PluginProbe
MxChat – AI Chatbot & Content Generation for WordPress / 2.4.6
MxChat – AI Chatbot & Content Generation for WordPress v2.4.6
3.2.21 3.2.20 3.2.19 3.2.18 3.2.17 3.2.16 3.2.15 3.2.14 3.2.12 3.2.13 3.2.11 3.2.10 3.2.9 3.2.8 3.2.7 3.2.6 3.2.5 3.2.4 3.2.3 3.2.2 3.2.1 2.0.3 2.0.4 2.0.5 2.0.6 All 152 releases
← All changes | includes/class-mxchat-utils.php +487 -901 3.2.1 → 2.4.6 View file →
@@ -1,902 +1,488 @@
1 -<?php
2 -if (!defined('ABSPATH')) {
3 - exit; // Exit if accessed directly
4 -}
5 -
6 -class MxChat_Utils {
7 -
8 -/**
9 - * UPDATED: Submit or update content (and its embedding) in the database.
10 - * Stores in Pinecone if enabled, otherwise stores in WordPress DB.
11 - *
12 - * @param string $content The content to be embedded.
13 - * @param string $source_url The source URL of the content.
14 - * @param string $api_key The API key used for generating embeddings.
15 - * @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
16 - * @param string $bot_id The bot ID for multi-bot support
17 - * @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.)
18 - * @return bool|WP_Error True on success, WP_Error on failure
19 - */
20 -public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
21 - global $wpdb;
22 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
23 -
24 - //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
25 - //error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
26 -
27 - // Sanitize the source URL
28 - $source_url = esc_url_raw($source_url);
29 -
30 - // Sanitize content_type
31 - $content_type = sanitize_key($content_type);
32 - if (empty($content_type)) {
33 - $content_type = 'content'; // Fallback for backwards compatibility
34 - }
35 -
36 - // Just ensure UTF-8 validity without aggressive escaping
37 - $safe_content = wp_check_invalid_utf8($content);
38 - // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
39 - $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
40 -
41 - // Check if chunking should be applied
42 - $chunker = MxChat_Chunker::from_settings();
43 - if ($chunker->should_chunk($safe_content)) {
44 - //error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission');
45 - return self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
46 - }
47 -
48 - // UPDATED: Generate the embedding using bot-specific configuration
49 - $embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
50 -
51 - if (!is_array($embedding_vector)) {
52 - //error_log('[MXCHAT-DB] Error: Embedding generation failed');
53 - return new WP_Error('embedding_failed', 'Failed to generate embedding for content');
54 - }
55 -
56 - //error_log('[MXCHAT-DB] Embedding generated successfully');
57 -
58 - // UPDATED: Check if Pinecone is enabled for this specific bot
59 - if (self::is_pinecone_enabled_for_bot($bot_id)) {
60 - //error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
61 - // Store in Pinecone only
62 - return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
63 - } else {
64 - //error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
65 - // Store in WordPress database only
66 - $embedding_vector_serialized = maybe_serialize($embedding_vector);
67 - return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
68 - }
69 -}
70 -
71 -/**
72 - * UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
73 - */
74 -private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
75 - // For default bot or when multi-bot is not active, use original method
76 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
77 - return self::is_pinecone_enabled();
78 - }
79 -
80 - // Get bot-specific Pinecone configuration
81 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
82 -
83 - if (empty($bot_pinecone_config)) {
84 - // Fallback to default configuration
85 - return self::is_pinecone_enabled();
86 - }
87 -
88 - $enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
89 - $api_key_check = !empty($bot_pinecone_config['api_key']);
90 - $host_check = !empty($bot_pinecone_config['host']);
91 -
92 - return $enabled_check && $api_key_check && $host_check;
93 -}
94 -
95 -/**
96 - * Check if Pinecone is enabled and properly configured (original method for default bot)
97 - */
98 -private static function is_pinecone_enabled() {
99 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
100 -
101 - if (empty($pinecone_options)) {
102 - return false;
103 - }
104 -
105 - $enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
106 - $api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
107 - $host_check = !empty($pinecone_options['mxchat_pinecone_host']);
108 -
109 - return $enabled_check && $api_key_check && $host_check;
110 -}
111 -
112 -/**
113 - * UPDATED: Store content in Pinecone only with bot support
114 - */
115 -private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
116 - //error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
117 -
118 - // Get bot-specific Pinecone configuration
119 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
120 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
121 - $api_key = $pinecone_options['mxchat_pinecone_api_key'];
122 - $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
123 - $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
124 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
125 - } else {
126 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
127 - if (empty($bot_pinecone_config)) {
128 - // Fallback to default configuration
129 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
130 - $api_key = $pinecone_options['mxchat_pinecone_api_key'];
131 - $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
132 - $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
133 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
134 - } else {
135 - $api_key = $bot_pinecone_config['api_key'];
136 - $environment = ''; // Not used in new Pinecone API
137 - $index_name = ''; // Not used in new Pinecone API
138 - $namespace = $bot_pinecone_config['namespace'] ?? '';
139 - }
140 - }
141 -
142 - $result = self::store_in_pinecone_main(
143 - $embedding_vector,
144 - $content,
145 - $source_url,
146 - $api_key,
147 - $environment,
148 - $index_name,
149 - $vector_id,
150 - $bot_id,
151 - $namespace,
152 - $content_type
153 - );
154 -
155 - if (is_wp_error($result)) {
156 - //error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
157 - return $result;
158 - }
159 -
160 - //error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
161 - return true;
162 -}
163 -
164 -/**
165 - * Store content in WordPress database with progressive fallback
166 - * UPDATED 2.5.6: Now includes content_type parameter
167 - */
168 -private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type = 'content') {
169 - global $wpdb;
170 -
171 - //error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
172 -
173 - // Sanitize content_type
174 - $content_type = sanitize_key($content_type);
175 - if (empty($content_type)) {
176 - $content_type = 'content'; // Fallback for backwards compatibility
177 - }
178 -
179 - // ===== FIXED: Generate unique identifier for manual content =====
180 - $original_source_url = $source_url;
181 - // Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
182 - // filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
183 - // Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
184 - $has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
185 - // Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
186 - $is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
187 - $is_manual_content = empty($source_url) || $source_url === '' || !$has_url_scheme || $is_legacy_mxchat_url;
188 -
189 - if ($is_manual_content) {
190 - // Generate unique identifier for manual content to prevent overwrites
191 - $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
192 - //error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
193 - }
194 -
195 - // Only check for duplicates if we have a valid source URL (not manual content)
196 - $existing_id = null;
197 - if (!$is_manual_content) {
198 - $existing_id = $wpdb->get_var(
199 - $wpdb->prepare(
200 - "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
201 - $source_url
202 - )
203 - );
204 - //error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
205 - } else {
206 - //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
207 - }
208 - // ===== END FIX =====
209 -
210 - // Progressive fallback mechanism for problematic content
211 - $attempt = 1;
212 - $max_attempts = 3;
213 - $current_content = $safe_content;
214 - $result = false;
215 -
216 - while ($attempt <= $max_attempts && $result === false) {
217 - try {
218 - if ($existing_id) {
219 - //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
220 -
221 - // Update the existing row - UPDATED 2.5.6: Added content_type
222 - $result = $wpdb->update(
223 - $table_name,
224 - array(
225 - 'url' => $source_url,
226 - 'article_content' => $current_content,
227 - 'embedding_vector' => $embedding_vector_serialized,
228 - 'source_url' => $source_url,
229 - 'content_type' => $content_type,
230 - 'timestamp' => current_time('mysql'),
231 - ),
232 - array('id' => $existing_id),
233 - array('%s','%s','%s','%s','%s','%s'),
234 - array('%d')
235 - );
236 - } else {
237 - //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
238 - //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
239 -
240 - // Insert a new row - UPDATED 2.5.6: Added content_type
241 - $result = $wpdb->insert(
242 - $table_name,
243 - array(
244 - 'url' => $source_url, // Now unique for manual content
245 - 'article_content' => $current_content,
246 - 'embedding_vector' => $embedding_vector_serialized,
247 - 'source_url' => $source_url, // Now unique for manual content
248 - 'content_type' => $content_type,
249 - 'timestamp' => current_time('mysql'),
250 - ),
251 - array('%s','%s','%s','%s','%s','%s')
252 - );
253 - }
254 -
255 - if ($result === false) {
256 - //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')');
257 - //error_log('[MXCHAT-DB] MySQL Error: ' . $wpdb->last_error);
258 - //error_log('[MXCHAT-DB] MySQL Error Number: ' . $wpdb->last_errno);
259 - //error_log('[MXCHAT-DB] Last Query: ' . substr($wpdb->last_query, 0, 500));
260 - //error_log('[MXCHAT-DB] Content length: ' . strlen($current_content) . ' bytes');
261 - //error_log('[MXCHAT-DB] Embedding vector length: ' . strlen($embedding_vector_serialized) . ' bytes');
262 -
263 - // Progressively apply more aggressive sanitization on failure
264 - if ($attempt === 1) {
265 - // First fallback: Use a more aggressive character filter and shorten
266 - $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
267 - $current_content = substr($current_content, 0, 50000);
268 - } else if ($attempt === 2) {
269 - // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
270 - $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
271 - $current_content = substr($current_content, 0, 30000);
272 - }
273 -
274 - $attempt++;
275 - }
276 - } catch (Exception $e) {
277 - //error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
278 - $attempt++;
279 - }
280 - }
281 -
282 -if ($result === false) {
283 - //error_log('[MXCHAT-DB] All database operation attempts failed');
284 - //error_log('[MXCHAT-DB] Final MySQL Error: ' . $wpdb->last_error);
285 -
286 - $detailed_error = sprintf(
287 - 'Failed to store content in WordPress database after %d attempts. MySQL Error: %s (Error #%d). Content size: %d bytes, Embedding size: %d bytes',
288 - $max_attempts,
289 - $wpdb->last_error,
290 - $wpdb->last_errno,
291 - strlen($current_content),
292 - strlen($embedding_vector_serialized)
293 - );
294 -
295 - return new WP_Error('database_failed', $detailed_error);
296 -}
297 -
298 - //error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
299 - return true;
300 -}
301 -
302 -/**
303 - * UPDATED: Store content in Pinecone database with bot support
304 - * UPDATED 2.5.6: Now accepts content_type parameter
305 - */
306 -private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '', $content_type = 'content') {
307 - //error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
308 -
309 - // ===== UPDATED: Handle manual content with unique vector IDs =====
310 - if ($vector_id) {
311 - // Use provided vector ID
312 - //error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
313 - } elseif (!empty($url) && preg_match('#^https?://#i', $url)) {
314 - // For URLs, use URL-based ID (existing behavior)
315 - $vector_id = md5($url);
316 - //error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
317 - } else {
318 - // For manual content (empty/no URL scheme), generate unique ID
319 - $vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
320 - //error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
321 - }
322 - // ===== END UPDATE =====
323 -
324 - // Get host from bot-specific config or fallback to default
325 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
326 - $options = get_option('mxchat_pinecone_addon_options');
327 - $host = $options['mxchat_pinecone_host'] ?? '';
328 - } else {
329 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
330 - if (!empty($bot_pinecone_config)) {
331 - $host = $bot_pinecone_config['host'] ?? '';
332 - } else {
333 - $options = get_option('mxchat_pinecone_addon_options');
334 - $host = $options['mxchat_pinecone_host'] ?? '';
335 - }
336 - }
337 -
338 - //error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
339 - //error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
340 - //error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
341 - //error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
342 -
343 - if (empty($host)) {
344 - //error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
345 - return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
346 - }
347 -
348 - // ===== UPDATED 2.5.6: Use passed content_type or determine from URL if not provided =====
349 - // Sanitize content_type
350 - $content_type = sanitize_key($content_type);
351 - if (empty($content_type)) {
352 - // Fallback to old detection logic for backwards compatibility
353 - $is_product = false;
354 - $content_type = 'manual'; // Default for manual content
355 -
356 - if (!empty($url) && preg_match('#^https?://#i', $url)) {
357 - $is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
358 - $content_type = $is_product ? 'product' : 'content';
359 - }
360 - }
361 -
362 - //error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
363 - // ===== END UPDATE =====
364 -
365 - $api_endpoint = "https://{$host}/vectors/upsert";
366 - //error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
367 -
368 - // UPDATED 2.5.6: Use provided content_type in metadata
369 - $metadata = array(
370 - 'text' => $content,
371 - 'source_url' => $url, // Can be empty for manual content
372 - 'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
373 - 'last_updated' => time(),
374 - 'created_at' => time(), // Add creation timestamp
375 - 'bot_id' => $bot_id // Add bot identification
376 - );
377 -
378 - $vector_data = array(
379 - 'id' => $vector_id,
380 - 'values' => $embedding_vector,
381 - 'metadata' => $metadata
382 - );
383 -
384 - $request_body = array(
385 - 'vectors' => array($vector_data)
386 - );
387 -
388 - // Add namespace if specified for multi-bot separation
389 - if (!empty($namespace)) {
390 - $request_body['namespace'] = $namespace;
391 - //error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
392 - }
393 -
394 - //error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
395 -
396 - $response = wp_remote_post($api_endpoint, array(
397 - 'headers' => array(
398 - 'Api-Key' => $api_key,
399 - 'accept' => 'application/json',
400 - 'content-type' => 'application/json'
401 - ),
402 - 'body' => wp_json_encode($request_body),
403 - 'timeout' => 30,
404 - 'data_format' => 'body'
405 - ));
406 -
407 - if (is_wp_error($response)) {
408 - //error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
409 - return new WP_Error('pinecone_request', $response->get_error_message());
410 - }
411 -
412 - $response_code = wp_remote_retrieve_response_code($response);
413 - //error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
414 -
415 - if ($response_code !== 200) {
416 - $body = wp_remote_retrieve_body($response);
417 - //error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
418 - return new WP_Error('pinecone_api', sprintf(
419 - 'Pinecone API error (HTTP %d): %s',
420 - $response_code,
421 - $body
422 - ));
423 - }
424 -
425 - $response_body = wp_remote_retrieve_body($response);
426 - //error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
427 - //error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
428 - //error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
429 -
430 - return true;
431 -}
432 -
433 -/**
434 - * UPDATED: Generate an embedding for the given text using bot-specific configuration.
435 - *
436 - * @param string $text The text to be embedded.
437 - * @param string $api_key The API key used for generating embeddings.
438 - * @param string $bot_id The bot ID for multi-bot support
439 - * @return array|null The embedding vector or null on failure.
440 - */
441 -private static function generate_embedding($text, $api_key, $bot_id = 'default') {
442 - // Get bot-specific options
443 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
444 - $options = get_option('mxchat_options');
445 - } else {
446 - $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
447 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
448 - }
449 -
450 - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
451 -
452 - // Determine endpoint and API key based on model
453 - if (strpos($selected_model, 'voyage') === 0) {
454 - $endpoint = 'https://api.voyageai.com/v1/embeddings';
455 - $api_key = $options['voyage_api_key'] ?? '';
456 - } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
457 - $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
458 - $api_key = $options['gemini_api_key'] ?? '';
459 - } else {
460 - $endpoint = 'https://api.openai.com/v1/embeddings';
461 - // Use the bot-specific API key or fallback to passed API key
462 - $api_key = $options['api_key'] ?? $api_key;
463 - }
464 -
465 - // Prepare request body based on provider
466 - if (strpos($selected_model, 'gemini-embedding') === 0) {
467 - // Gemini API format
468 - $request_body = [
469 - 'model' => 'models/' . $selected_model,
470 - 'content' => [
471 - 'parts' => [
472 - ['text' => $text]
473 - ]
474 - ],
475 - 'outputDimensionality' => 1536
476 - ];
477 -
478 - // Prepare headers for Gemini (API key as query parameter)
479 - $endpoint .= '?key=' . $api_key;
480 - $headers = [
481 - 'Content-Type' => 'application/json'
482 - ];
483 - } else {
484 - // OpenAI/Voyage API format
485 - $request_body = [
486 - 'input' => $text,
487 - 'model' => $selected_model
488 - ];
489 -
490 - // Add output_dimension for voyage-3-large
491 - if ($selected_model === 'voyage-3-large') {
492 - $request_body['output_dimension'] = 2048;
493 - }
494 -
495 - // Prepare headers for OpenAI/Voyage
496 - $headers = [
497 - 'Content-Type' => 'application/json',
498 - 'Authorization' => 'Bearer ' . $api_key
499 - ];
500 - }
501 -
502 - $args = [
503 - 'body' => wp_json_encode($request_body),
504 - 'headers' => $headers,
505 - 'timeout' => 60,
506 - 'redirection' => 5,
507 - 'blocking' => true,
508 - 'httpversion' => '1.0',
509 - 'sslverify' => true,
510 - ];
511 -
512 - $response = wp_remote_post($endpoint, $args);
513 -
514 - if (is_wp_error($response)) {
515 - //error_log('Error generating embedding for bot ' . $bot_id . ': ' . $response->get_error_message());
516 - return null;
517 - }
518 -
519 - $response_body = json_decode(wp_remote_retrieve_body($response), true);
520 -
521 - // Handle different response formats based on provider
522 - if (strpos($selected_model, 'gemini-embedding') === 0) {
523 - // Gemini API response format
524 - if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
525 - return $response_body['embedding']['values'];
526 - } else {
527 - //error_log('Invalid response received from Gemini embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
528 - return null;
529 - }
530 - } else {
531 - // OpenAI/Voyage API response format
532 - if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
533 - return $response_body['data'][0]['embedding'];
534 - } else {
535 - //error_log('Invalid response received from embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
536 - return null;
537 - }
538 - }
539 -}
540 -
541 -/**
542 - * Submit content as multiple chunks
543 - *
544 - * Splits large content into chunks, generates embeddings for each,
545 - * and stores them with chunk metadata for later reassembly.
546 - *
547 - * @param string $content The content to chunk and store
548 - * @param string $source_url The source URL
549 - * @param string $api_key The API key for embeddings
550 - * @param string $bot_id The bot ID
551 - * @param string $content_type The content type
552 - * @param MxChat_Chunker $chunker The chunker instance
553 - * @return bool|WP_Error True on success, WP_Error on failure
554 - */
555 -private static function submit_chunked_content($content, $source_url, $api_key, $bot_id, $content_type, $chunker) {
556 - global $wpdb;
557 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
558 -
559 - //error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url);
560 - //error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars');
561 -
562 - // First, delete any existing chunks for this URL (clean slate)
563 - $delete_result = self::delete_chunks_for_url($source_url, $bot_id);
564 - if (is_wp_error($delete_result)) {
565 - //error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message());
566 - // Continue anyway - we'll overwrite with upsert
567 - }
568 -
569 - // Split content into chunks
570 - $chunks = $chunker->chunk_text($content);
571 - $total_chunks = count($chunks);
572 -
573 - //error_log('[MXCHAT-CHUNK-DEBUG] Created ' . $total_chunks . ' chunks');
574 - foreach ($chunks as $i => $chunk) {
575 - //error_log('[MXCHAT-CHUNK-DEBUG] Chunk ' . $i . ' length: ' . strlen($chunk) . ' chars, preview: ' . substr($chunk, 0, 100));
576 - }
577 -
578 - //error_log('[MXCHAT-CHUNK] Split content into ' . $total_chunks . ' chunks');
579 -
580 - if ($total_chunks === 0) {
581 - return new WP_Error('chunking_failed', 'Content could not be split into chunks');
582 - }
583 -
584 - $errors = array();
585 - $is_pinecone = self::is_pinecone_enabled_for_bot($bot_id);
586 -
587 - foreach ($chunks as $index => $chunk_text) {
588 - // Generate chunk metadata
589 - $chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
590 - $chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
591 -
592 - //error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
593 -
594 - // Generate embedding for this chunk
595 - $embedding_vector = self::generate_embedding($chunk_text, $api_key, $bot_id);
596 -
597 - if (!is_array($embedding_vector)) {
598 - $errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index);
599 - //error_log('[MXCHAT-CHUNK] Failed to generate embedding for chunk ' . $index);
600 - continue;
601 - }
602 -
603 - if ($is_pinecone) {
604 - // Store in Pinecone with chunk metadata
605 - $result = self::store_chunk_in_pinecone(
606 - $embedding_vector,
607 - $chunk_text,
608 - $source_url,
609 - $chunk_vector_id,
610 - $bot_id,
611 - $content_type,
612 - $chunk_metadata
613 - );
614 - } else {
615 - // Store in WordPress DB with chunk metadata
616 - $content_with_metadata = MxChat_Chunker::format_chunk_for_storage($chunk_text, $chunk_metadata);
617 - $embedding_vector_serialized = maybe_serialize($embedding_vector);
618 -
619 - $result = self::store_chunk_in_wordpress_db(
620 - $content_with_metadata,
621 - $source_url,
622 - $embedding_vector_serialized,
623 - $table_name,
624 - $content_type,
625 - $chunk_metadata
626 - );
627 - }
628 -
629 - if (is_wp_error($result)) {
630 - $errors[] = $result;
631 - //error_log('[MXCHAT-CHUNK] Failed to store chunk ' . $index . ': ' . $result->get_error_message());
632 - }
633 - }
634 -
635 - if (count($errors) === $total_chunks) {
636 - return new WP_Error('chunking_failed', 'Failed to store any chunks');
637 - }
638 -
639 - if (!empty($errors)) {
640 - //error_log('[MXCHAT-CHUNK] Completed with ' . count($errors) . ' errors out of ' . $total_chunks . ' chunks');
641 - return new WP_Error('chunking_partial_failure',
642 - sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks));
643 - }
644 -
645 - //error_log('[MXCHAT-CHUNK] Successfully stored all ' . $total_chunks . ' chunks');
646 - return true;
647 -}
648 -
649 -/**
650 - * Store a single chunk in Pinecone with chunk-specific metadata
651 - */
652 -private static function store_chunk_in_pinecone($embedding_vector, $chunk_text, $source_url, $vector_id, $bot_id, $content_type, $chunk_metadata) {
653 - // Get Pinecone configuration
654 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
655 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
656 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
657 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
658 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
659 - } else {
660 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
661 - if (empty($bot_pinecone_config)) {
662 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
663 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
664 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
665 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
666 - } else {
667 - $api_key = $bot_pinecone_config['api_key'] ?? '';
668 - $host = $bot_pinecone_config['host'] ?? '';
669 - $namespace = $bot_pinecone_config['namespace'] ?? '';
670 - }
671 - }
672 -
673 - if (empty($host) || empty($api_key)) {
674 - return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
675 - }
676 -
677 - $api_endpoint = "https://{$host}/vectors/upsert";
678 -
679 - // Build metadata with chunk information
680 - $metadata = array(
681 - 'text' => $chunk_text,
682 - 'source_url' => $source_url,
683 - 'type' => $content_type,
684 - 'is_chunked' => true,
685 - 'chunk_index' => $chunk_metadata['chunk_index'],
686 - 'total_chunks' => $chunk_metadata['total_chunks'],
687 - 'parent_url_hash' => $chunk_metadata['parent_url_hash'],
688 - 'last_updated' => time(),
689 - 'created_at' => time(),
690 - 'bot_id' => $bot_id
691 - );
692 -
693 - $vector_data = array(
694 - 'id' => $vector_id,
695 - 'values' => $embedding_vector,
696 - 'metadata' => $metadata
697 - );
698 -
699 - $request_body = array(
700 - 'vectors' => array($vector_data)
701 - );
702 -
703 - if (!empty($namespace)) {
704 - $request_body['namespace'] = $namespace;
705 - }
706 -
707 - $response = wp_remote_post($api_endpoint, array(
708 - 'headers' => array(
709 - 'Api-Key' => $api_key,
710 - 'accept' => 'application/json',
711 - 'content-type' => 'application/json'
712 - ),
713 - 'body' => wp_json_encode($request_body),
714 - 'timeout' => 30
715 - ));
716 -
717 - if (is_wp_error($response)) {
718 - return $response;
719 - }
720 -
721 - $response_code = wp_remote_retrieve_response_code($response);
722 - if ($response_code !== 200) {
723 - return new WP_Error('pinecone_api', 'Pinecone API error: HTTP ' . $response_code);
724 - }
725 -
726 - return true;
727 -}
728 -
729 -/**
730 - * Store a single chunk in WordPress database
731 - */
732 -private static function store_chunk_in_wordpress_db($content_with_metadata, $source_url, $embedding_vector_serialized, $table_name, $content_type, $chunk_metadata) {
733 - global $wpdb;
734 -
735 - // For chunks, we always insert new rows (no duplicate checking)
736 - // The URL includes chunk info in the metadata, but source_url stays the same for grouping
737 - $result = $wpdb->insert(
738 - $table_name,
739 - array(
740 - 'url' => $source_url,
741 - 'article_content' => $content_with_metadata,
742 - 'embedding_vector' => $embedding_vector_serialized,
743 - 'source_url' => $source_url,
744 - 'content_type' => $content_type,
745 - 'timestamp' => current_time('mysql')
746 - ),
747 - array('%s', '%s', '%s', '%s', '%s', '%s')
748 - );
749 -
750 - if ($result === false) {
751 - return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error);
752 - }
753 -
754 - return true;
755 -}
756 -
757 -/**
758 - * Delete all chunks for a given URL
759 - *
760 - * @param string $source_url The source URL
761 - * @param string $bot_id The bot ID
762 - * @return bool|WP_Error True on success, WP_Error on failure
763 - */
764 -public static function delete_chunks_for_url($source_url, $bot_id = 'default') {
765 - //error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url);
766 -
767 - if (self::is_pinecone_enabled_for_bot($bot_id)) {
768 - return self::delete_pinecone_chunks_by_url($source_url, $bot_id);
769 - } else {
770 - return self::delete_wordpress_chunks_by_url($source_url);
771 - }
772 -}
773 -
774 -/**
775 - * Delete all chunks for a URL from Pinecone
776 - */
777 -private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
778 - // Get Pinecone configuration
779 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
780 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
781 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
782 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
783 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
784 - } else {
785 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
786 - if (empty($bot_pinecone_config)) {
787 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
788 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
789 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
790 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
791 - } else {
792 - $api_key = $bot_pinecone_config['api_key'] ?? '';
793 - $host = $bot_pinecone_config['host'] ?? '';
794 - $namespace = $bot_pinecone_config['namespace'] ?? '';
795 - }
796 - }
797 -
798 - if (empty($host) || empty($api_key)) {
799 - return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
800 - }
801 -
802 - $base_vector_id = md5($source_url);
803 - $vectors_to_delete = array();
804 -
805 - // Add the original single-vector ID (for non-chunked content)
806 - $vectors_to_delete[] = $base_vector_id;
807 -
808 - // Use Pinecone list API to find all chunk vectors with this prefix
809 - $list_url = "https://{$host}/vectors/list";
810 -
811 - $list_body = array(
812 - 'prefix' => $base_vector_id . '_chunk_',
813 - 'limit' => 100
814 - );
815 -
816 - if (!empty($namespace)) {
817 - $list_body['namespace'] = $namespace;
818 - }
819 -
820 - $list_response = wp_remote_post($list_url, array(
821 - 'headers' => array(
822 - 'Api-Key' => $api_key,
823 - 'accept' => 'application/json',
824 - 'content-type' => 'application/json'
825 - ),
826 - 'body' => wp_json_encode($list_body),
827 - 'timeout' => 30
828 - ));
829 -
830 - if (!is_wp_error($list_response)) {
831 - $list_data = json_decode(wp_remote_retrieve_body($list_response), true);
832 - if (!empty($list_data['vectors'])) {
833 - foreach ($list_data['vectors'] as $vector) {
834 - if (isset($vector['id'])) {
835 - $vectors_to_delete[] = $vector['id'];
836 - }
837 - }
838 - }
839 - }
840 -
841 - if (empty($vectors_to_delete)) {
842 - //error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
843 - return true;
844 - }
845 -
846 - //error_log('[MXCHAT-CHUNK-DELETE] Deleting ' . count($vectors_to_delete) . ' vectors from Pinecone');
847 -
848 - // Delete vectors
849 - $delete_url = "https://{$host}/vectors/delete";
850 -
851 - $delete_body = array(
852 - 'ids' => $vectors_to_delete
853 - );
854 -
855 - if (!empty($namespace)) {
856 - $delete_body['namespace'] = $namespace;
857 - }
858 -
859 - $delete_response = wp_remote_post($delete_url, array(
860 - 'headers' => array(
861 - 'Api-Key' => $api_key,
862 - 'accept' => 'application/json',
863 - 'content-type' => 'application/json'
864 - ),
865 - 'body' => wp_json_encode($delete_body),
866 - 'timeout' => 30
867 - ));
868 -
869 - if (is_wp_error($delete_response)) {
870 - return $delete_response;
871 - }
872 -
873 - $response_code = wp_remote_retrieve_response_code($delete_response);
874 - if ($response_code !== 200) {
875 - return new WP_Error('pinecone_delete', 'Failed to delete vectors: HTTP ' . $response_code);
876 - }
877 -
878 - return true;
879 -}
880 -
881 -/**
882 - * Delete all chunks for a URL from WordPress database
883 - */
884 -private static function delete_wordpress_chunks_by_url($source_url) {
885 - global $wpdb;
886 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
887 -
888 - // Delete all rows with this source_url (handles both chunked and non-chunked)
889 - $result = $wpdb->delete(
890 - $table_name,
891 - array('source_url' => $source_url),
892 - array('%s')
893 - );
894 -
895 - if ($result === false) {
896 - return new WP_Error('database_delete', 'Failed to delete chunks: ' . $wpdb->last_error);
897 - }
898 -
899 - //error_log('[MXCHAT-CHUNK-DELETE] Deleted ' . $result . ' rows from WordPress DB');
900 - return true;
901 -}
1 +<?php
2 +if (!defined('ABSPATH')) {
3 + exit; // Exit if accessed directly
4 +}
5 +
6 +class MxChat_Utils {
7 +
8 +/**
9 + * UPDATED: Submit or update content (and its embedding) in the database.
10 + * Stores in Pinecone if enabled, otherwise stores in WordPress DB.
11 + *
12 + * @param string $content The content to be embedded.
13 + * @param string $source_url The source URL of the content.
14 + * @param string $api_key The API key used for generating embeddings.
15 + * @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
16 + * @param string $bot_id The bot ID for multi-bot support
17 + * @return bool|WP_Error True on success, WP_Error on failure
18 + */
19 +public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default') {
20 + global $wpdb;
21 + $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
22 +
23 + //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ')');
24 + //error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
25 +
26 + // Sanitize the source URL
27 + $source_url = esc_url_raw($source_url);
28 +
29 + // Just ensure UTF-8 validity without aggressive escaping
30 + $safe_content = wp_check_invalid_utf8($content);
31 + // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
32 + $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
33 +
34 + // UPDATED: Generate the embedding using bot-specific configuration
35 + $embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
36 +
37 + if (!is_array($embedding_vector)) {
38 + //error_log('[MXCHAT-DB] Error: Embedding generation failed');
39 + return new WP_Error('embedding_failed', 'Failed to generate embedding for content');
40 + }
41 +
42 + //error_log('[MXCHAT-DB] Embedding generated successfully');
43 +
44 + // UPDATED: Check if Pinecone is enabled for this specific bot
45 + if (self::is_pinecone_enabled_for_bot($bot_id)) {
46 + //error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
47 + // Store in Pinecone only
48 + return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id);
49 + } else {
50 + //error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
51 + // Store in WordPress database only
52 + $embedding_vector_serialized = maybe_serialize($embedding_vector);
53 + return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name);
54 + }
55 +}
56 +
57 +/**
58 + * UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
59 + */
60 +private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
61 + // For default bot or when multi-bot is not active, use original method
62 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
63 + return self::is_pinecone_enabled();
64 + }
65 +
66 + // Get bot-specific Pinecone configuration
67 + $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
68 +
69 + if (empty($bot_pinecone_config)) {
70 + // Fallback to default configuration
71 + return self::is_pinecone_enabled();
72 + }
73 +
74 + $enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
75 + $api_key_check = !empty($bot_pinecone_config['api_key']);
76 + $host_check = !empty($bot_pinecone_config['host']);
77 +
78 + return $enabled_check && $api_key_check && $host_check;
79 +}
80 +
81 +/**
82 + * Check if Pinecone is enabled and properly configured (original method for default bot)
83 + */
84 +private static function is_pinecone_enabled() {
85 + $pinecone_options = get_option('mxchat_pinecone_addon_options');
86 +
87 + if (empty($pinecone_options)) {
88 + return false;
89 + }
90 +
91 + $enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
92 + $api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
93 + $host_check = !empty($pinecone_options['mxchat_pinecone_host']);
94 +
95 + return $enabled_check && $api_key_check && $host_check;
96 +}
97 +
98 +/**
99 + * UPDATED: Store content in Pinecone only with bot support
100 + */
101 +private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default') {
102 + //error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
103 +
104 + // Get bot-specific Pinecone configuration
105 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
106 + $pinecone_options = get_option('mxchat_pinecone_addon_options');
107 + $api_key = $pinecone_options['mxchat_pinecone_api_key'];
108 + $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
109 + $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
110 + $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
111 + } else {
112 + $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
113 + if (empty($bot_pinecone_config)) {
114 + // Fallback to default configuration
115 + $pinecone_options = get_option('mxchat_pinecone_addon_options');
116 + $api_key = $pinecone_options['mxchat_pinecone_api_key'];
117 + $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
118 + $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
119 + $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
120 + } else {
121 + $api_key = $bot_pinecone_config['api_key'];
122 + $environment = ''; // Not used in new Pinecone API
123 + $index_name = ''; // Not used in new Pinecone API
124 + $namespace = $bot_pinecone_config['namespace'] ?? '';
125 + }
126 + }
127 +
128 + $result = self::store_in_pinecone_main(
129 + $embedding_vector,
130 + $content,
131 + $source_url,
132 + $api_key,
133 + $environment,
134 + $index_name,
135 + $vector_id,
136 + $bot_id,
137 + $namespace
138 + );
139 +
140 + if (is_wp_error($result)) {
141 + //error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
142 + return $result;
143 + }
144 +
145 + //error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
146 + return true;
147 +}
148 +
149 +/**
150 + * Store content in WordPress database with progressive fallback (unchanged)
151 + */
152 +private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name) {
153 + global $wpdb;
154 +
155 + //error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
156 +
157 + // ===== FIXED: Generate unique identifier for manual content =====
158 + $original_source_url = $source_url;
159 + $is_manual_content = empty($source_url) || $source_url === '' || !filter_var($source_url, FILTER_VALIDATE_URL);
160 +
161 + if ($is_manual_content) {
162 + // Generate unique identifier for manual content to prevent overwrites
163 + $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
164 + //error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
165 + }
166 +
167 + // Only check for duplicates if we have a valid source URL (not manual content)
168 + $existing_id = null;
169 + if (!$is_manual_content) {
170 + $existing_id = $wpdb->get_var(
171 + $wpdb->prepare(
172 + "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
173 + $source_url
174 + )
175 + );
176 + //error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
177 + } else {
178 + //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
179 + }
180 + // ===== END FIX =====
181 +
182 + // Progressive fallback mechanism for problematic content
183 + $attempt = 1;
184 + $max_attempts = 3;
185 + $current_content = $safe_content;
186 + $result = false;
187 +
188 + while ($attempt <= $max_attempts && $result === false) {
189 + try {
190 + if ($existing_id) {
191 + //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
192 +
193 + // Update the existing row
194 + $result = $wpdb->update(
195 + $table_name,
196 + array(
197 + 'url' => $source_url,
198 + 'article_content' => $current_content,
199 + 'embedding_vector' => $embedding_vector_serialized,
200 + 'source_url' => $source_url,
201 + 'timestamp' => current_time('mysql'),
202 + ),
203 + array('id' => $existing_id),
204 + array('%s','%s','%s','%s','%s'),
205 + array('%d')
206 + );
207 + } else {
208 + //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
209 + //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
210 +
211 + // Insert a new row (using generated unique ID for manual content)
212 + $result = $wpdb->insert(
213 + $table_name,
214 + array(
215 + 'url' => $source_url, // Now unique for manual content
216 + 'article_content' => $current_content,
217 + 'embedding_vector' => $embedding_vector_serialized,
218 + 'source_url' => $source_url, // Now unique for manual content
219 + 'timestamp' => current_time('mysql'),
220 + ),
221 + array('%s','%s','%s','%s','%s')
222 + );
223 + }
224 +
225 + if ($result === false) {
226 + //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . '): ' . $wpdb->last_error);
227 +
228 + // Progressively apply more aggressive sanitization on failure
229 + if ($attempt === 1) {
230 + // First fallback: Use a more aggressive character filter and shorten
231 + $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
232 + $current_content = substr($current_content, 0, 50000);
233 + } else if ($attempt === 2) {
234 + // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
235 + $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
236 + $current_content = substr($current_content, 0, 30000);
237 + }
238 +
239 + $attempt++;
240 + }
241 + } catch (Exception $e) {
242 + //error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
243 + $attempt++;
244 + }
245 + }
246 +
247 + if ($result === false) {
248 + //error_log('[MXCHAT-DB] All database operation attempts failed');
249 + return new WP_Error('database_failed', 'Failed to store content in WordPress database after ' . $max_attempts . ' attempts');
250 + }
251 +
252 + //error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
253 + return true;
254 +}
255 +
256 +/**
257 + * UPDATED: Store content in Pinecone database with bot support
258 + */
259 +private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '') {
260 + //error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
261 +
262 + // ===== UPDATED: Handle manual content with unique vector IDs =====
263 + if ($vector_id) {
264 + // Use provided vector ID
265 + //error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
266 + } elseif (!empty($url) && filter_var($url, FILTER_VALIDATE_URL)) {
267 + // For valid URLs, use URL-based ID (existing behavior)
268 + $vector_id = md5($url);
269 + //error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
270 + } else {
271 + // For manual content (empty/invalid URL), generate unique ID
272 + $vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
273 + //error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
274 + }
275 + // ===== END UPDATE =====
276 +
277 + // Get host from bot-specific config or fallback to default
278 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
279 + $options = get_option('mxchat_pinecone_addon_options');
280 + $host = $options['mxchat_pinecone_host'] ?? '';
281 + } else {
282 + $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
283 + if (!empty($bot_pinecone_config)) {
284 + $host = $bot_pinecone_config['host'] ?? '';
285 + } else {
286 + $options = get_option('mxchat_pinecone_addon_options');
287 + $host = $options['mxchat_pinecone_host'] ?? '';
288 + }
289 + }
290 +
291 + //error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
292 + //error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
293 + //error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
294 + //error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
295 +
296 + if (empty($host)) {
297 + //error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
298 + return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
299 + }
300 +
301 + // ===== UPDATED: Determine content type more accurately =====
302 + $is_product = false;
303 + $content_type = 'manual'; // Default for manual content
304 +
305 + if (!empty($url) && filter_var($url, FILTER_VALIDATE_URL)) {
306 + $is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
307 + $content_type = $is_product ? 'product' : 'content';
308 + }
309 +
310 + //error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
311 + // ===== END UPDATE =====
312 +
313 + $api_endpoint = "https://{$host}/vectors/upsert";
314 + //error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
315 +
316 + // UPDATED: Add bot_id to metadata and handle namespace
317 + $metadata = array(
318 + 'text' => $content,
319 + 'source_url' => $url, // Can be empty for manual content
320 + 'type' => $content_type, // 'manual', 'content', or 'product'
321 + 'last_updated' => time(),
322 + 'created_at' => time(), // Add creation timestamp
323 + 'bot_id' => $bot_id // Add bot identification
324 + );
325 +
326 + $vector_data = array(
327 + 'id' => $vector_id,
328 + 'values' => $embedding_vector,
329 + 'metadata' => $metadata
330 + );
331 +
332 + $request_body = array(
333 + 'vectors' => array($vector_data)
334 + );
335 +
336 + // Add namespace if specified for multi-bot separation
337 + if (!empty($namespace)) {
338 + $request_body['namespace'] = $namespace;
339 + //error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
340 + }
341 +
342 + //error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
343 +
344 + $response = wp_remote_post($api_endpoint, array(
345 + 'headers' => array(
346 + 'Api-Key' => $api_key,
347 + 'accept' => 'application/json',
348 + 'content-type' => 'application/json'
349 + ),
350 + 'body' => wp_json_encode($request_body),
351 + 'timeout' => 30,
352 + 'data_format' => 'body'
353 + ));
354 +
355 + if (is_wp_error($response)) {
356 + //error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
357 + return new WP_Error('pinecone_request', $response->get_error_message());
358 + }
359 +
360 + $response_code = wp_remote_retrieve_response_code($response);
361 + //error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
362 +
363 + if ($response_code !== 200) {
364 + $body = wp_remote_retrieve_body($response);
365 + //error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
366 + return new WP_Error('pinecone_api', sprintf(
367 + 'Pinecone API error (HTTP %d): %s',
368 + $response_code,
369 + $body
370 + ));
371 + }
372 +
373 + $response_body = wp_remote_retrieve_body($response);
374 + //error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
375 + //error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
376 + //error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
377 +
378 + return true;
379 +}
380 +
381 +/**
382 + * UPDATED: Generate an embedding for the given text using bot-specific configuration.
383 + *
384 + * @param string $text The text to be embedded.
385 + * @param string $api_key The API key used for generating embeddings.
386 + * @param string $bot_id The bot ID for multi-bot support
387 + * @return array|null The embedding vector or null on failure.
388 + */
389 +private static function generate_embedding($text, $api_key, $bot_id = 'default') {
390 + // Get bot-specific options
391 + if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
392 + $options = get_option('mxchat_options');
393 + } else {
394 + $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
395 + $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
396 + }
397 +
398 + $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
399 +
400 + // Determine endpoint and API key based on model
401 + if (strpos($selected_model, 'voyage') === 0) {
402 + $endpoint = 'https://api.voyageai.com/v1/embeddings';
403 + $api_key = $options['voyage_api_key'] ?? '';
404 + } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
405 + $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
406 + $api_key = $options['gemini_api_key'] ?? '';
407 + } else {
408 + $endpoint = 'https://api.openai.com/v1/embeddings';
409 + // Use the bot-specific API key or fallback to passed API key
410 + $api_key = $options['api_key'] ?? $api_key;
411 + }
412 +
413 + // Prepare request body based on provider
414 + if (strpos($selected_model, 'gemini-embedding') === 0) {
415 + // Gemini API format
416 + $request_body = [
417 + 'model' => 'models/' . $selected_model,
418 + 'content' => [
419 + 'parts' => [
420 + ['text' => $text]
421 + ]
422 + ],
423 + 'outputDimensionality' => 1536
424 + ];
425 +
426 + // Prepare headers for Gemini (API key as query parameter)
427 + $endpoint .= '?key=' . $api_key;
428 + $headers = [
429 + 'Content-Type' => 'application/json'
430 + ];
431 + } else {
432 + // OpenAI/Voyage API format
433 + $request_body = [
434 + 'input' => $text,
435 + 'model' => $selected_model
436 + ];
437 +
438 + // Add output_dimension for voyage-3-large
439 + if ($selected_model === 'voyage-3-large') {
440 + $request_body['output_dimension'] = 2048;
441 + }
442 +
443 + // Prepare headers for OpenAI/Voyage
444 + $headers = [
445 + 'Content-Type' => 'application/json',
446 + 'Authorization' => 'Bearer ' . $api_key
447 + ];
448 + }
449 +
450 + $args = [
451 + 'body' => wp_json_encode($request_body),
452 + 'headers' => $headers,
453 + 'timeout' => 60,
454 + 'redirection' => 5,
455 + 'blocking' => true,
456 + 'httpversion' => '1.0',
457 + 'sslverify' => true,
458 + ];
459 +
460 + $response = wp_remote_post($endpoint, $args);
461 +
462 + if (is_wp_error($response)) {
463 + //error_log('Error generating embedding for bot ' . $bot_id . ': ' . $response->get_error_message());
464 + return null;
465 + }
466 +
467 + $response_body = json_decode(wp_remote_retrieve_body($response), true);
468 +
469 + // Handle different response formats based on provider
470 + if (strpos($selected_model, 'gemini-embedding') === 0) {
471 + // Gemini API response format
472 + if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
473 + return $response_body['embedding']['values'];
474 + } else {
475 + //error_log('Invalid response received from Gemini embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
476 + return null;
477 + }
478 + } else {
479 + // OpenAI/Voyage API response format
480 + if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
481 + return $response_body['data'][0]['embedding'];
482 + } else {
483 + //error_log('Invalid response received from embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
484 + return null;
485 + }
486 + }
487 +}
902 488 }