PluginProbe
MxChat – AI Chatbot & Content Generation for WordPress / 3.2.2
MxChat – AI Chatbot & Content Generation for WordPress v3.2.2
3.2.21 3.2.20 3.2.19 3.2.18 3.2.17 3.2.16 3.2.15 3.2.14 3.2.12 3.2.13 3.2.11 3.2.10 3.2.9 3.2.8 3.2.7 3.2.6 3.2.5 3.2.4 3.2.3 3.2.2 3.2.1 2.0.3 2.0.4 2.0.5 2.0.6 All 152 releases
mxchat-basic / includes / class-mxchat-utils.php

class-mxchat-utils.php in MxChat – AI Chatbot & Content Generation for WordPress 3.2.2, at includes/class-mxchat-utils.php

913 lines 36.4 KB
No matching file
Up and down to move Enter to open Esc to close
Raw Download Zip
1 <?php
2 if (!defined('ABSPATH')) {
3 exit; // Exit if accessed directly
4 }
5
6 class MxChat_Utils {
7
8 /**
9 * UPDATED: Submit or update content (and its embedding) in the database.
10 * Stores in Pinecone if enabled, otherwise stores in WordPress DB.
11 *
12 * @param string $content The content to be embedded.
13 * @param string $source_url The source URL of the content.
14 * @param string $api_key The API key used for generating embeddings.
15 * @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
16 * @param string $bot_id The bot ID for multi-bot support
17 * @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.)
18 * @return bool|WP_Error True on success, WP_Error on failure
19 */
20 public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
21 global $wpdb;
22 $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
23
24 //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
25 //error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
26
27 // Sanitize the source URL
28 $source_url = esc_url_raw($source_url);
29
30 // Sanitize content_type
31 $content_type = sanitize_key($content_type);
32 if (empty($content_type)) {
33 $content_type = 'content'; // Fallback for backwards compatibility
34 }
35
36 // Just ensure UTF-8 validity without aggressive escaping
37 $safe_content = wp_check_invalid_utf8($content);
38 // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
39 $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
40
41 // Check if chunking should be applied
42 $chunker = MxChat_Chunker::from_settings();
43 if ($chunker->should_chunk($safe_content)) {
44 //error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission');
45 return self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
46 }
47
48 // UPDATED: Generate the embedding using bot-specific configuration
49 $embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
50
51 if (!is_array($embedding_vector)) {
52 //error_log('[MXCHAT-DB] Error: Embedding generation failed');
53 return new WP_Error('embedding_failed', 'Failed to generate embedding for content');
54 }
55
56 //error_log('[MXCHAT-DB] Embedding generated successfully');
57
58 // UPDATED: Check if Pinecone is enabled for this specific bot
59 if (self::is_pinecone_enabled_for_bot($bot_id)) {
60 //error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
61 // Store in Pinecone only
62 return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
63 } else {
64 //error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
65 // Store in WordPress database only
66 $embedding_vector_serialized = maybe_serialize($embedding_vector);
67 return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
68 }
69 }
70
71 /**
72 * UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
73 */
74 private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
75 // For default bot or when multi-bot is not active, use original method
76 if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
77 return self::is_pinecone_enabled();
78 }
79
80 // Get bot-specific Pinecone configuration
81 $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
82
83 if (empty($bot_pinecone_config)) {
84 // Fallback to default configuration
85 return self::is_pinecone_enabled();
86 }
87
88 $enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
89 $api_key_check = !empty($bot_pinecone_config['api_key']);
90 $host_check = !empty($bot_pinecone_config['host']);
91
92 return $enabled_check && $api_key_check && $host_check;
93 }
94
95 /**
96 * Check if Pinecone is enabled and properly configured (original method for default bot)
97 */
98 private static function is_pinecone_enabled() {
99 $pinecone_options = get_option('mxchat_pinecone_addon_options');
100
101 if (empty($pinecone_options)) {
102 return false;
103 }
104
105 $enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
106 $api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
107 $host_check = !empty($pinecone_options['mxchat_pinecone_host']);
108
109 return $enabled_check && $api_key_check && $host_check;
110 }
111
112 /**
113 * UPDATED: Store content in Pinecone only with bot support
114 */
115 private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
116 //error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
117
118 // Get bot-specific Pinecone configuration
119 if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
120 $pinecone_options = get_option('mxchat_pinecone_addon_options');
121 $api_key = $pinecone_options['mxchat_pinecone_api_key'];
122 $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
123 $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
124 $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
125 } else {
126 $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
127 if (empty($bot_pinecone_config)) {
128 // Fallback to default configuration
129 $pinecone_options = get_option('mxchat_pinecone_addon_options');
130 $api_key = $pinecone_options['mxchat_pinecone_api_key'];
131 $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
132 $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
133 $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
134 } else {
135 $api_key = $bot_pinecone_config['api_key'];
136 $environment = ''; // Not used in new Pinecone API
137 $index_name = ''; // Not used in new Pinecone API
138 $namespace = $bot_pinecone_config['namespace'] ?? '';
139 }
140 }
141
142 $result = self::store_in_pinecone_main(
143 $embedding_vector,
144 $content,
145 $source_url,
146 $api_key,
147 $environment,
148 $index_name,
149 $vector_id,
150 $bot_id,
151 $namespace,
152 $content_type
153 );
154
155 if (is_wp_error($result)) {
156 //error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
157 return $result;
158 }
159
160 //error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
161 return true;
162 }
163
164 /**
165 * Store content in WordPress database with progressive fallback
166 * UPDATED 2.5.6: Now includes content_type parameter
167 */
168 private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type = 'content') {
169 global $wpdb;
170
171 //error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
172
173 // Sanitize content_type
174 $content_type = sanitize_key($content_type);
175 if (empty($content_type)) {
176 $content_type = 'content'; // Fallback for backwards compatibility
177 }
178
179 // ===== FIXED: Generate unique identifier for manual content =====
180 $original_source_url = $source_url;
181 // Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
182 // filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
183 // Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
184 $has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
185 // Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
186 $is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
187 $is_manual_content = empty($source_url) || $source_url === '' || !$has_url_scheme || $is_legacy_mxchat_url;
188
189 if ($is_manual_content) {
190 // Generate unique identifier for manual content to prevent overwrites
191 $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
192 //error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
193 }
194
195 // Only check for duplicates if we have a valid source URL (not manual content)
196 $existing_id = null;
197 if (!$is_manual_content) {
198 $existing_id = $wpdb->get_var(
199 $wpdb->prepare(
200 "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
201 $source_url
202 )
203 );
204 //error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
205 } else {
206 //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
207 }
208 // ===== END FIX =====
209
210 // Progressive fallback mechanism for problematic content
211 $attempt = 1;
212 $max_attempts = 3;
213 $current_content = $safe_content;
214 $result = false;
215
216 while ($attempt <= $max_attempts && $result === false) {
217 try {
218 if ($existing_id) {
219 //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
220
221 // Update the existing row - UPDATED 2.5.6: Added content_type
222 $result = $wpdb->update(
223 $table_name,
224 array(
225 'url' => $source_url,
226 'article_content' => $current_content,
227 'embedding_vector' => $embedding_vector_serialized,
228 'source_url' => $source_url,
229 'content_type' => $content_type,
230 'timestamp' => current_time('mysql'),
231 ),
232 array('id' => $existing_id),
233 array('%s','%s','%s','%s','%s','%s'),
234 array('%d')
235 );
236 } else {
237 //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
238 //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
239
240 // Insert a new row - UPDATED 2.5.6: Added content_type
241 $result = $wpdb->insert(
242 $table_name,
243 array(
244 'url' => $source_url, // Now unique for manual content
245 'article_content' => $current_content,
246 'embedding_vector' => $embedding_vector_serialized,
247 'source_url' => $source_url, // Now unique for manual content
248 'content_type' => $content_type,
249 'timestamp' => current_time('mysql'),
250 ),
251 array('%s','%s','%s','%s','%s','%s')
252 );
253 }
254
255 if ($result === false) {
256 //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')');
257 //error_log('[MXCHAT-DB] MySQL Error: ' . $wpdb->last_error);
258 //error_log('[MXCHAT-DB] MySQL Error Number: ' . $wpdb->last_errno);
259 //error_log('[MXCHAT-DB] Last Query: ' . substr($wpdb->last_query, 0, 500));
260 //error_log('[MXCHAT-DB] Content length: ' . strlen($current_content) . ' bytes');
261 //error_log('[MXCHAT-DB] Embedding vector length: ' . strlen($embedding_vector_serialized) . ' bytes');
262
263 // Progressively apply more aggressive sanitization on failure
264 if ($attempt === 1) {
265 // First fallback: Use a more aggressive character filter and shorten
266 $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
267 $current_content = substr($current_content, 0, 50000);
268 } else if ($attempt === 2) {
269 // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
270 $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
271 $current_content = substr($current_content, 0, 30000);
272 }
273
274 $attempt++;
275 }
276 } catch (Exception $e) {
277 //error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
278 $attempt++;
279 }
280 }
281
282 if ($result === false) {
283 //error_log('[MXCHAT-DB] All database operation attempts failed');
284 //error_log('[MXCHAT-DB] Final MySQL Error: ' . $wpdb->last_error);
285
286 $detailed_error = sprintf(
287 'Failed to store content in WordPress database after %d attempts. MySQL Error: %s (Error #%d). Content size: %d bytes, Embedding size: %d bytes',
288 $max_attempts,
289 $wpdb->last_error,
290 $wpdb->last_errno,
291 strlen($current_content),
292 strlen($embedding_vector_serialized)
293 );
294
295 return new WP_Error('database_failed', $detailed_error);
296 }
297
298 //error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
299 return true;
300 }
301
302 /**
303 * UPDATED: Store content in Pinecone database with bot support
304 * UPDATED 2.5.6: Now accepts content_type parameter
305 */
306 private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '', $content_type = 'content') {
307 //error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
308
309 // ===== UPDATED: Handle manual content with unique vector IDs =====
310 if ($vector_id) {
311 // Use provided vector ID
312 //error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
313 } elseif (!empty($url) && preg_match('#^https?://#i', $url)) {
314 // For URLs, use URL-based ID (existing behavior)
315 $vector_id = md5($url);
316 //error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
317 } else {
318 // For manual content (empty/no URL scheme), generate unique ID
319 $vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
320 //error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
321 }
322 // ===== END UPDATE =====
323
324 // Get host from bot-specific config or fallback to default
325 if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
326 $options = get_option('mxchat_pinecone_addon_options');
327 $host = $options['mxchat_pinecone_host'] ?? '';
328 } else {
329 $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
330 if (!empty($bot_pinecone_config)) {
331 $host = $bot_pinecone_config['host'] ?? '';
332 } else {
333 $options = get_option('mxchat_pinecone_addon_options');
334 $host = $options['mxchat_pinecone_host'] ?? '';
335 }
336 }
337
338 //error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
339 //error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
340 //error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
341 //error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
342
343 if (empty($host)) {
344 //error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
345 return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
346 }
347
348 // ===== UPDATED 2.5.6: Use passed content_type or determine from URL if not provided =====
349 // Sanitize content_type
350 $content_type = sanitize_key($content_type);
351 if (empty($content_type)) {
352 // Fallback to old detection logic for backwards compatibility
353 $is_product = false;
354 $content_type = 'manual'; // Default for manual content
355
356 if (!empty($url) && preg_match('#^https?://#i', $url)) {
357 $is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
358 $content_type = $is_product ? 'product' : 'content';
359 }
360 }
361
362 //error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
363 // ===== END UPDATE =====
364
365 $api_endpoint = "https://{$host}/vectors/upsert";
366 //error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
367
368 // UPDATED 2.5.6: Use provided content_type in metadata
369 $metadata = array(
370 'text' => $content,
371 'source_url' => $url, // Can be empty for manual content
372 'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
373 'last_updated' => time(),
374 'created_at' => time(), // Add creation timestamp
375 'bot_id' => $bot_id // Add bot identification
376 );
377
378 $vector_data = array(
379 'id' => $vector_id,
380 'values' => $embedding_vector,
381 'metadata' => $metadata
382 );
383
384 $request_body = array(
385 'vectors' => array($vector_data)
386 );
387
388 // Add namespace if specified for multi-bot separation
389 if (!empty($namespace)) {
390 $request_body['namespace'] = $namespace;
391 //error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
392 }
393
394 //error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
395
396 $response = wp_remote_post($api_endpoint, array(
397 'headers' => array(
398 'Api-Key' => $api_key,
399 'accept' => 'application/json',
400 'content-type' => 'application/json'
401 ),
402 'body' => wp_json_encode($request_body),
403 'timeout' => 30,
404 'data_format' => 'body'
405 ));
406
407 if (is_wp_error($response)) {
408 //error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
409 return new WP_Error('pinecone_request', $response->get_error_message());
410 }
411
412 $response_code = wp_remote_retrieve_response_code($response);
413 //error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
414
415 if ($response_code !== 200) {
416 $body = wp_remote_retrieve_body($response);
417 //error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
418 return new WP_Error('pinecone_api', sprintf(
419 'Pinecone API error (HTTP %d): %s',
420 $response_code,
421 $body
422 ));
423 }
424
425 $response_body = wp_remote_retrieve_body($response);
426 //error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
427 //error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
428 //error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
429
430 return true;
431 }
432
433 /**
434 * UPDATED: Generate an embedding for the given text using bot-specific configuration.
435 *
436 * @param string $text The text to be embedded.
437 * @param string $api_key The API key used for generating embeddings.
438 * @param string $bot_id The bot ID for multi-bot support
439 * @return array|null The embedding vector or null on failure.
440 */
441 private static function generate_embedding($text, $api_key, $bot_id = 'default') {
442 // Get bot-specific options
443 if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
444 $options = get_option('mxchat_options');
445 } else {
446 $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
447 $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
448 }
449
450 $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
451
452 // Determine endpoint and API key based on model
453 if (strpos($selected_model, 'voyage') === 0) {
454 $endpoint = 'https://api.voyageai.com/v1/embeddings';
455 $api_key = $options['voyage_api_key'] ?? '';
456 } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
457 $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
458 $api_key = $options['gemini_api_key'] ?? '';
459 } else {
460 $endpoint = 'https://api.openai.com/v1/embeddings';
461 // Use the bot-specific API key or fallback to passed API key
462 $api_key = $options['api_key'] ?? $api_key;
463 }
464
465 // Prepare request body based on provider
466 if (strpos($selected_model, 'gemini-embedding') === 0) {
467 // Gemini API format
468 $request_body = [
469 'model' => 'models/' . $selected_model,
470 'content' => [
471 'parts' => [
472 ['text' => $text]
473 ]
474 ],
475 'outputDimensionality' => 1536
476 ];
477
478 // Prepare headers for Gemini (API key as query parameter)
479 $endpoint .= '?key=' . $api_key;
480 $headers = [
481 'Content-Type' => 'application/json'
482 ];
483 } else {
484 // OpenAI/Voyage API format
485 $request_body = [
486 'input' => $text,
487 'model' => $selected_model
488 ];
489
490 // Add output_dimension for voyage-3-large
491 if ($selected_model === 'voyage-3-large') {
492 $request_body['output_dimension'] = 2048;
493 }
494
495 // Prepare headers for OpenAI/Voyage
496 $headers = [
497 'Content-Type' => 'application/json',
498 'Authorization' => 'Bearer ' . $api_key
499 ];
500 }
501
502 $args = [
503 'body' => wp_json_encode($request_body),
504 'headers' => $headers,
505 'timeout' => 60,
506 'redirection' => 5,
507 'blocking' => true,
508 'httpversion' => '1.0',
509 'sslverify' => true,
510 ];
511
512 $response = wp_remote_post($endpoint, $args);
513
514 if (is_wp_error($response)) {
515 //error_log('Error generating embedding for bot ' . $bot_id . ': ' . $response->get_error_message());
516 return null;
517 }
518
519 $response_body = json_decode(wp_remote_retrieve_body($response), true);
520
521 // Handle different response formats based on provider
522 if (strpos($selected_model, 'gemini-embedding') === 0) {
523 // Gemini API response format
524 if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
525 return $response_body['embedding']['values'];
526 } else {
527 //error_log('Invalid response received from Gemini embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
528 return null;
529 }
530 } else {
531 // OpenAI/Voyage API response format
532 if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
533 return $response_body['data'][0]['embedding'];
534 } else {
535 //error_log('Invalid response received from embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
536 return null;
537 }
538 }
539 }
540
541 /**
542 * Submit content as multiple chunks
543 *
544 * Splits large content into chunks, generates embeddings for each,
545 * and stores them with chunk metadata for later reassembly.
546 *
547 * @param string $content The content to chunk and store
548 * @param string $source_url The source URL
549 * @param string $api_key The API key for embeddings
550 * @param string $bot_id The bot ID
551 * @param string $content_type The content type
552 * @param MxChat_Chunker $chunker The chunker instance
553 * @return bool|WP_Error True on success, WP_Error on failure
554 */
555 private static function submit_chunked_content($content, $source_url, $api_key, $bot_id, $content_type, $chunker) {
556 global $wpdb;
557 $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
558
559 //error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url);
560 //error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars');
561
562 // First, delete any existing chunks for this URL (clean slate)
563 $delete_result = self::delete_chunks_for_url($source_url, $bot_id);
564 if (is_wp_error($delete_result)) {
565 //error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message());
566 // Continue anyway - we'll overwrite with upsert
567 }
568
569 // Split content into chunks
570 $chunks = $chunker->chunk_text($content);
571 $total_chunks = count($chunks);
572
573 //error_log('[MXCHAT-CHUNK-DEBUG] Created ' . $total_chunks . ' chunks');
574 foreach ($chunks as $i => $chunk) {
575 //error_log('[MXCHAT-CHUNK-DEBUG] Chunk ' . $i . ' length: ' . strlen($chunk) . ' chars, preview: ' . substr($chunk, 0, 100));
576 }
577
578 //error_log('[MXCHAT-CHUNK] Split content into ' . $total_chunks . ' chunks');
579
580 if ($total_chunks === 0) {
581 return new WP_Error('chunking_failed', 'Content could not be split into chunks');
582 }
583
584 $errors = array();
585 $is_pinecone = self::is_pinecone_enabled_for_bot($bot_id);
586
587 foreach ($chunks as $index => $chunk_text) {
588 // Generate chunk metadata
589 $chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
590 $chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
591
592 //error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
593
594 // Generate embedding for this chunk
595 $embedding_vector = self::generate_embedding($chunk_text, $api_key, $bot_id);
596
597 if (!is_array($embedding_vector)) {
598 $errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index);
599 //error_log('[MXCHAT-CHUNK] Failed to generate embedding for chunk ' . $index);
600 continue;
601 }
602
603 if ($is_pinecone) {
604 // Store in Pinecone with chunk metadata
605 $result = self::store_chunk_in_pinecone(
606 $embedding_vector,
607 $chunk_text,
608 $source_url,
609 $chunk_vector_id,
610 $bot_id,
611 $content_type,
612 $chunk_metadata
613 );
614 } else {
615 // Store in WordPress DB with chunk metadata
616 $content_with_metadata = MxChat_Chunker::format_chunk_for_storage($chunk_text, $chunk_metadata);
617 $embedding_vector_serialized = maybe_serialize($embedding_vector);
618
619 $result = self::store_chunk_in_wordpress_db(
620 $content_with_metadata,
621 $source_url,
622 $embedding_vector_serialized,
623 $table_name,
624 $content_type,
625 $chunk_metadata
626 );
627 }
628
629 if (is_wp_error($result)) {
630 $errors[] = $result;
631 //error_log('[MXCHAT-CHUNK] Failed to store chunk ' . $index . ': ' . $result->get_error_message());
632 }
633 }
634
635 if (count($errors) === $total_chunks) {
636 return new WP_Error('chunking_failed', 'Failed to store any chunks');
637 }
638
639 if (!empty($errors)) {
640 //error_log('[MXCHAT-CHUNK] Completed with ' . count($errors) . ' errors out of ' . $total_chunks . ' chunks');
641 return new WP_Error('chunking_partial_failure',
642 sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks));
643 }
644
645 //error_log('[MXCHAT-CHUNK] Successfully stored all ' . $total_chunks . ' chunks');
646 return true;
647 }
648
649 /**
650 * Store a single chunk in Pinecone with chunk-specific metadata
651 */
652 private static function store_chunk_in_pinecone($embedding_vector, $chunk_text, $source_url, $vector_id, $bot_id, $content_type, $chunk_metadata) {
653 // Get Pinecone configuration
654 if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
655 $pinecone_options = get_option('mxchat_pinecone_addon_options');
656 $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
657 $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
658 $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
659 } else {
660 $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
661 if (empty($bot_pinecone_config)) {
662 $pinecone_options = get_option('mxchat_pinecone_addon_options');
663 $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
664 $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
665 $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
666 } else {
667 $api_key = $bot_pinecone_config['api_key'] ?? '';
668 $host = $bot_pinecone_config['host'] ?? '';
669 $namespace = $bot_pinecone_config['namespace'] ?? '';
670 }
671 }
672
673 if (empty($host) || empty($api_key)) {
674 return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
675 }
676
677 $api_endpoint = "https://{$host}/vectors/upsert";
678
679 // Build metadata with chunk information
680 $metadata = array(
681 'text' => $chunk_text,
682 'source_url' => $source_url,
683 'type' => $content_type,
684 'is_chunked' => true,
685 'chunk_index' => $chunk_metadata['chunk_index'],
686 'total_chunks' => $chunk_metadata['total_chunks'],
687 'parent_url_hash' => $chunk_metadata['parent_url_hash'],
688 'last_updated' => time(),
689 'created_at' => time(),
690 'bot_id' => $bot_id
691 );
692
693 $vector_data = array(
694 'id' => $vector_id,
695 'values' => $embedding_vector,
696 'metadata' => $metadata
697 );
698
699 $request_body = array(
700 'vectors' => array($vector_data)
701 );
702
703 if (!empty($namespace)) {
704 $request_body['namespace'] = $namespace;
705 }
706
707 $response = wp_remote_post($api_endpoint, array(
708 'headers' => array(
709 'Api-Key' => $api_key,
710 'accept' => 'application/json',
711 'content-type' => 'application/json'
712 ),
713 'body' => wp_json_encode($request_body),
714 'timeout' => 30
715 ));
716
717 if (is_wp_error($response)) {
718 return $response;
719 }
720
721 $response_code = wp_remote_retrieve_response_code($response);
722 if ($response_code !== 200) {
723 return new WP_Error('pinecone_api', 'Pinecone API error: HTTP ' . $response_code);
724 }
725
726 return true;
727 }
728
729 /**
730 * Store a single chunk in WordPress database
731 */
732 private static function store_chunk_in_wordpress_db($content_with_metadata, $source_url, $embedding_vector_serialized, $table_name, $content_type, $chunk_metadata) {
733 global $wpdb;
734
735 // For chunks, we always insert new rows (no duplicate checking)
736 // The URL includes chunk info in the metadata, but source_url stays the same for grouping
737 $result = $wpdb->insert(
738 $table_name,
739 array(
740 'url' => $source_url,
741 'article_content' => $content_with_metadata,
742 'embedding_vector' => $embedding_vector_serialized,
743 'source_url' => $source_url,
744 'content_type' => $content_type,
745 'timestamp' => current_time('mysql')
746 ),
747 array('%s', '%s', '%s', '%s', '%s', '%s')
748 );
749
750 if ($result === false) {
751 return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error);
752 }
753
754 return true;
755 }
756
757 /**
758 * Delete all chunks for a given URL
759 *
760 * @param string $source_url The source URL
761 * @param string $bot_id The bot ID
762 * @return bool|WP_Error True on success, WP_Error on failure
763 */
764 public static function delete_chunks_for_url($source_url, $bot_id = 'default') {
765 //error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url);
766
767 if (self::is_pinecone_enabled_for_bot($bot_id)) {
768 return self::delete_pinecone_chunks_by_url($source_url, $bot_id);
769 } else {
770 return self::delete_wordpress_chunks_by_url($source_url);
771 }
772 }
773
774 /**
775 * Delete all chunks for a URL from Pinecone
776 */
777 private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
778 // Get Pinecone configuration
779 if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
780 $pinecone_options = get_option('mxchat_pinecone_addon_options');
781 $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
782 $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
783 $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
784 } else {
785 $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
786 if (empty($bot_pinecone_config)) {
787 $pinecone_options = get_option('mxchat_pinecone_addon_options');
788 $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
789 $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
790 $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
791 } else {
792 $api_key = $bot_pinecone_config['api_key'] ?? '';
793 $host = $bot_pinecone_config['host'] ?? '';
794 $namespace = $bot_pinecone_config['namespace'] ?? '';
795 }
796 }
797
798 if (empty($host) || empty($api_key)) {
799 return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
800 }
801
802 $base_vector_id = md5($source_url);
803 $vectors_to_delete = array();
804
805 // Add the original single-vector ID (for non-chunked content)
806 $vectors_to_delete[] = $base_vector_id;
807
808 // Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
809 // non-200 silently and we end up only deleting the base vector, leaving chunks orphaned.
810 $query_params = array(
811 'prefix' => $base_vector_id . '_chunk_',
812 'limit' => 100,
813 );
814 if (!empty($namespace)) {
815 $query_params['namespace'] = $namespace;
816 }
817
818 $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
819
820 // Paginate in case a URL has more than 100 chunks.
821 do {
822 $list_response = wp_remote_get($list_url, array(
823 'headers' => array(
824 'Api-Key' => $api_key,
825 'accept' => 'application/json',
826 ),
827 'timeout' => 30,
828 ));
829
830 if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) {
831 break;
832 }
833
834 $list_data = json_decode(wp_remote_retrieve_body($list_response), true);
835 if (!empty($list_data['vectors'])) {
836 foreach ($list_data['vectors'] as $vector) {
837 if (isset($vector['id'])) {
838 $vectors_to_delete[] = $vector['id'];
839 }
840 }
841 }
842
843 $next_token = $list_data['pagination']['next'] ?? '';
844 if (empty($next_token)) {
845 break;
846 }
847
848 $query_params['paginationToken'] = $next_token;
849 $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
850 } while (true);
851
852 if (empty($vectors_to_delete)) {
853 //error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
854 return true;
855 }
856
857 //error_log('[MXCHAT-CHUNK-DELETE] Deleting ' . count($vectors_to_delete) . ' vectors from Pinecone');
858
859 // Delete vectors
860 $delete_url = "https://{$host}/vectors/delete";
861
862 $delete_body = array(
863 'ids' => $vectors_to_delete
864 );
865
866 if (!empty($namespace)) {
867 $delete_body['namespace'] = $namespace;
868 }
869
870 $delete_response = wp_remote_post($delete_url, array(
871 'headers' => array(
872 'Api-Key' => $api_key,
873 'accept' => 'application/json',
874 'content-type' => 'application/json'
875 ),
876 'body' => wp_json_encode($delete_body),
877 'timeout' => 30
878 ));
879
880 if (is_wp_error($delete_response)) {
881 return $delete_response;
882 }
883
884 $response_code = wp_remote_retrieve_response_code($delete_response);
885 if ($response_code !== 200) {
886 return new WP_Error('pinecone_delete', 'Failed to delete vectors: HTTP ' . $response_code);
887 }
888
889 return true;
890 }
891
892 /**
893 * Delete all chunks for a URL from WordPress database
894 */
895 private static function delete_wordpress_chunks_by_url($source_url) {
896 global $wpdb;
897 $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
898
899 // Delete all rows with this source_url (handles both chunked and non-chunked)
900 $result = $wpdb->delete(
901 $table_name,
902 array('source_url' => $source_url),
903 array('%s')
904 );
905
906 if ($result === false) {
907 return new WP_Error('database_delete', 'Failed to delete chunks: ' . $wpdb->last_error);
908 }
909
910 //error_log('[MXCHAT-CHUNK-DELETE] Deleted ' . $result . ' rows from WordPress DB');
911 return true;
912 }
913 }