PluginProbe
MxChat – AI Chatbot & Content Generation for WordPress / 1.0.8
MxChat – AI Chatbot & Content Generation for WordPress v1.0.8
3.2.21 3.2.20 3.2.19 3.2.18 3.2.17 3.2.16 3.2.15 3.2.14 3.2.12 3.2.13 3.2.11 3.2.10 3.2.9 3.2.8 3.2.7 3.2.6 3.2.5 3.2.4 3.2.3 3.2.2 3.2.1 2.0.3 2.0.4 2.0.5 2.0.6 All 152 releases
← All changes | includes/class-mxchat-utils.php +93 -958 3.2.5 → 1.0.8 View file →
@@ -1,958 +1,93 @@
1 -<?php
2 -if (!defined('ABSPATH')) {
3 - exit; // Exit if accessed directly
4 -}
5 -
6 -class MxChat_Utils {
7 -
8 -/**
9 - * Centralized embedding model registry. Single source of truth for dimensions
10 - * and provider, so model-switch protection logic doesn't drift across files.
11 - */
12 -public static function embedding_model_registry() {
13 - return array(
14 - 'text-embedding-ada-002' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'Ada 2'),
15 - 'text-embedding-3-small' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'TE3 Small'),
16 - 'text-embedding-3-large' => array('dims' => 3072, 'provider' => 'openai', 'label' => 'TE3 Large'),
17 - 'voyage-3-large' => array('dims' => 2048, 'provider' => 'voyage', 'label' => 'Voyage-3 Large'),
18 - 'gemini-embedding-001' => array('dims' => 1536, 'provider' => 'gemini', 'label' => 'Gemini Embedding'),
19 - );
20 -}
21 -
22 -public static function embedding_model_dimensions($model) {
23 - $registry = self::embedding_model_registry();
24 - return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0;
25 -}
26 -
27 -public static function embedding_model_label($model) {
28 - $registry = self::embedding_model_registry();
29 - return isset($registry[$model]) ? $registry[$model]['label'] : $model;
30 -}
31 -
32 -/**
33 - * Returns the model that was last used to actually write embeddings into the
34 - * KB. Differs from the user-selected setting once a switch has happened but
35 - * no re-embed has occurred yet — that's the mismatch state we warn about.
36 - */
37 -public static function get_active_embedding_model() {
38 - return get_option('mxchat_active_embedding_model', '');
39 -}
40 -
41 -/**
42 - * Stamp the model that produced the most recent successful embedding. Called
43 - * from generate_embedding() right after the API responds with a valid vector.
44 - */
45 -public static function stamp_active_embedding_model($model) {
46 - if (!empty($model) && $model !== self::get_active_embedding_model()) {
47 - update_option('mxchat_active_embedding_model', $model, false);
48 - }
49 -}
50 -
51 -/**
52 - * UPDATED: Submit or update content (and its embedding) in the database.
53 - * Stores in Pinecone if enabled, otherwise stores in WordPress DB.
54 - *
55 - * @param string $content The content to be embedded.
56 - * @param string $source_url The source URL of the content.
57 - * @param string $api_key The API key used for generating embeddings.
58 - * @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL)
59 - * @param string $bot_id The bot ID for multi-bot support
60 - * @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.)
61 - * @return bool|WP_Error True on success, WP_Error on failure
62 - */
63 -public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
64 - global $wpdb;
65 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
66 -
67 - //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')');
68 - //error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes');
69 -
70 - // Sanitize the source URL
71 - $source_url = esc_url_raw($source_url);
72 -
73 - // Sanitize content_type
74 - $content_type = sanitize_key($content_type);
75 - if (empty($content_type)) {
76 - $content_type = 'content'; // Fallback for backwards compatibility
77 - }
78 -
79 - // Just ensure UTF-8 validity without aggressive escaping
80 - $safe_content = wp_check_invalid_utf8($content);
81 - // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D)
82 - $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content);
83 -
84 - // Check if chunking should be applied
85 - $chunker = MxChat_Chunker::from_settings();
86 - if ($chunker->should_chunk($safe_content)) {
87 - //error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission');
88 - return self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker);
89 - }
90 -
91 - // UPDATED: Generate the embedding using bot-specific configuration
92 - $embedding_vector = self::generate_embedding($content, $api_key, $bot_id);
93 -
94 - if (!is_array($embedding_vector)) {
95 - //error_log('[MXCHAT-DB] Error: Embedding generation failed');
96 - return new WP_Error('embedding_failed', 'Failed to generate embedding for content');
97 - }
98 -
99 - //error_log('[MXCHAT-DB] Embedding generated successfully');
100 -
101 - // UPDATED: Check if Pinecone is enabled for this specific bot
102 - if (self::is_pinecone_enabled_for_bot($bot_id)) {
103 - //error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage');
104 - // Store in Pinecone only
105 - return self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type);
106 - } else {
107 - //error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage');
108 - // Store in WordPress database only
109 - $embedding_vector_serialized = maybe_serialize($embedding_vector);
110 - return self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type);
111 - }
112 -}
113 -
114 -/**
115 - * UPDATED: Check if Pinecone is enabled and properly configured for a specific bot
116 - */
117 -private static function is_pinecone_enabled_for_bot($bot_id = 'default') {
118 - // For default bot or when multi-bot is not active, use original method
119 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
120 - return self::is_pinecone_enabled();
121 - }
122 -
123 - // Get bot-specific Pinecone configuration
124 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
125 -
126 - if (empty($bot_pinecone_config)) {
127 - // Fallback to default configuration
128 - return self::is_pinecone_enabled();
129 - }
130 -
131 - $enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone'];
132 - $api_key_check = !empty($bot_pinecone_config['api_key']);
133 - $host_check = !empty($bot_pinecone_config['host']);
134 -
135 - return $enabled_check && $api_key_check && $host_check;
136 -}
137 -
138 -/**
139 - * Check if Pinecone is enabled and properly configured (original method for default bot)
140 - */
141 -private static function is_pinecone_enabled() {
142 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
143 -
144 - if (empty($pinecone_options)) {
145 - return false;
146 - }
147 -
148 - $enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0';
149 - $api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']);
150 - $host_check = !empty($pinecone_options['mxchat_pinecone_host']);
151 -
152 - return $enabled_check && $api_key_check && $host_check;
153 -}
154 -
155 -/**
156 - * UPDATED: Store content in Pinecone only with bot support
157 - */
158 -private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default', $content_type = 'content') {
159 - //error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' =====');
160 -
161 - // Get bot-specific Pinecone configuration
162 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
163 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
164 - $api_key = $pinecone_options['mxchat_pinecone_api_key'];
165 - $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
166 - $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
167 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
168 - } else {
169 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
170 - if (empty($bot_pinecone_config)) {
171 - // Fallback to default configuration
172 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
173 - $api_key = $pinecone_options['mxchat_pinecone_api_key'];
174 - $environment = $pinecone_options['mxchat_pinecone_environment'] ?? '';
175 - $index_name = $pinecone_options['mxchat_pinecone_index'] ?? '';
176 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
177 - } else {
178 - $api_key = $bot_pinecone_config['api_key'];
179 - $environment = ''; // Not used in new Pinecone API
180 - $index_name = ''; // Not used in new Pinecone API
181 - $namespace = $bot_pinecone_config['namespace'] ?? '';
182 - }
183 - }
184 -
185 - $result = self::store_in_pinecone_main(
186 - $embedding_vector,
187 - $content,
188 - $source_url,
189 - $api_key,
190 - $environment,
191 - $index_name,
192 - $vector_id,
193 - $bot_id,
194 - $namespace,
195 - $content_type
196 - );
197 -
198 - if (is_wp_error($result)) {
199 - //error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message());
200 - return $result;
201 - }
202 -
203 - //error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id);
204 - return true;
205 -}
206 -
207 -/**
208 - * Store content in WordPress database with progressive fallback
209 - * UPDATED 2.5.6: Now includes content_type parameter
210 - */
211 -private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type = 'content') {
212 - global $wpdb;
213 -
214 - //error_log('[MXCHAT-DB] ===== Using WordPress-only storage =====');
215 -
216 - // Sanitize content_type
217 - $content_type = sanitize_key($content_type);
218 - if (empty($content_type)) {
219 - $content_type = 'content'; // Fallback for backwards compatibility
220 - }
221 -
222 - // ===== FIXED: Generate unique identifier for manual content =====
223 - $original_source_url = $source_url;
224 - // Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects
225 - // filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc.
226 - // Use a looser check: if it starts with http(s):// or has a scheme, it's a URL
227 - $has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url);
228 - // Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
229 - $is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false;
230 - $is_manual_content = empty($source_url) || $source_url === '' || !$has_url_scheme || $is_legacy_mxchat_url;
231 -
232 - if ($is_manual_content) {
233 - // Generate unique identifier for manual content to prevent overwrites
234 - $source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false);
235 - //error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url);
236 - }
237 -
238 - // Only check for duplicates if we have a valid source URL (not manual content)
239 - $existing_id = null;
240 - if (!$is_manual_content) {
241 - $existing_id = $wpdb->get_var(
242 - $wpdb->prepare(
243 - "SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1",
244 - $source_url
245 - )
246 - );
247 - //error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none'));
248 - } else {
249 - //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)');
250 - }
251 - // ===== END FIX =====
252 -
253 - // Progressive fallback mechanism for problematic content
254 - $attempt = 1;
255 - $max_attempts = 3;
256 - $current_content = $safe_content;
257 - $result = false;
258 -
259 - while ($attempt <= $max_attempts && $result === false) {
260 - try {
261 - if ($existing_id) {
262 - //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')');
263 -
264 - // Update the existing row - UPDATED 2.5.6: Added content_type
265 - $result = $wpdb->update(
266 - $table_name,
267 - array(
268 - 'url' => $source_url,
269 - 'article_content' => $current_content,
270 - 'embedding_vector' => $embedding_vector_serialized,
271 - 'source_url' => $source_url,
272 - 'content_type' => $content_type,
273 - 'timestamp' => current_time('mysql'),
274 - ),
275 - array('id' => $existing_id),
276 - array('%s','%s','%s','%s','%s','%s'),
277 - array('%d')
278 - );
279 - } else {
280 - //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')');
281 - //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000));
282 -
283 - // Insert a new row - UPDATED 2.5.6: Added content_type
284 - $result = $wpdb->insert(
285 - $table_name,
286 - array(
287 - 'url' => $source_url, // Now unique for manual content
288 - 'article_content' => $current_content,
289 - 'embedding_vector' => $embedding_vector_serialized,
290 - 'source_url' => $source_url, // Now unique for manual content
291 - 'content_type' => $content_type,
292 - 'timestamp' => current_time('mysql'),
293 - ),
294 - array('%s','%s','%s','%s','%s','%s')
295 - );
296 - }
297 -
298 - if ($result === false) {
299 - //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')');
300 - //error_log('[MXCHAT-DB] MySQL Error: ' . $wpdb->last_error);
301 - //error_log('[MXCHAT-DB] MySQL Error Number: ' . $wpdb->last_errno);
302 - //error_log('[MXCHAT-DB] Last Query: ' . substr($wpdb->last_query, 0, 500));
303 - //error_log('[MXCHAT-DB] Content length: ' . strlen($current_content) . ' bytes');
304 - //error_log('[MXCHAT-DB] Embedding vector length: ' . strlen($embedding_vector_serialized) . ' bytes');
305 -
306 - // Progressively apply more aggressive sanitization on failure
307 - if ($attempt === 1) {
308 - // First fallback: Use a more aggressive character filter and shorten
309 - $current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content);
310 - $current_content = substr($current_content, 0, 50000);
311 - } else if ($attempt === 2) {
312 - // Second fallback: Keep only alphanumeric and basic punctuation, shorten further
313 - $current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content);
314 - $current_content = substr($current_content, 0, 30000);
315 - }
316 -
317 - $attempt++;
318 - }
319 - } catch (Exception $e) {
320 - //error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage());
321 - $attempt++;
322 - }
323 - }
324 -
325 -if ($result === false) {
326 - //error_log('[MXCHAT-DB] All database operation attempts failed');
327 - //error_log('[MXCHAT-DB] Final MySQL Error: ' . $wpdb->last_error);
328 -
329 - $detailed_error = sprintf(
330 - 'Failed to store content in WordPress database after %d attempts. MySQL Error: %s (Error #%d). Content size: %d bytes, Embedding size: %d bytes',
331 - $max_attempts,
332 - $wpdb->last_error,
333 - $wpdb->last_errno,
334 - strlen($current_content),
335 - strlen($embedding_vector_serialized)
336 - );
337 -
338 - return new WP_Error('database_failed', $detailed_error);
339 -}
340 -
341 - //error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')');
342 - return true;
343 -}
344 -
345 -/**
346 - * UPDATED: Store content in Pinecone database with bot support
347 - * UPDATED 2.5.6: Now accepts content_type parameter
348 - */
349 -private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '', $content_type = 'content') {
350 - //error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' =====');
351 -
352 - // ===== UPDATED: Handle manual content with unique vector IDs =====
353 - if ($vector_id) {
354 - // Use provided vector ID
355 - //error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id);
356 - } elseif (!empty($url) && preg_match('#^https?://#i', $url)) {
357 - // For URLs, use URL-based ID (existing behavior)
358 - $vector_id = md5($url);
359 - //error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id);
360 - } else {
361 - // For manual content (empty/no URL scheme), generate unique ID
362 - $vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8);
363 - //error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id);
364 - }
365 - // ===== END UPDATE =====
366 -
367 - // Get host from bot-specific config or fallback to default
368 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
369 - $options = get_option('mxchat_pinecone_addon_options');
370 - $host = $options['mxchat_pinecone_host'] ?? '';
371 - } else {
372 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
373 - if (!empty($bot_pinecone_config)) {
374 - $host = $bot_pinecone_config['host'] ?? '';
375 - } else {
376 - $options = get_option('mxchat_pinecone_addon_options');
377 - $host = $options['mxchat_pinecone_host'] ?? '';
378 - }
379 - }
380 -
381 - //error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host);
382 - //error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key));
383 - //error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id);
384 - //error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace);
385 -
386 - if (empty($host)) {
387 - //error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty');
388 - return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.');
389 - }
390 -
391 - // ===== UPDATED 2.5.6: Use passed content_type or determine from URL if not provided =====
392 - // Sanitize content_type
393 - $content_type = sanitize_key($content_type);
394 - if (empty($content_type)) {
395 - // Fallback to old detection logic for backwards compatibility
396 - $is_product = false;
397 - $content_type = 'manual'; // Default for manual content
398 -
399 - if (!empty($url) && preg_match('#^https?://#i', $url)) {
400 - $is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
401 - $content_type = $is_product ? 'product' : 'content';
402 - }
403 - }
404 -
405 - //error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type);
406 - // ===== END UPDATE =====
407 -
408 - $api_endpoint = "https://{$host}/vectors/upsert";
409 - //error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint);
410 -
411 - // UPDATED 2.5.6: Use provided content_type in metadata
412 - $metadata = array(
413 - 'text' => $content,
414 - 'source_url' => $url, // Can be empty for manual content
415 - 'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc.
416 - 'last_updated' => time(),
417 - 'created_at' => time(), // Add creation timestamp
418 - 'bot_id' => $bot_id, // Add bot identification
419 - );
420 -
421 - $vector_data = array(
422 - 'id' => $vector_id,
423 - 'values' => $embedding_vector,
424 - 'metadata' => $metadata
425 - );
426 -
427 - $request_body = array(
428 - 'vectors' => array($vector_data)
429 - );
430 -
431 - // Add namespace if specified for multi-bot separation
432 - if (!empty($namespace)) {
433 - $request_body['namespace'] = $namespace;
434 - //error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace);
435 - }
436 -
437 - //error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')');
438 -
439 - $response = wp_remote_post($api_endpoint, array(
440 - 'headers' => array(
441 - 'Api-Key' => $api_key,
442 - 'accept' => 'application/json',
443 - 'content-type' => 'application/json'
444 - ),
445 - 'body' => wp_json_encode($request_body),
446 - 'timeout' => 30,
447 - 'data_format' => 'body'
448 - ));
449 -
450 - if (is_wp_error($response)) {
451 - //error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message());
452 - return new WP_Error('pinecone_request', $response->get_error_message());
453 - }
454 -
455 - $response_code = wp_remote_retrieve_response_code($response);
456 - //error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code);
457 -
458 - if ($response_code !== 200) {
459 - $body = wp_remote_retrieve_body($response);
460 - //error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body);
461 - return new WP_Error('pinecone_api', sprintf(
462 - 'Pinecone API error (HTTP %d): %s',
463 - $response_code,
464 - $body
465 - ));
466 - }
467 -
468 - $response_body = wp_remote_retrieve_body($response);
469 - //error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body);
470 - //error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id);
471 - //error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete =====');
472 -
473 - return true;
474 -}
475 -
476 -/**
477 - * UPDATED: Generate an embedding for the given text using bot-specific configuration.
478 - *
479 - * @param string $text The text to be embedded.
480 - * @param string $api_key The API key used for generating embeddings.
481 - * @param string $bot_id The bot ID for multi-bot support
482 - * @return array|null The embedding vector or null on failure.
483 - */
484 -private static function generate_embedding($text, $api_key, $bot_id = 'default') {
485 - // Get bot-specific options
486 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
487 - $options = get_option('mxchat_options');
488 - } else {
489 - $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
490 - $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
491 - }
492 -
493 - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
494 -
495 - // Determine endpoint and API key based on model
496 - if (strpos($selected_model, 'voyage') === 0) {
497 - $endpoint = 'https://api.voyageai.com/v1/embeddings';
498 - $api_key = $options['voyage_api_key'] ?? '';
499 - } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
500 - $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
501 - $api_key = $options['gemini_api_key'] ?? '';
502 - } else {
503 - $endpoint = 'https://api.openai.com/v1/embeddings';
504 - // Use the bot-specific API key or fallback to passed API key
505 - $api_key = $options['api_key'] ?? $api_key;
506 - }
507 -
508 - // Prepare request body based on provider
509 - if (strpos($selected_model, 'gemini-embedding') === 0) {
510 - // Gemini API format
511 - $request_body = [
512 - 'model' => 'models/' . $selected_model,
513 - 'content' => [
514 - 'parts' => [
515 - ['text' => $text]
516 - ]
517 - ],
518 - 'outputDimensionality' => 1536
519 - ];
520 -
521 - // Prepare headers for Gemini (API key as query parameter)
522 - $endpoint .= '?key=' . $api_key;
523 - $headers = [
524 - 'Content-Type' => 'application/json'
525 - ];
526 - } else {
527 - // OpenAI/Voyage API format
528 - $request_body = [
529 - 'input' => $text,
530 - 'model' => $selected_model
531 - ];
532 -
533 - // Add output_dimension for voyage-3-large
534 - if ($selected_model === 'voyage-3-large') {
535 - $request_body['output_dimension'] = 2048;
536 - }
537 -
538 - // Prepare headers for OpenAI/Voyage
539 - $headers = [
540 - 'Content-Type' => 'application/json',
541 - 'Authorization' => 'Bearer ' . $api_key
542 - ];
543 - }
544 -
545 - $args = [
546 - 'body' => wp_json_encode($request_body),
547 - 'headers' => $headers,
548 - 'timeout' => 60,
549 - 'redirection' => 5,
550 - 'blocking' => true,
551 - 'httpversion' => '1.0',
552 - 'sslverify' => true,
553 - ];
554 -
555 - $response = wp_remote_post($endpoint, $args);
556 -
557 - if (is_wp_error($response)) {
558 - //error_log('Error generating embedding for bot ' . $bot_id . ': ' . $response->get_error_message());
559 - return null;
560 - }
561 -
562 - $response_body = json_decode(wp_remote_retrieve_body($response), true);
563 -
564 - // Handle different response formats based on provider
565 - if (strpos($selected_model, 'gemini-embedding') === 0) {
566 - // Gemini API response format
567 - if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) {
568 - self::stamp_active_embedding_model($selected_model);
569 - return $response_body['embedding']['values'];
570 - } else {
571 - //error_log('Invalid response received from Gemini embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
572 - return null;
573 - }
574 - } else {
575 - // OpenAI/Voyage API response format
576 - if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
577 - self::stamp_active_embedding_model($selected_model);
578 - return $response_body['data'][0]['embedding'];
579 - } else {
580 - //error_log('Invalid response received from embedding API for bot ' . $bot_id . ': ' . wp_json_encode($response_body));
581 - return null;
582 - }
583 - }
584 -}
585 -
586 -/**
587 - * Submit content as multiple chunks
588 - *
589 - * Splits large content into chunks, generates embeddings for each,
590 - * and stores them with chunk metadata for later reassembly.
591 - *
592 - * @param string $content The content to chunk and store
593 - * @param string $source_url The source URL
594 - * @param string $api_key The API key for embeddings
595 - * @param string $bot_id The bot ID
596 - * @param string $content_type The content type
597 - * @param MxChat_Chunker $chunker The chunker instance
598 - * @return bool|WP_Error True on success, WP_Error on failure
599 - */
600 -private static function submit_chunked_content($content, $source_url, $api_key, $bot_id, $content_type, $chunker) {
601 - global $wpdb;
602 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
603 -
604 - //error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url);
605 - //error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars');
606 -
607 - // First, delete any existing chunks for this URL (clean slate)
608 - $delete_result = self::delete_chunks_for_url($source_url, $bot_id);
609 - if (is_wp_error($delete_result)) {
610 - //error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message());
611 - // Continue anyway - we'll overwrite with upsert
612 - }
613 -
614 - // Split content into chunks
615 - $chunks = $chunker->chunk_text($content);
616 - $total_chunks = count($chunks);
617 -
618 - //error_log('[MXCHAT-CHUNK-DEBUG] Created ' . $total_chunks . ' chunks');
619 - foreach ($chunks as $i => $chunk) {
620 - //error_log('[MXCHAT-CHUNK-DEBUG] Chunk ' . $i . ' length: ' . strlen($chunk) . ' chars, preview: ' . substr($chunk, 0, 100));
621 - }
622 -
623 - //error_log('[MXCHAT-CHUNK] Split content into ' . $total_chunks . ' chunks');
624 -
625 - if ($total_chunks === 0) {
626 - return new WP_Error('chunking_failed', 'Content could not be split into chunks');
627 - }
628 -
629 - $errors = array();
630 - $is_pinecone = self::is_pinecone_enabled_for_bot($bot_id);
631 -
632 - foreach ($chunks as $index => $chunk_text) {
633 - // Generate chunk metadata
634 - $chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url);
635 - $chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index);
636 -
637 - //error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')');
638 -
639 - // Generate embedding for this chunk
640 - $embedding_vector = self::generate_embedding($chunk_text, $api_key, $bot_id);
641 -
642 - if (!is_array($embedding_vector)) {
643 - $errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index);
644 - //error_log('[MXCHAT-CHUNK] Failed to generate embedding for chunk ' . $index);
645 - continue;
646 - }
647 -
648 - if ($is_pinecone) {
649 - // Store in Pinecone with chunk metadata
650 - $result = self::store_chunk_in_pinecone(
651 - $embedding_vector,
652 - $chunk_text,
653 - $source_url,
654 - $chunk_vector_id,
655 - $bot_id,
656 - $content_type,
657 - $chunk_metadata
658 - );
659 - } else {
660 - // Store in WordPress DB with chunk metadata
661 - $content_with_metadata = MxChat_Chunker::format_chunk_for_storage($chunk_text, $chunk_metadata);
662 - $embedding_vector_serialized = maybe_serialize($embedding_vector);
663 -
664 - $result = self::store_chunk_in_wordpress_db(
665 - $content_with_metadata,
666 - $source_url,
667 - $embedding_vector_serialized,
668 - $table_name,
669 - $content_type,
670 - $chunk_metadata
671 - );
672 - }
673 -
674 - if (is_wp_error($result)) {
675 - $errors[] = $result;
676 - //error_log('[MXCHAT-CHUNK] Failed to store chunk ' . $index . ': ' . $result->get_error_message());
677 - }
678 - }
679 -
680 - if (count($errors) === $total_chunks) {
681 - return new WP_Error('chunking_failed', 'Failed to store any chunks');
682 - }
683 -
684 - if (!empty($errors)) {
685 - //error_log('[MXCHAT-CHUNK] Completed with ' . count($errors) . ' errors out of ' . $total_chunks . ' chunks');
686 - return new WP_Error('chunking_partial_failure',
687 - sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks));
688 - }
689 -
690 - //error_log('[MXCHAT-CHUNK] Successfully stored all ' . $total_chunks . ' chunks');
691 - return true;
692 -}
693 -
694 -/**
695 - * Store a single chunk in Pinecone with chunk-specific metadata
696 - */
697 -private static function store_chunk_in_pinecone($embedding_vector, $chunk_text, $source_url, $vector_id, $bot_id, $content_type, $chunk_metadata) {
698 - // Get Pinecone configuration
699 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
700 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
701 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
702 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
703 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
704 - } else {
705 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
706 - if (empty($bot_pinecone_config)) {
707 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
708 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
709 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
710 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
711 - } else {
712 - $api_key = $bot_pinecone_config['api_key'] ?? '';
713 - $host = $bot_pinecone_config['host'] ?? '';
714 - $namespace = $bot_pinecone_config['namespace'] ?? '';
715 - }
716 - }
717 -
718 - if (empty($host) || empty($api_key)) {
719 - return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
720 - }
721 -
722 - $api_endpoint = "https://{$host}/vectors/upsert";
723 -
724 - // Build metadata with chunk information
725 - $metadata = array(
726 - 'text' => $chunk_text,
727 - 'source_url' => $source_url,
728 - 'type' => $content_type,
729 - 'is_chunked' => true,
730 - 'chunk_index' => $chunk_metadata['chunk_index'],
731 - 'total_chunks' => $chunk_metadata['total_chunks'],
732 - 'parent_url_hash' => $chunk_metadata['parent_url_hash'],
733 - 'last_updated' => time(),
734 - 'created_at' => time(),
735 - 'bot_id' => $bot_id,
736 - );
737 -
738 - $vector_data = array(
739 - 'id' => $vector_id,
740 - 'values' => $embedding_vector,
741 - 'metadata' => $metadata
742 - );
743 -
744 - $request_body = array(
745 - 'vectors' => array($vector_data)
746 - );
747 -
748 - if (!empty($namespace)) {
749 - $request_body['namespace'] = $namespace;
750 - }
751 -
752 - $response = wp_remote_post($api_endpoint, array(
753 - 'headers' => array(
754 - 'Api-Key' => $api_key,
755 - 'accept' => 'application/json',
756 - 'content-type' => 'application/json'
757 - ),
758 - 'body' => wp_json_encode($request_body),
759 - 'timeout' => 30
760 - ));
761 -
762 - if (is_wp_error($response)) {
763 - return $response;
764 - }
765 -
766 - $response_code = wp_remote_retrieve_response_code($response);
767 - if ($response_code !== 200) {
768 - return new WP_Error('pinecone_api', 'Pinecone API error: HTTP ' . $response_code);
769 - }
770 -
771 - return true;
772 -}
773 -
774 -/**
775 - * Store a single chunk in WordPress database
776 - */
777 -private static function store_chunk_in_wordpress_db($content_with_metadata, $source_url, $embedding_vector_serialized, $table_name, $content_type, $chunk_metadata) {
778 - global $wpdb;
779 -
780 - // For chunks, we always insert new rows (no duplicate checking)
781 - // The URL includes chunk info in the metadata, but source_url stays the same for grouping
782 - $result = $wpdb->insert(
783 - $table_name,
784 - array(
785 - 'url' => $source_url,
786 - 'article_content' => $content_with_metadata,
787 - 'embedding_vector' => $embedding_vector_serialized,
788 - 'source_url' => $source_url,
789 - 'content_type' => $content_type,
790 - 'timestamp' => current_time('mysql')
791 - ),
792 - array('%s', '%s', '%s', '%s', '%s', '%s')
793 - );
794 -
795 - if ($result === false) {
796 - return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error);
797 - }
798 -
799 - return true;
800 -}
801 -
802 -/**
803 - * Delete all chunks for a given URL
804 - *
805 - * @param string $source_url The source URL
806 - * @param string $bot_id The bot ID
807 - * @return bool|WP_Error True on success, WP_Error on failure
808 - */
809 -public static function delete_chunks_for_url($source_url, $bot_id = 'default') {
810 - //error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url);
811 -
812 - if (self::is_pinecone_enabled_for_bot($bot_id)) {
813 - return self::delete_pinecone_chunks_by_url($source_url, $bot_id);
814 - } else {
815 - return self::delete_wordpress_chunks_by_url($source_url);
816 - }
817 -}
818 -
819 -/**
820 - * Delete all chunks for a URL from Pinecone
821 - */
822 -private static function delete_pinecone_chunks_by_url($source_url, $bot_id) {
823 - // Get Pinecone configuration
824 - if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
825 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
826 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
827 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
828 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
829 - } else {
830 - $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
831 - if (empty($bot_pinecone_config)) {
832 - $pinecone_options = get_option('mxchat_pinecone_addon_options');
833 - $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
834 - $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
835 - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
836 - } else {
837 - $api_key = $bot_pinecone_config['api_key'] ?? '';
838 - $host = $bot_pinecone_config['host'] ?? '';
839 - $namespace = $bot_pinecone_config['namespace'] ?? '';
840 - }
841 - }
842 -
843 - if (empty($host) || empty($api_key)) {
844 - return new WP_Error('pinecone_config', 'Pinecone is not properly configured');
845 - }
846 -
847 - $base_vector_id = md5($source_url);
848 - $vectors_to_delete = array();
849 -
850 - // Add the original single-vector ID (for non-chunked content)
851 - $vectors_to_delete[] = $base_vector_id;
852 -
853 - // Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a
854 - // non-200 silently and we end up only deleting the base vector, leaving chunks orphaned.
855 - $query_params = array(
856 - 'prefix' => $base_vector_id . '_chunk_',
857 - 'limit' => 100,
858 - );
859 - if (!empty($namespace)) {
860 - $query_params['namespace'] = $namespace;
861 - }
862 -
863 - $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
864 -
865 - // Paginate in case a URL has more than 100 chunks.
866 - do {
867 - $list_response = wp_remote_get($list_url, array(
868 - 'headers' => array(
869 - 'Api-Key' => $api_key,
870 - 'accept' => 'application/json',
871 - ),
872 - 'timeout' => 30,
873 - ));
874 -
875 - if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) {
876 - break;
877 - }
878 -
879 - $list_data = json_decode(wp_remote_retrieve_body($list_response), true);
880 - if (!empty($list_data['vectors'])) {
881 - foreach ($list_data['vectors'] as $vector) {
882 - if (isset($vector['id'])) {
883 - $vectors_to_delete[] = $vector['id'];
884 - }
885 - }
886 - }
887 -
888 - $next_token = $list_data['pagination']['next'] ?? '';
889 - if (empty($next_token)) {
890 - break;
891 - }
892 -
893 - $query_params['paginationToken'] = $next_token;
894 - $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);
895 - } while (true);
896 -
897 - if (empty($vectors_to_delete)) {
898 - //error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete');
899 - return true;
900 - }
901 -
902 - //error_log('[MXCHAT-CHUNK-DELETE] Deleting ' . count($vectors_to_delete) . ' vectors from Pinecone');
903 -
904 - // Delete vectors
905 - $delete_url = "https://{$host}/vectors/delete";
906 -
907 - $delete_body = array(
908 - 'ids' => $vectors_to_delete
909 - );
910 -
911 - if (!empty($namespace)) {
912 - $delete_body['namespace'] = $namespace;
913 - }
914 -
915 - $delete_response = wp_remote_post($delete_url, array(
916 - 'headers' => array(
917 - 'Api-Key' => $api_key,
918 - 'accept' => 'application/json',
919 - 'content-type' => 'application/json'
920 - ),
921 - 'body' => wp_json_encode($delete_body),
922 - 'timeout' => 30
923 - ));
924 -
925 - if (is_wp_error($delete_response)) {
926 - return $delete_response;
927 - }
928 -
929 - $response_code = wp_remote_retrieve_response_code($delete_response);
930 - if ($response_code !== 200) {
931 - return new WP_Error('pinecone_delete', 'Failed to delete vectors: HTTP ' . $response_code);
932 - }
933 -
934 - return true;
935 -}
936 -
937 -/**
938 - * Delete all chunks for a URL from WordPress database
939 - */
940 -private static function delete_wordpress_chunks_by_url($source_url) {
941 - global $wpdb;
942 - $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
943 -
944 - // Delete all rows with this source_url (handles both chunked and non-chunked)
945 - $result = $wpdb->delete(
946 - $table_name,
947 - array('source_url' => $source_url),
948 - array('%s')
949 - );
950 -
951 - if ($result === false) {
952 - return new WP_Error('database_delete', 'Failed to delete chunks: ' . $wpdb->last_error);
953 - }
954 -
955 - //error_log('[MXCHAT-CHUNK-DELETE] Deleted ' . $result . ' rows from WordPress DB');
956 - return true;
957 -}
958 -}
1 +<?php
2 +if (!defined('ABSPATH')) {
3 + exit; // Exit if accessed directly
4 +}
5 +
6 +class MxChat_Utils {
7 +
8 + /**
9 + * Submit content and its embedding to the database.
10 + *
11 + * @param string $content The content to be embedded.
12 + * @param string $source_url The source URL of the content.
13 + * @param string $api_key The API key used for generating embeddings.
14 + */
15 + public static function submit_content_to_db($content, $source_url, $api_key) {
16 + global $wpdb;
17 + $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
18 +
19 + // Sanitize the source URL
20 + $source_url = esc_url_raw($source_url);
21 +
22 + // Generate the embedding using the API key
23 + $embedding_vector = self::generate_embedding($content, $api_key);
24 +
25 + if (is_array($embedding_vector)) {
26 + $embedding_vector_serialized = serialize($embedding_vector);
27 +
28 + // Insert or update the record in the database
29 + $wpdb->replace(
30 + $table_name,
31 + array(
32 + 'article_content' => $content,
33 + 'embedding_vector' => $embedding_vector_serialized,
34 + 'source_url' => $source_url,
35 + ),
36 + array(
37 + '%s',
38 + '%s',
39 + '%s',
40 + )
41 + );
42 + } else {
43 + //error_log('Embedding generation failed for content from ' . $source_url);
44 + }
45 + }
46 +
47 + /**
48 + * Generate an embedding for the given text.
49 + *
50 + * @param string $text The text to be embedded.
51 + * @param string $api_key The API key used for generating embeddings.
52 + * @return array|null The embedding vector or null on failure.
53 + */
54 + private static function generate_embedding($text, $api_key) {
55 + $endpoint = 'https://api.openai.com/v1/embeddings';
56 +
57 + $body = wp_json_encode([
58 + 'input' => $text,
59 + 'model' => 'text-embedding-ada-002'
60 + ]);
61 +
62 + $args = [
63 + 'body' => $body,
64 + 'headers' => [
65 + 'Content-Type' => 'application/json',
66 + 'Authorization' => 'Bearer ' . $api_key,
67 + ],
68 + 'timeout' => 60,
69 + 'redirection' => 5,
70 + 'blocking' => true,
71 + 'httpversion' => '1.0',
72 + 'sslverify' => true,
73 + ];
74 +
75 + $response = wp_remote_post($endpoint, $args);
76 +
77 + if (is_wp_error($response)) {
78 + //error_log('Error generating embedding: ' . $response->get_error_message());
79 + return null;
80 + }
81 +
82 + $response_body = json_decode(wp_remote_retrieve_body($response), true);
83 +
84 + if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) {
85 + return $response_body['data'][0]['embedding'];
86 + } else {
87 + //error_log('Invalid response received from embedding API.');
88 + return null;
89 + }
90 + }
91 +}
92 +
93 +?>