| 1 |
<?php |
| 2 |
if (!defined('ABSPATH')) { |
| 3 |
exit; // Exit if accessed directly |
| 4 |
} |
| 5 |
|
| 6 |
class MxChat_Utils { |
| 7 |
|
| 8 |
/** |
| 9 |
* True while a storage routine is re-writing an entry it is about to re-add |
| 10 |
* (submit_chunked_content's clean-slate delete). delete_chunks_for_url() skips |
| 11 |
* the Vector Store mirror-delete while set — an internal re-store is not an |
| 12 |
* entry removal, and mirroring it would delete-then-reupload the entry's file |
| 13 |
* on every chunked save (plan 15b5c6). |
| 14 |
*/ |
| 15 |
private static $vectorstore_mirror_suspended = false; |
| 16 |
|
| 17 |
/** |
| 18 |
* Validate a client-supplied session id (plan-mxchat-20260731-d42bec). |
| 19 |
* |
| 20 |
* sanitize_text_field() — which every session_id read site used before this — |
| 21 |
* preserves '/' and '..'. Harmless where the value is only an option or |
| 22 |
* transient key suffix, but mxchat_send_delayed_transcript() interpolates it |
| 23 |
* into a filesystem path, so '../../../../path/x' wrote, emailed and deleted a |
| 24 |
* file outside the uploads dir. |
| 25 |
* |
| 26 |
* REJECTS rather than rewrites: a silently-stripped id would orphan the |
| 27 |
* conversation it belongs to, which is harder to diagnose than a clean refusal. |
| 28 |
* Returns '' for anything malformed, so call sites fall into the empty-session |
| 29 |
* error paths they already have. |
| 30 |
* |
| 31 |
* The generator only ever emits 'mxchat_chat_' + 32 hex chars |
| 32 |
* (class-mxchat-integrator.php, js/chat-script.js), so this is not restrictive |
| 33 |
* in practice. Length ceiling is deliberate — session ids are also used as |
| 34 |
* option-name suffixes, and WP option names cap at 191 chars. |
| 35 |
* |
| 36 |
* @param mixed $raw Raw request value. |
| 37 |
* @return string The id if well-formed, '' otherwise. |
| 38 |
*/ |
| 39 |
public static function sanitize_session_id($raw) { |
| 40 |
if (!is_scalar($raw)) { |
| 41 |
return ''; |
| 42 |
} |
| 43 |
$val = trim((string) $raw); |
| 44 |
if ($val === '') { |
| 45 |
return ''; |
| 46 |
} |
| 47 |
return preg_match('/\A[A-Za-z0-9_-]{1,128}\z/', $val) ? $val : ''; |
| 48 |
} |
| 49 |
|
| 50 |
/** |
| 51 |
* Sanitize the lead-capture consent-checkbox label (plan b062c4). |
| 52 |
* |
| 53 |
* The label is owner-supplied and renders inside the widget's email form, so |
| 54 |
* this is a security boundary, not a formatting nicety. One explicit |
| 55 |
* allowlist, used at BOTH save time (options.php sanitize + autosave AJAX) |
| 56 |
* and render time (widget form, admin surfaces) so the two can never drift: |
| 57 |
* an anchor — the whole point is "I agree to the <a>Privacy Policy</a>" — |
| 58 |
* plus inline emphasis. No block tags, no images, no style attributes. |
| 59 |
* |
| 60 |
* The stored consent record keeps this exact sanitized string as "the text |
| 61 |
* the visitor saw", so it must be deterministic: same input, same output, |
| 62 |
* whichever path ran it. |
| 63 |
* |
| 64 |
* @param mixed $raw Owner-entered label. |
| 65 |
* @return string Sanitized label, capped at 1000 chars. |
| 66 |
*/ |
| 67 |
public static function sanitize_consent_label($raw) { |
| 68 |
if (!is_scalar($raw)) { |
| 69 |
return ''; |
| 70 |
} |
| 71 |
|
| 72 |
$allowed = array( |
| 73 |
'a' => array( |
| 74 |
'href' => true, |
| 75 |
'title' => true, |
| 76 |
'target' => true, |
| 77 |
'rel' => true, |
| 78 |
), |
| 79 |
'strong' => array(), |
| 80 |
'em' => array(), |
| 81 |
'br' => array(), |
| 82 |
); |
| 83 |
|
| 84 |
$label = wp_kses(trim((string) $raw), $allowed); |
| 85 |
|
| 86 |
return mb_substr($label, 0, 1000); |
| 87 |
} |
| 88 |
|
| 89 |
/** |
| 90 |
* Per-request cache for get_session_history(). Mirrors get_option()'s |
| 91 |
* request-scoped caching, which the mxchat_history_ option reads got for |
| 92 |
* free before plan 839c4c moved history reads onto the transcripts table. |
| 93 |
*/ |
| 94 |
private static $history_cache = array(); |
| 95 |
|
| 96 |
/** |
| 97 |
* Session chat history read from the transcripts table, in the exact array |
| 98 |
* shape the legacy mxchat_history_<sid> option stored (plan 839c4c). The |
| 99 |
* option was a second copy of state the table already held — measured |
| 100 |
* byte-identical in role/content/order on 174 of 177 real sessions, with |
| 101 |
* the table a superset on the rest — at up to 64 KB per option row. The |
| 102 |
* table is now the single store; nothing writes the option any more. |
| 103 |
* |
| 104 |
* Shape notes, load-bearing for the consumers: |
| 105 |
* - id: the transcripts row id (int). Integer ids make the pollers' |
| 106 |
* ">" comparisons correct where the old uniqid() strings only worked by |
| 107 |
* accident of hex ordering. |
| 108 |
* - timestamp: milliseconds, derived from the table's second-resolution GMT |
| 109 |
* column (x1000). Consumers comparing against a real-millisecond client |
| 110 |
* cutoff MUST floor the cutoff to the second and err inclusive — see the |
| 111 |
* persistence-off filters in class-mxchat-integrator.php. |
| 112 |
* - agent_name: the row's user_identifier, which the writer sets to the |
| 113 |
* same displayed_name value the option carried (agent name when present, |
| 114 |
* else email, else identifier). |
| 115 |
* |
| 116 |
* Public and static so mxchat-woo / mxchat-forms can call the same accessor |
| 117 |
* as core, guarded with method_exists against an older mxchat-basic. |
| 118 |
* |
| 119 |
* @param string $session_id |
| 120 |
* @return array[] Chronological entries: id, role, content, timestamp, agent_name. |
| 121 |
*/ |
| 122 |
public static function get_session_history($session_id) { |
| 123 |
global $wpdb; |
| 124 |
|
| 125 |
$session_id = self::sanitize_session_id($session_id); |
| 126 |
if ($session_id === '') { |
| 127 |
return array(); |
| 128 |
} |
| 129 |
|
| 130 |
if (array_key_exists($session_id, self::$history_cache)) { |
| 131 |
return self::$history_cache[$session_id]; |
| 132 |
} |
| 133 |
|
| 134 |
$table = $wpdb->prefix . 'mxchat_chat_transcripts'; |
| 135 |
|
| 136 |
// No SHOW TABLES guard: this is the chat hot path and the table is |
| 137 |
// created on activation (with an admin-load safety net). A genuinely |
| 138 |
// missing table fails the query and yields the same empty history the |
| 139 |
// old option read produced on a fresh session. |
| 140 |
$rows = $wpdb->get_results( |
| 141 |
$wpdb->prepare( |
| 142 |
"SELECT id, role, message, user_identifier, timestamp |
| 143 |
FROM `$table` WHERE session_id = %s ORDER BY id ASC", |
| 144 |
$session_id |
| 145 |
), |
| 146 |
ARRAY_A |
| 147 |
); |
| 148 |
|
| 149 |
$history = array(); |
| 150 |
if (is_array($rows)) { |
| 151 |
foreach ($rows as $row) { |
| 152 |
// The column stores GMT (current_time('mysql', 1) at the writer), |
| 153 |
// so pin the parse to UTC rather than the site timezone. |
| 154 |
$ts = strtotime($row['timestamp'] . ' +0000'); |
| 155 |
$history[] = array( |
| 156 |
'id' => (int) $row['id'], |
| 157 |
'role' => (string) $row['role'], |
| 158 |
'content' => (string) $row['message'], |
| 159 |
'timestamp' => ($ts ? $ts : 0) * 1000, |
| 160 |
'agent_name' => (string) $row['user_identifier'], |
| 161 |
); |
| 162 |
} |
| 163 |
} |
| 164 |
|
| 165 |
self::$history_cache[$session_id] = $history; |
| 166 |
|
| 167 |
return $history; |
| 168 |
} |
| 169 |
|
| 170 |
/** |
| 171 |
* Drop the cached history for one session (or all). The writer calls this |
| 172 |
* after every insert so a later read in the same request — e.g. the AI |
| 173 |
* context build that follows saving the user's message — sees the new row, |
| 174 |
* matching the read-your-own-write behavior update_option() gave the old |
| 175 |
* option copy. |
| 176 |
* |
| 177 |
* @param string|null $session_id Null flushes everything (test seam). |
| 178 |
*/ |
| 179 |
public static function flush_session_history_cache($session_id = null) { |
| 180 |
if ($session_id === null) { |
| 181 |
self::$history_cache = array(); |
| 182 |
return; |
| 183 |
} |
| 184 |
|
| 185 |
unset(self::$history_cache[(string) $session_id]); |
| 186 |
} |
| 187 |
|
| 188 |
/** |
| 189 |
* Append one message to a session's transcript on behalf of an add-on |
| 190 |
* (plan 4a664e). The Forms add-on's wizard steps travel over the add-on's |
| 191 |
* own AJAX action, never through the chat request, so nothing wrote them |
| 192 |
* to the transcript: Transcripts, the live-agent handoff card, Download |
| 193 |
* Transcript and every transcript email showed "[WIZARD_COMPLETE]" and no |
| 194 |
* answers. This is the write half of get_session_history() — same row |
| 195 |
* shape as the chat writer (visitor identity, captured name/email and |
| 196 |
* originating page from the session store, GMT timestamp), without the |
| 197 |
* new-session side effects (ownership, notification email, delayed |
| 198 |
* transcript) that only the visitor's first message should trigger. |
| 199 |
* |
| 200 |
* Public and static so add-ons can call it, guarded with method_exists |
| 201 |
* against an older mxchat-basic. |
| 202 |
* |
| 203 |
* @param string $session_id Chat session id. |
| 204 |
* @param string $role user|bot|system|agent. |
| 205 |
* @param string $message Stored as-is: bot rows may carry HTML, user rows |
| 206 |
* are rendered as text by every reader. |
| 207 |
* @return int The new transcripts row id, 0 when nothing was written. |
| 208 |
*/ |
| 209 |
public static function save_session_message($session_id, $role, $message) { |
| 210 |
global $wpdb; |
| 211 |
|
| 212 |
$session_id = self::sanitize_session_id($session_id); |
| 213 |
$role = is_string($role) ? strtolower(trim($role)) : ''; |
| 214 |
$message = is_scalar($message) ? trim((string) $message) : ''; |
| 215 |
|
| 216 |
if ($session_id === '' || $message === '' |
| 217 |
|| !in_array($role, array('user', 'bot', 'system', 'agent'), true)) { |
| 218 |
return 0; |
| 219 |
} |
| 220 |
|
| 221 |
$table = $wpdb->prefix . 'mxchat_chat_transcripts'; |
| 222 |
|
| 223 |
$saved_email = MxChat_Session_Store::get($session_id, 'email'); |
| 224 |
$saved_name = MxChat_Session_Store::get($session_id, 'name'); |
| 225 |
|
| 226 |
$row = array( |
| 227 |
'user_id' => is_user_logged_in() ? get_current_user_id() : 0, |
| 228 |
'user_identifier' => MxChat_User::mxchat_get_user_identifier(), |
| 229 |
'user_email' => $saved_email ?: MxChat_User::mxchat_get_user_email(), |
| 230 |
'user_name' => $saved_name ?: '', |
| 231 |
'session_id' => $session_id, |
| 232 |
'role' => $role, |
| 233 |
'message' => $message, |
| 234 |
'timestamp' => current_time('mysql', 1), |
| 235 |
); |
| 236 |
|
| 237 |
// Same page attribution as the rest of the session; the column pair was |
| 238 |
// added by an upgrade routine, so tolerate its absence like the chat |
| 239 |
// writer does. |
| 240 |
$originating = MxChat_Session_Store::get($session_id, 'originating_page'); |
| 241 |
if (is_array($originating) && !empty($originating['url']) |
| 242 |
&& $wpdb->get_var("SHOW COLUMNS FROM `$table` LIKE 'originating_page_url'")) { |
| 243 |
$row['originating_page_url'] = $originating['url']; |
| 244 |
$row['originating_page_title'] = isset($originating['title']) ? $originating['title'] : ''; |
| 245 |
} |
| 246 |
|
| 247 |
$inserted = $wpdb->insert($table, $row); |
| 248 |
if ($inserted === false) { |
| 249 |
return 0; |
| 250 |
} |
| 251 |
|
| 252 |
$row_id = (int) $wpdb->insert_id; |
| 253 |
self::flush_session_history_cache($session_id); |
| 254 |
|
| 255 |
return $row_id; |
| 256 |
} |
| 257 |
|
| 258 |
/** |
| 259 |
* Most recipients the Notification Email field will accept (plan 2f131a). |
| 260 |
* A settings field is not a mailing list. |
| 261 |
*/ |
| 262 |
const NOTIFICATION_EMAIL_MAX = 5; |
| 263 |
|
| 264 |
/** |
| 265 |
* Parse the Notification Email field into a list of recipients (plan 2f131a). |
| 266 |
* |
| 267 |
* THE TRAP THIS EXISTS TO CLOSE: sanitize_email() cannot be the validator for |
| 268 |
* this field, because its output for the failing input is VALID. WordPress |
| 269 |
* strips the separator and the surplus '@' and concatenates the remains: |
| 270 |
* |
| 271 |
* [email protected], [email protected] -> [email protected] |
| 272 |
* |
| 273 |
* and is_email() then returns true on that. So every guard in the plugin passed, |
| 274 |
* the address was stored, the autosave ticked green, and both the new-session |
| 275 |
* notification and the auto-emailed transcript went to a domain that does not |
| 276 |
* exist — with no error anywhere. Validating the RAW part BEFORE sanitizing is |
| 277 |
* the whole point; reversing those two lines silently restores the bug. |
| 278 |
* |
| 279 |
* All-or-nothing by design: if any entry is bad the caller must store NOTHING. |
| 280 |
* A partial accept — keeping the good addresses and dropping the bad one — is |
| 281 |
* the same defect in a new costume, because the owner still believes everyone |
| 282 |
* on their list is being notified. |
| 283 |
* |
| 284 |
* @param mixed $raw Raw field value, exactly as submitted. |
| 285 |
* @return array{emails: string[], error: string} Empty emails + empty error |
| 286 |
* means the field was empty. |
| 287 |
*/ |
| 288 |
public static function parse_notification_emails($raw) { |
| 289 |
$out = array('emails' => array(), 'error' => ''); |
| 290 |
|
| 291 |
if (!is_scalar($raw)) { |
| 292 |
$out['error'] = __('The notification email could not be read.', 'mxchat'); |
| 293 |
return $out; |
| 294 |
} |
| 295 |
|
| 296 |
$raw = trim((string) $raw); |
| 297 |
if ($raw === '') { |
| 298 |
return $out; // genuinely empty — the caller falls back to admin_email |
| 299 |
} |
| 300 |
|
| 301 |
$seen = array(); |
| 302 |
foreach (preg_split('/[,;]/', $raw) as $part) { |
| 303 |
$part = trim($part); |
| 304 |
if ($part === '') { |
| 305 |
// A trailing or doubled separator carries no address, so skipping it |
| 306 |
// cannot silently drop a recipient. This is the ONLY thing tolerated. |
| 307 |
continue; |
| 308 |
} |
| 309 |
|
| 310 |
// RAW first. See the note above — order is load-bearing. |
| 311 |
$clean = is_email($part) ? sanitize_email($part) : ''; |
| 312 |
if ($clean === '' || !is_email($clean)) { |
| 313 |
return array( |
| 314 |
'emails' => array(), |
| 315 |
'error' => sprintf( |
| 316 |
/* translators: %s: the email address the owner typed. */ |
| 317 |
__('"%s" is not a valid email address, so nothing was saved. Separate multiple addresses with a comma.', 'mxchat'), |
| 318 |
esc_html($part) |
| 319 |
), |
| 320 |
); |
| 321 |
} |
| 322 |
|
| 323 |
$key = strtolower($clean); |
| 324 |
if (isset($seen[$key])) { |
| 325 |
continue; // same address twice would simply mail them twice |
| 326 |
} |
| 327 |
$seen[$key] = true; |
| 328 |
$out['emails'][] = $clean; |
| 329 |
} |
| 330 |
|
| 331 |
if (count($out['emails']) > self::NOTIFICATION_EMAIL_MAX) { |
| 332 |
return array( |
| 333 |
'emails' => array(), |
| 334 |
'error' => sprintf( |
| 335 |
/* translators: %d: maximum number of notification recipients. */ |
| 336 |
__('Enter at most %d email addresses, separated by commas.', 'mxchat'), |
| 337 |
self::NOTIFICATION_EMAIL_MAX |
| 338 |
), |
| 339 |
); |
| 340 |
} |
| 341 |
|
| 342 |
return $out; |
| 343 |
} |
| 344 |
|
| 345 |
/** |
| 346 |
* The stored recipient list, ready to hand to wp_mail() (plan 2f131a). |
| 347 |
* |
| 348 |
* Fallback rule, and it is narrow on purpose: an EMPTY field falls back to the |
| 349 |
* site admin address, because that is the documented behaviour and an owner who |
| 350 |
* never filled the field in still wants their notifications. A field holding |
| 351 |
* something unusable does NOT fall back — it sends nowhere, exactly as before |
| 352 |
* this plan. Falling back on bad input would mean a typo silently redirects a |
| 353 |
* store's transcripts to a different mailbox than the one on screen. |
| 354 |
* |
| 355 |
* @param array|null $options mxchat_transcripts_options, or null to read it. |
| 356 |
* @return string[] Recipients; empty means do not send. |
| 357 |
*/ |
| 358 |
public static function notification_recipients($options = null) { |
| 359 |
if (!is_array($options)) { |
| 360 |
$options = get_option('mxchat_transcripts_options', array()); |
| 361 |
if (!is_array($options)) { |
| 362 |
$options = array(); |
| 363 |
} |
| 364 |
} |
| 365 |
|
| 366 |
$raw = isset($options['mxchat_notification_email']) ? $options['mxchat_notification_email'] : ''; |
| 367 |
$raw = is_scalar($raw) ? trim((string) $raw) : ''; |
| 368 |
|
| 369 |
if ($raw === '') { |
| 370 |
$admin = get_option('admin_email'); |
| 371 |
return is_email($admin) ? array($admin) : array(); |
| 372 |
} |
| 373 |
|
| 374 |
$parsed = self::parse_notification_emails($raw); |
| 375 |
return $parsed['error'] === '' ? $parsed['emails'] : array(); |
| 376 |
} |
| 377 |
|
| 378 |
/** |
| 379 |
* Centralized embedding model registry. Single source of truth for dimensions |
| 380 |
* and provider, so model-switch protection logic doesn't drift across files. |
| 381 |
*/ |
| 382 |
public static function embedding_model_registry() { |
| 383 |
return array( |
| 384 |
'text-embedding-ada-002' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'Ada 2'), |
| 385 |
'text-embedding-3-small' => array('dims' => 1536, 'provider' => 'openai', 'label' => 'TE3 Small'), |
| 386 |
'text-embedding-3-large' => array('dims' => 3072, 'provider' => 'openai', 'label' => 'TE3 Large'), |
| 387 |
'voyage-3-large' => array('dims' => 2048, 'provider' => 'voyage', 'label' => 'Voyage-3 Large'), |
| 388 |
'gemini-embedding-001' => array('dims' => 1536, 'provider' => 'gemini', 'label' => 'Gemini Embedding'), |
| 389 |
); |
| 390 |
} |
| 391 |
|
| 392 |
public static function embedding_model_dimensions($model) { |
| 393 |
$registry = self::embedding_model_registry(); |
| 394 |
return isset($registry[$model]) ? (int) $registry[$model]['dims'] : 0; |
| 395 |
} |
| 396 |
|
| 397 |
public static function embedding_model_label($model) { |
| 398 |
if (is_string($model) && strpos($model, 'custom:') === 0) { |
| 399 |
/* translators: %s: the embedding model name configured on the custom provider */ |
| 400 |
return sprintf(__('%s (custom provider)', 'mxchat'), substr($model, 7)); |
| 401 |
} |
| 402 |
$registry = self::embedding_model_registry(); |
| 403 |
return isset($registry[$model]) ? $registry[$model]['label'] : $model; |
| 404 |
} |
| 405 |
|
| 406 |
/** |
| 407 |
* Returns the model that was last used to actually write embeddings into the |
| 408 |
* KB. Differs from the user-selected setting once a switch has happened but |
| 409 |
* no re-embed has occurred yet — that's the mismatch state we warn about. |
| 410 |
*/ |
| 411 |
public static function get_active_embedding_model() { |
| 412 |
return get_option('mxchat_active_embedding_model', ''); |
| 413 |
} |
| 414 |
|
| 415 |
/** |
| 416 |
* Stamp the model that produced the most recent successful embedding. Called |
| 417 |
* from generate_embedding() right after the API responds with a valid vector. |
| 418 |
*/ |
| 419 |
public static function stamp_active_embedding_model($model) { |
| 420 |
if (!empty($model) && $model !== self::get_active_embedding_model()) { |
| 421 |
update_option('mxchat_active_embedding_model', $model, false); |
| 422 |
} |
| 423 |
} |
| 424 |
|
| 425 |
/** |
| 426 |
* The model name the custom-provider embedding path will send, mirroring the |
| 427 |
* fallback chain the request itself uses: dedicated custom embedding model, |
| 428 |
* else the custom chat model, else 'default'. Single source shared by |
| 429 |
* generate_embedding_custom() and the mismatch-warning "selected" side so the |
| 430 |
* two can never drift (plan ae02cb). |
| 431 |
*/ |
| 432 |
public static function resolve_custom_embedding_model($options) { |
| 433 |
if (isset($options['custom_provider_embedding_model']) && trim((string) $options['custom_provider_embedding_model']) !== '') { |
| 434 |
return trim((string) $options['custom_provider_embedding_model']); |
| 435 |
} |
| 436 |
if (isset($options['custom_provider_model']) && trim((string) $options['custom_provider_model']) !== '') { |
| 437 |
return trim((string) $options['custom_provider_model']); |
| 438 |
} |
| 439 |
return 'default'; |
| 440 |
} |
| 441 |
|
| 442 |
/** |
| 443 |
* The EFFECTIVE selected embedding model — what the next embed will actually |
| 444 |
* use. With custom-provider embeddings on this is the custom identity in the |
| 445 |
* same 'custom:<model>' form stamp_active_embedding_model() records, not the |
| 446 |
* inert standard dropdown value. Mismatch-warning comparisons must read this, |
| 447 |
* never $options['embedding_model'] directly — the dropdown cannot be |
| 448 |
* deselected, so reading it raw flags every correctly-configured custom setup. |
| 449 |
*/ |
| 450 |
public static function get_selected_embedding_model($options = null) { |
| 451 |
if (!is_array($options)) { |
| 452 |
$options = get_option('mxchat_options', array()); |
| 453 |
} |
| 454 |
if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') { |
| 455 |
return 'custom:' . self::resolve_custom_embedding_model($options); |
| 456 |
} |
| 457 |
return $options['embedding_model'] ?? ''; |
| 458 |
} |
| 459 |
|
| 460 |
/** |
| 461 |
* Extract the 11-character YouTube video ID from a URL, or '' if the URL is |
| 462 |
* not a single-video YouTube link. Single source of truth for both the KB |
| 463 |
* ingestion side and the chat render side — do not duplicate this parsing. |
| 464 |
* Channel, playlist, and search URLs deliberately return '' (only a URL that |
| 465 |
* identifies one video can be embedded). |
| 466 |
*/ |
| 467 |
public static function parse_youtube_id($url) { |
| 468 |
if (!is_string($url) || $url === '' || !preg_match('#^https?://#i', $url)) { |
| 469 |
return ''; |
| 470 |
} |
| 471 |
$host = strtolower((string) wp_parse_url($url, PHP_URL_HOST)); |
| 472 |
$host = preg_replace('/^(www|m)\./', '', $host); |
| 473 |
$path = (string) wp_parse_url($url, PHP_URL_PATH); |
| 474 |
$id = ''; |
| 475 |
if ($host === 'youtu.be') { |
| 476 |
$segments = explode('/', ltrim($path, '/')); |
| 477 |
$id = $segments[0] ?? ''; |
| 478 |
} elseif (in_array($host, array('youtube.com', 'youtube-nocookie.com'), true)) { |
| 479 |
if (preg_match('#^/(?:shorts|embed|live|v)/([A-Za-z0-9_-]+)#', $path, $m)) { |
| 480 |
$id = $m[1]; |
| 481 |
} elseif ($path === '/watch') { |
| 482 |
parse_str((string) wp_parse_url($url, PHP_URL_QUERY), $query_vars); |
| 483 |
$id = isset($query_vars['v']) ? (string) $query_vars['v'] : ''; |
| 484 |
} |
| 485 |
} |
| 486 |
$id = preg_replace('/[^A-Za-z0-9_-]/', '', (string) $id); |
| 487 |
return (strlen($id) === 11) ? $id : ''; |
| 488 |
} |
| 489 |
|
| 490 |
/** |
| 491 |
* plan-mxchat-20260813-f52492 — video-card gating. |
| 492 |
* |
| 493 |
* Two standalone options (NOT mxchat_options — they skip the sanitize/autosave |
| 494 |
* traps entirely), read here so the gate in the integrator and the fields on |
| 495 |
* Knowledge -> Chunking & Retrieval can never disagree about a default. |
| 496 |
* |
| 497 |
* Master switch. Default ON: the card is existing behavior, and this is an |
| 498 |
* opt-out for owners who never want one, not a new feature to opt into. |
| 499 |
*/ |
| 500 |
public static function video_embed_enabled() { |
| 501 |
return get_option('mxchat_video_embed_enabled', 'on') === 'on'; |
| 502 |
} |
| 503 |
|
| 504 |
/** |
| 505 |
* The video card's OWN confidence floor, as a 0-1 cosine — deliberately not |
| 506 |
* the site-wide Similarity Threshold (default 35). "Good enough to quote in |
| 507 |
* the answer" and "good enough to put a video on screen" are different |
| 508 |
* questions: retrieval is allowed to be generous because the model still |
| 509 |
* decides what to say, whereas the card is asserted to the visitor with no |
| 510 |
* such filter. Stored as an int percentage to match the site-wide slider's |
| 511 |
* convention; the default (55) sits above it on purpose. |
| 512 |
* |
| 513 |
* MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT is the single source of that number. |
| 514 |
*/ |
| 515 |
public static function video_embed_threshold() { |
| 516 |
$stored = get_option('mxchat_video_embed_threshold', null); |
| 517 |
$percent = ($stored === null || $stored === '') |
| 518 |
? MXCHAT_VIDEO_EMBED_THRESHOLD_DEFAULT |
| 519 |
: (int) $stored; |
| 520 |
if ($percent < 0) { $percent = 0; } |
| 521 |
if ($percent > 100) { $percent = 100; } |
| 522 |
// Cast: PHP evaluates 100/100 to int(1), so an unclamped return type would |
| 523 |
// vary with the stored value. Callers compare against a cosine — keep it float. |
| 524 |
return (float) $percent / 100; |
| 525 |
} |
| 526 |
|
| 527 |
/** |
| 528 |
* UPDATED: Submit or update content (and its embedding) in the database. |
| 529 |
* Stores in Pinecone if enabled, otherwise stores in WordPress DB. |
| 530 |
* |
| 531 |
* @param string $content The content to be embedded. |
| 532 |
* @param string $source_url The source URL of the content. |
| 533 |
* @param string $api_key The API key used for generating embeddings. |
| 534 |
* @param string $vector_id Optional vector ID for Pinecone (if not provided, will use md5 of URL) |
| 535 |
* @param string $bot_id The bot ID for multi-bot support |
| 536 |
* @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.) |
| 537 |
* @return bool|WP_Error True on success, WP_Error on failure |
| 538 |
*/ |
| 539 |
public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content', $title = '') { |
| 540 |
global $wpdb; |
| 541 |
$table_name = $wpdb->prefix . 'mxchat_system_prompt_content'; |
| 542 |
|
| 543 |
//error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')'); |
| 544 |
//error_log('[MXCHAT-DB] Content length: ' . strlen($content) . ' bytes'); |
| 545 |
|
| 546 |
// Sanitize the source URL. Internal identities (mxchat:// manual docs, |
| 547 |
// upload:// file uploads) are NOT web URLs — esc_url_raw EMPTIES them |
| 548 |
// because its protocol list is statically cached and effectively |
| 549 |
// unfilterable (measured, plan 945406 / 0485e5) — sanitize those as text |
| 550 |
// so upserts stay keyed to a stable identity across re-imports. |
| 551 |
if (preg_match('#^(mxchat|upload)://#i', $source_url)) { |
| 552 |
$source_url = sanitize_text_field($source_url); |
| 553 |
} else { |
| 554 |
$source_url = esc_url_raw($source_url); |
| 555 |
} |
| 556 |
|
| 557 |
// Sanitize content_type |
| 558 |
$content_type = sanitize_key($content_type); |
| 559 |
if (empty($content_type)) { |
| 560 |
$content_type = 'content'; // Fallback for backwards compatibility |
| 561 |
} |
| 562 |
|
| 563 |
// Page title captured by the importer (plan d9ee66), registered under the |
| 564 |
// SANITIZED url so the storage writers below find it by the same key. |
| 565 |
if (is_string($title) && trim($title) !== '') { |
| 566 |
self::set_source_title($source_url, $title); |
| 567 |
} |
| 568 |
|
| 569 |
// Just ensure UTF-8 validity without aggressive escaping |
| 570 |
$safe_content = wp_check_invalid_utf8($content); |
| 571 |
// Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D) |
| 572 |
$safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content); |
| 573 |
|
| 574 |
// Check if chunking should be applied |
| 575 |
$chunker = MxChat_Chunker::from_settings(); |
| 576 |
if ($chunker->should_chunk($safe_content)) { |
| 577 |
//error_log('[MXCHAT-DB] Content exceeds chunk threshold, using chunked submission'); |
| 578 |
$chunk_result = self::submit_chunked_content($safe_content, $source_url, $api_key, $bot_id, $content_type, $chunker); |
| 579 |
// Vector Store mirror gets the entry WHOLE — one file per KB entry, |
| 580 |
// OpenAI chunks server-side; local chunking is our own embedding |
| 581 |
// concern and never reaches the store (plan 15b5c6). |
| 582 |
if ($chunk_result === true && class_exists('MxChat_Vectorstore_Manager')) { |
| 583 |
MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type); |
| 584 |
} |
| 585 |
return $chunk_result; |
| 586 |
} |
| 587 |
|
| 588 |
// UPDATED: Generate the embedding using bot-specific configuration |
| 589 |
$embedding_vector = self::generate_embedding($content, $api_key, $bot_id); |
| 590 |
|
| 591 |
if (!is_array($embedding_vector)) { |
| 592 |
// Surface the provider's real reason instead of a fixed string (4a7c0a). |
| 593 |
$reason = is_wp_error($embedding_vector) |
| 594 |
? $embedding_vector->get_error_message() |
| 595 |
: 'Failed to generate embedding for content'; |
| 596 |
return new WP_Error('embedding_failed', $reason); |
| 597 |
} |
| 598 |
|
| 599 |
//error_log('[MXCHAT-DB] Embedding generated successfully'); |
| 600 |
|
| 601 |
// UPDATED: Check if Pinecone is enabled for this specific bot |
| 602 |
if (self::is_pinecone_enabled_for_bot($bot_id)) { |
| 603 |
//error_log('[MXCHAT-DB] Pinecone is enabled for bot ' . $bot_id . ' - using Pinecone storage'); |
| 604 |
// Store in Pinecone only |
| 605 |
$result = self::store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id, $bot_id, $content_type); |
| 606 |
// If this URL previously stored as CHUNKED content and the new content fits in a |
| 607 |
// single vector, the upsert above only overwrote the base id — the old |
| 608 |
// md5(url)_chunk_N vectors would keep serving the stale text. Sweep them. |
| 609 |
// Only for a real source_url: chunk ids derive from it, and md5('') is shared |
| 610 |
// by legacy URL-less entries so a blind sweep there could hit other entries. |
| 611 |
if ($result === true && !empty($source_url)) { |
| 612 |
self::cleanup_pinecone_chunk_stragglers($source_url, $bot_id); |
| 613 |
} |
| 614 |
if ($result === true && class_exists('MxChat_Vectorstore_Manager')) { |
| 615 |
MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type); |
| 616 |
} |
| 617 |
return $result; |
| 618 |
} else { |
| 619 |
//error_log('[MXCHAT-DB] Pinecone not enabled for bot ' . $bot_id . ' - using WordPress storage'); |
| 620 |
// Store in WordPress database only |
| 621 |
$embedding_vector_serialized = maybe_serialize($embedding_vector); |
| 622 |
$result = self::store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type); |
| 623 |
if ($result === true && class_exists('MxChat_Vectorstore_Manager')) { |
| 624 |
MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, $safe_content, $bot_id, $content_type); |
| 625 |
} |
| 626 |
return $result; |
| 627 |
} |
| 628 |
} |
| 629 |
|
| 630 |
/* ------------------------------------------------------------------------- |
| 631 |
* Source titles (3.2.23, plan d9ee66) |
| 632 |
* |
| 633 |
* A knowledge entry imported from a URL or sitemap remembers the page's own |
| 634 |
* title, so an add-on that names a source (AI Search's source cards) can show |
| 635 |
* "About Us" instead of a guess from the slug ("Aboutus"). Stored in the |
| 636 |
* `title` column of the WP knowledge table and as `title` metadata on Pinecone |
| 637 |
* vectors / documents. The importer hands the title to submit_content_to_db(), |
| 638 |
* which registers it under the source URL; the writers look it up by that key. |
| 639 |
* ---------------------------------------------------------------------- */ |
| 640 |
|
| 641 |
/** @var array<string,string> source_url => title, for the current request */ |
| 642 |
private static $source_titles = array(); |
| 643 |
|
| 644 |
public static function set_source_title($source_url, $title) { |
| 645 |
$key = trim((string) $source_url); |
| 646 |
$title = self::clean_title($title); |
| 647 |
if ($key === '' || $title === '') { |
| 648 |
return; |
| 649 |
} |
| 650 |
self::$source_titles[$key] = $title; |
| 651 |
} |
| 652 |
|
| 653 |
public static function source_title_for($source_url) { |
| 654 |
$key = trim((string) $source_url); |
| 655 |
return ($key !== '' && isset(self::$source_titles[$key])) ? self::$source_titles[$key] : ''; |
| 656 |
} |
| 657 |
|
| 658 |
/** Entities decoded, tags gone, whitespace collapsed, capped at the column width. */ |
| 659 |
public static function clean_title($title) { |
| 660 |
$title = html_entity_decode((string) $title, ENT_QUOTES | ENT_HTML5, 'UTF-8'); |
| 661 |
$title = wp_strip_all_tags($title); |
| 662 |
$title = trim((string) preg_replace('/\s+/u', ' ', $title)); |
| 663 |
if (mb_strlen($title, 'UTF-8') > 255) { |
| 664 |
$title = rtrim(mb_substr($title, 0, 255, 'UTF-8')); |
| 665 |
} |
| 666 |
return $title; |
| 667 |
} |
| 668 |
|
| 669 |
/** |
| 670 |
* The page's own title from fetched HTML: og:title, else <title> with a |
| 671 |
* trailing "| Site Name" style suffix removed, else the first <h1>. Returns '' |
| 672 |
* when nothing usable is there, so the caller keeps its own fallback. |
| 673 |
*/ |
| 674 |
public static function extract_page_title($html) { |
| 675 |
$html = (string) $html; |
| 676 |
if ($html === '') { |
| 677 |
return ''; |
| 678 |
} |
| 679 |
$head = substr($html, 0, 200000); |
| 680 |
$site = ''; |
| 681 |
if (preg_match('/<meta\s+[^>]*property=["\']og:site_name["\'][^>]*content=["\']([^"\']{1,120})["\']/i', $head, $m) |
| 682 |
|| preg_match('/<meta\s+[^>]*content=["\']([^"\']{1,120})["\'][^>]*property=["\']og:site_name["\']/i', $head, $m)) { |
| 683 |
$site = self::clean_title($m[1]); |
| 684 |
} |
| 685 |
$candidate = ''; |
| 686 |
if (preg_match('/<meta\s+[^>]*property=["\']og:title["\'][^>]*content=["\']([^"\']{1,500})["\']/i', $head, $m) |
| 687 |
|| preg_match('/<meta\s+[^>]*content=["\']([^"\']{1,500})["\'][^>]*property=["\']og:title["\']/i', $head, $m)) { |
| 688 |
$candidate = self::clean_title($m[1]); |
| 689 |
} |
| 690 |
if ($candidate === '' && preg_match('/<title[^>]*>(.*?)<\/title>/is', $head, $m)) { |
| 691 |
$candidate = self::clean_title($m[1]); |
| 692 |
} |
| 693 |
if ($candidate !== '') { |
| 694 |
$candidate = self::strip_site_suffix($candidate, $site); |
| 695 |
} |
| 696 |
if ($candidate === '' && preg_match('/<h1[^>]*>(.*?)<\/h1>/is', $html, $m)) { |
| 697 |
$candidate = self::clean_title($m[1]); |
| 698 |
} |
| 699 |
return $candidate; |
| 700 |
} |
| 701 |
|
| 702 |
/** |
| 703 |
* "About Us | Just Facts" -> "About Us". Only a TRAILING segment after a spaced |
| 704 |
* separator goes, only when what is left is a real title (3+ characters) and the |
| 705 |
* removed part is short enough to be a site name; a known og:site_name wins. |
| 706 |
* Byte offsets throughout: every separator starts with an ASCII space, so a cut |
| 707 |
* there never lands inside a multibyte character. |
| 708 |
*/ |
| 709 |
private static function strip_site_suffix($title, $site_name = '') { |
| 710 |
$seps = array(' | ', ' – ', ' — ', ' - ', ' :: ', ' · ', ' » ', ' • '); |
| 711 |
if ($site_name !== '') { |
| 712 |
foreach ($seps as $sep) { |
| 713 |
$tail = $sep . $site_name; |
| 714 |
if (strlen($title) > strlen($tail) && substr($title, -strlen($tail)) === $tail) { |
| 715 |
return trim(substr($title, 0, strlen($title) - strlen($tail))); |
| 716 |
} |
| 717 |
} |
| 718 |
} |
| 719 |
$best_pos = -1; |
| 720 |
$best_sep = ''; |
| 721 |
foreach ($seps as $sep) { |
| 722 |
$pos = strrpos($title, $sep); |
| 723 |
if ($pos !== false && $pos > $best_pos) { |
| 724 |
$best_pos = $pos; |
| 725 |
$best_sep = $sep; |
| 726 |
} |
| 727 |
} |
| 728 |
if ($best_pos > 0) { |
| 729 |
$left = trim(substr($title, 0, $best_pos)); |
| 730 |
$right = trim(substr($title, $best_pos + strlen($best_sep))); |
| 731 |
if (mb_strlen($left, 'UTF-8') >= 3 && $right !== '' && mb_strlen($right, 'UTF-8') <= 60) { |
| 732 |
return $left; |
| 733 |
} |
| 734 |
} |
| 735 |
return $title; |
| 736 |
} |
| 737 |
|
| 738 |
/** |
| 739 |
* Make sure the knowledge table has the `title` column (3.2.23, plan d9ee66). |
| 740 |
* Activation adds it; this is the safety net for a copy updated in place, and |
| 741 |
* the one place add-ons ask "may I read/write title?". A positive answer is |
| 742 |
* remembered in an option, so after the first time this costs one option read. |
| 743 |
*/ |
| 744 |
public static function ensure_kb_title_column() { |
| 745 |
global $wpdb; |
| 746 |
static $ready = null; |
| 747 |
if ($ready !== null) { |
| 748 |
return $ready; |
| 749 |
} |
| 750 |
if (get_option('mxchat_kb_title_column') === '1') { |
| 751 |
$ready = true; |
| 752 |
return true; |
| 753 |
} |
| 754 |
$table = $wpdb->prefix . 'mxchat_system_prompt_content'; |
| 755 |
$cols = $wpdb->get_col("SHOW COLUMNS FROM {$table}", 0); |
| 756 |
if (empty($cols)) { |
| 757 |
$ready = false; |
| 758 |
return false; |
| 759 |
} |
| 760 |
if (!in_array('title', $cols, true)) { |
| 761 |
$wpdb->query("ALTER TABLE {$table} ADD COLUMN title VARCHAR(255) DEFAULT NULL AFTER source_url"); |
| 762 |
$cols = $wpdb->get_col("SHOW COLUMNS FROM {$table}", 0); |
| 763 |
} |
| 764 |
$ready = in_array('title', $cols, true); |
| 765 |
if ($ready) { |
| 766 |
update_option('mxchat_kb_title_column', '1', false); |
| 767 |
} |
| 768 |
return $ready; |
| 769 |
} |
| 770 |
|
| 771 |
/** |
| 772 |
* UPDATED: Check if Pinecone is enabled and properly configured for a specific bot |
| 773 |
*/ |
| 774 |
private static function is_pinecone_enabled_for_bot($bot_id = 'default') { |
| 775 |
// For default bot or when multi-bot is not active, use original method |
| 776 |
if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) { |
| 777 |
return self::is_pinecone_enabled(); |
| 778 |
} |
| 779 |
|
| 780 |
// Get bot-specific Pinecone configuration |
| 781 |
$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id); |
| 782 |
|
| 783 |
if (empty($bot_pinecone_config)) { |
| 784 |
// Fallback to default configuration |
| 785 |
return self::is_pinecone_enabled(); |
| 786 |
} |
| 787 |
|
| 788 |
$enabled_check = !empty($bot_pinecone_config['use_pinecone']) && $bot_pinecone_config['use_pinecone']; |
| 789 |
$api_key_check = !empty($bot_pinecone_config['api_key']); |
| 790 |
$host_check = !empty($bot_pinecone_config['host']); |
| 791 |
|
| 792 |
return $enabled_check && $api_key_check && $host_check; |
| 793 |
} |
| 794 |
|
| 795 |
/** |
| 796 |
* Check if Pinecone is enabled and properly configured (original method for default bot) |
| 797 |
*/ |
| 798 |
private static function is_pinecone_enabled() { |
| 799 |
$pinecone_options = get_option('mxchat_pinecone_addon_options'); |
| 800 |
|
| 801 |
if (empty($pinecone_options)) { |
| 802 |
return false; |
| 803 |
} |
| 804 |
|
| 805 |
$enabled_check = !empty($pinecone_options['mxchat_use_pinecone']) && $pinecone_options['mxchat_use_pinecone'] !== '0'; |
| 806 |
$api_key_check = !empty($pinecone_options['mxchat_pinecone_api_key']); |
| 807 |
$host_check = !empty($pinecone_options['mxchat_pinecone_host']); |
| 808 |
|
| 809 |
return $enabled_check && $api_key_check && $host_check; |
| 810 |
} |
| 811 |
|
| 812 |
/** |
| 813 |
* UPDATED: Store content in Pinecone only with bot support |
| 814 |
*/ |
| 815 |
private static function store_in_pinecone_only($embedding_vector, $content, $source_url, $vector_id = null, $bot_id = 'default', $content_type = 'content') { |
| 816 |
//error_log('[MXCHAT-PINECONE] ===== Using Pinecone-only storage for bot ' . $bot_id . ' ====='); |
| 817 |
|
| 818 |
// Get bot-specific Pinecone configuration |
| 819 |
if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) { |
| 820 |
$pinecone_options = get_option('mxchat_pinecone_addon_options'); |
| 821 |
$api_key = $pinecone_options['mxchat_pinecone_api_key']; |
| 822 |
$environment = $pinecone_options['mxchat_pinecone_environment'] ?? ''; |
| 823 |
$index_name = $pinecone_options['mxchat_pinecone_index'] ?? ''; |
| 824 |
$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; |
| 825 |
} else { |
| 826 |
$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id); |
| 827 |
if (empty($bot_pinecone_config)) { |
| 828 |
// Fallback to default configuration |
| 829 |
$pinecone_options = get_option('mxchat_pinecone_addon_options'); |
| 830 |
$api_key = $pinecone_options['mxchat_pinecone_api_key']; |
| 831 |
$environment = $pinecone_options['mxchat_pinecone_environment'] ?? ''; |
| 832 |
$index_name = $pinecone_options['mxchat_pinecone_index'] ?? ''; |
| 833 |
$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; |
| 834 |
} else { |
| 835 |
$api_key = $bot_pinecone_config['api_key']; |
| 836 |
$environment = ''; // Not used in new Pinecone API |
| 837 |
$index_name = ''; // Not used in new Pinecone API |
| 838 |
$namespace = $bot_pinecone_config['namespace'] ?? ''; |
| 839 |
} |
| 840 |
} |
| 841 |
|
| 842 |
$result = self::store_in_pinecone_main( |
| 843 |
$embedding_vector, |
| 844 |
$content, |
| 845 |
$source_url, |
| 846 |
$api_key, |
| 847 |
$environment, |
| 848 |
$index_name, |
| 849 |
$vector_id, |
| 850 |
$bot_id, |
| 851 |
$namespace, |
| 852 |
$content_type |
| 853 |
); |
| 854 |
|
| 855 |
if (is_wp_error($result)) { |
| 856 |
//error_log('[MXCHAT-PINECONE] Pinecone storage failed for bot ' . $bot_id . ': ' . $result->get_error_message()); |
| 857 |
return $result; |
| 858 |
} |
| 859 |
|
| 860 |
//error_log('[MXCHAT-PINECONE] Pinecone storage completed successfully for bot ' . $bot_id); |
| 861 |
return true; |
| 862 |
} |
| 863 |
|
| 864 |
/** |
| 865 |
* Store content in WordPress database with progressive fallback |
| 866 |
* UPDATED 2.5.6: Now includes content_type parameter |
| 867 |
*/ |
| 868 |
private static function store_in_wordpress_db($safe_content, $source_url, $embedding_vector_serialized, $table_name, $content_type = 'content') { |
| 869 |
global $wpdb; |
| 870 |
|
| 871 |
//error_log('[MXCHAT-DB] ===== Using WordPress-only storage ====='); |
| 872 |
|
| 873 |
// Sanitize content_type |
| 874 |
$content_type = sanitize_key($content_type); |
| 875 |
if (empty($content_type)) { |
| 876 |
$content_type = 'content'; // Fallback for backwards compatibility |
| 877 |
} |
| 878 |
|
| 879 |
// ===== FIXED: Generate unique identifier for manual content ===== |
| 880 |
$original_source_url = $source_url; |
| 881 |
// Check if this is truly manual content (no URL at all) vs a real URL that filter_var rejects |
| 882 |
// filter_var(FILTER_VALIDATE_URL) rejects valid URLs with encoded chars, non-ASCII, fragments, etc. |
| 883 |
// Use a looser check: if it starts with http(s):// or has a scheme, it's a URL |
| 884 |
$has_url_scheme = !empty($source_url) && preg_match('#^https?://#i', $source_url); |
| 885 |
// upload:// identities (admin document/PDF uploads, plan 0485e5) are stable |
| 886 |
// and deduplicable — treat them like URLs so a re-upload UPDATES the row |
| 887 |
// instead of minting a fresh manual identity (which would duplicate). |
| 888 |
$has_stable_identity = $has_url_scheme || (!empty($source_url) && preg_match('#^upload://#i', $source_url)); |
| 889 |
// Treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries |
| 890 |
$is_legacy_mxchat_url = $has_url_scheme && strpos($source_url, 'mxchat.ai') !== false; |
| 891 |
$is_manual_content = empty($source_url) || $source_url === '' || !$has_stable_identity || $is_legacy_mxchat_url; |
| 892 |
|
| 893 |
if ($is_manual_content) { |
| 894 |
// Generate unique identifier for manual content to prevent overwrites |
| 895 |
$source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false); |
| 896 |
//error_log('[MXCHAT-DB] Generated unique ID for manual content: ' . $source_url); |
| 897 |
} |
| 898 |
|
| 899 |
// Only check for duplicates if we have a valid source URL (not manual content) |
| 900 |
$existing_id = null; |
| 901 |
if (!$is_manual_content) { |
| 902 |
$existing_id = $wpdb->get_var( |
| 903 |
$wpdb->prepare( |
| 904 |
"SELECT id FROM {$table_name} WHERE source_url = %s LIMIT 1", |
| 905 |
$source_url |
| 906 |
) |
| 907 |
); |
| 908 |
//error_log('[MXCHAT-DB] Checked for existing URL, found ID: ' . ($existing_id ?: 'none')); |
| 909 |
} else { |
| 910 |
//error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)'); |
| 911 |
} |
| 912 |
// ===== END FIX ===== |
| 913 |
|
| 914 |
// Page title captured at import (plan d9ee66): written only when the |
| 915 |
// importer supplied one, so a re-index without a title keeps the old value. |
| 916 |
$title_value = $is_manual_content ? '' : self::source_title_for($source_url); |
| 917 |
$title_ready = ($title_value !== '') && self::ensure_kb_title_column(); |
| 918 |
|
| 919 |
// Progressive fallback mechanism for problematic content |
| 920 |
$attempt = 1; |
| 921 |
$max_attempts = 3; |
| 922 |
$current_content = $safe_content; |
| 923 |
$result = false; |
| 924 |
|
| 925 |
while ($attempt <= $max_attempts && $result === false) { |
| 926 |
try { |
| 927 |
if ($existing_id) { |
| 928 |
//error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')'); |
| 929 |
|
| 930 |
// Update the existing row - UPDATED 2.5.6: Added content_type |
| 931 |
$row = array( |
| 932 |
'url' => $source_url, |
| 933 |
'article_content' => $current_content, |
| 934 |
'embedding_vector' => $embedding_vector_serialized, |
| 935 |
'source_url' => $source_url, |
| 936 |
'content_type' => $content_type, |
| 937 |
'timestamp' => current_time('mysql'), |
| 938 |
); |
| 939 |
$formats = array('%s','%s','%s','%s','%s','%s'); |
| 940 |
if ($title_ready) { |
| 941 |
$row['title'] = $title_value; |
| 942 |
$formats[] = '%s'; |
| 943 |
} |
| 944 |
$result = $wpdb->update($table_name, $row, array('id' => $existing_id), $formats, array('%d')); |
| 945 |
} else { |
| 946 |
//error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')'); |
| 947 |
//error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000)); |
| 948 |
|
| 949 |
// Insert a new row - UPDATED 2.5.6: Added content_type |
| 950 |
$row = array( |
| 951 |
'url' => $source_url, // Now unique for manual content |
| 952 |
'article_content' => $current_content, |
| 953 |
'embedding_vector' => $embedding_vector_serialized, |
| 954 |
'source_url' => $source_url, // Now unique for manual content |
| 955 |
'content_type' => $content_type, |
| 956 |
'timestamp' => current_time('mysql'), |
| 957 |
); |
| 958 |
$formats = array('%s','%s','%s','%s','%s','%s'); |
| 959 |
if ($title_ready) { |
| 960 |
$row['title'] = $title_value; |
| 961 |
$formats[] = '%s'; |
| 962 |
} |
| 963 |
$result = $wpdb->insert($table_name, $row, $formats); |
| 964 |
} |
| 965 |
|
| 966 |
if ($result === false) { |
| 967 |
//error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')'); |
| 968 |
//error_log('[MXCHAT-DB] MySQL Error: ' . $wpdb->last_error); |
| 969 |
//error_log('[MXCHAT-DB] MySQL Error Number: ' . $wpdb->last_errno); |
| 970 |
//error_log('[MXCHAT-DB] Last Query: ' . substr($wpdb->last_query, 0, 500)); |
| 971 |
//error_log('[MXCHAT-DB] Content length: ' . strlen($current_content) . ' bytes'); |
| 972 |
//error_log('[MXCHAT-DB] Embedding vector length: ' . strlen($embedding_vector_serialized) . ' bytes'); |
| 973 |
|
| 974 |
// Progressively apply more aggressive sanitization on failure |
| 975 |
if ($attempt === 1) { |
| 976 |
// First fallback: Use a more aggressive character filter and shorten |
| 977 |
$current_content = preg_replace('/[^\p{L}\p{N}\s.,;:!?()-]/u', '', $current_content); |
| 978 |
$current_content = substr($current_content, 0, 50000); |
| 979 |
} else if ($attempt === 2) { |
| 980 |
// Second fallback: Keep only alphanumeric and basic punctuation, shorten further |
| 981 |
$current_content = preg_replace('/[^a-zA-Z0-9\s.,;:!?()-]/u', '', $current_content); |
| 982 |
$current_content = substr($current_content, 0, 30000); |
| 983 |
} |
| 984 |
|
| 985 |
$attempt++; |
| 986 |
} |
| 987 |
} catch (Exception $e) { |
| 988 |
//error_log('[MXCHAT-DB] Exception during database operation: ' . $e->getMessage()); |
| 989 |
$attempt++; |
| 990 |
} |
| 991 |
} |
| 992 |
|
| 993 |
if ($result === false) { |
| 994 |
//error_log('[MXCHAT-DB] All database operation attempts failed'); |
| 995 |
//error_log('[MXCHAT-DB] Final MySQL Error: ' . $wpdb->last_error); |
| 996 |
|
| 997 |
$detailed_error = sprintf( |
| 998 |
'Failed to store content in WordPress database after %d attempts. MySQL Error: %s (Error #%d). Content size: %d bytes, Embedding size: %d bytes', |
| 999 |
$max_attempts, |
| 1000 |
$wpdb->last_error, |
| 1001 |
$wpdb->last_errno, |
| 1002 |
strlen($current_content), |
| 1003 |
strlen($embedding_vector_serialized) |
| 1004 |
); |
| 1005 |
|
| 1006 |
return new WP_Error('database_failed', $detailed_error); |
| 1007 |
} |
| 1008 |
|
| 1009 |
//error_log('[MXCHAT-DB] WordPress database operation completed successfully (Attempt ' . ($attempt - 1) . ')'); |
| 1010 |
return true; |
| 1011 |
} |
| 1012 |
|
| 1013 |
/** |
| 1014 |
* UPDATED: Store content in Pinecone database with bot support |
| 1015 |
* UPDATED 2.5.6: Now accepts content_type parameter |
| 1016 |
*/ |
| 1017 |
private static function store_in_pinecone_main($embedding_vector, $content, $url, $api_key, $environment, $index_name, $vector_id = null, $bot_id = 'default', $namespace = '', $content_type = 'content') { |
| 1018 |
//error_log('[MXCHAT-PINECONE-MAIN] ===== Starting Pinecone storage for bot ' . $bot_id . ' ====='); |
| 1019 |
|
| 1020 |
// ===== UPDATED: Handle manual content with unique vector IDs ===== |
| 1021 |
if ($vector_id) { |
| 1022 |
// Use provided vector ID |
| 1023 |
//error_log('[MXCHAT-PINECONE-MAIN] Using provided vector ID: ' . $vector_id); |
| 1024 |
} elseif (!empty($url) && preg_match('#^(https?|upload)://#i', $url)) { |
| 1025 |
// For URLs — and stable upload:// identities (plan 0485e5) — use an |
| 1026 |
// identity-derived ID so a re-import upserts the same vector. |
| 1027 |
$vector_id = md5($url); |
| 1028 |
//error_log('[MXCHAT-PINECONE-MAIN] Generated vector ID from URL: ' . $vector_id); |
| 1029 |
} else { |
| 1030 |
// For manual content (empty/no URL scheme), generate unique ID |
| 1031 |
$vector_id = 'manual_' . time() . '_' . substr(md5($content . microtime(true)), 0, 8); |
| 1032 |
//error_log('[MXCHAT-PINECONE-MAIN] Generated unique vector ID for manual content: ' . $vector_id); |
| 1033 |
} |
| 1034 |
// ===== END UPDATE ===== |
| 1035 |
|
| 1036 |
// Get host from bot-specific config or fallback to default |
| 1037 |
if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) { |
| 1038 |
$options = get_option('mxchat_pinecone_addon_options'); |
| 1039 |
$host = $options['mxchat_pinecone_host'] ?? ''; |
| 1040 |
} else { |
| 1041 |
$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id); |
| 1042 |
if (!empty($bot_pinecone_config)) { |
| 1043 |
$host = $bot_pinecone_config['host'] ?? ''; |
| 1044 |
} else { |
| 1045 |
$options = get_option('mxchat_pinecone_addon_options'); |
| 1046 |
$host = $options['mxchat_pinecone_host'] ?? ''; |
| 1047 |
} |
| 1048 |
} |
| 1049 |
|
| 1050 |
//error_log('[MXCHAT-PINECONE-MAIN] Host: ' . $host); |
| 1051 |
//error_log('[MXCHAT-PINECONE-MAIN] API key length: ' . strlen($api_key)); |
| 1052 |
//error_log('[MXCHAT-PINECONE-MAIN] Bot ID: ' . $bot_id); |
| 1053 |
//error_log('[MXCHAT-PINECONE-MAIN] Namespace: ' . $namespace); |
| 1054 |
|
| 1055 |
if (empty($host)) { |
| 1056 |
//error_log('[MXCHAT-PINECONE-MAIN] ERROR: Host is empty'); |
| 1057 |
return new WP_Error('pinecone_config', 'Pinecone host is not configured. Please set the host in your bot settings.'); |
| 1058 |
} |
| 1059 |
|
| 1060 |
// ===== UPDATED 2.5.6: Use passed content_type or determine from URL if not provided ===== |
| 1061 |
// Sanitize content_type |
| 1062 |
$content_type = sanitize_key($content_type); |
| 1063 |
if (empty($content_type)) { |
| 1064 |
// Fallback to old detection logic for backwards compatibility |
| 1065 |
$is_product = false; |
| 1066 |
$content_type = 'manual'; // Default for manual content |
| 1067 |
|
| 1068 |
if (!empty($url) && preg_match('#^https?://#i', $url)) { |
| 1069 |
$is_product = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false); |
| 1070 |
$content_type = $is_product ? 'product' : 'content'; |
| 1071 |
} |
| 1072 |
} |
| 1073 |
|
| 1074 |
//error_log('[MXCHAT-PINECONE-MAIN] Content type: ' . $content_type); |
| 1075 |
// ===== END UPDATE ===== |
| 1076 |
|
| 1077 |
$api_endpoint = "https://{$host}/vectors/upsert"; |
| 1078 |
//error_log('[MXCHAT-PINECONE-MAIN] API endpoint: ' . $api_endpoint); |
| 1079 |
|
| 1080 |
// UPDATED 2.5.6: Use provided content_type in metadata |
| 1081 |
$metadata = array( |
| 1082 |
'text' => $content, |
| 1083 |
'source_url' => $url, // Can be empty for manual content |
| 1084 |
'type' => $content_type, // Now supports: post, page, pdf, url, manual, product, etc. |
| 1085 |
'last_updated' => time(), |
| 1086 |
'created_at' => time(), // Add creation timestamp |
| 1087 |
'bot_id' => $bot_id, // Add bot identification |
| 1088 |
); |
| 1089 |
$page_title = self::source_title_for($url); |
| 1090 |
if ($page_title !== '') { |
| 1091 |
$metadata['title'] = $page_title; // page title captured at import (plan d9ee66) |
| 1092 |
} |
| 1093 |
|
| 1094 |
// Document index (plan 362c31): same id, same metadata, through the |
| 1095 |
// documents API. A site on the classic vector index never enters here. |
| 1096 |
if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) { |
| 1097 |
return MxChat_Pinecone_Documents::upsert_one($vector_id, $content, $embedding_vector, $metadata, array( |
| 1098 |
'api_key' => $api_key, |
| 1099 |
'host' => $host, |
| 1100 |
'namespace' => $namespace, |
| 1101 |
)); |
| 1102 |
} |
| 1103 |
|
| 1104 |
$vector_data = array( |
| 1105 |
'id' => $vector_id, |
| 1106 |
'values' => $embedding_vector, |
| 1107 |
'metadata' => $metadata |
| 1108 |
); |
| 1109 |
|
| 1110 |
$request_body = array( |
| 1111 |
'vectors' => array($vector_data) |
| 1112 |
); |
| 1113 |
|
| 1114 |
// Add namespace if specified for multi-bot separation |
| 1115 |
if (!empty($namespace)) { |
| 1116 |
$request_body['namespace'] = $namespace; |
| 1117 |
//error_log('[MXCHAT-PINECONE-MAIN] Using namespace: ' . $namespace); |
| 1118 |
} |
| 1119 |
|
| 1120 |
//error_log('[MXCHAT-PINECONE-MAIN] Request body prepared (embedding dimensions: ' . count($embedding_vector) . ')'); |
| 1121 |
|
| 1122 |
$response = wp_remote_post($api_endpoint, array( |
| 1123 |
'headers' => array( |
| 1124 |
'Api-Key' => $api_key, |
| 1125 |
'accept' => 'application/json', |
| 1126 |
'content-type' => 'application/json' |
| 1127 |
), |
| 1128 |
'body' => wp_json_encode($request_body), |
| 1129 |
'timeout' => 30, |
| 1130 |
'data_format' => 'body' |
| 1131 |
)); |
| 1132 |
|
| 1133 |
if (is_wp_error($response)) { |
| 1134 |
//error_log('[MXCHAT-PINECONE-MAIN] WordPress request error: ' . $response->get_error_message()); |
| 1135 |
return new WP_Error('pinecone_request', $response->get_error_message()); |
| 1136 |
} |
| 1137 |
|
| 1138 |
$response_code = wp_remote_retrieve_response_code($response); |
| 1139 |
//error_log('[MXCHAT-PINECONE-MAIN] Response code: ' . $response_code); |
| 1140 |
|
| 1141 |
if ($response_code !== 200) { |
| 1142 |
$body = wp_remote_retrieve_body($response); |
| 1143 |
//error_log('[MXCHAT-PINECONE-MAIN] API error - Response body: ' . $body); |
| 1144 |
return new WP_Error('pinecone_api', sprintf( |
| 1145 |
'Pinecone API error (HTTP %d): %s', |
| 1146 |
$response_code, |
| 1147 |
$body |
| 1148 |
)); |
| 1149 |
} |
| 1150 |
|
| 1151 |
$response_body = wp_remote_retrieve_body($response); |
| 1152 |
//error_log('[MXCHAT-PINECONE-MAIN] Success response: ' . $response_body); |
| 1153 |
//error_log('[MXCHAT-PINECONE-MAIN] Successfully stored in Pinecone for bot ' . $bot_id); |
| 1154 |
//error_log('[MXCHAT-PINECONE-MAIN] ===== Pinecone storage complete ====='); |
| 1155 |
|
| 1156 |
return true; |
| 1157 |
} |
| 1158 |
|
| 1159 |
/** |
| 1160 |
* Caller-side pre-flight for KB ingestion: can an embedding request be made |
| 1161 |
* with these options, and which API key should travel downstream? |
| 1162 |
* |
| 1163 |
* Custom-provider-aware — generate_embedding() below routes to the custom |
| 1164 |
* endpoint FIRST and ignores the passed cloud key entirely when |
| 1165 |
* custom_provider_for_embeddings is on, so on that branch the only real |
| 1166 |
* requirement is a Base URL. Ingestion callers that gated on a cloud API key |
| 1167 |
* were killing keyless custom-embeddings sites (local Ollama / LM Studio |
| 1168 |
* class) before the embed layer could route (plan cbd5fd). |
| 1169 |
* |
| 1170 |
* NOTE: reads $options['embedding_model'] raw on purpose — this mirrors |
| 1171 |
* generate_embedding()'s own routing read, NOT the mismatch-banner's |
| 1172 |
* "selected" chain (get_selected_embedding_model). The helper must predict |
| 1173 |
* what the very next embed call will do, byte-for-byte. |
| 1174 |
* |
| 1175 |
* Decision only — callers keep their own error-surfacing shape (admin-notice |
| 1176 |
* transient + redirect, wp_send_json_error, WP_Error, silent return). |
| 1177 |
* |
| 1178 |
* @param array|null $options Resolved options (bot-specific where the caller |
| 1179 |
* has them); null loads the default bot's options. |
| 1180 |
* @return array { |
| 1181 |
* @type bool $ok Whether ingestion can proceed. |
| 1182 |
* @type string $api_key Key to pass downstream ('' on the custom branch — |
| 1183 |
* generate_embedding() ignores it there). |
| 1184 |
* @type string $reason Human-readable blocker; '' when $ok. |
| 1185 |
* @type string $provider Short provider label ('OpenAI', 'Voyage AI', |
| 1186 |
* 'Google Gemini', 'Custom Provider'). |
| 1187 |
* } |
| 1188 |
*/ |
| 1189 |
public static function embedding_preflight($options = null) { |
| 1190 |
if (!is_array($options)) { |
| 1191 |
$options = get_option('mxchat_options'); |
| 1192 |
$options = is_array($options) ? $options : array(); |
| 1193 |
} |
| 1194 |
|
| 1195 |
// Custom branch mirrors generate_embedding()'s routing order (custom first). |
| 1196 |
if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') { |
| 1197 |
$base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : ''; |
| 1198 |
if ($base_url === '') { |
| 1199 |
return array( |
| 1200 |
'ok' => false, |
| 1201 |
'api_key' => '', |
| 1202 |
// Same string generate_embedding_custom() returns for this state. |
| 1203 |
'reason' => __('Custom provider Base URL is not configured.', 'mxchat'), |
| 1204 |
'provider' => 'Custom Provider', |
| 1205 |
); |
| 1206 |
} |
| 1207 |
return array('ok' => true, 'api_key' => '', 'reason' => '', 'provider' => 'Custom Provider'); |
| 1208 |
} |
| 1209 |
|
| 1210 |
$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002'; |
| 1211 |
if (strpos($selected_model, 'voyage') === 0) { |
| 1212 |
$api_key = $options['voyage_api_key'] ?? ''; |
| 1213 |
$provider = 'Voyage AI'; |
| 1214 |
} elseif (strpos($selected_model, 'gemini-embedding') === 0) { |
| 1215 |
$api_key = $options['gemini_api_key'] ?? ''; |
| 1216 |
$provider = 'Google Gemini'; |
| 1217 |
} else { |
| 1218 |
$api_key = $options['api_key'] ?? ''; |
| 1219 |
$provider = 'OpenAI'; |
| 1220 |
} |
| 1221 |
|
| 1222 |
if (empty($api_key)) { |
| 1223 |
return array( |
| 1224 |
'ok' => false, |
| 1225 |
'api_key' => '', |
| 1226 |
'reason' => sprintf( |
| 1227 |
/* translators: %s: embedding provider name */ |
| 1228 |
__('%s API key is not configured. Please add your API key in the settings before submitting content.', 'mxchat'), |
| 1229 |
$provider |
| 1230 |
), |
| 1231 |
'provider' => $provider, |
| 1232 |
); |
| 1233 |
} |
| 1234 |
|
| 1235 |
return array('ok' => true, 'api_key' => $api_key, 'reason' => '', 'provider' => $provider); |
| 1236 |
} |
| 1237 |
|
| 1238 |
/** |
| 1239 |
* Public QUERY-side entry point (plan 876edb). The chat pipeline's |
| 1240 |
* MxChat_Integrator::mxchat_generate_embedding() adapter routes through here |
| 1241 |
* so the query and index sides share ONE provider-routing implementation — |
| 1242 |
* the same endpoints, request bodies, and stamping semantics. The Integrator |
| 1243 |
* keeps its own error vocabulary by translating the WP_Error this returns |
| 1244 |
* (see the structured error data on every failure path below). |
| 1245 |
* |
| 1246 |
* @param string $text The text to be embedded. |
| 1247 |
* @param string $api_key Caller-resolved API key (per-bot on the query side). |
| 1248 |
* @param string $bot_id The bot ID for multi-bot support. |
| 1249 |
* @return array|WP_Error The embedding vector, or WP_Error carrying the reason. |
| 1250 |
*/ |
| 1251 |
public static function generate_query_embedding($text, $api_key, $bot_id = 'default') { |
| 1252 |
return self::generate_embedding($text, $api_key, $bot_id); |
| 1253 |
} |
| 1254 |
|
| 1255 |
/** |
| 1256 |
* UPDATED: Generate an embedding for the given text using bot-specific configuration. |
| 1257 |
* |
| 1258 |
* @param string $text The text to be embedded. |
| 1259 |
* @param string $api_key The API key used for generating embeddings. |
| 1260 |
* @param string $bot_id The bot ID for multi-bot support |
| 1261 |
* @return array|null The embedding vector or null on failure. |
| 1262 |
*/ |
| 1263 |
private static function generate_embedding($text, $api_key, $bot_id = 'default') { |
| 1264 |
// Get bot-specific options |
| 1265 |
if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) { |
| 1266 |
$options = get_option('mxchat_options'); |
| 1267 |
} else { |
| 1268 |
$bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id); |
| 1269 |
$options = !empty($bot_options) ? $bot_options : get_option('mxchat_options'); |
| 1270 |
} |
| 1271 |
|
| 1272 |
// Opt-in: when the custom provider is selected for embeddings, route the KB |
| 1273 |
// INDEX side through the same custom endpoint the query side uses, so stored |
| 1274 |
// vectors and query vectors come from the same model. Default-off behavior |
| 1275 |
// below is untouched. |
| 1276 |
if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') { |
| 1277 |
$custom = self::generate_embedding_custom($text, $options); |
| 1278 |
// The custom path already returns a human-readable error string — |
| 1279 |
// carry it instead of collapsing to null (plan 4a7c0a). The 'custom' |
| 1280 |
// branch marker lets the Integrator adapter map the string back onto |
| 1281 |
// its own error codes (876edb). |
| 1282 |
return is_array($custom) ? $custom : new WP_Error('embedding_failed', (string) $custom, array('branch' => 'custom')); |
| 1283 |
} |
| 1284 |
|
| 1285 |
$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002'; |
| 1286 |
|
| 1287 |
// Determine endpoint and API key based on model |
| 1288 |
if (strpos($selected_model, 'voyage') === 0) { |
| 1289 |
$endpoint = 'https://api.voyageai.com/v1/embeddings'; |
| 1290 |
$api_key = $options['voyage_api_key'] ?? ''; |
| 1291 |
} elseif (strpos($selected_model, 'gemini-embedding') === 0) { |
| 1292 |
$endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent'; |
| 1293 |
$api_key = $options['gemini_api_key'] ?? ''; |
| 1294 |
} else { |
| 1295 |
$endpoint = 'https://api.openai.com/v1/embeddings'; |
| 1296 |
// Prefer the caller-resolved key when one was passed — the query side |
| 1297 |
// resolves per-bot keys at its call sites (integrator adapter, 876edb). |
| 1298 |
// Index callers pass the preflight key, which equals this options read, |
| 1299 |
// so nothing changes for them. |
| 1300 |
$api_key = !empty($api_key) ? $api_key : ($options['api_key'] ?? ''); |
| 1301 |
} |
| 1302 |
|
| 1303 |
// Prepare request body based on provider |
| 1304 |
if (strpos($selected_model, 'gemini-embedding') === 0) { |
| 1305 |
// Gemini API format |
| 1306 |
$request_body = [ |
| 1307 |
'model' => 'models/' . $selected_model, |
| 1308 |
'content' => [ |
| 1309 |
'parts' => [ |
| 1310 |
['text' => $text] |
| 1311 |
] |
| 1312 |
], |
| 1313 |
'outputDimensionality' => 1536 |
| 1314 |
]; |
| 1315 |
|
| 1316 |
// Prepare headers for Gemini (API key as query parameter) |
| 1317 |
$endpoint .= '?key=' . $api_key; |
| 1318 |
$headers = [ |
| 1319 |
'Content-Type' => 'application/json' |
| 1320 |
]; |
| 1321 |
} else { |
| 1322 |
// OpenAI/Voyage API format |
| 1323 |
$request_body = [ |
| 1324 |
'input' => $text, |
| 1325 |
'model' => $selected_model |
| 1326 |
]; |
| 1327 |
|
| 1328 |
// Add output_dimension for voyage-3-large |
| 1329 |
if ($selected_model === 'voyage-3-large') { |
| 1330 |
$request_body['output_dimension'] = 2048; |
| 1331 |
} |
| 1332 |
|
| 1333 |
// Prepare headers for OpenAI/Voyage |
| 1334 |
$headers = [ |
| 1335 |
'Content-Type' => 'application/json', |
| 1336 |
'Authorization' => 'Bearer ' . $api_key |
| 1337 |
]; |
| 1338 |
} |
| 1339 |
|
| 1340 |
$args = [ |
| 1341 |
'body' => wp_json_encode($request_body), |
| 1342 |
'headers' => $headers, |
| 1343 |
'timeout' => 60, |
| 1344 |
'redirection' => 5, |
| 1345 |
'blocking' => true, |
| 1346 |
'httpversion' => '1.0', |
| 1347 |
'sslverify' => true, |
| 1348 |
]; |
| 1349 |
|
| 1350 |
$response = wp_remote_post($endpoint, $args); |
| 1351 |
|
| 1352 |
if (is_wp_error($response)) { |
| 1353 |
$message = 'Embedding request failed (connection): ' . $response->get_error_message(); |
| 1354 |
if (class_exists('MxChat_Admin')) { |
| 1355 |
MxChat_Admin::mxchat_log_debug('embedding_error', $message, array('model' => $selected_model, 'bot_id' => $bot_id)); |
| 1356 |
} |
| 1357 |
return new WP_Error('embedding_failed', $message, array( |
| 1358 |
'branch' => 'cloud', |
| 1359 |
'kind' => 'connection', |
| 1360 |
'reason' => $response->get_error_message(), |
| 1361 |
'model' => $selected_model, |
| 1362 |
)); |
| 1363 |
} |
| 1364 |
|
| 1365 |
$response_body = json_decode(wp_remote_retrieve_body($response), true); |
| 1366 |
|
| 1367 |
// Handle different response formats based on provider |
| 1368 |
if (strpos($selected_model, 'gemini-embedding') === 0) { |
| 1369 |
// Gemini API response format |
| 1370 |
if (isset($response_body['embedding']['values']) && is_array($response_body['embedding']['values'])) { |
| 1371 |
self::stamp_active_embedding_model($selected_model); |
| 1372 |
return $response_body['embedding']['values']; |
| 1373 |
} else { |
| 1374 |
return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id); |
| 1375 |
} |
| 1376 |
} else { |
| 1377 |
// OpenAI/Voyage API response format |
| 1378 |
if (isset($response_body['data'][0]['embedding']) && is_array($response_body['data'][0]['embedding'])) { |
| 1379 |
self::stamp_active_embedding_model($selected_model); |
| 1380 |
return $response_body['data'][0]['embedding']; |
| 1381 |
} else { |
| 1382 |
return self::embedding_failure_error($response, $selected_model, $api_key, $bot_id); |
| 1383 |
} |
| 1384 |
} |
| 1385 |
} |
| 1386 |
|
| 1387 |
/** |
| 1388 |
* Build a WP_Error carrying the embedding provider's REAL failure reason, |
| 1389 |
* and record it in the Debug Mode log. Previously every failure path |
| 1390 |
* returned bare null, so customers saw only "Failed to generate embedding |
| 1391 |
* for content" / "Failed to store any chunks" with no cause (plan 4a7c0a). |
| 1392 |
* |
| 1393 |
* The API key never appears in provider response bodies (it travels in the |
| 1394 |
* request headers), but the reason is scrubbed for it anyway before it can |
| 1395 |
* reach a notice or the debug log. |
| 1396 |
*/ |
| 1397 |
private static function embedding_failure_error($response, $selected_model, $api_key, $bot_id) { |
| 1398 |
$status = (int) wp_remote_retrieve_response_code($response); |
| 1399 |
$raw = (string) wp_remote_retrieve_body($response); |
| 1400 |
$decoded = json_decode($raw, true); |
| 1401 |
|
| 1402 |
// Provider error shapes: OpenAI + Gemini use {"error":{"message":…}}; |
| 1403 |
// Voyage uses {"detail":…}. |
| 1404 |
$reason = ''; |
| 1405 |
if (is_array($decoded)) { |
| 1406 |
if (isset($decoded['error']['message']) && is_string($decoded['error']['message'])) { |
| 1407 |
$reason = $decoded['error']['message']; |
| 1408 |
} elseif (isset($decoded['detail']) && is_string($decoded['detail'])) { |
| 1409 |
$reason = $decoded['detail']; |
| 1410 |
} |
| 1411 |
} |
| 1412 |
if ($reason === '') { |
| 1413 |
$reason = ($raw !== '') ? substr($raw, 0, 200) : 'empty or malformed response'; |
| 1414 |
} |
| 1415 |
if (is_string($api_key) && $api_key !== '') { |
| 1416 |
$reason = str_replace($api_key, '[redacted]', $reason); |
| 1417 |
} |
| 1418 |
$reason = substr($reason, 0, 300); |
| 1419 |
$message = sprintf('Embedding failed (%s, HTTP %d): %s', $selected_model, $status, $reason); |
| 1420 |
|
| 1421 |
if (class_exists('MxChat_Admin')) { |
| 1422 |
MxChat_Admin::mxchat_log_debug('embedding_error', $message, array( |
| 1423 |
'model' => $selected_model, |
| 1424 |
'status' => $status, |
| 1425 |
'bot_id' => $bot_id, |
| 1426 |
)); |
| 1427 |
} |
| 1428 |
|
| 1429 |
// Structured data so the Integrator's query-side adapter can rebuild its |
| 1430 |
// typed error contract (auth/rate-limit/quota/invalid-response) without a |
| 1431 |
// second transport implementation (876edb). Additive — message unchanged. |
| 1432 |
return new WP_Error('embedding_failed', $message, array( |
| 1433 |
'branch' => 'cloud', |
| 1434 |
'status' => $status, |
| 1435 |
'error_type' => (is_array($decoded) && isset($decoded['error']['type']) && is_string($decoded['error']['type'])) ? $decoded['error']['type'] : '', |
| 1436 |
'reason' => $reason, |
| 1437 |
'model' => $selected_model, |
| 1438 |
)); |
| 1439 |
} |
| 1440 |
|
| 1441 |
/** |
| 1442 |
* Generate an embedding via a Custom (OpenAI-compatible) provider's /embeddings route. |
| 1443 |
* Shared by every embedding entry point so the KNOWLEDGE-BASE INDEX side and the |
| 1444 |
* QUERY side route through the same model when the opt-in |
| 1445 |
* 'custom_provider_for_embeddings' setting is on. Mirrors the query-path logic in |
| 1446 |
* MxChat_Integrator::mxchat_generate_embedding_custom() but takes an explicit |
| 1447 |
* $options array so it is callable statically from utils + knowledge-manager. |
| 1448 |
* |
| 1449 |
* Returns a numeric array (the embedding vector) on success, or a human-readable |
| 1450 |
* error string on failure (so callers expecting a string error, like the |
| 1451 |
* knowledge-manager, can surface it directly; callers expecting array|null wrap it). |
| 1452 |
* |
| 1453 |
* @param string $text Text to embed. |
| 1454 |
* @param array $options The resolved mxchat options (must contain the custom_provider_* keys). |
| 1455 |
* @return array|string Embedding vector on success; error string on failure. |
| 1456 |
*/ |
| 1457 |
public static function generate_embedding_custom($text, $options) { |
| 1458 |
if (empty($text)) { |
| 1459 |
return 'No text provided for embedding generation'; |
| 1460 |
} |
| 1461 |
|
| 1462 |
$base_url = isset($options['custom_provider_base_url']) ? rtrim(trim((string) $options['custom_provider_base_url']), '/') : ''; |
| 1463 |
if (empty($base_url)) { |
| 1464 |
return 'Custom provider Base URL is not configured.'; |
| 1465 |
} |
| 1466 |
|
| 1467 |
$api_key = isset($options['custom_provider_api_key']) ? trim((string) $options['custom_provider_api_key']) : ''; |
| 1468 |
$auth_scheme = isset($options['custom_provider_auth_scheme']) ? $options['custom_provider_auth_scheme'] : 'bearer'; |
| 1469 |
$api_version = isset($options['custom_provider_api_version']) ? trim((string) $options['custom_provider_api_version']) : ''; |
| 1470 |
|
| 1471 |
// Embedding model: shared resolver (dedicated embedding model -> chat model |
| 1472 |
// -> 'default') — the mismatch warning's "selected" side reads the same chain. |
| 1473 |
$model = self::resolve_custom_embedding_model($options); |
| 1474 |
|
| 1475 |
$embed_url = $base_url . '/embeddings'; |
| 1476 |
if (!empty($api_version)) { |
| 1477 |
$embed_url .= (strpos($embed_url, '?') === false ? '?' : '&') . 'api-version=' . rawurlencode($api_version); |
| 1478 |
} |
| 1479 |
|
| 1480 |
$headers = ['Content-Type' => 'application/json']; |
| 1481 |
if (!empty($api_key)) { |
| 1482 |
if ($auth_scheme === 'api-key') { |
| 1483 |
$headers['api-key'] = $api_key; |
| 1484 |
} else { |
| 1485 |
$headers['Authorization'] = 'Bearer ' . $api_key; |
| 1486 |
} |
| 1487 |
} |
| 1488 |
|
| 1489 |
$response = wp_remote_post($embed_url, [ |
| 1490 |
'headers' => $headers, |
| 1491 |
'body' => wp_json_encode(['input' => $text, 'model' => $model]), |
| 1492 |
'timeout' => 60, |
| 1493 |
]); |
| 1494 |
if (is_wp_error($response)) { |
| 1495 |
return self::log_custom_embedding_failure( |
| 1496 |
'Connection error when generating embeddings (custom provider): ' . $response->get_error_message(), |
| 1497 |
$model, |
| 1498 |
$api_key |
| 1499 |
); |
| 1500 |
} |
| 1501 |
|
| 1502 |
$status = wp_remote_retrieve_response_code($response); |
| 1503 |
$body = json_decode(wp_remote_retrieve_body($response), true); |
| 1504 |
if ($status !== 200) { |
| 1505 |
$msg = isset($body['error']['message']) ? $body['error']['message'] : 'HTTP ' . $status; |
| 1506 |
return self::log_custom_embedding_failure( |
| 1507 |
'Custom embedding endpoint error: ' . $msg, |
| 1508 |
$model, |
| 1509 |
$api_key, |
| 1510 |
(int) $status |
| 1511 |
); |
| 1512 |
} |
| 1513 |
if (isset($body['data'][0]['embedding']) && is_array($body['data'][0]['embedding'])) { |
| 1514 |
// Stamp the custom model identity so the active-embedding-model mismatch |
| 1515 |
// warning reflects the real (custom) model rather than the built-in setting. |
| 1516 |
self::stamp_active_embedding_model('custom:' . $model); |
| 1517 |
return $body['data'][0]['embedding']; |
| 1518 |
} |
| 1519 |
return self::log_custom_embedding_failure('Invalid embedding response from custom provider.', $model, $api_key); |
| 1520 |
} |
| 1521 |
|
| 1522 |
/** |
| 1523 |
* Record a custom-provider embedding failure in the Debug Mode log, then |
| 1524 |
* return the message unchanged so callers keep their string-error contract. |
| 1525 |
* The cloud branch has logged its failures since 4a7c0a; the custom branch |
| 1526 |
* never did, so chat-side failures on Custom-provider installs were |
| 1527 |
* invisible to Debug Mode despite the 3.2.18 readme saying otherwise |
| 1528 |
* (plan 71e4b6). Same scrub-then-log shape as embedding_failure_error(). |
| 1529 |
* |
| 1530 |
* @param string $message Human-readable failure (the caller's return value). |
| 1531 |
* @param string $model Resolved custom embedding model. |
| 1532 |
* @param string $api_key Scrubbed out of the logged message if it ever appears. |
| 1533 |
* @param int $status HTTP status when one was received, 0 otherwise. |
| 1534 |
* @return string The (scrubbed) message. |
| 1535 |
*/ |
| 1536 |
private static function log_custom_embedding_failure($message, $model, $api_key, $status = 0) { |
| 1537 |
if (is_string($api_key) && $api_key !== '') { |
| 1538 |
$message = str_replace($api_key, '[redacted]', $message); |
| 1539 |
} |
| 1540 |
|
| 1541 |
if (class_exists('MxChat_Admin')) { |
| 1542 |
$context = array('model' => 'custom:' . $model); |
| 1543 |
if ($status > 0) { |
| 1544 |
$context['status'] = $status; |
| 1545 |
} |
| 1546 |
MxChat_Admin::mxchat_log_debug('embedding_error', $message, $context); |
| 1547 |
} |
| 1548 |
|
| 1549 |
return $message; |
| 1550 |
} |
| 1551 |
|
| 1552 |
/** |
| 1553 |
* Submit content as multiple chunks |
| 1554 |
* |
| 1555 |
* Splits large content into chunks, generates embeddings for each, |
| 1556 |
* and stores them with chunk metadata for later reassembly. |
| 1557 |
* |
| 1558 |
* @param string $content The content to chunk and store |
| 1559 |
* @param string $source_url The source URL |
| 1560 |
* @param string $api_key The API key for embeddings |
| 1561 |
* @param string $bot_id The bot ID |
| 1562 |
* @param string $content_type The content type |
| 1563 |
* @param MxChat_Chunker $chunker The chunker instance |
| 1564 |
* @return bool|WP_Error True on success, WP_Error on failure |
| 1565 |
*/ |
| 1566 |
private static function submit_chunked_content($content, $source_url, $api_key, $bot_id, $content_type, $chunker) { |
| 1567 |
global $wpdb; |
| 1568 |
$table_name = $wpdb->prefix . 'mxchat_system_prompt_content'; |
| 1569 |
|
| 1570 |
//error_log('[MXCHAT-CHUNK-DEBUG] Starting chunked submission for: ' . $source_url); |
| 1571 |
//error_log('[MXCHAT-CHUNK-DEBUG] Content length: ' . strlen($content) . ' chars'); |
| 1572 |
|
| 1573 |
// URL-less (manual) content needs a minted identity BEFORE chunk ids are derived: |
| 1574 |
// every chunk id is md5(source_url)_chunk_N, so with source_url = '' EVERY long |
| 1575 |
// manual document shared the md5('') prefix — and the clean-slate delete below |
| 1576 |
// wiped the PREVIOUS manual entry's chunks each time a new one was added. The |
| 1577 |
// single-vector paths already mint (mxchat:// in WP, manual_* in Pinecone); this |
| 1578 |
// was the one storage path that didn't. Keep an mxchat:// identity if the caller |
| 1579 |
// already carries one. |
| 1580 |
if (strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, 'upload://') !== 0 && !preg_match('#^https?://#i', $source_url)) { |
| 1581 |
$source_url = 'mxchat://manual-content/' . time() . '-' . wp_generate_password(8, false); |
| 1582 |
} |
| 1583 |
|
| 1584 |
// First, delete any existing chunks for this URL (clean slate). |
| 1585 |
// Mirror-suspended: this is a re-store, not an entry removal — the Vector |
| 1586 |
// Store file is replaced (or kept, on hash match) by the caller's |
| 1587 |
// sync_upsert_entry after storage succeeds. |
| 1588 |
self::$vectorstore_mirror_suspended = true; |
| 1589 |
$delete_result = self::delete_chunks_for_url($source_url, $bot_id); |
| 1590 |
self::$vectorstore_mirror_suspended = false; |
| 1591 |
if (is_wp_error($delete_result)) { |
| 1592 |
//error_log('[MXCHAT-CHUNK-DEBUG] Warning: Failed to delete existing chunks: ' . $delete_result->get_error_message()); |
| 1593 |
// Continue anyway - we'll overwrite with upsert |
| 1594 |
} |
| 1595 |
|
| 1596 |
// Split content into chunks |
| 1597 |
$chunks = $chunker->chunk_text($content); |
| 1598 |
$total_chunks = count($chunks); |
| 1599 |
|
| 1600 |
//error_log('[MXCHAT-CHUNK-DEBUG] Created ' . $total_chunks . ' chunks'); |
| 1601 |
foreach ($chunks as $i => $chunk) { |
| 1602 |
//error_log('[MXCHAT-CHUNK-DEBUG] Chunk ' . $i . ' length: ' . strlen($chunk) . ' chars, preview: ' . substr($chunk, 0, 100)); |
| 1603 |
} |
| 1604 |
|
| 1605 |
//error_log('[MXCHAT-CHUNK] Split content into ' . $total_chunks . ' chunks'); |
| 1606 |
|
| 1607 |
if ($total_chunks === 0) { |
| 1608 |
return new WP_Error('chunking_failed', 'Content could not be split into chunks'); |
| 1609 |
} |
| 1610 |
|
| 1611 |
$errors = array(); |
| 1612 |
$embed_failures = 0; |
| 1613 |
$first_embed_reason = ''; |
| 1614 |
$first_store_reason = ''; |
| 1615 |
$is_pinecone = self::is_pinecone_enabled_for_bot($bot_id); |
| 1616 |
|
| 1617 |
foreach ($chunks as $index => $chunk_text) { |
| 1618 |
// Generate chunk metadata |
| 1619 |
$chunk_metadata = MxChat_Chunker::create_chunk_metadata($index, $total_chunks, $source_url); |
| 1620 |
|
| 1621 |
// AI-Engine-style aliases so external consumers (Pinecone/Qdrant/Chroma) can rely on |
| 1622 |
// a stable shorthand ('source'/'part_index'/'part_total') without parsing our internal names. |
| 1623 |
$chunk_metadata['source'] = $source_url; |
| 1624 |
$chunk_metadata['part_index'] = (int) $index; |
| 1625 |
$chunk_metadata['part_total'] = (int) $total_chunks; |
| 1626 |
|
| 1627 |
/** |
| 1628 |
* Filter the per-chunk metadata blob before it's written to the KB store. |
| 1629 |
* |
| 1630 |
* @param array $chunk_metadata Metadata array (source, part_index, part_total, chunk_index, total_chunks, source_url, parent_url_hash, document_type, ...). |
| 1631 |
* @param string $chunk_text The chunk text being stored. |
| 1632 |
* @param array $context ['bot_id' => string, 'content_type' => string, 'source_url' => string, 'part_index' => int, 'part_total' => int] |
| 1633 |
* @return array Updated metadata array. |
| 1634 |
*/ |
| 1635 |
$chunk_metadata = apply_filters( |
| 1636 |
'mxchat_embedding_chunk_metadata', |
| 1637 |
$chunk_metadata, |
| 1638 |
$chunk_text, |
| 1639 |
array( |
| 1640 |
'bot_id' => $bot_id, |
| 1641 |
'content_type' => $content_type, |
| 1642 |
'source_url' => $source_url, |
| 1643 |
'part_index' => (int) $index, |
| 1644 |
'part_total' => (int) $total_chunks, |
| 1645 |
) |
| 1646 |
); |
| 1647 |
|
| 1648 |
$chunk_vector_id = MxChat_Chunker::generate_chunk_vector_id($source_url, $index); |
| 1649 |
|
| 1650 |
//error_log('[MXCHAT-CHUNK] Processing chunk ' . ($index + 1) . '/' . $total_chunks . ' (ID: ' . $chunk_vector_id . ')'); |
| 1651 |
|
| 1652 |
// Generate embedding for this chunk |
| 1653 |
$embedding_vector = self::generate_embedding($chunk_text, $api_key, $bot_id); |
| 1654 |
|
| 1655 |
if (!is_array($embedding_vector)) { |
| 1656 |
// Track embedding failures separately from storage failures, and |
| 1657 |
// keep the first provider reason seen — the two failure classes |
| 1658 |
// have opposite remedies (API key vs Pinecone/DB) (plan 4a7c0a). |
| 1659 |
$embed_failures++; |
| 1660 |
$reason = is_wp_error($embedding_vector) ? $embedding_vector->get_error_message() : ''; |
| 1661 |
if ($reason !== '' && $first_embed_reason === '') { |
| 1662 |
$first_embed_reason = $reason; |
| 1663 |
} |
| 1664 |
$errors[] = new WP_Error('embedding_failed', 'Failed to generate embedding for chunk ' . $index . ($reason !== '' ? ' — ' . $reason : '')); |
| 1665 |
continue; |
| 1666 |
} |
| 1667 |
|
| 1668 |
if ($is_pinecone) { |
| 1669 |
// Store in Pinecone with chunk metadata |
| 1670 |
$result = self::store_chunk_in_pinecone( |
| 1671 |
$embedding_vector, |
| 1672 |
$chunk_text, |
| 1673 |
$source_url, |
| 1674 |
$chunk_vector_id, |
| 1675 |
$bot_id, |
| 1676 |
$content_type, |
| 1677 |
$chunk_metadata |
| 1678 |
); |
| 1679 |
} else { |
| 1680 |
// Store in WordPress DB with chunk metadata |
| 1681 |
$content_with_metadata = MxChat_Chunker::format_chunk_for_storage($chunk_text, $chunk_metadata); |
| 1682 |
$embedding_vector_serialized = maybe_serialize($embedding_vector); |
| 1683 |
|
| 1684 |
$result = self::store_chunk_in_wordpress_db( |
| 1685 |
$content_with_metadata, |
| 1686 |
$source_url, |
| 1687 |
$embedding_vector_serialized, |
| 1688 |
$table_name, |
| 1689 |
$content_type, |
| 1690 |
$chunk_metadata |
| 1691 |
); |
| 1692 |
} |
| 1693 |
|
| 1694 |
if (is_wp_error($result)) { |
| 1695 |
$errors[] = $result; |
| 1696 |
if ($first_store_reason === '') { |
| 1697 |
$first_store_reason = $result->get_error_message(); |
| 1698 |
} |
| 1699 |
} |
| 1700 |
} |
| 1701 |
|
| 1702 |
if (count($errors) === $total_chunks) { |
| 1703 |
// Say WHICH stage failed — "failed to store" used to cover pure |
| 1704 |
// embedding failures too, sending customers to debug Pinecone when |
| 1705 |
// the problem was their embedding API key (plan 4a7c0a). |
| 1706 |
if ($embed_failures === $total_chunks) { |
| 1707 |
return new WP_Error('chunking_failed', |
| 1708 |
'Failed to store any chunks — every chunk failed to embed' |
| 1709 |
. ($first_embed_reason !== '' ? ': ' . $first_embed_reason : '') |
| 1710 |
. ' Check the embedding provider API key and model under MxChat Settings.'); |
| 1711 |
} |
| 1712 |
if ($embed_failures === 0) { |
| 1713 |
return new WP_Error('chunking_failed', |
| 1714 |
'Failed to store any chunks — embeddings generated but storage failed' |
| 1715 |
. ($first_store_reason !== '' ? ': ' . $first_store_reason : '') |
| 1716 |
. ' Check the knowledge base storage (Pinecone index or database).'); |
| 1717 |
} |
| 1718 |
return new WP_Error('chunking_failed', sprintf( |
| 1719 |
'Failed to store any chunks — %d failed to embed%s and %d failed to store%s', |
| 1720 |
$embed_failures, |
| 1721 |
$first_embed_reason !== '' ? ' (' . $first_embed_reason . ')' : '', |
| 1722 |
$total_chunks - $embed_failures, |
| 1723 |
$first_store_reason !== '' ? ' (' . $first_store_reason . ')' : '' |
| 1724 |
)); |
| 1725 |
} |
| 1726 |
|
| 1727 |
if (!empty($errors)) { |
| 1728 |
$detail = $first_embed_reason !== '' ? $first_embed_reason : $first_store_reason; |
| 1729 |
return new WP_Error('chunking_partial_failure', |
| 1730 |
sprintf('Failed to store %d of %d chunks', count($errors), $total_chunks) |
| 1731 |
. ($detail !== '' ? ' — first error: ' . $detail : '')); |
| 1732 |
} |
| 1733 |
|
| 1734 |
//error_log('[MXCHAT-CHUNK] Successfully stored all ' . $total_chunks . ' chunks'); |
| 1735 |
return true; |
| 1736 |
} |
| 1737 |
|
| 1738 |
/** |
| 1739 |
* Store a single chunk in Pinecone with chunk-specific metadata |
| 1740 |
*/ |
| 1741 |
private static function store_chunk_in_pinecone($embedding_vector, $chunk_text, $source_url, $vector_id, $bot_id, $content_type, $chunk_metadata) { |
| 1742 |
// Get Pinecone configuration |
| 1743 |
if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) { |
| 1744 |
$pinecone_options = get_option('mxchat_pinecone_addon_options'); |
| 1745 |
$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? ''; |
| 1746 |
$host = $pinecone_options['mxchat_pinecone_host'] ?? ''; |
| 1747 |
$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; |
| 1748 |
} else { |
| 1749 |
$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id); |
| 1750 |
if (empty($bot_pinecone_config)) { |
| 1751 |
$pinecone_options = get_option('mxchat_pinecone_addon_options'); |
| 1752 |
$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? ''; |
| 1753 |
$host = $pinecone_options['mxchat_pinecone_host'] ?? ''; |
| 1754 |
$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; |
| 1755 |
} else { |
| 1756 |
$api_key = $bot_pinecone_config['api_key'] ?? ''; |
| 1757 |
$host = $bot_pinecone_config['host'] ?? ''; |
| 1758 |
$namespace = $bot_pinecone_config['namespace'] ?? ''; |
| 1759 |
} |
| 1760 |
} |
| 1761 |
|
| 1762 |
if (empty($host) || empty($api_key)) { |
| 1763 |
return new WP_Error('pinecone_config', 'Pinecone is not properly configured'); |
| 1764 |
} |
| 1765 |
|
| 1766 |
$api_endpoint = "https://{$host}/vectors/upsert"; |
| 1767 |
|
| 1768 |
// Build metadata with chunk information |
| 1769 |
$metadata = array( |
| 1770 |
'text' => $chunk_text, |
| 1771 |
'source_url' => $source_url, |
| 1772 |
'type' => $content_type, |
| 1773 |
'is_chunked' => true, |
| 1774 |
'chunk_index' => $chunk_metadata['chunk_index'], |
| 1775 |
'total_chunks' => $chunk_metadata['total_chunks'], |
| 1776 |
'parent_url_hash' => $chunk_metadata['parent_url_hash'], |
| 1777 |
'last_updated' => time(), |
| 1778 |
'created_at' => time(), |
| 1779 |
'bot_id' => $bot_id, |
| 1780 |
); |
| 1781 |
$page_title = self::source_title_for($source_url); |
| 1782 |
if ($page_title !== '') { |
| 1783 |
$metadata['title'] = $page_title; // page title captured at import (plan d9ee66) |
| 1784 |
} |
| 1785 |
|
| 1786 |
// Document index (plan 362c31): same chunk id and metadata, documents API. |
| 1787 |
if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) { |
| 1788 |
return MxChat_Pinecone_Documents::upsert_one($vector_id, $chunk_text, $embedding_vector, $metadata, array( |
| 1789 |
'api_key' => $api_key, |
| 1790 |
'host' => $host, |
| 1791 |
'namespace' => $namespace, |
| 1792 |
)); |
| 1793 |
} |
| 1794 |
|
| 1795 |
$vector_data = array( |
| 1796 |
'id' => $vector_id, |
| 1797 |
'values' => $embedding_vector, |
| 1798 |
'metadata' => $metadata |
| 1799 |
); |
| 1800 |
|
| 1801 |
$request_body = array( |
| 1802 |
'vectors' => array($vector_data) |
| 1803 |
); |
| 1804 |
|
| 1805 |
if (!empty($namespace)) { |
| 1806 |
$request_body['namespace'] = $namespace; |
| 1807 |
} |
| 1808 |
|
| 1809 |
$response = wp_remote_post($api_endpoint, array( |
| 1810 |
'headers' => array( |
| 1811 |
'Api-Key' => $api_key, |
| 1812 |
'accept' => 'application/json', |
| 1813 |
'content-type' => 'application/json' |
| 1814 |
), |
| 1815 |
'body' => wp_json_encode($request_body), |
| 1816 |
'timeout' => 30 |
| 1817 |
)); |
| 1818 |
|
| 1819 |
if (is_wp_error($response)) { |
| 1820 |
return $response; |
| 1821 |
} |
| 1822 |
|
| 1823 |
$response_code = wp_remote_retrieve_response_code($response); |
| 1824 |
if ($response_code !== 200) { |
| 1825 |
return new WP_Error('pinecone_api', 'Pinecone API error: HTTP ' . $response_code); |
| 1826 |
} |
| 1827 |
|
| 1828 |
return true; |
| 1829 |
} |
| 1830 |
|
| 1831 |
/** |
| 1832 |
* Store a single chunk in WordPress database |
| 1833 |
*/ |
| 1834 |
private static function store_chunk_in_wordpress_db($content_with_metadata, $source_url, $embedding_vector_serialized, $table_name, $content_type, $chunk_metadata) { |
| 1835 |
global $wpdb; |
| 1836 |
|
| 1837 |
// For chunks, we always insert new rows (no duplicate checking) |
| 1838 |
// The URL includes chunk info in the metadata, but source_url stays the same for grouping |
| 1839 |
$row = array( |
| 1840 |
'url' => $source_url, |
| 1841 |
'article_content' => $content_with_metadata, |
| 1842 |
'embedding_vector' => $embedding_vector_serialized, |
| 1843 |
'source_url' => $source_url, |
| 1844 |
'content_type' => $content_type, |
| 1845 |
'timestamp' => current_time('mysql') |
| 1846 |
); |
| 1847 |
$formats = array('%s', '%s', '%s', '%s', '%s', '%s'); |
| 1848 |
// Every chunk row carries the page title too (plan d9ee66) - readers group by |
| 1849 |
// source_url and may pick any chunk as the representative. |
| 1850 |
$page_title = self::source_title_for($source_url); |
| 1851 |
if ($page_title !== '' && self::ensure_kb_title_column()) { |
| 1852 |
$row['title'] = $page_title; |
| 1853 |
$formats[] = '%s'; |
| 1854 |
} |
| 1855 |
$result = $wpdb->insert($table_name, $row, $formats); |
| 1856 |
|
| 1857 |
if ($result === false) { |
| 1858 |
return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error); |
| 1859 |
} |
| 1860 |
|
| 1861 |
return true; |
| 1862 |
} |
| 1863 |
|
| 1864 |
/** |
| 1865 |
* Delete all chunks for a given URL |
| 1866 |
* |
| 1867 |
* @param string $source_url The source URL |
| 1868 |
* @param string $bot_id The bot ID |
| 1869 |
* @return bool|WP_Error True on success, WP_Error on failure |
| 1870 |
*/ |
| 1871 |
public static function delete_chunks_for_url($source_url, $bot_id = 'default') { |
| 1872 |
//error_log('[MXCHAT-CHUNK-DELETE] Deleting chunks for URL: ' . $source_url); |
| 1873 |
|
| 1874 |
// Entry removal — mirror it to the Vector Store (unless a storage routine |
| 1875 |
// is mid-re-store, see $vectorstore_mirror_suspended). |
| 1876 |
if (!self::$vectorstore_mirror_suspended && class_exists('MxChat_Vectorstore_Manager')) { |
| 1877 |
MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id); |
| 1878 |
} |
| 1879 |
|
| 1880 |
if (self::is_pinecone_enabled_for_bot($bot_id)) { |
| 1881 |
return self::delete_pinecone_chunks_by_url($source_url, $bot_id); |
| 1882 |
} else { |
| 1883 |
return self::delete_wordpress_chunks_by_url($source_url); |
| 1884 |
} |
| 1885 |
} |
| 1886 |
|
| 1887 |
/** |
| 1888 |
* Delete all chunks for a URL from Pinecone |
| 1889 |
*/ |
| 1890 |
/** |
| 1891 |
* Delete leftover md5(url)_chunk_N vectors after a URL's content was re-stored |
| 1892 |
* as a SINGLE vector (content shrank below the chunk threshold on edit/re-import). |
| 1893 |
* Unlike delete_pinecone_chunks_by_url this leaves the base id alone — the caller |
| 1894 |
* just upserted the new content there. Failure is logged, not fatal: the save |
| 1895 |
* itself succeeded, and the next save retries the sweep. |
| 1896 |
*/ |
| 1897 |
private static function cleanup_pinecone_chunk_stragglers($source_url, $bot_id) { |
| 1898 |
if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) { |
| 1899 |
$pinecone_options = get_option('mxchat_pinecone_addon_options'); |
| 1900 |
$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? ''; |
| 1901 |
$host = $pinecone_options['mxchat_pinecone_host'] ?? ''; |
| 1902 |
$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; |
| 1903 |
} else { |
| 1904 |
$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id); |
| 1905 |
if (empty($bot_pinecone_config)) { |
| 1906 |
$pinecone_options = get_option('mxchat_pinecone_addon_options'); |
| 1907 |
$api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? ''; |
| 1908 |
$host = $pinecone_options['mxchat_pinecone_host'] ?? ''; |
| 1909 |
$namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; |
| 1910 |
} else { |
| 1911 |
$api_key = $bot_pinecone_config['api_key'] ?? ''; |
| 1912 |
$host = $bot_pinecone_config['host'] ?? ''; |
| 1913 |
$namespace = $bot_pinecone_config['namespace'] ?? ''; |
| 1914 |
} |
| 1915 |
} |
| 1916 |
|
| 1917 |
if (empty($host) || empty($api_key)) { |
| 1918 |
return; |
| 1919 |
} |
| 1920 |
|
| 1921 |
// Document index (plan 362c31): list the chunk ids and delete them through the documents API. |
| 1922 |
if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) { |
| 1923 |
$docs_cfg = array('api_key' => $api_key, 'host' => $host, 'namespace' => $namespace); |
| 1924 |
$doc_stragglers = MxChat_Pinecone_Documents::list_ids(md5($source_url) . '_chunk_', $docs_cfg); |
| 1925 |
if (!empty($doc_stragglers)) { |
| 1926 |
$doc_result = MxChat_Pinecone_Documents::delete_ids($doc_stragglers, $docs_cfg); |
| 1927 |
if (is_wp_error($doc_result) && class_exists('MxChat_Admin') && method_exists('MxChat_Admin', 'mxchat_log_debug')) { |
| 1928 |
MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to sweep stale chunk documents after single-vector re-store', array('source_url' => $source_url, 'bot_id' => $bot_id, 'count' => count($doc_stragglers))); |
| 1929 |
} |
| 1930 |
} |
| 1931 |
return; |
| 1932 |
} |
| 1933 |
|
| 1934 |
$stragglers = array(); |
| 1935 |
|
| 1936 |
// Pinecone /vectors/list is a GET endpoint with query-string parameters (a POST |
| 1937 |
// answers 200-with-an-empty-body, which reads as "no stragglers"). |
| 1938 |
$query_params = array( |
| 1939 |
'prefix' => md5($source_url) . '_chunk_', |
| 1940 |
'limit' => 100, |
| 1941 |
); |
| 1942 |
if (!empty($namespace)) { |
| 1943 |
$query_params['namespace'] = $namespace; |
| 1944 |
} |
| 1945 |
|
| 1946 |
$list_url = "https://{$host}/vectors/list?" . http_build_query($query_params); |
| 1947 |
|
| 1948 |
do { |
| 1949 |
$list_response = wp_remote_get($list_url, array( |
| 1950 |
'headers' => array( |
| 1951 |
'Api-Key' => $api_key, |
| 1952 |
'accept' => 'application/json', |
| 1953 |
), |
| 1954 |
'timeout' => 30, |
| 1955 |
)); |
| 1956 |
|
| 1957 |
if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) { |
| 1958 |
break; |
| 1959 |
} |
| 1960 |
|
| 1961 |
$list_data = json_decode(wp_remote_retrieve_body($list_response), true); |
| 1962 |
if (!empty($list_data['vectors'])) { |
| 1963 |
foreach ($list_data['vectors'] as $vector) { |
| 1964 |
if (isset($vector['id'])) { |
| 1965 |
$stragglers[] = $vector['id']; |
| 1966 |
} |
| 1967 |
} |
| 1968 |
} |
| 1969 |
|
| 1970 |
$next_token = $list_data['pagination']['next'] ?? ''; |
| 1971 |
if (empty($next_token)) { |
| 1972 |
break; |
| 1973 |
} |
| 1974 |
|
| 1975 |
$query_params['paginationToken'] = $next_token; |
| 1976 |
$list_url = "https://{$host}/vectors/list?" . http_build_query($query_params); |
| 1977 |
} while (true); |
| 1978 |
|
| 1979 |
if (empty($stragglers)) { |
| 1980 |
return; |
| 1981 |
} |
| 1982 |
|
| 1983 |
$delete_body = array('ids' => $stragglers); |
| 1984 |
if (!empty($namespace)) { |
| 1985 |
$delete_body['namespace'] = $namespace; |
| 1986 |
} |
| 1987 |
|
| 1988 |
$delete_response = wp_remote_post("https://{$host}/vectors/delete", array( |
| 1989 |
'headers' => array( |
| 1990 |
'Api-Key' => $api_key, |
| 1991 |
'accept' => 'application/json', |
| 1992 |
'content-type' => 'application/json' |
| 1993 |
), |
| 1994 |
'body' => wp_json_encode($delete_body), |
| 1995 |
'timeout' => 30 |
| 1996 |
)); |
| 1997 |
|
| 1998 |
if ((is_wp_error($delete_response) || wp_remote_retrieve_response_code($delete_response) !== 200) |
| 1999 |
&& class_exists('MxChat_Admin') && method_exists('MxChat_Admin', 'mxchat_log_debug')) { |
| 2000 |
MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to sweep stale chunk vectors after single-vector re-store', array('source_url' => $source_url, 'bot_id' => $bot_id, 'count' => count($stragglers))); |
| 2001 |
} |
| 2002 |
} |
| 2003 |
|
| 2004 |
/** |
| 2005 |
* Resolve the Pinecone credentials for a bot: the bot's own config when the |
| 2006 |
* multi-bot add-on supplies one, else the site-wide options. Extracted so the |
| 2007 |
* delete path and the count path can never resolve different credentials. |
| 2008 |
* |
| 2009 |
* @param string $bot_id |
| 2010 |
* @return array{api_key:string,host:string,namespace:string} |
| 2011 |
*/ |
| 2012 |
private static function pinecone_config_for_bot($bot_id) { |
| 2013 |
if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) { |
| 2014 |
$bot_pinecone_config = array(); |
| 2015 |
} else { |
| 2016 |
$bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id); |
| 2017 |
} |
| 2018 |
|
| 2019 |
if (empty($bot_pinecone_config)) { |
| 2020 |
$pinecone_options = get_option('mxchat_pinecone_addon_options'); |
| 2021 |
return array( |
| 2022 |
'api_key' => $pinecone_options['mxchat_pinecone_api_key'] ?? '', |
| 2023 |
'host' => $pinecone_options['mxchat_pinecone_host'] ?? '', |
| 2024 |
'namespace' => $pinecone_options['mxchat_pinecone_namespace'] ?? '', |
| 2025 |
// 'vector' | 'document' (plan 362c31) — the delete / count paths branch on it. |
| 2026 |
'index_type' => (($pinecone_options['mxchat_pinecone_index_type'] ?? 'vector') === 'document') ? 'document' : 'vector', |
| 2027 |
); |
| 2028 |
} |
| 2029 |
|
| 2030 |
return array( |
| 2031 |
'api_key' => $bot_pinecone_config['api_key'] ?? '', |
| 2032 |
'host' => $bot_pinecone_config['host'] ?? '', |
| 2033 |
'namespace' => $bot_pinecone_config['namespace'] ?? '', |
| 2034 |
'index_type' => (($bot_pinecone_config['index_type'] ?? 'vector') === 'document') ? 'document' : 'vector', |
| 2035 |
); |
| 2036 |
} |
| 2037 |
|
| 2038 |
/** |
| 2039 |
* List every Pinecone vector id under a prefix, following pagination. |
| 2040 |
* |
| 2041 |
* Pinecone /vectors/list is a GET endpoint with query-string parameters; a POST here returns a |
| 2042 |
* non-200 silently and we end up seeing only the base vector, leaving chunks orphaned. |
| 2043 |
* |
| 2044 |
* @param string $prefix Vector-id prefix to match. |
| 2045 |
* @param array $cfg From pinecone_config_for_bot(). |
| 2046 |
* @return array List of vector ids (empty when none match or the API is unreachable). |
| 2047 |
*/ |
| 2048 |
private static function pinecone_list_ids_by_prefix($prefix, array $cfg) { |
| 2049 |
// Document index (plan 362c31): the documents API lists by prefix too. |
| 2050 |
if (($cfg['index_type'] ?? 'vector') === 'document' && class_exists('MxChat_Pinecone_Documents')) { |
| 2051 |
return MxChat_Pinecone_Documents::list_ids($prefix, $cfg); |
| 2052 |
} |
| 2053 |
|
| 2054 |
$ids = array(); |
| 2055 |
|
| 2056 |
$query_params = array( |
| 2057 |
'prefix' => $prefix, |
| 2058 |
'limit' => 100, |
| 2059 |
); |
| 2060 |
if (!empty($cfg['namespace'])) { |
| 2061 |
$query_params['namespace'] = $cfg['namespace']; |
| 2062 |
} |
| 2063 |
|
| 2064 |
$list_url = "https://{$cfg['host']}/vectors/list?" . http_build_query($query_params); |
| 2065 |
|
| 2066 |
// Paginate in case a URL has more than 100 chunks. |
| 2067 |
do { |
| 2068 |
$list_response = wp_remote_get($list_url, array( |
| 2069 |
'headers' => array( |
| 2070 |
'Api-Key' => $cfg['api_key'], |
| 2071 |
'accept' => 'application/json', |
| 2072 |
), |
| 2073 |
'timeout' => 30, |
| 2074 |
)); |
| 2075 |
|
| 2076 |
if (is_wp_error($list_response) || wp_remote_retrieve_response_code($list_response) !== 200) { |
| 2077 |
break; |
| 2078 |
} |
| 2079 |
|
| 2080 |
$list_data = json_decode(wp_remote_retrieve_body($list_response), true); |
| 2081 |
if (!empty($list_data['vectors'])) { |
| 2082 |
foreach ($list_data['vectors'] as $vector) { |
| 2083 |
if (isset($vector['id'])) { |
| 2084 |
$ids[] = $vector['id']; |
| 2085 |
} |
| 2086 |
} |
| 2087 |
} |
| 2088 |
|
| 2089 |
$next_token = $list_data['pagination']['next'] ?? ''; |
| 2090 |
if (empty($next_token)) { |
| 2091 |
break; |
| 2092 |
} |
| 2093 |
|
| 2094 |
$query_params['paginationToken'] = $next_token; |
| 2095 |
$list_url = "https://{$cfg['host']}/vectors/list?" . http_build_query($query_params); |
| 2096 |
} while (true); |
| 2097 |
|
| 2098 |
return $ids; |
| 2099 |
} |
| 2100 |
|
| 2101 |
/** |
| 2102 |
* How many stored pieces a URL currently has, in whichever backend holds it. |
| 2103 |
* |
| 2104 |
* Branches on the SAME condition delete_chunks_for_url() branches on, so the |
| 2105 |
* number reported is the number that call removes. Used by the REST DELETE |
| 2106 |
* route to answer "was anything actually there?" without a second delete path. |
| 2107 |
* |
| 2108 |
* @param string $source_url |
| 2109 |
* @param string $bot_id |
| 2110 |
* @return int |
| 2111 |
*/ |
| 2112 |
public static function count_chunks_for_url($source_url, $bot_id = 'default') { |
| 2113 |
if (self::is_pinecone_enabled_for_bot($bot_id)) { |
| 2114 |
$cfg = self::pinecone_config_for_bot($bot_id); |
| 2115 |
if (empty($cfg['host']) || empty($cfg['api_key'])) { |
| 2116 |
return 0; |
| 2117 |
} |
| 2118 |
// The base id and every "<base>_chunk_N" share the base as a prefix, |
| 2119 |
// so one prefix listing covers chunked and non-chunked content alike. |
| 2120 |
return count(self::pinecone_list_ids_by_prefix(md5($source_url), $cfg)); |
| 2121 |
} |
| 2122 |
|
| 2123 |
global $wpdb; |
| 2124 |
$table_name = $wpdb->prefix . 'mxchat_system_prompt_content'; |
| 2125 |
|
| 2126 |
return (int) $wpdb->get_var($wpdb->prepare( |
| 2127 |
"SELECT COUNT(*) FROM {$table_name} WHERE source_url = %s", |
| 2128 |
$source_url |
| 2129 |
)); |
| 2130 |
} |
| 2131 |
|
| 2132 |
private static function delete_pinecone_chunks_by_url($source_url, $bot_id) { |
| 2133 |
$cfg = self::pinecone_config_for_bot($bot_id); |
| 2134 |
$api_key = $cfg['api_key']; |
| 2135 |
$host = $cfg['host']; |
| 2136 |
$namespace = $cfg['namespace']; |
| 2137 |
|
| 2138 |
if (empty($host) || empty($api_key)) { |
| 2139 |
return new WP_Error('pinecone_config', 'Pinecone is not properly configured'); |
| 2140 |
} |
| 2141 |
|
| 2142 |
// Document index (plan 362c31): base id + every <md5>_chunk_N, documents API. |
| 2143 |
if (($cfg['index_type'] ?? 'vector') === 'document' && class_exists('MxChat_Pinecone_Documents')) { |
| 2144 |
$doc_ids = array_merge(array(md5($source_url)), MxChat_Pinecone_Documents::list_ids(md5($source_url) . '_chunk_', $cfg)); |
| 2145 |
return MxChat_Pinecone_Documents::delete_ids($doc_ids, $cfg); |
| 2146 |
} |
| 2147 |
|
| 2148 |
$base_vector_id = md5($source_url); |
| 2149 |
$vectors_to_delete = array(); |
| 2150 |
|
| 2151 |
// Add the original single-vector ID (for non-chunked content) |
| 2152 |
$vectors_to_delete[] = $base_vector_id; |
| 2153 |
|
| 2154 |
$vectors_to_delete = array_merge( |
| 2155 |
$vectors_to_delete, |
| 2156 |
self::pinecone_list_ids_by_prefix($base_vector_id . '_chunk_', $cfg) |
| 2157 |
); |
| 2158 |
|
| 2159 |
if (empty($vectors_to_delete)) { |
| 2160 |
//error_log('[MXCHAT-CHUNK-DELETE] No vectors found to delete'); |
| 2161 |
return true; |
| 2162 |
} |
| 2163 |
|
| 2164 |
//error_log('[MXCHAT-CHUNK-DELETE] Deleting ' . count($vectors_to_delete) . ' vectors from Pinecone'); |
| 2165 |
|
| 2166 |
// Delete vectors |
| 2167 |
$delete_url = "https://{$host}/vectors/delete"; |
| 2168 |
|
| 2169 |
$delete_body = array( |
| 2170 |
'ids' => $vectors_to_delete |
| 2171 |
); |
| 2172 |
|
| 2173 |
if (!empty($namespace)) { |
| 2174 |
$delete_body['namespace'] = $namespace; |
| 2175 |
} |
| 2176 |
|
| 2177 |
$delete_response = wp_remote_post($delete_url, array( |
| 2178 |
'headers' => array( |
| 2179 |
'Api-Key' => $api_key, |
| 2180 |
'accept' => 'application/json', |
| 2181 |
'content-type' => 'application/json' |
| 2182 |
), |
| 2183 |
'body' => wp_json_encode($delete_body), |
| 2184 |
'timeout' => 30 |
| 2185 |
)); |
| 2186 |
|
| 2187 |
if (is_wp_error($delete_response)) { |
| 2188 |
return $delete_response; |
| 2189 |
} |
| 2190 |
|
| 2191 |
$response_code = wp_remote_retrieve_response_code($delete_response); |
| 2192 |
if ($response_code !== 200) { |
| 2193 |
return new WP_Error('pinecone_delete', 'Failed to delete vectors: HTTP ' . $response_code); |
| 2194 |
} |
| 2195 |
|
| 2196 |
return true; |
| 2197 |
} |
| 2198 |
|
| 2199 |
/** |
| 2200 |
* Delete all chunks for a URL from WordPress database |
| 2201 |
*/ |
| 2202 |
private static function delete_wordpress_chunks_by_url($source_url) { |
| 2203 |
global $wpdb; |
| 2204 |
$table_name = $wpdb->prefix . 'mxchat_system_prompt_content'; |
| 2205 |
|
| 2206 |
// Delete all rows with this source_url (handles both chunked and non-chunked) |
| 2207 |
$result = $wpdb->delete( |
| 2208 |
$table_name, |
| 2209 |
array('source_url' => $source_url), |
| 2210 |
array('%s') |
| 2211 |
); |
| 2212 |
|
| 2213 |
if ($result === false) { |
| 2214 |
return new WP_Error('database_delete', 'Failed to delete chunks: ' . $wpdb->last_error); |
| 2215 |
} |
| 2216 |
|
| 2217 |
//error_log('[MXCHAT-CHUNK-DELETE] Deleted ' . $result . ' rows from WordPress DB'); |
| 2218 |
return true; |
| 2219 |
} |
| 2220 |
|
| 2221 |
/** |
| 2222 |
* Hybrid keyword boost (plan-38ffa1): detect whether the WP-DB knowledge |
| 2223 |
* table can serve the keyword leg via a MySQL FULLTEXT index, creating the |
| 2224 |
* index if needed. Detection runs once and caches the answer in the |
| 2225 |
* mxchat_hybrid_keyword_capability option ('fulltext' | 'like'); pass |
| 2226 |
* $force to re-detect. LIKE is the graceful fallback for shared hosts |
| 2227 |
* whose ALTER fails — the feature works either way, FULLTEXT just ranks |
| 2228 |
* better and scales. |
| 2229 |
* |
| 2230 |
* @param bool $force Re-run detection even if a cached answer exists. |
| 2231 |
* @return string 'fulltext' or 'like' |
| 2232 |
*/ |
| 2233 |
public static function mxchat_hybrid_detect_capability($force = false) { |
| 2234 |
$cached = get_option('mxchat_hybrid_keyword_capability', ''); |
| 2235 |
if (!$force && in_array($cached, array('fulltext', 'like'), true)) { |
| 2236 |
return $cached; |
| 2237 |
} |
| 2238 |
|
| 2239 |
global $wpdb; |
| 2240 |
$table = $wpdb->prefix . 'mxchat_system_prompt_content'; |
| 2241 |
|
| 2242 |
$index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'"); |
| 2243 |
if (!$index_exists) { |
| 2244 |
// Suppress the visible error on hosts where this is not permitted — |
| 2245 |
// failure is an expected, handled outcome (LIKE fallback). |
| 2246 |
$suppress = $wpdb->suppress_errors(true); |
| 2247 |
$wpdb->query("ALTER TABLE {$table} ADD FULLTEXT INDEX mxchat_content_ft (article_content)"); |
| 2248 |
$wpdb->suppress_errors($suppress); |
| 2249 |
$index_exists = $wpdb->get_var("SHOW INDEX FROM {$table} WHERE Key_name = 'mxchat_content_ft'"); |
| 2250 |
} |
| 2251 |
|
| 2252 |
$capability = $index_exists ? 'fulltext' : 'like'; |
| 2253 |
update_option('mxchat_hybrid_keyword_capability', $capability); |
| 2254 |
return $capability; |
| 2255 |
} |
| 2256 |
|
| 2257 |
/** |
| 2258 |
* Public entry for re-embedding already-stored content in place (wp mxchat |
| 2259 |
* rtl-repair, plan d1e6f7). Thin wrapper so the repair CLI gets the exact |
| 2260 |
* provider routing the import path uses — the repaired vector must come from |
| 2261 |
* the same model family the bot indexes with, or retrieval stays broken. |
| 2262 |
*/ |
| 2263 |
public static function regenerate_embedding($text, $api_key, $bot_id = 'default') { |
| 2264 |
return self::generate_embedding($text, $api_key, $bot_id); |
| 2265 |
} |
| 2266 |
|
| 2267 |
/** |
| 2268 |
* Restore logical character order in PDF-extracted RTL text (plan 32bf9e). |
| 2269 |
* |
| 2270 |
* The bundled Smalot parser only un-reverses text runs tagged with the |
| 2271 |
* ReversedChars marked-content operator (Word emits it; LibreOffice and most |
| 2272 |
* other producers do not), so their Hebrew/Arabic PDFs extract in visual |
| 2273 |
* (reversed) order and embed/search as garbage. This is OUR post-processing |
| 2274 |
* seam over getText() — the parser itself is never patched (it gets replaced |
| 2275 |
* wholesale on library updates). |
| 2276 |
* |
| 2277 |
* Heuristic and deliberately conservative, per line: |
| 2278 |
* - lines without strong RTL codepoints are untouched (a fully-Latin line in |
| 2279 |
* an RTL document therefore stays as extracted — accepted limitation); |
| 2280 |
* - Arabic presentation forms are a definitive visual-order signal (they only |
| 2281 |
* appear in shaped output): de-shape to base letters and reverse; |
| 2282 |
* - otherwise flip only on positive evidence — Hebrew final-letter position |
| 2283 |
* (a sofit at word START only happens in reversed text) or sentence |
| 2284 |
* punctuation position (leading in visual order, trailing in logical); |
| 2285 |
* - ambiguous lines are left alone: a conservative miss beats corrupting a |
| 2286 |
* Word-produced extraction the parser already handled (the double-flip |
| 2287 |
* guard this plan's approval named mandatory). |
| 2288 |
* |
| 2289 |
* @param string $text One extracted page string, straight from getText(). |
| 2290 |
* @param string $context Caller tag for the Debug Mode entry (site + page). |
| 2291 |
* @return string Text with RTL lines restored to logical order. |
| 2292 |
*/ |
| 2293 |
public static function normalize_pdf_rtl($text, $context = '') { |
| 2294 |
if (!is_string($text) || '' === $text) { |
| 2295 |
return $text; |
| 2296 |
} |
| 2297 |
// Escape hatch for sites whose PDFs already extract logically. |
| 2298 |
if (!apply_filters('mxchat_pdf_rtl_normalize', true, $text)) { |
| 2299 |
return $text; |
| 2300 |
} |
| 2301 |
// Fast bail: nothing RTL anywhere in the page. |
| 2302 |
if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $text)) { |
| 2303 |
return $text; |
| 2304 |
} |
| 2305 |
|
| 2306 |
$parts = preg_split('/(\R)/u', $text, -1, PREG_SPLIT_DELIM_CAPTURE); |
| 2307 |
if (false === $parts) { |
| 2308 |
return $text; |
| 2309 |
} |
| 2310 |
|
| 2311 |
$flipped_lines = 0; |
| 2312 |
$deshaped_lines = 0; |
| 2313 |
foreach ($parts as $i => $part) { |
| 2314 |
if ('' === $part || preg_match('/^\R$/u', $part)) { |
| 2315 |
continue; |
| 2316 |
} |
| 2317 |
$was_flipped = false; |
| 2318 |
$was_deshaped = false; |
| 2319 |
$new = self::pdf_rtl_normalize_line($part, $was_flipped, $was_deshaped); |
| 2320 |
if ($new !== $part) { |
| 2321 |
$parts[$i] = $new; |
| 2322 |
} |
| 2323 |
if ($was_flipped) { |
| 2324 |
$flipped_lines++; |
| 2325 |
} |
| 2326 |
if ($was_deshaped) { |
| 2327 |
$deshaped_lines++; |
| 2328 |
} |
| 2329 |
} |
| 2330 |
|
| 2331 |
if (($flipped_lines || $deshaped_lines) && class_exists('MxChat_Admin')) { |
| 2332 |
MxChat_Admin::mxchat_log_debug('pdf_rtl_normalized', 'RTL PDF text restored to logical order', array( |
| 2333 |
'context' => (string) $context, |
| 2334 |
'lines_flipped' => $flipped_lines, |
| 2335 |
'lines_deshaped' => $deshaped_lines, |
| 2336 |
'decision' => 'visual-order extraction detected', |
| 2337 |
)); |
| 2338 |
} |
| 2339 |
|
| 2340 |
return implode('', $parts); |
| 2341 |
} |
| 2342 |
|
| 2343 |
/** |
| 2344 |
* Normalize one line. Sets $flipped/$deshaped for the caller's debug entry. |
| 2345 |
*/ |
| 2346 |
private static function pdf_rtl_normalize_line($line, &$flipped, &$deshaped) { |
| 2347 |
$flipped = false; |
| 2348 |
$deshaped = false; |
| 2349 |
|
| 2350 |
if (!preg_match('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line)) { |
| 2351 |
return $line; |
| 2352 |
} |
| 2353 |
|
| 2354 |
$has_forms = (bool) preg_match('/[\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u', $line); |
| 2355 |
$work = $line; |
| 2356 |
if ($has_forms) { |
| 2357 |
$work = strtr($work, self::pdf_rtl_deshape_map()); |
| 2358 |
$deshaped = ($work !== $line); |
| 2359 |
} |
| 2360 |
|
| 2361 |
$verdict = 'ambiguous'; |
| 2362 |
if ($has_forms) { |
| 2363 |
// Shaped glyph codepoints only exist in visual-order output. |
| 2364 |
$verdict = 'visual'; |
| 2365 |
} else { |
| 2366 |
// Strong-direction dominance gate first: an LTR-dominant line with an |
| 2367 |
// embedded RTL word is not flip material. |
| 2368 |
$rtl_count = preg_match_all('/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}]/u', $work, $m_rtl); |
| 2369 |
$ltr_count = preg_match_all('/[A-Za-z]/u', $work, $m_ltr); |
| 2370 |
if ($rtl_count < 1 || $rtl_count <= $ltr_count) { |
| 2371 |
return $line; |
| 2372 |
} |
| 2373 |
|
| 2374 |
// Hebrew final letters (ך ם ן ף ץ) end words in logical text; one at |
| 2375 |
// a word START (Hebrew letter follows, none precedes) is reversal |
| 2376 |
// evidence. Positional, so it survives the line being reversed. |
| 2377 |
$sofit_initial = preg_match_all('/(?<![\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?=[\x{05D0}-\x{05EA}])/u', $work, $m_i); |
| 2378 |
$sofit_terminal = preg_match_all('/(?<=[\x{05D0}-\x{05EA}])[\x{05DA}\x{05DD}\x{05DF}\x{05E3}\x{05E5}](?![\x{05D0}-\x{05EA}])/u', $work, $m_t); |
| 2379 |
if ($sofit_initial > $sofit_terminal) { |
| 2380 |
$verdict = 'visual'; |
| 2381 |
} elseif ($sofit_terminal > $sofit_initial) { |
| 2382 |
$verdict = 'logical'; |
| 2383 |
} else { |
| 2384 |
// Sentence punctuation lands at the visual LEFT edge of an RTL |
| 2385 |
// line, i.e. the START of a visual-order extraction. |
| 2386 |
$trimmed = trim($work); |
| 2387 |
$starts_punct = (bool) preg_match('/^[.?!:;,]/u', $trimmed); |
| 2388 |
$ends_punct = (bool) preg_match('/[.?!:;,]$/u', $trimmed); |
| 2389 |
if ($starts_punct && !$ends_punct) { |
| 2390 |
$verdict = 'visual'; |
| 2391 |
} elseif ($ends_punct && !$starts_punct) { |
| 2392 |
$verdict = 'logical'; |
| 2393 |
} |
| 2394 |
} |
| 2395 |
} |
| 2396 |
|
| 2397 |
if ('visual' !== $verdict) { |
| 2398 |
// Ambiguous or logical: hand back the original line UNLESS we |
| 2399 |
// de-shaped (de-shaping alone is always safe — same letters, same |
| 2400 |
// order, un-ligated). |
| 2401 |
return $deshaped ? $work : $line; |
| 2402 |
} |
| 2403 |
|
| 2404 |
$flipped = true; |
| 2405 |
return self::pdf_rtl_flip_line($work); |
| 2406 |
} |
| 2407 |
|
| 2408 |
/** |
| 2409 |
* Reverse a visual-order line into logical order: full character reversal, |
| 2410 |
* mirror paired punctuation, then re-reverse embedded LTR runs (Latin words |
| 2411 |
* and digit sequences, incl. Arabic-Indic digits) so they stay readable. |
| 2412 |
*/ |
| 2413 |
private static function pdf_rtl_flip_line($line) { |
| 2414 |
$chars = preg_split('//u', $line, -1, PREG_SPLIT_NO_EMPTY); |
| 2415 |
if (false === $chars) { |
| 2416 |
return $line; |
| 2417 |
} |
| 2418 |
$reversed = implode('', array_reverse($chars)); |
| 2419 |
$reversed = strtr($reversed, array( |
| 2420 |
'(' => ')', ')' => '(', |
| 2421 |
'[' => ']', ']' => '[', |
| 2422 |
'{' => '}', '}' => '{', |
| 2423 |
'<' => '>', '>' => '<', |
| 2424 |
)); |
| 2425 |
$restored = preg_replace_callback( |
| 2426 |
'/[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}](?:[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9} .,\'"%\-:\/]*[0-9A-Za-z\x{0660}-\x{0669}\x{06F0}-\x{06F9}])?/u', |
| 2427 |
function ($m) { |
| 2428 |
$run = preg_split('//u', $m[0], -1, PREG_SPLIT_NO_EMPTY); |
| 2429 |
return false === $run ? $m[0] : implode('', array_reverse($run)); |
| 2430 |
}, |
| 2431 |
$reversed |
| 2432 |
); |
| 2433 |
return null === $restored ? $reversed : $restored; |
| 2434 |
} |
| 2435 |
|
| 2436 |
/** |
| 2437 |
* Arabic presentation forms (A + B) -> base letters. Built once from range |
| 2438 |
* specs rather than ~120 hand-written literal entries; every codepoint in a |
| 2439 |
* range maps to the same base sequence (isolated/final/initial/medial forms |
| 2440 |
* of one letter are contiguous in the FE70 block). |
| 2441 |
*/ |
| 2442 |
private static function pdf_rtl_deshape_map() { |
| 2443 |
static $map = null; |
| 2444 |
if (null !== $map) { |
| 2445 |
return $map; |
| 2446 |
} |
| 2447 |
$ranges = array( |
| 2448 |
// Form B harakat (each pair = standalone + tatweel-joined form). |
| 2449 |
array(0xFE70, 0xFE71, array(0x064B)), array(0xFE72, 0xFE72, array(0x064C)), |
| 2450 |
array(0xFE74, 0xFE74, array(0x064D)), array(0xFE76, 0xFE77, array(0x064E)), |
| 2451 |
array(0xFE78, 0xFE79, array(0x064F)), array(0xFE7A, 0xFE7B, array(0x0650)), |
| 2452 |
array(0xFE7C, 0xFE7D, array(0x0651)), array(0xFE7E, 0xFE7F, array(0x0652)), |
| 2453 |
// Form B letters. |
| 2454 |
array(0xFE80, 0xFE80, array(0x0621)), array(0xFE81, 0xFE82, array(0x0622)), |
| 2455 |
array(0xFE83, 0xFE84, array(0x0623)), array(0xFE85, 0xFE86, array(0x0624)), |
| 2456 |
array(0xFE87, 0xFE88, array(0x0625)), array(0xFE89, 0xFE8C, array(0x0626)), |
| 2457 |
array(0xFE8D, 0xFE8E, array(0x0627)), array(0xFE8F, 0xFE92, array(0x0628)), |
| 2458 |
array(0xFE93, 0xFE94, array(0x0629)), array(0xFE95, 0xFE98, array(0x062A)), |
| 2459 |
array(0xFE99, 0xFE9C, array(0x062B)), array(0xFE9D, 0xFEA0, array(0x062C)), |
| 2460 |
array(0xFEA1, 0xFEA4, array(0x062D)), array(0xFEA5, 0xFEA8, array(0x062E)), |
| 2461 |
array(0xFEA9, 0xFEAA, array(0x062F)), array(0xFEAB, 0xFEAC, array(0x0630)), |
| 2462 |
array(0xFEAD, 0xFEAE, array(0x0631)), array(0xFEAF, 0xFEB0, array(0x0632)), |
| 2463 |
array(0xFEB1, 0xFEB4, array(0x0633)), array(0xFEB5, 0xFEB8, array(0x0634)), |
| 2464 |
array(0xFEB9, 0xFEBC, array(0x0635)), array(0xFEBD, 0xFEC0, array(0x0636)), |
| 2465 |
array(0xFEC1, 0xFEC4, array(0x0637)), array(0xFEC5, 0xFEC8, array(0x0638)), |
| 2466 |
array(0xFEC9, 0xFECC, array(0x0639)), array(0xFECD, 0xFED0, array(0x063A)), |
| 2467 |
array(0xFED1, 0xFED4, array(0x0641)), array(0xFED5, 0xFED8, array(0x0642)), |
| 2468 |
array(0xFED9, 0xFEDC, array(0x0643)), array(0xFEDD, 0xFEE0, array(0x0644)), |
| 2469 |
array(0xFEE1, 0xFEE4, array(0x0645)), array(0xFEE5, 0xFEE8, array(0x0646)), |
| 2470 |
array(0xFEE9, 0xFEEC, array(0x0647)), array(0xFEED, 0xFEEE, array(0x0648)), |
| 2471 |
array(0xFEEF, 0xFEF0, array(0x0649)), array(0xFEF1, 0xFEF4, array(0x064A)), |
| 2472 |
// Form B lam-alef ligatures decompose to two letters. |
| 2473 |
array(0xFEF5, 0xFEF6, array(0x0644, 0x0622)), array(0xFEF7, 0xFEF8, array(0x0644, 0x0623)), |
| 2474 |
array(0xFEF9, 0xFEFA, array(0x0644, 0x0625)), array(0xFEFB, 0xFEFC, array(0x0644, 0x0627)), |
| 2475 |
// Form A: Persian / Urdu letters in common use. |
| 2476 |
array(0xFB56, 0xFB59, array(0x067E)), array(0xFB66, 0xFB69, array(0x0679)), |
| 2477 |
array(0xFB7A, 0xFB7D, array(0x0686)), array(0xFB88, 0xFB89, array(0x0688)), |
| 2478 |
array(0xFB8A, 0xFB8B, array(0x0698)), array(0xFB8E, 0xFB91, array(0x06A9)), |
| 2479 |
array(0xFB92, 0xFB95, array(0x06AF)), array(0xFBA6, 0xFBA9, array(0x06C1)), |
| 2480 |
array(0xFBAA, 0xFBAD, array(0x06BE)), array(0xFBAE, 0xFBAF, array(0x06D2)), |
| 2481 |
array(0xFBFC, 0xFBFF, array(0x06CC)), |
| 2482 |
); |
| 2483 |
$map = array(); |
| 2484 |
foreach ($ranges as $range) { |
| 2485 |
$base = ''; |
| 2486 |
foreach ($range[2] as $cp) { |
| 2487 |
$base .= self::pdf_rtl_cp_to_utf8($cp); |
| 2488 |
} |
| 2489 |
for ($cp = $range[0]; $cp <= $range[1]; $cp++) { |
| 2490 |
$map[self::pdf_rtl_cp_to_utf8($cp)] = $base; |
| 2491 |
} |
| 2492 |
} |
| 2493 |
return $map; |
| 2494 |
} |
| 2495 |
|
| 2496 |
/** |
| 2497 |
* Codepoint to UTF-8 without ext-intl / mbstring entity tricks (PHP 7.2 floor). |
| 2498 |
*/ |
| 2499 |
private static function pdf_rtl_cp_to_utf8($cp) { |
| 2500 |
if ($cp < 0x80) { |
| 2501 |
return chr($cp); |
| 2502 |
} |
| 2503 |
if ($cp < 0x800) { |
| 2504 |
return chr(0xC0 | ($cp >> 6)) . chr(0x80 | ($cp & 0x3F)); |
| 2505 |
} |
| 2506 |
if ($cp < 0x10000) { |
| 2507 |
return chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F)); |
| 2508 |
} |
| 2509 |
return chr(0xF0 | ($cp >> 18)) . chr(0x80 | (($cp >> 12) & 0x3F)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F)); |
| 2510 |
} |
| 2511 |
} |
