| 1 |
<?php |
| 2 |
/** |
| 3 |
* Pinecone document index adapter (plan 362c31). |
| 4 |
* |
| 5 |
* A Pinecone "document index" is a schema-based index (API version 2026-07) |
| 6 |
* whose string fields can carry a BM25 full-text index. MxChat offers it as a |
| 7 |
* second, OPT-IN index type beside the classic vector index: same API key, |
| 8 |
* host and namespace fields, an "Index type" choice on the Pinecone card, and |
| 9 |
* a copy-migration that moves an existing classic index across with zero |
| 10 |
* embedding calls (the classic index already holds the chunk text in metadata |
| 11 |
* and the vector in values). |
| 12 |
* |
| 13 |
* Everything the plugin does against a document index goes through this class: |
| 14 |
* the data plane (documents/upsert, /search, /fetch, /list, /delete), the |
| 15 |
* control plane (create / describe / delete index) and the migration. The |
| 16 |
* classic vector code paths are NOT modified — each of them carries a one-line |
| 17 |
* guard that hands a document-index configuration to the matching method here, |
| 18 |
* so a site on the classic index runs exactly the code it ran before. |
| 19 |
* |
| 20 |
* Field layout of a document (fixed — the schema is created by this class): |
| 21 |
* _id same ids as the classic index (md5(source_url), <md5>_chunk_N, manual_*) |
| 22 |
* text the chunk text, full-text indexed (string field, language from settings) |
| 23 |
* embedding the dense vector, dimension = the active embedding model's (client-supplied) |
| 24 |
* everything else (source_url, type, bot_id, chunk_index, total_chunks, is_chunked, |
| 25 |
* parent_url_hash, role_restriction, created_at, last_updated) = auto-indexed metadata. |
| 26 |
* |
| 27 |
* Response shapes measured 2026-09-19 against a live document index: |
| 28 |
* search → {matches:[{_id,_score,<fields…>}], namespace, usage} (fields are FLAT on the hit) |
| 29 |
* fetch → {documents:{<id>:{_id,<fields…>}}} (embedding included unless include_fields) |
| 30 |
* list → POST body {prefix,limit,pagination_token} → {documents:[{_id}], pagination:{next}} |
| 31 |
* upsert → 202 {upserted_count}; delete → 202 {}; a classic /query on a document index → 400. |
| 32 |
* Numeric fields come back as floats (chunk_index 0.0) — normalize_metadata() casts them. |
| 33 |
* A dense score_by cannot be combined with a text score_by in one search; hybrid is |
| 34 |
* a dense search plus a text-match FILTER, or two searches fused client-side. |
| 35 |
*/ |
| 36 |
|
| 37 |
if (!defined('ABSPATH')) { |
| 38 |
exit; |
| 39 |
} |
| 40 |
|
| 41 |
class MxChat_Pinecone_Documents { |
| 42 |
|
| 43 |
const API_VERSION = '2026-07'; |
| 44 |
const TEXT_FIELD = 'text'; |
| 45 |
const VECTOR_FIELD = 'embedding'; |
| 46 |
const CODES_FIELD = 'codes'; // plan 3e83e4: normalised part-number tokens stored beside the text |
| 47 |
const CODES_MAX = 60; // per document — a long spec sheet must not bloat the 40 KB metadata cap |
| 48 |
const CODE_MIN_LEN = 4; // shorter tokens would match everything |
| 49 |
const CODE_MAX_LEN = 32; |
| 50 |
const MIGRATION_KEY = 'mxchat_pinecone_docs_migration'; |
| 51 |
const UPSERT_BATCH = 40; // 1536-dim vectors are ~30 KB of JSON each; the request cap is 2 MB |
| 52 |
const MIGRATE_BATCH = 40; |
| 53 |
const MIGRATE_STEP_SECONDS = 12; // pages copied per AJAX step until this much wall time is used (3e83e4) |
| 54 |
const MIGRATE_RETRIES = 3; // attempts per Pinecone call inside a step, 1 s / 2 s / 4 s apart |
| 55 |
const OPTION = 'mxchat_pinecone_addon_options'; |
| 56 |
|
| 57 |
/** Fields the retrieval and listing paths ask for (never the vector unless needed). */ |
| 58 |
const RECORD_FIELDS = array('text', 'source_url', 'title', 'type', 'bot_id', 'is_chunked', 'chunk_index', 'total_chunks', 'parent_url_hash', 'role_restriction', 'created_at', 'last_updated'); |
| 59 |
|
| 60 |
public static function clouds() { |
| 61 |
return array('aws' => 'AWS', 'gcp' => 'GCP', 'azure' => 'Azure'); |
| 62 |
} |
| 63 |
|
| 64 |
public static function languages() { |
| 65 |
return array( |
| 66 |
'en' => 'English', 'de' => 'German', 'fr' => 'French', 'es' => 'Spanish', 'it' => 'Italian', |
| 67 |
'nl' => 'Dutch', 'pt' => 'Portuguese', 'sv' => 'Swedish', 'da' => 'Danish', 'no' => 'Norwegian', |
| 68 |
'fi' => 'Finnish', 'ru' => 'Russian', 'tr' => 'Turkish', 'ar' => 'Arabic', 'hu' => 'Hungarian', |
| 69 |
'ro' => 'Romanian', 'el' => 'Greek', |
| 70 |
); |
| 71 |
} |
| 72 |
|
| 73 |
// ===================================================================== |
| 74 |
// Configuration |
| 75 |
// ===================================================================== |
| 76 |
|
| 77 |
/** |
| 78 |
* 'vector' | 'document' from a mxchat_pinecone_addon_options-shaped array. |
| 79 |
*/ |
| 80 |
public static function options_index_type(array $options) { |
| 81 |
return (isset($options['mxchat_pinecone_index_type']) && $options['mxchat_pinecone_index_type'] === 'document') ? 'document' : 'vector'; |
| 82 |
} |
| 83 |
|
| 84 |
/** |
| 85 |
* Resolve the Pinecone connection for a bot: the bot's own config when the |
| 86 |
* Multi-Bot add-on supplies one (it may carry 'index_type'), else the |
| 87 |
* site-wide option. Same precedence the classic paths use. |
| 88 |
* |
| 89 |
* @return array{api_key:string,host:string,namespace:string,index_type:string,use_pinecone:bool} |
| 90 |
*/ |
| 91 |
public static function config($bot_id = 'default') { |
| 92 |
if ($bot_id === 'testing') { |
| 93 |
$bot_id = 'default'; |
| 94 |
} |
| 95 |
if ($bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager')) { |
| 96 |
$bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id); |
| 97 |
if (is_array($bot_config) && !empty($bot_config)) { |
| 98 |
return array( |
| 99 |
'api_key' => $bot_config['api_key'] ?? '', |
| 100 |
'host' => $bot_config['host'] ?? '', |
| 101 |
'namespace' => $bot_config['namespace'] ?? '', |
| 102 |
'index_type' => (($bot_config['index_type'] ?? 'vector') === 'document') ? 'document' : 'vector', |
| 103 |
'use_pinecone' => !empty($bot_config['use_pinecone']), |
| 104 |
); |
| 105 |
} |
| 106 |
} |
| 107 |
$options = get_option(self::OPTION, array()); |
| 108 |
if (!is_array($options)) { |
| 109 |
$options = array(); |
| 110 |
} |
| 111 |
return self::cfg_from_options($options); |
| 112 |
} |
| 113 |
|
| 114 |
/** |
| 115 |
* Same shape as config(), from an options array (the KB screen passes those around). |
| 116 |
*/ |
| 117 |
public static function cfg_from_options(array $options) { |
| 118 |
return array( |
| 119 |
'api_key' => $options['mxchat_pinecone_api_key'] ?? '', |
| 120 |
'host' => $options['mxchat_pinecone_host'] ?? '', |
| 121 |
'namespace' => $options['mxchat_pinecone_namespace'] ?? '', |
| 122 |
'index_type' => self::options_index_type($options), |
| 123 |
'use_pinecone' => isset($options['mxchat_use_pinecone']) && (string) $options['mxchat_use_pinecone'] === '1', |
| 124 |
); |
| 125 |
} |
| 126 |
|
| 127 |
/** Is this bot's Pinecone configuration a document index? */ |
| 128 |
public static function is_document_index($bot_id = 'default') { |
| 129 |
$cfg = self::config($bot_id); |
| 130 |
return $cfg['index_type'] === 'document' && !empty($cfg['host']) && !empty($cfg['api_key']); |
| 131 |
} |
| 132 |
|
| 133 |
/** |
| 134 |
* Is this host the site's document index? For the delete helpers that |
| 135 |
* receive bare credentials (no options array, no bot id). |
| 136 |
*/ |
| 137 |
public static function is_document_host($host) { |
| 138 |
$host = strtolower(trim((string) $host, '/')); |
| 139 |
if ($host === '') { |
| 140 |
return false; |
| 141 |
} |
| 142 |
$options = get_option(self::OPTION, array()); |
| 143 |
$is = is_array($options) |
| 144 |
&& self::options_index_type($options) === 'document' |
| 145 |
&& strtolower(trim((string) ($options['mxchat_pinecone_host'] ?? ''), '/')) === $host; |
| 146 |
return (bool) apply_filters('mxchat_pinecone_is_document_host', $is, $host); |
| 147 |
} |
| 148 |
|
| 149 |
/** The "processed content" row the Knowledge screens build per indexed post. */ |
| 150 |
public static function processed_row($vector_id, $url, array $metadata) { |
| 151 |
$created_at = $metadata['created_at'] ?? ''; |
| 152 |
$processed_date = 'Recently'; |
| 153 |
$timestamp = current_time('timestamp'); |
| 154 |
if (!empty($created_at)) { |
| 155 |
$ts = is_numeric($created_at) ? (int) $created_at : strtotime($created_at); |
| 156 |
if ($ts) { |
| 157 |
$timestamp = $ts; |
| 158 |
$processed_date = human_time_diff($ts, current_time('timestamp')) . ' ago'; |
| 159 |
} |
| 160 |
} |
| 161 |
return array( |
| 162 |
'db_id' => $vector_id, |
| 163 |
'processed_date' => $processed_date, |
| 164 |
'url' => $url, |
| 165 |
'source' => 'pinecone', |
| 166 |
'timestamp' => $timestamp, |
| 167 |
); |
| 168 |
} |
| 169 |
|
| 170 |
/** Is this options array (mxchat_pinecone_addon_options shape) a document index? */ |
| 171 |
public static function is_document_options($options) { |
| 172 |
if (!is_array($options)) { |
| 173 |
return false; |
| 174 |
} |
| 175 |
$cfg = self::cfg_from_options($options); |
| 176 |
return $cfg['index_type'] === 'document' && !empty($cfg['host']) && !empty($cfg['api_key']); |
| 177 |
} |
| 178 |
|
| 179 |
/** Namespace path segment: Pinecone addresses the default namespace as __default__. */ |
| 180 |
public static function namespace_path($namespace) { |
| 181 |
$namespace = (string) $namespace; |
| 182 |
return $namespace === '' ? '__default__' : rawurlencode($namespace); |
| 183 |
} |
| 184 |
|
| 185 |
/** Stats report the default namespace as "__default__" — the classic shape uses "". */ |
| 186 |
private static function namespace_stats_key($namespace) { |
| 187 |
return (string) $namespace === '' ? '__default__' : (string) $namespace; |
| 188 |
} |
| 189 |
|
| 190 |
// ===================================================================== |
| 191 |
// Transport |
| 192 |
// ===================================================================== |
| 193 |
|
| 194 |
/** |
| 195 |
* One HTTP call with the document-API version header. |
| 196 |
* |
| 197 |
* @return array{code:int,data:mixed,raw:string,error:string} |
| 198 |
*/ |
| 199 |
public static function request($method, $url, $body, $api_key, $timeout = 30) { |
| 200 |
$args = array( |
| 201 |
'method' => $method, |
| 202 |
'headers' => array( |
| 203 |
'Api-Key' => $api_key, |
| 204 |
'accept' => 'application/json', |
| 205 |
'content-type' => 'application/json', |
| 206 |
'X-Pinecone-Api-Version' => self::API_VERSION, |
| 207 |
), |
| 208 |
'timeout' => $timeout, |
| 209 |
); |
| 210 |
if ($body !== null) { |
| 211 |
$args['body'] = wp_json_encode($body); |
| 212 |
} |
| 213 |
$response = wp_remote_request($url, $args); |
| 214 |
if (is_wp_error($response)) { |
| 215 |
return array('code' => 0, 'data' => null, 'raw' => '', 'error' => $response->get_error_message()); |
| 216 |
} |
| 217 |
$code = (int) wp_remote_retrieve_response_code($response); |
| 218 |
$raw = (string) wp_remote_retrieve_body($response); |
| 219 |
$data = $raw === '' ? array() : json_decode($raw, true); |
| 220 |
$error = ''; |
| 221 |
if ($code < 200 || $code >= 300) { |
| 222 |
$error = is_array($data) && isset($data['error']['message']) ? $data['error']['message'] |
| 223 |
: (is_array($data) && isset($data['message']) ? $data['message'] : substr($raw, 0, 300)); |
| 224 |
$error = 'Pinecone API error (HTTP ' . $code . '): ' . $error; |
| 225 |
} |
| 226 |
return array('code' => $code, 'data' => $data, 'raw' => $raw, 'error' => $error); |
| 227 |
} |
| 228 |
|
| 229 |
private static function data_url(array $cfg, $path) { |
| 230 |
return 'https://' . $cfg['host'] . '/namespaces/' . self::namespace_path($cfg['namespace'] ?? '') . '/documents/' . ltrim($path, '/'); |
| 231 |
} |
| 232 |
|
| 233 |
private static function require_cfg(array $cfg) { |
| 234 |
if (empty($cfg['host']) || empty($cfg['api_key'])) { |
| 235 |
return new WP_Error('pinecone_config', 'Pinecone is not properly configured (host or API key missing).'); |
| 236 |
} |
| 237 |
return true; |
| 238 |
} |
| 239 |
|
| 240 |
// ===================================================================== |
| 241 |
// Document operations |
| 242 |
// ===================================================================== |
| 243 |
|
| 244 |
/** |
| 245 |
* Build one document from the pieces the classic upsert would have sent. |
| 246 |
* Metadata keys travel 1:1 (except 'text', which is the schema field). |
| 247 |
*/ |
| 248 |
public static function build_document($id, $text, array $embedding, array $metadata) { |
| 249 |
$doc = array('_id' => (string) $id); |
| 250 |
foreach ($metadata as $key => $value) { |
| 251 |
if ($key === self::TEXT_FIELD || $key === self::VECTOR_FIELD || $key === '_id') { |
| 252 |
continue; |
| 253 |
} |
| 254 |
if (is_scalar($value) || $value === null) { |
| 255 |
$doc[$key] = $value; |
| 256 |
} elseif (is_array($value)) { |
| 257 |
$doc[$key] = $value; |
| 258 |
} |
| 259 |
} |
| 260 |
$doc[self::TEXT_FIELD] = (string) $text; |
| 261 |
$doc[self::VECTOR_FIELD] = array_values(array_map('floatval', $embedding)); |
| 262 |
// 3e83e4: the normalised code set travels with every write (new content |
| 263 |
// and copied records alike) so a part number typed without its |
| 264 |
// separators can still reach the record — see index_codes(). |
| 265 |
$codes = self::index_codes((string) $text); |
| 266 |
if (!empty($codes)) { |
| 267 |
$doc[self::CODES_FIELD] = $codes; |
| 268 |
} else { |
| 269 |
unset($doc[self::CODES_FIELD]); |
| 270 |
} |
| 271 |
return $doc; |
| 272 |
} |
| 273 |
|
| 274 |
/** |
| 275 |
* Upsert documents in batches. Same id → the document is replaced. |
| 276 |
* |
| 277 |
* @return true|WP_Error |
| 278 |
*/ |
| 279 |
public static function upsert(array $documents, array $cfg) { |
| 280 |
$ok = self::require_cfg($cfg); |
| 281 |
if (is_wp_error($ok)) { |
| 282 |
return $ok; |
| 283 |
} |
| 284 |
if (empty($documents)) { |
| 285 |
return true; |
| 286 |
} |
| 287 |
foreach (array_chunk(array_values($documents), self::UPSERT_BATCH) as $batch) { |
| 288 |
$res = self::request('POST', self::data_url($cfg, 'upsert'), array('documents' => $batch), $cfg['api_key'], 60); |
| 289 |
if ($res['error'] !== '') { |
| 290 |
return new WP_Error('pinecone_api', $res['error']); |
| 291 |
} |
| 292 |
} |
| 293 |
return true; |
| 294 |
} |
| 295 |
|
| 296 |
public static function upsert_one($id, $text, array $embedding, array $metadata, array $cfg) { |
| 297 |
return self::upsert(array(self::build_document($id, $text, $embedding, $metadata)), $cfg); |
| 298 |
} |
| 299 |
|
| 300 |
/** |
| 301 |
* Search. Returns matches normalized to the CLASSIC shape |
| 302 |
* [{id, score, metadata:{…}}] so the grouping code can be shared. |
| 303 |
* |
| 304 |
* @return array|WP_Error |
| 305 |
*/ |
| 306 |
public static function search(array $score_by, $top_k, $filter, array $cfg, $include_fields = null) { |
| 307 |
$ok = self::require_cfg($cfg); |
| 308 |
if (is_wp_error($ok)) { |
| 309 |
return $ok; |
| 310 |
} |
| 311 |
$top_k = max(1, min(10000, (int) $top_k)); |
| 312 |
$body = array( |
| 313 |
'score_by' => $score_by, |
| 314 |
'top_k' => $top_k, |
| 315 |
'include_fields' => $include_fields === null ? self::RECORD_FIELDS : array_values($include_fields), |
| 316 |
); |
| 317 |
if (!empty($filter) && is_array($filter)) { |
| 318 |
$body['filter'] = $filter; |
| 319 |
} |
| 320 |
$res = self::request('POST', self::data_url($cfg, 'search'), $body, $cfg['api_key'], 30); |
| 321 |
if ($res['error'] !== '') { |
| 322 |
return new WP_Error('pinecone_api', $res['error']); |
| 323 |
} |
| 324 |
$matches = array(); |
| 325 |
if (is_array($res['data']) && !empty($res['data']['matches'])) { |
| 326 |
foreach ($res['data']['matches'] as $hit) { |
| 327 |
$matches[] = self::normalize_hit($hit); |
| 328 |
} |
| 329 |
} |
| 330 |
return $matches; |
| 331 |
} |
| 332 |
|
| 333 |
public static function search_dense(array $vector, $top_k, $filter, array $cfg, $include_fields = null) { |
| 334 |
return self::search( |
| 335 |
array(array('type' => 'dense_vector', 'fields' => array(self::VECTOR_FIELD), 'values' => array_values($vector))), |
| 336 |
$top_k, $filter, $cfg, $include_fields |
| 337 |
); |
| 338 |
} |
| 339 |
|
| 340 |
public static function search_text($query, $top_k, $filter, array $cfg, $include_fields = null) { |
| 341 |
return self::search( |
| 342 |
array(array('type' => 'text', 'fields' => array(self::TEXT_FIELD), 'query' => (string) $query)), |
| 343 |
$top_k, $filter, $cfg, $include_fields |
| 344 |
); |
| 345 |
} |
| 346 |
|
| 347 |
/** |
| 348 |
* Fetch documents by id. Returns the classic /vectors/fetch shape: |
| 349 |
* [id => ['id' => …, 'values' => [...] (only when asked), 'metadata' => [...]]]. |
| 350 |
*/ |
| 351 |
public static function fetch(array $ids, array $cfg, $include_fields = null, $with_vector = false) { |
| 352 |
$ok = self::require_cfg($cfg); |
| 353 |
if (is_wp_error($ok) || empty($ids)) { |
| 354 |
return array(); |
| 355 |
} |
| 356 |
$fields = $include_fields === null ? self::RECORD_FIELDS : array_values($include_fields); |
| 357 |
if ($with_vector && !in_array(self::VECTOR_FIELD, $fields, true)) { |
| 358 |
$fields[] = self::VECTOR_FIELD; |
| 359 |
} |
| 360 |
$out = array(); |
| 361 |
foreach (array_chunk(array_values(array_unique(array_map('strval', $ids))), 1000) as $chunk) { |
| 362 |
$res = self::request('POST', self::data_url($cfg, 'fetch'), array('ids' => $chunk, 'include_fields' => $fields), $cfg['api_key'], 30); |
| 363 |
if ($res['error'] !== '' || !is_array($res['data']) || empty($res['data']['documents'])) { |
| 364 |
continue; |
| 365 |
} |
| 366 |
foreach ($res['data']['documents'] as $doc_id => $doc) { |
| 367 |
if (!is_array($doc)) { |
| 368 |
continue; |
| 369 |
} |
| 370 |
$id = isset($doc['_id']) ? (string) $doc['_id'] : (string) $doc_id; |
| 371 |
$row = array('id' => $id, 'metadata' => self::normalize_metadata($doc)); |
| 372 |
if (isset($doc[self::VECTOR_FIELD]) && is_array($doc[self::VECTOR_FIELD])) { |
| 373 |
$row['values'] = $doc[self::VECTOR_FIELD]; |
| 374 |
} |
| 375 |
$out[$id] = $row; |
| 376 |
} |
| 377 |
} |
| 378 |
return $out; |
| 379 |
} |
| 380 |
|
| 381 |
/** |
| 382 |
* One page of ids. @return array{ids:array,next:string}|WP_Error |
| 383 |
*/ |
| 384 |
public static function list_page($prefix, $limit, $token, array $cfg) { |
| 385 |
$ok = self::require_cfg($cfg); |
| 386 |
if (is_wp_error($ok)) { |
| 387 |
return $ok; |
| 388 |
} |
| 389 |
$body = array('limit' => max(1, min(100, (int) $limit))); |
| 390 |
if ((string) $prefix !== '') { |
| 391 |
$body['prefix'] = (string) $prefix; |
| 392 |
} |
| 393 |
if ((string) $token !== '') { |
| 394 |
$body['pagination_token'] = (string) $token; |
| 395 |
} |
| 396 |
$res = self::request('POST', self::data_url($cfg, 'list'), $body, $cfg['api_key'], 30); |
| 397 |
if ($res['error'] !== '') { |
| 398 |
return new WP_Error('pinecone_api', $res['error']); |
| 399 |
} |
| 400 |
$ids = array(); |
| 401 |
if (is_array($res['data']) && !empty($res['data']['documents'])) { |
| 402 |
foreach ($res['data']['documents'] as $doc) { |
| 403 |
if (isset($doc['_id'])) { |
| 404 |
$ids[] = (string) $doc['_id']; |
| 405 |
} |
| 406 |
} |
| 407 |
} |
| 408 |
$next = is_array($res['data']) && isset($res['data']['pagination']['next']) ? (string) $res['data']['pagination']['next'] : ''; |
| 409 |
return array('ids' => $ids, 'next' => $next); |
| 410 |
} |
| 411 |
|
| 412 |
/** Every id under a prefix, following pagination (capped). */ |
| 413 |
public static function list_ids($prefix, array $cfg, $max = 10000) { |
| 414 |
$ids = array(); |
| 415 |
$token = ''; |
| 416 |
do { |
| 417 |
$page = self::list_page($prefix, 100, $token, $cfg); |
| 418 |
if (is_wp_error($page)) { |
| 419 |
break; |
| 420 |
} |
| 421 |
$ids = array_merge($ids, $page['ids']); |
| 422 |
$token = $page['next']; |
| 423 |
} while ($token !== '' && count($ids) < $max); |
| 424 |
return $ids; |
| 425 |
} |
| 426 |
|
| 427 |
/** @return true|WP_Error */ |
| 428 |
public static function delete_ids(array $ids, array $cfg) { |
| 429 |
$ok = self::require_cfg($cfg); |
| 430 |
if (is_wp_error($ok)) { |
| 431 |
return $ok; |
| 432 |
} |
| 433 |
$ids = array_values(array_unique(array_map('strval', $ids))); |
| 434 |
if (empty($ids)) { |
| 435 |
return true; |
| 436 |
} |
| 437 |
foreach (array_chunk($ids, 1000) as $chunk) { |
| 438 |
$res = self::request('POST', self::data_url($cfg, 'delete'), array('ids' => $chunk), $cfg['api_key'], 60); |
| 439 |
// 404 = the namespace does not exist yet — nothing to delete. |
| 440 |
if ($res['error'] !== '' && $res['code'] !== 404) { |
| 441 |
return new WP_Error('pinecone_api', $res['error']); |
| 442 |
} |
| 443 |
} |
| 444 |
return true; |
| 445 |
} |
| 446 |
|
| 447 |
/** Delete every document in the namespace, or those matching a metadata filter. @return true|WP_Error */ |
| 448 |
public static function delete_all(array $cfg, $filter = null) { |
| 449 |
$ok = self::require_cfg($cfg); |
| 450 |
if (is_wp_error($ok)) { |
| 451 |
return $ok; |
| 452 |
} |
| 453 |
$body = (!empty($filter) && is_array($filter)) ? array('filter' => $filter) : array('delete_all' => true); |
| 454 |
$res = self::request('POST', self::data_url($cfg, 'delete'), $body, $cfg['api_key'], 60); |
| 455 |
if ($res['error'] !== '' && $res['code'] !== 404) { |
| 456 |
return new WP_Error('pinecone_api', $res['error']); |
| 457 |
} |
| 458 |
return true; |
| 459 |
} |
| 460 |
|
| 461 |
/** |
| 462 |
* describe_index_stats, classic-shaped: namespaces keyed the classic way |
| 463 |
* ('' for the default namespace), totalVectorCount, dimension. |
| 464 |
* |
| 465 |
* @return array|WP_Error |
| 466 |
*/ |
| 467 |
public static function stats(array $cfg) { |
| 468 |
$ok = self::require_cfg($cfg); |
| 469 |
if (is_wp_error($ok)) { |
| 470 |
return $ok; |
| 471 |
} |
| 472 |
$res = self::request('POST', 'https://' . $cfg['host'] . '/describe_index_stats', new stdClass(), $cfg['api_key'], 15); |
| 473 |
if ($res['error'] !== '' || !is_array($res['data'])) { |
| 474 |
return new WP_Error('pinecone_api', $res['error'] !== '' ? $res['error'] : 'Empty stats response'); |
| 475 |
} |
| 476 |
$namespaces = array(); |
| 477 |
if (!empty($res['data']['namespaces']) && is_array($res['data']['namespaces'])) { |
| 478 |
foreach ($res['data']['namespaces'] as $name => $info) { |
| 479 |
$key = ($name === '__default__') ? '' : (string) $name; |
| 480 |
$namespaces[$key] = array('vectorCount' => (int) ($info['vectorCount'] ?? $info['recordCount'] ?? 0)); |
| 481 |
} |
| 482 |
} |
| 483 |
return array( |
| 484 |
'namespaces' => $namespaces, |
| 485 |
'totalVectorCount' => (int) ($res['data']['totalVectorCount'] ?? 0), |
| 486 |
'dimension' => (int) ($res['data']['dimension'] ?? 0), |
| 487 |
); |
| 488 |
} |
| 489 |
|
| 490 |
/** Record count for the configured namespace (0 when unknown / not listed). */ |
| 491 |
public static function count(array $cfg) { |
| 492 |
$stats = self::stats($cfg); |
| 493 |
if (is_wp_error($stats)) { |
| 494 |
return 0; |
| 495 |
} |
| 496 |
$ns = (string) ($cfg['namespace'] ?? ''); |
| 497 |
if ($ns !== '') { |
| 498 |
return isset($stats['namespaces'][$ns]['vectorCount']) ? (int) $stats['namespaces'][$ns]['vectorCount'] : 0; |
| 499 |
} |
| 500 |
return (int) $stats['totalVectorCount']; |
| 501 |
} |
| 502 |
|
| 503 |
/** |
| 504 |
* Reassemble a chunked entry's text from its <md5>_chunk_N documents — |
| 505 |
* the document-index twin of the integrator's reassemble_chunks_from_pinecone(). |
| 506 |
*/ |
| 507 |
public static function reassemble_chunks($source_url, array $cfg, $max_chunks = 0, &$chunk_count = 0) { |
| 508 |
$chunk_count = 0; |
| 509 |
if (empty($cfg['host']) || empty($cfg['api_key'])) { |
| 510 |
return ''; |
| 511 |
} |
| 512 |
$ids = self::list_ids(md5($source_url) . '_chunk_', $cfg, 500); |
| 513 |
if (empty($ids)) { |
| 514 |
return ''; |
| 515 |
} |
| 516 |
$docs = self::fetch($ids, $cfg, array(self::TEXT_FIELD, 'chunk_index')); |
| 517 |
if (empty($docs)) { |
| 518 |
return ''; |
| 519 |
} |
| 520 |
$chunks = array(); |
| 521 |
foreach ($docs as $doc) { |
| 522 |
$chunks[(int) ($doc['metadata']['chunk_index'] ?? 0)] = (string) ($doc['metadata']['text'] ?? ''); |
| 523 |
} |
| 524 |
ksort($chunks); |
| 525 |
if ($max_chunks > 0 && count($chunks) > $max_chunks) { |
| 526 |
$chunks = array_slice($chunks, 0, $max_chunks, true); |
| 527 |
} |
| 528 |
$chunk_count = count($chunks); |
| 529 |
return implode("\n\n", $chunks); |
| 530 |
} |
| 531 |
|
| 532 |
/** Flat search hit → classic {id, score, metadata}. */ |
| 533 |
public static function normalize_hit(array $hit) { |
| 534 |
$id = isset($hit['_id']) ? (string) $hit['_id'] : ''; |
| 535 |
$score = isset($hit['_score']) ? (float) $hit['_score'] : 0.0; |
| 536 |
$row = array('id' => $id, 'score' => $score, 'metadata' => self::normalize_metadata($hit)); |
| 537 |
if (isset($hit[self::VECTOR_FIELD]) && is_array($hit[self::VECTOR_FIELD])) { |
| 538 |
$row['values'] = $hit[self::VECTOR_FIELD]; |
| 539 |
} |
| 540 |
return $row; |
| 541 |
} |
| 542 |
|
| 543 |
/** Document fields → the metadata array the classic code reads (ints where it expects ints). */ |
| 544 |
public static function normalize_metadata(array $fields) { |
| 545 |
$meta = array(); |
| 546 |
foreach ($fields as $key => $value) { |
| 547 |
if ($key === '_id' || $key === '_score' || $key === self::VECTOR_FIELD) { |
| 548 |
continue; |
| 549 |
} |
| 550 |
$meta[$key] = $value; |
| 551 |
} |
| 552 |
foreach (array('chunk_index', 'total_chunks', 'created_at', 'last_updated') as $int_key) { |
| 553 |
if (isset($meta[$int_key]) && is_numeric($meta[$int_key])) { |
| 554 |
$meta[$int_key] = (int) $meta[$int_key]; |
| 555 |
} |
| 556 |
} |
| 557 |
if (isset($meta['is_chunked'])) { |
| 558 |
$meta['is_chunked'] = filter_var($meta['is_chunked'], FILTER_VALIDATE_BOOLEAN); |
| 559 |
} |
| 560 |
return $meta; |
| 561 |
} |
| 562 |
|
| 563 |
/** Classic {id, score, metadata} rows → the KB screen's record objects. */ |
| 564 |
public static function records_from_matches(array $matches, $bot_id = 'default', $content_type = '') { |
| 565 |
$records = array(); |
| 566 |
foreach ($matches as $match) { |
| 567 |
$metadata = $match['metadata'] ?? array(); |
| 568 |
if ($content_type !== '' && (($metadata['type'] ?? 'content') !== $content_type)) { |
| 569 |
continue; |
| 570 |
} |
| 571 |
$created_at = $metadata['created_at'] ?? $metadata['last_updated'] ?? time(); |
| 572 |
if (!is_numeric($created_at)) { |
| 573 |
$created_at = strtotime($created_at) ?: time(); |
| 574 |
} |
| 575 |
$records[] = (object) array( |
| 576 |
'id' => $match['id'] ?? '', |
| 577 |
'article_content' => $metadata['text'] ?? '', |
| 578 |
'source_url' => $metadata['source_url'] ?? '', |
| 579 |
'role_restriction' => $metadata['role_restriction'] ?? 'public', |
| 580 |
'type' => $metadata['type'] ?? 'content', |
| 581 |
'bot_id' => $bot_id, |
| 582 |
'created_at' => (int) $created_at, |
| 583 |
'data_source' => 'pinecone', |
| 584 |
'relevance_score' => $match['score'] ?? 0, |
| 585 |
'chunk_index' => isset($metadata['chunk_index']) ? (int) $metadata['chunk_index'] : null, |
| 586 |
'total_chunks' => isset($metadata['total_chunks']) ? (int) $metadata['total_chunks'] : null, |
| 587 |
'is_chunked' => !empty($metadata['is_chunked']), |
| 588 |
); |
| 589 |
} |
| 590 |
return $records; |
| 591 |
} |
| 592 |
|
| 593 |
/** |
| 594 |
* The KB screen listing on a document index (twin of |
| 595 |
* MxChat_Pinecone_Manager::mxchat_fetch_pinecone_records, cursor-paged). |
| 596 |
* With a search query the BM25 index answers it — no embedding call. |
| 597 |
*/ |
| 598 |
public static function fetch_records(array $pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type) { |
| 599 |
$cfg = self::cfg_from_options($pinecone_options); |
| 600 |
$page = max(1, (int) $page); |
| 601 |
$per_page = max(1, (int) $per_page); |
| 602 |
$empty = array('data' => array(), 'total' => 0, 'total_in_database' => 0, 'showing_recent_only' => false, 'has_next' => false); |
| 603 |
if (empty($cfg['host']) || empty($cfg['api_key'])) { |
| 604 |
return $empty; |
| 605 |
} |
| 606 |
$total_in_database = self::count($cfg); |
| 607 |
$type_filter = $content_type !== '' ? array('type' => array('$eq' => (string) $content_type)) : null; |
| 608 |
|
| 609 |
if (trim((string) $search_query) !== '') { |
| 610 |
$limit = min(($page * $per_page) + 100, 500); |
| 611 |
$matches = self::search_text($search_query, $limit, $type_filter, $cfg); |
| 612 |
if (is_wp_error($matches)) { |
| 613 |
$matches = array(); |
| 614 |
} |
| 615 |
$records = self::records_from_matches($matches, $bot_id, ''); |
| 616 |
$offset = ($page - 1) * $per_page; |
| 617 |
return array( |
| 618 |
'data' => array_slice($records, $offset, $per_page), |
| 619 |
'total' => count($records), |
| 620 |
'total_in_database' => $total_in_database, |
| 621 |
'showing_recent_only' => false, |
| 622 |
'has_next' => ($offset + $per_page) < count($records), |
| 623 |
); |
| 624 |
} |
| 625 |
|
| 626 |
// Browse: one cursor step per page (plan dd6e10's rule), tokens kept per page. |
| 627 |
$limit = $content_type !== '' ? $per_page * 2 : $per_page; |
| 628 |
$pagination_key = 'mxchat_pinecone_page_' . md5($cfg['host'] . $cfg['namespace'] . $content_type . '_docs'); |
| 629 |
$token = ''; |
| 630 |
if ($page > 1) { |
| 631 |
$stored = get_transient($pagination_key); |
| 632 |
if ($stored && isset($stored[$page])) { |
| 633 |
$token = $stored[$page]; |
| 634 |
} |
| 635 |
} |
| 636 |
$listing = self::list_page('', $limit, $token, $cfg); |
| 637 |
if (is_wp_error($listing)) { |
| 638 |
error_log('MxChat Pinecone (document index): documents/list failed — ' . $listing->get_error_message()); |
| 639 |
$empty['total_in_database'] = $total_in_database; |
| 640 |
return $empty; |
| 641 |
} |
| 642 |
$has_next = $listing['next'] !== ''; |
| 643 |
if ($has_next) { |
| 644 |
$stored = get_transient($pagination_key) ?: array(); |
| 645 |
$stored[$page + 1] = $listing['next']; |
| 646 |
set_transient($pagination_key, $stored, 300); |
| 647 |
} |
| 648 |
$records = array(); |
| 649 |
if (!empty($listing['ids'])) { |
| 650 |
$docs = self::fetch($listing['ids'], $cfg); |
| 651 |
$matches = array(); |
| 652 |
foreach ($docs as $doc) { |
| 653 |
$matches[] = array('id' => $doc['id'], 'score' => 0, 'metadata' => $doc['metadata']); |
| 654 |
} |
| 655 |
$records = self::records_from_matches($matches, $bot_id, (string) $content_type); |
| 656 |
usort($records, function ($a, $b) { |
| 657 |
return $b->created_at <=> $a->created_at; |
| 658 |
}); |
| 659 |
$records = array_slice($records, 0, $per_page); |
| 660 |
} |
| 661 |
return array( |
| 662 |
'data' => $records, |
| 663 |
'total' => $total_in_database, |
| 664 |
'total_in_database' => $total_in_database, |
| 665 |
'showing_recent_only' => false, |
| 666 |
'count_unit' => 'vectors', |
| 667 |
'cursor_pagination' => true, |
| 668 |
'has_next' => $has_next, |
| 669 |
); |
| 670 |
} |
| 671 |
|
| 672 |
// ===================================================================== |
| 673 |
// Retrieval helpers (hybrid keyword leg) |
| 674 |
// ===================================================================== |
| 675 |
|
| 676 |
/** |
| 677 |
* Code-like tokens in a question: runs that mix letters and digits, or |
| 678 |
* hyphen / dot joined codes (HGH25CA, SR25W, R1621-314-20, ISO-9001-2015). |
| 679 |
* Plain numbers and plain words are not codes. Minimum 4 characters. |
| 680 |
*/ |
| 681 |
public static function extract_code_tokens($text) { |
| 682 |
$tokens = array(); |
| 683 |
if (!preg_match_all('/(?<![A-Za-z0-9])([A-Za-z0-9]+(?:[.\-][A-Za-z0-9]+)*)(?![A-Za-z0-9])/u', (string) $text, $m)) { |
| 684 |
return $tokens; |
| 685 |
} |
| 686 |
foreach ($m[1] as $candidate) { |
| 687 |
$candidate = rtrim($candidate, '.'); |
| 688 |
if (strlen($candidate) < 4 || !preg_match('/\d/', $candidate)) { |
| 689 |
continue; |
| 690 |
} |
| 691 |
$has_letter = (bool) preg_match('/[A-Za-z]/', $candidate); |
| 692 |
$separators = preg_match_all('/[.\-]/', $candidate); |
| 693 |
if (!$has_letter && $separators < 2) { |
| 694 |
continue; // 2024, 26.5, 10-20 |
| 695 |
} |
| 696 |
$tokens[strtoupper($candidate)] = $candidate; |
| 697 |
} |
| 698 |
return array_values($tokens); |
| 699 |
} |
| 700 |
|
| 701 |
/** |
| 702 |
* Text-match filter that requires at least one of the codes. Hyphenated or |
| 703 |
* dotted codes are phrase-matched (the tokenizer splits them, and "20" on |
| 704 |
* its own would match everything); single runs use $match_any. |
| 705 |
* |
| 706 |
* 3e83e4: a second leg matches the NORMALISED code set stored on each |
| 707 |
* document (codes $in …), which is what lets "R165321320" reach a record |
| 708 |
* that only ever says "R1653" — see query_codes() for the expansion. |
| 709 |
*/ |
| 710 |
public static function keyword_filter(array $tokens) { |
| 711 |
$clauses = array(); |
| 712 |
foreach ($tokens as $token) { |
| 713 |
$op = preg_match('/[.\-]/', $token) ? '$match_phrase' : '$match_any'; |
| 714 |
$clauses[] = array(self::TEXT_FIELD => array($op => $token)); |
| 715 |
} |
| 716 |
$query_codes = self::query_codes($tokens); |
| 717 |
if (!empty($query_codes)) { |
| 718 |
$clauses[] = array(self::CODES_FIELD => array('$in' => $query_codes)); |
| 719 |
} |
| 720 |
if (empty($clauses)) { |
| 721 |
return null; |
| 722 |
} |
| 723 |
return count($clauses) === 1 ? $clauses[0] : array('$or' => $clauses); |
| 724 |
} |
| 725 |
|
| 726 |
/** Upper-case, separators stripped: "R1653 213 20" → "R165321320", "hgh25ca" → "HGH25CA". */ |
| 727 |
public static function normalize_code($code) { |
| 728 |
return strtoupper(preg_replace('/[\s.\-_\/]+/u', '', (string) $code)); |
| 729 |
} |
| 730 |
|
| 731 |
/** |
| 732 |
* Code-like runs in a piece of text, INCLUDING digit groups that follow a |
| 733 |
* code separated by a space, hyphen or dot ("R1653 213 20" is one number |
| 734 |
* the way Rexroth prints it). Returns [ [head, full], … ] where head is |
| 735 |
* the first run and full is the whole joined run, both un-normalised. |
| 736 |
*/ |
| 737 |
private static function code_runs($text) { |
| 738 |
$runs = array(); |
| 739 |
$pattern = '/(?<![A-Za-z0-9])([A-Za-z0-9]+(?:[.\-][A-Za-z0-9]+)*)((?:[ .\-]\d{2,6}(?![A-Za-z0-9])){1,3})?(?![A-Za-z0-9])/u'; |
| 740 |
if (!preg_match_all($pattern, (string) $text, $m, PREG_SET_ORDER)) { |
| 741 |
return $runs; |
| 742 |
} |
| 743 |
foreach ($m as $match) { |
| 744 |
$head = rtrim($match[1], '.'); |
| 745 |
if (strlen($head) < self::CODE_MIN_LEN || !preg_match('/\d/', $head)) { |
| 746 |
continue; |
| 747 |
} |
| 748 |
$has_letter = (bool) preg_match('/[A-Za-z]/', $head); |
| 749 |
$separators = preg_match_all('/[.\-]/', $head); |
| 750 |
if (!$has_letter && $separators < 2) { |
| 751 |
continue; // 2024, 26.5, 10-20 |
| 752 |
} |
| 753 |
$tail = isset($match[2]) ? $match[2] : ''; |
| 754 |
$runs[] = array($head, $head . $tail); |
| 755 |
} |
| 756 |
return $runs; |
| 757 |
} |
| 758 |
|
| 759 |
/** |
| 760 |
* The normalised token set stored on a document at write time: for every |
| 761 |
* code-like run in the text, the separator-stripped form, the leading stem |
| 762 |
* (the run before the first separator) and the letters+digits stem |
| 763 |
* ("HGH25" of "HGH25CA"). Unique, minimum 4 characters, capped. |
| 764 |
*/ |
| 765 |
public static function index_codes($text) { |
| 766 |
$codes = array(); |
| 767 |
foreach (self::code_runs($text) as $run) { |
| 768 |
list($head, $full) = $run; |
| 769 |
$candidates = array(self::normalize_code($full), self::normalize_code($head)); |
| 770 |
$stem = preg_split('/[.\-\s]/', $head)[0]; |
| 771 |
$candidates[] = self::normalize_code($stem); |
| 772 |
if (preg_match('/^([A-Za-z]+\d+)/', $head, $sm)) { |
| 773 |
$candidates[] = self::normalize_code($sm[1]); |
| 774 |
} |
| 775 |
foreach ($candidates as $c) { |
| 776 |
if (strlen($c) >= self::CODE_MIN_LEN && strlen($c) <= self::CODE_MAX_LEN && preg_match('/\d/', $c)) { |
| 777 |
$codes[$c] = true; |
| 778 |
} |
| 779 |
} |
| 780 |
if (count($codes) >= self::CODES_MAX) { |
| 781 |
break; |
| 782 |
} |
| 783 |
} |
| 784 |
return array_slice(array_keys($codes), 0, self::CODES_MAX); |
| 785 |
} |
| 786 |
|
| 787 |
/** |
| 788 |
* What a question's tokens expand to for the codes $in filter: the |
| 789 |
* stripped form, every prefix of it from 4 characters up (a stored stem |
| 790 |
* that is a prefix of what the visitor typed then matches — "R1653" for |
| 791 |
* "R165321320") and the letters+digits stem. Plain-language questions |
| 792 |
* carry no tokens and return nothing, so they are unchanged. |
| 793 |
*/ |
| 794 |
public static function query_codes(array $tokens) { |
| 795 |
$codes = array(); |
| 796 |
foreach ($tokens as $token) { |
| 797 |
$full = self::normalize_code($token); |
| 798 |
if (strlen($full) < self::CODE_MIN_LEN || !preg_match('/\d/', $full)) { |
| 799 |
continue; |
| 800 |
} |
| 801 |
$full = substr($full, 0, self::CODE_MAX_LEN); |
| 802 |
$codes[$full] = true; |
| 803 |
for ($len = self::CODE_MIN_LEN; $len < strlen($full); $len++) { |
| 804 |
$codes[substr($full, 0, $len)] = true; |
| 805 |
} |
| 806 |
if (preg_match('/^([A-Z]+\d+)/', $full, $sm) && strlen($sm[1]) >= self::CODE_MIN_LEN) { |
| 807 |
$codes[$sm[1]] = true; |
| 808 |
} |
| 809 |
if (count($codes) >= 200) { |
| 810 |
break; |
| 811 |
} |
| 812 |
} |
| 813 |
return array_slice(array_keys($codes), 0, 200); |
| 814 |
} |
| 815 |
|
| 816 |
/** |
| 817 |
* Order keyword-leg hits so an exact code match outranks a prefix-only |
| 818 |
* match: a hit is exact when its stored code set (or its text) carries the |
| 819 |
* stripped question token itself. Dense order is kept inside each group. |
| 820 |
* |
| 821 |
* @return array{hits:array,exact:int,prefix:int} |
| 822 |
*/ |
| 823 |
public static function rank_keyword_hits(array $hits, array $tokens) { |
| 824 |
$exact_codes = array(); |
| 825 |
foreach ($tokens as $token) { |
| 826 |
$n = self::normalize_code($token); |
| 827 |
if (strlen($n) >= self::CODE_MIN_LEN) { |
| 828 |
$exact_codes[$n] = true; |
| 829 |
} |
| 830 |
} |
| 831 |
$exact = array(); |
| 832 |
$prefix = array(); |
| 833 |
foreach ($hits as $hit) { |
| 834 |
$meta = is_array($hit['metadata'] ?? null) ? $hit['metadata'] : array(); |
| 835 |
$stored = is_array($meta[self::CODES_FIELD] ?? null) ? $meta[self::CODES_FIELD] : array(); |
| 836 |
$is_exact = false; |
| 837 |
foreach ($stored as $code) { |
| 838 |
if (isset($exact_codes[strtoupper((string) $code)])) { |
| 839 |
$is_exact = true; |
| 840 |
break; |
| 841 |
} |
| 842 |
} |
| 843 |
if (!$is_exact && !empty($tokens) && isset($meta[self::TEXT_FIELD])) { |
| 844 |
$haystack = self::normalize_code($meta[self::TEXT_FIELD]); |
| 845 |
foreach ($exact_codes as $code => $_) { |
| 846 |
if ($haystack !== '' && strpos($haystack, $code) !== false) { |
| 847 |
$is_exact = true; |
| 848 |
break; |
| 849 |
} |
| 850 |
} |
| 851 |
} |
| 852 |
$hit['code_match'] = $is_exact ? 'exact' : 'prefix'; |
| 853 |
if ($is_exact) { |
| 854 |
$exact[] = $hit; |
| 855 |
} else { |
| 856 |
$prefix[] = $hit; |
| 857 |
} |
| 858 |
} |
| 859 |
return array('hits' => array_merge($exact, $prefix), 'exact' => count($exact), 'prefix' => count($prefix)); |
| 860 |
} |
| 861 |
|
| 862 |
/** AND two filters (either may be empty). */ |
| 863 |
public static function merge_filters($a, $b) { |
| 864 |
$a = (!empty($a) && is_array($a)) ? $a : null; |
| 865 |
$b = (!empty($b) && is_array($b)) ? $b : null; |
| 866 |
if ($a === null) { |
| 867 |
return $b; |
| 868 |
} |
| 869 |
if ($b === null) { |
| 870 |
return $a; |
| 871 |
} |
| 872 |
return array('$and' => array($a, $b)); |
| 873 |
} |
| 874 |
|
| 875 |
/** |
| 876 |
* Translate the classic query body (what the mxchat_pinecone_query_body |
| 877 |
* filter sees) into what a document search needs: topK → top_k, filter → filter. |
| 878 |
* |
| 879 |
* @return array{top_k:int,filter:array|null} |
| 880 |
*/ |
| 881 |
public static function translate_classic_body(array $body, $default_top_k) { |
| 882 |
$top_k = isset($body['topK']) ? absint($body['topK']) : 0; |
| 883 |
if ($top_k < 1 || $top_k > 1000) { |
| 884 |
$top_k = $default_top_k; |
| 885 |
} |
| 886 |
$filter = (!empty($body['filter']) && is_array($body['filter'])) ? $body['filter'] : null; |
| 887 |
return array('top_k' => $top_k, 'filter' => $filter); |
| 888 |
} |
| 889 |
|
| 890 |
// ===================================================================== |
| 891 |
// Control plane |
| 892 |
// ===================================================================== |
| 893 |
|
| 894 |
private static function control_request($method, $path, $body, $api_key) { |
| 895 |
return self::request($method, 'https://api.pinecone.io/' . ltrim($path, '/'), $body, $api_key, 30); |
| 896 |
} |
| 897 |
|
| 898 |
/** @return array|WP_Error the IndexModel (2026-07 shape, with schema) */ |
| 899 |
public static function describe_index($name, $api_key) { |
| 900 |
$name = sanitize_text_field($name); |
| 901 |
if ($name === '') { |
| 902 |
return new WP_Error('pinecone_index', 'Index name is empty.'); |
| 903 |
} |
| 904 |
$res = self::control_request('GET', 'indexes/' . rawurlencode($name), null, $api_key); |
| 905 |
if ($res['error'] !== '' || !is_array($res['data'])) { |
| 906 |
return new WP_Error('pinecone_api', $res['error'] !== '' ? $res['error'] : 'Empty describe response', array('code' => $res['code'])); |
| 907 |
} |
| 908 |
return $res['data']; |
| 909 |
} |
| 910 |
|
| 911 |
/** Find an index in the project by its host. @return array|WP_Error */ |
| 912 |
public static function find_index_by_host($host, $api_key) { |
| 913 |
$host = strtolower(trim(str_replace(array('https://', 'http://'), '', (string) $host), '/')); |
| 914 |
if ($host === '') { |
| 915 |
return new WP_Error('pinecone_index', 'Host is empty.'); |
| 916 |
} |
| 917 |
$res = self::control_request('GET', 'indexes', null, $api_key); |
| 918 |
if ($res['error'] !== '' || !is_array($res['data'])) { |
| 919 |
return new WP_Error('pinecone_api', $res['error'] !== '' ? $res['error'] : 'Could not list indexes'); |
| 920 |
} |
| 921 |
foreach (($res['data']['indexes'] ?? array()) as $index) { |
| 922 |
if (isset($index['host']) && strtolower($index['host']) === $host) { |
| 923 |
return $index; |
| 924 |
} |
| 925 |
} |
| 926 |
return new WP_Error('pinecone_index', sprintf('No index in this Pinecone project answers to the host %s. Check the host and that the API key belongs to the same project.', $host)); |
| 927 |
} |
| 928 |
|
| 929 |
/** |
| 930 |
* Read a described index's schema. Classic vector indexes carry the |
| 931 |
* reserved _values / _sparse_values fields; a document index names its own. |
| 932 |
*/ |
| 933 |
public static function inspect_schema(array $index) { |
| 934 |
$out = array('is_document' => false, 'text_field' => '', 'dense_field' => '', 'dimension' => 0, 'metric' => '', 'language' => ''); |
| 935 |
$fields = $index['schema']['fields'] ?? array(); |
| 936 |
if (!is_array($fields)) { |
| 937 |
return $out; |
| 938 |
} |
| 939 |
foreach ($fields as $name => $spec) { |
| 940 |
if (!is_array($spec)) { |
| 941 |
continue; |
| 942 |
} |
| 943 |
if (($spec['type'] ?? '') === 'dense_vector' && strpos($name, '_') !== 0) { |
| 944 |
$out['dense_field'] = (string) $name; |
| 945 |
$out['dimension'] = (int) ($spec['dimension'] ?? 0); |
| 946 |
$out['metric'] = (string) ($spec['metric'] ?? ''); |
| 947 |
} |
| 948 |
if (($spec['type'] ?? '') === 'string' && !empty($spec['full_text_search']) && $out['text_field'] === '') { |
| 949 |
$out['text_field'] = (string) $name; |
| 950 |
$out['language'] = (string) ($spec['full_text_search']['language'] ?? ''); |
| 951 |
} |
| 952 |
} |
| 953 |
$out['is_document'] = $out['dense_field'] !== '' && $out['text_field'] !== ''; |
| 954 |
return $out; |
| 955 |
} |
| 956 |
|
| 957 |
/** |
| 958 |
* Assert an index is a document index MxChat can use. Refuses with the |
| 959 |
* exact mismatch otherwise. |
| 960 |
* |
| 961 |
* @return array{ok:bool,message:string,host:string,name:string,dimension:int,expected:int} |
| 962 |
*/ |
| 963 |
public static function check_document_index($host, $api_key, $expected_dimension) { |
| 964 |
$index = self::find_index_by_host($host, $api_key); |
| 965 |
if (is_wp_error($index)) { |
| 966 |
return array('ok' => false, 'message' => $index->get_error_message(), 'host' => (string) $host, 'name' => '', 'dimension' => 0, 'expected' => (int) $expected_dimension); |
| 967 |
} |
| 968 |
$name = (string) ($index['name'] ?? ''); |
| 969 |
$schema = self::inspect_schema($index); |
| 970 |
$base = array('host' => (string) ($index['host'] ?? $host), 'name' => $name, 'dimension' => $schema['dimension'], 'expected' => (int) $expected_dimension); |
| 971 |
if (!$schema['is_document']) { |
| 972 |
$fields = is_array($index['schema']['fields'] ?? null) ? implode(', ', array_keys($index['schema']['fields'])) : 'none'; |
| 973 |
return $base + array('ok' => false, 'message' => sprintf('"%s" is a classic vector index (schema fields: %s), not a document index with full-text search. Create a document index with the button above, or switch the index type back to Vector index.', $name, $fields)); |
| 974 |
} |
| 975 |
if ($schema['text_field'] !== self::TEXT_FIELD || $schema['dense_field'] !== self::VECTOR_FIELD) { |
| 976 |
return $base + array('ok' => false, 'message' => sprintf('"%s" is a document index, but its fields are named "%s" (text) and "%s" (vector). MxChat writes to fields named "%s" and "%s" — create the index with the button above so the schema matches.', $name, $schema['text_field'], $schema['dense_field'], self::TEXT_FIELD, self::VECTOR_FIELD)); |
| 977 |
} |
| 978 |
if ((int) $schema['dimension'] !== (int) $expected_dimension) { |
| 979 |
return $base + array('ok' => false, 'message' => sprintf('"%s" stores %d-dimension vectors, but the active embedding model produces %d dimensions. Pick a matching embedding model or create a new document index for this one.', $name, $schema['dimension'], $expected_dimension)); |
| 980 |
} |
| 981 |
if (empty($index['status']['ready'])) { |
| 982 |
return $base + array('ok' => false, 'message' => sprintf('"%s" is a matching document index but Pinecone reports it as %s. Wait a moment and check again.', $name, (string) ($index['status']['state'] ?? 'not ready'))); |
| 983 |
} |
| 984 |
return $base + array('ok' => true, 'message' => sprintf('"%s" is a document index with full-text search (%s, %d dimensions, %s). Ready to use.', $name, $schema['language'] !== '' ? 'language ' . $schema['language'] : 'full-text search', $schema['dimension'], $schema['metric'] ?: 'cosine')); |
| 985 |
} |
| 986 |
|
| 987 |
/** |
| 988 |
* The plain-language "that name is taken" message (3e83e4) — never a bare |
| 989 |
* HTTP 409. Carries a free suggestion so the UI can fill it in. |
| 990 |
*/ |
| 991 |
public static function name_taken_message($name, $suggested = '') { |
| 992 |
$message = sprintf( |
| 993 |
/* translators: %s: Pinecone index name */ |
| 994 |
__('An index named "%s" already exists in this Pinecone project. Pick a different name, or paste that index\'s host above and press Check index instead of creating one.', 'mxchat'), |
| 995 |
$name |
| 996 |
); |
| 997 |
if ($suggested !== '') { |
| 998 |
/* translators: %s: suggested Pinecone index name */ |
| 999 |
$message .= ' ' . sprintf(__('Suggested free name: %s.', 'mxchat'), $suggested); |
| 1000 |
} |
| 1001 |
return $message; |
| 1002 |
} |
| 1003 |
|
| 1004 |
/** |
| 1005 |
* A free index name derived from the current one: <base>-docs, then |
| 1006 |
* <base>-docs-2, -3 … (an existing -docs suffix is not doubled). Lists the |
| 1007 |
* project's indexes once. @return array{name:string,taken:bool,existing:array} |
| 1008 |
*/ |
| 1009 |
public static function suggest_index_name($base, $api_key) { |
| 1010 |
$base = strtolower(sanitize_text_field($base)); |
| 1011 |
$base = preg_replace('/[^a-z0-9-]+/', '-', $base); |
| 1012 |
$base = trim(preg_replace('/-{2,}/', '-', $base), '-'); |
| 1013 |
$base = preg_replace('/-docs(-\d+)?$/', '', $base); |
| 1014 |
if ($base === '') { |
| 1015 |
$base = 'mxchat'; |
| 1016 |
} |
| 1017 |
$existing = array(); |
| 1018 |
$res = self::control_request('GET', 'indexes', null, $api_key); |
| 1019 |
if ($res['error'] === '' && is_array($res['data'])) { |
| 1020 |
foreach (($res['data']['indexes'] ?? array()) as $index) { |
| 1021 |
if (isset($index['name'])) { |
| 1022 |
$existing[strtolower((string) $index['name'])] = true; |
| 1023 |
} |
| 1024 |
} |
| 1025 |
} |
| 1026 |
$stem = substr($base, 0, 45 - strlen('-docs-99')); |
| 1027 |
$stem = rtrim($stem, '-'); |
| 1028 |
$candidate = $stem . '-docs'; |
| 1029 |
for ($n = 2; isset($existing[$candidate]) && $n < 100; $n++) { |
| 1030 |
$candidate = $stem . '-docs-' . $n; |
| 1031 |
} |
| 1032 |
return array('name' => $candidate, 'taken' => isset($existing[$base]), 'existing' => array_keys($existing)); |
| 1033 |
} |
| 1034 |
|
| 1035 |
/** |
| 1036 |
* Create a document index for the active embedding dimension. |
| 1037 |
* @return array|WP_Error the IndexModel |
| 1038 |
*/ |
| 1039 |
public static function create_document_index($name, $dimension, $cloud, $region, $language, $api_key) { |
| 1040 |
$name = strtolower(sanitize_text_field($name)); |
| 1041 |
if (!preg_match('/^[a-z0-9]([a-z0-9-]{0,43}[a-z0-9])?$/', $name)) { |
| 1042 |
return new WP_Error('pinecone_index', 'Index names are 1-45 lowercase letters, digits and hyphens.'); |
| 1043 |
} |
| 1044 |
// 3e83e4: say so in plain words BEFORE Pinecone can answer 409 — the |
| 1045 |
// name field is prefilled with the vector index being migrated away |
| 1046 |
// from, so the first click used to collide for exactly that customer. |
| 1047 |
$taken = self::describe_index($name, $api_key); |
| 1048 |
if (!is_wp_error($taken)) { |
| 1049 |
$suggest = self::suggest_index_name($name, $api_key); |
| 1050 |
return new WP_Error('pinecone_exists', self::name_taken_message($name, $suggest['name']), array('code' => 409, 'suggested' => $suggest['name'])); |
| 1051 |
} |
| 1052 |
$cloud = array_key_exists($cloud, self::clouds()) ? $cloud : 'aws'; |
| 1053 |
$region = sanitize_text_field($region) ?: 'us-east-1'; |
| 1054 |
$language = array_key_exists($language, self::languages()) ? $language : 'en'; |
| 1055 |
$dimension = (int) $dimension; |
| 1056 |
if ($dimension < 1 || $dimension > 20000) { |
| 1057 |
return new WP_Error('pinecone_index', 'Embedding dimension is out of range.'); |
| 1058 |
} |
| 1059 |
$body = array( |
| 1060 |
'name' => $name, |
| 1061 |
'deployment' => array('deployment_type' => 'managed', 'cloud' => $cloud, 'region' => $region), |
| 1062 |
'schema' => array('fields' => array( |
| 1063 |
self::TEXT_FIELD => array('type' => 'string', 'full_text_search' => array('language' => $language)), |
| 1064 |
self::VECTOR_FIELD => array('type' => 'dense_vector', 'dimension' => $dimension, 'metric' => 'cosine'), |
| 1065 |
)), |
| 1066 |
); |
| 1067 |
$res = self::control_request('POST', 'indexes', $body, $api_key); |
| 1068 |
if ($res['code'] === 409) { |
| 1069 |
$suggest = self::suggest_index_name($name, $api_key); |
| 1070 |
return new WP_Error('pinecone_exists', self::name_taken_message($name, $suggest['name']), array('code' => 409, 'suggested' => $suggest['name'])); |
| 1071 |
} |
| 1072 |
if ($res['error'] !== '' || !is_array($res['data'])) { |
| 1073 |
return new WP_Error('pinecone_api', $res['error'] !== '' ? $res['error'] : 'Empty create response', array('code' => $res['code'])); |
| 1074 |
} |
| 1075 |
return $res['data']; |
| 1076 |
} |
| 1077 |
|
| 1078 |
/** @return true|WP_Error */ |
| 1079 |
public static function delete_index($name, $api_key) { |
| 1080 |
$name = sanitize_text_field($name); |
| 1081 |
if ($name === '') { |
| 1082 |
return new WP_Error('pinecone_index', 'Index name is empty.'); |
| 1083 |
} |
| 1084 |
$res = self::control_request('DELETE', 'indexes/' . rawurlencode($name), null, $api_key); |
| 1085 |
if ($res['error'] !== '' && $res['code'] !== 404) { |
| 1086 |
return new WP_Error('pinecone_api', $res['error']); |
| 1087 |
} |
| 1088 |
return true; |
| 1089 |
} |
| 1090 |
|
| 1091 |
/** Dimension of the active embedding model (same table the KB screen uses). */ |
| 1092 |
public static function expected_dimension() { |
| 1093 |
$options = get_option('mxchat_options', array()); |
| 1094 |
$selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002'; |
| 1095 |
$model_dimensions = array( |
| 1096 |
'text-embedding-ada-002' => 1536, |
| 1097 |
'text-embedding-3-small' => 1536, |
| 1098 |
'text-embedding-3-large' => 3072, |
| 1099 |
'voyage-2' => 1024, |
| 1100 |
'voyage-large-2' => 1536, |
| 1101 |
'voyage-3-large' => 2048, |
| 1102 |
'gemini-embedding-001' => 1536, |
| 1103 |
); |
| 1104 |
if (strpos($selected_model, 'voyage-3-large') === 0) { |
| 1105 |
return intval($options['voyage_output_dimension'] ?? 2048); |
| 1106 |
} |
| 1107 |
if (strpos($selected_model, 'gemini-embedding') === 0) { |
| 1108 |
return intval($options['gemini_output_dimension'] ?? 1536); |
| 1109 |
} |
| 1110 |
$dimension = apply_filters('mxchat_pinecone_expected_dimension', $model_dimensions[$selected_model] ?? 1536, $selected_model); |
| 1111 |
return (int) $dimension; |
| 1112 |
} |
| 1113 |
|
| 1114 |
// ===================================================================== |
| 1115 |
// Options (raw writes — the registered sanitizer rebuilds the array and |
| 1116 |
// would flip mxchat_use_pinecone on any isset key; the autosave handler |
| 1117 |
// writes this option the same raw way for the same reason) |
| 1118 |
// ===================================================================== |
| 1119 |
|
| 1120 |
public static function persist_options(array $changes) { |
| 1121 |
global $wpdb; |
| 1122 |
$raw = $wpdb->get_var($wpdb->prepare("SELECT option_value FROM {$wpdb->options} WHERE option_name = %s", self::OPTION)); |
| 1123 |
$current = $raw === null ? array() : maybe_unserialize($raw); |
| 1124 |
if (!is_array($current)) { |
| 1125 |
$current = array(); |
| 1126 |
} |
| 1127 |
foreach ($changes as $key => $value) { |
| 1128 |
$current[$key] = $value; |
| 1129 |
} |
| 1130 |
$serialized = maybe_serialize($current); |
| 1131 |
if ($raw === null) { |
| 1132 |
$wpdb->insert($wpdb->options, array('option_name' => self::OPTION, 'option_value' => $serialized, 'autoload' => 'off'), array('%s', '%s', '%s')); |
| 1133 |
} else { |
| 1134 |
$wpdb->update($wpdb->options, array('option_value' => $serialized), array('option_name' => self::OPTION), array('%s'), array('%s')); |
| 1135 |
} |
| 1136 |
wp_cache_delete(self::OPTION, 'options'); |
| 1137 |
wp_cache_delete('alloptions', 'options'); |
| 1138 |
return $current; |
| 1139 |
} |
| 1140 |
|
| 1141 |
// ===================================================================== |
| 1142 |
// Copy-migration (classic vector index → document index, no embedding calls) |
| 1143 |
// ===================================================================== |
| 1144 |
|
| 1145 |
public static function migration_get() { |
| 1146 |
$state = get_transient(self::MIGRATION_KEY); |
| 1147 |
return is_array($state) ? $state : array(); |
| 1148 |
} |
| 1149 |
|
| 1150 |
public static function migration_set(array $state) { |
| 1151 |
$state['updated'] = time(); |
| 1152 |
set_transient(self::MIGRATION_KEY, $state, DAY_IN_SECONDS); |
| 1153 |
return $state; |
| 1154 |
} |
| 1155 |
|
| 1156 |
public static function migration_clear() { |
| 1157 |
delete_transient(self::MIGRATION_KEY); |
| 1158 |
} |
| 1159 |
|
| 1160 |
/** |
| 1161 |
* 3e83e4: is there an unfinished copy INTO this host? The settings save |
| 1162 |
* and the autosave refuse to switch the live chatbot onto a document index |
| 1163 |
* while its copy is still running (unless the owner ticks Switch anyway), |
| 1164 |
* so a production bot never answers from a half-filled index. |
| 1165 |
* |
| 1166 |
* @return array|null the running state, or null when nothing blocks |
| 1167 |
*/ |
| 1168 |
public static function migration_blocks_switch($host) { |
| 1169 |
$host = strtolower(trim(str_replace(array('https://', 'http://'), '', (string) $host), '/')); |
| 1170 |
$state = self::migration_get(); |
| 1171 |
if ($host === '' || empty($state) || ($state['status'] ?? '') !== 'running') { |
| 1172 |
return null; |
| 1173 |
} |
| 1174 |
if (strtolower((string) ($state['target_host'] ?? '')) !== $host) { |
| 1175 |
return null; |
| 1176 |
} |
| 1177 |
return $state; |
| 1178 |
} |
| 1179 |
|
| 1180 |
/** Plain-language line for the switch gate: "The copy into X is still running (481 of 4,503 records)…". */ |
| 1181 |
public static function switch_blocked_message(array $state) { |
| 1182 |
return sprintf( |
| 1183 |
/* translators: 1: document index name, 2: records copied, 3: records in total */ |
| 1184 |
__('The copy into "%1$s" is still running (%2$s of %3$s records). Let it finish before switching the chatbot to the document index, or tick Switch anyway to move it now and copy the rest afterwards.', 'mxchat'), |
| 1185 |
(string) ($state['target_name'] ?: $state['target_host']), |
| 1186 |
number_format_i18n((int) ($state['copied'] ?? 0)), |
| 1187 |
number_format_i18n((int) ($state['total'] ?? 0)) |
| 1188 |
); |
| 1189 |
} |
| 1190 |
|
| 1191 |
/** |
| 1192 |
* Run one Pinecone call up to MIGRATE_RETRIES times, 1 s / 2 s / 4 s apart, |
| 1193 |
* so a single dropped connection or a 5xx does not end the copy. |
| 1194 |
* |
| 1195 |
* @param callable $call returns array|true|WP_Error |
| 1196 |
* @return array{result:mixed,attempts:int} |
| 1197 |
*/ |
| 1198 |
private static function with_retry(callable $call) { |
| 1199 |
$result = null; |
| 1200 |
$attempts = 0; |
| 1201 |
for ($attempt = 1; $attempt <= self::MIGRATE_RETRIES; $attempt++) { |
| 1202 |
$attempts = $attempt; |
| 1203 |
$result = call_user_func($call); |
| 1204 |
if (!is_wp_error($result)) { |
| 1205 |
return array('result' => $result, 'attempts' => $attempts); |
| 1206 |
} |
| 1207 |
if ($attempt < self::MIGRATE_RETRIES) { |
| 1208 |
sleep((int) pow(2, $attempt - 1)); |
| 1209 |
} |
| 1210 |
} |
| 1211 |
return array('result' => $result, 'attempts' => $attempts); |
| 1212 |
} |
| 1213 |
|
| 1214 |
/** Seconds one AJAX step may spend copying pages (bounded by PHP's own limit). */ |
| 1215 |
private static function step_budget() { |
| 1216 |
$budget = (float) apply_filters('mxchat_pinecone_migrate_step_seconds', self::MIGRATE_STEP_SECONDS); |
| 1217 |
$limit = (int) ini_get('max_execution_time'); |
| 1218 |
if ($limit > 0 && $limit - 8 < $budget) { |
| 1219 |
$budget = max(3, $limit - 8); |
| 1220 |
} |
| 1221 |
return $budget; |
| 1222 |
} |
| 1223 |
|
| 1224 |
/** Classic GET /vectors/list page. @return array{ids:array,next:string}|WP_Error */ |
| 1225 |
public static function classic_list_page($host, $api_key, $namespace, $limit, $token) { |
| 1226 |
$params = array('limit' => max(1, min(100, (int) $limit))); |
| 1227 |
if ((string) $namespace !== '') { |
| 1228 |
$params['namespace'] = (string) $namespace; |
| 1229 |
} |
| 1230 |
if ((string) $token !== '') { |
| 1231 |
$params['paginationToken'] = (string) $token; |
| 1232 |
} |
| 1233 |
$response = wp_remote_get('https://' . $host . '/vectors/list?' . http_build_query($params), array( |
| 1234 |
'headers' => array('Api-Key' => $api_key, 'accept' => 'application/json'), |
| 1235 |
'timeout' => 30, |
| 1236 |
)); |
| 1237 |
if (is_wp_error($response)) { |
| 1238 |
return new WP_Error('pinecone_api', $response->get_error_message()); |
| 1239 |
} |
| 1240 |
$code = (int) wp_remote_retrieve_response_code($response); |
| 1241 |
$data = json_decode((string) wp_remote_retrieve_body($response), true); |
| 1242 |
if ($code !== 200 || !is_array($data)) { |
| 1243 |
return new WP_Error('pinecone_api', 'Source index list failed (HTTP ' . $code . '): ' . substr((string) wp_remote_retrieve_body($response), 0, 200)); |
| 1244 |
} |
| 1245 |
$ids = array(); |
| 1246 |
foreach (($data['vectors'] ?? array()) as $vector) { |
| 1247 |
if (isset($vector['id'])) { |
| 1248 |
$ids[] = (string) $vector['id']; |
| 1249 |
} |
| 1250 |
} |
| 1251 |
return array('ids' => $ids, 'next' => (string) ($data['pagination']['next'] ?? '')); |
| 1252 |
} |
| 1253 |
|
| 1254 |
/** Classic GET /vectors/fetch (values + metadata). @return array|WP_Error id => vector */ |
| 1255 |
public static function classic_fetch($host, $api_key, $namespace, array $ids) { |
| 1256 |
$vectors = array(); |
| 1257 |
foreach (array_chunk(array_values($ids), 100) as $chunk) { |
| 1258 |
$query = array(); |
| 1259 |
foreach ($chunk as $id) { |
| 1260 |
$query[] = 'ids=' . rawurlencode($id); |
| 1261 |
} |
| 1262 |
if ((string) $namespace !== '') { |
| 1263 |
$query[] = 'namespace=' . rawurlencode($namespace); |
| 1264 |
} |
| 1265 |
$response = wp_remote_get('https://' . $host . '/vectors/fetch?' . implode('&', $query), array( |
| 1266 |
'headers' => array('Api-Key' => $api_key, 'accept' => 'application/json'), |
| 1267 |
'timeout' => 60, |
| 1268 |
)); |
| 1269 |
if (is_wp_error($response)) { |
| 1270 |
return new WP_Error('pinecone_api', $response->get_error_message()); |
| 1271 |
} |
| 1272 |
$code = (int) wp_remote_retrieve_response_code($response); |
| 1273 |
$data = json_decode((string) wp_remote_retrieve_body($response), true); |
| 1274 |
if ($code !== 200 || !is_array($data)) { |
| 1275 |
return new WP_Error('pinecone_api', 'Source index fetch failed (HTTP ' . $code . ')'); |
| 1276 |
} |
| 1277 |
if (!empty($data['vectors']) && is_array($data['vectors'])) { |
| 1278 |
$vectors += $data['vectors']; |
| 1279 |
} |
| 1280 |
} |
| 1281 |
return $vectors; |
| 1282 |
} |
| 1283 |
|
| 1284 |
/** Classic POST /describe_index_stats (no version header). @return array|WP_Error */ |
| 1285 |
public static function classic_stats($host, $api_key) { |
| 1286 |
$response = wp_remote_post('https://' . $host . '/describe_index_stats', array( |
| 1287 |
'headers' => array('Api-Key' => $api_key, 'Content-Type' => 'application/json'), |
| 1288 |
'body' => '{}', |
| 1289 |
'timeout' => 15, |
| 1290 |
)); |
| 1291 |
if (is_wp_error($response)) { |
| 1292 |
return new WP_Error('pinecone_api', $response->get_error_message()); |
| 1293 |
} |
| 1294 |
$code = (int) wp_remote_retrieve_response_code($response); |
| 1295 |
$data = json_decode((string) wp_remote_retrieve_body($response), true); |
| 1296 |
if ($code !== 200 || !is_array($data)) { |
| 1297 |
return new WP_Error('pinecone_api', 'Source index stats failed (HTTP ' . $code . '): ' . substr((string) wp_remote_retrieve_body($response), 0, 200)); |
| 1298 |
} |
| 1299 |
return $data; |
| 1300 |
} |
| 1301 |
|
| 1302 |
/** Display key of a namespace in the per-namespace map — Pinecone's console names the default one __default__. */ |
| 1303 |
public static function ns_key($namespace) { |
| 1304 |
return (string) $namespace === '' ? '__default__' : (string) $namespace; |
| 1305 |
} |
| 1306 |
|
| 1307 |
/** Raw namespace for a display key (the default namespace is '' on the wire). */ |
| 1308 |
public static function ns_raw($key) { |
| 1309 |
return (string) $key === '__default__' ? '' : (string) $key; |
| 1310 |
} |
| 1311 |
|
| 1312 |
/** Classic describe_index_stats → [raw namespace => record count], the default namespace as ''. */ |
| 1313 |
private static function namespace_counts(array $stats) { |
| 1314 |
$counts = array(); |
| 1315 |
foreach ((array) ($stats['namespaces'] ?? array()) as $ns_name => $ns_info) { |
| 1316 |
$counts[self::ns_raw($ns_name)] = (int) ($ns_info['vectorCount'] ?? $ns_info['recordCount'] ?? 0); |
| 1317 |
} |
| 1318 |
return $counts; |
| 1319 |
} |
| 1320 |
|
| 1321 |
/** |
| 1322 |
* e50d2e: the order namespaces are copied in when every namespace is |
| 1323 |
* asked for — the default namespace first, then the rest by name. Only |
| 1324 |
* namespaces that hold records are queued. |
| 1325 |
*/ |
| 1326 |
private static function namespace_queue(array $counts) { |
| 1327 |
$queue = array(); |
| 1328 |
if (!empty($counts[''])) { |
| 1329 |
$queue[] = ''; |
| 1330 |
} |
| 1331 |
$named = array(); |
| 1332 |
foreach ($counts as $ns => $count) { |
| 1333 |
if ($ns !== '' && $count > 0) { |
| 1334 |
$named[] = (string) $ns; |
| 1335 |
} |
| 1336 |
} |
| 1337 |
sort($named, SORT_STRING); |
| 1338 |
return array_merge($queue, $named); |
| 1339 |
} |
| 1340 |
|
| 1341 |
private static function new_namespace_entry($namespace, $total, $status = 'queued') { |
| 1342 |
return array('namespace' => (string) $namespace, 'total' => (int) $total, 'copied' => 0, 'status' => $status, 'target_count' => null); |
| 1343 |
} |
| 1344 |
|
| 1345 |
/** |
| 1346 |
* e50d2e: the per-namespace map of a migration state — display key → |
| 1347 |
* namespace / total / copied / status / target_count. A state written |
| 1348 |
* before this build copied one namespace and carries no map, so one is |
| 1349 |
* synthesised from its single namespace; the Delete guard and the step |
| 1350 |
* loop read every state through here. |
| 1351 |
*/ |
| 1352 |
public static function per_namespace_map(array $state) { |
| 1353 |
if (!empty($state['per_namespace']) && is_array($state['per_namespace'])) { |
| 1354 |
return $state['per_namespace']; |
| 1355 |
} |
| 1356 |
$namespace = (string) ($state['namespace'] ?? ''); |
| 1357 |
$status = ($state['status'] ?? '') === 'running' ? 'running' : (in_array($state['status'] ?? '', array('done', 'source_deleted'), true) ? 'done' : 'queued'); |
| 1358 |
$entry = self::new_namespace_entry($namespace, (int) ($state['total'] ?? 0), $status); |
| 1359 |
$entry['copied'] = (int) ($state['copied'] ?? 0); |
| 1360 |
$entry['target_count'] = isset($state['target_count']) ? $state['target_count'] : null; |
| 1361 |
return array(self::ns_key($namespace) => $entry); |
| 1362 |
} |
| 1363 |
|
| 1364 |
/** |
| 1365 |
* e50d2e: widen a single-namespace copy — running or finished — to every |
| 1366 |
* namespace of the source that holds records. Namespaces already copied |
| 1367 |
* keep their entry, the rest queue behind the current one, the total |
| 1368 |
* becomes the sum, and "other namespaces" empties because none are left |
| 1369 |
* out any more. A finished copy with nothing left to queue stays done. |
| 1370 |
* |
| 1371 |
* @return array|WP_Error state |
| 1372 |
*/ |
| 1373 |
private static function extend_to_all_namespaces(array $state, $api_key) { |
| 1374 |
$stats = self::classic_stats((string) ($state['source_host'] ?? ''), $api_key); |
| 1375 |
if (is_wp_error($stats)) { |
| 1376 |
return $stats; |
| 1377 |
} |
| 1378 |
$ns_counts = self::namespace_counts($stats); |
| 1379 |
$map = self::per_namespace_map($state); |
| 1380 |
$queue = array_values((array) ($state['queue'] ?? array())); |
| 1381 |
$current = isset($state['current']) ? (string) $state['current'] : (string) ($state['namespace'] ?? ''); |
| 1382 |
foreach (self::namespace_queue($ns_counts) as $ns) { |
| 1383 |
if (isset($map[self::ns_key($ns)]) || $ns === $current || in_array($ns, $queue, true)) { |
| 1384 |
continue; |
| 1385 |
} |
| 1386 |
$map[self::ns_key($ns)] = self::new_namespace_entry($ns, $ns_counts[$ns]); |
| 1387 |
$queue[] = $ns; |
| 1388 |
} |
| 1389 |
$total = 0; |
| 1390 |
foreach ($map as $entry) { |
| 1391 |
$total += (int) ($entry['total'] ?? 0); |
| 1392 |
} |
| 1393 |
$state['per_namespace'] = $map; |
| 1394 |
$state['queue'] = $queue; |
| 1395 |
$state['current'] = $current; |
| 1396 |
$state['namespace'] = $current; |
| 1397 |
$state['total'] = $total; |
| 1398 |
$state['total_all'] = (int) ($stats['totalVectorCount'] ?? $total); |
| 1399 |
$state['other_namespaces'] = array(); |
| 1400 |
$state['all_namespaces'] = true; |
| 1401 |
if (($state['status'] ?? '') === 'done' && !empty($queue)) { |
| 1402 |
$next = array_shift($queue); |
| 1403 |
$state['queue'] = $queue; |
| 1404 |
$state['current'] = $next; |
| 1405 |
$state['namespace'] = $next; |
| 1406 |
$state['token'] = ''; |
| 1407 |
$state['status'] = 'running'; |
| 1408 |
$state['error'] = ''; |
| 1409 |
$state['error_at'] = 0; |
| 1410 |
$state['per_namespace'][self::ns_key($next)]['status'] = 'running'; |
| 1411 |
unset($state['finished']); |
| 1412 |
} |
| 1413 |
return $state; |
| 1414 |
} |
| 1415 |
|
| 1416 |
/** |
| 1417 |
* Validate and start (or resume) a migration. @return array|WP_Error state |
| 1418 |
* |
| 1419 |
* e50d2e: $all_namespaces copies every namespace of the source that holds |
| 1420 |
* records — the default namespace first, then the rest by name — each into |
| 1421 |
* the same-named namespace of the document index, one after another, in |
| 1422 |
* one run. Without it the copy is exactly 3e83e4's: the one configured |
| 1423 |
* namespace. Asking for every namespace on a copy that is already running |
| 1424 |
* or finished keeps what was copied and queues the rest. |
| 1425 |
*/ |
| 1426 |
public static function migrate_start($source_host, array $target_cfg, $restart = false, $all_namespaces = false) { |
| 1427 |
$source_host = strtolower(trim(str_replace(array('https://', 'http://'), '', (string) $source_host), '/')); |
| 1428 |
if ($source_host === '') { |
| 1429 |
return new WP_Error('migrate', 'Enter the host of the vector index to copy from.'); |
| 1430 |
} |
| 1431 |
if (empty($target_cfg['host']) || empty($target_cfg['api_key'])) { |
| 1432 |
return new WP_Error('migrate', 'The document index host and API key must be saved first.'); |
| 1433 |
} |
| 1434 |
if ($source_host === strtolower($target_cfg['host'])) { |
| 1435 |
return new WP_Error('migrate', 'The source and the document index are the same host.'); |
| 1436 |
} |
| 1437 |
$existing = self::migration_get(); |
| 1438 |
$same_copy = !empty($existing) && ($existing['source_host'] ?? '') === $source_host && strtolower((string) ($existing['target_host'] ?? '')) === strtolower($target_cfg['host']); |
| 1439 |
if (!$restart && $same_copy && ($existing['status'] ?? '') === 'running') { |
| 1440 |
// Resume from the recorded cursor — whether the last step ended |
| 1441 |
// cleanly or with an error (3e83e4: a second click continues, |
| 1442 |
// it never starts over unless asked). |
| 1443 |
$existing['error'] = ''; |
| 1444 |
if ($all_namespaces && empty($existing['all_namespaces'])) { |
| 1445 |
$existing = self::extend_to_all_namespaces($existing, $target_cfg['api_key']); |
| 1446 |
if (is_wp_error($existing)) { |
| 1447 |
return $existing; |
| 1448 |
} |
| 1449 |
} |
| 1450 |
return self::migration_set($existing); |
| 1451 |
} |
| 1452 |
if (!$restart && $same_copy && $all_namespaces && ($existing['status'] ?? '') === 'done') { |
| 1453 |
$extended = self::extend_to_all_namespaces($existing, $target_cfg['api_key']); |
| 1454 |
if (is_wp_error($extended)) { |
| 1455 |
return $extended; |
| 1456 |
} |
| 1457 |
return self::migration_set($extended); |
| 1458 |
} |
| 1459 |
$stats = self::classic_stats($source_host, $target_cfg['api_key']); |
| 1460 |
if (is_wp_error($stats)) { |
| 1461 |
return $stats; |
| 1462 |
} |
| 1463 |
$namespace = (string) ($target_cfg['namespace'] ?? ''); |
| 1464 |
// 3e83e4: the total is the SOURCE NAMESPACE's count, never the whole |
| 1465 |
// index — the copy lists one namespace, so an index with records in |
| 1466 |
// other namespaces used to report "done" with copied < total and no |
| 1467 |
// explanation. The other namespaces are recorded for the UI instead. |
| 1468 |
$ns_counts = self::namespace_counts($stats); |
| 1469 |
$queue = array(); |
| 1470 |
$per_namespace = array(); |
| 1471 |
$other_namespaces = array(); |
| 1472 |
if ($all_namespaces) { |
| 1473 |
$queue = self::namespace_queue($ns_counts); |
| 1474 |
if (empty($queue)) { |
| 1475 |
$queue = array($namespace); // stats listed no namespaces — copy the configured one, as before |
| 1476 |
} |
| 1477 |
$total = 0; |
| 1478 |
foreach ($queue as $ns) { |
| 1479 |
if (isset($ns_counts[$ns])) { |
| 1480 |
$count = (int) $ns_counts[$ns]; |
| 1481 |
} elseif ($ns === '' && empty($ns_counts)) { |
| 1482 |
$count = (int) ($stats['totalVectorCount'] ?? 0); |
| 1483 |
} else { |
| 1484 |
$count = 0; |
| 1485 |
} |
| 1486 |
$per_namespace[self::ns_key($ns)] = self::new_namespace_entry($ns, $count); |
| 1487 |
$total += $count; |
| 1488 |
} |
| 1489 |
$current = (string) array_shift($queue); |
| 1490 |
$per_namespace[self::ns_key($current)]['status'] = 'running'; |
| 1491 |
} else { |
| 1492 |
$current = $namespace; |
| 1493 |
if (isset($ns_counts[$namespace])) { |
| 1494 |
$total = (int) $ns_counts[$namespace]; |
| 1495 |
} elseif ($namespace === '' && empty($ns_counts)) { |
| 1496 |
$total = (int) ($stats['totalVectorCount'] ?? 0); |
| 1497 |
} else { |
| 1498 |
$total = 0; |
| 1499 |
} |
| 1500 |
$per_namespace[self::ns_key($namespace)] = self::new_namespace_entry($namespace, $total, 'running'); |
| 1501 |
foreach ($ns_counts as $ns_name => $ns_count) { |
| 1502 |
if ($ns_name !== $namespace && $ns_count > 0) { |
| 1503 |
$other_namespaces[self::ns_key($ns_name)] = $ns_count; |
| 1504 |
} |
| 1505 |
} |
| 1506 |
} |
| 1507 |
$source_dimension = (int) ($stats['dimension'] ?? 0); |
| 1508 |
$target_index = self::find_index_by_host($target_cfg['host'], $target_cfg['api_key']); |
| 1509 |
if (is_wp_error($target_index)) { |
| 1510 |
return $target_index; |
| 1511 |
} |
| 1512 |
$schema = self::inspect_schema($target_index); |
| 1513 |
if (!$schema['is_document']) { |
| 1514 |
return new WP_Error('migrate', 'The configured host is not a document index. Run Check index first.'); |
| 1515 |
} |
| 1516 |
if ($source_dimension > 0 && $schema['dimension'] > 0 && $source_dimension !== (int) $schema['dimension']) { |
| 1517 |
return new WP_Error('migrate', sprintf('Dimensions differ (source %d, document index %d) — the vectors cannot be copied. Re-import your content into the document index instead.', $source_dimension, $schema['dimension'])); |
| 1518 |
} |
| 1519 |
$state = array( |
| 1520 |
'status' => 'running', |
| 1521 |
'source_host' => $source_host, |
| 1522 |
'target_host' => $target_cfg['host'], |
| 1523 |
'target_name' => (string) ($target_index['name'] ?? ''), |
| 1524 |
'namespace' => $current, |
| 1525 |
'total' => $total, |
| 1526 |
'total_all' => (int) ($stats['totalVectorCount'] ?? $total), |
| 1527 |
'other_namespaces' => $other_namespaces, |
| 1528 |
// e50d2e: the namespace being copied, the ones still queued and |
| 1529 |
// a per-namespace ledger — the Delete guard checks every entry. |
| 1530 |
'all_namespaces' => (bool) $all_namespaces, |
| 1531 |
'current' => $current, |
| 1532 |
'queue' => $queue, |
| 1533 |
'per_namespace' => $per_namespace, |
| 1534 |
'copied' => 0, |
| 1535 |
'skipped' => 0, |
| 1536 |
'token' => '', |
| 1537 |
'batches' => 0, |
| 1538 |
'retries' => 0, |
| 1539 |
'failed_steps' => 0, |
| 1540 |
'started' => time(), |
| 1541 |
'error' => '', |
| 1542 |
'error_at' => 0, |
| 1543 |
'target_count' => null, |
| 1544 |
'source_name' => '', |
| 1545 |
); |
| 1546 |
$source_index = self::find_index_by_host($source_host, $target_cfg['api_key']); |
| 1547 |
if (!is_wp_error($source_index)) { |
| 1548 |
$state['source_name'] = (string) ($source_index['name'] ?? ''); |
| 1549 |
} |
| 1550 |
return self::migration_set($state); |
| 1551 |
} |
| 1552 |
|
| 1553 |
/** |
| 1554 |
* Copy pages until the step's time budget is used or the source is |
| 1555 |
* exhausted: list → fetch (values + metadata) → documents/upsert, each |
| 1556 |
* call retried with backoff. Idempotent — the same ids land in the same |
| 1557 |
* place on a re-run. The cursor is persisted after EVERY page, so a step |
| 1558 |
* that dies mid-way (proxy timeout, killed request) loses at most one |
| 1559 |
* page of work and the next click continues from the recorded position. |
| 1560 |
* |
| 1561 |
* @return array|WP_Error state |
| 1562 |
*/ |
| 1563 |
public static function migrate_step(array $state, array $target_cfg, $batch = self::MIGRATE_BATCH) { |
| 1564 |
if (($state['status'] ?? '') !== 'running') { |
| 1565 |
return $state; |
| 1566 |
} |
| 1567 |
if (strtolower((string) ($state['target_host'] ?? '')) !== strtolower((string) ($target_cfg['host'] ?? ''))) { |
| 1568 |
return new WP_Error('migrate', 'The document index host changed since the migration started. Start it again.'); |
| 1569 |
} |
| 1570 |
$started = microtime(true); |
| 1571 |
$budget = self::step_budget(); |
| 1572 |
$pages = 0; |
| 1573 |
$api_key = $target_cfg['api_key']; |
| 1574 |
foreach (array('retries', 'skipped', 'failed_steps', 'error_at') as $counter) { |
| 1575 |
$state[$counter] = (int) ($state[$counter] ?? 0); // states written before 3e83e4 lack these |
| 1576 |
} |
| 1577 |
if (empty($state['per_namespace']) || !is_array($state['per_namespace'])) { |
| 1578 |
// e50d2e: a state written before this build copied one namespace and has no ledger |
| 1579 |
$state['per_namespace'] = self::per_namespace_map($state); |
| 1580 |
$state['current'] = (string) ($state['namespace'] ?? ''); |
| 1581 |
$state['queue'] = array(); |
| 1582 |
} |
| 1583 |
if (!isset($state['current'])) { |
| 1584 |
$state['current'] = (string) ($state['namespace'] ?? ''); |
| 1585 |
} |
| 1586 |
do { |
| 1587 |
$source_host = $state['source_host']; |
| 1588 |
$namespace = (string) $state['current']; |
| 1589 |
$ns_key = self::ns_key($namespace); |
| 1590 |
$token = $state['token']; |
| 1591 |
// e50d2e: the same namespace on the document side — every bot keeps reading its own |
| 1592 |
$page_cfg = $target_cfg; |
| 1593 |
$page_cfg['namespace'] = $namespace; |
| 1594 |
$listed = self::with_retry(function () use ($source_host, $api_key, $namespace, $batch, $token) { |
| 1595 |
return self::classic_list_page($source_host, $api_key, $namespace, $batch, $token); |
| 1596 |
}); |
| 1597 |
$state['retries'] += max(0, $listed['attempts'] - 1); |
| 1598 |
$page = $listed['result']; |
| 1599 |
if (is_wp_error($page)) { |
| 1600 |
return self::migration_fail($state, 'Listing the source index failed after ' . $listed['attempts'] . ' attempts: ' . $page->get_error_message()); |
| 1601 |
} |
| 1602 |
$ids = $page['ids']; |
| 1603 |
if (!empty($ids)) { |
| 1604 |
$fetched = self::with_retry(function () use ($source_host, $api_key, $namespace, $ids) { |
| 1605 |
return self::classic_fetch($source_host, $api_key, $namespace, $ids); |
| 1606 |
}); |
| 1607 |
$state['retries'] += max(0, $fetched['attempts'] - 1); |
| 1608 |
$vectors = $fetched['result']; |
| 1609 |
if (is_wp_error($vectors)) { |
| 1610 |
return self::migration_fail($state, 'Reading ' . count($ids) . ' records from the source index failed after ' . $fetched['attempts'] . ' attempts: ' . $vectors->get_error_message()); |
| 1611 |
} |
| 1612 |
$documents = array(); |
| 1613 |
$skipped = 0; |
| 1614 |
foreach ($vectors as $id => $vector) { |
| 1615 |
if (empty($vector['values']) || !is_array($vector['values'])) { |
| 1616 |
$skipped++; |
| 1617 |
continue; |
| 1618 |
} |
| 1619 |
$metadata = is_array($vector['metadata'] ?? null) ? $vector['metadata'] : array(); |
| 1620 |
$documents[] = self::build_document((string) ($vector['id'] ?? $id), (string) ($metadata['text'] ?? ''), $vector['values'], $metadata); |
| 1621 |
} |
| 1622 |
$skipped += max(0, count($ids) - count($vectors)); |
| 1623 |
if (!empty($documents)) { |
| 1624 |
$upserted = self::with_retry(function () use ($documents, $page_cfg) { |
| 1625 |
return self::upsert($documents, $page_cfg); |
| 1626 |
}); |
| 1627 |
$state['retries'] += max(0, $upserted['attempts'] - 1); |
| 1628 |
if (is_wp_error($upserted['result'])) { |
| 1629 |
return self::migration_fail($state, 'Writing ' . count($documents) . ' records to the document index failed after ' . $upserted['attempts'] . ' attempts: ' . $upserted['result']->get_error_message()); |
| 1630 |
} |
| 1631 |
} |
| 1632 |
$state['copied'] = (int) $state['copied'] + count($documents); |
| 1633 |
$state['skipped'] = (int) ($state['skipped'] ?? 0) + $skipped; |
| 1634 |
$state['per_namespace'][$ns_key]['copied'] = (int) ($state['per_namespace'][$ns_key]['copied'] ?? 0) + count($documents); |
| 1635 |
} |
| 1636 |
$state['batches'] = (int) $state['batches'] + 1; |
| 1637 |
$state['token'] = $page['next']; |
| 1638 |
$state['error'] = ''; |
| 1639 |
$state['error_at'] = 0; |
| 1640 |
$pages++; |
| 1641 |
if ($page['next'] === '') { |
| 1642 |
// This namespace is copied: record what the document index |
| 1643 |
// reports for it, then move on to the next queued namespace |
| 1644 |
// inside the same time budget (e50d2e), or finish. |
| 1645 |
$state['per_namespace'][$ns_key]['status'] = 'done'; |
| 1646 |
$state['per_namespace'][$ns_key]['finished'] = time(); |
| 1647 |
$ns_target = self::namespace_count($page_cfg); |
| 1648 |
$state['per_namespace'][$ns_key]['target_count'] = $ns_target > 0 ? $ns_target : null; |
| 1649 |
$queue = array_values((array) ($state['queue'] ?? array())); |
| 1650 |
if (!empty($queue)) { |
| 1651 |
$next = (string) array_shift($queue); |
| 1652 |
$state['queue'] = $queue; |
| 1653 |
$state['current'] = $next; |
| 1654 |
$state['namespace'] = $next; |
| 1655 |
$state['token'] = ''; |
| 1656 |
if (!isset($state['per_namespace'][self::ns_key($next)])) { |
| 1657 |
$state['per_namespace'][self::ns_key($next)] = self::new_namespace_entry($next, 0); |
| 1658 |
} |
| 1659 |
$state['per_namespace'][self::ns_key($next)]['status'] = 'running'; |
| 1660 |
self::migration_set($state); |
| 1661 |
continue; // re-checks the budget, then lists the next namespace from its first page |
| 1662 |
} |
| 1663 |
$state['status'] = 'done'; |
| 1664 |
$state['finished'] = time(); |
| 1665 |
// The finished state names what both sides report, so the UI |
| 1666 |
// can say "5,000 copied; the document index holds 5,000". |
| 1667 |
$target_count = 0; |
| 1668 |
foreach ($state['per_namespace'] as $entry) { |
| 1669 |
$target_count += (int) ($entry['target_count'] ?? 0); |
| 1670 |
} |
| 1671 |
$state['target_count'] = $target_count > 0 ? $target_count : null; |
| 1672 |
$state['target_count_at'] = time(); |
| 1673 |
break; |
| 1674 |
} |
| 1675 |
self::migration_set($state); // cursor persisted per page, not per step |
| 1676 |
} while ((microtime(true) - $started) < $budget); |
| 1677 |
$state['last_step_pages'] = $pages; |
| 1678 |
$state['last_step_seconds'] = round(microtime(true) - $started, 2); |
| 1679 |
return self::migration_set($state); |
| 1680 |
} |
| 1681 |
|
| 1682 |
/** Record count of ONE namespace on a document index (0 when not listed) — count() reads the whole index for the default namespace. */ |
| 1683 |
private static function namespace_count(array $cfg) { |
| 1684 |
$stats = self::stats($cfg); |
| 1685 |
if (is_wp_error($stats)) { |
| 1686 |
return 0; |
| 1687 |
} |
| 1688 |
$ns = (string) ($cfg['namespace'] ?? ''); |
| 1689 |
return isset($stats['namespaces'][$ns]['vectorCount']) ? (int) $stats['namespaces'][$ns]['vectorCount'] : 0; |
| 1690 |
} |
| 1691 |
|
| 1692 |
/** |
| 1693 |
* e50d2e: namespaces of the OLD index that still hold records without a |
| 1694 |
* finished, count-matching copy — read from the live source stats, so a |
| 1695 |
* namespace filled after the copy counts too. Empty array = every record |
| 1696 |
* on the old index has a copy; the Delete old index button refuses on |
| 1697 |
* anything else and names what is missing. |
| 1698 |
* |
| 1699 |
* @return array|WP_Error display key → ['namespace','source','copied','status'] |
| 1700 |
*/ |
| 1701 |
public static function uncopied_namespaces(array $state, $api_key) { |
| 1702 |
$stats = self::classic_stats((string) ($state['source_host'] ?? ''), $api_key); |
| 1703 |
if (is_wp_error($stats)) { |
| 1704 |
return $stats; |
| 1705 |
} |
| 1706 |
$counts = self::namespace_counts($stats); |
| 1707 |
if (empty($counts) && (int) ($stats['totalVectorCount'] ?? 0) > 0) { |
| 1708 |
$counts[''] = (int) $stats['totalVectorCount']; |
| 1709 |
} |
| 1710 |
$map = self::per_namespace_map($state); |
| 1711 |
$missing = array(); |
| 1712 |
foreach ($counts as $ns => $count) { |
| 1713 |
if ($count <= 0) { |
| 1714 |
continue; |
| 1715 |
} |
| 1716 |
$key = self::ns_key($ns); |
| 1717 |
$entry = isset($map[$key]) && is_array($map[$key]) ? $map[$key] : null; |
| 1718 |
if ($entry === null || ($entry['status'] ?? '') !== 'done' || (int) ($entry['copied'] ?? 0) < $count) { |
| 1719 |
$missing[$key] = array( |
| 1720 |
'namespace' => (string) $ns, |
| 1721 |
'source' => (int) $count, |
| 1722 |
'copied' => $entry ? (int) ($entry['copied'] ?? 0) : 0, |
| 1723 |
'status' => $entry ? (string) ($entry['status'] ?? '') : 'not copied', |
| 1724 |
); |
| 1725 |
} |
| 1726 |
} |
| 1727 |
// Named in the copy order (default first, then by name) — Pinecone's stats object order varies call to call. |
| 1728 |
$ordered = array(); |
| 1729 |
foreach (self::namespace_queue(array_combine(array_map(array(__CLASS__, 'ns_raw'), array_keys($missing)), array_column($missing, 'source')) ?: array()) as $ns) { |
| 1730 |
$ordered[self::ns_key($ns)] = $missing[self::ns_key($ns)]; |
| 1731 |
} |
| 1732 |
return $ordered + $missing; |
| 1733 |
} |
| 1734 |
|
| 1735 |
/** Plain-language refusal naming every namespace the copy is missing. (Plain __(): the card's JS text-escapes it.) */ |
| 1736 |
public static function uncopied_message(array $missing) { |
| 1737 |
$parts = array(); |
| 1738 |
foreach ($missing as $key => $m) { |
| 1739 |
$parts[] = sprintf( |
| 1740 |
/* translators: 1: namespace name, 2: records copied, 3: records on the old index */ |
| 1741 |
__('%1$s (%2$s of %3$s records copied)', 'mxchat'), |
| 1742 |
$key, |
| 1743 |
number_format_i18n((int) $m['copied']), |
| 1744 |
number_format_i18n((int) $m['source']) |
| 1745 |
); |
| 1746 |
} |
| 1747 |
return sprintf( |
| 1748 |
/* translators: %s: comma-separated namespace names with their counts */ |
| 1749 |
_n( |
| 1750 |
'Namespace %s on the old index has records that are not in the document index. Tick Copy every namespace and run Migrate again, then delete. The old index stays.', |
| 1751 |
'Namespaces %s on the old index have records that are not in the document index. Tick Copy every namespace and run Migrate again, then delete. The old index stays.', |
| 1752 |
count($missing), |
| 1753 |
'mxchat' |
| 1754 |
), |
| 1755 |
implode(', ', $parts) |
| 1756 |
); |
| 1757 |
} |
| 1758 |
|
| 1759 |
/** Record a step failure without ending the copy: status stays running, the cursor stays put. */ |
| 1760 |
private static function migration_fail(array $state, $message) { |
| 1761 |
$state['error'] = (string) $message; |
| 1762 |
$state['error_at'] = time(); |
| 1763 |
$state['failed_steps'] = (int) ($state['failed_steps'] ?? 0) + 1; |
| 1764 |
return self::migration_set($state); |
| 1765 |
} |
| 1766 |
|
| 1767 |
// ===================================================================== |
| 1768 |
// AJAX (Knowledge → Pinecone card) |
| 1769 |
// ===================================================================== |
| 1770 |
|
| 1771 |
public static function boot() { |
| 1772 |
add_action('wp_ajax_mxchat_pinecone_docs_create_index', array(__CLASS__, 'ajax_create_index')); |
| 1773 |
add_action('wp_ajax_mxchat_pinecone_docs_check_index', array(__CLASS__, 'ajax_check_index')); |
| 1774 |
add_action('wp_ajax_mxchat_pinecone_docs_suggest_name', array(__CLASS__, 'ajax_suggest_name')); |
| 1775 |
add_action('wp_ajax_mxchat_pinecone_docs_migrate', array(__CLASS__, 'ajax_migrate')); |
| 1776 |
add_action('wp_ajax_mxchat_pinecone_docs_delete_old_index', array(__CLASS__, 'ajax_delete_old_index')); |
| 1777 |
} |
| 1778 |
|
| 1779 |
/** |
| 1780 |
* Suggest a free document-index name for the name in the field (3e83e4). |
| 1781 |
* With a host that already answers in the project, the name is that |
| 1782 |
* index's own — the owner is pointing at an existing document index, not |
| 1783 |
* about to create one. |
| 1784 |
*/ |
| 1785 |
public static function ajax_suggest_name() { |
| 1786 |
self::ajax_guard(); |
| 1787 |
$api_key = self::posted_api_key(); |
| 1788 |
$base = isset($_POST['base']) ? sanitize_text_field(wp_unslash($_POST['base'])) : ''; |
| 1789 |
$host = isset($_POST['host']) ? sanitize_text_field(wp_unslash($_POST['host'])) : ''; |
| 1790 |
if ($api_key === '') { |
| 1791 |
wp_send_json_error(array('message' => esc_html__('Enter your Pinecone API key first.', 'mxchat'))); |
| 1792 |
} |
| 1793 |
if ($host !== '') { |
| 1794 |
$index = self::find_index_by_host($host, $api_key); |
| 1795 |
if (!is_wp_error($index) && !empty($index['name'])) { |
| 1796 |
wp_send_json_success(array('name' => strtolower((string) $index['name']), 'resolved' => true, 'taken' => false, 'base' => strtolower($base))); |
| 1797 |
} |
| 1798 |
} |
| 1799 |
$suggest = self::suggest_index_name($base, $api_key); |
| 1800 |
wp_send_json_success(array('name' => $suggest['name'], 'resolved' => false, 'taken' => $suggest['taken'], 'base' => strtolower($base))); |
| 1801 |
} |
| 1802 |
|
| 1803 |
/** The document-index target for a migration call: the host in the form, else the saved one. */ |
| 1804 |
private static function posted_target_host(array $options) { |
| 1805 |
$posted = isset($_POST['target_host']) ? sanitize_text_field(wp_unslash($_POST['target_host'])) : ''; |
| 1806 |
$posted = strtolower(trim(str_replace(array('https://', 'http://'), '', $posted), '/')); |
| 1807 |
if ($posted !== '') { |
| 1808 |
return $posted; |
| 1809 |
} |
| 1810 |
return strtolower(trim((string) ($options['mxchat_pinecone_host'] ?? ''), '/')); |
| 1811 |
} |
| 1812 |
|
| 1813 |
private static function ajax_guard() { |
| 1814 |
check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce'); |
| 1815 |
if (!current_user_can('manage_options')) { |
| 1816 |
wp_send_json_error(array('message' => esc_html__('Permission denied.', 'mxchat')), 403); |
| 1817 |
} |
| 1818 |
} |
| 1819 |
|
| 1820 |
private static function posted_api_key() { |
| 1821 |
$posted = isset($_POST['api_key']) ? sanitize_text_field(wp_unslash($_POST['api_key'])) : ''; |
| 1822 |
if ($posted !== '') { |
| 1823 |
return $posted; |
| 1824 |
} |
| 1825 |
$options = get_option(self::OPTION, array()); |
| 1826 |
return is_array($options) ? (string) ($options['mxchat_pinecone_api_key'] ?? '') : ''; |
| 1827 |
} |
| 1828 |
|
| 1829 |
/** |
| 1830 |
* Create index for me: control-plane create with the active embedding |
| 1831 |
* dimension, then fill in (and persist) the host. |
| 1832 |
*/ |
| 1833 |
public static function ajax_create_index() { |
| 1834 |
self::ajax_guard(); |
| 1835 |
$api_key = self::posted_api_key(); |
| 1836 |
$name = isset($_POST['index_name']) ? sanitize_text_field(wp_unslash($_POST['index_name'])) : ''; |
| 1837 |
$cloud = isset($_POST['cloud']) ? sanitize_key($_POST['cloud']) : 'aws'; |
| 1838 |
$region = isset($_POST['region']) ? sanitize_text_field(wp_unslash($_POST['region'])) : 'us-east-1'; |
| 1839 |
$language = isset($_POST['language']) ? sanitize_key($_POST['language']) : 'en'; |
| 1840 |
if ($api_key === '') { |
| 1841 |
wp_send_json_error(array('message' => esc_html__('Enter your Pinecone API key first.', 'mxchat'))); |
| 1842 |
} |
| 1843 |
if ($name === '') { |
| 1844 |
wp_send_json_error(array('message' => esc_html__('Enter an index name first — the new document index will be created under that name.', 'mxchat'))); |
| 1845 |
} |
| 1846 |
$dimension = self::expected_dimension(); |
| 1847 |
$index = self::create_document_index($name, $dimension, $cloud, $region, $language, $api_key); |
| 1848 |
if (is_wp_error($index)) { |
| 1849 |
$data = $index->get_error_data(); |
| 1850 |
wp_send_json_error(array( |
| 1851 |
'message' => $index->get_error_message(), |
| 1852 |
'code' => $index->get_error_code(), |
| 1853 |
'suggested' => is_array($data) ? (string) ($data['suggested'] ?? '') : '', |
| 1854 |
)); |
| 1855 |
} |
| 1856 |
$host = (string) ($index['host'] ?? ''); |
| 1857 |
// Give Pinecone a few seconds — small serverless indexes are usually Ready quickly. |
| 1858 |
$ready = !empty($index['status']['ready']); |
| 1859 |
for ($i = 0; !$ready && $i < 5; $i++) { |
| 1860 |
sleep(2); |
| 1861 |
$described = self::describe_index($name, $api_key); |
| 1862 |
if (!is_wp_error($described)) { |
| 1863 |
$ready = !empty($described['status']['ready']); |
| 1864 |
$host = (string) ($described['host'] ?? $host); |
| 1865 |
} |
| 1866 |
} |
| 1867 |
$options = get_option(self::OPTION, array()); |
| 1868 |
$options = is_array($options) ? $options : array(); |
| 1869 |
// 3e83e4: creating the index no longer moves the live chatbot. The |
| 1870 |
// host and index type are only written when the owner saves the |
| 1871 |
// form; here we record that this host passed the check (the save |
| 1872 |
// gate) and the settings the index was created with. |
| 1873 |
$changes = array( |
| 1874 |
'mxchat_pinecone_docs_verified_host' => $host, |
| 1875 |
'mxchat_pinecone_docs_cloud' => array_key_exists($cloud, self::clouds()) ? $cloud : 'aws', |
| 1876 |
'mxchat_pinecone_docs_region' => $region ?: 'us-east-1', |
| 1877 |
'mxchat_pinecone_docs_language' => array_key_exists($language, self::languages()) ? $language : 'en', |
| 1878 |
); |
| 1879 |
if (empty($options['mxchat_pinecone_api_key'])) { |
| 1880 |
$changes['mxchat_pinecone_api_key'] = $api_key; |
| 1881 |
} |
| 1882 |
// Remember the classic index so Migrate can copy from it. |
| 1883 |
$previous_host = (string) ($options['mxchat_pinecone_host'] ?? ''); |
| 1884 |
if ($previous_host !== '' && strtolower($previous_host) !== strtolower($host) && self::options_index_type($options) === 'vector' && empty($options['mxchat_pinecone_vector_host'])) { |
| 1885 |
$changes['mxchat_pinecone_vector_host'] = $previous_host; |
| 1886 |
$changes['mxchat_pinecone_vector_index'] = (string) ($options['mxchat_pinecone_index'] ?? ''); |
| 1887 |
} |
| 1888 |
self::persist_options($changes); |
| 1889 |
wp_send_json_success(array( |
| 1890 |
'host' => $host, |
| 1891 |
'name' => strtolower($name), |
| 1892 |
'dimension' => $dimension, |
| 1893 |
'ready' => $ready, |
| 1894 |
'vector_host' => $changes['mxchat_pinecone_vector_host'] ?? ($options['mxchat_pinecone_vector_host'] ?? ''), |
| 1895 |
// Plain text: the card's JS escapes it on render (esc_html__ here double-escaped the quotes to a literal "). |
| 1896 |
'message' => $ready |
| 1897 |
? sprintf(__('Document index "%1$s" created (%2$d dimensions, cosine, full-text search on the chunk text). The host is filled in above. Your chatbot keeps answering from the current index until you save the Pinecone settings — copy your records across with Migrate first.', 'mxchat'), strtolower($name), $dimension) |
| 1898 |
: sprintf(__('Document index "%1$s" is being created (%2$d dimensions). The host is filled in above; run Check index in a minute to confirm it is ready. Your chatbot keeps answering from the current index until you save the Pinecone settings.', 'mxchat'), strtolower($name), $dimension), |
| 1899 |
)); |
| 1900 |
} |
| 1901 |
|
| 1902 |
/** Check index: assert the host is a document index MxChat can use. */ |
| 1903 |
public static function ajax_check_index() { |
| 1904 |
self::ajax_guard(); |
| 1905 |
$api_key = self::posted_api_key(); |
| 1906 |
$host = isset($_POST['host']) ? sanitize_text_field(wp_unslash($_POST['host'])) : ''; |
| 1907 |
$host = trim(str_replace(array('https://', 'http://'), '', $host), '/'); |
| 1908 |
if ($api_key === '' || $host === '') { |
| 1909 |
wp_send_json_error(array('message' => esc_html__('Enter the Pinecone API key and host first.', 'mxchat'))); |
| 1910 |
} |
| 1911 |
$check = self::check_document_index($host, $api_key, self::expected_dimension()); |
| 1912 |
if (!$check['ok']) { |
| 1913 |
wp_send_json_error($check); |
| 1914 |
} |
| 1915 |
$options = get_option(self::OPTION, array()); |
| 1916 |
$options = is_array($options) ? $options : array(); |
| 1917 |
$changes = array( |
| 1918 |
'mxchat_pinecone_docs_verified_host' => $check['host'], |
| 1919 |
'mxchat_pinecone_index' => $check['name'], |
| 1920 |
); |
| 1921 |
$previous_host = (string) ($options['mxchat_pinecone_host'] ?? ''); |
| 1922 |
if ($previous_host !== '' && strtolower($previous_host) !== strtolower($check['host']) && self::options_index_type($options) === 'vector' && empty($options['mxchat_pinecone_vector_host'])) { |
| 1923 |
$changes['mxchat_pinecone_vector_host'] = $previous_host; |
| 1924 |
$changes['mxchat_pinecone_vector_index'] = (string) ($options['mxchat_pinecone_index'] ?? ''); |
| 1925 |
} |
| 1926 |
self::persist_options($changes); |
| 1927 |
$check['vector_host'] = $changes['mxchat_pinecone_vector_host'] ?? ($options['mxchat_pinecone_vector_host'] ?? ''); |
| 1928 |
wp_send_json_success($check); |
| 1929 |
} |
| 1930 |
|
| 1931 |
/** |
| 1932 |
* Migrate: mode start | step | status | reset. |
| 1933 |
* |
| 1934 |
* 3e83e4: the copy is DECOUPLED from the live retrieval path. The target |
| 1935 |
* is the document-index host in the form (posted as target_host) plus the |
| 1936 |
* API key, checked against Pinecone at start — the saved index type is |
| 1937 |
* never consulted, so the site keeps answering from its vector index for |
| 1938 |
* the whole copy and the owner switches by saving the settings afterwards. |
| 1939 |
* Each step copies pages for up to MIGRATE_STEP_SECONDS. |
| 1940 |
*/ |
| 1941 |
public static function ajax_migrate() { |
| 1942 |
self::ajax_guard(); |
| 1943 |
$mode = isset($_POST['mode']) ? sanitize_key($_POST['mode']) : 'status'; |
| 1944 |
$options = get_option(self::OPTION, array()); |
| 1945 |
$options = is_array($options) ? $options : array(); |
| 1946 |
if ($mode === 'reset') { |
| 1947 |
self::migration_clear(); |
| 1948 |
wp_send_json_success(array('state' => array())); |
| 1949 |
} |
| 1950 |
if ($mode === 'status') { |
| 1951 |
wp_send_json_success(array('state' => self::migration_get())); |
| 1952 |
} |
| 1953 |
$api_key = self::posted_api_key(); |
| 1954 |
$namespace = (string) ($options['mxchat_pinecone_namespace'] ?? ''); |
| 1955 |
if ($api_key === '') { |
| 1956 |
wp_send_json_error(array('message' => esc_html__('Enter your Pinecone API key first.', 'mxchat'))); |
| 1957 |
} |
| 1958 |
if ($mode === 'start') { |
| 1959 |
$target_host = self::posted_target_host($options); |
| 1960 |
if ($target_host === '') { |
| 1961 |
wp_send_json_error(array('message' => esc_html__('Enter the host of the document index to copy into (or create one with Create index for me).', 'mxchat'))); |
| 1962 |
} |
| 1963 |
$check = self::check_document_index($target_host, $api_key, self::expected_dimension()); |
| 1964 |
if (!$check['ok']) { |
| 1965 |
wp_send_json_error(array('message' => $check['message'])); |
| 1966 |
} |
| 1967 |
$target = array('api_key' => $api_key, 'host' => strtolower($check['host']), 'namespace' => $namespace, 'index_type' => 'document'); |
| 1968 |
$source_host = isset($_POST['source_host']) ? sanitize_text_field(wp_unslash($_POST['source_host'])) : ''; |
| 1969 |
$restart = !empty($_POST['restart']); |
| 1970 |
$all_ns = !empty($_POST['all_namespaces']); // e50d2e: Copy every namespace |
| 1971 |
$state = self::migrate_start($source_host, $target, $restart, $all_ns); |
| 1972 |
if (is_wp_error($state)) { |
| 1973 |
wp_send_json_error(array('message' => $state->get_error_message())); |
| 1974 |
} |
| 1975 |
$changes = array('mxchat_pinecone_docs_verified_host' => $target['host']); |
| 1976 |
if ($source_host !== '' && strtolower($state['source_host']) !== strtolower((string) ($options['mxchat_pinecone_vector_host'] ?? ''))) { |
| 1977 |
$changes['mxchat_pinecone_vector_host'] = $state['source_host']; |
| 1978 |
$changes['mxchat_pinecone_vector_index'] = $state['source_name']; |
| 1979 |
} |
| 1980 |
self::persist_options($changes); |
| 1981 |
wp_send_json_success(array('state' => $state, 'live_index_type' => self::options_index_type($options))); |
| 1982 |
} |
| 1983 |
if ($mode === 'step') { |
| 1984 |
$state = self::migration_get(); |
| 1985 |
if (empty($state)) { |
| 1986 |
wp_send_json_error(array('message' => esc_html__('No migration in progress. Click Migrate to start one.', 'mxchat'))); |
| 1987 |
} |
| 1988 |
$target = array('api_key' => $api_key, 'host' => (string) ($state['target_host'] ?? ''), 'namespace' => (string) ($state['current'] ?? ($state['namespace'] ?? $namespace)), 'index_type' => 'document'); |
| 1989 |
$state = self::migrate_step($state, $target); |
| 1990 |
if (is_wp_error($state)) { |
| 1991 |
wp_send_json_error(array('message' => $state->get_error_message())); |
| 1992 |
} |
| 1993 |
if (!empty($state['error'])) { |
| 1994 |
wp_send_json_error(array('message' => $state['error'], 'state' => $state)); |
| 1995 |
} |
| 1996 |
wp_send_json_success(array('state' => $state, 'live_index_type' => self::options_index_type($options))); |
| 1997 |
} |
| 1998 |
wp_send_json_error(array('message' => esc_html__('Unknown migration action.', 'mxchat'))); |
| 1999 |
} |
| 2000 |
|
| 2001 |
/** |
| 2002 |
* Delete old index — only after a finished copy whose count matches, only |
| 2003 |
* with the index name typed back, never the configured document index. |
| 2004 |
*/ |
| 2005 |
public static function ajax_delete_old_index() { |
| 2006 |
self::ajax_guard(); |
| 2007 |
$confirm = isset($_POST['confirm_name']) ? strtolower(sanitize_text_field(wp_unslash($_POST['confirm_name']))) : ''; |
| 2008 |
$options = get_option(self::OPTION, array()); |
| 2009 |
$options = is_array($options) ? $options : array(); |
| 2010 |
$target = self::cfg_from_options($options); |
| 2011 |
$state = self::migration_get(); |
| 2012 |
if (empty($state) || ($state['status'] ?? '') !== 'done') { |
| 2013 |
wp_send_json_error(array('message' => esc_html__('Finish the migration first — the old index can only be deleted after a completed copy.', 'mxchat'))); |
| 2014 |
} |
| 2015 |
if ((int) $state['copied'] < (int) $state['total'] || (int) $state['total'] === 0) { |
| 2016 |
wp_send_json_error(array('message' => sprintf(esc_html__('The copy count does not match (%1$d of %2$d copied). The old index stays.', 'mxchat'), (int) $state['copied'], (int) $state['total']))); |
| 2017 |
} |
| 2018 |
if ($target['index_type'] !== 'document' || strtolower($target['host']) !== strtolower((string) $state['target_host'])) { |
| 2019 |
wp_send_json_error(array('message' => esc_html__('The saved document index no longer matches the migration. The old index stays.', 'mxchat'))); |
| 2020 |
} |
| 2021 |
// e50d2e: EVERY namespace that holds records on the old index needs a |
| 2022 |
// finished copy whose count matches — not just the one copied last. |
| 2023 |
// Before this, a Multi-Bot or legacy-namespace site could delete |
| 2024 |
// records that were never copied. |
| 2025 |
$missing = self::uncopied_namespaces($state, $target['api_key']); |
| 2026 |
if (is_wp_error($missing)) { |
| 2027 |
wp_send_json_error(array('message' => sprintf(__('Could not read the old index before deleting it (%s). The old index stays.', 'mxchat'), $missing->get_error_message()))); |
| 2028 |
} |
| 2029 |
if (!empty($missing)) { |
| 2030 |
wp_send_json_error(array('message' => self::uncopied_message($missing), 'uncopied' => $missing)); |
| 2031 |
} |
| 2032 |
$source = self::find_index_by_host($state['source_host'], $target['api_key']); |
| 2033 |
if (is_wp_error($source)) { |
| 2034 |
wp_send_json_error(array('message' => $source->get_error_message())); |
| 2035 |
} |
| 2036 |
$source_name = strtolower((string) ($source['name'] ?? '')); |
| 2037 |
if ($source_name === '' || $confirm !== $source_name) { |
| 2038 |
wp_send_json_error(array('message' => sprintf(esc_html__('Type the old index name exactly (%s) to confirm.', 'mxchat'), $source_name))); |
| 2039 |
} |
| 2040 |
if (strtolower((string) ($source['host'] ?? '')) === strtolower($target['host']) || $source_name === strtolower((string) ($state['target_name'] ?? ''))) { |
| 2041 |
wp_send_json_error(array('message' => esc_html__('Refusing: that is the document index currently in use.', 'mxchat'))); |
| 2042 |
} |
| 2043 |
$result = self::delete_index($source_name, $target['api_key']); |
| 2044 |
if (is_wp_error($result)) { |
| 2045 |
wp_send_json_error(array('message' => $result->get_error_message())); |
| 2046 |
} |
| 2047 |
self::persist_options(array('mxchat_pinecone_vector_host' => '', 'mxchat_pinecone_vector_index' => '')); |
| 2048 |
$state['status'] = 'source_deleted'; |
| 2049 |
self::migration_set($state); |
| 2050 |
wp_send_json_success(array('message' => sprintf(esc_html__('Old vector index "%s" deleted.', 'mxchat'), $source_name), 'state' => $state)); |
| 2051 |
} |
| 2052 |
} |
| 2053 |
|
| 2054 |
MxChat_Pinecone_Documents::boot(); |
| 2055 |
|
