| @@ -14,8 +14,10 @@ | ||
| 14 | 14 | /** |
| 15 | 15 | * Constructor |
| 16 | 16 | */ |
| 17 | 17 | public function __construct() { |
| 18 | + // Hook into WordPress actions if needed | |
| 19 | + add_action('mxchat_delete_content', array($this, 'mxchat_delete_from_pinecone_by_url'), 10, 1); | |
| 18 | 20 | } |
| 19 | 21 | |
| 20 | 22 | // ======================================== |
| 21 | 23 | // PINECONE FETCH OPERATIONS |
| @@ -493,12 +495,8 @@ | ||
| 493 | 495 | $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; |
| 494 | 496 | |
| 495 | 497 | // Pinecone's list endpoint returns vector IDs with pagination |
| 496 | 498 | // We then fetch the metadata for those specific IDs |
| 497 | - // NOTE (plan 793b82): /vectors/list is a GET endpoint with query parameters. | |
| 498 | - // A POST is answered 200-with-an-empty-body, which reads as "no vectors" — | |
| 499 | - // this call used to POST and only ever "worked" because the empty result | |
| 500 | - // tripped the query-based fallback below. | |
| 501 | 499 | $list_url = "https://{$host}/vectors/list"; |
| 502 | 500 | |
| 503 | 501 | // Calculate pagination token from page number |
| 504 | 502 | // Pinecone uses cursor-based pagination, so we need to handle this differently |
| @@ -522,20 +520,19 @@ | ||
| 522 | 520 | $list_params['paginationToken'] = $stored_tokens[$page]; |
| 523 | 521 | } |
| 524 | 522 | } |
| 525 | 523 | |
| 526 | - $response = wp_remote_get($list_url . '?' . http_build_query($list_params), array( | |
| 524 | + $response = wp_remote_post($list_url, array( | |
| 527 | 525 | 'headers' => array( |
| 528 | 526 | 'Api-Key' => $api_key, |
| 529 | - 'accept' => 'application/json' | |
| 527 | + 'Content-Type' => 'application/json' | |
| 530 | 528 | ), |
| 529 | + 'body' => json_encode($list_params), | |
| 531 | 530 | 'timeout' => 15 |
| 532 | 531 | )); |
| 533 | 532 | |
| 534 | 533 | if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) { |
| 535 | - // Genuine transport/API failure — fall back to the query-based approach, | |
| 536 | - // but say so: a silent fallback is what masked the POST bug for a year. | |
| 537 | - error_log('MxChat Pinecone: /vectors/list GET failed (' . (is_wp_error($response) ? $response->get_error_message() : wp_remote_retrieve_response_code($response)) . ') — falling back to query-based listing.'); | |
| 534 | + // Fallback to query-based approach | |
| 538 | 535 | return $this->mxchat_query_based_list($pinecone_options, $page, $per_page, $bot_id, $content_type); |
| 539 | 536 | } |
| 540 | 537 | |
| 541 | 538 | $body = wp_remote_retrieve_body($response); |
| @@ -554,26 +551,15 @@ | ||
| 554 | 551 | $vector_ids[] = $vector['id']; |
| 555 | 552 | } |
| 556 | 553 | } |
| 557 | 554 | |
| 558 | - // A 200 with no vector ids is a REAL answer now (the namespace is empty), | |
| 559 | - // not the failure signature it was under POST — return the empty listing | |
| 560 | - // rather than routing into the fallback and masking a broken primary. | |
| 555 | + // If list endpoint returned empty, fall back to query-based approach | |
| 561 | 556 | if (empty($vector_ids)) { |
| 562 | - return array('data' => array(), 'total' => 0); | |
| 557 | + return $this->mxchat_query_based_list($pinecone_options, $page, $per_page, $bot_id, $content_type); | |
| 563 | 558 | } |
| 564 | 559 | |
| 565 | 560 | // Fetch metadata for these vector IDs |
| 566 | - $fetched = $this->mxchat_fetch_vectors_by_ids_for_list($pinecone_options, $vector_ids, $page, $per_page, $bot_id, $content_type); | |
| 567 | - | |
| 568 | - // null = the fetch itself failed in transport (distinct from "ids had no | |
| 569 | - // metadata matches") — log and fall back so the screen still lists. | |
| 570 | - if ($fetched === null) { | |
| 571 | - error_log('MxChat Pinecone: /vectors/fetch GET failed for the listing — falling back to query-based listing.'); | |
| 572 | - return $this->mxchat_query_based_list($pinecone_options, $page, $per_page, $bot_id, $content_type); | |
| 573 | - } | |
| 574 | - | |
| 575 | - return $fetched; | |
| 561 | + return $this->mxchat_fetch_vectors_by_ids_for_list($pinecone_options, $vector_ids, $page, $per_page, $bot_id, $content_type); | |
| 576 | 562 | } |
| 577 | 563 | |
| 578 | 564 | /** |
| 579 | 565 | * Query-based listing fallback when list endpoint fails |
| @@ -677,31 +663,29 @@ | ||
| 677 | 663 | $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? ''; |
| 678 | 664 | $host = $pinecone_options['mxchat_pinecone_host'] ?? ''; |
| 679 | 665 | $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; |
| 680 | 666 | |
| 681 | - // NOTE (plan 793b82): /vectors/fetch is a GET endpoint, and Pinecone expects | |
| 682 | - // the ids repeated (ids=a&ids=b) — http_build_query would emit ids[0]=a. | |
| 683 | - // A POST here is answered 200-with-an-empty-body, which reads as "no vectors". | |
| 684 | - $fetch_query = array(); | |
| 685 | - foreach ($vector_ids as $fetch_vid) { | |
| 686 | - $fetch_query[] = 'ids=' . rawurlencode($fetch_vid); | |
| 687 | - } | |
| 667 | + $fetch_url = "https://{$host}/vectors/fetch"; | |
| 668 | + | |
| 669 | + $fetch_data = array( | |
| 670 | + 'ids' => $vector_ids | |
| 671 | + ); | |
| 672 | + | |
| 688 | 673 | if (!empty($namespace)) { |
| 689 | - $fetch_query[] = 'namespace=' . rawurlencode($namespace); | |
| 674 | + $fetch_data['namespace'] = $namespace; | |
| 690 | 675 | } |
| 691 | 676 | |
| 692 | - $response = wp_remote_get("https://{$host}/vectors/fetch?" . implode('&', $fetch_query), array( | |
| 677 | + $response = wp_remote_post($fetch_url, array( | |
| 693 | 678 | 'headers' => array( |
| 694 | 679 | 'Api-Key' => $api_key, |
| 695 | - 'accept' => 'application/json' | |
| 680 | + 'Content-Type' => 'application/json' | |
| 696 | 681 | ), |
| 682 | + 'body' => json_encode($fetch_data), | |
| 697 | 683 | 'timeout' => 15 |
| 698 | 684 | )); |
| 699 | 685 | |
| 700 | - // Transport/API failure is distinct from "nothing matched": return null so | |
| 701 | - // the caller can tell the difference (plan 793b82's empty-vs-failed rule). | |
| 702 | 686 | if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) { |
| 703 | - return null; | |
| 687 | + return array('data' => array(), 'total' => 0); | |
| 704 | 688 | } |
| 705 | 689 | |
| 706 | 690 | $body = wp_remote_retrieve_body($response); |
| 707 | 691 | $data = json_decode($body, true); |
| @@ -1122,45 +1106,31 @@ | ||
| 1122 | 1106 | if (empty($vector_id_map)) { |
| 1123 | 1107 | return array(); |
| 1124 | 1108 | } |
| 1125 | 1109 | |
| 1126 | - // Batch check Pinecone using fetch API. | |
| 1127 | - // NOTE (plan 793b82): /vectors/fetch is a GET endpoint with the ids | |
| 1128 | - // repeated in the query string (ids=a&ids=b); the old POST here was | |
| 1129 | - // answered 200-with-an-empty-body, so this scan saw NOTHING as indexed | |
| 1130 | - // and callers re-embedded content Pinecone already had. Chunk size 100: | |
| 1131 | - // measured on a live serverless index, ~9KB of URL is accepted and | |
| 1132 | - // ~18KB draws HTTP 414, so 100 32-char ids (~4KB) leaves real margin. | |
| 1133 | - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; | |
| 1110 | + // Batch check Pinecone using fetch API (max 1000 IDs per request) | |
| 1134 | 1111 | $all_vector_ids = array_keys($vector_id_map); |
| 1135 | - $chunks = array_chunk($all_vector_ids, 100); | |
| 1112 | + $chunks = array_chunk($all_vector_ids, 1000); | |
| 1136 | 1113 | $processed_data = array(); |
| 1137 | 1114 | |
| 1138 | 1115 | foreach ($chunks as $chunk) { |
| 1139 | - $fetch_query = array(); | |
| 1140 | - foreach ($chunk as $fetch_vid) { | |
| 1141 | - $fetch_query[] = 'ids=' . rawurlencode($fetch_vid); | |
| 1142 | - } | |
| 1143 | - if (!empty($namespace)) { | |
| 1144 | - $fetch_query[] = 'namespace=' . rawurlencode($namespace); | |
| 1145 | - } | |
| 1116 | + $fetch_url = "https://{$host}/vectors/fetch"; | |
| 1146 | 1117 | |
| 1147 | - $response = wp_remote_get("https://{$host}/vectors/fetch?" . implode('&', $fetch_query), array( | |
| 1118 | + $response = wp_remote_post($fetch_url, array( | |
| 1148 | 1119 | 'headers' => array( |
| 1149 | 1120 | 'Api-Key' => $api_key, |
| 1150 | - 'accept' => 'application/json' | |
| 1121 | + 'Content-Type' => 'application/json' | |
| 1151 | 1122 | ), |
| 1123 | + 'body' => json_encode(array('ids' => $chunk)), | |
| 1152 | 1124 | 'timeout' => 30 |
| 1153 | 1125 | )); |
| 1154 | 1126 | |
| 1155 | 1127 | if (is_wp_error($response)) { |
| 1156 | - error_log('MxChat Pinecone: processed-content scan /vectors/fetch GET failed: ' . $response->get_error_message()); | |
| 1157 | 1128 | continue; |
| 1158 | 1129 | } |
| 1159 | 1130 | |
| 1160 | 1131 | $response_code = wp_remote_retrieve_response_code($response); |
| 1161 | 1132 | if ($response_code !== 200) { |
| 1162 | - error_log('MxChat Pinecone: processed-content scan /vectors/fetch GET returned HTTP ' . $response_code); | |
| 1163 | 1133 | continue; |
| 1164 | 1134 | } |
| 1165 | 1135 | |
| 1166 | 1136 | $body = wp_remote_retrieve_body($response); |
| @@ -1343,9 +1313,8 @@ | ||
| 1343 | 1313 | */ |
| 1344 | 1314 | public function fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) { |
| 1345 | 1315 | $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? ''; |
| 1346 | 1316 | $host = $pinecone_options['mxchat_pinecone_host'] ?? ''; |
| 1347 | - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; | |
| 1348 | 1317 | |
| 1349 | 1318 | if (empty($api_key) || empty($host) || empty($vector_ids)) { |
| 1350 | 1319 | return array(); |
| 1351 | 1320 | } |
| @@ -1350,53 +1319,44 @@ | ||
| 1350 | 1319 | return array(); |
| 1351 | 1320 | } |
| 1352 | 1321 | |
| 1353 | 1322 | try { |
| 1354 | - // NOTE (plan 793b82): /vectors/fetch is a GET endpoint with the ids | |
| 1355 | - // repeated in the query string; the old POST here was answered | |
| 1356 | - // 200-with-an-empty-body. Chunked at 100 ids to stay well under the | |
| 1357 | - // measured HTTP 414 URL-length boundary. | |
| 1358 | - $vectors = array(); | |
| 1359 | - foreach (array_chunk(array_values($vector_ids), 100) as $chunk) { | |
| 1360 | - $fetch_query = array(); | |
| 1361 | - foreach ($chunk as $fetch_vid) { | |
| 1362 | - $fetch_query[] = 'ids=' . rawurlencode($fetch_vid); | |
| 1363 | - } | |
| 1364 | - if (!empty($namespace)) { | |
| 1365 | - $fetch_query[] = 'namespace=' . rawurlencode($namespace); | |
| 1366 | - } | |
| 1323 | + $fetch_url = "https://{$host}/vectors/fetch"; | |
| 1367 | 1324 | |
| 1368 | - $response = wp_remote_get("https://{$host}/vectors/fetch?" . implode('&', $fetch_query), array( | |
| 1369 | - 'headers' => array( | |
| 1370 | - 'Api-Key' => $api_key, | |
| 1371 | - 'accept' => 'application/json' | |
| 1372 | - ), | |
| 1373 | - 'timeout' => 30 | |
| 1374 | - )); | |
| 1325 | + // Pinecone fetch API allows fetching specific vectors by ID | |
| 1326 | + $fetch_data = array( | |
| 1327 | + 'ids' => array_values($vector_ids) | |
| 1328 | + ); | |
| 1375 | 1329 | |
| 1376 | - if (is_wp_error($response)) { | |
| 1377 | - error_log('MxChat Pinecone: fetch_pinecone_vectors_by_ids GET failed: ' . $response->get_error_message()); | |
| 1378 | - continue; | |
| 1379 | - } | |
| 1330 | + $response = wp_remote_post($fetch_url, array( | |
| 1331 | + 'headers' => array( | |
| 1332 | + 'Api-Key' => $api_key, | |
| 1333 | + 'Content-Type' => 'application/json' | |
| 1334 | + ), | |
| 1335 | + 'body' => json_encode($fetch_data), | |
| 1336 | + 'timeout' => 30 | |
| 1337 | + )); | |
| 1380 | 1338 | |
| 1381 | - if (wp_remote_retrieve_response_code($response) !== 200) { | |
| 1382 | - error_log('MxChat Pinecone: fetch_pinecone_vectors_by_ids GET returned HTTP ' . wp_remote_retrieve_response_code($response)); | |
| 1383 | - continue; | |
| 1384 | - } | |
| 1339 | + if (is_wp_error($response)) { | |
| 1340 | + return array(); | |
| 1341 | + } | |
| 1385 | 1342 | |
| 1386 | - $data = json_decode(wp_remote_retrieve_body($response), true); | |
| 1387 | - if (isset($data['vectors']) && is_array($data['vectors'])) { | |
| 1388 | - $vectors += $data['vectors']; | |
| 1389 | - } | |
| 1343 | + $response_code = wp_remote_retrieve_response_code($response); | |
| 1344 | + | |
| 1345 | + if ($response_code !== 200) { | |
| 1346 | + return array(); | |
| 1390 | 1347 | } |
| 1391 | 1348 | |
| 1392 | - if (empty($vectors)) { | |
| 1349 | + $body = wp_remote_retrieve_body($response); | |
| 1350 | + $data = json_decode($body, true); | |
| 1351 | + | |
| 1352 | + if (!isset($data['vectors'])) { | |
| 1393 | 1353 | return array(); |
| 1394 | 1354 | } |
| 1395 | 1355 | |
| 1396 | 1356 | $processed_data = array(); |
| 1397 | 1357 | |
| 1398 | - foreach ($vectors as $vector_id => $vector_data) { | |
| 1358 | + foreach ($data['vectors'] as $vector_id => $vector_data) { | |
| 1399 | 1359 | $metadata = $vector_data['metadata'] ?? array(); |
| 1400 | 1360 | $source_url = $metadata['source_url'] ?? ''; |
| 1401 | 1361 | |
| 1402 | 1362 | if (!empty($source_url)) { |
| @@ -1439,9 +1399,8 @@ | ||
| 1439 | 1399 | */ |
| 1440 | 1400 | public function mxchat_delete_all_from_pinecone($pinecone_options, $content_type_filter = '') { |
| 1441 | 1401 | $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? ''; |
| 1442 | 1402 | $host = $pinecone_options['mxchat_pinecone_host'] ?? ''; |
| 1443 | - $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? ''; | |
| 1444 | 1403 | |
| 1445 | 1404 | if (empty($api_key) || empty($host)) { |
| 1446 | 1405 | return array( |
| 1447 | 1406 | 'success' => false, |
| @@ -1486,9 +1445,9 @@ | ||
| 1486 | 1445 | $batch_size = 100; |
| 1487 | 1446 | $batches = array_chunk($vector_ids, $batch_size); |
| 1488 | 1447 | |
| 1489 | 1448 | foreach ($batches as $batch) { |
| 1490 | - $result = $this->mxchat_delete_pinecone_batch($batch, $api_key, $host, $namespace); | |
| 1449 | + $result = $this->mxchat_delete_pinecone_batch($batch, $api_key, $host); | |
| 1491 | 1450 | if ($result['success']) { |
| 1492 | 1451 | $total_deleted += count($batch); |
| 1493 | 1452 | } else { |
| 1494 | 1453 | $failed_batches++; |
| @@ -1529,9 +1488,9 @@ | ||
| 1529 | 1488 | |
| 1530 | 1489 | /** |
| 1531 | 1490 | * Deletes batch of vectors from Pinecone database |
| 1532 | 1491 | */ |
| 1533 | - public function mxchat_delete_pinecone_batch($vector_ids, $api_key, $host, $namespace = '') { | |
| 1492 | + public function mxchat_delete_pinecone_batch($vector_ids, $api_key, $host) { | |
| 1534 | 1493 | // Build the API endpoint |
| 1535 | 1494 | $api_endpoint = "https://{$host}/vectors/delete"; |
| 1536 | 1495 | |
| 1537 | 1496 | // Prepare the request body with the IDs |
| @@ -1538,13 +1497,8 @@ | ||
| 1538 | 1497 | $request_body = array( |
| 1539 | 1498 | 'ids' => $vector_ids |
| 1540 | 1499 | ); |
| 1541 | 1500 | |
| 1542 | - // Add namespace if provided — omitting it targets the default namespace | |
| 1543 | - if (!empty($namespace)) { | |
| 1544 | - $request_body['namespace'] = $namespace; | |
| 1545 | - } | |
| 1546 | - | |
| 1547 | 1501 | // Make the deletion request |
| 1548 | 1502 | $response = wp_remote_post($api_endpoint, array( |
| 1549 | 1503 | 'headers' => array( |
| 1550 | 1504 | 'Api-Key' => $api_key, |
| @@ -1725,8 +1679,52 @@ | ||
| 1725 | 1679 | 'success' => true, |
| 1726 | 1680 | 'message' => 'Vector deleted successfully from Pinecone' |
| 1727 | 1681 | ); |
| 1728 | 1682 | } |
| 1683 | + | |
| 1684 | + /** | |
| 1685 | + * Deletes data from Pinecone using a source URL | |
| 1686 | + */ | |
| 1687 | + public function mxchat_delete_from_pinecone_by_url($source_url, $pinecone_options) { | |
| 1688 | + $host = $pinecone_options['mxchat_pinecone_host'] ?? ''; | |
| 1689 | + $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? ''; | |
| 1690 | + | |
| 1691 | + if (empty($host) || empty($api_key)) { | |
| 1692 | + //error_log('MXChat: Pinecone deletion failed - missing configuration'); | |
| 1693 | + return false; | |
| 1694 | + } | |
| 1695 | + | |
| 1696 | + $api_endpoint = "https://{$host}/vectors/delete"; | |
| 1697 | + $vector_id = md5($source_url); | |
| 1698 | + | |
| 1699 | + $request_body = array( | |
| 1700 | + 'ids' => array($vector_id) | |
| 1701 | + ); | |
| 1702 | + | |
| 1703 | + $response = wp_remote_post($api_endpoint, array( | |
| 1704 | + 'headers' => array( | |
| 1705 | + 'Api-Key' => $api_key, | |
| 1706 | + 'accept' => 'application/json', | |
| 1707 | + 'content-type' => 'application/json' | |
| 1708 | + ), | |
| 1709 | + 'body' => wp_json_encode($request_body), | |
| 1710 | + 'timeout' => 30 | |
| 1711 | + )); | |
| 1712 | + | |
| 1713 | + if (is_wp_error($response)) { | |
| 1714 | + //error_log('MXChat: Pinecone deletion error - ' . $response->get_error_message()); | |
| 1715 | + return false; | |
| 1716 | + } | |
| 1717 | + | |
| 1718 | + $response_code = wp_remote_retrieve_response_code($response); | |
| 1719 | + if ($response_code !== 200) { | |
| 1720 | + //error_log('MXChat: Pinecone deletion failed with status ' . $response_code); | |
| 1721 | + return false; | |
| 1722 | + } | |
| 1723 | + | |
| 1724 | + return true; | |
| 1725 | + } | |
| 1726 | + | |
| 1729 | 1727 | |
| 1730 | 1728 | /** |
| 1731 | 1729 | * Deletes data from Pinecone index using API key |
| 1732 | 1730 | */ |
