| @@ -42,15 +42,21 @@ | ||
| 42 | 42 | |
| 43 | 43 | const API_VERSION = '2026-07'; |
| 44 | 44 | const TEXT_FIELD = 'text'; |
| 45 | 45 | const VECTOR_FIELD = 'embedding'; |
| 46 | + const CODES_FIELD = 'codes'; // plan 3e83e4: normalised part-number tokens stored beside the text | |
| 47 | + const CODES_MAX = 60; // per document — a long spec sheet must not bloat the 40 KB metadata cap | |
| 48 | + const CODE_MIN_LEN = 4; // shorter tokens would match everything | |
| 49 | + const CODE_MAX_LEN = 32; | |
| 46 | 50 | const MIGRATION_KEY = 'mxchat_pinecone_docs_migration'; |
| 47 | 51 | const UPSERT_BATCH = 40; // 1536-dim vectors are ~30 KB of JSON each; the request cap is 2 MB |
| 48 | 52 | const MIGRATE_BATCH = 40; |
| 53 | + const MIGRATE_STEP_SECONDS = 12; // pages copied per AJAX step until this much wall time is used (3e83e4) | |
| 54 | + const MIGRATE_RETRIES = 3; // attempts per Pinecone call inside a step, 1 s / 2 s / 4 s apart | |
| 49 | 55 | const OPTION = 'mxchat_pinecone_addon_options'; |
| 50 | 56 | |
| 51 | 57 | /** Fields the retrieval and listing paths ask for (never the vector unless needed). */ |
| 52 | - const RECORD_FIELDS = array('text', 'source_url', 'type', 'bot_id', 'is_chunked', 'chunk_index', 'total_chunks', 'parent_url_hash', 'role_restriction', 'created_at', 'last_updated'); | |
| 58 | + const RECORD_FIELDS = array('text', 'source_url', 'title', 'type', 'bot_id', 'is_chunked', 'chunk_index', 'total_chunks', 'parent_url_hash', 'role_restriction', 'created_at', 'last_updated'); | |
| 53 | 59 | |
| 54 | 60 | public static function clouds() { |
| 55 | 61 | return array('aws' => 'AWS', 'gcp' => 'GCP', 'azure' => 'Azure'); |
| 56 | 62 | } |
| @@ -252,8 +258,17 @@ | ||
| 252 | 258 | } |
| 253 | 259 | } |
| 254 | 260 | $doc[self::TEXT_FIELD] = (string) $text; |
| 255 | 261 | $doc[self::VECTOR_FIELD] = array_values(array_map('floatval', $embedding)); |
| 262 | + // 3e83e4: the normalised code set travels with every write (new content | |
| 263 | + // and copied records alike) so a part number typed without its | |
| 264 | + // separators can still reach the record — see index_codes(). | |
| 265 | + $codes = self::index_codes((string) $text); | |
| 266 | + if (!empty($codes)) { | |
| 267 | + $doc[self::CODES_FIELD] = $codes; | |
| 268 | + } else { | |
| 269 | + unset($doc[self::CODES_FIELD]); | |
| 270 | + } | |
| 256 | 271 | return $doc; |
| 257 | 272 | } |
| 258 | 273 | |
| 259 | 274 | /** |
| @@ -686,8 +701,12 @@ | ||
| 686 | 701 | /** |
| 687 | 702 | * Text-match filter that requires at least one of the codes. Hyphenated or |
| 688 | 703 | * dotted codes are phrase-matched (the tokenizer splits them, and "20" on |
| 689 | 704 | * its own would match everything); single runs use $match_any. |
| 705 | + * | |
| 706 | + * 3e83e4: a second leg matches the NORMALISED code set stored on each | |
| 707 | + * document (codes $in …), which is what lets "R165321320" reach a record | |
| 708 | + * that only ever says "R1653" — see query_codes() for the expansion. | |
| 690 | 709 | */ |
| 691 | 710 | public static function keyword_filter(array $tokens) { |
| 692 | 711 | $clauses = array(); |
| 693 | 712 | foreach ($tokens as $token) { |
| @@ -693,8 +712,12 @@ | ||
| 693 | 712 | foreach ($tokens as $token) { |
| 694 | 713 | $op = preg_match('/[.\-]/', $token) ? '$match_phrase' : '$match_any'; |
| 695 | 714 | $clauses[] = array(self::TEXT_FIELD => array($op => $token)); |
| 696 | 715 | } |
| 716 | + $query_codes = self::query_codes($tokens); | |
| 717 | + if (!empty($query_codes)) { | |
| 718 | + $clauses[] = array(self::CODES_FIELD => array('$in' => $query_codes)); | |
| 719 | + } | |
| 697 | 720 | if (empty($clauses)) { |
| 698 | 721 | return null; |
| 699 | 722 | } |
| 700 | 723 | return count($clauses) === 1 ? $clauses[0] : array('$or' => $clauses); |
| @@ -699,8 +722,155 @@ | ||
| 699 | 722 | } |
| 700 | 723 | return count($clauses) === 1 ? $clauses[0] : array('$or' => $clauses); |
| 701 | 724 | } |
| 702 | 725 | |
| 726 | + /** Upper-case, separators stripped: "R1653 213 20" → "R165321320", "hgh25ca" → "HGH25CA". */ | |
| 727 | + public static function normalize_code($code) { | |
| 728 | + return strtoupper(preg_replace('/[\s.\-_\/]+/u', '', (string) $code)); | |
| 729 | + } | |
| 730 | + | |
| 731 | + /** | |
| 732 | + * Code-like runs in a piece of text, INCLUDING digit groups that follow a | |
| 733 | + * code separated by a space, hyphen or dot ("R1653 213 20" is one number | |
| 734 | + * the way Rexroth prints it). Returns [ [head, full], … ] where head is | |
| 735 | + * the first run and full is the whole joined run, both un-normalised. | |
| 736 | + */ | |
| 737 | + private static function code_runs($text) { | |
| 738 | + $runs = array(); | |
| 739 | + $text = (string) $text; | |
| 740 | + $pattern = '/(?<![A-Za-z0-9])([A-Za-z0-9]+(?:[.\-][A-Za-z0-9]+)*)((?:[ .\-]\d{2,6}(?![A-Za-z0-9])){1,3})?(?![A-Za-z0-9])/u'; | |
| 741 | + $offset = 0; | |
| 742 | + $length = strlen($text); | |
| 743 | + // 0d7bd4: walked one match at a time. Only a head that IS a code keeps | |
| 744 | + // its trailing digit groups; a plain word ("part 4471-AB-12") used to | |
| 745 | + // take "4471" as its tail and the digit-led number was never stored. | |
| 746 | + while ($offset < $length && preg_match($pattern, $text, $match, PREG_OFFSET_CAPTURE, $offset)) { | |
| 747 | + $head = rtrim($match[1][0], '.'); | |
| 748 | + $head_end = $match[1][1] + strlen($match[1][0]); | |
| 749 | + $is_code = strlen($head) >= self::CODE_MIN_LEN && preg_match('/\d/', $head); | |
| 750 | + if ($is_code) { | |
| 751 | + $has_letter = (bool) preg_match('/[A-Za-z]/', $head); | |
| 752 | + $separators = preg_match_all('/[.\-]/', $head); | |
| 753 | + if (!$has_letter && $separators < 2) { | |
| 754 | + $is_code = false; // 2024, 26.5, 10-20 | |
| 755 | + } | |
| 756 | + } | |
| 757 | + if (!$is_code) { | |
| 758 | + $offset = $head_end; | |
| 759 | + continue; | |
| 760 | + } | |
| 761 | + $tail = isset($match[2]) && $match[2][1] >= 0 ? $match[2][0] : ''; | |
| 762 | + $runs[] = array($head, $head . $tail); | |
| 763 | + $offset = $match[0][1] + strlen($match[0][0]); | |
| 764 | + } | |
| 765 | + return $runs; | |
| 766 | + } | |
| 767 | + | |
| 768 | + /** | |
| 769 | + * The normalised token set stored on a document at write time: for every | |
| 770 | + * code-like run in the text, the separator-stripped form, the leading stem | |
| 771 | + * (the run before the first separator) and the letters+digits stem | |
| 772 | + * ("HGH25" of "HGH25CA"). Unique, minimum 4 characters, capped. | |
| 773 | + */ | |
| 774 | + public static function index_codes($text) { | |
| 775 | + $codes = array(); | |
| 776 | + foreach (self::code_runs($text) as $run) { | |
| 777 | + list($head, $full) = $run; | |
| 778 | + $candidates = array(self::normalize_code($full), self::normalize_code($head)); | |
| 779 | + $stem = preg_split('/[.\-\s]/', $head)[0]; | |
| 780 | + $candidates[] = self::normalize_code($stem); | |
| 781 | + if (preg_match('/^([A-Za-z]+\d+)/', $head, $sm)) { | |
| 782 | + $candidates[] = self::normalize_code($sm[1]); | |
| 783 | + } | |
| 784 | + foreach ($candidates as $c) { | |
| 785 | + if (strlen($c) >= self::CODE_MIN_LEN && strlen($c) <= self::CODE_MAX_LEN && preg_match('/\d/', $c)) { | |
| 786 | + $codes[$c] = true; | |
| 787 | + } | |
| 788 | + } | |
| 789 | + if (count($codes) >= self::CODES_MAX) { | |
| 790 | + break; | |
| 791 | + } | |
| 792 | + } | |
| 793 | + // 0d7bd4: strval — PHP turns an all-digit array key into an int, and | |
| 794 | + // Pinecone refuses a list that mixes strings and numbers. | |
| 795 | + return array_slice(array_map('strval', array_keys($codes)), 0, self::CODES_MAX); | |
| 796 | + } | |
| 797 | + | |
| 798 | + /** | |
| 799 | + * What a question's tokens expand to for the codes $in filter: the | |
| 800 | + * stripped form, every prefix of it from 4 characters up (a stored stem | |
| 801 | + * that is a prefix of what the visitor typed then matches — "R1653" for | |
| 802 | + * "R165321320") and the letters+digits stem. Plain-language questions | |
| 803 | + * carry no tokens and return nothing, so they are unchanged. | |
| 804 | + */ | |
| 805 | + public static function query_codes(array $tokens) { | |
| 806 | + $codes = array(); | |
| 807 | + foreach ($tokens as $token) { | |
| 808 | + $full = self::normalize_code($token); | |
| 809 | + if (strlen($full) < self::CODE_MIN_LEN || !preg_match('/\d/', $full)) { | |
| 810 | + continue; | |
| 811 | + } | |
| 812 | + $full = substr($full, 0, self::CODE_MAX_LEN); | |
| 813 | + $codes[$full] = true; | |
| 814 | + for ($len = self::CODE_MIN_LEN; $len < strlen($full); $len++) { | |
| 815 | + $codes[substr($full, 0, $len)] = true; | |
| 816 | + } | |
| 817 | + if (preg_match('/^([A-Z]+\d+)/', $full, $sm) && strlen($sm[1]) >= self::CODE_MIN_LEN) { | |
| 818 | + $codes[$sm[1]] = true; | |
| 819 | + } | |
| 820 | + if (count($codes) >= 200) { | |
| 821 | + break; | |
| 822 | + } | |
| 823 | + } | |
| 824 | + return array_slice(array_map('strval', array_keys($codes)), 0, 200); | |
| 825 | + } | |
| 826 | + | |
| 827 | + /** | |
| 828 | + * Order keyword-leg hits so an exact code match outranks a prefix-only | |
| 829 | + * match: a hit is exact when its stored code set (or its text) carries the | |
| 830 | + * stripped question token itself. Dense order is kept inside each group. | |
| 831 | + * | |
| 832 | + * @return array{hits:array,exact:int,prefix:int} | |
| 833 | + */ | |
| 834 | + public static function rank_keyword_hits(array $hits, array $tokens) { | |
| 835 | + $exact_codes = array(); | |
| 836 | + foreach ($tokens as $token) { | |
| 837 | + $n = self::normalize_code($token); | |
| 838 | + if (strlen($n) >= self::CODE_MIN_LEN) { | |
| 839 | + $exact_codes[$n] = true; | |
| 840 | + } | |
| 841 | + } | |
| 842 | + $exact = array(); | |
| 843 | + $prefix = array(); | |
| 844 | + foreach ($hits as $hit) { | |
| 845 | + $meta = is_array($hit['metadata'] ?? null) ? $hit['metadata'] : array(); | |
| 846 | + $stored = is_array($meta[self::CODES_FIELD] ?? null) ? $meta[self::CODES_FIELD] : array(); | |
| 847 | + $is_exact = false; | |
| 848 | + foreach ($stored as $code) { | |
| 849 | + if (isset($exact_codes[strtoupper((string) $code)])) { | |
| 850 | + $is_exact = true; | |
| 851 | + break; | |
| 852 | + } | |
| 853 | + } | |
| 854 | + if (!$is_exact && !empty($tokens) && isset($meta[self::TEXT_FIELD])) { | |
| 855 | + $haystack = self::normalize_code($meta[self::TEXT_FIELD]); | |
| 856 | + foreach ($exact_codes as $code => $_) { | |
| 857 | + if ($haystack !== '' && strpos($haystack, $code) !== false) { | |
| 858 | + $is_exact = true; | |
| 859 | + break; | |
| 860 | + } | |
| 861 | + } | |
| 862 | + } | |
| 863 | + $hit['code_match'] = $is_exact ? 'exact' : 'prefix'; | |
| 864 | + if ($is_exact) { | |
| 865 | + $exact[] = $hit; | |
| 866 | + } else { | |
| 867 | + $prefix[] = $hit; | |
| 868 | + } | |
| 869 | + } | |
| 870 | + return array('hits' => array_merge($exact, $prefix), 'exact' => count($exact), 'prefix' => count($prefix)); | |
| 871 | + } | |
| 872 | + | |
| 703 | 873 | /** AND two filters (either may be empty). */ |
| 704 | 874 | public static function merge_filters($a, $b) { |
| 705 | 875 | $a = (!empty($a) && is_array($a)) ? $a : null; |
| 706 | 876 | $b = (!empty($b) && is_array($b)) ? $b : null; |
| @@ -825,8 +995,56 @@ | ||
| 825 | 995 | return $base + array('ok' => true, 'message' => sprintf('"%s" is a document index with full-text search (%s, %d dimensions, %s). Ready to use.', $name, $schema['language'] !== '' ? 'language ' . $schema['language'] : 'full-text search', $schema['dimension'], $schema['metric'] ?: 'cosine')); |
| 826 | 996 | } |
| 827 | 997 | |
| 828 | 998 | /** |
| 999 | + * The plain-language "that name is taken" message (3e83e4) — never a bare | |
| 1000 | + * HTTP 409. Carries a free suggestion so the UI can fill it in. | |
| 1001 | + */ | |
| 1002 | + public static function name_taken_message($name, $suggested = '') { | |
| 1003 | + $message = sprintf( | |
| 1004 | + /* translators: %s: Pinecone index name */ | |
| 1005 | + __('An index named "%s" already exists in this Pinecone project. Pick a different name, or paste that index\'s host above and press Check index instead of creating one.', 'mxchat'), | |
| 1006 | + $name | |
| 1007 | + ); | |
| 1008 | + if ($suggested !== '') { | |
| 1009 | + /* translators: %s: suggested Pinecone index name */ | |
| 1010 | + $message .= ' ' . sprintf(__('Suggested free name: %s.', 'mxchat'), $suggested); | |
| 1011 | + } | |
| 1012 | + return $message; | |
| 1013 | + } | |
| 1014 | + | |
| 1015 | + /** | |
| 1016 | + * A free index name derived from the current one: <base>-docs, then | |
| 1017 | + * <base>-docs-2, -3 … (an existing -docs suffix is not doubled). Lists the | |
| 1018 | + * project's indexes once. @return array{name:string,taken:bool,existing:array} | |
| 1019 | + */ | |
| 1020 | + public static function suggest_index_name($base, $api_key) { | |
| 1021 | + $base = strtolower(sanitize_text_field($base)); | |
| 1022 | + $base = preg_replace('/[^a-z0-9-]+/', '-', $base); | |
| 1023 | + $base = trim(preg_replace('/-{2,}/', '-', $base), '-'); | |
| 1024 | + $base = preg_replace('/-docs(-\d+)?$/', '', $base); | |
| 1025 | + if ($base === '') { | |
| 1026 | + $base = 'mxchat'; | |
| 1027 | + } | |
| 1028 | + $existing = array(); | |
| 1029 | + $res = self::control_request('GET', 'indexes', null, $api_key); | |
| 1030 | + if ($res['error'] === '' && is_array($res['data'])) { | |
| 1031 | + foreach (($res['data']['indexes'] ?? array()) as $index) { | |
| 1032 | + if (isset($index['name'])) { | |
| 1033 | + $existing[strtolower((string) $index['name'])] = true; | |
| 1034 | + } | |
| 1035 | + } | |
| 1036 | + } | |
| 1037 | + $stem = substr($base, 0, 45 - strlen('-docs-99')); | |
| 1038 | + $stem = rtrim($stem, '-'); | |
| 1039 | + $candidate = $stem . '-docs'; | |
| 1040 | + for ($n = 2; isset($existing[$candidate]) && $n < 100; $n++) { | |
| 1041 | + $candidate = $stem . '-docs-' . $n; | |
| 1042 | + } | |
| 1043 | + return array('name' => $candidate, 'taken' => isset($existing[$base]), 'existing' => array_keys($existing)); | |
| 1044 | + } | |
| 1045 | + | |
| 1046 | + /** | |
| 829 | 1047 | * Create a document index for the active embedding dimension. |
| 830 | 1048 | * @return array|WP_Error the IndexModel |
| 831 | 1049 | */ |
| 832 | 1050 | public static function create_document_index($name, $dimension, $cloud, $region, $language, $api_key) { |
| @@ -833,8 +1051,16 @@ | ||
| 833 | 1051 | $name = strtolower(sanitize_text_field($name)); |
| 834 | 1052 | if (!preg_match('/^[a-z0-9]([a-z0-9-]{0,43}[a-z0-9])?$/', $name)) { |
| 835 | 1053 | return new WP_Error('pinecone_index', 'Index names are 1-45 lowercase letters, digits and hyphens.'); |
| 836 | 1054 | } |
| 1055 | + // 3e83e4: say so in plain words BEFORE Pinecone can answer 409 — the | |
| 1056 | + // name field is prefilled with the vector index being migrated away | |
| 1057 | + // from, so the first click used to collide for exactly that customer. | |
| 1058 | + $taken = self::describe_index($name, $api_key); | |
| 1059 | + if (!is_wp_error($taken)) { | |
| 1060 | + $suggest = self::suggest_index_name($name, $api_key); | |
| 1061 | + return new WP_Error('pinecone_exists', self::name_taken_message($name, $suggest['name']), array('code' => 409, 'suggested' => $suggest['name'])); | |
| 1062 | + } | |
| 837 | 1063 | $cloud = array_key_exists($cloud, self::clouds()) ? $cloud : 'aws'; |
| 838 | 1064 | $region = sanitize_text_field($region) ?: 'us-east-1'; |
| 839 | 1065 | $language = array_key_exists($language, self::languages()) ? $language : 'en'; |
| 840 | 1066 | $dimension = (int) $dimension; |
| @@ -849,8 +1075,12 @@ | ||
| 849 | 1075 | self::VECTOR_FIELD => array('type' => 'dense_vector', 'dimension' => $dimension, 'metric' => 'cosine'), |
| 850 | 1076 | )), |
| 851 | 1077 | ); |
| 852 | 1078 | $res = self::control_request('POST', 'indexes', $body, $api_key); |
| 1079 | + if ($res['code'] === 409) { | |
| 1080 | + $suggest = self::suggest_index_name($name, $api_key); | |
| 1081 | + return new WP_Error('pinecone_exists', self::name_taken_message($name, $suggest['name']), array('code' => 409, 'suggested' => $suggest['name'])); | |
| 1082 | + } | |
| 853 | 1083 | if ($res['error'] !== '' || !is_array($res['data'])) { |
| 854 | 1084 | return new WP_Error('pinecone_api', $res['error'] !== '' ? $res['error'] : 'Empty create response', array('code' => $res['code'])); |
| 855 | 1085 | } |
| 856 | 1086 | return $res['data']; |
| @@ -870,29 +1100,55 @@ | ||
| 870 | 1100 | } |
| 871 | 1101 | |
| 872 | 1102 | /** Dimension of the active embedding model (same table the KB screen uses). */ |
| 873 | 1103 | public static function expected_dimension() { |
| 1104 | + $dimension = self::expected_dimension_checked(); | |
| 1105 | + return is_wp_error($dimension) ? 0 : (int) $dimension; | |
| 1106 | + } | |
| 1107 | + | |
| 1108 | + /** | |
| 1109 | + * 418afd: the dimension the site's embeddings really have, or why it is | |
| 1110 | + * not known. A Custom Provider embedding model is asked for its size | |
| 1111 | + * (MxChat_Utils::expected_embedding_dimension()); the standard dropdown is | |
| 1112 | + * disabled on such a site and used to size the index anyway. The filter | |
| 1113 | + * still has the last word, and can supply a number when the model could | |
| 1114 | + * not be asked. | |
| 1115 | + * | |
| 1116 | + * @param bool $force Ask a custom model again even after a recent failure (the card's buttons). | |
| 1117 | + * @return int|WP_Error | |
| 1118 | + */ | |
| 1119 | + public static function expected_dimension_checked($force = false) { | |
| 874 | 1120 | $options = get_option('mxchat_options', array()); |
| 875 | - $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002'; | |
| 876 | - $model_dimensions = array( | |
| 877 | - 'text-embedding-ada-002' => 1536, | |
| 878 | - 'text-embedding-3-small' => 1536, | |
| 879 | - 'text-embedding-3-large' => 3072, | |
| 880 | - 'voyage-2' => 1024, | |
| 881 | - 'voyage-large-2' => 1536, | |
| 882 | - 'voyage-3-large' => 2048, | |
| 883 | - 'gemini-embedding-001' => 1536, | |
| 884 | - ); | |
| 885 | - if (strpos($selected_model, 'voyage-3-large') === 0) { | |
| 886 | - return intval($options['voyage_output_dimension'] ?? 2048); | |
| 1121 | + $options = is_array($options) ? $options : array(); | |
| 1122 | + $custom = isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on'; | |
| 1123 | + $selected_model = $custom | |
| 1124 | + ? MxChat_Utils::get_selected_embedding_model($options) | |
| 1125 | + : ($options['embedding_model'] ?? 'text-embedding-ada-002'); | |
| 1126 | + $dimension = MxChat_Utils::expected_embedding_dimension($options, $force); | |
| 1127 | + if (is_wp_error($dimension)) { | |
| 1128 | + $supplied = (int) apply_filters('mxchat_pinecone_expected_dimension', 0, $selected_model); | |
| 1129 | + return $supplied > 0 ? $supplied : $dimension; | |
| 887 | 1130 | } |
| 888 | - if (strpos($selected_model, 'gemini-embedding') === 0) { | |
| 889 | - return intval($options['gemini_output_dimension'] ?? 1536); | |
| 1131 | + if (!$custom && (strpos($selected_model, 'voyage-3-large') === 0 || strpos($selected_model, 'gemini-embedding') === 0)) { | |
| 1132 | + return (int) $dimension; // adjustable-size models: the owner's own setting, never filtered (as before) | |
| 890 | 1133 | } |
| 891 | - $dimension = apply_filters('mxchat_pinecone_expected_dimension', $model_dimensions[$selected_model] ?? 1536, $selected_model); | |
| 892 | - return (int) $dimension; | |
| 1134 | + return (int) apply_filters('mxchat_pinecone_expected_dimension', $dimension, $selected_model); | |
| 893 | 1135 | } |
| 894 | 1136 | |
| 1137 | + /** Plain-language refusal for the card when a custom model's size is unknown. (Plain __(): the card's JS text-escapes it.) */ | |
| 1138 | + public static function dimension_unknown_message(WP_Error $error, $action) { | |
| 1139 | + $reason = trim($error->get_error_message()); | |
| 1140 | + $tail = $action === 'create' | |
| 1141 | + ? __('The index was not created.', 'mxchat') | |
| 1142 | + : __('The index was not checked.', 'mxchat'); | |
| 1143 | + return sprintf( | |
| 1144 | + /* translators: 1: the error the embedding endpoint returned, 2: what did not happen */ | |
| 1145 | + __('Could not determine the dimension of your Custom Provider embedding model (%1$s). %2$s Check the Custom Provider embedding settings, then try again.', 'mxchat'), | |
| 1146 | + $reason !== '' ? $reason : __('no answer from the embedding endpoint', 'mxchat'), | |
| 1147 | + $tail | |
| 1148 | + ); | |
| 1149 | + } | |
| 1150 | + | |
| 895 | 1151 | // ===================================================================== |
| 896 | 1152 | // Options (raw writes — the registered sanitizer rebuilds the array and |
| 897 | 1153 | // would flip mxchat_use_pinecone on any isset key; the autosave handler |
| 898 | 1154 | // writes this option the same raw way for the same reason) |
| @@ -937,8 +1193,72 @@ | ||
| 937 | 1193 | public static function migration_clear() { |
| 938 | 1194 | delete_transient(self::MIGRATION_KEY); |
| 939 | 1195 | } |
| 940 | 1196 | |
| 1197 | + /** | |
| 1198 | + * 3e83e4: is there an unfinished copy INTO this host? The settings save | |
| 1199 | + * and the autosave refuse to switch the live chatbot onto a document index | |
| 1200 | + * while its copy is still running (unless the owner ticks Switch anyway), | |
| 1201 | + * so a production bot never answers from a half-filled index. | |
| 1202 | + * | |
| 1203 | + * @return array|null the running state, or null when nothing blocks | |
| 1204 | + */ | |
| 1205 | + public static function migration_blocks_switch($host) { | |
| 1206 | + $host = strtolower(trim(str_replace(array('https://', 'http://'), '', (string) $host), '/')); | |
| 1207 | + $state = self::migration_get(); | |
| 1208 | + if ($host === '' || empty($state) || ($state['status'] ?? '') !== 'running') { | |
| 1209 | + return null; | |
| 1210 | + } | |
| 1211 | + if (strtolower((string) ($state['target_host'] ?? '')) !== $host) { | |
| 1212 | + return null; | |
| 1213 | + } | |
| 1214 | + return $state; | |
| 1215 | + } | |
| 1216 | + | |
| 1217 | + /** Plain-language line for the switch gate: "The copy into X is still running (481 of 4,503 records)…". */ | |
| 1218 | + public static function switch_blocked_message(array $state) { | |
| 1219 | + return sprintf( | |
| 1220 | + /* translators: 1: document index name, 2: records copied, 3: records in total */ | |
| 1221 | + __('The copy into "%1$s" is still running (%2$s of %3$s records). Let it finish before switching the chatbot to the document index, or tick Switch anyway to move it now and copy the rest afterwards.', 'mxchat'), | |
| 1222 | + (string) ($state['target_name'] ?: $state['target_host']), | |
| 1223 | + number_format_i18n((int) ($state['copied'] ?? 0)), | |
| 1224 | + number_format_i18n((int) ($state['total'] ?? 0)) | |
| 1225 | + ); | |
| 1226 | + } | |
| 1227 | + | |
| 1228 | + /** | |
| 1229 | + * Run one Pinecone call up to MIGRATE_RETRIES times, 1 s / 2 s / 4 s apart, | |
| 1230 | + * so a single dropped connection or a 5xx does not end the copy. | |
| 1231 | + * | |
| 1232 | + * @param callable $call returns array|true|WP_Error | |
| 1233 | + * @return array{result:mixed,attempts:int} | |
| 1234 | + */ | |
| 1235 | + private static function with_retry(callable $call) { | |
| 1236 | + $result = null; | |
| 1237 | + $attempts = 0; | |
| 1238 | + for ($attempt = 1; $attempt <= self::MIGRATE_RETRIES; $attempt++) { | |
| 1239 | + $attempts = $attempt; | |
| 1240 | + $result = call_user_func($call); | |
| 1241 | + if (!is_wp_error($result)) { | |
| 1242 | + return array('result' => $result, 'attempts' => $attempts); | |
| 1243 | + } | |
| 1244 | + if ($attempt < self::MIGRATE_RETRIES) { | |
| 1245 | + sleep((int) pow(2, $attempt - 1)); | |
| 1246 | + } | |
| 1247 | + } | |
| 1248 | + return array('result' => $result, 'attempts' => $attempts); | |
| 1249 | + } | |
| 1250 | + | |
| 1251 | + /** Seconds one AJAX step may spend copying pages (bounded by PHP's own limit). */ | |
| 1252 | + private static function step_budget() { | |
| 1253 | + $budget = (float) apply_filters('mxchat_pinecone_migrate_step_seconds', self::MIGRATE_STEP_SECONDS); | |
| 1254 | + $limit = (int) ini_get('max_execution_time'); | |
| 1255 | + if ($limit > 0 && $limit - 8 < $budget) { | |
| 1256 | + $budget = max(3, $limit - 8); | |
| 1257 | + } | |
| 1258 | + return $budget; | |
| 1259 | + } | |
| 1260 | + | |
| 941 | 1261 | /** Classic GET /vectors/list page. @return array{ids:array,next:string}|WP_Error */ |
| 942 | 1262 | public static function classic_list_page($host, $api_key, $namespace, $limit, $token) { |
| 943 | 1263 | $params = array('limit' => max(1, min(100, (int) $limit))); |
| 944 | 1264 | if ((string) $namespace !== '') { |
| @@ -1015,12 +1335,133 @@ | ||
| 1015 | 1335 | } |
| 1016 | 1336 | return $data; |
| 1017 | 1337 | } |
| 1018 | 1338 | |
| 1339 | + /** Display key of a namespace in the per-namespace map — Pinecone's console names the default one __default__. */ | |
| 1340 | + public static function ns_key($namespace) { | |
| 1341 | + return (string) $namespace === '' ? '__default__' : (string) $namespace; | |
| 1342 | + } | |
| 1343 | + | |
| 1344 | + /** Raw namespace for a display key (the default namespace is '' on the wire). */ | |
| 1345 | + public static function ns_raw($key) { | |
| 1346 | + return (string) $key === '__default__' ? '' : (string) $key; | |
| 1347 | + } | |
| 1348 | + | |
| 1349 | + /** Classic describe_index_stats → [raw namespace => record count], the default namespace as ''. */ | |
| 1350 | + private static function namespace_counts(array $stats) { | |
| 1351 | + $counts = array(); | |
| 1352 | + foreach ((array) ($stats['namespaces'] ?? array()) as $ns_name => $ns_info) { | |
| 1353 | + $counts[self::ns_raw($ns_name)] = (int) ($ns_info['vectorCount'] ?? $ns_info['recordCount'] ?? 0); | |
| 1354 | + } | |
| 1355 | + return $counts; | |
| 1356 | + } | |
| 1357 | + | |
| 1019 | 1358 | /** |
| 1359 | + * e50d2e: the order namespaces are copied in when every namespace is | |
| 1360 | + * asked for — the default namespace first, then the rest by name. Only | |
| 1361 | + * namespaces that hold records are queued. | |
| 1362 | + */ | |
| 1363 | + private static function namespace_queue(array $counts) { | |
| 1364 | + $queue = array(); | |
| 1365 | + if (!empty($counts[''])) { | |
| 1366 | + $queue[] = ''; | |
| 1367 | + } | |
| 1368 | + $named = array(); | |
| 1369 | + foreach ($counts as $ns => $count) { | |
| 1370 | + if ($ns !== '' && $count > 0) { | |
| 1371 | + $named[] = (string) $ns; | |
| 1372 | + } | |
| 1373 | + } | |
| 1374 | + sort($named, SORT_STRING); | |
| 1375 | + return array_merge($queue, $named); | |
| 1376 | + } | |
| 1377 | + | |
| 1378 | + private static function new_namespace_entry($namespace, $total, $status = 'queued') { | |
| 1379 | + return array('namespace' => (string) $namespace, 'total' => (int) $total, 'copied' => 0, 'status' => $status, 'target_count' => null); | |
| 1380 | + } | |
| 1381 | + | |
| 1382 | + /** | |
| 1383 | + * e50d2e: the per-namespace map of a migration state — display key → | |
| 1384 | + * namespace / total / copied / status / target_count. A state written | |
| 1385 | + * before this build copied one namespace and carries no map, so one is | |
| 1386 | + * synthesised from its single namespace; the Delete guard and the step | |
| 1387 | + * loop read every state through here. | |
| 1388 | + */ | |
| 1389 | + public static function per_namespace_map(array $state) { | |
| 1390 | + if (!empty($state['per_namespace']) && is_array($state['per_namespace'])) { | |
| 1391 | + return $state['per_namespace']; | |
| 1392 | + } | |
| 1393 | + $namespace = (string) ($state['namespace'] ?? ''); | |
| 1394 | + $status = ($state['status'] ?? '') === 'running' ? 'running' : (in_array($state['status'] ?? '', array('done', 'source_deleted'), true) ? 'done' : 'queued'); | |
| 1395 | + $entry = self::new_namespace_entry($namespace, (int) ($state['total'] ?? 0), $status); | |
| 1396 | + $entry['copied'] = (int) ($state['copied'] ?? 0); | |
| 1397 | + $entry['target_count'] = isset($state['target_count']) ? $state['target_count'] : null; | |
| 1398 | + return array(self::ns_key($namespace) => $entry); | |
| 1399 | + } | |
| 1400 | + | |
| 1401 | + /** | |
| 1402 | + * e50d2e: widen a single-namespace copy — running or finished — to every | |
| 1403 | + * namespace of the source that holds records. Namespaces already copied | |
| 1404 | + * keep their entry, the rest queue behind the current one, the total | |
| 1405 | + * becomes the sum, and "other namespaces" empties because none are left | |
| 1406 | + * out any more. A finished copy with nothing left to queue stays done. | |
| 1407 | + * | |
| 1408 | + * @return array|WP_Error state | |
| 1409 | + */ | |
| 1410 | + private static function extend_to_all_namespaces(array $state, $api_key) { | |
| 1411 | + $stats = self::classic_stats((string) ($state['source_host'] ?? ''), $api_key); | |
| 1412 | + if (is_wp_error($stats)) { | |
| 1413 | + return $stats; | |
| 1414 | + } | |
| 1415 | + $ns_counts = self::namespace_counts($stats); | |
| 1416 | + $map = self::per_namespace_map($state); | |
| 1417 | + $queue = array_values((array) ($state['queue'] ?? array())); | |
| 1418 | + $current = isset($state['current']) ? (string) $state['current'] : (string) ($state['namespace'] ?? ''); | |
| 1419 | + foreach (self::namespace_queue($ns_counts) as $ns) { | |
| 1420 | + if (isset($map[self::ns_key($ns)]) || $ns === $current || in_array($ns, $queue, true)) { | |
| 1421 | + continue; | |
| 1422 | + } | |
| 1423 | + $map[self::ns_key($ns)] = self::new_namespace_entry($ns, $ns_counts[$ns]); | |
| 1424 | + $queue[] = $ns; | |
| 1425 | + } | |
| 1426 | + $total = 0; | |
| 1427 | + foreach ($map as $entry) { | |
| 1428 | + $total += (int) ($entry['total'] ?? 0); | |
| 1429 | + } | |
| 1430 | + $state['per_namespace'] = $map; | |
| 1431 | + $state['queue'] = $queue; | |
| 1432 | + $state['current'] = $current; | |
| 1433 | + $state['namespace'] = $current; | |
| 1434 | + $state['total'] = $total; | |
| 1435 | + $state['total_all'] = (int) ($stats['totalVectorCount'] ?? $total); | |
| 1436 | + $state['other_namespaces'] = array(); | |
| 1437 | + $state['all_namespaces'] = true; | |
| 1438 | + if (($state['status'] ?? '') === 'done' && !empty($queue)) { | |
| 1439 | + $next = array_shift($queue); | |
| 1440 | + $state['queue'] = $queue; | |
| 1441 | + $state['current'] = $next; | |
| 1442 | + $state['namespace'] = $next; | |
| 1443 | + $state['token'] = ''; | |
| 1444 | + $state['status'] = 'running'; | |
| 1445 | + $state['error'] = ''; | |
| 1446 | + $state['error_at'] = 0; | |
| 1447 | + $state['per_namespace'][self::ns_key($next)]['status'] = 'running'; | |
| 1448 | + unset($state['finished']); | |
| 1449 | + } | |
| 1450 | + return $state; | |
| 1451 | + } | |
| 1452 | + | |
| 1453 | + /** | |
| 1020 | 1454 | * Validate and start (or resume) a migration. @return array|WP_Error state |
| 1455 | + * | |
| 1456 | + * e50d2e: $all_namespaces copies every namespace of the source that holds | |
| 1457 | + * records — the default namespace first, then the rest by name — each into | |
| 1458 | + * the same-named namespace of the document index, one after another, in | |
| 1459 | + * one run. Without it the copy is exactly 3e83e4's: the one configured | |
| 1460 | + * namespace. Asking for every namespace on a copy that is already running | |
| 1461 | + * or finished keeps what was copied and queues the rest. | |
| 1021 | 1462 | */ |
| 1022 | - public static function migrate_start($source_host, array $target_cfg, $restart = false) { | |
| 1463 | + public static function migrate_start($source_host, array $target_cfg, $restart = false, $all_namespaces = false) { | |
| 1023 | 1464 | $source_host = strtolower(trim(str_replace(array('https://', 'http://'), '', (string) $source_host), '/')); |
| 1024 | 1465 | if ($source_host === '') { |
| 1025 | 1466 | return new WP_Error('migrate', 'Enter the host of the vector index to copy from.'); |
| 1026 | 1467 | } |
| @@ -1029,21 +1470,79 @@ | ||
| 1029 | 1470 | } |
| 1030 | 1471 | if ($source_host === strtolower($target_cfg['host'])) { |
| 1031 | 1472 | return new WP_Error('migrate', 'The source and the document index are the same host.'); |
| 1032 | 1473 | } |
| 1033 | - $existing = self::migration_get(); | |
| 1034 | - if (!$restart && !empty($existing) && ($existing['source_host'] ?? '') === $source_host && ($existing['target_host'] ?? '') === $target_cfg['host'] && ($existing['status'] ?? '') === 'running') { | |
| 1035 | - return $existing; // resume | |
| 1474 | + $existing = self::migration_get(); | |
| 1475 | + $same_copy = !empty($existing) && ($existing['source_host'] ?? '') === $source_host && strtolower((string) ($existing['target_host'] ?? '')) === strtolower($target_cfg['host']); | |
| 1476 | + if (!$restart && $same_copy && ($existing['status'] ?? '') === 'running') { | |
| 1477 | + // Resume from the recorded cursor — whether the last step ended | |
| 1478 | + // cleanly or with an error (3e83e4: a second click continues, | |
| 1479 | + // it never starts over unless asked). | |
| 1480 | + $existing['error'] = ''; | |
| 1481 | + if ($all_namespaces && empty($existing['all_namespaces'])) { | |
| 1482 | + $existing = self::extend_to_all_namespaces($existing, $target_cfg['api_key']); | |
| 1483 | + if (is_wp_error($existing)) { | |
| 1484 | + return $existing; | |
| 1485 | + } | |
| 1486 | + } | |
| 1487 | + return self::migration_set($existing); | |
| 1036 | 1488 | } |
| 1489 | + if (!$restart && $same_copy && $all_namespaces && ($existing['status'] ?? '') === 'done') { | |
| 1490 | + $extended = self::extend_to_all_namespaces($existing, $target_cfg['api_key']); | |
| 1491 | + if (is_wp_error($extended)) { | |
| 1492 | + return $extended; | |
| 1493 | + } | |
| 1494 | + return self::migration_set($extended); | |
| 1495 | + } | |
| 1037 | 1496 | $stats = self::classic_stats($source_host, $target_cfg['api_key']); |
| 1038 | 1497 | if (is_wp_error($stats)) { |
| 1039 | 1498 | return $stats; |
| 1040 | 1499 | } |
| 1041 | 1500 | $namespace = (string) ($target_cfg['namespace'] ?? ''); |
| 1042 | - if ($namespace !== '') { | |
| 1043 | - $total = isset($stats['namespaces'][$namespace]['vectorCount']) ? (int) $stats['namespaces'][$namespace]['vectorCount'] : 0; | |
| 1501 | + // 3e83e4: the total is the SOURCE NAMESPACE's count, never the whole | |
| 1502 | + // index — the copy lists one namespace, so an index with records in | |
| 1503 | + // other namespaces used to report "done" with copied < total and no | |
| 1504 | + // explanation. The other namespaces are recorded for the UI instead. | |
| 1505 | + $ns_counts = self::namespace_counts($stats); | |
| 1506 | + $queue = array(); | |
| 1507 | + $per_namespace = array(); | |
| 1508 | + $other_namespaces = array(); | |
| 1509 | + if ($all_namespaces) { | |
| 1510 | + $queue = self::namespace_queue($ns_counts); | |
| 1511 | + if (empty($queue)) { | |
| 1512 | + $queue = array($namespace); // stats listed no namespaces — copy the configured one, as before | |
| 1513 | + } | |
| 1514 | + $total = 0; | |
| 1515 | + foreach ($queue as $ns) { | |
| 1516 | + if (isset($ns_counts[$ns])) { | |
| 1517 | + $count = (int) $ns_counts[$ns]; | |
| 1518 | + } elseif ($ns === '' && empty($ns_counts)) { | |
| 1519 | + $count = (int) ($stats['totalVectorCount'] ?? 0); | |
| 1520 | + } else { | |
| 1521 | + $count = 0; | |
| 1522 | + } | |
| 1523 | + $per_namespace[self::ns_key($ns)] = self::new_namespace_entry($ns, $count); | |
| 1524 | + $total += $count; | |
| 1525 | + } | |
| 1526 | + $current = (string) array_shift($queue); | |
| 1527 | + $per_namespace[self::ns_key($current)]['status'] = 'running'; | |
| 1044 | 1528 | } else { |
| 1045 | - $total = (int) ($stats['totalVectorCount'] ?? 0); | |
| 1529 | + $current = $namespace; | |
| 1530 | + if (isset($ns_counts[$namespace])) { | |
| 1531 | + $total = (int) $ns_counts[$namespace]; | |
| 1532 | + } elseif ($namespace === '' && empty($ns_counts)) { | |
| 1533 | + $total = (int) ($stats['totalVectorCount'] ?? 0); | |
| 1534 | + } else { | |
| 1535 | + $total = 0; | |
| 1536 | + } | |
| 1537 | + $per_namespace[self::ns_key($namespace)] = self::new_namespace_entry($namespace, $total, 'running'); | |
| 1538 | + // 47c61f: in the copy order (default first, then by name). Pinecone's | |
| 1539 | + // stats object lists namespaces in a different order call to call. | |
| 1540 | + foreach (self::namespace_queue($ns_counts) as $ns_name) { | |
| 1541 | + if ((string) $ns_name !== (string) $namespace) { | |
| 1542 | + $other_namespaces[self::ns_key($ns_name)] = (int) $ns_counts[$ns_name]; | |
| 1543 | + } | |
| 1544 | + } | |
| 1046 | 1545 | } |
| 1047 | 1546 | $source_dimension = (int) ($stats['dimension'] ?? 0); |
| 1048 | 1547 | $target_index = self::find_index_by_host($target_cfg['host'], $target_cfg['api_key']); |
| 1049 | 1548 | if (is_wp_error($target_index)) { |
| @@ -1056,20 +1555,33 @@ | ||
| 1056 | 1555 | if ($source_dimension > 0 && $schema['dimension'] > 0 && $source_dimension !== (int) $schema['dimension']) { |
| 1057 | 1556 | return new WP_Error('migrate', sprintf('Dimensions differ (source %d, document index %d) — the vectors cannot be copied. Re-import your content into the document index instead.', $source_dimension, $schema['dimension'])); |
| 1058 | 1557 | } |
| 1059 | 1558 | $state = array( |
| 1060 | - 'status' => 'running', | |
| 1061 | - 'source_host' => $source_host, | |
| 1062 | - 'target_host' => $target_cfg['host'], | |
| 1063 | - 'target_name' => (string) ($target_index['name'] ?? ''), | |
| 1064 | - 'namespace' => $namespace, | |
| 1065 | - 'total' => $total, | |
| 1066 | - 'copied' => 0, | |
| 1067 | - 'token' => '', | |
| 1068 | - 'batches' => 0, | |
| 1069 | - 'started' => time(), | |
| 1070 | - 'error' => '', | |
| 1071 | - 'source_name' => '', | |
| 1559 | + 'status' => 'running', | |
| 1560 | + 'source_host' => $source_host, | |
| 1561 | + 'target_host' => $target_cfg['host'], | |
| 1562 | + 'target_name' => (string) ($target_index['name'] ?? ''), | |
| 1563 | + 'namespace' => $current, | |
| 1564 | + 'total' => $total, | |
| 1565 | + 'total_all' => (int) ($stats['totalVectorCount'] ?? $total), | |
| 1566 | + 'other_namespaces' => $other_namespaces, | |
| 1567 | + // e50d2e: the namespace being copied, the ones still queued and | |
| 1568 | + // a per-namespace ledger — the Delete guard checks every entry. | |
| 1569 | + 'all_namespaces' => (bool) $all_namespaces, | |
| 1570 | + 'current' => $current, | |
| 1571 | + 'queue' => $queue, | |
| 1572 | + 'per_namespace' => $per_namespace, | |
| 1573 | + 'copied' => 0, | |
| 1574 | + 'skipped' => 0, | |
| 1575 | + 'token' => '', | |
| 1576 | + 'batches' => 0, | |
| 1577 | + 'retries' => 0, | |
| 1578 | + 'failed_steps' => 0, | |
| 1579 | + 'started' => time(), | |
| 1580 | + 'error' => '', | |
| 1581 | + 'error_at' => 0, | |
| 1582 | + 'target_count' => null, | |
| 1583 | + 'source_name' => '', | |
| 1072 | 1584 | ); |
| 1073 | 1585 | $source_index = self::find_index_by_host($source_host, $target_cfg['api_key']); |
| 1074 | 1586 | if (!is_wp_error($source_index)) { |
| 1075 | 1587 | $state['source_name'] = (string) ($source_index['name'] ?? ''); |
| @@ -1077,10 +1589,15 @@ | ||
| 1077 | 1589 | return self::migration_set($state); |
| 1078 | 1590 | } |
| 1079 | 1591 | |
| 1080 | 1592 | /** |
| 1081 | - * Copy one page: list → fetch (values + metadata) → documents/upsert. | |
| 1082 | - * Idempotent — the same ids land in the same place on a re-run. | |
| 1593 | + * Copy pages until the step's time budget is used or the source is | |
| 1594 | + * exhausted: list → fetch (values + metadata) → documents/upsert, each | |
| 1595 | + * call retried with backoff. Idempotent — the same ids land in the same | |
| 1596 | + * place on a re-run. The cursor is persisted after EVERY page, so a step | |
| 1597 | + * that dies mid-way (proxy timeout, killed request) loses at most one | |
| 1598 | + * page of work and the next click continues from the recorded position. | |
| 1599 | + * | |
| 1083 | 1600 | * @return array|WP_Error state |
| 1084 | 1601 | */ |
| 1085 | 1602 | public static function migrate_step(array $state, array $target_cfg, $batch = self::MIGRATE_BATCH) { |
| 1086 | 1603 | if (($state['status'] ?? '') !== 'running') { |
| @@ -1085,44 +1602,245 @@ | ||
| 1085 | 1602 | public static function migrate_step(array $state, array $target_cfg, $batch = self::MIGRATE_BATCH) { |
| 1086 | 1603 | if (($state['status'] ?? '') !== 'running') { |
| 1087 | 1604 | return $state; |
| 1088 | 1605 | } |
| 1089 | - if (($state['target_host'] ?? '') !== ($target_cfg['host'] ?? '')) { | |
| 1606 | + if (strtolower((string) ($state['target_host'] ?? '')) !== strtolower((string) ($target_cfg['host'] ?? ''))) { | |
| 1090 | 1607 | return new WP_Error('migrate', 'The document index host changed since the migration started. Start it again.'); |
| 1091 | 1608 | } |
| 1092 | - $page = self::classic_list_page($state['source_host'], $target_cfg['api_key'], $state['namespace'], $batch, $state['token']); | |
| 1093 | - if (is_wp_error($page)) { | |
| 1094 | - $state['error'] = $page->get_error_message(); | |
| 1095 | - return self::migration_set($state); | |
| 1609 | + $started = microtime(true); | |
| 1610 | + $budget = self::step_budget(); | |
| 1611 | + $pages = 0; | |
| 1612 | + $api_key = $target_cfg['api_key']; | |
| 1613 | + foreach (array('retries', 'skipped', 'failed_steps', 'error_at') as $counter) { | |
| 1614 | + $state[$counter] = (int) ($state[$counter] ?? 0); // states written before 3e83e4 lack these | |
| 1096 | 1615 | } |
| 1097 | - if (!empty($page['ids'])) { | |
| 1098 | - $vectors = self::classic_fetch($state['source_host'], $target_cfg['api_key'], $state['namespace'], $page['ids']); | |
| 1099 | - if (is_wp_error($vectors)) { | |
| 1100 | - $state['error'] = $vectors->get_error_message(); | |
| 1101 | - return self::migration_set($state); | |
| 1616 | + if (empty($state['per_namespace']) || !is_array($state['per_namespace'])) { | |
| 1617 | + // e50d2e: a state written before this build copied one namespace and has no ledger | |
| 1618 | + $state['per_namespace'] = self::per_namespace_map($state); | |
| 1619 | + $state['current'] = (string) ($state['namespace'] ?? ''); | |
| 1620 | + $state['queue'] = array(); | |
| 1621 | + } | |
| 1622 | + if (!isset($state['current'])) { | |
| 1623 | + $state['current'] = (string) ($state['namespace'] ?? ''); | |
| 1624 | + } | |
| 1625 | + do { | |
| 1626 | + $source_host = $state['source_host']; | |
| 1627 | + $namespace = (string) $state['current']; | |
| 1628 | + $ns_key = self::ns_key($namespace); | |
| 1629 | + $token = $state['token']; | |
| 1630 | + // e50d2e: the same namespace on the document side — every bot keeps reading its own | |
| 1631 | + $page_cfg = $target_cfg; | |
| 1632 | + $page_cfg['namespace'] = $namespace; | |
| 1633 | + $listed = self::with_retry(function () use ($source_host, $api_key, $namespace, $batch, $token) { | |
| 1634 | + return self::classic_list_page($source_host, $api_key, $namespace, $batch, $token); | |
| 1635 | + }); | |
| 1636 | + $state['retries'] += max(0, $listed['attempts'] - 1); | |
| 1637 | + $page = $listed['result']; | |
| 1638 | + if (is_wp_error($page)) { | |
| 1639 | + return self::migration_fail($state, 'Listing the source index failed after ' . $listed['attempts'] . ' attempts: ' . $page->get_error_message()); | |
| 1102 | 1640 | } |
| 1103 | - $documents = array(); | |
| 1104 | - foreach ($vectors as $id => $vector) { | |
| 1105 | - if (empty($vector['values']) || !is_array($vector['values'])) { | |
| 1106 | - continue; | |
| 1641 | + $ids = $page['ids']; | |
| 1642 | + if (!empty($ids)) { | |
| 1643 | + $fetched = self::with_retry(function () use ($source_host, $api_key, $namespace, $ids) { | |
| 1644 | + return self::classic_fetch($source_host, $api_key, $namespace, $ids); | |
| 1645 | + }); | |
| 1646 | + $state['retries'] += max(0, $fetched['attempts'] - 1); | |
| 1647 | + $vectors = $fetched['result']; | |
| 1648 | + if (is_wp_error($vectors)) { | |
| 1649 | + return self::migration_fail($state, 'Reading ' . count($ids) . ' records from the source index failed after ' . $fetched['attempts'] . ' attempts: ' . $vectors->get_error_message()); | |
| 1107 | 1650 | } |
| 1108 | - $metadata = is_array($vector['metadata'] ?? null) ? $vector['metadata'] : array(); | |
| 1109 | - $documents[] = self::build_document((string) ($vector['id'] ?? $id), (string) ($metadata['text'] ?? ''), $vector['values'], $metadata); | |
| 1651 | + $documents = array(); | |
| 1652 | + $skipped = 0; | |
| 1653 | + foreach ($vectors as $id => $vector) { | |
| 1654 | + if (empty($vector['values']) || !is_array($vector['values'])) { | |
| 1655 | + $skipped++; | |
| 1656 | + continue; | |
| 1657 | + } | |
| 1658 | + $metadata = is_array($vector['metadata'] ?? null) ? $vector['metadata'] : array(); | |
| 1659 | + $documents[] = self::build_document((string) ($vector['id'] ?? $id), (string) ($metadata['text'] ?? ''), $vector['values'], $metadata); | |
| 1660 | + } | |
| 1661 | + $skipped += max(0, count($ids) - count($vectors)); | |
| 1662 | + if (!empty($documents)) { | |
| 1663 | + $upserted = self::with_retry(function () use ($documents, $page_cfg) { | |
| 1664 | + return self::upsert($documents, $page_cfg); | |
| 1665 | + }); | |
| 1666 | + $state['retries'] += max(0, $upserted['attempts'] - 1); | |
| 1667 | + if (is_wp_error($upserted['result'])) { | |
| 1668 | + return self::migration_fail($state, 'Writing ' . count($documents) . ' records to the document index failed after ' . $upserted['attempts'] . ' attempts: ' . $upserted['result']->get_error_message()); | |
| 1669 | + } | |
| 1670 | + } | |
| 1671 | + $state['copied'] = (int) $state['copied'] + count($documents); | |
| 1672 | + $state['skipped'] = (int) ($state['skipped'] ?? 0) + $skipped; | |
| 1673 | + $state['per_namespace'][$ns_key]['copied'] = (int) ($state['per_namespace'][$ns_key]['copied'] ?? 0) + count($documents); | |
| 1110 | 1674 | } |
| 1111 | - $result = self::upsert($documents, $target_cfg); | |
| 1112 | - if (is_wp_error($result)) { | |
| 1113 | - $state['error'] = $result->get_error_message(); | |
| 1114 | - return self::migration_set($state); | |
| 1675 | + $state['batches'] = (int) $state['batches'] + 1; | |
| 1676 | + $state['token'] = $page['next']; | |
| 1677 | + $state['error'] = ''; | |
| 1678 | + $state['error_at'] = 0; | |
| 1679 | + $pages++; | |
| 1680 | + if ($page['next'] === '') { | |
| 1681 | + // This namespace is copied: record what the document index | |
| 1682 | + // reports for it, then move on to the next queued namespace | |
| 1683 | + // inside the same time budget (e50d2e), or finish. | |
| 1684 | + $state['per_namespace'][$ns_key]['status'] = 'done'; | |
| 1685 | + $state['per_namespace'][$ns_key]['finished'] = time(); | |
| 1686 | + $ns_target = self::namespace_count($page_cfg); | |
| 1687 | + $state['per_namespace'][$ns_key]['target_count'] = $ns_target > 0 ? $ns_target : null; | |
| 1688 | + $queue = array_values((array) ($state['queue'] ?? array())); | |
| 1689 | + if (!empty($queue)) { | |
| 1690 | + $next = (string) array_shift($queue); | |
| 1691 | + $state['queue'] = $queue; | |
| 1692 | + $state['current'] = $next; | |
| 1693 | + $state['namespace'] = $next; | |
| 1694 | + $state['token'] = ''; | |
| 1695 | + if (!isset($state['per_namespace'][self::ns_key($next)])) { | |
| 1696 | + $state['per_namespace'][self::ns_key($next)] = self::new_namespace_entry($next, 0); | |
| 1697 | + } | |
| 1698 | + $state['per_namespace'][self::ns_key($next)]['status'] = 'running'; | |
| 1699 | + self::migration_set($state); | |
| 1700 | + continue; // re-checks the budget, then lists the next namespace from its first page | |
| 1701 | + } | |
| 1702 | + $state['status'] = 'done'; | |
| 1703 | + $state['finished'] = time(); | |
| 1704 | + // The finished state names what both sides report, so the UI | |
| 1705 | + // can say "5,000 copied; the document index holds 5,000". | |
| 1706 | + $target_count = 0; | |
| 1707 | + foreach ($state['per_namespace'] as $entry) { | |
| 1708 | + $target_count += (int) ($entry['target_count'] ?? 0); | |
| 1709 | + } | |
| 1710 | + $state['target_count'] = $target_count > 0 ? $target_count : null; | |
| 1711 | + $state['target_count_at'] = time(); | |
| 1712 | + break; | |
| 1115 | 1713 | } |
| 1116 | - $state['copied'] = (int) $state['copied'] + count($documents); | |
| 1714 | + self::migration_set($state); // cursor persisted per page, not per step | |
| 1715 | + } while ((microtime(true) - $started) < $budget); | |
| 1716 | + $state['last_step_pages'] = $pages; | |
| 1717 | + $state['last_step_seconds'] = round(microtime(true) - $started, 2); | |
| 1718 | + return self::migration_set($state); | |
| 1719 | + } | |
| 1720 | + | |
| 1721 | + /** Record count of ONE namespace on a document index (0 when not listed) — count() reads the whole index for the default namespace. */ | |
| 1722 | + private static function namespace_count(array $cfg) { | |
| 1723 | + $stats = self::stats($cfg); | |
| 1724 | + if (is_wp_error($stats)) { | |
| 1725 | + return 0; | |
| 1117 | 1726 | } |
| 1118 | - $state['batches'] = (int) $state['batches'] + 1; | |
| 1119 | - $state['token'] = $page['next']; | |
| 1120 | - $state['error'] = ''; | |
| 1121 | - if ($page['next'] === '') { | |
| 1122 | - $state['status'] = 'done'; | |
| 1123 | - $state['finished'] = time(); | |
| 1727 | + $ns = (string) ($cfg['namespace'] ?? ''); | |
| 1728 | + return isset($stats['namespaces'][$ns]['vectorCount']) ? (int) $stats['namespaces'][$ns]['vectorCount'] : 0; | |
| 1729 | + } | |
| 1730 | + | |
| 1731 | + /** | |
| 1732 | + * e50d2e: namespaces of the OLD index that still hold records without a | |
| 1733 | + * finished, count-matching copy — read from the live source stats, so a | |
| 1734 | + * namespace filled after the copy counts too. Empty array = every record | |
| 1735 | + * on the old index has a copy; the Delete old index button refuses on | |
| 1736 | + * anything else and names what is missing. | |
| 1737 | + * | |
| 1738 | + * @return array|WP_Error display key → ['namespace','source','copied','status'] | |
| 1739 | + */ | |
| 1740 | + public static function uncopied_namespaces(array $state, $api_key) { | |
| 1741 | + $stats = self::classic_stats((string) ($state['source_host'] ?? ''), $api_key); | |
| 1742 | + if (is_wp_error($stats)) { | |
| 1743 | + return $stats; | |
| 1124 | 1744 | } |
| 1745 | + $counts = self::namespace_counts($stats); | |
| 1746 | + if (empty($counts) && (int) ($stats['totalVectorCount'] ?? 0) > 0) { | |
| 1747 | + $counts[''] = (int) $stats['totalVectorCount']; | |
| 1748 | + } | |
| 1749 | + $map = self::per_namespace_map($state); | |
| 1750 | + $missing = array(); | |
| 1751 | + foreach ($counts as $ns => $count) { | |
| 1752 | + if ($count <= 0) { | |
| 1753 | + continue; | |
| 1754 | + } | |
| 1755 | + $key = self::ns_key($ns); | |
| 1756 | + $entry = isset($map[$key]) && is_array($map[$key]) ? $map[$key] : null; | |
| 1757 | + if ($entry === null || ($entry['status'] ?? '') !== 'done' || (int) ($entry['copied'] ?? 0) < $count) { | |
| 1758 | + $missing[$key] = array( | |
| 1759 | + 'namespace' => (string) $ns, | |
| 1760 | + 'source' => (int) $count, | |
| 1761 | + 'copied' => $entry ? (int) ($entry['copied'] ?? 0) : 0, | |
| 1762 | + 'status' => $entry ? (string) ($entry['status'] ?? '') : 'not copied', | |
| 1763 | + ); | |
| 1764 | + } | |
| 1765 | + } | |
| 1766 | + // Named in the copy order (default first, then by name) — Pinecone's stats object order varies call to call. | |
| 1767 | + $ordered = array(); | |
| 1768 | + foreach (self::namespace_queue(array_combine(array_map(array(__CLASS__, 'ns_raw'), array_keys($missing)), array_column($missing, 'source')) ?: array()) as $ns) { | |
| 1769 | + $ordered[self::ns_key($ns)] = $missing[self::ns_key($ns)]; | |
| 1770 | + } | |
| 1771 | + return $ordered + $missing; | |
| 1772 | + } | |
| 1773 | + | |
| 1774 | + /** Plain-language refusal naming every namespace the copy is missing. (Plain __(): the card's JS text-escapes it.) */ | |
| 1775 | + public static function uncopied_message(array $missing) { | |
| 1776 | + $parts = array(); | |
| 1777 | + foreach ($missing as $key => $m) { | |
| 1778 | + $parts[] = sprintf( | |
| 1779 | + /* translators: 1: namespace name, 2: records copied, 3: records on the old index */ | |
| 1780 | + __('%1$s (%2$s of %3$s records copied)', 'mxchat'), | |
| 1781 | + $key, | |
| 1782 | + number_format_i18n((int) $m['copied']), | |
| 1783 | + number_format_i18n((int) $m['source']) | |
| 1784 | + ); | |
| 1785 | + } | |
| 1786 | + return sprintf( | |
| 1787 | + /* translators: %s: comma-separated namespace names with their counts */ | |
| 1788 | + _n( | |
| 1789 | + 'Namespace %s on the old index has records that are not in the document index. Tick Copy every namespace and run Migrate again, then delete. The old index stays.', | |
| 1790 | + 'Namespaces %s on the old index have records that are not in the document index. Tick Copy every namespace and run Migrate again, then delete. The old index stays.', | |
| 1791 | + count($missing), | |
| 1792 | + 'mxchat' | |
| 1793 | + ), | |
| 1794 | + implode(', ', $parts) | |
| 1795 | + ); | |
| 1796 | + } | |
| 1797 | + | |
| 1798 | + /** | |
| 1799 | + * 5872ef: bots that still keep the given Pinecone host as their own (the | |
| 1800 | + * Multi-Bot add-on stores a host per bot). Core knows nothing about the | |
| 1801 | + * bots table — the add-on answers the filter, and with it inactive the | |
| 1802 | + * list is empty. Delete old index refuses while this is not empty. | |
| 1803 | + * | |
| 1804 | + * @return array bot id → bot name | |
| 1805 | + */ | |
| 1806 | + public static function bots_on_host($host) { | |
| 1807 | + $host = strtolower(trim(str_replace(array('https://', 'http://'), '', (string) $host), '/')); | |
| 1808 | + if ($host === '') { | |
| 1809 | + return array(); | |
| 1810 | + } | |
| 1811 | + $bots = apply_filters('mxchat_pinecone_bots_on_host', array(), $host); | |
| 1812 | + if (!is_array($bots)) { | |
| 1813 | + return array(); | |
| 1814 | + } | |
| 1815 | + $named = array(); | |
| 1816 | + foreach ($bots as $bot_id => $name) { | |
| 1817 | + $name = is_scalar($name) ? trim((string) $name) : ''; | |
| 1818 | + $named[(string) $bot_id] = $name !== '' ? $name : (string) $bot_id; | |
| 1819 | + } | |
| 1820 | + return $named; | |
| 1821 | + } | |
| 1822 | + | |
| 1823 | + /** Plain-language refusal naming every bot still on the old index. (Plain __(): the card's JS text-escapes it.) */ | |
| 1824 | + public static function bots_on_host_message(array $bots) { | |
| 1825 | + return sprintf( | |
| 1826 | + /* translators: 1: number of bots, 2: comma-separated bot names */ | |
| 1827 | + _n( | |
| 1828 | + '%1$s bot still uses the old index: %2$s. Move it to the new index first (Multi-Bot, All Bots, Move these bots to the new index), or it will stop answering. The old index stays.', | |
| 1829 | + '%1$s bots still use the old index: %2$s. Move them to the new index first (Multi-Bot, All Bots, Move these bots to the new index), or they will stop answering. The old index stays.', | |
| 1830 | + count($bots), | |
| 1831 | + 'mxchat' | |
| 1832 | + ), | |
| 1833 | + number_format_i18n(count($bots)), | |
| 1834 | + implode(', ', $bots) | |
| 1835 | + ); | |
| 1836 | + } | |
| 1837 | + | |
| 1838 | + /** Record a step failure without ending the copy: status stays running, the cursor stays put. */ | |
| 1839 | + private static function migration_fail(array $state, $message) { | |
| 1840 | + $state['error'] = (string) $message; | |
| 1841 | + $state['error_at'] = time(); | |
| 1842 | + $state['failed_steps'] = (int) ($state['failed_steps'] ?? 0) + 1; | |
| 1125 | 1843 | return self::migration_set($state); |
| 1126 | 1844 | } |
| 1127 | 1845 | |
| 1128 | 1846 | // ===================================================================== |
| @@ -1131,12 +1849,47 @@ | ||
| 1131 | 1849 | |
| 1132 | 1850 | public static function boot() { |
| 1133 | 1851 | add_action('wp_ajax_mxchat_pinecone_docs_create_index', array(__CLASS__, 'ajax_create_index')); |
| 1134 | 1852 | add_action('wp_ajax_mxchat_pinecone_docs_check_index', array(__CLASS__, 'ajax_check_index')); |
| 1853 | + add_action('wp_ajax_mxchat_pinecone_docs_suggest_name', array(__CLASS__, 'ajax_suggest_name')); | |
| 1135 | 1854 | add_action('wp_ajax_mxchat_pinecone_docs_migrate', array(__CLASS__, 'ajax_migrate')); |
| 1136 | 1855 | add_action('wp_ajax_mxchat_pinecone_docs_delete_old_index', array(__CLASS__, 'ajax_delete_old_index')); |
| 1137 | 1856 | } |
| 1138 | 1857 | |
| 1858 | + /** | |
| 1859 | + * Suggest a free document-index name for the name in the field (3e83e4). | |
| 1860 | + * With a host that already answers in the project, the name is that | |
| 1861 | + * index's own — the owner is pointing at an existing document index, not | |
| 1862 | + * about to create one. | |
| 1863 | + */ | |
| 1864 | + public static function ajax_suggest_name() { | |
| 1865 | + self::ajax_guard(); | |
| 1866 | + $api_key = self::posted_api_key(); | |
| 1867 | + $base = isset($_POST['base']) ? sanitize_text_field(wp_unslash($_POST['base'])) : ''; | |
| 1868 | + $host = isset($_POST['host']) ? sanitize_text_field(wp_unslash($_POST['host'])) : ''; | |
| 1869 | + if ($api_key === '') { | |
| 1870 | + wp_send_json_error(array('message' => esc_html__('Enter your Pinecone API key first.', 'mxchat'))); | |
| 1871 | + } | |
| 1872 | + if ($host !== '') { | |
| 1873 | + $index = self::find_index_by_host($host, $api_key); | |
| 1874 | + if (!is_wp_error($index) && !empty($index['name'])) { | |
| 1875 | + wp_send_json_success(array('name' => strtolower((string) $index['name']), 'resolved' => true, 'taken' => false, 'base' => strtolower($base))); | |
| 1876 | + } | |
| 1877 | + } | |
| 1878 | + $suggest = self::suggest_index_name($base, $api_key); | |
| 1879 | + wp_send_json_success(array('name' => $suggest['name'], 'resolved' => false, 'taken' => $suggest['taken'], 'base' => strtolower($base))); | |
| 1880 | + } | |
| 1881 | + | |
| 1882 | + /** The document-index target for a migration call: the host in the form, else the saved one. */ | |
| 1883 | + private static function posted_target_host(array $options) { | |
| 1884 | + $posted = isset($_POST['target_host']) ? sanitize_text_field(wp_unslash($_POST['target_host'])) : ''; | |
| 1885 | + $posted = strtolower(trim(str_replace(array('https://', 'http://'), '', $posted), '/')); | |
| 1886 | + if ($posted !== '') { | |
| 1887 | + return $posted; | |
| 1888 | + } | |
| 1889 | + return strtolower(trim((string) ($options['mxchat_pinecone_host'] ?? ''), '/')); | |
| 1890 | + } | |
| 1891 | + | |
| 1139 | 1892 | private static function ajax_guard() { |
| 1140 | 1893 | check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce'); |
| 1141 | 1894 | if (!current_user_can('manage_options')) { |
| 1142 | 1895 | wp_send_json_error(array('message' => esc_html__('Permission denied.', 'mxchat')), 403); |
| @@ -1168,12 +1921,21 @@ | ||
| 1168 | 1921 | } |
| 1169 | 1922 | if ($name === '') { |
| 1170 | 1923 | wp_send_json_error(array('message' => esc_html__('Enter an index name first — the new document index will be created under that name.', 'mxchat'))); |
| 1171 | 1924 | } |
| 1172 | - $dimension = self::expected_dimension(); | |
| 1925 | + $dimension = self::expected_dimension_checked(true); | |
| 1926 | + if (is_wp_error($dimension)) { | |
| 1927 | + // 418afd: never size an index from a guess | |
| 1928 | + wp_send_json_error(array('message' => self::dimension_unknown_message($dimension, 'create'), 'code' => 'dimension_unknown')); | |
| 1929 | + } | |
| 1173 | 1930 | $index = self::create_document_index($name, $dimension, $cloud, $region, $language, $api_key); |
| 1174 | 1931 | if (is_wp_error($index)) { |
| 1175 | - wp_send_json_error(array('message' => $index->get_error_message())); | |
| 1932 | + $data = $index->get_error_data(); | |
| 1933 | + wp_send_json_error(array( | |
| 1934 | + 'message' => $index->get_error_message(), | |
| 1935 | + 'code' => $index->get_error_code(), | |
| 1936 | + 'suggested' => is_array($data) ? (string) ($data['suggested'] ?? '') : '', | |
| 1937 | + )); | |
| 1176 | 1938 | } |
| 1177 | 1939 | $host = (string) ($index['host'] ?? ''); |
| 1178 | 1940 | // Give Pinecone a few seconds — small serverless indexes are usually Ready quickly. |
| 1179 | 1941 | $ready = !empty($index['status']['ready']); |
| @@ -1186,21 +1948,24 @@ | ||
| 1186 | 1948 | } |
| 1187 | 1949 | } |
| 1188 | 1950 | $options = get_option(self::OPTION, array()); |
| 1189 | 1951 | $options = is_array($options) ? $options : array(); |
| 1952 | + // 3e83e4: creating the index no longer moves the live chatbot. The | |
| 1953 | + // host and index type are only written when the owner saves the | |
| 1954 | + // form; here we record that this host passed the check (the save | |
| 1955 | + // gate) and the settings the index was created with. | |
| 1190 | 1956 | $changes = array( |
| 1191 | - 'mxchat_pinecone_api_key' => $api_key, | |
| 1192 | - 'mxchat_pinecone_host' => $host, | |
| 1193 | - 'mxchat_pinecone_index' => strtolower($name), | |
| 1194 | - 'mxchat_pinecone_index_type' => 'document', | |
| 1195 | 1957 | 'mxchat_pinecone_docs_verified_host' => $host, |
| 1196 | 1958 | 'mxchat_pinecone_docs_cloud' => array_key_exists($cloud, self::clouds()) ? $cloud : 'aws', |
| 1197 | 1959 | 'mxchat_pinecone_docs_region' => $region ?: 'us-east-1', |
| 1198 | 1960 | 'mxchat_pinecone_docs_language' => array_key_exists($language, self::languages()) ? $language : 'en', |
| 1199 | 1961 | ); |
| 1962 | + if (empty($options['mxchat_pinecone_api_key'])) { | |
| 1963 | + $changes['mxchat_pinecone_api_key'] = $api_key; | |
| 1964 | + } | |
| 1200 | 1965 | // Remember the classic index so Migrate can copy from it. |
| 1201 | 1966 | $previous_host = (string) ($options['mxchat_pinecone_host'] ?? ''); |
| 1202 | - if ($previous_host !== '' && $previous_host !== $host && self::options_index_type($options) === 'vector' && empty($options['mxchat_pinecone_vector_host'])) { | |
| 1967 | + if ($previous_host !== '' && strtolower($previous_host) !== strtolower($host) && self::options_index_type($options) === 'vector' && empty($options['mxchat_pinecone_vector_host'])) { | |
| 1203 | 1968 | $changes['mxchat_pinecone_vector_host'] = $previous_host; |
| 1204 | 1969 | $changes['mxchat_pinecone_vector_index'] = (string) ($options['mxchat_pinecone_index'] ?? ''); |
| 1205 | 1970 | } |
| 1206 | 1971 | self::persist_options($changes); |
| @@ -1209,11 +1974,12 @@ | ||
| 1209 | 1974 | 'name' => strtolower($name), |
| 1210 | 1975 | 'dimension' => $dimension, |
| 1211 | 1976 | 'ready' => $ready, |
| 1212 | 1977 | 'vector_host' => $changes['mxchat_pinecone_vector_host'] ?? ($options['mxchat_pinecone_vector_host'] ?? ''), |
| 1978 | + // Plain text: the card's JS escapes it on render (esc_html__ here double-escaped the quotes to a literal "). | |
| 1213 | 1979 | 'message' => $ready |
| 1214 | - ? sprintf(esc_html__('Document index "%1$s" created (%2$d dimensions, cosine, full-text search on the chunk text). Host filled in and saved.', 'mxchat'), strtolower($name), $dimension) | |
| 1215 | - : sprintf(esc_html__('Document index "%1$s" is being created (%2$d dimensions). The host is filled in and saved; run Check index in a minute to confirm it is ready.', 'mxchat'), strtolower($name), $dimension), | |
| 1980 | + ? sprintf(__('Document index "%1$s" created (%2$d dimensions, cosine, full-text search on the chunk text). The host is filled in above. Your chatbot keeps answering from the current index until you save the Pinecone settings — copy your records across with Migrate first.', 'mxchat'), strtolower($name), $dimension) | |
| 1981 | + : sprintf(__('Document index "%1$s" is being created (%2$d dimensions). The host is filled in above; run Check index in a minute to confirm it is ready. Your chatbot keeps answering from the current index until you save the Pinecone settings.', 'mxchat'), strtolower($name), $dimension), | |
| 1216 | 1982 | )); |
| 1217 | 1983 | } |
| 1218 | 1984 | |
| 1219 | 1985 | /** Check index: assert the host is a document index MxChat can use. */ |
| @@ -1224,9 +1990,13 @@ | ||
| 1224 | 1990 | $host = trim(str_replace(array('https://', 'http://'), '', $host), '/'); |
| 1225 | 1991 | if ($api_key === '' || $host === '') { |
| 1226 | 1992 | wp_send_json_error(array('message' => esc_html__('Enter the Pinecone API key and host first.', 'mxchat'))); |
| 1227 | 1993 | } |
| 1228 | - $check = self::check_document_index($host, $api_key, self::expected_dimension()); | |
| 1994 | + $dimension = self::expected_dimension_checked(true); | |
| 1995 | + if (is_wp_error($dimension)) { | |
| 1996 | + wp_send_json_error(array('ok' => false, 'message' => self::dimension_unknown_message($dimension, 'check'), 'code' => 'dimension_unknown')); | |
| 1997 | + } | |
| 1998 | + $check = self::check_document_index($host, $api_key, $dimension); | |
| 1229 | 1999 | if (!$check['ok']) { |
| 1230 | 2000 | wp_send_json_error($check); |
| 1231 | 2001 | } |
| 1232 | 2002 | $options = get_option(self::OPTION, array()); |
| @@ -1244,15 +2014,23 @@ | ||
| 1244 | 2014 | $check['vector_host'] = $changes['mxchat_pinecone_vector_host'] ?? ($options['mxchat_pinecone_vector_host'] ?? ''); |
| 1245 | 2015 | wp_send_json_success($check); |
| 1246 | 2016 | } |
| 1247 | 2017 | |
| 1248 | - /** Migrate: mode start | step | status | reset. Each step copies one page. */ | |
| 2018 | + /** | |
| 2019 | + * Migrate: mode start | step | status | reset. | |
| 2020 | + * | |
| 2021 | + * 3e83e4: the copy is DECOUPLED from the live retrieval path. The target | |
| 2022 | + * is the document-index host in the form (posted as target_host) plus the | |
| 2023 | + * API key, checked against Pinecone at start — the saved index type is | |
| 2024 | + * never consulted, so the site keeps answering from its vector index for | |
| 2025 | + * the whole copy and the owner switches by saving the settings afterwards. | |
| 2026 | + * Each step copies pages for up to MIGRATE_STEP_SECONDS. | |
| 2027 | + */ | |
| 1249 | 2028 | public static function ajax_migrate() { |
| 1250 | 2029 | self::ajax_guard(); |
| 1251 | 2030 | $mode = isset($_POST['mode']) ? sanitize_key($_POST['mode']) : 'status'; |
| 1252 | 2031 | $options = get_option(self::OPTION, array()); |
| 1253 | 2032 | $options = is_array($options) ? $options : array(); |
| 1254 | - $target = self::cfg_from_options($options); | |
| 1255 | 2033 | if ($mode === 'reset') { |
| 1256 | 2034 | self::migration_clear(); |
| 1257 | 2035 | wp_send_json_success(array('state' => array())); |
| 1258 | 2036 | } |
| @@ -1258,25 +2036,41 @@ | ||
| 1258 | 2036 | } |
| 1259 | 2037 | if ($mode === 'status') { |
| 1260 | 2038 | wp_send_json_success(array('state' => self::migration_get())); |
| 1261 | 2039 | } |
| 1262 | - if ($target['index_type'] !== 'document') { | |
| 1263 | - wp_send_json_error(array('message' => esc_html__('Save the Pinecone settings with the index type set to Document index first.', 'mxchat'))); | |
| 2040 | + $api_key = self::posted_api_key(); | |
| 2041 | + $namespace = (string) ($options['mxchat_pinecone_namespace'] ?? ''); | |
| 2042 | + if ($api_key === '') { | |
| 2043 | + wp_send_json_error(array('message' => esc_html__('Enter your Pinecone API key first.', 'mxchat'))); | |
| 1264 | 2044 | } |
| 1265 | - if (strtolower((string) ($options['mxchat_pinecone_docs_verified_host'] ?? '')) !== strtolower($target['host'])) { | |
| 1266 | - wp_send_json_error(array('message' => esc_html__('Run Check index on the document index host before migrating.', 'mxchat'))); | |
| 1267 | - } | |
| 1268 | 2045 | if ($mode === 'start') { |
| 2046 | + $target_host = self::posted_target_host($options); | |
| 2047 | + if ($target_host === '') { | |
| 2048 | + wp_send_json_error(array('message' => esc_html__('Enter the host of the document index to copy into (or create one with Create index for me).', 'mxchat'))); | |
| 2049 | + } | |
| 2050 | + $dimension = self::expected_dimension_checked(true); | |
| 2051 | + if (is_wp_error($dimension)) { | |
| 2052 | + wp_send_json_error(array('message' => self::dimension_unknown_message($dimension, 'check'), 'code' => 'dimension_unknown')); | |
| 2053 | + } | |
| 2054 | + $check = self::check_document_index($target_host, $api_key, $dimension); | |
| 2055 | + if (!$check['ok']) { | |
| 2056 | + wp_send_json_error(array('message' => $check['message'])); | |
| 2057 | + } | |
| 2058 | + $target = array('api_key' => $api_key, 'host' => strtolower($check['host']), 'namespace' => $namespace, 'index_type' => 'document'); | |
| 1269 | 2059 | $source_host = isset($_POST['source_host']) ? sanitize_text_field(wp_unslash($_POST['source_host'])) : ''; |
| 1270 | 2060 | $restart = !empty($_POST['restart']); |
| 1271 | - $state = self::migrate_start($source_host, $target, $restart); | |
| 2061 | + $all_ns = !empty($_POST['all_namespaces']); // e50d2e: Copy every namespace | |
| 2062 | + $state = self::migrate_start($source_host, $target, $restart, $all_ns); | |
| 1272 | 2063 | if (is_wp_error($state)) { |
| 1273 | 2064 | wp_send_json_error(array('message' => $state->get_error_message())); |
| 1274 | 2065 | } |
| 1275 | - if ($source_host !== '' && $source_host !== (string) ($options['mxchat_pinecone_vector_host'] ?? '')) { | |
| 1276 | - self::persist_options(array('mxchat_pinecone_vector_host' => $state['source_host'], 'mxchat_pinecone_vector_index' => $state['source_name'])); | |
| 2066 | + $changes = array('mxchat_pinecone_docs_verified_host' => $target['host']); | |
| 2067 | + if ($source_host !== '' && strtolower($state['source_host']) !== strtolower((string) ($options['mxchat_pinecone_vector_host'] ?? ''))) { | |
| 2068 | + $changes['mxchat_pinecone_vector_host'] = $state['source_host']; | |
| 2069 | + $changes['mxchat_pinecone_vector_index'] = $state['source_name']; | |
| 1277 | 2070 | } |
| 1278 | - wp_send_json_success(array('state' => $state)); | |
| 2071 | + self::persist_options($changes); | |
| 2072 | + wp_send_json_success(array('state' => $state, 'live_index_type' => self::options_index_type($options))); | |
| 1279 | 2073 | } |
| 1280 | 2074 | if ($mode === 'step') { |
| 1281 | 2075 | $state = self::migration_get(); |
| 1282 | 2076 | if (empty($state)) { |
| @@ -1281,8 +2075,9 @@ | ||
| 1281 | 2075 | $state = self::migration_get(); |
| 1282 | 2076 | if (empty($state)) { |
| 1283 | 2077 | wp_send_json_error(array('message' => esc_html__('No migration in progress. Click Migrate to start one.', 'mxchat'))); |
| 1284 | 2078 | } |
| 2079 | + $target = array('api_key' => $api_key, 'host' => (string) ($state['target_host'] ?? ''), 'namespace' => (string) ($state['current'] ?? ($state['namespace'] ?? $namespace)), 'index_type' => 'document'); | |
| 1285 | 2080 | $state = self::migrate_step($state, $target); |
| 1286 | 2081 | if (is_wp_error($state)) { |
| 1287 | 2082 | wp_send_json_error(array('message' => $state->get_error_message())); |
| 1288 | 2083 | } |
| @@ -1288,9 +2083,9 @@ | ||
| 1288 | 2083 | } |
| 1289 | 2084 | if (!empty($state['error'])) { |
| 1290 | 2085 | wp_send_json_error(array('message' => $state['error'], 'state' => $state)); |
| 1291 | 2086 | } |
| 1292 | - wp_send_json_success(array('state' => $state)); | |
| 2087 | + wp_send_json_success(array('state' => $state, 'live_index_type' => self::options_index_type($options))); | |
| 1293 | 2088 | } |
| 1294 | 2089 | wp_send_json_error(array('message' => esc_html__('Unknown migration action.', 'mxchat'))); |
| 1295 | 2090 | } |
| 1296 | 2091 | |
| @@ -1312,8 +2107,32 @@ | ||
| 1312 | 2107 | wp_send_json_error(array('message' => sprintf(esc_html__('The copy count does not match (%1$d of %2$d copied). The old index stays.', 'mxchat'), (int) $state['copied'], (int) $state['total']))); |
| 1313 | 2108 | } |
| 1314 | 2109 | if ($target['index_type'] !== 'document' || strtolower($target['host']) !== strtolower((string) $state['target_host'])) { |
| 1315 | 2110 | wp_send_json_error(array('message' => esc_html__('The saved document index no longer matches the migration. The old index stays.', 'mxchat'))); |
| 2111 | + } | |
| 2112 | + // e50d2e: EVERY namespace that holds records on the old index needs a | |
| 2113 | + // finished copy whose count matches — not just the one copied last. | |
| 2114 | + // Before this, a Multi-Bot or legacy-namespace site could delete | |
| 2115 | + // records that were never copied. | |
| 2116 | + $missing = self::uncopied_namespaces($state, $target['api_key']); | |
| 2117 | + // 5872ef: a bot that still keeps the old index as its own host stops | |
| 2118 | + // answering the moment that index is deleted, however complete the | |
| 2119 | + // copy is. Refuse and name the bots; when namespaces are uncopied too, | |
| 2120 | + // say both in one answer. | |
| 2121 | + $bots = self::bots_on_host((string) ($state['source_host'] ?? '')); | |
| 2122 | + if (!empty($bots)) { | |
| 2123 | + $refusal = array('message' => self::bots_on_host_message($bots), 'bots' => $bots); | |
| 2124 | + if (is_array($missing) && !empty($missing)) { | |
| 2125 | + $refusal['message'] .= ' ' . self::uncopied_message($missing); | |
| 2126 | + $refusal['uncopied'] = $missing; | |
| 2127 | + } | |
| 2128 | + wp_send_json_error($refusal); | |
| 2129 | + } | |
| 2130 | + if (is_wp_error($missing)) { | |
| 2131 | + wp_send_json_error(array('message' => sprintf(__('Could not read the old index before deleting it (%s). The old index stays.', 'mxchat'), $missing->get_error_message()))); | |
| 2132 | + } | |
| 2133 | + if (!empty($missing)) { | |
| 2134 | + wp_send_json_error(array('message' => self::uncopied_message($missing), 'uncopied' => $missing)); | |
| 1316 | 2135 | } |
| 1317 | 2136 | $source = self::find_index_by_host($state['source_host'], $target['api_key']); |
| 1318 | 2137 | if (is_wp_error($source)) { |
| 1319 | 2138 | wp_send_json_error(array('message' => $source->get_error_message())); |
