| @@ -535,9 +535,9 @@ | ||
| 535 | 535 | * @param string $bot_id The bot ID for multi-bot support |
| 536 | 536 | * @param string $content_type The type of content (post, page, pdf, url, manual, product, etc.) |
| 537 | 537 | * @return bool|WP_Error True on success, WP_Error on failure |
| 538 | 538 | */ |
| 539 | -public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content') { | |
| 539 | +public static function submit_content_to_db($content, $source_url, $api_key, $vector_id = null, $bot_id = 'default', $content_type = 'content', $title = '') { | |
| 540 | 540 | global $wpdb; |
| 541 | 541 | $table_name = $wpdb->prefix . 'mxchat_system_prompt_content'; |
| 542 | 542 | |
| 543 | 543 | //error_log('[MXCHAT-DB] Starting database submission for URL: ' . $source_url . ' (Bot: ' . $bot_id . ', Type: ' . $content_type . ')'); |
| @@ -559,8 +559,14 @@ | ||
| 559 | 559 | if (empty($content_type)) { |
| 560 | 560 | $content_type = 'content'; // Fallback for backwards compatibility |
| 561 | 561 | } |
| 562 | 562 | |
| 563 | + // Page title captured by the importer (plan d9ee66), registered under the | |
| 564 | + // SANITIZED url so the storage writers below find it by the same key. | |
| 565 | + if (is_string($title) && trim($title) !== '') { | |
| 566 | + self::set_source_title($source_url, $title); | |
| 567 | + } | |
| 568 | + | |
| 563 | 569 | // Just ensure UTF-8 validity without aggressive escaping |
| 564 | 570 | $safe_content = wp_check_invalid_utf8($content); |
| 565 | 571 | // Remove only null bytes and other control characters, but preserve newlines (\n = \x0A) and carriage returns (\r = \x0D) |
| 566 | 572 | $safe_content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $safe_content); |
| @@ -620,9 +626,150 @@ | ||
| 620 | 626 | return $result; |
| 621 | 627 | } |
| 622 | 628 | } |
| 623 | 629 | |
| 630 | +/* ------------------------------------------------------------------------- | |
| 631 | + * Source titles (3.2.23, plan d9ee66) | |
| 632 | + * | |
| 633 | + * A knowledge entry imported from a URL or sitemap remembers the page's own | |
| 634 | + * title, so an add-on that names a source (AI Search's source cards) can show | |
| 635 | + * "About Us" instead of a guess from the slug ("Aboutus"). Stored in the | |
| 636 | + * `title` column of the WP knowledge table and as `title` metadata on Pinecone | |
| 637 | + * vectors / documents. The importer hands the title to submit_content_to_db(), | |
| 638 | + * which registers it under the source URL; the writers look it up by that key. | |
| 639 | + * ---------------------------------------------------------------------- */ | |
| 640 | + | |
| 641 | +/** @var array<string,string> source_url => title, for the current request */ | |
| 642 | +private static $source_titles = array(); | |
| 643 | + | |
| 644 | +public static function set_source_title($source_url, $title) { | |
| 645 | + $key = trim((string) $source_url); | |
| 646 | + $title = self::clean_title($title); | |
| 647 | + if ($key === '' || $title === '') { | |
| 648 | + return; | |
| 649 | + } | |
| 650 | + self::$source_titles[$key] = $title; | |
| 651 | +} | |
| 652 | + | |
| 653 | +public static function source_title_for($source_url) { | |
| 654 | + $key = trim((string) $source_url); | |
| 655 | + return ($key !== '' && isset(self::$source_titles[$key])) ? self::$source_titles[$key] : ''; | |
| 656 | +} | |
| 657 | + | |
| 658 | +/** Entities decoded, tags gone, whitespace collapsed, capped at the column width. */ | |
| 659 | +public static function clean_title($title) { | |
| 660 | + $title = html_entity_decode((string) $title, ENT_QUOTES | ENT_HTML5, 'UTF-8'); | |
| 661 | + $title = wp_strip_all_tags($title); | |
| 662 | + $title = trim((string) preg_replace('/\s+/u', ' ', $title)); | |
| 663 | + if (mb_strlen($title, 'UTF-8') > 255) { | |
| 664 | + $title = rtrim(mb_substr($title, 0, 255, 'UTF-8')); | |
| 665 | + } | |
| 666 | + return $title; | |
| 667 | +} | |
| 668 | + | |
| 624 | 669 | /** |
| 670 | + * The page's own title from fetched HTML: og:title, else <title> with a | |
| 671 | + * trailing "| Site Name" style suffix removed, else the first <h1>. Returns '' | |
| 672 | + * when nothing usable is there, so the caller keeps its own fallback. | |
| 673 | + */ | |
| 674 | +public static function extract_page_title($html) { | |
| 675 | + $html = (string) $html; | |
| 676 | + if ($html === '') { | |
| 677 | + return ''; | |
| 678 | + } | |
| 679 | + $head = substr($html, 0, 200000); | |
| 680 | + $site = ''; | |
| 681 | + if (preg_match('/<meta\s+[^>]*property=["\']og:site_name["\'][^>]*content=["\']([^"\']{1,120})["\']/i', $head, $m) | |
| 682 | + || preg_match('/<meta\s+[^>]*content=["\']([^"\']{1,120})["\'][^>]*property=["\']og:site_name["\']/i', $head, $m)) { | |
| 683 | + $site = self::clean_title($m[1]); | |
| 684 | + } | |
| 685 | + $candidate = ''; | |
| 686 | + if (preg_match('/<meta\s+[^>]*property=["\']og:title["\'][^>]*content=["\']([^"\']{1,500})["\']/i', $head, $m) | |
| 687 | + || preg_match('/<meta\s+[^>]*content=["\']([^"\']{1,500})["\'][^>]*property=["\']og:title["\']/i', $head, $m)) { | |
| 688 | + $candidate = self::clean_title($m[1]); | |
| 689 | + } | |
| 690 | + if ($candidate === '' && preg_match('/<title[^>]*>(.*?)<\/title>/is', $head, $m)) { | |
| 691 | + $candidate = self::clean_title($m[1]); | |
| 692 | + } | |
| 693 | + if ($candidate !== '') { | |
| 694 | + $candidate = self::strip_site_suffix($candidate, $site); | |
| 695 | + } | |
| 696 | + if ($candidate === '' && preg_match('/<h1[^>]*>(.*?)<\/h1>/is', $html, $m)) { | |
| 697 | + $candidate = self::clean_title($m[1]); | |
| 698 | + } | |
| 699 | + return $candidate; | |
| 700 | +} | |
| 701 | + | |
| 702 | +/** | |
| 703 | + * "About Us | Just Facts" -> "About Us". Only a TRAILING segment after a spaced | |
| 704 | + * separator goes, only when what is left is a real title (3+ characters) and the | |
| 705 | + * removed part is short enough to be a site name; a known og:site_name wins. | |
| 706 | + * Byte offsets throughout: every separator starts with an ASCII space, so a cut | |
| 707 | + * there never lands inside a multibyte character. | |
| 708 | + */ | |
| 709 | +private static function strip_site_suffix($title, $site_name = '') { | |
| 710 | + $seps = array(' | ', ' – ', ' — ', ' - ', ' :: ', ' · ', ' » ', ' • '); | |
| 711 | + if ($site_name !== '') { | |
| 712 | + foreach ($seps as $sep) { | |
| 713 | + $tail = $sep . $site_name; | |
| 714 | + if (strlen($title) > strlen($tail) && substr($title, -strlen($tail)) === $tail) { | |
| 715 | + return trim(substr($title, 0, strlen($title) - strlen($tail))); | |
| 716 | + } | |
| 717 | + } | |
| 718 | + } | |
| 719 | + $best_pos = -1; | |
| 720 | + $best_sep = ''; | |
| 721 | + foreach ($seps as $sep) { | |
| 722 | + $pos = strrpos($title, $sep); | |
| 723 | + if ($pos !== false && $pos > $best_pos) { | |
| 724 | + $best_pos = $pos; | |
| 725 | + $best_sep = $sep; | |
| 726 | + } | |
| 727 | + } | |
| 728 | + if ($best_pos > 0) { | |
| 729 | + $left = trim(substr($title, 0, $best_pos)); | |
| 730 | + $right = trim(substr($title, $best_pos + strlen($best_sep))); | |
| 731 | + if (mb_strlen($left, 'UTF-8') >= 3 && $right !== '' && mb_strlen($right, 'UTF-8') <= 60) { | |
| 732 | + return $left; | |
| 733 | + } | |
| 734 | + } | |
| 735 | + return $title; | |
| 736 | +} | |
| 737 | + | |
| 738 | +/** | |
| 739 | + * Make sure the knowledge table has the `title` column (3.2.23, plan d9ee66). | |
| 740 | + * Activation adds it; this is the safety net for a copy updated in place, and | |
| 741 | + * the one place add-ons ask "may I read/write title?". A positive answer is | |
| 742 | + * remembered in an option, so after the first time this costs one option read. | |
| 743 | + */ | |
| 744 | +public static function ensure_kb_title_column() { | |
| 745 | + global $wpdb; | |
| 746 | + static $ready = null; | |
| 747 | + if ($ready !== null) { | |
| 748 | + return $ready; | |
| 749 | + } | |
| 750 | + if (get_option('mxchat_kb_title_column') === '1') { | |
| 751 | + $ready = true; | |
| 752 | + return true; | |
| 753 | + } | |
| 754 | + $table = $wpdb->prefix . 'mxchat_system_prompt_content'; | |
| 755 | + $cols = $wpdb->get_col("SHOW COLUMNS FROM {$table}", 0); | |
| 756 | + if (empty($cols)) { | |
| 757 | + $ready = false; | |
| 758 | + return false; | |
| 759 | + } | |
| 760 | + if (!in_array('title', $cols, true)) { | |
| 761 | + $wpdb->query("ALTER TABLE {$table} ADD COLUMN title VARCHAR(255) DEFAULT NULL AFTER source_url"); | |
| 762 | + $cols = $wpdb->get_col("SHOW COLUMNS FROM {$table}", 0); | |
| 763 | + } | |
| 764 | + $ready = in_array('title', $cols, true); | |
| 765 | + if ($ready) { | |
| 766 | + update_option('mxchat_kb_title_column', '1', false); | |
| 767 | + } | |
| 768 | + return $ready; | |
| 769 | +} | |
| 770 | + | |
| 771 | +/** | |
| 625 | 772 | * UPDATED: Check if Pinecone is enabled and properly configured for a specific bot |
| 626 | 773 | */ |
| 627 | 774 | private static function is_pinecone_enabled_for_bot($bot_id = 'default') { |
| 628 | 775 | // For default bot or when multi-bot is not active, use original method |
| @@ -763,8 +910,13 @@ | ||
| 763 | 910 | //error_log('[MXCHAT-DB] Manual content - will create new entry (no duplicate check)'); |
| 764 | 911 | } |
| 765 | 912 | // ===== END FIX ===== |
| 766 | 913 | |
| 914 | + // Page title captured at import (plan d9ee66): written only when the | |
| 915 | + // importer supplied one, so a re-index without a title keeps the old value. | |
| 916 | + $title_value = $is_manual_content ? '' : self::source_title_for($source_url); | |
| 917 | + $title_ready = ($title_value !== '') && self::ensure_kb_title_column(); | |
| 918 | + | |
| 767 | 919 | // Progressive fallback mechanism for problematic content |
| 768 | 920 | $attempt = 1; |
| 769 | 921 | $max_attempts = 3; |
| 770 | 922 | $current_content = $safe_content; |
| @@ -775,39 +927,41 @@ | ||
| 775 | 927 | if ($existing_id) { |
| 776 | 928 | //error_log('[MXCHAT-DB] Found existing entry (ID: ' . $existing_id . '). Updating... (Attempt ' . $attempt . ')'); |
| 777 | 929 | |
| 778 | 930 | // Update the existing row - UPDATED 2.5.6: Added content_type |
| 779 | - $result = $wpdb->update( | |
| 780 | - $table_name, | |
| 781 | - array( | |
| 782 | - 'url' => $source_url, | |
| 783 | - 'article_content' => $current_content, | |
| 784 | - 'embedding_vector' => $embedding_vector_serialized, | |
| 785 | - 'source_url' => $source_url, | |
| 786 | - 'content_type' => $content_type, | |
| 787 | - 'timestamp' => current_time('mysql'), | |
| 788 | - ), | |
| 789 | - array('id' => $existing_id), | |
| 790 | - array('%s','%s','%s','%s','%s','%s'), | |
| 791 | - array('%d') | |
| 931 | + $row = array( | |
| 932 | + 'url' => $source_url, | |
| 933 | + 'article_content' => $current_content, | |
| 934 | + 'embedding_vector' => $embedding_vector_serialized, | |
| 935 | + 'source_url' => $source_url, | |
| 936 | + 'content_type' => $content_type, | |
| 937 | + 'timestamp' => current_time('mysql'), | |
| 792 | 938 | ); |
| 939 | + $formats = array('%s','%s','%s','%s','%s','%s'); | |
| 940 | + if ($title_ready) { | |
| 941 | + $row['title'] = $title_value; | |
| 942 | + $formats[] = '%s'; | |
| 943 | + } | |
| 944 | + $result = $wpdb->update($table_name, $row, array('id' => $existing_id), $formats, array('%d')); | |
| 793 | 945 | } else { |
| 794 | 946 | //error_log('[MXCHAT-DB] No existing entry found. Inserting new row... (Attempt ' . $attempt . ')'); |
| 795 | 947 | //error_log('[MXCHAT-DB] Content sample: ' . substr($current_content, 0, 1000)); |
| 796 | 948 | |
| 797 | 949 | // Insert a new row - UPDATED 2.5.6: Added content_type |
| 798 | - $result = $wpdb->insert( | |
| 799 | - $table_name, | |
| 800 | - array( | |
| 801 | - 'url' => $source_url, // Now unique for manual content | |
| 802 | - 'article_content' => $current_content, | |
| 803 | - 'embedding_vector' => $embedding_vector_serialized, | |
| 804 | - 'source_url' => $source_url, // Now unique for manual content | |
| 805 | - 'content_type' => $content_type, | |
| 806 | - 'timestamp' => current_time('mysql'), | |
| 807 | - ), | |
| 808 | - array('%s','%s','%s','%s','%s','%s') | |
| 950 | + $row = array( | |
| 951 | + 'url' => $source_url, // Now unique for manual content | |
| 952 | + 'article_content' => $current_content, | |
| 953 | + 'embedding_vector' => $embedding_vector_serialized, | |
| 954 | + 'source_url' => $source_url, // Now unique for manual content | |
| 955 | + 'content_type' => $content_type, | |
| 956 | + 'timestamp' => current_time('mysql'), | |
| 809 | 957 | ); |
| 958 | + $formats = array('%s','%s','%s','%s','%s','%s'); | |
| 959 | + if ($title_ready) { | |
| 960 | + $row['title'] = $title_value; | |
| 961 | + $formats[] = '%s'; | |
| 962 | + } | |
| 963 | + $result = $wpdb->insert($table_name, $row, $formats); | |
| 810 | 964 | } |
| 811 | 965 | |
| 812 | 966 | if ($result === false) { |
| 813 | 967 | //error_log('[MXCHAT-DB] Database operation failed (Attempt ' . $attempt . ')'); |
| @@ -931,8 +1085,12 @@ | ||
| 931 | 1085 | 'last_updated' => time(), |
| 932 | 1086 | 'created_at' => time(), // Add creation timestamp |
| 933 | 1087 | 'bot_id' => $bot_id, // Add bot identification |
| 934 | 1088 | ); |
| 1089 | + $page_title = self::source_title_for($url); | |
| 1090 | + if ($page_title !== '') { | |
| 1091 | + $metadata['title'] = $page_title; // page title captured at import (plan d9ee66) | |
| 1092 | + } | |
| 935 | 1093 | |
| 936 | 1094 | // Document index (plan 362c31): same id, same metadata, through the |
| 937 | 1095 | // documents API. A site on the classic vector index never enters here. |
| 938 | 1096 | if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) { |
| @@ -1619,8 +1777,12 @@ | ||
| 1619 | 1777 | 'last_updated' => time(), |
| 1620 | 1778 | 'created_at' => time(), |
| 1621 | 1779 | 'bot_id' => $bot_id, |
| 1622 | 1780 | ); |
| 1781 | + $page_title = self::source_title_for($source_url); | |
| 1782 | + if ($page_title !== '') { | |
| 1783 | + $metadata['title'] = $page_title; // page title captured at import (plan d9ee66) | |
| 1784 | + } | |
| 1623 | 1785 | |
| 1624 | 1786 | // Document index (plan 362c31): same chunk id and metadata, documents API. |
| 1625 | 1787 | if (class_exists('MxChat_Pinecone_Documents') && MxChat_Pinecone_Documents::is_document_index($bot_id)) { |
| 1626 | 1788 | return MxChat_Pinecone_Documents::upsert_one($vector_id, $chunk_text, $embedding_vector, $metadata, array( |
| @@ -1673,20 +1835,25 @@ | ||
| 1673 | 1835 | global $wpdb; |
| 1674 | 1836 | |
| 1675 | 1837 | // For chunks, we always insert new rows (no duplicate checking) |
| 1676 | 1838 | // The URL includes chunk info in the metadata, but source_url stays the same for grouping |
| 1677 | - $result = $wpdb->insert( | |
| 1678 | - $table_name, | |
| 1679 | - array( | |
| 1680 | - 'url' => $source_url, | |
| 1681 | - 'article_content' => $content_with_metadata, | |
| 1682 | - 'embedding_vector' => $embedding_vector_serialized, | |
| 1683 | - 'source_url' => $source_url, | |
| 1684 | - 'content_type' => $content_type, | |
| 1685 | - 'timestamp' => current_time('mysql') | |
| 1686 | - ), | |
| 1687 | - array('%s', '%s', '%s', '%s', '%s', '%s') | |
| 1839 | + $row = array( | |
| 1840 | + 'url' => $source_url, | |
| 1841 | + 'article_content' => $content_with_metadata, | |
| 1842 | + 'embedding_vector' => $embedding_vector_serialized, | |
| 1843 | + 'source_url' => $source_url, | |
| 1844 | + 'content_type' => $content_type, | |
| 1845 | + 'timestamp' => current_time('mysql') | |
| 1688 | 1846 | ); |
| 1847 | + $formats = array('%s', '%s', '%s', '%s', '%s', '%s'); | |
| 1848 | + // Every chunk row carries the page title too (plan d9ee66) - readers group by | |
| 1849 | + // source_url and may pick any chunk as the representative. | |
| 1850 | + $page_title = self::source_title_for($source_url); | |
| 1851 | + if ($page_title !== '' && self::ensure_kb_title_column()) { | |
| 1852 | + $row['title'] = $page_title; | |
| 1853 | + $formats[] = '%s'; | |
| 1854 | + } | |
| 1855 | + $result = $wpdb->insert($table_name, $row, $formats); | |
| 1689 | 1856 | |
| 1690 | 1857 | if ($result === false) { |
| 1691 | 1858 | return new WP_Error('database_failed', 'Failed to insert chunk: ' . $wpdb->last_error); |
| 1692 | 1859 | } |
