<?php
/**
 * File: admin/class-knowledge-manager.php
 * 
 * Handles all knowledge base content processing for MxChat
 * Including PDF, sitemap, content processing, and WordPress post management
 */
if (!defined('ABSPATH')) {
    exit; // Exit if accessed directly
}

class MxChat_Knowledge_Manager {

    private $options;

    // Post IDs whose vectors were already deleted by mxchat_handle_status_transition this
    // request, so the transient-based branch in mxchat_handle_post_update can skip the
    // redundant (idempotent but network-visible) second deletion.
    private $transition_deleted_posts = array();

    // Post IDs already INDEXED by mxchat_handle_status_transition's arrival edge this
    // request. Normal editor publishes fire transition_post_status first, then
    // post_updated — without this guard every editor publish would embed twice.
    private $transition_indexed_posts = array();

    // Post IDs core has announced an in-flight UPDATE for. pre_post_update fires only
    // inside wp_insert_post's update branch and always before wp_transition_post_status,
    // so this is an exact "a post_updated is coming later this request" signal — which is
    // what makes it safe to arm transition_indexed_posts (plan a664f3).
    private $pending_post_update = array();

    /**
     * Constructor - Register hooks for content processing
     */
public function __construct() {
    $this->options = get_option('mxchat_options', array());
    $this->mxchat_init_hooks();
    
    $this->mxchat_init_role_hooks();
}
    
/**
 * Initialize WordPress hooks for content processing
 *
 */
private function mxchat_init_hooks() {
    // Admin post handlers for form submissions
    add_action('admin_post_mxchat_submit_content', array($this, 'mxchat_handle_content_submission'));
    add_action('admin_post_mxchat_submit_sitemap', array($this, 'mxchat_handle_sitemap_submission'));
    add_action('admin_post_mxchat_submit_pdf_file', array($this, 'mxchat_handle_pdf_file_submission'));
    add_action('admin_post_mxchat_submit_document_file', array($this, 'mxchat_handle_document_file_submission'));
    add_action('admin_post_mxchat_submit_youtube', array($this, 'mxchat_handle_youtube_submission'));
    add_action('admin_post_mxchat_stop_processing', array($this, 'mxchat_stop_processing'));
    
    // AJAX handlers for real-time processing and status updates
    add_action('wp_ajax_mxchat_get_status_updates', array($this, 'mxchat_ajax_get_status_updates'));
    add_action('wp_ajax_mxchat_dismiss_completed_status', array($this, 'mxchat_ajax_dismiss_completed_status'));
    add_action('wp_ajax_mxchat_get_content_list', array($this, 'ajax_mxchat_get_content_list'));
    add_action('wp_ajax_mxchat_process_selected_content', array($this, 'ajax_mxchat_process_selected_content'));
    add_action('wp_ajax_mxchat_save_inline_prompt', array($this, 'mxchat_save_inline_prompt'));
    add_action('admin_post_mxchat_delete_pinecone_prompt', array($this, 'mxchat_handle_pinecone_prompt_delete'));
    add_action('wp_ajax_mxchat_delete_pinecone_prompt', array($this, 'ajax_mxchat_delete_pinecone_prompt'));
    add_action('wp_ajax_mxchat_delete_chunks_by_url', array($this, 'ajax_mxchat_delete_chunks_by_url'));
    add_action('wp_ajax_mxchat_delete_wordpress_prompt', array($this, 'ajax_mxchat_delete_wordpress_prompt'));
    add_action('wp_ajax_mxchat_bulk_delete_knowledge', array($this, 'ajax_mxchat_bulk_delete_knowledge'));
    add_action('wp_ajax_mxchat_update_role_restriction', array($this, 'ajax_mxchat_update_role_restriction'));
    
    //   Queue-based processing AJAX handlers
    add_action('wp_ajax_mxchat_get_next_queue_item', array($this, 'ajax_mxchat_get_next_queue_item'));
    add_action('wp_ajax_mxchat_process_queue_item', array($this, 'ajax_mxchat_process_queue_item'));
    add_action('wp_ajax_mxchat_get_queue_status', array($this, 'ajax_mxchat_get_queue_status'));
    add_action('wp_ajax_mxchat_clear_queue', array($this, 'ajax_mxchat_clear_queue'));
    add_action('wp_ajax_mxchat_retry_failed', array($this, 'ajax_mxchat_retry_failed'));
    add_action('wp_ajax_mxchat_get_recent_entries', array($this, 'ajax_mxchat_get_recent_entries'));
    add_action('wp_ajax_mxchat_detect_sitemaps', array($this, 'ajax_mxchat_detect_sitemaps'));
    add_action('wp_ajax_mxchat_refresh_pinecone_entries', array($this, 'ajax_mxchat_refresh_pinecone_entries'));
    add_action('wp_ajax_mxchat_paginate_entries', array($this, 'ajax_mxchat_paginate_entries'));
    add_action('wp_ajax_mxchat_get_entry_content', array($this, 'ajax_mxchat_get_entry_content'));
    add_action('wp_ajax_mxchat_save_entry_content', array($this, 'ajax_mxchat_save_entry_content'));
    add_action('wp_ajax_mxchat_inspect_entry', array($this, 'ajax_mxchat_inspect_entry'));

    // WordPress post management hooks
    add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
    add_action('post_updated', array($this, 'mxchat_handle_post_update'), 10, 3);
    add_action('before_delete_post', array($this, 'mxchat_handle_post_delete'));
    add_action('wp_trash_post', array($this, 'mxchat_handle_post_delete'));
    // Authoritative unpublish detection: core hands this hook the REAL previous status, so
    // removal no longer depends on the mxchat_prev_status_* transients (evictable by persistent
    // object caches, never written by paths that bypass wp_update_post, e.g. plugins flipping
    // post_status directly and calling wp_transition_post_status themselves).
    add_action('transition_post_status', array($this, 'mxchat_handle_status_transition'), 10, 3);

    // ACF hook - fires AFTER ACF fields are saved, ensuring ACF data is available
    // Priority 20 to run after ACF's own save (which runs at priority 10)
    add_action('acf/save_post', array($this, 'mxchat_handle_acf_save'), 20);

    // One-time cleanup for vectors orphaned by unpublishes that predate the
    // transition_post_status handler (plan 816fb1): wp mxchat prune-unpublished
    if (defined('WP_CLI') && WP_CLI) {
        WP_CLI::add_command('mxchat prune-unpublished', array($this, 'cli_prune_unpublished'));
        // In-place repair for RTL KB rows imported in visual order before the
        // 32bf9e normalizer existed: wp mxchat rtl-repair (plan d1e6f7)
        WP_CLI::add_command('mxchat rtl-repair', array($this, 'cli_rtl_repair'));
    }

    add_action('wp_ajax_mxchat_mark_queue_complete', array($this, 'ajax_mxchat_mark_queue_complete'));

    // WooCommerce product hooks (if WooCommerce is active)
    if (class_exists('WooCommerce')) {
        add_action('pre_post_update', array($this, 'mxchat_store_pre_update_status'), 10, 2);
        add_action('save_post_product', array($this, 'mxchat_handle_product_change'), 10, 3);
        add_action('wp_trash_post', array($this, 'mxchat_handle_product_delete'));
        add_action('before_delete_post', array($this, 'mxchat_handle_product_delete'));
    }
}
    
    /**
     * Get current options (refreshed)
     */
    private function mxchat_get_options() {
        if (empty($this->options)) {
            $this->options = get_option('mxchat_options', array());
        }
        return $this->options;
    }
    

    // ========================================
    // MAIN CONTENT SUBMISSION HANDLERS
    // ========================================
    
public function mxchat_handle_content_submission() {
    // Check if the form was submitted and the user has permission.
    if (!isset($_POST['submit_content']) || !current_user_can('manage_options')) {
        return;
    }
    
    // Verify the nonce.
    $nonce = isset($_POST['mxchat_submit_content_nonce']) ? sanitize_text_field(wp_unslash($_POST['mxchat_submit_content_nonce'])) : '';
    if (!wp_verify_nonce($nonce, 'mxchat_submit_content_action')) {
        wp_die(esc_html__('Nonce verification failed.', 'mxchat'));
    }
    
    // Sanitize the inputs.
    // Use wp_kses_post to allow safe HTML and wp_unslash to remove WordPress added slashes
    $article_content = wp_kses_post(wp_unslash($_POST['article_content']));
    $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
    
    //  Get bot_id from form submission
    $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
    
    //  Get bot-specific options and API key
    $bot_options = $this->get_bot_options($bot_id);
    $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');

    // Custom-provider-aware decision; keyless custom sites must pass (plan cbd5fd).
    $preflight = MxChat_Utils::embedding_preflight($options);
    if (!$preflight['ok']) {
        set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
        wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
        exit;
    }
    $api_key = $preflight['api_key'];
    
    //  Use centralized utility function with bot_id
    $result = MxChat_Utils::submit_content_to_db($article_content, $article_url, $api_key, null, $bot_id);
    
    if (is_wp_error($result)) {
        set_transient('mxchat_admin_notice_error',
            esc_html__('Error storing content: ', 'mxchat') . $result->get_error_message(),
            30
        );
    } else {
        set_transient('mxchat_admin_notice_success',
            esc_html__('Content successfully submitted!', 'mxchat'),
            30
        );
    }
    
    wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
    exit;
}

/**
 * Handle the "YouTube" KB import source (admin-post form submission).
 *
 * Per-video description mode:
 *  - auto:   fetch oEmbed metadata (reliable) + best-effort captions transcript.
 *            If no usable transcript, index the metadata anyway, tell the admin,
 *            and bounce back with the manual box pre-filled (never fail silently).
 *  - manual: the admin's own description is what gets indexed; metadata rides along.
 *
 * The row is stored with content_type 'youtube' and source_url = the canonical
 * watch URL, so re-importing the same video UPDATES the entry (source_url
 * duplicate handling in MxChat_Utils::store_in_wordpress_db) — that is also the
 * "augment a metadata-only entry" path.
 */
public function mxchat_handle_youtube_submission() {
    if (!isset($_POST['submit_youtube']) || !current_user_can('manage_options')) {
        wp_die(esc_html__('Unauthorized access', 'mxchat'));
    }

    check_admin_referer('mxchat_submit_youtube_action', 'mxchat_submit_youtube_nonce');

    $redirect_url = admin_url('admin.php?page=mxchat-prompts');

    $youtube_url = isset($_POST['youtube_url']) ? esc_url_raw(wp_unslash($_POST['youtube_url'])) : '';
    $video_id = MxChat_Utils::parse_youtube_id($youtube_url);

    if (empty($video_id)) {
        set_transient('mxchat_admin_notice_error',
            esc_html__('That does not look like a link to a single YouTube video. Please paste a watch, youtu.be, or Shorts URL.', 'mxchat'),
            30
        );
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    $canonical_url = 'https://www.youtube.com/watch?v=' . $video_id;

    $description_mode = (isset($_POST['youtube_description_mode']) && $_POST['youtube_description_mode'] === 'manual') ? 'manual' : 'auto';
    $manual_description = isset($_POST['youtube_description']) ? trim(wp_kses_post(wp_unslash($_POST['youtube_description']))) : '';

    $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';

    // Resolve the embedding decision exactly like the sibling handlers —
    // custom-provider-aware (plan cbd5fd).
    $bot_options = $this->get_bot_options($bot_id);
    $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');

    $preflight = MxChat_Utils::embedding_preflight($options);
    if (!$preflight['ok']) {
        set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }
    $api_key = $preflight['api_key'];

    // Metadata is fetched in BOTH modes — it is the reliable half of auto, and in
    // manual mode it enriches the indexed text with the real title/channel.
    $meta = $this->mxchat_fetch_youtube_oembed($video_id);
    $video_title = isset($meta['title']) ? sanitize_text_field($meta['title']) : '';
    $video_channel = isset($meta['author_name']) ? sanitize_text_field($meta['author_name']) : '';

    $header_lines = 'YouTube Video: ' . ($video_title !== '' ? $video_title : $canonical_url) . "\n";
    if ($video_channel !== '') {
        $header_lines .= 'Channel: ' . $video_channel . "\n";
    }
    $header_lines .= 'URL: ' . $canonical_url . "\n\n";

    $transcript_missing = false;

    if ($description_mode === 'manual') {
        if ($manual_description === '') {
            set_transient('mxchat_admin_notice_error',
                esc_html__('Please write a description for the video, or switch to auto-fetch.', 'mxchat'),
                30
            );
            wp_safe_redirect(esc_url($redirect_url));
            exit;
        }
        $indexed_text = $header_lines . $manual_description;
    } else {
        $transcript = $this->mxchat_fetch_youtube_transcript($video_id);

        if (strlen($transcript) >= 200) {
            $indexed_text = $header_lines . $transcript;
        } else {
            // Graceful fallback: captions disabled / blocked / no speech. Auto
            // reliably gets metadata; it does NOT guarantee a transcript.
            $transcript_missing = true;

            if ($video_title === '' && $video_channel === '') {
                // Both halves failed — nothing meaningful to index.
                set_transient('mxchat_admin_notice_error',
                    esc_html__('Could not retrieve any information for that video (no metadata and no captions). Please check the URL, or use the manual description option.', 'mxchat'),
                    30
                );
                wp_safe_redirect(esc_url($redirect_url));
                exit;
            }

            $indexed_text = $header_lines . sprintf(
                /* translators: 1: video title, 2: channel name */
                __('A YouTube video titled "%1$s" from the channel %2$s.', 'mxchat'),
                $video_title !== '' ? $video_title : $canonical_url,
                $video_channel !== '' ? $video_channel : 'YouTube'
            );
        }
    }

    $result = MxChat_Utils::submit_content_to_db($indexed_text, $canonical_url, $api_key, null, $bot_id, 'youtube');

    if (is_wp_error($result)) {
        set_transient('mxchat_admin_notice_error',
            esc_html__('Error storing video in the knowledge base: ', 'mxchat') . $result->get_error_message(),
            30
        );
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    if ($transcript_missing) {
        set_transient('mxchat_admin_notice_success',
            esc_html__('Video indexed from its title and channel — no captions were available for a transcript. The form below is pre-filled: write your own description and import again to improve matching (it updates the same entry).', 'mxchat'),
            30
        );
        // Bounce back with prefill args so the page reopens the YouTube form in
        // manual mode with the URL + fetched title ready to augment.
        $redirect_url = add_query_arg(array(
            'mxchat_yt_prefill' => '1',
            'yt_url'            => rawurlencode($canonical_url),
            'yt_title'          => rawurlencode($video_title),
        ), $redirect_url);
    } else {
        set_transient('mxchat_admin_notice_success',
            esc_html__('YouTube video successfully added to the knowledge base!', 'mxchat'),
            30
        );
    }

    wp_safe_redirect(esc_url_raw($redirect_url));
    exit;
}

/**
 * Fetch YouTube oEmbed metadata for a video (no API key required).
 * Returns the decoded array (title, author_name, thumbnail_url, ...) or array().
 */
private function mxchat_fetch_youtube_oembed($video_id) {
    $oembed_url = 'https://www.youtube.com/oembed?url=' . rawurlencode('https://www.youtube.com/watch?v=' . $video_id) . '&format=json';
    $response = wp_remote_get($oembed_url, array('timeout' => 15));
    if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
        return array();
    }
    $data = json_decode(wp_remote_retrieve_body($response), true);
    return is_array($data) ? $data : array();
}

/**
 * Best-effort captions transcript for a video. Deliberately ISOLATED: this uses
 * YouTube's unofficial timedtext route (the caption track list embedded in the
 * watch page), which YouTube has broken before and will break again. Every
 * failure mode returns '' so a break degrades to the metadata-only import path
 * instead of erroring the whole submission. Do not let anything in here throw.
 */
private function mxchat_fetch_youtube_transcript($video_id) {
    $watch_url = 'https://www.youtube.com/watch?v=' . $video_id . '&hl=en';

    // First try the honest ingest UA; some responses omit the player config for
    // bot UAs, so retry once with a browser UA before giving up.
    $user_agents = array(
        mxchat_ingest_user_agent(),
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
    );

    $tracks = array();
    foreach ($user_agents as $ua) {
        $response = wp_remote_get($watch_url, array(
            'timeout'    => 20,
            'user-agent' => $ua,
        ));
        if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
            continue;
        }
        $body = wp_remote_retrieve_body($response);
        if (!is_string($body) || $body === '' || !preg_match('/"captionTracks":(\[.*?\])(?=,")/s', $body, $m)) {
            continue;
        }
        $decoded = json_decode($m[1], true);
        if (is_array($decoded) && !empty($decoded)) {
            $tracks = $decoded;
            break;
        }
    }

    if (empty($tracks)) {
        return '';
    }

    // Prefer an English track, else take the first offered.
    $chosen = null;
    foreach ($tracks as $track) {
        if (isset($track['languageCode']) && strpos($track['languageCode'], 'en') === 0) {
            $chosen = $track;
            break;
        }
    }
    if ($chosen === null) {
        $chosen = $tracks[0];
    }
    if (empty($chosen['baseUrl']) || !is_string($chosen['baseUrl'])) {
        return '';
    }

    $timedtext = wp_remote_get($chosen['baseUrl'], array('timeout' => 20));
    if (is_wp_error($timedtext) || wp_remote_retrieve_response_code($timedtext) !== 200) {
        return '';
    }
    $xml = wp_remote_retrieve_body($timedtext);
    if (!is_string($xml) || strpos($xml, '<text') === false) {
        return '';
    }

    // <text start=".." dur="..">caption</text> — strip tags, decode the
    // double-encoded entities timedtext ships, collapse whitespace.
    $text = preg_replace('/<[^>]+>/', ' ', $xml);
    $text = html_entity_decode(html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'), ENT_QUOTES | ENT_HTML5, 'UTF-8');
    $text = trim(preg_replace('/\s+/u', ' ', $text));

    return $text;
}

public function mxchat_is_pdf_url($url, $response) {
    $content_type = wp_remote_retrieve_header($response, 'content-type');
    $file_extension = strtolower(pathinfo($url, PATHINFO_EXTENSION));

    // Check Content-Disposition header for .pdf filename (Google Drive sends this)
    $disposition = wp_remote_retrieve_header($response, 'content-disposition');
    $has_pdf_disposition = ! empty($disposition) && stripos($disposition, '.pdf') !== false;

    return strpos($content_type, 'pdf') !== false || $file_extension === 'pdf' || $has_pdf_disposition;
}


public function mxchat_handle_pdf_for_knowledge_base($pdf_url, $response, $bot_id = 'default') {
    if (!current_user_can('manage_options')) {
        return false;
    }

    $pdf_url = esc_url_raw($pdf_url);
    $upload_dir = wp_upload_dir();

    if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
        return false;
    }

    $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
    $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;

    $response_body = wp_remote_retrieve_body($response);
    if (empty($response_body)) {
        return false;
    }

    if (!wp_mkdir_p(dirname($pdf_path))) {
        return false;
    }

    try {
        file_put_contents($pdf_path, $response_body);

        if (!file_exists($pdf_path)) {
            throw new Exception(__('Failed to save PDF file', 'mxchat'));
        }

        $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);
        
        if ($total_pages === false || $total_pages < 1) {
            throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
        }

        // Create unique queue ID
        $queue_id = 'pdf_' . md5($pdf_url . time());

        // Create array of pages to process
        $pages = array();
        for ($i = 1; $i <= $total_pages; $i++) {
            $pages[] = array(
                'pdf_path' => $pdf_path,
                'pdf_url' => $pdf_url,
                'page_number' => $i,
                'total_pages' => $total_pages
            );
        }

        // Add pages to queue
        $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);

        if ($queued_count === 0) {
            wp_delete_file($pdf_path);
            throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
        }

        // Store queue metadata
        $this->mxchat_set_queue_meta($queue_id, 'source_url', $pdf_url);
        $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
        $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
        $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
        $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
        $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));

        // Store queue ID in transient for status tracking
        set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
        set_transient('mxchat_last_pdf_url', $pdf_url, DAY_IN_SECONDS);

        return 'queued';

    } catch (Exception $e) {
        if (file_exists($pdf_path)) {
            wp_delete_file($pdf_path);
        }
        return $e->getMessage();
    }
}

/**
 * Handle direct PDF file upload from the knowledge base page
 */
public function mxchat_handle_pdf_file_submission() {
    if (!isset($_POST['submit_pdf_file']) || !current_user_can('manage_options')) {
        wp_die(esc_html__('Unauthorized access', 'mxchat'));
    }

    check_admin_referer('mxchat_submit_pdf_file_action', 'mxchat_submit_pdf_file_nonce');

    $redirect_url = admin_url('admin.php?page=mxchat-prompts');

    // Validate file upload
    if (empty($_FILES['pdf_file']) || $_FILES['pdf_file']['error'] !== UPLOAD_ERR_OK) {
        $error_code = isset($_FILES['pdf_file']['error']) ? $_FILES['pdf_file']['error'] : UPLOAD_ERR_NO_FILE;
        $error_messages = array(
            UPLOAD_ERR_INI_SIZE   => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
            UPLOAD_ERR_FORM_SIZE  => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
            UPLOAD_ERR_PARTIAL    => __('The file was only partially uploaded.', 'mxchat'),
            UPLOAD_ERR_NO_FILE    => __('No file was uploaded. Please select a PDF file.', 'mxchat'),
            UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
            UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
        );
        $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
        set_transient('mxchat_admin_notice_error', $error_msg, 30);
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    $file = $_FILES['pdf_file'];

    // Validate MIME type
    $finfo = finfo_open(FILEINFO_MIME_TYPE);
    $mime_type = finfo_file($finfo, $file['tmp_name']);
    finfo_close($finfo);

    if ($mime_type !== 'application/pdf') {
        set_transient('mxchat_admin_notice_error',
            esc_html__('Invalid file type. Only PDF files are accepted.', 'mxchat'),
            30
        );
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    // Validate extension
    $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
    if ($ext !== 'pdf') {
        set_transient('mxchat_admin_notice_error',
            esc_html__('Invalid file extension. Only .pdf files are accepted.', 'mxchat'),
            30
        );
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
    $original_filename = sanitize_file_name($file['name']);

    $upload_dir = wp_upload_dir();
    if (isset($upload_dir['error']) && $upload_dir['error'] !== false) {
        set_transient('mxchat_admin_notice_error',
            esc_html__('WordPress upload directory is not writable.', 'mxchat'),
            30
        );
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    $pdf_filename = sanitize_file_name('mxchat_kb_' . time() . '.pdf');
    $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;

    if (!wp_mkdir_p(dirname($pdf_path))) {
        set_transient('mxchat_admin_notice_error',
            esc_html__('Failed to create upload directory.', 'mxchat'),
            30
        );
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    // Move uploaded file
    if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
        set_transient('mxchat_admin_notice_error',
            esc_html__('Failed to save uploaded PDF file.', 'mxchat'),
            30
        );
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    try {
        $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);

        if ($total_pages === false || $total_pages < 1) {
            throw new Exception(__('Invalid PDF: Unable to parse or no pages found', 'mxchat'));
        }

        // Use original filename as the source identifier
        $source_label = 'upload://' . $original_filename;

        $queue_id = 'pdf_' . md5($source_label . time());

        $pages = array();
        for ($i = 1; $i <= $total_pages; $i++) {
            $pages[] = array(
                'pdf_path'    => $pdf_path,
                'pdf_url'     => $source_label,
                'page_number' => $i,
                'total_pages' => $total_pages,
            );
        }

        $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);

        if ($queued_count === 0) {
            wp_delete_file($pdf_path);
            throw new Exception(__('Failed to add PDF pages to processing queue', 'mxchat'));
        }

        $this->mxchat_set_queue_meta($queue_id, 'source_url', $source_label);
        $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'pdf');
        $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_pages);
        $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
        $this->mxchat_set_queue_meta($queue_id, 'pdf_path', $pdf_path);
        $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));

        set_transient('mxchat_active_queue_pdf', $queue_id, DAY_IN_SECONDS);
        set_transient('mxchat_last_pdf_url', $source_label, DAY_IN_SECONDS);

        set_transient('mxchat_admin_notice_success',
            sprintf(
                esc_html__('PDF "%s" (%d pages) queued for processing. Processing will start automatically.', 'mxchat'),
                esc_html($original_filename),
                $total_pages
            ),
            30
        );

    } catch (Exception $e) {
        if (file_exists($pdf_path)) {
            wp_delete_file($pdf_path);
        }
        set_transient('mxchat_admin_notice_error',
            esc_html__('Failed to process uploaded PDF: ', 'mxchat') . esc_html($e->getMessage()),
            30
        );
    }

    wp_safe_redirect(esc_url($redirect_url));
    exit;
}

/**
 * Handle direct document upload (.docx / .txt / .md) from the knowledge base
 * page (plan 0485e5). Unlike PDF Upload there is no per-page queue: the text
 * extracts in one pass and routes through submit_content_to_db, whose chunker
 * takes over for long content. The uploaded file is read from the PHP temp
 * file and never persisted — only its extracted text enters the KB.
 *
 * Source identity matches PDF Upload's scheme: upload://<filename>, stable
 * across re-uploads so a re-import REPLACES (delete_chunks_for_url + upsert
 * per identity) instead of duplicating.
 */
public function mxchat_handle_document_file_submission() {
    if (!isset($_POST['submit_document_file']) || !current_user_can('manage_options')) {
        wp_die(esc_html__('Unauthorized access', 'mxchat'));
    }

    check_admin_referer('mxchat_submit_document_file_action', 'mxchat_submit_document_file_nonce');

    $redirect_url = admin_url('admin.php?page=mxchat-prompts');

    if (empty($_FILES['document_file']) || $_FILES['document_file']['error'] !== UPLOAD_ERR_OK) {
        $error_code = isset($_FILES['document_file']['error']) ? $_FILES['document_file']['error'] : UPLOAD_ERR_NO_FILE;
        $error_messages = array(
            UPLOAD_ERR_INI_SIZE   => __('The uploaded file exceeds the server upload_max_filesize limit.', 'mxchat'),
            UPLOAD_ERR_FORM_SIZE  => __('The uploaded file exceeds the form MAX_FILE_SIZE limit.', 'mxchat'),
            UPLOAD_ERR_PARTIAL    => __('The file was only partially uploaded.', 'mxchat'),
            UPLOAD_ERR_NO_FILE    => __('No file was uploaded. Please select a document.', 'mxchat'),
            UPLOAD_ERR_NO_TMP_DIR => __('Server missing temporary folder.', 'mxchat'),
            UPLOAD_ERR_CANT_WRITE => __('Server failed to write file to disk.', 'mxchat'),
        );
        $error_msg = isset($error_messages[$error_code]) ? $error_messages[$error_code] : __('Unknown upload error.', 'mxchat');
        set_transient('mxchat_admin_notice_error', $error_msg, 30);
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    $file = $_FILES['document_file'];
    $ext  = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));

    $finfo = finfo_open(FILEINFO_MIME_TYPE);
    $mime_type = finfo_file($finfo, $file['tmp_name']);
    finfo_close($finfo);

    // Per-extension MIME expectations. finfo commonly reports .docx as
    // application/zip (it IS a Zip container) and .md as plain text.
    $mime_ok = false;
    if ($ext === 'docx') {
        $mime_ok = in_array($mime_type, array(
            'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
            'application/zip',
        ), true);
    } elseif ($ext === 'txt' || $ext === 'md') {
        $mime_ok = (strpos((string) $mime_type, 'text/') === 0);
    }

    if (!$mime_ok) {
        set_transient('mxchat_admin_notice_error',
            esc_html__('Invalid or unreadable document. Accepted types: .docx, .txt, .md.', 'mxchat'),
            30
        );
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
    $original_filename = sanitize_file_name($file['name']);

    // ---- Extract text (ONE extractor for .docx — the word handler's) ----
    if ($ext === 'docx') {
        $text = MXChat_Word_Handler::extract_docx_text($file['tmp_name']);
        if ($text === false) {
            set_transient('mxchat_admin_notice_error',
                esc_html__('The .docx file could not be read. It may be corrupt, empty, or not a real Word document.', 'mxchat'),
                30
            );
            wp_safe_redirect(esc_url($redirect_url));
            exit;
        }
    } else {
        // .txt / .md read as-is. Markdown keeps its syntax on purpose —
        // headings are useful retrieval signal.
        $text = (string) file_get_contents($file['tmp_name']);
        $text = wp_check_invalid_utf8($text);
        $text = trim($text);
    }

    if ($text === '') {
        set_transient('mxchat_admin_notice_error',
            esc_html__('The uploaded document contains no readable text.', 'mxchat'),
            30
        );
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    // Size cap — same pdf_max_pages setting the PDF/toolbar paths use, but
    // estimated by CHARACTERS (~2500/page): the .docx cleaner collapses all
    // newlines to spaces, so a paragraph count reads 1 for any Word file.
    // Processing is synchronous — an unbounded document risks a timeout.
    $options = get_option('mxchat_options', array());
    $max_pages = isset($options['pdf_max_pages']) ? intval($options['pdf_max_pages']) : 69;
    $estimated_pages = (int) ceil(strlen($text) / 2500);
    if ($estimated_pages > $max_pages) {
        set_transient('mxchat_admin_notice_error',
            sprintf(
                esc_html__('The document is too large (about %1$d pages; the limit is %2$d). Split it into smaller files, or raise the PDF max pages setting.', 'mxchat'),
                $estimated_pages,
                $max_pages
            ),
            30
        );
        wp_safe_redirect(esc_url($redirect_url));
        exit;
    }

    // Embedding API key — bot-aware, same shape as the direct-content handler.
    $api_key = '';
    if ($bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager')) {
        $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
        $api_key = $bot_options['api_key'] ?? '';
    }
    if (empty($api_key)) {
        $api_key = $options['api_key'] ?? '';
    }

    // Stable identity — PDF Upload's scheme. A re-upload of the same filename
    // replaces: clear old chunks first (covers a doc shrinking below the chunk
    // threshold, where the single-vector path would not clean them), then
    // submit — the chunked path re-deletes harmlessly.
    $source_label = 'upload://' . $original_filename;
    MxChat_Utils::delete_chunks_for_url($source_label, $bot_id);
    $result = MxChat_Utils::submit_content_to_db($text, $source_label, $api_key, null, $bot_id, 'document');

    if (is_wp_error($result)) {
        set_transient('mxchat_admin_notice_error',
            esc_html__('Failed to import the document: ', 'mxchat') . esc_html($result->get_error_message()),
            30
        );
    } else {
        set_transient('mxchat_admin_notice_success',
            sprintf(
                esc_html__('Document "%s" imported into the knowledge base.', 'mxchat'),
                esc_html($original_filename)
            ),
            30
        );
    }

    wp_safe_redirect(esc_url($redirect_url));
    exit;
}

/**
 *   Validate PDF and count pages with multiple parser attempts
 */
private function mxchat_validate_and_count_pdf_pages($pdf_path) {
    // Method 1: Try with Smalot PDF Parser (your current method)
    try {
        mxchat_load_pdf_parser();
        $parser = new \Smalot\PdfParser\Parser();
        $pdf = $parser->parseFile($pdf_path);
        $pages = $pdf->getPages();
        $page_count = count($pages);
        
        if ($page_count > 0) {
            //error_log('PDF parsed successfully with Smalot parser: ' . $page_count . ' pages');
            return $page_count;
        }
    } catch (Exception $e) {
        //error_log('Smalot PDF parser failed: ' . $e->getMessage());
    }

    // Method 2: Try with pdfinfo command (if available)
    if (function_exists('shell_exec') && !$this->mxchat_is_shell_disabled()) {
        try {
            $command = 'pdfinfo ' . escapeshellarg($pdf_path) . ' 2>&1';
            $output = shell_exec($command);
            
            if ($output && preg_match('/Pages:\s*(\d+)/', $output, $matches)) {
                $page_count = intval($matches[1]);
                if ($page_count > 0) {
                    //error_log('PDF parsed successfully with pdfinfo: ' . $page_count . ' pages');
                    return $page_count;
                }
            }
        } catch (Exception $e) {
            //error_log('pdfinfo command failed: ' . $e->getMessage());
        }
    }

    // Method 3: Try to repair PDF and parse again
    try {
        $repaired_path = $this->mxchat_attempt_pdf_repair($pdf_path);
        if ($repaired_path && $repaired_path !== $pdf_path) {
            mxchat_load_pdf_parser();
            $parser = new \Smalot\PdfParser\Parser();
            $pdf = $parser->parseFile($repaired_path);
            $pages = $pdf->getPages();
            $page_count = count($pages);
            
            if ($page_count > 0) {
                // Replace original with repaired version
                copy($repaired_path, $pdf_path);
                unlink($repaired_path);
                //error_log('PDF repaired and parsed successfully: ' . $page_count . ' pages');
                return $page_count;
            }
            
            // Clean up repaired file if it didn't work
            unlink($repaired_path);
        }
    } catch (Exception $e) {
        //error_log('PDF repair attempt failed: ' . $e->getMessage());
    }

    // Method 4: Manual PDF structure analysis (basic page count)
    try {
        $page_count = $this->mxchat_manual_pdf_page_count($pdf_path);
        if ($page_count > 0) {
            //error_log('PDF page count determined manually: ' . $page_count . ' pages');
            return $page_count;
        }
    } catch (Exception $e) {
        //error_log('Manual PDF analysis failed: ' . $e->getMessage());
    }

    //error_log('All PDF parsing methods failed for: ' . $pdf_path);
    return false;
}

/**
 *   Check if shell_exec is disabled
 */
private function mxchat_is_shell_disabled() {
    $disabled = explode(',', ini_get('disable_functions'));
    return in_array('shell_exec', $disabled);
}

/**
 *   Attempt to repair PDF using basic methods
 */
private function mxchat_attempt_pdf_repair($pdf_path) {
    try {
        $content = file_get_contents($pdf_path);
        if (!$content) {
            return false;
        }

        // Check if PDF starts with proper header
        if (substr($content, 0, 4) !== '%PDF') {
            // Try to find PDF header in the content
            $header_pos = strpos($content, '%PDF');
            if ($header_pos !== false && $header_pos < 1024) {
                // Remove junk before PDF header
                $content = substr($content, $header_pos);
                $repaired_path = $pdf_path . '.repaired';
                file_put_contents($repaired_path, $content);
                return $repaired_path;
            }
        }

        // Check for EOF marker
        $content = rtrim($content);
        if (!preg_match('/%%EOF\s*$/', $content)) {
            // Add EOF marker if missing
            $content .= "\n%%EOF";
            $repaired_path = $pdf_path . '.repaired';
            file_put_contents($repaired_path, $content);
            return $repaired_path;
        }

    } catch (Exception $e) {
        //error_log('PDF repair error: ' . $e->getMessage());
    }

    return false;
}

/**
 *   Manual PDF page counting by analyzing PDF structure
 */
private function mxchat_manual_pdf_page_count($pdf_path) {
    try {
        $content = file_get_contents($pdf_path);
        if (!$content) {
            return 0;
        }

        // Method 1: Count /Type /Page objects
        $page_count = preg_match_all('/\/Type\s*\/Page[^s]/', $content);
        if ($page_count > 0) {
            return $page_count;
        }

        // Method 2: Look for /Count in pages object
        if (preg_match('/\/Type\s*\/Pages.*?\/Count\s+(\d+)/', $content, $matches)) {
            return intval($matches[1]);
        }

        // Method 3: Count page references
        $page_count = preg_match_all('/\d+\s+0\s+obj\s*<<[^>]*\/Type\s*\/Page/', $content);
        if ($page_count > 0) {
            return $page_count;
        }

    } catch (Exception $e) {
        //error_log('Manual PDF analysis error: ' . $e->getMessage());
    }

    return 0;
}


public function mxchat_save_inline_prompt() {
    // DEBUG: Log what we're receiving
    //error_log('=== MXCHAT DEBUG ===');
    //error_log('POST data: ' . print_r($_POST, true));
    //error_log('Nonce from POST: ' . ($_POST['_ajax_nonce'] ?? 'NOT FOUND'));
    
    // Check for nonce security
    check_ajax_referer('mxchat_save_inline_nonce', '_ajax_nonce');
    
    // If we get here, nonce passed
    //error_log('Nonce verification PASSED');
    
    // Verify permissions
    if (!current_user_can('manage_options')) {
        wp_send_json_error(esc_html__('Permission denied.', 'mxchat'));
        return;
    }
    
    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
    
    // Validate and sanitize input data
    $prompt_id = isset($_POST['id']) ? absint($_POST['id']) : 0;
    $article_content = isset($_POST['article_content']) ? wp_kses_post(wp_unslash($_POST['article_content'])) : '';
    $article_url = isset($_POST['article_url']) ? esc_url_raw($_POST['article_url']) : '';
    
    if ($prompt_id > 0 && !empty($article_content)) {
        // Re-generate the embedding vector for the updated content
        $embedding_vector = $this->mxchat_generate_embedding($article_content);
        if (is_array($embedding_vector)) {
            // Serialize the embedding vector before storing it
            $embedding_vector_serialized = serialize($embedding_vector);
            // Update the prompt in the database
            $updated = $wpdb->update(
                $table_name,
                array(
                    'article_content'   => $article_content,
                    'embedding_vector'  => $embedding_vector_serialized,
                    'source_url'        => $article_url,
                ),
                array('id' => $prompt_id),
                array('%s', '%s', '%s'),
                array('%d')
            );
            if ($updated !== false) {
                wp_send_json_success();
            } else {
                MxChat_Admin::mxchat_log_debug('knowledge_error', 'Database update failed when saving knowledge entry');
                wp_send_json_error(esc_html__('Database update failed.', 'mxchat'));
            }
        } else {
            MxChat_Admin::mxchat_log_debug('embedding_error', 'Embedding generation failed for knowledge entry');
            wp_send_json_error(esc_html__('Embedding generation failed.', 'mxchat'));
        }
    } else {
        wp_send_json_error(esc_html__('Invalid data.', 'mxchat'));
    }
}


/**
 * AJAX: Get full content for editing — reassembles chunks if needed.
 * Works for both WordPress DB and Pinecone entries.
 */
/**
 * Sanitize a knowledge entry's source_url from an AJAX request WITHOUT destroying
 * its identity. sanitize_text_field() strips percent-encoded octets (%20, %D7%A9…),
 * so a percent-encoded URL — every non-ASCII permalink — would md5 to a DIFFERENT
 * id than the one it was stored under: reads miss the entry and saves write an
 * orphan copy while the original keeps its stale text. URLs get esc_url_raw
 * (identity-preserving, matches what import stored); non-URL keys (mxchat://,
 * _ungrouped_) keep the old sanitizer.
 */
private function sanitize_entry_source_url( $raw ) {
    $raw = trim( (string) $raw );
    if ( preg_match( '#^https?://#i', $raw ) ) {
        return esc_url_raw( $raw );
    }
    return sanitize_text_field( $raw );
}

public function ajax_mxchat_get_entry_content() {
    check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');

    if ( ! current_user_can('manage_options') ) {
        wp_send_json_error( array( 'message' => 'Permission denied.' ) );
    }

    $source_url  = $this->sanitize_entry_source_url( isset($_POST['source_url']) ? wp_unslash($_POST['source_url']) : '' );
    $entry_id    = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
    $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
    $bot_id      = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';

    if ( $data_source === 'pinecone' ) {
        // Pinecone ids are strings (md5 hashes, manual_* ids) — absint() would
        // destroy them, so re-read the raw value for this branch only.
        $vector_id = isset($_POST['entry_id']) ? sanitize_text_field( wp_unslash($_POST['entry_id']) ) : '';
        // Pinecone: fetch vectors by source_url, reassemble chunks
        $content = $this->get_pinecone_entry_content( $source_url, $vector_id, $bot_id );
    } else {
        // WordPress DB
        $content = $this->get_wordpress_entry_content( $source_url, $entry_id );
    }

    if ( is_wp_error( $content ) ) {
        wp_send_json_error( array( 'message' => $content->get_error_message() ) );
    }

    wp_send_json_success( $content );
}

/**
 * Get content from WordPress DB — reassembles chunks by source_url.
 */
private function get_wordpress_entry_content( $source_url, $entry_id ) {
    global $wpdb;
    $table = $wpdb->prefix . 'mxchat_system_prompt_content';

    // If we have a source_url, check for chunks
    if ( ! empty( $source_url ) && strpos( $source_url, 'mxchat://' ) !== 0 ) {
        $rows = $wpdb->get_results( $wpdb->prepare(
            "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
            $source_url
        ) );

        if ( $rows && count( $rows ) > 1 ) {
            // Multiple rows = chunked. Reassemble.
            $chunks = array();
            foreach ( $rows as $row ) {
                $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
                $index  = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
                $chunks[ $index ] = $parsed['text'];
            }
            ksort( $chunks );
            return array(
                'content'      => implode( "\n\n", $chunks ),
                'source_url'   => $source_url,
                'is_chunked'   => true,
                'chunk_count'  => count( $chunks ),
                'content_type' => $rows[0]->content_type,
            );
        } elseif ( $rows && count( $rows ) === 1 ) {
            $parsed = MxChat_Chunker::parse_stored_chunk( $rows[0]->article_content );
            return array(
                'content'      => $parsed['text'],
                'source_url'   => $source_url,
                'entry_id'     => $rows[0]->id,
                'is_chunked'   => false,
                'content_type' => $rows[0]->content_type,
            );
        }
    }

    // Fallback: fetch by ID
    if ( $entry_id > 0 ) {
        $row = $wpdb->get_row( $wpdb->prepare(
            "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
            $entry_id
        ) );
        if ( $row ) {
            $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
            return array(
                'content'      => $parsed['text'],
                'source_url'   => $row->source_url,
                'entry_id'     => $row->id,
                'is_chunked'   => false,
                'content_type' => $row->content_type,
            );
        }
    }

    return new WP_Error( 'not_found', 'Entry not found.' );
}

/**
 * Get content from Pinecone — fetches vectors by source_url, reassembles chunks.
 */
private function get_pinecone_entry_content( $source_url, $entry_id, $bot_id ) {
    if ( ! class_exists('MxChat_Pinecone_Manager') ) {
        return new WP_Error( 'pinecone_unavailable', 'Pinecone manager not available.' );
    }

    // Get Pinecone config
    if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
        $pinecone_options = get_option('mxchat_pinecone_addon_options');
        $api_key   = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
        $host      = $pinecone_options['mxchat_pinecone_host'] ?? '';
        $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
    } else {
        $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
        $api_key   = $bot_config['api_key'] ?? '';
        $host      = $bot_config['host'] ?? '';
        $namespace = $bot_config['namespace'] ?? '';
    }

    if ( empty($host) || empty($api_key) ) {
        return new WP_Error( 'pinecone_config', 'Pinecone not configured.' );
    }

    // Manual entries carry no source_url (their vector id is a minted manual_* string,
    // not md5 of anything the row can hand us) — fetch the exact vector instead.
    // '_ungrouped_' is the table view's synthetic display key for such rows.
    if ( ( empty($source_url) || strpos($source_url, '_ungrouped_') === 0 ) && ! empty($entry_id) && is_string($entry_id) ) {
        $vector_ids = array( $entry_id );
    } else {
        // List vectors with the source_url prefix
        $base_id = md5( $source_url );
        $vector_ids = array( $base_id );

        // Find chunk vectors
        // NOTE: Pinecone's /vectors/list is a GET endpoint with query parameters.
        // A POST is answered 200-with-an-empty-body, which reads as "no vectors".
        $list_url    = "https://{$host}/vectors/list";
        $list_params = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
        if ( ! empty($namespace) ) {
            $list_params['namespace'] = $namespace;
        }

        $list_resp = wp_remote_get( $list_url . '?' . http_build_query( $list_params ), array(
            'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
            'timeout' => 15,
        ) );

        if ( ! is_wp_error($list_resp) ) {
            $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
            if ( ! empty($list_data['vectors']) ) {
                foreach ( $list_data['vectors'] as $v ) {
                    $vector_ids[] = $v['id'];
                }
            }
        }
    }

    // Fetch vectors with metadata
    // NOTE: /vectors/fetch is a GET endpoint too, and Pinecone expects the ids
    // repeated (ids=a&ids=b) — http_build_query would emit ids[0]=a, so build
    // the query string explicitly.
    $fetch_query = array();
    foreach ( $vector_ids as $fetch_vid ) {
        $fetch_query[] = 'ids=' . rawurlencode( $fetch_vid );
    }
    if ( ! empty($namespace) ) {
        $fetch_query[] = 'namespace=' . rawurlencode( $namespace );
    }

    $fetch_resp = wp_remote_get( "https://{$host}/vectors/fetch?" . implode( '&', $fetch_query ), array(
        'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
        'timeout' => 15,
    ) );

    if ( is_wp_error($fetch_resp) ) {
        return new WP_Error( 'pinecone_fetch', 'Failed to fetch from Pinecone.' );
    }

    $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
    $vectors    = $fetch_data['vectors'] ?? array();

    if ( empty($vectors) ) {
        return new WP_Error( 'not_found', 'Entry not found in Pinecone.' );
    }

    // Reassemble chunks
    $chunks = array();
    $content_type = 'content';
    foreach ( $vectors as $vid => $vector ) {
        $meta  = $vector['metadata'] ?? array();
        $text  = $meta['text'] ?? '';
        $index = $meta['chunk_index'] ?? 0;
        $content_type = $meta['type'] ?? 'content';
        $chunks[ intval($index) ] = $text;
    }
    ksort( $chunks );

    return array(
        'content'      => implode( "\n\n", $chunks ),
        'source_url'   => $source_url,
        'is_chunked'   => count($chunks) > 1,
        'chunk_count'  => count($chunks),
        'content_type' => $content_type,
    );
}

/**
 * AJAX: Inspect a knowledge entry — returns the per-chunk stored text + metadata
 * WITHOUT collapsing it, so a site owner can see exactly what was indexed for an
 * entry (plan-mxchat-20260628-d8cb4b). READ-ONLY: never re-embeds or mutates.
 */
public function ajax_mxchat_inspect_entry() {
    check_ajax_referer('mxchat_inspect_entry_nonce', 'nonce');

    if ( ! current_user_can('manage_options') ) {
        wp_send_json_error( array( 'message' => esc_html__('Permission denied.', 'mxchat') ) );
    }

    $source_url  = isset($_POST['source_url']) ? sanitize_text_field( wp_unslash($_POST['source_url']) ) : '';
    $entry_id    = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
    $data_source = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
    $bot_id      = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';

    if ( $data_source === 'pinecone' ) {
        $result = $this->inspect_pinecone_entry( $source_url, $entry_id, $bot_id );
    } else {
        $result = $this->inspect_wordpress_entry( $source_url, $entry_id );
    }

    if ( is_wp_error( $result ) ) {
        wp_send_json_error( array( 'message' => $result->get_error_message() ) );
    }

    wp_send_json_success( $result );
}

/**
 * Read-only inspector for WordPress-DB entries. Mirrors get_wordpress_entry_content()
 * but returns each STORED chunk's exact text + length (no implode), plus the assembled
 * embedded text. This shows what is actually in the index, not a re-derivation from the post.
 */
private function inspect_wordpress_entry( $source_url, $entry_id ) {
    global $wpdb;
    $table = $wpdb->prefix . 'mxchat_system_prompt_content';

    $rows = array();

    // Group by the real stored source_url — this INCLUDES "mxchat://" manual
    // Direct Content entries (the spec's manual-entry case), which share one
    // source_url across their chunk rows. Only the synthetic "_ungrouped_<id>"
    // display key (invented by the table view for rows with no source_url) is
    // excluded; those fall through to the entry_id lookup below.
    if ( ! empty( $source_url ) && strpos( $source_url, '_ungrouped_' ) !== 0 ) {
        $rows = $wpdb->get_results( $wpdb->prepare(
            "SELECT id, article_content, source_url, content_type FROM {$table} WHERE source_url = %s ORDER BY id ASC",
            $source_url
        ) );
    }

    // Fallback / manual "Direct Content" entries: fetch the single row by id.
    if ( empty( $rows ) && $entry_id > 0 ) {
        $row = $wpdb->get_row( $wpdb->prepare(
            "SELECT id, article_content, source_url, content_type FROM {$table} WHERE id = %d",
            $entry_id
        ) );
        if ( $row ) {
            $rows = array( $row );
        }
    }

    if ( empty( $rows ) ) {
        return new WP_Error( 'not_found', esc_html__('Entry not found in the local knowledge database.', 'mxchat') );
    }

    $chunks       = array();
    $content_type = '';
    foreach ( $rows as $row ) {
        $parsed = MxChat_Chunker::parse_stored_chunk( $row->article_content );
        $text   = isset( $parsed['text'] ) ? $parsed['text'] : '';
        $index  = isset( $parsed['metadata']['chunk_index'] ) ? intval( $parsed['metadata']['chunk_index'] ) : count( $chunks );
        $content_type = $row->content_type;
        $chunks[] = array(
            'index'  => $index,
            'text'   => $text,
            'length' => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
            'row_id' => intval( $row->id ),
        );
    }

    usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );

    $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );

    return array(
        'store'            => 'wordpress',
        'source_url'       => $source_url,
        'content_type'     => $content_type,
        'is_chunked'       => count( $chunks ) > 1,
        'chunk_count'      => count( $chunks ),
        'assembled'        => $assembled,
        'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
        'chunks'           => array_values( $chunks ),
        // WP-DB storage carries no separate vector metadata; surface that fact
        // rather than letting the owner guess (the spec's taxonomy question).
        'metadata'         => array(),
        'metadata_note'    => esc_html__('Stored in the local WordPress database. Only the assembled text shown here is embedded — there are no separate vector metadata fields (e.g. taxonomy terms are not stored unless they were injected into the text itself).', 'mxchat'),
    );
}

/**
 * Read-only inspector for Pinecone entries. Mirrors get_pinecone_entry_content()
 * but keeps each vector's text + metadata instead of imploding, so the owner can
 * confirm exactly which metadata fields (text/source_url/type/last_updated/created_at/bot_id)
 * are present per chunk. READ-ONLY.
 */
private function inspect_pinecone_entry( $source_url, $entry_id, $bot_id ) {
    if ( ! class_exists('MxChat_Pinecone_Manager') ) {
        return new WP_Error( 'pinecone_unavailable', esc_html__('Pinecone manager not available.', 'mxchat') );
    }

    if ( $bot_id === 'default' || ! class_exists('MxChat_Multi_Bot_Manager') ) {
        $pinecone_options = get_option('mxchat_pinecone_addon_options');
        $api_key   = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
        $host      = $pinecone_options['mxchat_pinecone_host'] ?? '';
        $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';
    } else {
        $bot_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
        $api_key   = $bot_config['api_key'] ?? '';
        $host      = $bot_config['host'] ?? '';
        $namespace = $bot_config['namespace'] ?? '';
    }

    if ( empty($host) || empty($api_key) ) {
        return new WP_Error( 'pinecone_config', esc_html__('Pinecone not configured.', 'mxchat') );
    }

    $base_id    = md5( $source_url );
    $vector_ids = array( $base_id );

    // NOTE: Pinecone's /vectors/list is a GET endpoint with query parameters.
    // A POST is answered 200-with-an-empty-body, which reads as "no vectors".
    $list_url    = "https://{$host}/vectors/list";
    $list_params = array( 'prefix' => $base_id . '_chunk_', 'limit' => 100 );
    if ( ! empty($namespace) ) {
        $list_params['namespace'] = $namespace;
    }

    $list_resp = wp_remote_get( $list_url . '?' . http_build_query( $list_params ), array(
        'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
        'timeout' => 15,
    ) );

    if ( ! is_wp_error($list_resp) ) {
        $list_data = json_decode( wp_remote_retrieve_body($list_resp), true );
        if ( ! empty($list_data['vectors']) ) {
            foreach ( $list_data['vectors'] as $v ) {
                $vector_ids[] = $v['id'];
            }
        }
    }

    // NOTE: /vectors/fetch is a GET endpoint too, and Pinecone expects the ids
    // repeated (ids=a&ids=b) — http_build_query would emit ids[0]=a, so build
    // the query string explicitly.
    $fetch_query = array();
    foreach ( $vector_ids as $fetch_vid ) {
        $fetch_query[] = 'ids=' . rawurlencode( $fetch_vid );
    }
    if ( ! empty($namespace) ) {
        $fetch_query[] = 'namespace=' . rawurlencode( $namespace );
    }

    $fetch_resp = wp_remote_get( "https://{$host}/vectors/fetch?" . implode( '&', $fetch_query ), array(
        'headers' => array( 'Api-Key' => $api_key, 'accept' => 'application/json' ),
        'timeout' => 15,
    ) );

    if ( is_wp_error($fetch_resp) ) {
        return new WP_Error( 'pinecone_fetch', esc_html__('Failed to fetch from Pinecone.', 'mxchat') );
    }

    $fetch_data = json_decode( wp_remote_retrieve_body($fetch_resp), true );
    $vectors    = $fetch_data['vectors'] ?? array();

    if ( empty($vectors) ) {
        return new WP_Error( 'not_found', esc_html__('Entry not found in Pinecone.', 'mxchat') );
    }

    // Whitelisted metadata fields the spec calls out — shown so devs can confirm
    // what is (and is NOT) stored per vector.
    $meta_fields  = array( 'text', 'source_url', 'type', 'last_updated', 'created_at', 'bot_id', 'chunk_index', 'total_chunks' );
    $chunks       = array();
    $content_type = '';
    foreach ( $vectors as $vid => $vector ) {
        $meta  = isset($vector['metadata']) && is_array($vector['metadata']) ? $vector['metadata'] : array();
        $text  = $meta['text'] ?? '';
        $index = isset($meta['chunk_index']) ? intval($meta['chunk_index']) : count($chunks);
        $content_type = $meta['type'] ?? $content_type;

        $clean_meta = array();
        foreach ( $meta_fields as $field ) {
            if ( array_key_exists( $field, $meta ) && $field !== 'text' ) {
                $clean_meta[ $field ] = is_scalar( $meta[ $field ] ) ? (string) $meta[ $field ] : wp_json_encode( $meta[ $field ] );
            }
        }

        $chunks[] = array(
            'index'     => $index,
            'text'      => $text,
            'length'    => function_exists('mb_strlen') ? mb_strlen( $text ) : strlen( $text ),
            'vector_id' => (string) $vid,
            'metadata'  => $clean_meta,
        );
    }

    usort( $chunks, function( $a, $b ) { return $a['index'] - $b['index']; } );

    $assembled = implode( "\n\n", wp_list_pluck( $chunks, 'text' ) );

    return array(
        'store'            => 'pinecone',
        'source_url'       => $source_url,
        'content_type'     => $content_type,
        'is_chunked'       => count( $chunks ) > 1,
        'chunk_count'      => count( $chunks ),
        'assembled'        => $assembled,
        'assembled_length' => function_exists('mb_strlen') ? mb_strlen( $assembled ) : strlen( $assembled ),
        'chunks'           => array_values( $chunks ),
        'metadata'         => array(),
        'metadata_note'    => esc_html__('Stored in Pinecone. Each chunk above lists the vector metadata fields actually present — if a field you expect (such as taxonomy terms) is missing here, it was not stored as metadata and is only searchable if it appears in the embedded text.', 'mxchat'),
    );
}

/**
 * AJAX: Save edited content — re-chunks and re-embeds as needed.
 * Works for both WordPress DB and Pinecone entries.
 */
public function ajax_mxchat_save_entry_content() {
    check_ajax_referer('mxchat_edit_entry_nonce', 'nonce');

    if ( ! current_user_can('manage_options') ) {
        wp_send_json_error( array( 'message' => 'Permission denied.' ) );
    }

    $source_url   = $this->sanitize_entry_source_url( isset($_POST['source_url']) ? wp_unslash($_POST['source_url']) : '' );
    $entry_id     = isset($_POST['entry_id']) ? absint($_POST['entry_id']) : 0;
    $content      = isset($_POST['content']) ? wp_kses_post( wp_unslash($_POST['content']) ) : '';
    $data_source  = isset($_POST['data_source']) ? sanitize_key($_POST['data_source']) : 'wordpress';
    $bot_id       = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
    $content_type = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : 'content';

    if ( empty($content) ) {
        wp_send_json_error( array( 'message' => 'Content cannot be empty.' ) );
    }

    // Get the embedding API key
    $options = get_option('mxchat_options', array());
    $api_key = '';

    if ( $bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager') ) {
        $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
        $api_key = $bot_options['api_key'] ?? '';
    }
    if ( empty($api_key) ) {
        $api_key = $options['api_key'] ?? '';
    }

    if ( $data_source === 'pinecone' ) {
        // Pinecone branch. The WP-DB manual-entry delete below must never run here:
        // Pinecone ids are strings, and absint() on a digit-leading md5 hash would
        // yield a real (unrelated) WP row id.
        $raw_vector_id     = isset($_POST['entry_id']) ? sanitize_text_field( wp_unslash($_POST['entry_id']) ) : '';
        $is_manual_single  = empty($source_url) || strpos($source_url, '_ungrouped_') === 0;
        $is_manual_chunked = strpos($source_url, 'mxchat://') === 0;

        if ( $is_manual_single || $is_manual_chunked ) {
            // Manual content: remove the old vectors first, then store as fresh manual
            // content — submit_content_to_db mints a new unique identity (manual_* id
            // for a single vector, an mxchat:// chunk prefix if it now chunks).
            if ( $is_manual_chunked ) {
                // Minted identity: base + chunk vectors share the md5(mxchat://...) prefix.
                MxChat_Utils::delete_chunks_for_url( $source_url, $bot_id );
            } elseif ( ! empty($raw_vector_id) && class_exists('MxChat_Pinecone_Manager') ) {
                $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
                $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options( $bot_id );
                if ( ! empty($pinecone_options['mxchat_pinecone_api_key']) && ! empty($pinecone_options['mxchat_pinecone_host']) ) {
                    $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
                        $raw_vector_id,
                        $pinecone_options['mxchat_pinecone_api_key'],
                        $pinecone_options['mxchat_pinecone_host'],
                        $pinecone_options['mxchat_pinecone_namespace'] ?? ''
                    );
                }
            }
            $result = MxChat_Utils::submit_content_to_db( $content, '', $api_key, null, $bot_id, $content_type );
        } else {
            // URL-sourced entry: identity is md5(source_url). submit_content_to_db
            // handles delete-old-chunks → re-chunk → re-embed → store, and sweeps
            // stale chunk vectors when the content now fits in a single vector.
            $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, md5($source_url), $bot_id, $content_type );
        }
    } else {
        global $wpdb;
        $table = $wpdb->prefix . 'mxchat_system_prompt_content';

        // If source_url is empty but we have an entry_id, look it up
        if ( empty($source_url) && $entry_id > 0 ) {
            $row = $wpdb->get_row( $wpdb->prepare( "SELECT source_url FROM {$table} WHERE id = %d", $entry_id ) );
            if ( $row && ! empty($row->source_url) ) {
                $source_url = $row->source_url;
            }
        }

        // For manual entries (no source_url or mxchat:// prefix), delete the old entry by ID first
        // so submit_content_to_db creates a replacement instead of a duplicate
        // Also treat legacy mxchat.ai source URLs as manual — old bug assigned the site URL to manual entries
        $is_legacy_manual = !empty($source_url) && strpos($source_url, 'mxchat.ai') !== false && strpos($source_url, 'mxchat://') !== 0;
        if ( $entry_id > 0 && (empty($source_url) || strpos($source_url, 'mxchat://') === 0 || $is_legacy_manual) ) {
            $wpdb->delete( $table, array( 'id' => $entry_id ), array( '%d' ) );
            // Clear legacy URL so submit_content_to_db generates a unique mxchat:// identifier
            // instead of reusing the shared URL (which would mass-delete other entries with the same URL)
            if ( $is_legacy_manual ) {
                $source_url = '';
            }
        }

        // Use the existing submit_content_to_db which handles chunking, Pinecone, and WP DB
        $vector_id = ! empty($source_url) ? md5($source_url) : md5('mxchat_manual_' . $entry_id);

        // submit_content_to_db already handles: delete old chunks → re-chunk → re-embed → store
        $result = MxChat_Utils::submit_content_to_db( $content, $source_url, $api_key, $vector_id, $bot_id, $content_type );
    }

    if ( is_wp_error($result) ) {
        wp_send_json_error( array( 'message' => $result->get_error_message() ) );
    }

    wp_send_json_success( array( 'message' => 'Content saved and re-embedded successfully.' ) );
}

public function mxchat_get_pdf_processing_status($pdf_url) {
    $pdf_url = esc_url_raw($pdf_url);
    $status = get_transient(sanitize_key('mxchat_pdf_status_' . md5($pdf_url)));

    if (!$status || !is_array($status)) {
        return false;
    }

    // Check for stalled processing (no updates for 5 minutes)
    if ($status['status'] === 'processing' && (time() - absint($status['last_update'])) > 300) {
        $status['status'] = 'error';
        $status['error'] = __('PDF processing appears to be stalled. No updates for over 5 minutes.', 'mxchat');
        
        // Save the updated status
        set_transient(
            sanitize_key('mxchat_pdf_status_' . md5($pdf_url)),
            array_map('sanitize_text_field', $status),
            DAY_IN_SECONDS
        );
    }

    $result = array(
        'total_pages' => absint($status['total_pages']),
        'processed_pages' => absint($status['processed_pages']),
        'failed_pages' => absint($status['failed_pages'] ?? 0),
        'percentage' => ($status['total_pages'] > 0)
            ? round((absint($status['processed_pages']) / absint($status['total_pages'])) * 100)
            : 0,
        'status' => sanitize_text_field($status['status']),
        'last_update' => human_time_diff(absint($status['last_update']), time()) . ' ' . esc_html__('ago', 'mxchat'),
        'failed_pages_list' => isset($status['failed_pages_list']) ? $status['failed_pages_list'] : array(),
        'completion_summary' => isset($status['completion_summary']) ? $status['completion_summary'] : null
    );
    
    // Add error message if present
    if (isset($status['error']) && !empty($status['error'])) {
        $result['error'] = sanitize_text_field($status['error']);
    }
    
    return $result;
}

    
public function mxchat_handle_sitemap_submission() {
    // Check if the form was submitted and verify permissions
    if (!isset($_POST['submit_sitemap']) || !current_user_can('manage_options')) {
        wp_die(esc_html__('Unauthorized access', 'mxchat'));
    }

    // Verify nonce
    check_admin_referer('mxchat_submit_sitemap_action', 'mxchat_submit_sitemap_nonce');

    // Validate URL
    if (!isset($_POST['sitemap_url']) || empty($_POST['sitemap_url'])) {
        set_transient('mxchat_admin_notice_error',
            esc_html__('Please provide a valid URL.', 'mxchat'),
            30
        );
        wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
        exit;
    }

    $submitted_url = esc_url_raw($_POST['sitemap_url']);

    // Convert Google Drive sharing URLs to direct download URLs
    if ( strpos($submitted_url, 'drive.google.com') !== false ) {
        $file_id = '';
        if ( preg_match('/[?&]id=([a-zA-Z0-9_-]+)/', $submitted_url, $m) ) {
            $file_id = $m[1];
        } elseif ( preg_match('#/file/d/([a-zA-Z0-9_-]+)#', $submitted_url, $m) ) {
            $file_id = $m[1];
        }
        if ( ! empty($file_id) ) {
            $submitted_url = 'https://drive.google.com/uc?export=download&id=' . $file_id;
        }
    }

    // Get bot_id from form submission
    $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';

    // Get bot-specific options and validate the embedding decision —
    // custom-provider-aware (plan cbd5fd).
    $bot_options = $this->get_bot_options($bot_id);
    $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');

    $preflight = MxChat_Utils::embedding_preflight($options);
    if (!$preflight['ok']) {
        set_transient('mxchat_admin_notice_error', esc_html($preflight['reason']), 30);
        wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
        exit;
    }
    $api_key = $preflight['api_key'];

    // Fetch URL — send an honest, versioned MXChat crawler UA (not a spoofed
    // browser). Stale browser UAs are exactly what WAFs like SiteGround's
    // ModSecurity flag as scrapers, 403-ing the fetch (including PDFs served
    // from the site's own media library, which route through this same call).
    // See mxchat_ingest_user_agent(). Accept is kept for content negotiation;
    // the browser-only Accept-Language fingerprint is dropped so it stays
    // coherent with a bot identity.
    $response = wp_remote_get($submitted_url, array(
        'timeout' => 30,
        'sslverify' => false,
        'user-agent' => mxchat_ingest_user_agent(),
        'headers' => array(
            'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        ),
    ));

    if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
        $error_message = is_wp_error($response) ? $response->get_error_message() : 'HTTP Status: ' . wp_remote_retrieve_response_code($response);
        set_transient('mxchat_admin_notice_error',
            sprintf(
                esc_html__('Failed to fetch the URL: %s', 'mxchat'),
                esc_html($error_message)
            ),
            30
        );
        wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
        exit;
    }

    $content_type = wp_remote_retrieve_header($response, 'content-type');
    $body_content = wp_remote_retrieve_body($response);

    if (empty($body_content)) {
        set_transient('mxchat_admin_notice_error',
            esc_html__('Empty response received from URL.', 'mxchat'),
            30
        );
        wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
        exit;
    }

    // Handle PDF URL
    if ($this->mxchat_is_pdf_url($submitted_url, $response)) {
        $result = $this->mxchat_handle_pdf_for_knowledge_base($submitted_url, $response, $bot_id);

        if ($result === 'queued') {
            set_transient('mxchat_admin_notice_success',
                esc_html__('PDF queued for processing. Processing will start automatically.', 'mxchat'),
                30
            );
        } else {
            set_transient('mxchat_admin_notice_error',
                esc_html__('Failed to queue PDF processing: ', 'mxchat') . esc_html($result),
                30
            );
        }

        wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
        exit;
    }

    // Handle Sitemap XML
    if (strpos($content_type, 'xml') !== false || strpos($body_content, '<urlset') !== false) {
        libxml_use_internal_errors(true);
        $xml = simplexml_load_string($body_content);
        $xml_errors = libxml_get_errors();
        libxml_clear_errors();

        if ($xml === false || !empty($xml_errors)) {
            set_transient('mxchat_admin_notice_error',
                esc_html__('Invalid sitemap XML. Please provide a valid sitemap.', 'mxchat'),
                30
            );
            wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
            exit;
        }

        $result = $this->mxchat_handle_sitemap_for_knowledge_base($xml, $submitted_url, $bot_id);

        if ($result === 'queued') {
            set_transient('mxchat_admin_notice_success',
                esc_html__('Sitemap queued for processing. Processing will start automatically.', 'mxchat'),
                30
            );
        } else {
            // Surface the reason the handler already computed (embedding pre-flight,
            // empty sitemap, queue failure). The old message pointed at the status
            // area, which is empty on this path — nothing was ever queued.
            if (is_string($result) && $result !== '') {
                set_transient('mxchat_admin_notice_error',
                    esc_html__('Failed to queue sitemap processing: ', 'mxchat') . esc_html($result),
                    30
                );
            } else {
                set_transient('mxchat_admin_notice_error',
                    esc_html__('Failed to queue sitemap processing.', 'mxchat'),
                    30
                );
            }
        }

        wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
        exit;
    }

    // Handle Regular URL (single page)
    $page_content = $this->mxchat_extract_main_content($body_content);
    $sanitized_content = $this->mxchat_sanitize_content_for_api($page_content);

    //error_log('[MXCHAT-URL-DEBUG] URL: ' . $submitted_url);
    //error_log('[MXCHAT-URL-DEBUG] Extracted page_content length: ' . strlen($page_content));
    //error_log('[MXCHAT-URL-DEBUG] Sanitized content length: ' . strlen($sanitized_content));
    //error_log('[MXCHAT-URL-DEBUG] Content preview: ' . substr($sanitized_content, 0, 500));

    if (empty($sanitized_content)) {
        set_transient('mxchat_admin_notice_error',
            esc_html__('No valid content found on the provided URL.', 'mxchat'),
            30
        );
        wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
        exit;
    }

    // For single URLs, process immediately using submit_content_to_db
    // This handles chunking automatically for large content
    $db_result = MxChat_Utils::submit_content_to_db(
        $sanitized_content,
        $submitted_url,
        $api_key,
        null,
        $bot_id,
        'url' // content_type
    );

    if (is_wp_error($db_result)) {
        $error_message = esc_html__('Failed to store content in database: ', 'mxchat') . esc_html($db_result->get_error_message());
        set_transient('mxchat_admin_notice_error', $error_message, 30);
    } else {
        $success_message = esc_html__('URL content successfully submitted!', 'mxchat');
        set_transient('mxchat_admin_notice_success', $success_message, 30);
    }

    wp_safe_redirect(esc_url(admin_url('admin.php?page=mxchat-prompts')));
    exit;
}


public function mxchat_get_single_url_status() {
    $status = get_transient('mxchat_single_url_status');
    if (!$status) {
        return null;
    }
    
    // Add human-readable time
    if (isset($status['timestamp'])) {
        $status['human_time'] = human_time_diff(strtotime($status['timestamp']), current_time('timestamp')) . ' ' . __('ago', 'mxchat');
    }
    
    return $status;
}

public function mxchat_handle_sitemap_for_knowledge_base($xml, $sitemap_url, $bot_id = 'default') {
    if (!current_user_can('manage_options')) {
        return false;
    }

    try {
        $sitemap_url = esc_url_raw($sitemap_url);

        if (!$xml || !is_object($xml)) {
            throw new Exception(__('Invalid XML object provided', 'mxchat'));
        }

        // Get bot-specific embedding API for validation
        $bot_options = $this->get_bot_options($bot_id);
        $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
        
        // Test the embedding API before processing
        $test_phrase = "Test embedding generation for MxChat";
        $test_result = $this->mxchat_generate_embedding($test_phrase, $bot_id);
        
        if (is_string($test_result)) {
            throw new Exception(__('Embedding API validation failed: ', 'mxchat') . $test_result);
        }
        
        if (!is_array($test_result)) {
            throw new Exception(__('Embedding API returned unexpected result type. Please check your configuration.', 'mxchat'));
        }

        // Extract URLs from sitemap
        $urls = array();
        foreach ($xml->url as $url_element) {
            $url = esc_url_raw((string)$url_element->loc);
            if ($url) {
                $urls[] = array('url' => $url);
            }
        }

        $total_urls = count($urls);

        if ($total_urls < 1) {
            throw new Exception(__('No valid URLs found in sitemap', 'mxchat'));
        }

        // Create unique queue ID
        $queue_id = 'sitemap_' . md5($sitemap_url . time());

        // Add URLs to queue
        $queued_count = $this->mxchat_add_to_queue($queue_id, 'url', $urls, $bot_id);

        if ($queued_count === 0) {
            throw new Exception(__('Failed to add URLs to processing queue', 'mxchat'));
        }

        // Store queue metadata
        $this->mxchat_set_queue_meta($queue_id, 'source_url', $sitemap_url);
        $this->mxchat_set_queue_meta($queue_id, 'queue_type', 'sitemap');
        $this->mxchat_set_queue_meta($queue_id, 'total_items', $total_urls);
        $this->mxchat_set_queue_meta($queue_id, 'bot_id', $bot_id);
        $this->mxchat_set_queue_meta($queue_id, 'created_at', current_time('mysql'));

        // Store queue ID in transient for status tracking
        set_transient('mxchat_active_queue_sitemap', $queue_id, DAY_IN_SECONDS);
        set_transient('mxchat_last_sitemap_url', $sitemap_url, DAY_IN_SECONDS);

        return 'queued';

    } catch (Exception $e) {
        $error_message = $e->getMessage();
        //error_log(sprintf(esc_html__('Error preparing sitemap for processing: %s', 'mxchat'), esc_html($error_message)));
        
        return $error_message;
    }
    
}

/**
 * Remove shortcode tags but preserve the content inside them
 * Example: [vc_column]Hello World[/vc_column] becomes "Hello World"
 *
 * @param string $content The content containing shortcodes
 * @return string Content with shortcode tags removed but inner content preserved
 */
/**
 * Single-pass HTML entity decode for text entering the knowledge base.
 * The corpus should hold what a human reads: a stored `&amp;` consumes
 * extra tokens, distorts the vector away from the form a visitor's
 * question uses, and can be quoted back verbatim in an answer.
 * Deliberately NOT looped to a fixed point — a stored `&amp;amp;` is a
 * legitimate literal `&amp;` and must not collapse further (data loss).
 * UTF-8 charset keeps multibyte (CJK/RTL) text untouched. Both assembly
 * paths call this at their output points so the treatment cannot drift.
 * (Plan d2c92e.)
 */
private function mxchat_decode_entities_for_indexing($text) {
    return html_entity_decode((string) $text, ENT_QUOTES | ENT_HTML5, 'UTF-8');
}

/**
 * Price lines for a product's indexed text, pinned to the store's BASE currency.
 *
 * The four product assembly paths each used to call get_woocommerce_currency_symbol()
 * with no argument, which resolves the currency active on the CURRENT request.
 * Multi-currency plugins (CURCY, WOOCS, Aelia, WPML Multicurrency) filter that per
 * request, so whichever currency the store happened to be serving when an import ran
 * was frozen into every product it indexed. The amounts have the mirror problem: the
 * woocommerce_product_get_* filters convert prices in the 'view' context but not in
 * 'edit', so a converted amount could be paired with an unconverted symbol and produce
 * a price that is not merely wrong but incoherent.
 *
 * Base currency option + 'edit' context makes both halves agree and makes the output
 * independent of when the import ran. The currency CODE is emitted alongside the symbol
 * so the model cannot read a bare "$" as USD on a store whose plugin swapped symbols.
 * (Plan 7403ec.)
 */
private function mxchat_product_price_lines($product) {
    if (!is_object($product) || !method_exists($product, 'get_regular_price')) {
        return '';
    }

    $currency = get_option('woocommerce_currency');
    $currency = is_string($currency) ? trim($currency) : '';
    $symbol   = ($currency !== '')
        ? get_woocommerce_currency_symbol($currency)
        : get_woocommerce_currency_symbol();
    $symbol   = $this->mxchat_decode_entities_for_indexing($symbol);

    $regular_price = $product->get_regular_price('edit');
    $sale_price    = $product->get_sale_price('edit');
    $price         = $product->get_price('edit');

    $lines = '';

    if (!empty($regular_price)) {
        $lines .= "Price: " . $this->mxchat_format_indexed_price($regular_price, $currency, $symbol) . "\n";
    } elseif (!empty($price)) {
        $lines .= "Price: " . $this->mxchat_format_indexed_price($price, $currency, $symbol) . "\n";
    }

    if (!empty($sale_price) && $sale_price !== $regular_price) {
        $lines .= "Sale Price: " . $this->mxchat_format_indexed_price($sale_price, $currency, $symbol) . "\n";
    }

    if ($product->is_type('variable')) {
        list($min_price, $max_price) = $this->mxchat_variation_price_range($product);
        if ($min_price !== null && $max_price !== null && (float) $min_price !== (float) $max_price) {
            $lines .= "Price Range: " . $this->mxchat_format_indexed_price($min_price, $currency, $symbol)
                . " - " . $this->mxchat_format_indexed_price($max_price, $currency, $symbol) . "\n";
        }
    }

    return $lines;
}

/**
 * One indexed price amount, labelled with its currency code.
 *
 * "INR 1299.00 (Rs.1299.00)" — the code is what the model should reason from; the symbol
 * is kept so a quoted price still reads naturally. Falls back to the old symbol-only
 * shape when WooCommerce has no base currency configured, and drops the parenthetical
 * when the symbol is absent or IS the code (several currencies have no distinct glyph).
 */
private function mxchat_format_indexed_price($amount, $currency, $symbol) {
    $amount = (string) $amount;

    if ($currency === '') {
        return $symbol . $amount;
    }

    if ($symbol === '' || $symbol === $currency) {
        return $currency . ' ' . $amount;
    }

    return $currency . ' ' . $amount . ' (' . $symbol . $amount . ')';
}

/**
 * Min/max variation price read from the variations themselves in 'edit' context.
 *
 * get_variation_price() reads WooCommerce's display price cache, which multi-currency
 * plugins populate with converted values — the same defect the rest of this helper
 * exists to remove. Returns raw stored strings (not floats) so the indexed text keeps
 * the store's own price formatting, and (null, null) when no variation carries a price.
 */
private function mxchat_variation_price_range($product) {
    $min_raw = null;
    $max_raw = null;
    $min_val = null;
    $max_val = null;

    $children = method_exists($product, 'get_children') ? $product->get_children() : array();

    foreach ($children as $child_id) {
        $variation = wc_get_product($child_id);
        if (!$variation) {
            continue;
        }
        $raw = $variation->get_price('edit');
        if ($raw === '' || $raw === null) {
            continue;
        }
        $val = (float) $raw;
        if ($min_val === null || $val < $min_val) {
            $min_val = $val;
            $min_raw = $raw;
        }
        if ($max_val === null || $val > $max_val) {
            $max_val = $val;
            $max_raw = $raw;
        }
    }

    return array($min_raw, $max_raw);
}

private function strip_shortcode_tags_preserve_content($content) {
    // Single-pass regex removes all shortcode brackets: [tag], [tag attr="val"], [/tag], [tag /]
    // Content between tags is inherently preserved since only brackets are targeted
    $result = preg_replace('/\[\/?\w[\w-]*[^\]]*\]/', '', $content);
    return ($result !== null) ? $result : $content;
}

public function mxchat_sanitize_content_for_api($content) {
    //error_log('[MXCHAT-SANITIZE] Original content preview: ' . substr($content, 0, 500) . '...');

    // Remove shortcode tags but PRESERVE content inside them
    $content = $this->strip_shortcode_tags_preserve_content($content);

    // Remove script, style tags, and HTML comments
    $content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
    $content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
    $content = preg_replace('/<!--(.|\s)*?-->/', '', $content);

    // Remove all HTML tags and decode HTML entities
    $content = wp_strip_all_tags($content);
    $content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5);
    
    // Normalize whitespace but preserve paragraph breaks
    // First, normalize line endings to \n
    $content = str_replace(["\r\n", "\r"], "\n", $content);
    // Replace multiple spaces/tabs with single space, but preserve newlines
    $content = preg_replace('/[ \t]+/', ' ', $content);
    // Replace 3+ newlines with 2 newlines (max 2 blank lines)
    $content = preg_replace('/\n{3,}/', "\n\n", $content);
    // Trim each line
    $lines = explode("\n", $content);
    $lines = array_map('trim', $lines);
    $content = implode("\n", $lines);
    // Final trim
    $content = trim($content);
    
    // Remove control characters (which can cause database issues) but preserve \n (0x0A) and \r (0x0D)
    $content = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $content);
    
    // Remove NULL bytes which can cause database errors
    $content = str_replace("\0", "", $content);
    
    // Ensure valid UTF-8 encoding
    $content = wp_check_invalid_utf8($content);
    
    // Remove extremely long runs with no whitespace (base64 blobs, minified JS).
    // Counts CHARACTERS (/u), and never strips a run containing characters from a
    // script written without spaces — Japanese, Chinese, Thai, Khmer, Lao, Myanmar —
    // where a normal paragraph is legitimately one unbroken run.
    $content = preg_replace_callback('/\S{300,}/u', function ($m) {
        return preg_match('/[\p{Han}\p{Hiragana}\p{Katakana}\p{Thai}\p{Khmer}\p{Lao}\p{Myanmar}]/u', $m[0]) ? $m[0] : ' ';
    }, $content);

    // Remove emoji/symbol blocks only — not the whole supplementary plane, which
    // also holds CJK Extension B ideographs used in real Chinese/Japanese names.
    // A ZWJ (U+200D) BETWEEN stripped pictographs is consumed with them, so a
    // family sequence like 👨‍👩‍👧 leaves no invisible zero-width residue behind
    // (the joiner between NON-emoji characters — Hindi conjuncts — is untouched).
    $content = preg_replace('/[\x{1F000}-\x{1F0FF}\x{1F300}-\x{1FAFF}](?:\x{200D}[\x{1F000}-\x{1F0FF}\x{1F300}-\x{1FAFF}])*/u', '', $content);

    // There is deliberately NO catch-all character allowlist here (plan 209e57;
    // one existed until 3.2.20). Every genuinely dangerous byte is already gone:
    // control characters, null bytes, invalid UTF-8 and the emoji blocks are all
    // stripped above. The allowlist's only remaining effect was to damage scripts
    // nobody thought to enumerate — Unicode Cf (Format) was missing, so it
    // replaced the zero-width joiner/non-joiner with spaces and silently split
    // Persian words (می‌روم → می روم) and broke Hindi conjuncts (क्‍ष → क् ष).
    // Do not add one back; the failure mode of an allowlist is exactly this.

    // Limit to reasonable length if needed (byte limit — MySQL TEXT is byte-sized,
    // but cut on a character boundary so a multibyte char is never split mid-sequence)
    $max_length = 65000; // Just under MySQL TEXT field limit
    if (strlen($content) > $max_length) {
        $content = mb_strcut($content, 0, $max_length, 'UTF-8');
    }
    
    //error_log('[MXCHAT-SANITIZE] Sanitized content preview: ' . substr($content, 0, 500) . '...');
    return $content;
}
public function mxchat_extract_main_content($html) {
    if (empty($html)) {
        return '';
    }
    try {
        $dom = new DOMDocument;
        libxml_use_internal_errors(true); // Suppress HTML parsing errors
        @$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
        $xpath = new DOMXPath($dom);
        
        // For debugging purposes
        $debugEnabled = true; // Set to true to enable debugging output
        $debug = function($message) use ($debugEnabled) {
            if ($debugEnabled) {
                //error_log('[MXCHAT-EXTRACT-DEBUG] ' . $message);
            }
        };
        
        // Direct targeting for Gerow theme posts
        $post_text = $xpath->query('//div[contains(@class, "post-text")]');
        if ($post_text && $post_text->length > 0) {
            $debug("Found post-text directly");
            $content = '';
            foreach ($post_text as $node) {
                $content .= $dom->saveHTML($node);
            }
            if (!empty($content)) {
                $debug("Returning post-text content");
                return $content;
            }
        }
        
        // Try to get the blog details content which contains the post-text
        $blog_details = $xpath->query('//div[contains(@class, "blog-details-content")]');
        if ($blog_details && $blog_details->length > 0) {
            $debug("Found blog-details-content");
            $content = '';
            foreach ($blog_details as $node) {
                $content .= $dom->saveHTML($node);
            }
            if (!empty($content)) {
                $debug("Returning blog-details-content");
                return $content;
            }
        }
        
        // Try to get the article which contains the blog details
        $article = $xpath->query('//article[contains(@class, "blog-details-wrap")]');
        if ($article && $article->length > 0) {
            $debug("Found article with blog-details-wrap");
            $content = '';
            foreach ($article as $node) {
                $content .= $dom->saveHTML($node);
            }
            if (!empty($content)) {
                $debug("Returning article content");
                return $content;
            }
        }
        
        // Try even broader with the blog-item-wrap
        $blog_item = $xpath->query('//div[contains(@class, "blog-item-wrap")]');
        if ($blog_item && $blog_item->length > 0) {
            $debug("Found blog-item-wrap");
            $content = '';
            foreach ($blog_item as $node) {
                $content .= $dom->saveHTML($node);
            }
            if (!empty($content)) {
                $debug("Returning blog-item-wrap content");
                return $content;
            }
        }
        
        // Specific Gerow theme path
        $gerow_path = $xpath->query('//section[contains(@class, "blog-area")]//div[contains(@class, "post-text")]');
        if ($gerow_path && $gerow_path->length > 0) {
            $debug("Found Gerow theme path to post-text");
            $content = '';
            foreach ($gerow_path as $node) {
                $content .= $dom->saveHTML($node);
            }
            if (!empty($content)) {
                $debug("Returning Gerow post-text content");
                return $content;
            }
        }
        
        // Generic blog post selectors
        $selectors = [
            // Blog post specific selectors
            '//div[contains(@class, "post-text")]',
            '//article[contains(@class, "blog-post-item")]//div[contains(@class, "post-text")]',
            '//div[contains(@class, "blog-details-content")]',
            '//article[contains(@class, "blog-details-wrap")]',
            '//div[contains(@class, "entry-content")]',
            '//div[contains(@class, "blog-content")]',
            '//div[contains(@class, "blog-item-wrap")]',
            
            // More general content selectors
            '//div[contains(@class, "page__content")]',
            '//div[contains(@class, "elementor-widget-container")]',
            '//div[contains(@class, "elementor-text-editor")]',
            '//div[contains(@class, "elementor-widget-text-editor")]',
            '//*[contains(@class, "entry-content")]',
            '//*[contains(@class, "post-content")]',
            '//*[contains(@class, "article-content")]',
            '//*[@id="content"]',
            '//*[@id="main-content"]',
            '//section[contains(@class, "blog-area")]',
            '//article',
            '//main',
            '//div[contains(@class, "content")]'
        ];
        
        // First handle Elementor content - get only leaf widget containers to avoid duplicates
        $debug("Checking for Elementor content");
        // Get widget containers that are direct children of widgets (not nested inside other widget containers)
        $elementor_widgets = $xpath->query('//div[contains(@class, "elementor-widget")]//div[contains(@class, "elementor-widget-container")]');
        if ($elementor_widgets && $elementor_widgets->length > 0) {
            $debug("Found Elementor widgets");
            $seen_content = array(); // Track seen content to avoid duplicates
            $combined_content = '';
            foreach ($elementor_widgets as $widget) {
                $widget_content = $dom->saveHTML($widget);
                if (!empty($widget_content)) {
                    // Create a hash of the content to detect duplicates
                    $content_hash = md5($widget_content);
                    if (!isset($seen_content[$content_hash])) {
                        $seen_content[$content_hash] = true;
                        $combined_content .= $widget_content;
                    }
                }
            }
            if (!empty($combined_content)) {
                $debug("Returning Elementor content");
                return $combined_content;
            }
        }
        
        // Try standard selectors one by one
        foreach ($selectors as $selector) {
            $debug("Trying selector: " . $selector);
            $nodes = $xpath->query($selector);
            if ($nodes && $nodes->length > 0) {
                $debug("Found " . $nodes->length . " matches for selector: " . $selector);
                // Only take the FIRST matching node to avoid duplicate content
                // (pages often have nested or multiple containers with same class)
                $content = $dom->saveHTML($nodes->item(0));
                if (!empty($content)) {
                    $debug("Returning content from selector: " . $selector . " (first match only)");
                    return $content;
                }
            }
        }
        
        // Manual regex fallback for post-text if DOM methods fail
        $debug("Trying regex fallback");
        if (preg_match('/<div class="post-text">(.*?)<\/div>\s*<\/div>\s*<\/div>/s', $html, $matches)) {
            $debug("Found post-text via regex");
            return '<div class="post-text">' . $matches[1] . '</div>';
        }
        
        // Try to extract the blog section as a whole
        $blog_section = $xpath->query('//section[contains(@class, "blog-area")]');
        if ($blog_section && $blog_section->length > 0) {
            $debug("Found blog-area section");
            $content = '';
            foreach ($blog_section as $node) {
                $content .= $dom->saveHTML($node);
            }
            if (!empty($content)) {
                $debug("Returning blog-area section content");
                return $content;
            }
        }

        // Generic container selectors for non-CMS sites (like .asp pages)
        $debug("Trying generic container selectors");
        $generic_selectors = [
            '//div[@id="main"]',
            '//div[@id="wrapper"]',
            '//div[@id="page"]',
            '//div[@id="site-content"]',
            '//div[contains(@class, "main-content")]',
            '//div[contains(@class, "page-content")]',
            '//div[contains(@class, "site-content")]',
        ];

        foreach ($generic_selectors as $selector) {
            $debug("Trying generic selector: " . $selector);
            $nodes = $xpath->query($selector);
            if ($nodes && $nodes->length > 0) {
                $content = $dom->saveHTML($nodes->item(0));
                if (!empty($content)) {
                    $debug("Returning content from generic selector: " . $selector);
                    return $content;
                }
            }
        }

        // Paragraph-based content detection - find regions with substantial text
        $debug("Trying paragraph-based content detection");
        $paragraphs = $xpath->query('//p[string-length(normalize-space()) > 50]');
        if ($paragraphs && $paragraphs->length >= 3) {
            $debug("Found " . $paragraphs->length . " substantial paragraphs");
            // Collect all substantial paragraphs and their content
            $paragraph_content = '';
            foreach ($paragraphs as $p) {
                $paragraph_content .= $dom->saveHTML($p) . "\n";
            }
            if (!empty($paragraph_content)) {
                $debug("Returning paragraph-based content");
                return $paragraph_content;
            }
        }

        // Improved body fallback - strip nav/header/footer elements first
        $debug("Using improved body fallback");
        $body = $dom->getElementsByTagName('body');
        if ($body->length > 0) {
            // Clone the body to avoid modifying the original DOM
            $body_clone = $body->item(0)->cloneNode(true);

            // Remove common non-content elements by tag name
            $remove_tags = ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript'];
            foreach ($remove_tags as $tag) {
                $elements = $body_clone->getElementsByTagName($tag);
                // Iterate backwards to safely remove elements
                for ($i = $elements->length - 1; $i >= 0; $i--) {
                    $el = $elements->item($i);
                    if ($el && $el->parentNode) {
                        $el->parentNode->removeChild($el);
                    }
                }
            }

            // Remove elements with common non-content class names using XPath on the cloned body
            $temp_dom = new DOMDocument();
            @$temp_dom->appendChild($temp_dom->importNode($body_clone, true));
            $temp_xpath = new DOMXPath($temp_dom);

            $remove_class_patterns = [
                '//*[contains(@class, "nav")]',
                '//*[contains(@class, "menu")]',
                '//*[contains(@class, "sidebar")]',
                '//*[contains(@class, "footer")]',
                '//*[contains(@class, "header")]',
                '//*[contains(@id, "nav")]',
                '//*[contains(@id, "menu")]',
                '//*[contains(@id, "sidebar")]',
                '//*[contains(@id, "footer")]',
                '//*[contains(@id, "header")]',
            ];

            foreach ($remove_class_patterns as $pattern) {
                $elements = $temp_xpath->query($pattern);
                if ($elements) {
                    for ($i = $elements->length - 1; $i >= 0; $i--) {
                        $el = $elements->item($i);
                        if ($el && $el->parentNode) {
                            $el->parentNode->removeChild($el);
                        }
                    }
                }
            }

            $cleaned_content = $temp_dom->saveHTML();
            if (!empty($cleaned_content)) {
                $debug("Returning cleaned body content");
                return $cleaned_content;
            }
        }

        // Last resort: return the original HTML
        $debug("Returning original HTML");
        return $html;
    } catch (Exception $e) {
        //error_log('[MXCHAT-ERROR] Content extraction failed: ' . $e->getMessage());
        return $html; // Return original HTML if parsing fails
    } finally {
        libxml_clear_errors();
    }
}
public function mxchat_get_sitemap_processing_status($sitemap_url) {
    $sitemap_url = esc_url_raw($sitemap_url);
    $status_key = sanitize_key('mxchat_sitemap_status_' . md5($sitemap_url));
    $status = get_transient($status_key);
    
    if (!$status || !is_array($status)) {
        return false;
    }
    
    // Auto-complete check: if all URLs are processed but status isn't complete
    if (isset($status['processed_urls']) && isset($status['total_urls']) && 
        $status['processed_urls'] >= $status['total_urls'] && 
        isset($status['status']) && $status['status'] !== 'complete' && 
        $status['status'] !== 'error') {
        
        // Mark as complete
        $status['status'] = 'complete';
        $status['processed_urls'] = $status['total_urls']; // Ensure exact match
        
        // Update the transient with the corrected status
        set_transient($status_key, $status, DAY_IN_SECONDS);
    }
    
    return array(
        'total_urls' => absint($status['total_urls']),
        'processed_urls' => absint($status['processed_urls']),
        'failed_urls' => absint($status['failed_urls'] ?? 0),
        'percentage' => ($status['total_urls'] > 0)
            ? round((absint($status['processed_urls']) / absint($status['total_urls'])) * 100)
            : 0,
        'status' => sanitize_text_field($status['status']),
        'last_update' => human_time_diff(absint($status['last_update']), time()) . ' ' . esc_html__('ago', 'mxchat'),
        'error' => isset($status['error']) ? sanitize_text_field($status['error']) : '',
        'last_error' => isset($status['last_error']) ? sanitize_text_field($status['last_error']) : '',
        'failed_urls_list' => isset($status['failed_urls_list']) ? $status['failed_urls_list'] : array()
    );
}

public function mxchat_ajax_get_status_updates() {
    try {
        // Verify the request
        check_ajax_referer('mxchat_status_nonce', 'nonce');
        
        // Get active queue IDs
        $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
        $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
        
        $sitemap_status = false;
        $pdf_status = false;
        
        // Get sitemap queue status
        if ($sitemap_queue_id) {
            $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
        }
        
        // Get PDF queue status
        if ($pdf_queue_id) {
            $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
        }
        
        $is_active_processing = 
            ($sitemap_status && $sitemap_status['status'] === 'processing') || 
            ($pdf_status && $pdf_status['status'] === 'processing');
        
        // Return JSON response with the status data
        wp_send_json(array(
            'pdf_status' => $pdf_status,
            'sitemap_status' => $sitemap_status,
            'is_processing' => $is_active_processing,
            'sitemap_queue_id' => $sitemap_queue_id,
            'pdf_queue_id' => $pdf_queue_id
        ));
        
    } catch (Exception $e) {
        //error_log('MxChat Status Update Error: ' . $e->getMessage());
        
        wp_send_json_error(array(
            'message' => 'Error getting status updates: ' . $e->getMessage(),
            'status' => 'error'
        ));
    }
}

/**
 * Helper function to get queue status data
 */
private function mxchat_get_queue_status_data($queue_id, $type = 'sitemap') {
    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_processing_queue';
    
    // Get counts by status
    $counts = $wpdb->get_results($wpdb->prepare(
        "SELECT status, COUNT(*) as count 
        FROM $table_name 
        WHERE queue_id = %s 
        GROUP BY status",
        $queue_id
    ), OBJECT_K);
    
    $total = 0;
    $completed = 0;
    $failed = 0;
    $processing = 0;
    $pending = 0;
    
    foreach ($counts as $status => $data) {
        $count = absint($data->count);
        $total += $count;
        
        switch ($status) {
            case 'completed':
                $completed = $count;
                break;
            case 'failed':
                $failed = $count;
                break;
            case 'processing':
                $processing = $count;
                break;
            case 'pending':
                $pending = $count;
                break;
        }
    }
    
    if ($total === 0) {
        return false;
    }
    
    // Calculate percentage
    $percentage = round((($completed + $failed) / $total) * 100);
    
    // Get failed items details (limit to 50)
    $failed_items = array();
    if ($failed > 0) {
        $failed_results = $wpdb->get_results($wpdb->prepare(
            "SELECT item_type, item_data, error_message, attempts, completed_at 
            FROM $table_name 
            WHERE queue_id = %s 
            AND status = 'failed'
            AND attempts >= max_attempts
            ORDER BY id DESC
            LIMIT 50",
            $queue_id
        ));
        
        foreach ($failed_results as $item) {
            $data = json_decode($item->item_data, true);
            $url = $type === 'pdf' ? ($data['pdf_url'] ?? '') : ($data['url'] ?? '');
            
            $failed_items[] = array(
                'url' => $url,
                'page' => $type === 'pdf' ? ($data['page_number'] ?? 0) : 0,
                'error' => $item->error_message,
                'retries' => $item->attempts,
                'time' => strtotime($item->completed_at)
            );
        }
    }
    
    // Get queue metadata
    $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
    
    // Determine if queue is complete
    $is_complete = ($pending === 0 && $processing === 0);
    
    // Get last update time
    $last_update = $wpdb->get_var($wpdb->prepare(
        "SELECT MAX(UNIX_TIMESTAMP(COALESCE(completed_at, started_at, created_at))) 
        FROM $table_name 
        WHERE queue_id = %s",
        $queue_id
    ));
    
    $last_update_text = $last_update ? human_time_diff($last_update, time()) . ' ' . __('ago', 'mxchat') : __('Just now', 'mxchat');
    
    // Format based on type
    if ($type === 'pdf') {
        return array(
            'total_pages' => $total,
            'processed_pages' => $completed + $failed,
            'failed_pages' => $failed,
            'percentage' => $percentage,
            'status' => $is_complete ? 'complete' : 'processing',
            'last_update' => $last_update_text,
            'failed_pages_list' => $failed_items,
            'pdf_url' => $source_url,
            'queue_id' => $queue_id
        );
    } else {
        return array(
            'total_urls' => $total,
            'processed_urls' => $completed + $failed,
            'failed_urls' => $failed,
            'percentage' => $percentage,
            'status' => $is_complete ? 'complete' : 'processing',
            'last_update' => $last_update_text,
            'failed_urls_list' => $failed_items,
            'sitemap_url' => $source_url,
            'queue_id' => $queue_id
        );
    }
}

/**
 * Public method to get processing status for both sitemap and PDF queues
 * Used by admin pages to display processing status
 *
 * @return array Array with 'sitemap_status', 'pdf_status', and 'is_processing' keys
 */
public function mxchat_get_processing_statuses() {
    $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
    $pdf_queue_id = get_transient('mxchat_active_queue_pdf');

    $sitemap_status = false;
    $pdf_status = false;

    if ($sitemap_queue_id) {
        $sitemap_status = $this->mxchat_get_queue_status_data($sitemap_queue_id, 'sitemap');
    }

    if ($pdf_queue_id) {
        $pdf_status = $this->mxchat_get_queue_status_data($pdf_queue_id, 'pdf');
    }

    $is_processing =
        ($sitemap_status && $sitemap_status['status'] === 'processing') ||
        ($pdf_status && $pdf_status['status'] === 'processing');

    return array(
        'sitemap_status' => $sitemap_status,
        'pdf_status' => $pdf_status,
        'is_processing' => $is_processing
    );
}

/**
 * AJAX handler to get recent knowledge entries for real-time table updates
 * UPDATED: Now supports both WordPress DB and Pinecone data sources
 */
public function ajax_mxchat_get_recent_entries() {
    check_ajax_referer('mxchat_entries_nonce', 'nonce');

    if (!current_user_can('manage_options')) {
        wp_send_json_error(array('message' => 'Unauthorized'));
        return;
    }

    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';

    // Get parameters
    $last_id = isset($_POST['last_id']) ? absint($_POST['last_id']) : 0;
    $limit = isset($_POST['limit']) ? min(absint($_POST['limit']), 50) : 10;
    $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';

    // Check if Pinecone is enabled for this bot
    $pinecone_manager = $this->mxchat_get_pinecone_manager();
    $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
    $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
    $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);

    if ($use_pinecone && $has_pinecone_api) {
        // PINECONE DATA SOURCE - Get grouped entry count (not raw vector count)
        // Use mxchat_fetch_pinecone_records which returns total_unique_entries
        $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, '', 1, 10, $bot_id, '');
        $total_count = $records['total'] ?? 0;

        // For Pinecone, we don't return individual entries during polling
        // (entries are already displayed on page load via mxchat_fetch_pinecone_records)
        // We just return the updated count
        wp_send_json_success(array(
            'entries' => array(),
            'total_count' => absint($total_count),
            'max_id' => $last_id,
            'data_source' => 'pinecone'
        ));
        return;
    }

    // WORDPRESS DB DATA SOURCE
    // Build query to get entries newer than last_id
    $where_clauses = array('1=1');
    $where_values = array();

    if ($last_id > 0) {
        $where_clauses[] = 'id > %d';
        $where_values[] = $last_id;
    }

    // Note: WordPress DB table doesn't have bot_id column
    // Multi-bot filtering is handled via Pinecone namespaces

    $where_sql = implode(' AND ', $where_clauses);

    // Get recent entries
    $query = "SELECT id, article_content, source_url, timestamp
              FROM $table_name
              WHERE $where_sql
              ORDER BY id DESC
              LIMIT %d";

    $where_values[] = $limit;

    $entries = $wpdb->get_results($wpdb->prepare($query, $where_values));

    // Get total count of GROUPED entries (by source_url) - matches pagination display
    // Count unique source_urls (excluding mxchat:// internal refs) + count of ungrouped rows
    $total_count = $wpdb->get_var(
        "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
                (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
    );

    // Format entries for response
    $formatted_entries = array();
    $preview_length = 150;
    foreach ($entries as $entry) {
        // Parse chunk metadata using the proper chunker method (same as initial page load)
        if (class_exists('MxChat_Chunker')) {
            $chunk_meta = MxChat_Chunker::parse_stored_chunk($entry->article_content);
            $display_content = $chunk_meta['text'];
            $chunk_metadata = $chunk_meta['metadata'];
        } else {
            $display_content = $entry->article_content;
            $chunk_metadata = array();
        }

        $content_preview = mb_strlen($display_content) > $preview_length
            ? mb_substr($display_content, 0, $preview_length) . '...'
            : $display_content;

        $formatted_entries[] = array(
            'id' => $entry->id,
            'preview' => esc_html($content_preview),
            'full_content' => wp_kses_post(wpautop($display_content)),
            'content_length' => mb_strlen($display_content),
            'preview_length' => $preview_length,
            'source_url' => $entry->source_url,
            'has_link' => !empty($entry->source_url) && strpos($entry->source_url, 'mxchat://') !== 0,
            'chunk_metadata' => $chunk_metadata,
            'bot_id' => $entry->bot_id ?? 'default',
            'edit_nonce' => wp_create_nonce('mxchat_edit_entry_nonce'),
            'delete_nonce' => wp_create_nonce('mxchat_delete_prompt_nonce')
        );
    }

    wp_send_json_success(array(
        'entries' => $formatted_entries,
        'total_count' => absint($total_count),
        'max_id' => !empty($entries) ? $entries[0]->id : $last_id,
        'data_source' => 'wordpress'
    ));
}

/**
 * Get Pinecone total count from stats API
 * Helper function for ajax_mxchat_get_recent_entries
 */
private function mxchat_get_pinecone_count_from_stats($pinecone_options) {
    $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
    $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
    $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';

    if (empty($api_key) || empty($host)) {
        return 0;
    }

    try {
        $stats_url = "https://{$host}/describe_index_stats";

        $response = wp_remote_post($stats_url, array(
            'headers' => array(
                'Api-Key' => $api_key,
                'Content-Type' => 'application/json'
            ),
            'body' => '{}',
            'timeout' => 10
        ));

        if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
            $body = wp_remote_retrieve_body($response);
            $stats_data = json_decode($body, true);

            // If namespace is specified, get count from that specific namespace
            if (!empty($namespace) && isset($stats_data['namespaces'][$namespace]['vectorCount'])) {
                return intval($stats_data['namespaces'][$namespace]['vectorCount']);
            }

            // If no namespace specified or namespace not found in response, use total
            return intval($stats_data['totalVectorCount'] ?? 0);
        }

        return 0;

    } catch (Exception $e) {
        return 0;
    }
}

/**
 * AJAX handler to refresh Pinecone entries table via AJAX
 * Returns the table HTML for updating the UI without a full page reload
 */
public function ajax_mxchat_refresh_pinecone_entries() {
    check_ajax_referer('mxchat_entries_nonce', 'nonce');

    if (!current_user_can('manage_options')) {
        wp_send_json_error(array('message' => 'Unauthorized'));
        return;
    }

    $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
    $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
    $per_page = 25;
    $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
    $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';

    // Get Pinecone manager and options
    $pinecone_manager = $this->mxchat_get_pinecone_manager();
    if (!$pinecone_manager) {
        wp_send_json_error(array('message' => 'Pinecone manager not available'));
        return;
    }

    $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
    $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
    $pinecone_api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';

    if (!$use_pinecone || empty($pinecone_api_key)) {
        wp_send_json_error(array('message' => 'Pinecone not configured'));
        return;
    }

    // Fetch records from Pinecone
    $records = $pinecone_manager->mxchat_fetch_pinecone_records($pinecone_options, $search_query, $page, $per_page, $bot_id, $content_type_filter);
    $prompts = $records['data'] ?? array();
    $total_records = $records['total'] ?? 0;

    // Preprocess Pinecone records — set chunk_metadata and display_content
    // (matches admin-knowledge-page.php preprocessing)
    foreach ($prompts as $prompt) {
        if (isset($prompt->chunk_index) && $prompt->chunk_index !== null) {
            $prompt->chunk_metadata = array(
                'chunk_index' => intval($prompt->chunk_index),
                'total_chunks' => isset($prompt->total_chunks) ? intval($prompt->total_chunks) : null,
                'is_chunked' => isset($prompt->is_chunked) ? (bool) $prompt->is_chunked : true,
                'source_url' => $prompt->source_url ?? ''
            );
            $prompt->display_content = $prompt->article_content;
        } else {
            $prompt->chunk_metadata = array();
            $prompt->display_content = $prompt->article_content ?? '';
        }
    }

    // Group prompts by source_url
    $grouped_prompts = array();
    foreach ($prompts as $prompt) {
        $source_url = '';
        if (!empty($prompt->chunk_metadata['source_url'])) {
            $source_url = $prompt->chunk_metadata['source_url'];
        } elseif (!empty($prompt->source_url)) {
            $source_url = $prompt->source_url;
        }

        if (!empty($source_url)) {
            if (!isset($grouped_prompts[$source_url])) {
                $grouped_prompts[$source_url] = array();
            }
            $grouped_prompts[$source_url][] = $prompt;
        } else {
            $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
        }
    }

    // Sort each group by chunk_index
    foreach ($grouped_prompts as $source_url => &$group) {
        usort($group, function($a, $b) {
            $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
            $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
            return $index_a - $index_b;
        });
    }
    unset($group);

    // Build HTML for the table rows - must match admin-knowledge-page.php structure exactly
    ob_start();
    $display_index = 0;
    $current_page = $page;
    $data_source = 'pinecone';
    $current_bot_id = $bot_id;
    $preview_length = 150;

    if (empty($grouped_prompts)) {
        echo '<tr><td colspan="4" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
        esc_html_e('No knowledge entries found in Pinecone.', 'mxchat');
        echo '</td></tr>';
    } else {
        foreach ($grouped_prompts as $source_url => $group) {
            $chunk_count = count($group);
            $first_prompt = $group[0];
            $display_index++;

            if ($chunk_count > 1) {
                // Multiple chunks - show grouped row with expand button
                $group_id = 'group-' . md5($source_url);
                ?>
                <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
                    class="mxchat-chunk-group-header"
                    data-source="<?php echo esc_attr($data_source); ?>"
                    data-group-id="<?php echo esc_attr($group_id); ?>"
                    style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
                    <td style="padding: 12px 16px; text-align: center;">
                        <input type="checkbox"
                               class="mxchat-entry-checkbox"
                               data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
                               data-source="<?php echo esc_attr($data_source); ?>"
                               data-source-url="<?php echo esc_attr($source_url); ?>"
                               data-is-group="true"
                               data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
                    </td>
                    <td style="padding: 12px 16px; font-size: 13px;">
                        <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
                    </td>
                    <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
                        <div class="mxchat-chunk-group-info">
                            <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
                                <span class="dashicons dashicons-arrow-right-alt2"></span>
                            </button>
                            <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
                            <span class="mxchat-chunk-preview">
                                <?php
                                $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : (isset($first_prompt->article_content) ? $first_prompt->article_content : '');
                                $content_preview = mb_substr($parent_content, 0, 100);
                                echo esc_html($content_preview . '...');
                                ?>
                            </span>
                        </div>
                    </td>
                    <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
                        <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
                            <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
                                <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
                                <?php esc_html_e('View Source', 'mxchat'); ?>
                            </a>
                        <?php else : ?>
                            <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
                        <?php endif; ?>
                    </td>
                    <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
                        <button type="button"
                                class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-inspect-entry-btn"
                                data-source-url="<?php echo esc_attr($source_url); ?>"
                                data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
                                data-data-source="<?php echo esc_attr($data_source); ?>"
                                data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
                                data-nonce="<?php echo wp_create_nonce('mxchat_inspect_entry_nonce'); ?>"
                                title="<?php esc_attr_e('View indexed content', 'mxchat'); ?>">
                            <span class="dashicons dashicons-visibility" style="font-size: 14px;"></span>
                        </button>
                        <button type="button"
                                class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
                                data-source-url="<?php echo esc_attr($source_url); ?>"
                                data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
                                data-data-source="<?php echo esc_attr($data_source); ?>"
                                data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
                                data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
                                title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
                            <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
                        </button>
                        <button type="button"
                                class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
                                data-source-url="<?php echo esc_attr($source_url); ?>"
                                data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
                                data-data-source="<?php echo esc_attr($data_source); ?>"
                                data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
                                data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
                                style="color: var(--mxch-error);"
                                title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
                            <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
                        </button>
                    </td>
                </tr>
                <?php
                // Render hidden chunk rows
                foreach ($group as $chunk_index => $chunk) {
                    $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
                    $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
                    $content = isset($chunk->display_content) ? $chunk->display_content : (isset($chunk->article_content) ? $chunk->article_content : '');
                    $content_preview = mb_strlen($content) > $preview_length
                        ? mb_substr($content, 0, $preview_length) . '...'
                        : $content;
                    ?>
                    <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
                        class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
                        data-source="<?php echo esc_attr($data_source); ?>"
                        style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
                        <td style="padding: 12px 16px; text-align: center;">
                            <!-- Checkbox column placeholder for chunks (managed by group) -->
                        </td>
                        <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
                            <!-- Hidden ID column for chunks -->
                        </td>
                        <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
                            <div class="mxchat-accordion-wrapper">
                                <div class="mxchat-content-preview">
                                    <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
                                        <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
                                    </span>
                                    <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
                                    <?php if (mb_strlen($content) > $preview_length) : ?>
                                        <button class="mxchat-expand-toggle" type="button">
                                            <span class="dashicons dashicons-arrow-down-alt2"></span>
                                        </button>
                                    <?php endif; ?>
                                </div>
                                <div class="mxchat-content-full" style="display: none;">
                                    <div class="content-view">
                                        <?php
                                        if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
                                            echo '<div dir="rtl" lang="he" class="rtl-content">';
                                            echo wp_kses_post(wpautop($content));
                                            echo '</div>';
                                        } else {
                                            echo wp_kses_post(wpautop($content));
                                        }
                                        ?>
                                    </div>
                                </div>
                            </div>
                        </td>
                        <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
                            <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
                        </td>
                        <td class="mxchat-actions-cell" style="padding: 12px 16px;">
                            <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
                        </td>
                    </tr>
                    <?php
                }
            } else {
                // Single entry - display normally with accordion
                $prompt = $first_prompt;
                $content = isset($prompt->display_content) ? $prompt->display_content : (isset($prompt->article_content) ? $prompt->article_content : '');
                $content_preview = mb_strlen($content) > $preview_length
                    ? mb_substr($content, 0, $preview_length) . '...'
                    : $content;
                ?>
                <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
                    data-source="<?php echo esc_attr($data_source); ?>"
                    style="border-bottom: 1px solid var(--mxch-card-border); background: rgba(33, 150, 243, 0.02);">
                    <td style="padding: 12px 16px; text-align: center;">
                        <input type="checkbox"
                               class="mxchat-entry-checkbox"
                               data-entry-id="<?php echo esc_attr($prompt->id); ?>"
                               data-source="<?php echo esc_attr($data_source); ?>"
                               data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
                               data-is-group="false"
                               data-chunk-count="1">
                    </td>
                    <td style="padding: 12px 16px; font-size: 13px;">
                        <?php echo esc_html($display_index + (($current_page - 1) * $per_page)); ?>
                    </td>
                    <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
                        <div class="mxchat-accordion-wrapper">
                            <div class="mxchat-content-preview">
                                <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
                                <?php if (mb_strlen($content) > $preview_length) : ?>
                                    <button class="mxchat-expand-toggle" type="button">
                                        <span class="dashicons dashicons-arrow-down-alt2"></span>
                                    </button>
                                <?php endif; ?>
                            </div>
                            <div class="mxchat-content-full" style="display: none;">
                                <div class="content-view">
                                    <?php
                                    if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
                                        echo '<div dir="rtl" lang="he" class="rtl-content">';
                                        echo wp_kses_post(wpautop($content));
                                        echo '</div>';
                                    } else {
                                        echo wp_kses_post(wpautop($content));
                                    }
                                    ?>
                                </div>
                            </div>
                        </div>
                    </td>
                    <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
                        <?php
                        $actual_source = $source_url;
                        if (strpos($source_url, '_ungrouped_') === 0) {
                            $actual_source = $prompt->source_url ?? '';
                        }
                        if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
                            <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
                                <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
                                <?php esc_html_e('View', 'mxchat'); ?>
                            </a>
                        <?php else : ?>
                            <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
                        <?php endif; ?>
                    </td>
                    <td style="padding: 12px 16px; white-space: nowrap;">
                        <button type="button"
                                class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-inspect-entry-btn"
                                data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
                                data-entry-id="<?php echo esc_attr($prompt->id); ?>"
                                data-data-source="<?php echo esc_attr($data_source); ?>"
                                data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
                                data-nonce="<?php echo wp_create_nonce('mxchat_inspect_entry_nonce'); ?>"
                                title="<?php esc_attr_e('View indexed content', 'mxchat'); ?>">
                            <span class="dashicons dashicons-visibility" style="font-size: 14px;"></span>
                        </button>
                        <button type="button"
                                class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
                                data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
                                data-entry-id="<?php echo esc_attr($prompt->id); ?>"
                                data-data-source="<?php echo esc_attr($data_source); ?>"
                                data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
                                data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
                                title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
                            <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
                        </button>
                        <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-ajax" data-vector-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_pinecone_prompt_nonce'); ?>" style="color: var(--mxch-error);">
                            <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
                        </button>
                    </td>
                </tr>
                <?php
            }
        }
    }
    $html = ob_get_clean();

    // Generate pagination HTML for Pinecone
    $total_pages = ceil($total_records / $per_page);
    $pagination_html = '';
    if ($total_pages > 1) {
        $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';

        // Previous button
        if ($page > 1) {
            $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
        }

        // Page numbers
        $start_page = max(1, $page - 2);
        $end_page = min($total_pages, $page + 2);

        if ($start_page > 1) {
            $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
            if ($start_page > 2) {
                $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
            }
        }

        for ($i = $start_page; $i <= $end_page; $i++) {
            if ($i == $page) {
                $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
            } else {
                $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
            }
        }

        if ($end_page < $total_pages) {
            if ($end_page < $total_pages - 1) {
                $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
            }
            $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
        }

        // Next button
        if ($page < $total_pages) {
            $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
        }

        $pagination_html .= '</div>';
    }

    wp_send_json_success(array(
        'html' => $html,
        'pagination_html' => $pagination_html,
        'total_count' => $total_records,
        'total_pages' => $total_pages,
        'page' => $page,
        'per_page' => $per_page,
        'data_source' => 'pinecone'
    ));
}

/**
 * AJAX handler for pagination - handles both WordPress DB and Pinecone data sources
 * Returns paginated entries without requiring a full page reload
 */
public function ajax_mxchat_paginate_entries() {
    check_ajax_referer('mxchat_entries_nonce', 'nonce');

    if (!current_user_can('manage_options')) {
        wp_send_json_error(array('message' => 'Unauthorized'));
        return;
    }

    $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
    $page = isset($_POST['page']) ? absint($_POST['page']) : 1;
    $search_query = isset($_POST['search']) ? sanitize_text_field($_POST['search']) : '';
    $content_type_filter = isset($_POST['content_type']) ? sanitize_key($_POST['content_type']) : '';
    $per_page = 25;

    // Check if Pinecone is enabled for this bot
    $pinecone_manager = $this->mxchat_get_pinecone_manager();
    $pinecone_options = $pinecone_manager ? $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id) : array();
    $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
    $has_pinecone_api = !empty($pinecone_options['mxchat_pinecone_api_key']);

    if ($use_pinecone && $has_pinecone_api) {
        // Delegate to Pinecone pagination handler (pass search params)
        $_POST['page'] = $page;
        $_POST['search'] = $search_query;
        $_POST['content_type'] = $content_type_filter;
        $this->ajax_mxchat_refresh_pinecone_entries();
        return;
    }

    // WordPress DB pagination - MUST match initial page load logic exactly
    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
    $offset = ($page - 1) * $per_page;

    // Build WHERE clause for search and content type filtering
    $where_clauses = array();
    $where_values = array();

    if ($search_query) {
        $where_clauses[] = "article_content LIKE %s";
        $where_values[] = '%' . $wpdb->esc_like($search_query) . '%';
    }

    if ($content_type_filter) {
        switch ($content_type_filter) {
            case 'manual':
                $where_clauses[] = "(source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')";
                break;
            case 'pdf':
                $where_clauses[] = "source_url LIKE '%.pdf'";
                break;
            case 'url':
                $where_clauses[] = "source_url != '' AND source_url IS NOT NULL AND source_url NOT LIKE 'mxchat://%' AND source_url NOT LIKE '%.pdf'";
                break;
        }
    }

    $where_sql = !empty($where_clauses) ? 'WHERE ' . implode(' AND ', $where_clauses) : '';

    // Count grouped entries with filters applied
    if (!empty($where_values)) {
        $count_args = array_merge($where_values, $where_values);
        $count_query = $wpdb->prepare(
            "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
                    (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))",
            ...$count_args
        );
        $total_records = $wpdb->get_var($count_query);
    } else if (!empty($where_sql)) {
        // Content type filter only (no search), no prepared values needed
        $total_records = $wpdb->get_var(
            "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} {$where_sql} AND source_url != '' AND source_url NOT LIKE 'mxchat://%') +
                    (SELECT COUNT(*) FROM {$table_name} {$where_sql} AND (source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%'))"
        );
    } else {
        // No filters
        $total_records = $wpdb->get_var(
            "SELECT (SELECT COUNT(DISTINCT source_url) FROM {$table_name} WHERE source_url != '' AND source_url NOT LIKE 'mxchat://%') +
                    (SELECT COUNT(*) FROM {$table_name} WHERE source_url = '' OR source_url IS NULL OR source_url LIKE 'mxchat://%')"
        );
    }
    $total_pages = ceil($total_records / $per_page);

    // Step 1: Get unique source_urls for this page (ordered by latest timestamp) with filters
    if (!empty($where_values)) {
        $query_args = array_merge($where_values, array($per_page, $offset));
        $urls_query = $wpdb->prepare(
            "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
             {$where_sql}
             GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
            ...$query_args
        );
    } else if (!empty($where_sql)) {
        $urls_query = $wpdb->prepare(
            "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
             {$where_sql}
             GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
            $per_page, $offset
        );
    } else {
        $urls_query = $wpdb->prepare(
            "SELECT source_url, MAX(timestamp) as latest_ts FROM {$table_name}
             GROUP BY source_url ORDER BY latest_ts DESC LIMIT %d OFFSET %d",
            $per_page, $offset
        );
    }
    $page_urls = $wpdb->get_results($urls_query);

    // Step 2: Build list of source_urls to fetch
    $url_list = array();
    $url_order_map = array();
    $order_index = 0;
    foreach ($page_urls as $url_row) {
        $url = $url_row->source_url;
        $url_list[] = $url;
        $url_order_map[$url] = $order_index++;
    }

    // Step 3: Fetch all rows for these source_urls (with search filter if applicable)
    $prompts = array();
    if (!empty($url_list)) {
        $placeholders = implode(',', array_fill(0, count($url_list), '%s'));
        if ($search_query) {
            // Include search filter in the final fetch
            $prompts_query = $wpdb->prepare(
                "SELECT id, article_content, source_url, timestamp, role_restriction
                 FROM {$table_name}
                 WHERE source_url IN ($placeholders) AND article_content LIKE %s
                 ORDER BY timestamp DESC",
                ...array_merge($url_list, ['%' . $wpdb->esc_like($search_query) . '%'])
            );
        } else {
            $prompts_query = $wpdb->prepare(
                "SELECT id, article_content, source_url, timestamp, role_restriction
                 FROM {$table_name}
                 WHERE source_url IN ($placeholders)
                 ORDER BY timestamp DESC",
                $url_list
            );
        }
        $prompts = $wpdb->get_results($prompts_query);
    }

    // Group prompts by source_url for chunk display
    $grouped_prompts = array();
    foreach ($prompts as $prompt) {
        $source_url = $prompt->source_url ?? '';

        // Parse chunk metadata using the proper chunker method (same as initial page load)
        if (class_exists('MxChat_Chunker')) {
            $chunk_meta = MxChat_Chunker::parse_stored_chunk($prompt->article_content);
            $prompt->chunk_metadata = $chunk_meta['metadata'];
            $prompt->display_content = $chunk_meta['text'];
        } else {
            $prompt->chunk_metadata = array();
            $prompt->display_content = $prompt->article_content;
        }

        if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0) {
            if (!isset($grouped_prompts[$source_url])) {
                $grouped_prompts[$source_url] = array();
            }
            $grouped_prompts[$source_url][] = $prompt;
        } else {
            // Ungrouped entries
            $grouped_prompts['_ungrouped_' . $prompt->id] = array($prompt);
        }
    }

    // Sort groups by the original URL order (newest first)
    uksort($grouped_prompts, function($a, $b) use ($url_order_map) {
        $order_a = isset($url_order_map[$a]) ? $url_order_map[$a] : PHP_INT_MAX;
        $order_b = isset($url_order_map[$b]) ? $url_order_map[$b] : PHP_INT_MAX;
        return $order_a - $order_b;
    });

    // Sort each group internally by chunk_index
    foreach ($grouped_prompts as $source_url => &$group) {
        usort($group, function($a, $b) {
            $index_a = isset($a->chunk_metadata['chunk_index']) ? intval($a->chunk_metadata['chunk_index']) : 0;
            $index_b = isset($b->chunk_metadata['chunk_index']) ? intval($b->chunk_metadata['chunk_index']) : 0;
            return $index_a - $index_b;
        });
    }
    unset($group);

    // Build HTML for the table rows
    ob_start();
    $display_index = 0;
    $current_page = $page;
    $data_source = 'wordpress';
    $current_bot_id = $bot_id;
    $preview_length = 150;

    if (empty($grouped_prompts)) {
        echo '<tr><td colspan="5" style="padding: 40px; text-align: center; color: var(--mxch-text-muted);">';
        esc_html_e('No knowledge entries found. Use the Import Options to add content.', 'mxchat');
        echo '</td></tr>';
    } else {
        foreach ($grouped_prompts as $source_url => $group) {
            $chunk_count = count($group);
            $first_prompt = $group[0];
            $display_index++;

            if ($chunk_count > 1) {
                // Multiple chunks - show grouped row with expand button
                $group_id = 'group-' . md5($source_url);
                ?>
                <tr id="prompt-<?php echo esc_attr($first_prompt->id); ?>"
                    class="mxchat-chunk-group-header"
                    data-source="<?php echo esc_attr($data_source); ?>"
                    data-group-id="<?php echo esc_attr($group_id); ?>"
                    style="border-bottom: 1px solid var(--mxch-card-border);">
                    <td style="padding: 12px 16px; text-align: center;">
                        <input type="checkbox"
                               class="mxchat-entry-checkbox"
                               data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
                               data-source="<?php echo esc_attr($data_source); ?>"
                               data-source-url="<?php echo esc_attr($source_url); ?>"
                               data-is-group="true"
                               data-chunk-count="<?php echo esc_attr($chunk_count); ?>">
                    </td>
                    <td style="padding: 12px 16px; font-size: 13px;">
                        <?php echo esc_html($first_prompt->id); ?>
                    </td>
                    <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
                        <div class="mxchat-chunk-group-info">
                            <button type="button" class="mxchat-chunk-toggle" data-group-id="<?php echo esc_attr($group_id); ?>">
                                <span class="dashicons dashicons-arrow-right-alt2"></span>
                            </button>
                            <span class="mxchat-chunk-badge"><?php echo esc_html($chunk_count); ?> <?php esc_html_e('chunks', 'mxchat'); ?></span>
                            <span class="mxchat-chunk-preview">
                                <?php
                                $parent_content = isset($first_prompt->display_content) ? $first_prompt->display_content : $first_prompt->article_content;
                                $content_preview = mb_substr($parent_content, 0, 100);
                                echo esc_html($content_preview . '...');
                                ?>
                            </span>
                        </div>
                    </td>
                    <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
                        <?php if (!empty($source_url) && strpos($source_url, 'mxchat://') !== 0 && strpos($source_url, '_ungrouped_') !== 0) : ?>
                            <a href="<?php echo esc_url($source_url); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
                                <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
                                <?php esc_html_e('View Source', 'mxchat'); ?>
                            </a>
                        <?php else : ?>
                            <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual Content', 'mxchat'); ?></span>
                        <?php endif; ?>
                    </td>
                    <td class="mxchat-actions-cell" style="padding: 12px 16px; white-space: nowrap;">
                        <?php if ($data_source !== 'pinecone') : ?>
                        <button type="button"
                                class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
                                data-source-url="<?php echo esc_attr($source_url); ?>"
                                data-entry-id="<?php echo esc_attr($first_prompt->id); ?>"
                                data-data-source="<?php echo esc_attr($data_source); ?>"
                                data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
                                data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
                                title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
                            <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
                        </button>
                        <?php endif; ?>
                        <button type="button"
                                class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-group"
                                data-source-url="<?php echo esc_attr($source_url); ?>"
                                data-chunk-count="<?php echo esc_attr($chunk_count); ?>"
                                data-data-source="<?php echo esc_attr($data_source); ?>"
                                data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
                                data-nonce="<?php echo wp_create_nonce('mxchat_delete_chunks_nonce'); ?>"
                                style="color: var(--mxch-error);"
                                title="<?php esc_attr_e('Delete all chunks', 'mxchat'); ?>">
                            <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
                        </button>
                    </td>
                </tr>
                <?php
                // Render hidden chunk rows
                foreach ($group as $chunk_index => $chunk) {
                    $meta_chunk_index = isset($chunk->chunk_metadata['chunk_index']) ? intval($chunk->chunk_metadata['chunk_index']) : $chunk_index;
                    $meta_total_chunks = isset($chunk->chunk_metadata['total_chunks']) ? intval($chunk->chunk_metadata['total_chunks']) : $chunk_count;
                    $content = isset($chunk->display_content) ? $chunk->display_content : $chunk->article_content;
                    $content_preview = mb_strlen($content) > $preview_length
                        ? mb_substr($content, 0, $preview_length) . '...'
                        : $content;
                    ?>
                    <tr id="prompt-<?php echo esc_attr($chunk->id); ?>"
                        class="mxchat-chunk-row <?php echo esc_attr($group_id); ?>"
                        data-source="<?php echo esc_attr($data_source); ?>"
                        style="display: none; background: #f8f9fa; border-bottom: 1px solid var(--mxch-card-border);">
                        <td style="padding: 12px 16px; text-align: center;">
                            <!-- Checkbox column placeholder for chunks (managed by group) -->
                        </td>
                        <td style="padding: 12px 16px 12px 30px; font-size: 13px;">
                            <!-- Hidden ID column for chunks -->
                        </td>
                        <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
                            <div class="mxchat-accordion-wrapper">
                                <div class="mxchat-content-preview">
                                    <span class="mxchat-chunk-indicator" style="margin-right: 10px; color: var(--mxch-text-secondary); font-size: 12px;">
                                        <?php printf(esc_html__('Chunk %d of %d', 'mxchat'), $meta_chunk_index + 1, $meta_total_chunks); ?>
                                    </span>
                                    <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
                                    <?php if (mb_strlen($content) > $preview_length) : ?>
                                        <button class="mxchat-expand-toggle" type="button">
                                            <span class="dashicons dashicons-arrow-down-alt2"></span>
                                        </button>
                                    <?php endif; ?>
                                </div>
                                <div class="mxchat-content-full" style="display: none;">
                                    <div class="content-view">
                                        <?php
                                        if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
                                            echo '<div dir="rtl" lang="he" class="rtl-content">';
                                            echo wp_kses_post(wpautop($content));
                                            echo '</div>';
                                        } else {
                                            echo wp_kses_post(wpautop($content));
                                        }
                                        ?>
                                    </div>
                                </div>
                            </div>
                        </td>
                        <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
                            <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Same as parent', 'mxchat'); ?></span>
                        </td>
                        <td class="mxchat-actions-cell" style="padding: 12px 16px;">
                            <span class="mxchat-chunk-label" style="color: var(--mxch-text-muted);"><?php esc_html_e('Managed by group', 'mxchat'); ?></span>
                        </td>
                    </tr>
                    <?php
                }
            } else {
                // Single entry - display normally with accordion
                $prompt = $first_prompt;
                $content = isset($prompt->display_content) ? $prompt->display_content : $prompt->article_content;
                $content_preview = mb_strlen($content) > $preview_length
                    ? mb_substr($content, 0, $preview_length) . '...'
                    : $content;
                ?>
                <tr id="prompt-<?php echo esc_attr($prompt->id); ?>"
                    data-source="<?php echo esc_attr($data_source); ?>"
                    style="border-bottom: 1px solid var(--mxch-card-border);">
                    <td style="padding: 12px 16px; text-align: center;">
                        <input type="checkbox"
                               class="mxchat-entry-checkbox"
                               data-entry-id="<?php echo esc_attr($prompt->id); ?>"
                               data-source="<?php echo esc_attr($data_source); ?>"
                               data-source-url="<?php echo esc_attr($source_url); ?>"
                               data-is-group="false">
                    </td>
                    <td style="padding: 12px 16px; font-size: 13px;">
                        <?php echo esc_html($prompt->id); ?>
                    </td>
                    <td class="mxchat-content-cell" style="padding: 12px 16px; font-size: 13px;">
                        <div class="mxchat-accordion-wrapper">
                            <div class="mxchat-content-preview">
                                <span class="preview-text"><?php echo esc_html($content_preview); ?></span>
                                <?php if (mb_strlen($content) > $preview_length) : ?>
                                    <button class="mxchat-expand-toggle" type="button">
                                        <span class="dashicons dashicons-arrow-down-alt2"></span>
                                    </button>
                                <?php endif; ?>
                            </div>
                            <div class="mxchat-content-full" style="display: none;">
                                <div class="content-view">
                                    <?php
                                    if (preg_match('/[\x{0590}-\x{05FF}]/u', $content)) {
                                        echo '<div dir="rtl" lang="he" class="rtl-content">';
                                        echo wp_kses_post(wpautop($content));
                                        echo '</div>';
                                    } else {
                                        echo wp_kses_post(wpautop($content));
                                    }
                                    ?>
                                </div>
                            </div>
                        </div>
                    </td>
                    <td class="mxchat-url-cell" style="padding: 12px 16px; font-size: 13px;">
                        <?php
                        $actual_source = $source_url;
                        if (strpos($source_url, '_ungrouped_') === 0) {
                            $actual_source = $prompt->source_url ?? '';
                        }
                        if (!empty($actual_source) && strpos($actual_source, 'mxchat://') !== 0) : ?>
                            <a href="<?php echo esc_url($actual_source); ?>" target="_blank" style="color: var(--mxch-primary); text-decoration: none;">
                                <span class="dashicons dashicons-external" style="font-size: 14px;"></span>
                                <?php esc_html_e('View', 'mxchat'); ?>
                            </a>
                        <?php else : ?>
                            <span style="color: var(--mxch-text-muted);"><?php esc_html_e('Manual', 'mxchat'); ?></span>
                        <?php endif; ?>
                    </td>
                    <td style="padding: 12px 16px; white-space: nowrap;">
                        <button type="button"
                                class="mxch-btn mxch-btn-ghost mxch-btn-sm mxchat-edit-entry-btn"
                                data-source-url="<?php echo esc_attr($prompt->source_url ?? ''); ?>"
                                data-entry-id="<?php echo esc_attr($prompt->id); ?>"
                                data-data-source="<?php echo esc_attr($data_source); ?>"
                                data-bot-id="<?php echo esc_attr($current_bot_id); ?>"
                                data-nonce="<?php echo wp_create_nonce('mxchat_edit_entry_nonce'); ?>"
                                title="<?php esc_attr_e('Edit content', 'mxchat'); ?>">
                            <span class="dashicons dashicons-edit" style="font-size: 14px;"></span>
                        </button>
                        <button type="button" class="mxch-btn mxch-btn-ghost mxch-btn-sm delete-button-wordpress" data-entry-id="<?php echo esc_attr($prompt->id); ?>" data-bot-id="<?php echo esc_attr($current_bot_id); ?>" data-nonce="<?php echo wp_create_nonce('mxchat_delete_wordpress_prompt_nonce'); ?>" style="color: var(--mxch-error);">
                            <span class="dashicons dashicons-trash" style="font-size: 14px;"></span>
                        </button>
                    </td>
                </tr>
                <?php
            }
        }
    }
    $html = ob_get_clean();

    // Generate pagination HTML (include search/filter data for subsequent pages)
    $pagination_html = '';
    if ($total_pages > 1) {
        $pagination_html = '<div class="mxchat-ajax-pagination" data-current-page="' . esc_attr($page) . '" data-total-pages="' . esc_attr($total_pages) . '" data-search="' . esc_attr($search_query) . '" data-content-type="' . esc_attr($content_type_filter) . '">';

        // Previous button
        if ($page > 1) {
            $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page - 1) . '">' . esc_html__('&laquo; Previous', 'mxchat') . '</a> ';
        }

        // Page numbers
        $start_page = max(1, $page - 2);
        $end_page = min($total_pages, $page + 2);

        if ($start_page > 1) {
            $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="1">1</a> ';
            if ($start_page > 2) {
                $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
            }
        }

        for ($i = $start_page; $i <= $end_page; $i++) {
            if ($i == $page) {
                $pagination_html .= '<span class="mxchat-page-current">' . $i . '</span> ';
            } else {
                $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $i . '">' . $i . '</a> ';
            }
        }

        if ($end_page < $total_pages) {
            if ($end_page < $total_pages - 1) {
                $pagination_html .= '<span class="mxchat-page-dots">...</span> ';
            }
            $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . $total_pages . '">' . $total_pages . '</a> ';
        }

        // Next button
        if ($page < $total_pages) {
            $pagination_html .= '<a href="#" class="mxchat-page-link" data-page="' . ($page + 1) . '">' . esc_html__('Next &raquo;', 'mxchat') . '</a>';
        }

        $pagination_html .= '</div>';
    }

    wp_send_json_success(array(
        'html' => $html,
        'pagination_html' => $pagination_html,
        'total_count' => $total_records,
        'total_pages' => $total_pages,
        'page' => $page,
        'per_page' => $per_page,
        'data_source' => 'wordpress'
    ));
}

/**
 * AJAX handler to detect available sitemaps on the site
 * Optimized for speed - only checks primary sitemap indexes first
 */
public function ajax_mxchat_detect_sitemaps() {
    check_ajax_referer('mxchat_detect_sitemaps_nonce', 'nonce');

    if (!current_user_can('manage_options')) {
        wp_send_json_error(array('message' => 'Unauthorized'));
        return;
    }

    $site_url = get_site_url();
    $sitemaps = array();
    $found_index = false;

    // Only check the main sitemap index files first (much faster)
    // These are the primary entry points that contain sub-sitemaps
    $primary_indexes = array(
        'sitemap_index.xml' => 'Yoast SEO / Rank Math',  // Yoast & Rank Math
        'wp-sitemap.xml' => 'WordPress Core',            // WordPress Core
        'sitemap.xml' => 'Standard',                     // Generic/AIOSEO
    );

    foreach ($primary_indexes as $path => $source) {
        $url = trailingslashit($site_url) . $path;

        $response = wp_remote_head($url, array(
            'timeout' => 10,
            'sslverify' => false,
            'redirection' => 1,
            'user-agent' => mxchat_ingest_user_agent(),
        ));

        if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
            // Found a sitemap index - parse it to get sub-sitemaps
            $sub_sitemaps = $this->parse_sitemap_index($url);
            if (!empty($sub_sitemaps)) {
                $sitemaps[] = array(
                    'url' => $url,
                    'type' => 'index',
                    'source' => $source,
                    'sub_sitemaps' => $sub_sitemaps
                );
                $found_index = true;
                // Found a valid index, no need to check others
                break;
            }
        }
    }

    // If no sitemap index found, check for standalone sitemaps
    if (!$found_index) {
        $standalone_sitemaps = array(
            'post-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
            'page-sitemap.xml' => array('type' => 'content', 'source' => 'Yoast SEO'),
        );

        foreach ($standalone_sitemaps as $path => $info) {
            $url = trailingslashit($site_url) . $path;

            $response = wp_remote_head($url, array(
                'timeout' => 2,
                'sslverify' => false
            ));

            if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
                $sitemaps[] = array(
                    'url' => $url,
                    'type' => $info['type'],
                    'source' => $info['source'],
                    'url_count' => 0  // Skip URL count for speed
                );
            }
        }
    }

    wp_send_json_success(array(
        'sitemaps' => $sitemaps,
        'site_url' => $site_url
    ));
}

/**
 * Parse a sitemap index to get sub-sitemaps
 * Optimized: doesn't fetch URL count for each sub-sitemap (too slow)
 */
private function parse_sitemap_index($url) {
    $sub_sitemaps = array();

    $response = wp_remote_get($url, array(
        'timeout' => 30,
        'sslverify' => false,
        'user-agent' => mxchat_ingest_user_agent(),
        'headers' => array(
            'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        ),
    ));

    if (is_wp_error($response)) {
        return $sub_sitemaps;
    }

    $body = wp_remote_retrieve_body($response);
    if (empty($body)) {
        return $sub_sitemaps;
    }

    // Suppress XML errors
    libxml_use_internal_errors(true);
    $xml = simplexml_load_string($body);
    libxml_clear_errors();

    if ($xml === false) {
        return $sub_sitemaps;
    }

    // Check if it's a sitemap index (contains <sitemap> elements)
    if (isset($xml->sitemap)) {
        foreach ($xml->sitemap as $sitemap) {
            $loc = (string) $sitemap->loc;
            if (!empty($loc)) {
                // Try to determine the type from the URL
                $type = 'content';
                if (strpos($loc, 'category') !== false || strpos($loc, 'tag') !== false || strpos($loc, 'taxonomy') !== false) {
                    $type = 'taxonomy';
                } elseif (strpos($loc, 'author') !== false || strpos($loc, 'user') !== false) {
                    $type = 'author';
                }

                // Skip URL count - too slow to fetch for each sitemap
                $sub_sitemaps[] = array(
                    'url' => $loc,
                    'type' => $type,
                    'url_count' => 0,  // Don't fetch - takes too long
                    'name' => basename(parse_url($loc, PHP_URL_PATH))
                );
            }
        }
    }

    return $sub_sitemaps;
}

/**
 * Get URL count from a sitemap
 */
private function get_sitemap_url_count($url) {
    $response = wp_remote_get($url, array(
        'timeout' => 30,
        'sslverify' => false,
        'user-agent' => mxchat_ingest_user_agent(),
        'headers' => array(
            'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        ),
    ));

    if (is_wp_error($response)) {
        return 0;
    }

    $body = wp_remote_retrieve_body($response);
    if (empty($body)) {
        return 0;
    }

    // Count <url> or <loc> elements
    $count = preg_match_all('/<url>/i', $body, $matches);
    return $count ?: 0;
}

/**
 * Get sitemaps declared in robots.txt
 */
private function get_sitemaps_from_robots($site_url) {
    $sitemaps = array();
    $robots_url = trailingslashit($site_url) . 'robots.txt';

    $response = wp_remote_get($robots_url, array(
        'timeout' => 15,
        'sslverify' => false,
        'user-agent' => mxchat_ingest_user_agent(),
    ));

    if (is_wp_error($response)) {
        return $sitemaps;
    }

    $body = wp_remote_retrieve_body($response);
    if (empty($body)) {
        return $sitemaps;
    }

    // Find Sitemap: declarations
    if (preg_match_all('/^Sitemap:\s*(.+)$/mi', $body, $matches)) {
        foreach ($matches[1] as $sitemap_url) {
            $sitemap_url = trim($sitemap_url);
            if (filter_var($sitemap_url, FILTER_VALIDATE_URL)) {
                $sitemaps[] = $sitemap_url;
            }
        }
    }

    return $sitemaps;
}

public function mxchat_stop_processing() {
    // Verify permissions
    if (!current_user_can('manage_options')) {
        wp_die(esc_html__('Unauthorized access', 'mxchat'));
    }

    // Verify nonce
    check_admin_referer('mxchat_stop_processing_action', 'mxchat_stop_processing_nonce');

    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_processing_queue';
    
    // Get active queue IDs
    $sitemap_queue_id = get_transient('mxchat_active_queue_sitemap');
    $pdf_queue_id = get_transient('mxchat_active_queue_pdf');
    
    // Delete all pending items from active queues
    if ($sitemap_queue_id) {
        $wpdb->delete(
            $table_name,
            array(
                'queue_id' => $sitemap_queue_id,
                'status' => 'pending'
            ),
            array('%s', '%s')
        );
        
        delete_transient('mxchat_active_queue_sitemap');
        delete_transient('mxchat_last_sitemap_url');
    }
    
    if ($pdf_queue_id) {
        // Get PDF path before deleting
        $pdf_path = $this->mxchat_get_queue_meta($pdf_queue_id, 'pdf_path');
        
        $wpdb->delete(
            $table_name,
            array(
                'queue_id' => $pdf_queue_id,
                'status' => 'pending'
            ),
            array('%s', '%s')
        );
        
        // Delete PDF file
        if ($pdf_path && file_exists($pdf_path)) {
            wp_delete_file($pdf_path);
        }
        
        delete_transient('mxchat_active_queue_pdf');
        delete_transient('mxchat_last_pdf_url');
    }

    // Redirect back with a success message
    set_transient('mxchat_admin_notice_success',
        esc_html__('Processing has been stopped successfully.', 'mxchat'),
        30
    );
    wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
    exit;
}

/**
 * Get content list for processing
 */
public function ajax_mxchat_get_content_list() {
    // Verify the nonce
    check_ajax_referer('mxchat_content_selector_nonce', 'nonce');
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error(__('Unauthorized access', 'mxchat'));
    }
    
    $page = isset($_GET['page']) ? absint($_GET['page']) : 1;
    $per_page = isset($_GET['per_page']) ? absint($_GET['per_page']) : 100;
    $search = isset($_GET['search']) ? sanitize_text_field($_GET['search']) : '';
    $post_type = isset($_GET['post_type']) ? sanitize_text_field($_GET['post_type']) : 'all';
    $post_status = isset($_GET['post_status']) ? sanitize_text_field($_GET['post_status']) : 'publish';
    $processed_filter = isset($_GET['processed_filter']) ? sanitize_text_field($_GET['processed_filter']) : 'all';
    
    // Build query args
    $args = array(
        'posts_per_page' => $per_page,
        'paged' => $page,
        'post_status' => $post_status !== 'all' ? $post_status : array('publish', 'draft', 'pending'),
        'orderby' => 'date',
        'order' => 'DESC',
    );
    
    // Handle post types - IMPROVED VERSION
    if ($post_type !== 'all') {
        $args['post_type'] = $post_type;
    } else {
        // Get all available post types that might contain content
        $all_post_types = array();
        
        // First get all public post types
        $public_types = get_post_types(array('public' => true), 'names');
        $all_post_types = array_merge($all_post_types, $public_types);
        
        // Add common forum/community post types
        $forum_types = array('topic', 'reply', 'forum', 'wpforo_topic', 'wpforo_post');
        foreach ($forum_types as $forum_type) {
            if (post_type_exists($forum_type)) {
                $all_post_types[] = $forum_type;
            }
        }
        
        // Add other commonly used post types
        $common_types = array('product', 'job_listing', 'event', 'portfolio');
        foreach ($common_types as $common_type) {
            if (post_type_exists($common_type)) {
                $all_post_types[] = $common_type;
            }
        }
        
        // Remove duplicates and ensure we have at least some post types
        $all_post_types = array_unique($all_post_types);
        
        if (empty($all_post_types)) {
            // Fallback to basic post types
            $all_post_types = array('post', 'page');
        }
        
        $args['post_type'] = $all_post_types;
        
        // Debug logging to see what post types are being queried
        //error_log('MxChat Debug: Querying post types: ' . implode(', ', $all_post_types));
    }
    
    if (!empty($search)) {
        $args['s'] = $search;
    }
    
    // Get processed data from storage
    $processed_data = array();
    
    $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
    $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';

    if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
        // Get fresh data from Pinecone - no caching
        $processed_data = $this->mxchat_get_pinecone_processed_content($pinecone_options);
    } else {
        // WordPress DB checking with better URL matching for all post types
        global $wpdb;
        $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
        $processed_items = $wpdb->get_results("SELECT id, source_url, article_content, timestamp FROM {$table_name}");

        // Group items by source_url to count chunks
        $url_chunk_counts = array();
        $url_latest_timestamp = array();
        $url_first_id = array();

        if (!empty($processed_items)) {
            foreach ($processed_items as $item) {
                $url = $item->source_url;
                if (empty($url)) continue;

                // Count chunks per URL
                if (!isset($url_chunk_counts[$url])) {
                    $url_chunk_counts[$url] = 0;
                    $url_latest_timestamp[$url] = $item->timestamp;
                    $url_first_id[$url] = $item->id;
                }
                $url_chunk_counts[$url]++;

                // Track latest timestamp
                if (strtotime($item->timestamp) > strtotime($url_latest_timestamp[$url])) {
                    $url_latest_timestamp[$url] = $item->timestamp;
                }
            }

            // Now build processed_data with chunk counts
            foreach ($url_chunk_counts as $url => $chunk_count) {
                $post_id = $this->mxchat_url_to_post_id_improved($url);

                if ($post_id) {
                    $processed_data[$post_id] = array(
                        'db_id' => $url_first_id[$url],
                        'timestamp' => $url_latest_timestamp[$url],
                        'url' => $url,
                        'source' => 'wordpress',
                        'chunk_count' => $chunk_count
                    );
                }
            }
        }
    }
    
    // Get processed IDs as a simple array for in_array checks
    $processed_ids = array_keys($processed_data);
    
    // Handle processed/unprocessed filter
    if ($processed_filter === 'processed' && !empty($processed_ids)) {
        $args['post__in'] = $processed_ids;
    } elseif ($processed_filter === 'unprocessed' && !empty($processed_ids)) {
        $args['post__not_in'] = $processed_ids;
    }
    
    // Run the query
    $query = new WP_Query($args);
    $content_items = array();
    
    if ($query->have_posts()) {
        while ($query->have_posts()) {
            $query->the_post();
            $id = get_the_ID();
            $post_date = get_the_date();
            $excerpt = wp_trim_words(get_the_excerpt(), 20, '...');
            $word_count = str_word_count(strip_tags(get_the_content()));
            
            $is_processed = in_array($id, $processed_ids);
            $processed_date = '';
            $db_record_id = 0;
            $data_source = 'none';
            
            if ($is_processed && isset($processed_data[$id])) {
                $item_data = $processed_data[$id];
                $data_source = $item_data['source'];
                
                if ($data_source === 'wordpress' && isset($item_data['timestamp'])) {
                    // WordPress DB format
                    $timestamp = strtotime($item_data['timestamp']);
                    $processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
                    $db_record_id = $item_data['db_id'];
                } elseif ($data_source === 'pinecone') {
                    // Pinecone format
                    $processed_date = $item_data['processed_date'];
                    $db_record_id = $item_data['db_id'];
                }
            }
            
            // Get chunk count for this item
            $chunk_count = 0;
            if ($is_processed && isset($processed_data[$id]['chunk_count'])) {
                $chunk_count = intval($processed_data[$id]['chunk_count']);
            }

            $content_items[] = array(
                'id' => $id,
                'title' => get_the_title(),
                'permalink' => get_permalink(),
                'date' => $post_date,
                'type' => get_post_type(),
                'status' => get_post_status(),
                'excerpt' => $excerpt,
                'word_count' => $word_count,
                'already_processed' => $is_processed,
                'processed_date' => $processed_date,
                'db_record_id' => $db_record_id,
                'data_source' => $data_source,
                'chunk_count' => $chunk_count
            );
        }
        wp_reset_postdata();
    }
    
    $response = array(
        'items' => $content_items,
        'total' => $query->found_posts,
        'total_pages' => $query->max_num_pages,
        'current_page' => $page,
        'processed_count' => count($processed_ids)
    );
    
    wp_send_json_success($response);
    exit;
}


/**
 * This function handles various WooCommerce URL formats and permalink structures
 */
private function mxchat_url_to_post_id_improved($url) {
    // First try the standard WordPress function
    $post_id = url_to_postid($url);
    
    if ($post_id > 0) {
        return $post_id;
    }
    
    // If that fails, try more aggressive URL matching
    // Remove trailing slashes and query parameters for better matching
    $clean_url = rtrim($url, '/');
    $clean_url = strtok($clean_url, '?'); // Remove query parameters
    
    // Try again with cleaned URL
    $post_id = url_to_postid($clean_url);
    if ($post_id > 0) {
        return $post_id;
    }
    
    // For bbPress forum topics, try extracting slug from URL
    if (strpos($url, '/topic/') !== false || strpos($url, '/forums/') !== false) {
        // Handle bbPress URLs: /forums/topic/topic-name/
        if (preg_match('/\/forums\/topic\/([^\/\?]+)/', $url, $matches)) {
            $topic_slug = $matches[1];
            
            // Look up topic by slug
            $topic = get_page_by_path($topic_slug, OBJECT, 'topic');
            if ($topic) {
                return $topic->ID;
            }
            
            // Alternative method: query by post_name
            global $wpdb;
            $post_id = $wpdb->get_var($wpdb->prepare(
                "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
                $topic_slug
            ));
            
            if ($post_id) {
                return intval($post_id);
            }
        }
        
        // Handle simpler topic URLs: /topic/topic-name/
        if (preg_match('/\/topic\/([^\/\?]+)/', $url, $matches)) {
            $topic_slug = $matches[1];
            
            global $wpdb;
            $post_id = $wpdb->get_var($wpdb->prepare(
                "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'topic' AND post_status IN ('publish', 'closed')",
                $topic_slug
            ));
            
            if ($post_id) {
                return intval($post_id);
            }
        }
    }
    
    // For WooCommerce products
    if (strpos($url, '/product/') !== false || strpos($url, 'product=') !== false) {
        // Extract product slug from various URL formats
        $product_slug = '';
        
        // Handle pretty permalinks: /product/product-name/
        if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
            $product_slug = $matches[1];
        }
        // Handle query parameters: ?product=product-name
        elseif (preg_match('/[\?&]product=([^&]+)/', $url, $matches)) {
            $product_slug = $matches[1];
        }
        
        if (!empty($product_slug)) {
            // Look up product by slug
            $product = get_page_by_path($product_slug, OBJECT, 'product');
            if ($product) {
                return $product->ID;
            }
            
            // Alternative method: query by post_name
            global $wpdb;
            $post_id = $wpdb->get_var($wpdb->prepare(
                "SELECT ID FROM {$wpdb->posts} WHERE post_name = %s AND post_type = 'product' AND post_status = 'publish'",
                $product_slug
            ));
            
            if ($post_id) {
                return intval($post_id);
            }
        }
    }
    
    // Generic approach: try to extract slug and match against all post types
    $parsed_url = wp_parse_url($clean_url);
    $path = $parsed_url['path'] ?? '';
    
    if (!empty($path)) {
        // Get the last part of the path as potential slug
        $path_parts = array_filter(explode('/', trim($path, '/')));
        $potential_slug = end($path_parts);
        
        if (!empty($potential_slug)) {
            global $wpdb;
            
            // Try to find any post with this slug
            $post_id = $wpdb->get_var($wpdb->prepare(
                "SELECT ID FROM {$wpdb->posts} 
                 WHERE post_name = %s 
                 AND post_status IN ('publish', 'closed', 'private') 
                 AND post_type NOT IN ('revision', 'attachment', 'nav_menu_item')
                 ORDER BY CASE 
                    WHEN post_type = 'post' THEN 1
                    WHEN post_type = 'page' THEN 2
                    WHEN post_type = 'topic' THEN 3
                    WHEN post_type = 'product' THEN 4
                    ELSE 5
                 END
                 LIMIT 1",
                $potential_slug
            ));
            
            if ($post_id) {
                return intval($post_id);
            }
        }
    }
    
    // ADDITIONAL: Try direct database lookup by URL variations
    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
    
    // Try variations of the URL (with/without trailing slash, http/https)
    $url_variations = array(
        $url,
        rtrim($url, '/'),
        $url . '/',
        str_replace('http://', 'https://', $url),
        str_replace('https://', 'http://', $url),
        str_replace('http://', 'https://', rtrim($url, '/')),
        str_replace('https://', 'http://', rtrim($url, '/'))
    );
    
    // Remove duplicates
    $url_variations = array_unique($url_variations);
    
    foreach ($url_variations as $variation) {
        $existing_record = $wpdb->get_row($wpdb->prepare(
            "SELECT id, source_url FROM $table_name WHERE source_url = %s",
            $variation
        ));
        
        if ($existing_record) {
            // Try to get post ID from this stored URL
            $stored_post_id = url_to_postid($existing_record->source_url);
            if ($stored_post_id > 0) {
                return $stored_post_id;
            }
        }
    }
    
    return 0; // No match found
}
/**
 * Process selected content via AJAX
 */
public function ajax_mxchat_process_selected_content() {
    // Basic request validation
    if (!check_ajax_referer('mxchat_content_selector_nonce', 'nonce', false)) {
        wp_send_json_error('Invalid nonce');
        exit;
    }
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
        exit;
    }
    
    // Get post IDs - safely parse the array
    $post_ids = array();
    if (isset($_POST['post_ids']) && is_array($_POST['post_ids'])) {
        foreach ($_POST['post_ids'] as $id) {
            $post_ids[] = absint($id);
        }
    }
    
    if (empty($post_ids)) {
        wp_send_json_error('No content selected');
        exit;
    }
    
    //  Get bot_id from request
    $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';

    // ACF→PDF extraction is an install-level setting (Knowledge → ACF Fields,
    // plan 11720c). The import modal shows a passive status line pointing
    // there; the old per-batch checkbox and its remembered default are gone.
    $extract_acf_pdfs = get_option('mxchat_acf_pdf_extraction', '0') === '1';

    // Process only ONE post at a time to avoid request size issues
    $post_id = reset($post_ids);
    $post = get_post($post_id);
    
    if (!$post) {
        wp_send_json_error('Post not found');
        exit;
    }

    /**
     * Allow developers to modify post data before processing into the knowledge base.
     * Applied on BOTH content-preparation paths (this manual bulk import and the
     * auto-sync path in mxchat_handle_post_update) with the same signature, so a
     * callback registered once covers every indexing route. Purely additive —
     * zero behaviour change when unhooked.
     *
     * @param WP_Post $post   The post about to be indexed.
     * @param string  $bot_id Bot context for this import.
     */
    $post = apply_filters('mxchat_before_process_post', $post, $bot_id);
    if (!($post instanceof WP_Post)) {
        $post = get_post($post_id); // defend against a bad callback return
    }

    // Assemble the indexable text via the shared post-kind assembler (a3d60c).
    // Bulk import reads raw post fields, has always included product custom tabs,
    // and passes the install-level ACF→PDF option.
    $prepared = $this->mxchat_prepare_post_content_for_indexing($post_id, $post, array(
        'read_display'         => false,
        'extract_acf_pdfs'     => $extract_acf_pdfs,
        'include_product_tabs' => true,
    ));
    $content             = $prepared['content'];
    $pdf_extracted_count = $prepared['pdf_extracted_count'];

    // Debug logging for WordPress Import content
    //error_log('[MXCHAT-WP-IMPORT-DEBUG] Post ID: ' . $post_id . ' Title: ' . $post->post_title);
    //error_log('[MXCHAT-WP-IMPORT-DEBUG] Raw post_content length: ' . strlen($post->post_content));
    //error_log('[MXCHAT-WP-IMPORT-DEBUG] Final content length: ' . strlen($content));
    //error_log('[MXCHAT-WP-IMPORT-DEBUG] Content preview: ' . substr($content, 0, 300));

    // Note: Removed 10,000 char limit - chunking now handles large content properly

    //  Get bot-specific embedding decision — custom-provider-aware (plan cbd5fd)
    $bot_options = $this->get_bot_options($bot_id);
    $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');

    $preflight = MxChat_Utils::embedding_preflight($options);
    if (!$preflight['ok']) {
        MxChat_Admin::mxchat_log_debug('api_error', $preflight['reason'] . ' (knowledge processing)');
        wp_send_json_error($preflight['reason']);
        exit;
    }
    $api_key = $preflight['api_key'];
    
    $source_url = get_permalink($post_id);
    $vector_id = md5($source_url); // Vector ID for Pinecone
    
    //  Check for existing content in bot-specific storage
    $is_update = false;
    
    // Get bot-specific Pinecone configuration
    $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
    $use_pinecone = !empty($bot_pinecone_config) && ($bot_pinecone_config['use_pinecone'] ?? false);
    
    if ($use_pinecone && !empty($bot_pinecone_config['api_key'])) {
        // Check Pinecone for this bot
        $pinecone_data = $this->mxchat_get_pinecone_processed_content($bot_pinecone_config);
        if (isset($pinecone_data[$post_id])) {
            $is_update = true;
        }
    } else {
        // Check WordPress DB (same as before since it's shared)
        global $wpdb;
        $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
        $existing_record = $wpdb->get_row($wpdb->prepare(
            "SELECT id FROM $table_name WHERE source_url = %s",
            $source_url
        ));
        
        if ($existing_record) {
            $is_update = true;
        }
    }

    // UPDATED 2.5.6: Determine content type based on post_type
    $post_type = $post->post_type;
    $content_type = 'content'; // Default fallback

    // Map WordPress post types to content types
    switch ($post_type) {
        case 'post':
            $content_type = 'post';
            break;
        case 'page':
            $content_type = 'page';
            break;
        case 'product':
            $content_type = 'product';
            break;
        default:
            // For custom post types, use the post type name
            $content_type = sanitize_key($post_type);
            break;
    }

    //  Use the centralized utility function with bot_id and content_type
    $result = MxChat_Utils::submit_content_to_db(
        $content,
        $source_url,
        $api_key,
        $vector_id,
        $bot_id,
        $content_type
    );

    if (is_wp_error($result)) {
        MxChat_Admin::mxchat_log_debug('storage_error', 'Knowledge storage failed: ' . $result->get_error_message(), array('source_url' => $source_url));
        wp_send_json_error('Storage failed: ' . $result->get_error_message());
        exit;
    }

    //   Automatically apply role restriction based on tags
    $this->apply_role_restriction_to_post($post_id, $source_url);

    $operation_type = $is_update ? 'update' : 'new';

    // Count ACF fields for debugging
    $acf_field_count = $prepared['acf_fields_found'];
    
    // Success response with minimal data
    wp_send_json_success(array(
        'message' => $operation_type === 'update' ? 'Content updated successfully' : 'Content processed successfully',
        'post_id' => $post_id,
        'title'   => $post->post_title,
        'operation_type' => $operation_type,
        'vector_id' => $vector_id,
        'acf_fields_found' => $acf_field_count,
        'pdf_extracted_count' => (int) $pdf_extracted_count,
        'content_preview' => substr($content, 0, 100) . '...',
        'bot_id' => $bot_id
    ));
    exit;
}

private function apply_role_restriction_to_post($post_id, $source_url) {
    // Get tag-role mappings
    $mappings = get_option('mxchat_tag_role_mappings', array());
    
    if (empty($mappings)) {
        return; // No mappings, leave as public
    }
    
    // Get all tags for the post
    $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
    
    if (empty($post_tags)) {
        return; // No tags, leave as public
    }
    
    // Determine the highest role restriction based on tags
    $highest_role = 'public';
    $role_hierarchy = array(
        'public' => 0,
        'logged_in' => 1,
        'subscriber' => 2,
        'contributor' => 3,
        'author' => 4,
        'editor' => 5,
        'administrator' => 6
    );
    
    foreach ($post_tags as $tag_slug) {
        if (isset($mappings[$tag_slug])) {
            $role = $mappings[$tag_slug];
            if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
                $highest_role = $role;
            }
        }
    }
    
    // If no restricted tags found, return (leave as public)
    if ($highest_role === 'public') {
        return;
    }
    
    // Update the role restriction in the database
    global $wpdb;
    
    // Check if using Pinecone
    $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
    $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
    
    if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
        // Update Pinecone role restriction
        $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
        $vector_id = md5($source_url);
        
        $wpdb->replace(
            $roles_table,
            array(
                'vector_id' => $vector_id,
                'role_restriction' => $highest_role,
                'updated_at' => current_time('mysql')
            ),
            array('%s', '%s', '%s')
        );
    } else {
        // Update WordPress DB
        $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';

        $wpdb->update(
            $table_name,
            array('role_restriction' => $highest_role),
            array('source_url' => $source_url),
            array('%s'),
            array('%s')
        );
    }

    // The entry's restriction just changed — keep the OpenAI Vector Store
    // mirror consistent: non-public pulls the file (file_search has no
    // per-role filtering), public re-mirrors it (plan 15b5c6).
    if (class_exists('MxChat_Vectorstore_Manager')) {
        MxChat_Vectorstore_Manager::handle_role_change($source_url, 'default', $highest_role);
    }
}

public function mxchat_get_public_post_types() {
    // Get all public post types
    $post_types = get_post_types(array('public' => true), 'objects');
    $post_type_options = array();
    
    foreach ($post_types as $post_type) {
        $post_type_options[$post_type->name] = $post_type->label;
    }
    
    // Also include common forum/community post types that might not be marked as public
    $additional_types = array(
        'topic' => 'Forum Topics (bbPress)',
        'reply' => 'Forum Replies (bbPress)', 
        'forum' => 'Forums (bbPress)',
        'wpforo_topic' => 'wpForo Topics',
        'wpforo_post' => 'wpForo Posts'
    );
    
    foreach ($additional_types as $type_name => $type_label) {
        if (post_type_exists($type_name) && !isset($post_type_options[$type_name])) {
            $post_type_options[$type_name] = $type_label;
        }
    }
    
    return $post_type_options;
}

/**
 * Retrieves processed content from Pinecone API
 */
public function mxchat_get_pinecone_processed_content($pinecone_options) {
    $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
    $host = $pinecone_options['mxchat_pinecone_host'] ?? '';

    if (empty($api_key) || empty($host)) {
        return array();
    }

    $pinecone_data = array();

    try {
        // Always get fresh data from Pinecone
        $pinecone_data = $this->mxchat_scan_pinecone_for_processed_content($pinecone_options);

        // Method 2: Final fallback - try stats endpoint (if available)
        if (empty($pinecone_data)) {
            $stats_url = "https://{$host}/describe_index_stats";

            $response = wp_remote_post($stats_url, array(
                'headers' => array(
                    'Api-Key' => $api_key,
                    'Content-Type' => 'application/json'
                ),
                'body' => json_encode(array()),
                'timeout' => 30
            ));

            if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
                $body = wp_remote_retrieve_body($response);
                $stats_data = json_decode($body, true);
            }
        }

    } catch (Exception $e) {
        // Log error but return fresh data only
    }

    return $pinecone_data;
}
public function mxchat_fetch_pinecone_vectors_by_ids($pinecone_options, $vector_ids) {
    $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
    $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
    $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';

    if (empty($api_key) || empty($host) || empty($vector_ids)) {
        return array();
    }

    try {
        // NOTE (plan 793b82): /vectors/fetch is a GET endpoint with the ids
        // repeated in the query string (ids=a&ids=b — http_build_query would
        // emit ids[0]=a); the old POST here was answered 200-with-an-empty-body,
        // which read as "nothing indexed". Chunked at 100 ids to stay well
        // under the measured HTTP 414 URL-length boundary.
        $vectors = array();
        foreach (array_chunk(array_values($vector_ids), 100) as $chunk) {
            $fetch_query = array();
            foreach ($chunk as $fetch_vid) {
                $fetch_query[] = 'ids=' . rawurlencode($fetch_vid);
            }
            if (!empty($namespace)) {
                $fetch_query[] = 'namespace=' . rawurlencode($namespace);
            }

            $response = wp_remote_get("https://{$host}/vectors/fetch?" . implode('&', $fetch_query), array(
                'headers' => array(
                    'Api-Key' => $api_key,
                    'accept' => 'application/json'
                ),
                'timeout' => 30
            ));

            if (is_wp_error($response)) {
                error_log('MxChat Pinecone: mxchat_fetch_pinecone_vectors_by_ids GET failed: ' . $response->get_error_message());
                continue;
            }

            if (wp_remote_retrieve_response_code($response) !== 200) {
                error_log('MxChat Pinecone: mxchat_fetch_pinecone_vectors_by_ids GET returned HTTP ' . wp_remote_retrieve_response_code($response));
                continue;
            }

            $data = json_decode(wp_remote_retrieve_body($response), true);
            if (isset($data['vectors']) && is_array($data['vectors'])) {
                $vectors += $data['vectors'];
            }
        }

        if (empty($vectors)) {
            return array();
        }

        $processed_data = array();

        foreach ($vectors as $vector_id => $vector_data) {
            $metadata = $vector_data['metadata'] ?? array();
            $source_url = $metadata['source_url'] ?? '';

            if (!empty($source_url)) {
                $post_id = url_to_postid($source_url);
                if ($post_id) {
                    $created_at = $metadata['created_at'] ?? '';
                    $processed_date = 'Recently';

                    if (!empty($created_at)) {
                        $timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
                        if ($timestamp) {
                            $processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
                        }
                    }

                    $processed_data[$post_id] = array(
                        'db_id' => $vector_id,
                        'processed_date' => $processed_date,
                        'url' => $source_url,
                        'source' => 'pinecone',
                        'timestamp' => $timestamp ?? current_time('timestamp')
                    );
                }
            }
        }

        //error_log('DEBUG: Processed ' . count($processed_data) . ' vectors from fetch');
        return $processed_data;

    } catch (Exception $e) {
        //error_log('DEBUG: Exception in fetch_pinecone_vectors_by_ids: ' . $e->getMessage());
        return array();
    }
}

/**
 * Get embedding dimensions based on the selected model.
 */
private function mxchat_get_embedding_dimensions() {
    $options = get_option('mxchat_options', array());
    $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';

    $model_dimensions = array(
        'text-embedding-ada-002' => 1536,
        'text-embedding-3-small' => 1536,
        'text-embedding-3-large' => 3072,
        'voyage-2' => 1024,
        'voyage-large-2' => 1536,
        'voyage-3-large' => 2048,
        'gemini-embedding-001' => 1536,
    );

    if (strpos($selected_model, 'voyage-3-large') === 0) {
        $custom_dimensions = $options['voyage_output_dimension'] ?? 2048;
        return intval($custom_dimensions);
    }

    if (strpos($selected_model, 'gemini-embedding') === 0) {
        $custom_dimensions = $options['gemini_output_dimension'] ?? 1536;
        return intval($custom_dimensions);
    }

    return $model_dimensions[$selected_model] ?? 1536;
}

/**
 * Scan Pinecone for processed content
 */
public function mxchat_scan_pinecone_for_processed_content($pinecone_options) {
    $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
    $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
    // plan 793b82: this scan was namespace-blind — on a namespaced setup it
    // surveyed the default namespace and reported the wrong content as indexed.
    $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';

    if (empty($api_key) || empty($host)) {
        return array();
    }

    try {
        // Use multiple random vectors to get better coverage
        $all_matches = array();
        $seen_ids = array();

        // Get correct dimensions for the configured embedding model
        $dimensions = $this->mxchat_get_embedding_dimensions();

        // Try 3 different random vectors to get better coverage
        for ($i = 0; $i < 3; $i++) {
            $query_url = "https://{$host}/query";

            // Generate a random unit vector instead of zeros
            $random_vector = array();
            for ($j = 0; $j < $dimensions; $j++) {
                $random_vector[] = (rand(-1000, 1000) / 1000.0);
            }

            // Normalize the vector to unit length
            $magnitude = sqrt(array_sum(array_map(function($x) { return $x * $x; }, $random_vector)));
            if ($magnitude > 0) {
                $random_vector = array_map(function($x) use ($magnitude) { return $x / $magnitude; }, $random_vector);
            }

            $query_data = array(
                'includeMetadata' => true,
                'includeValues' => false,
                'topK' => 10000,
                'vector' => $random_vector
            );

            if (!empty($namespace)) {
                $query_data['namespace'] = $namespace;
            }

            $response = wp_remote_post($query_url, array(
                'headers' => array(
                    'Api-Key' => $api_key,
                    'Content-Type' => 'application/json'
                ),
                'body' => json_encode($query_data),
                'timeout' => 30
            ));

            if (is_wp_error($response)) {
                continue;
            }

            $response_code = wp_remote_retrieve_response_code($response);

            if ($response_code !== 200) {
                continue;
            }

            $body = wp_remote_retrieve_body($response);
            $data = json_decode($body, true);

            if (isset($data['matches'])) {
                foreach ($data['matches'] as $match) {
                    $match_id = $match['id'] ?? '';
                    if (!empty($match_id) && !isset($seen_ids[$match_id])) {
                        $all_matches[] = $match;
                        $seen_ids[$match_id] = true;
                    }
                }
            }
        }

        // Convert matches to processed data format, grouping by URL to count chunks
        $processed_data = array();
        $url_chunk_counts = array();

        foreach ($all_matches as $match) {
            $metadata = $match['metadata'] ?? array();
            $source_url = $metadata['source_url'] ?? '';
            $match_id = $match['id'] ?? '';

            if (!empty($source_url) && !empty($match_id)) {
                $post_id = url_to_postid($source_url);
                if ($post_id) {
                    // Count chunks per post_id
                    if (!isset($url_chunk_counts[$post_id])) {
                        $url_chunk_counts[$post_id] = 0;
                    }
                    $url_chunk_counts[$post_id]++;

                    $created_at = $metadata['created_at'] ?? '';
                    $processed_date = 'Recently';

                    if (!empty($created_at)) {
                        $timestamp = is_numeric($created_at) ? $created_at : strtotime($created_at);
                        if ($timestamp) {
                            $processed_date = human_time_diff($timestamp, current_time('timestamp')) . ' ago';
                        }
                    }

                    // Only store if not already set, or update with newer timestamp
                    if (!isset($processed_data[$post_id]) ||
                        ($timestamp ?? 0) > ($processed_data[$post_id]['timestamp'] ?? 0)) {
                        $processed_data[$post_id] = array(
                            'db_id' => $match_id,
                            'processed_date' => $processed_date,
                            'url' => $source_url,
                            'source' => 'pinecone',
                            'timestamp' => $timestamp ?? current_time('timestamp')
                        );
                    }
                }
            }
        }

        // Add chunk counts to processed data
        foreach ($url_chunk_counts as $post_id => $chunk_count) {
            if (isset($processed_data[$post_id])) {
                $processed_data[$post_id]['chunk_count'] = $chunk_count;
            }
        }

        return $processed_data;

    } catch (Exception $e) {
        return array();
    }
}
/**
 *  Generate embeddings from input text for MXChat with bot support
 */
private function mxchat_generate_embedding($text, $bot_id = 'default') {
    // Enable detailed logging for debugging
    //error_log('[MXCHAT-EMBED] Starting embedding generation for bot: ' . $bot_id . '. Text length: ' . strlen($text) . ' bytes');
    //error_log('[MXCHAT-EMBED] Text preview: ' . substr($text, 0, 100) . '...');

    //  Get bot-specific options
    $bot_options = $this->get_bot_options($bot_id);
    $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');

    // Opt-in: when the custom provider is selected for embeddings, index through
    // the same custom endpoint the query path uses so stored vectors and query
    // vectors share a model. Returns the vector array on success, or an error
    // string on failure (this function's existing failure contract).
    if (isset($options['custom_provider_for_embeddings']) && $options['custom_provider_for_embeddings'] === 'on') {
        if (!class_exists('MxChat_Utils')) {
            require_once dirname(__FILE__) . '/../includes/class-mxchat-utils.php';
        }
        return MxChat_Utils::generate_embedding_custom($text, $options);
    }

    $selected_model = $options['embedding_model'] ?? 'text-embedding-ada-002';
    //error_log('[MXCHAT-EMBED] Selected embedding model for bot ' . $bot_id . ': ' . $selected_model);

    // Determine provider and endpoint
    if (strpos($selected_model, 'voyage') === 0) {
        $api_key = $options['voyage_api_key'] ?? '';
        $endpoint = 'https://api.voyageai.com/v1/embeddings';
        $provider_name = 'Voyage AI';
        //error_log('[MXCHAT-EMBED] Using Voyage AI API for bot ' . $bot_id);
    } elseif (strpos($selected_model, 'gemini-embedding') === 0) {
        $api_key = $options['gemini_api_key'] ?? '';
        $endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $selected_model . ':embedContent';
        $provider_name = 'Google Gemini';
        //error_log('[MXCHAT-EMBED] Using Google Gemini API for bot ' . $bot_id);
    } else {
        $api_key = $options['api_key'] ?? '';
        $endpoint = 'https://api.openai.com/v1/embeddings';
        $provider_name = 'OpenAI';
        //error_log('[MXCHAT-EMBED] Using OpenAI API for bot ' . $bot_id);
    }

    //error_log('[MXCHAT-EMBED] Using endpoint: ' . $endpoint);

    if (empty($api_key)) {
        $error_message = sprintf('Missing %s API key for bot %s. Please configure your API key in the bot settings.', $provider_name, $bot_id);
        //error_log('[MXCHAT-EMBED] Error: ' . $error_message);
        return $error_message;
    }

    // Check if text is too long (for OpenAI, roughly estimate tokens as words/0.75)
    $estimated_tokens = ceil(str_word_count($text) / 0.75);
    //error_log('[MXCHAT-EMBED] Estimated token count: ~' . $estimated_tokens);

    if ($estimated_tokens > 8000 && strpos($selected_model, 'voyage') === false && strpos($selected_model, 'gemini-embedding') === false) {
        //error_log('[MXCHAT-EMBED] Warning: Text may exceed OpenAI token limits (8K for most models)');
        // Consider truncating text here
    }

    // Prepare request body based on provider
    if (strpos($selected_model, 'gemini-embedding') === 0) {
        // Gemini API format
        $request_body = array(
            'model' => 'models/' . $selected_model,
            'content' => array(
                'parts' => array(
                    array('text' => $text)
                )
            )
        );

        // Set output dimensionality to 1536 for consistency with other models
        $request_body['outputDimensionality'] = 1536;
    } else {
        // OpenAI/Voyage API format
        $request_body = array(
            'model' => $selected_model,
            'input' => $text
        );

        // Add output_dimension for voyage-3-large model
        if ($selected_model === 'voyage-3-large') {
            $request_body['output_dimension'] = 2048;
        }
    }

    //error_log('[MXCHAT-EMBED] Request prepared with model: ' . $selected_model);

    // Prepare headers based on provider
    if (strpos($selected_model, 'gemini-embedding') === 0) {
        // Gemini uses API key as query parameter
        $endpoint .= '?key=' . $api_key;
        $headers = array(
            'Content-Type' => 'application/json'
        );
    } else {
        // OpenAI/Voyage use Bearer token
        $headers = array(
            'Authorization' => 'Bearer ' . $api_key,
            'Content-Type' => 'application/json'
        );
    }

    // Make API request
    //error_log('[MXCHAT-EMBED] Sending API request to: ' . $endpoint);
    $response = wp_remote_post($endpoint, array(
        'body' => wp_json_encode($request_body),
        'headers' => $headers,
        'timeout' => 60 // Increased timeout for large inputs
    ));

    // Handle wp_remote_post errors
    if (is_wp_error($response)) {
        $error_message = $response->get_error_message();
        //error_log('[MXCHAT-EMBED] WP Remote Post Error: ' . $error_message);
        return 'Connection error: ' . $error_message;
    }

    // Get and check HTTP response code
    $http_code = wp_remote_retrieve_response_code($response);
    //error_log('[MXCHAT-EMBED] API Response Code: ' . $http_code);

    if ($http_code !== 200) {
        $error_body = wp_remote_retrieve_body($response);
        //error_log('[MXCHAT-EMBED] API Error Response Body: ' . $error_body);

        // Try to parse error for more details
        $error_json = json_decode($error_body, true);
        if (json_last_error() === JSON_ERROR_NONE && isset($error_json['error'])) {
            $error_type = $error_json['error']['type'] ?? 'unknown';
            $error_message = $error_json['error']['message'] ?? 'No message';
            //error_log('[MXCHAT-EMBED] API Error Type: ' . $error_type);
            //error_log('[MXCHAT-EMBED] API Error Message: ' . $error_message);

            // Keep the provider's own diagnostic — a restricted-key 401 names the
            // exact missing scope, and replacing it with "check your API key" sent
            // a customer to regenerate two keys (plan 46b596). Same shape as
            // MxChat_Utils::embedding_failure_error() so both ingestion paths read
            // identically. Key never appears in provider messages, but scrub anyway.
            if ($error_type === 'invalid_request_error' || $error_type === 'authentication_error') {
                if (is_string($api_key) && $api_key !== '') {
                    $error_message = str_replace($api_key, '[redacted]', $error_message);
                }
                $error_message = sprintf(
                    'Embedding failed (%s, HTTP %d): %s',
                    $selected_model,
                    $http_code,
                    substr($error_message, 0, 300)
                );
            }

            //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
            return $error_message;
        }

        $error_message = sprintf("API Error (HTTP %d): Unable to generate embedding for bot %s", $http_code, $bot_id);
        //error_log('[MXCHAT-EMBED] Returning error: ' . $error_message);
        return $error_message;
    }

    // Parse response body
    $response_body = wp_remote_retrieve_body($response);
    //error_log('[MXCHAT-EMBED] Received response length: ' . strlen($response_body) . ' bytes');

    $response_data = json_decode($response_body, true);

    if (json_last_error() !== JSON_ERROR_NONE) {
        $error = json_last_error_msg();
        //error_log('[MXCHAT-EMBED] JSON Parse Error: ' . $error);
        //error_log('[MXCHAT-EMBED] Response preview: ' . substr($response_body, 0, 200));
        return "Failed to parse API response: $error";
    }

    // Handle different response formats based on provider
    if (strpos($selected_model, 'gemini-embedding') === 0) {
        // Gemini API response format
        if (isset($response_data['embedding']['values'])) {
            $embedding_dimensions = count($response_data['embedding']['values']);
            //error_log('[MXCHAT-EMBED] Successfully extracted Gemini embedding with ' . $embedding_dimensions . ' dimensions');

            // Check if embedding dimensions are as expected (should be 1536)
            if ($embedding_dimensions !== 1536) {
                //error_log('[MXCHAT-EMBED] Warning: Unexpected Gemini embedding dimensions: ' . $embedding_dimensions);
            }

            return $response_data['embedding']['values'];
        } else {
            //error_log('[MXCHAT-EMBED] Error: No embedding found in Gemini response');
            //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));

            if (isset($response_data['error'])) {
                $error_message = "Gemini API Error in response: " . wp_json_encode($response_data['error']);
                //error_log('[MXCHAT-EMBED] ' . $error_message);
                return $error_message;
            }

            $error_message = "Invalid Gemini API response format: No embedding found";
            //error_log('[MXCHAT-EMBED] ' . $error_message);
            return $error_message;
        }
    } else {
        // OpenAI/Voyage API response format
        if (isset($response_data['data'][0]['embedding'])) {
            $embedding_dimensions = count($response_data['data'][0]['embedding']);
            //error_log('[MXCHAT-EMBED] Successfully extracted embedding with ' . $embedding_dimensions . ' dimensions');

            // Check if embedding dimensions are as expected
            if (($selected_model === 'text-embedding-ada-002' && $embedding_dimensions !== 1536) ||
                ($selected_model === 'voyage-3-large' && $embedding_dimensions !== 2048)) {
                //error_log('[MXCHAT-EMBED] Warning: Unexpected embedding dimensions');
            }

            return $response_data['data'][0]['embedding'];
        } else {
            //error_log('[MXCHAT-EMBED] Error: No embedding found in response');
            //error_log('[MXCHAT-EMBED] Response structure: ' . wp_json_encode(array_keys($response_data)));

            if (isset($response_data['error'])) {
                $error_message = "API Error in response: " . wp_json_encode($response_data['error']);
                //error_log('[MXCHAT-EMBED] ' . $error_message);
                return $error_message;
            }

            $error_message = "Invalid API response format: No embedding found";
            //error_log('[MXCHAT-EMBED] ' . $error_message);
            return $error_message;
        }
    }
}

/**
 * Get bot-specific options for multi-bot functionality
 * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
 */
private function get_bot_options($bot_id = 'default') {
    //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
    
    if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
        //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
        return array();
    }
    
    $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
    
    if (!empty($bot_options)) {
        //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
        if (isset($bot_options['similarity_threshold'])) {
            //error_log("  - similarity_threshold: " . $bot_options['similarity_threshold']);
        }
    }
    
    return is_array($bot_options) ? $bot_options : array();
}

/**
 * Get bot-specific Pinecone configuration
 * Used in the knowledge retrieval functions
 */
// Also add debugging to your get_bot_pinecone_config function
private function get_bot_pinecone_config($bot_id = 'default') {
    //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
    
    // If default bot or multi-bot add-on not active, use default Pinecone config
    if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
        //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
        $addon_options = get_option('mxchat_pinecone_addon_options', array());
        $config = array(
            'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
            'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
            'host' => $addon_options['mxchat_pinecone_host'] ?? '',
            'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
        );
        //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
        return $config;
    }
    
    //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
    
    // Hook for multi-bot add-on to provide bot-specific Pinecone config
    $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
    
    if (!empty($bot_pinecone_config)) {
        //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
        //error_log("  - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
        //error_log("  - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
        //error_log("  - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
    } else {
        //error_log("MXCHAT DEBUG: Filter returned empty config!");
    }
    
    return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
}


public function mxchat_ajax_dismiss_completed_status() {
    try {
        // Verify the request
        check_ajax_referer('mxchat_status_nonce', 'nonce');
        
        if (!current_user_can('manage_options')) {
            wp_send_json_error('Unauthorized access');
            exit;
        }
        
        $card_type = isset($_POST['card_type']) ? sanitize_text_field($_POST['card_type']) : '';
        
        if ($card_type === 'pdf') {
            // Clear PDF status
            $pdf_url = get_transient('mxchat_last_pdf_url');
            if ($pdf_url) {
                delete_transient('mxchat_pdf_status_' . md5($pdf_url));
                delete_transient('mxchat_last_pdf_url');
            }
        } elseif ($card_type === 'sitemap') {
            // Clear sitemap status
            $sitemap_url = get_transient('mxchat_last_sitemap_url');
            if ($sitemap_url) {
                delete_transient('mxchat_sitemap_status_' . md5($sitemap_url));
                delete_transient('mxchat_last_sitemap_url');
            }
        }
        
        wp_send_json_success(array('message' => 'Status dismissed successfully'));
        
    } catch (Exception $e) {
        wp_send_json_error(array('message' => 'Error dismissing status: ' . $e->getMessage()));
    }
}

/**
 * Render completed status cards on page load
 * This ensures completed processing status persists through page refreshes
 */
public function mxchat_render_completed_status_cards() {
    $output = '';
    
    // Check for completed PDF status
    $pdf_url = get_transient('mxchat_last_pdf_url');
    if ($pdf_url) {
        $pdf_status = $this->mxchat_get_pdf_processing_status($pdf_url);
        if ($pdf_status && ($pdf_status['status'] === 'complete' || $pdf_status['status'] === 'error')) {
            $output .= $this->mxchat_render_pdf_status_card($pdf_status, $pdf_url);
        }
    }
    
    // Check for completed sitemap status
    $sitemap_url = get_transient('mxchat_last_sitemap_url');
    if ($sitemap_url) {
        $sitemap_status = $this->mxchat_get_sitemap_processing_status($sitemap_url);
        if ($sitemap_status && ($sitemap_status['status'] === 'complete' || $sitemap_status['status'] === 'error')) {
            $output .= $this->mxchat_render_sitemap_status_card($sitemap_status, $sitemap_url);
        }
    }
    
    return $output;
}

/**
 * Render PDF status card HTML
 */
private function mxchat_render_pdf_status_card($status, $pdf_url) {
    $html = '<div class="mxchat-status-card" data-card-type="pdf">';
    $html .= '<div class="mxchat-status-header">';
    $html .= '<h4>' . esc_html__('PDF Processing Status', 'mxchat') . '</h4>';
    
    // Add dismiss button for completed status
    if ($status['status'] === 'complete' || $status['status'] === 'error') {
        $html .= '<button type="button" class="mxchat-dismiss-button">' . esc_html__('Dismiss', 'mxchat') . '</button>';
    }
    
    // Process Batch button for processing status
    if ($status['status'] === 'processing') {
        $html .= '<button type="button" class="mxchat-manual-batch-btn" 
                  data-process-type="pdf"
                  data-url="' . esc_attr($pdf_url) . '">
                  ' . esc_html__('Process Batch', 'mxchat') . '</button>';
    }
    
    // Add status badges
    if ($status['status'] === 'error') {
        $html .= '<span class="mxchat-status-badge mxchat-status-failed">' . esc_html__('Error', 'mxchat') . '</span>';
    } elseif ($status['status'] === 'complete') {
        if ($status['failed_pages'] > 0) {
            $html .= '<span class="mxchat-status-badge mxchat-status-warning">' . 
                     sprintf(esc_html__('Completed with %d failures', 'mxchat'), $status['failed_pages']) . '</span>';
        } else {
            $html .= '<span class="mxchat-status-badge mxchat-status-success">' . esc_html__('Complete', 'mxchat') . '</span>';
        }
    }
    
    $html .= '</div>'; // End header
    
    // Progress bar
    $html .= '<div class="mxchat-progress-bar">';
    $html .= '<div class="mxchat-progress-fill" style="width: ' . esc_attr($status['percentage']) . '%"></div>';
    $html .= '</div>';
    
    // Status details
    $html .= '<div class="mxchat-status-details">';
    $html .= '<p>' . sprintf(
        esc_html__('Progress: %d of %d pages (%d%%)', 'mxchat'),
        $status['processed_pages'],
        $status['total_pages'],
        $status['percentage']
    ) . '</p>';
    
    // Show failed pages count if any
    if ($status['failed_pages'] > 0) {
        $html .= '<p><strong>' . esc_html__('Failed pages:', 'mxchat') . '</strong> ' . esc_html($status['failed_pages']) . '</p>';
    }
    
    $html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
    $html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
    
    // Add completion summary if available AND it's an array
    if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
        $summary = $status['completion_summary'];
        $html .= '<div class="mxchat-completion-summary">';
        $html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
        $html .= '<p><strong>' . esc_html__('Total Pages:', 'mxchat') . '</strong> ' . esc_html($summary['total_pages']) . '</p>';
        $html .= '<p><strong>' . esc_html__('Successful:', 'mxchat') . '</strong> ' . esc_html($summary['successful_pages']) . '</p>';
        $html .= '<p><strong>' . esc_html__('Failed:', 'mxchat') . '</strong> ' . esc_html($summary['failed_pages']) . '</p>';
        $html .= '<p><strong>' . esc_html__('Completed:', 'mxchat') . '</strong> ' . esc_html($summary['completion_time']) . '</p>';
        $html .= '</div>';
    }
    
    // Add failed pages list if any AND it's an array
    if (isset($status['failed_pages_list']) && is_array($status['failed_pages_list']) && !empty($status['failed_pages_list'])) {
        $html .= $this->mxchat_render_failed_pages_list($status['failed_pages_list']);
    }
    
    // Add error message if any
    if (isset($status['error']) && !empty($status['error'])) {
        $html .= '<div class="mxchat-error-notice">';
        $html .= '<p class="error">' . esc_html($status['error']) . '</p>';
        $html .= '</div>';
    }
    
    $html .= '</div>'; // End details
    $html .= '</div>'; // End card
    
    return $html;
}
/**
 * Render sitemap status card HTML
 */
private function mxchat_render_sitemap_status_card($status, $sitemap_url) {
    $html = '<div class="mxchat-status-card" data-card-type="sitemap">';
    $html .= '<div class="mxchat-status-header">';
    $html .= '<h4>' . esc_html__('Sitemap Processing Status', 'mxchat') . '</h4>';
    
    // Add dismiss button for completed status  
    if ($status['status'] === 'complete' || $status['status'] === 'error') {
        $html .= '<button type="button" class="mxchat-dismiss-button">' . esc_html__('Dismiss', 'mxchat') . '</button>';
    }
    
    // Process Batch button for processing status
    if ($status['status'] === 'processing') {
        $html .= '<button type="button" class="mxchat-manual-batch-btn" 
                  data-process-type="sitemap"
                  data-url="' . esc_attr($sitemap_url) . '">
                  ' . esc_html__('Process Batch', 'mxchat') . '</button>';
    }

    // Add status badges
    if ($status['status'] === 'error') {
        $html .= '<span class="mxchat-status-badge mxchat-status-failed">' . esc_html__('Error', 'mxchat') . '</span>';
    } elseif ($status['status'] === 'complete') {
        if ($status['failed_urls'] > 0) {
            $html .= '<span class="mxchat-status-badge mxchat-status-warning">' . 
                     sprintf(esc_html__('Completed with %d failures', 'mxchat'), $status['failed_urls']) . '</span>';
        } else {
            $html .= '<span class="mxchat-status-badge mxchat-status-success">' . esc_html__('Complete', 'mxchat') . '</span>';
        }
    }
    
    $html .= '</div>'; // End header
    
    // Progress bar
    $html .= '<div class="mxchat-progress-bar">';
    $html .= '<div class="mxchat-progress-fill" style="width: ' . esc_attr($status['percentage']) . '%"></div>';
    $html .= '</div>';
    
    // Status details
    $html .= '<div class="mxchat-status-details">';
    $html .= '<p>' . sprintf(
        esc_html__('Progress: %d of %d URLs (%d%%)', 'mxchat'),
        $status['processed_urls'],
        $status['total_urls'],
        $status['percentage']
    ) . '</p>';
    
    // Show failed URLs count if any
    if ($status['failed_urls'] > 0) {
        $html .= '<p><strong>' . esc_html__('Failed URLs:', 'mxchat') . '</strong> ' . esc_html($status['failed_urls']) . '</p>';
    }
    
    $html .= '<p><strong>' . esc_html__('Status:', 'mxchat') . '</strong> ' . esc_html(ucfirst($status['status'])) . '</p>';
    $html .= '<p><strong>' . esc_html__('Last update:', 'mxchat') . '</strong> ' . esc_html($status['last_update']) . '</p>';
    
    // Add completion summary if available AND it's an array
    if (isset($status['completion_summary']) && is_array($status['completion_summary']) && !empty($status['completion_summary'])) {
        $summary = $status['completion_summary'];
        $html .= '<div class="mxchat-completion-summary">';
        $html .= '<h5>' . esc_html__('Processing Summary', 'mxchat') . '</h5>';
        $html .= '<p><strong>' . esc_html__('Total URLs:', 'mxchat') . '</strong> ' . esc_html($summary['total_urls']) . '</p>';
        $html .= '<p><strong>' . esc_html__('Successful:', 'mxchat') . '</strong> ' . esc_html($summary['successful_urls']) . '</p>';
        $html .= '<p><strong>' . esc_html__('Failed:', 'mxchat') . '</strong> ' . esc_html($summary['failed_urls']) . '</p>';
        $html .= '<p><strong>' . esc_html__('Completed:', 'mxchat') . '</strong> ' . esc_html($summary['completion_time']) . '</p>';
        $html .= '</div>';
    }
    
    // Add error messages if any (but not the failed URLs list)
    if (!empty($status['error']) || !empty($status['last_error'])) {
        $html .= '<div class="mxchat-error-notice">';
        
        if (!empty($status['error'])) {
            $html .= '<p class="error">' . esc_html($status['error']) . '</p>';
        }
        
        if (!empty($status['last_error'])) {
            $html .= '<p class="last-error">' . esc_html__('Last error:', 'mxchat') . ' ' . esc_html($status['last_error']) . '</p>';
        }
        
        $html .= '</div>';
    }
    
    $html .= '</div>'; // End details
    $html .= '</div>'; // End card
    
    return $html;
}


/**
 * Render failed pages list
 */
private function mxchat_render_failed_pages_list($failed_pages_list) {
    // Validate that $failed_pages_list is an array and not empty
    if (!is_array($failed_pages_list) || empty($failed_pages_list)) {
        return '';
    }
    
    $html = '<div class="mxchat-error-notice">';
    $html .= '<div class="mxchat-failed-pages-container">';
    $html .= '<h5>' . sprintf(esc_html__('Failed Pages (%d)', 'mxchat'), count($failed_pages_list)) . '</h5>';
    $html .= '<details>';
    $html .= '<summary>' . esc_html__('Show Failed Pages', 'mxchat') . '</summary>';
    $html .= '<div class="mxchat-failed-pages-list">';
    
    // Create table for failed pages
    $html .= '<table class="widefat striped">';
    $html .= '<thead><tr>';
    $html .= '<th>' . esc_html__('Page', 'mxchat') . '</th>';
    $html .= '<th>' . esc_html__('Error', 'mxchat') . '</th>';
    $html .= '<th>' . esc_html__('Retries', 'mxchat') . '</th>';
    $html .= '<th>' . esc_html__('Time', 'mxchat') . '</th>';
    $html .= '</tr></thead><tbody>';
    
    // Sort failed pages by most recent
    $sorted_failed_pages = $failed_pages_list;
    usort($sorted_failed_pages, function($a, $b) {
        return ($b['time'] ?? 0) - ($a['time'] ?? 0);
    });
    
    foreach ($sorted_failed_pages as $item) {
        // Ensure $item is an array before accessing its elements
        if (!is_array($item)) {
            continue;
        }
        
        $time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
        $html .= '<tr>';
        $html .= '<td>' . esc_html__('Page', 'mxchat') . ' ' . esc_html($item['page'] ?? 'Unknown') . '</td>';
        $html .= '<td style="word-break: break-word;">' . esc_html($item['error'] ?? 'Unknown error') . '</td>';
        $html .= '<td>' . esc_html($item['retries'] ?? 'N/A') . '</td>';
        $html .= '<td>' . esc_html($time_ago) . '</td>';
        $html .= '</tr>';
    }
    
    $html .= '</tbody></table>';
    $html .= '</div></details></div></div>';
    
    return $html;
}

/**
 * Render failed URLs list
 */
private function mxchat_render_failed_urls_list($failed_urls_list) {
    // Validate that $failed_urls_list is an array and not empty
    if (!is_array($failed_urls_list) || empty($failed_urls_list)) {
        return '';
    }
    
    $html = '<div class="mxchat-failed-urls-container">';
    $html .= '<h5>' . sprintf(esc_html__('Failed URLs (%d)', 'mxchat'), count($failed_urls_list)) . '</h5>';
    $html .= '<details>';
    $html .= '<summary>' . esc_html__('Show Failed URLs', 'mxchat') . '</summary>';
    $html .= '<div class="mxchat-failed-urls-list">';
    
    // Create table for failed URLs
    $html .= '<table class="widefat striped">';
    $html .= '<thead><tr>';
    $html .= '<th>' . esc_html__('URL', 'mxchat') . '</th>';
    $html .= '<th>' . esc_html__('Error', 'mxchat') . '</th>';
    $html .= '<th>' . esc_html__('Retries', 'mxchat') . '</th>';
    $html .= '<th>' . esc_html__('Time', 'mxchat') . '</th>';
    $html .= '</tr></thead><tbody>';
    
    // Sort failed URLs by most recent
    $sorted_failed_urls = $failed_urls_list;
    usort($sorted_failed_urls, function($a, $b) {
        return ($b['time'] ?? 0) - ($a['time'] ?? 0);
    });
    
    // Show up to 50 failed URLs
    $display_urls = array_slice($sorted_failed_urls, 0, 50);
    
    foreach ($display_urls as $item) {
        // Ensure $item is an array before accessing its elements
        if (!is_array($item)) {
            continue;
        }
        
        $url = $item['url'] ?? '';
        $time_ago = isset($item['time']) ? human_time_diff($item['time'], current_time('timestamp')) . ' ' . esc_html__('ago', 'mxchat') : 'Unknown';
        
        // Truncate URL for display
        $display_url = strlen($url) > 50 ? substr($url, 0, 47) . '...' : $url;
        
        $html .= '<tr>';
        $html .= '<td style="word-break: break-all;">';
        if (!empty($url)) {
            $html .= '<a href="' . esc_url($url) . '" target="_blank" rel="noopener noreferrer">' . esc_html($display_url) . '</a>';
        } else {
            $html .= esc_html__('Unknown URL', 'mxchat');
        }
        $html .= '</td>';
        $html .= '<td style="word-break: break-word;">' . esc_html($item['error'] ?? 'Unknown error') . '</td>';
        $html .= '<td>' . esc_html($item['retries'] ?? 'N/A') . '</td>';
        $html .= '<td>' . esc_html($time_ago) . '</td>';
        $html .= '</tr>';
    }
    
    $html .= '</tbody></table>';
    
    if (count($failed_urls_list) > 50) {
        $html .= '<div class="mxchat-failed-urls-more">+ ' . 
                 (count($failed_urls_list) - 50) . 
                 ' ' . esc_html__('more failed URLs not shown', 'mxchat') . '</div>';
    }
    
    $html .= '</div></details></div>';
    
    return $html;
}

/**
 * Get all ACF fields for a specific post, excluding any fields the user has disabled
 */
public function mxchat_get_acf_fields_for_post($post_id) {
    if (!function_exists('get_field_objects')) {
        return array();
    }

    // Field OBJECTS, not get_fields(): exclusion matches on the field KEY
    // (unique per field) rather than the name (shared across groups — plan
    // 30e81f). ACF's own get_fields() is implemented as get_field_objects()
    // reduced to name => value, so the un-excluded reduction below is the
    // identical shape and order the previous get_fields() call produced.
    $field_objects = get_field_objects($post_id);
    if (!$field_objects || !is_array($field_objects)) {
        return array();
    }

    $excluded_fields = get_option('mxchat_acf_excluded_fields', array());
    if (!is_array($excluded_fields)) {
        $excluded_fields = array();
    }

    $fields = array();
    foreach ($field_objects as $field_name => $field_object) {
        if (!empty($excluded_fields)) {
            $field_key = isset($field_object['key']) ? $field_object['key'] : '';
            // Legacy name entries stay honored: a stored name whose group was
            // inactive at migration time still excludes every field wearing it.
            if (in_array($field_key, $excluded_fields, true) || in_array($field_name, $excluded_fields, true)) {
                continue;
            }
        }
        $fields[$field_name] = isset($field_object['value']) ? $field_object['value'] : null;
    }

    return $fields;
}

/**
 * Get all registered ACF field groups and their fields for the settings UI
 */
public function mxchat_get_all_acf_fields() {
    if (!function_exists('acf_get_field_groups') || !function_exists('acf_get_fields')) {
        return array();
    }

    // Keyed by GROUP KEY, not title (titles are not unique), and each field
    // entry carries its ACF field key — the unique identifier every toggle,
    // save, and index-time exclusion now runs on (plans 30e81f / bf57e0).
    $all_fields = array();
    $field_groups = acf_get_field_groups();

    if (!empty($field_groups)) {
        foreach ($field_groups as $group) {
            $group_fields = acf_get_fields($group['key']);
            if (!empty($group_fields)) {
                $entry = array(
                    'title'  => $group['title'],
                    'fields' => array(),
                );
                foreach ($group_fields as $field) {
                    $entry['fields'][] = array(
                        'key'   => $field['key'],
                        'name'  => $field['name'],
                        'label' => $field['label'],
                        'type'  => $field['type']
                    );
                }
                $all_fields[$group['key']] = $entry;
            }
        }
    }

    return $all_fields;
}

/**
 * Get whitelisted custom post meta for a given post
 * This allows non-ACF meta fields (like OptionTree, theme meta boxes) to be included in embeddings
 */
public function mxchat_get_whitelisted_post_meta($post_id) {
    $whitelist = get_option('mxchat_custom_meta_whitelist', '');

    if (empty($whitelist)) {
        return array();
    }

    // Parse the whitelist - one meta key per line
    $meta_keys = array_filter(array_map('trim', explode("\n", $whitelist)));

    if (empty($meta_keys)) {
        return array();
    }

    $result = array();

    foreach ($meta_keys as $key) {
        // Skip empty keys
        if (empty($key)) {
            continue;
        }

        $value = get_post_meta($post_id, $key, true);

        // Only include non-empty string values
        if (!empty($value) && is_string($value)) {
            $result[$key] = $value;
        } elseif (!empty($value) && is_array($value)) {
            // Handle array values by joining them
            $flat_value = $this->mxchat_flatten_meta_array($value);
            if (!empty($flat_value)) {
                $result[$key] = $flat_value;
            }
        }
    }

    return $result;
}

/**
 * Flatten array meta values into a readable string
 */
private function mxchat_flatten_meta_array($array, $depth = 0) {
    if ($depth > 3) {
        return ''; // Prevent infinite recursion
    }

    $parts = array();

    foreach ($array as $key => $value) {
        if (is_string($value) && !empty($value)) {
            $parts[] = $value;
        } elseif (is_array($value)) {
            $nested = $this->mxchat_flatten_meta_array($value, $depth + 1);
            if (!empty($nested)) {
                $parts[] = $nested;
            }
        }
    }

    return implode(', ', $parts);
}

/**
 * Format ACF field values for content extraction
 */
public function mxchat_format_acf_field_value($value, $field_name = '', $post_id = 0) {
    if (empty($value)) {
        return '';
    }
    
    // Handle WP_Post objects first (THIS IS THE KEY FIX)
    if ($value instanceof WP_Post) {
        return $value->post_title ?: '';
    }
    
    // Handle other WP objects
    if (is_object($value)) {
        if (isset($value->post_title)) {
            return $value->post_title;
        } elseif (isset($value->display_name)) {
            return $value->display_name;
        } elseif (isset($value->name)) {
            return $value->name;
        } elseif (method_exists($value, '__toString')) {
            try {
                return (string) $value;
            } catch (Exception $e) {
                return '';
            }
        }
        // For any other objects, return empty string
        return '';
    }
    
    // Handle different ACF field types
    if (is_array($value)) {
        // Check if it's an image/file field
        if (isset($value['url'])) {
            // Image field - return alt text, title, or caption
            if (!empty($value['alt'])) {
                return $value['alt'];
            } elseif (!empty($value['title'])) {
                return $value['title'];
            } elseif (!empty($value['caption'])) {
                return $value['caption'];
            } else {
                return ''; // Don't include just the URL
            }
        }
        
        // Check if it's a post object or relationship field
        if (isset($value['post_title'])) {
            return $value['post_title'];
        }
        
        // Check if it's a user field
        if (isset($value['display_name'])) {
            return $value['display_name'];
        }
        
        // Check if it's a taxonomy term
        if (isset($value['name']) && isset($value['taxonomy'])) {
            return $value['name'];
        }
        
        // Check if it's a select field with label
        if (isset($value['label'])) {
            return $value['label'];
        }
        
        // Check for repeater field or flexible content
        if (is_numeric(key($value))) {
            $sub_values = array();
            foreach ($value as $sub_item) {
                if (is_array($sub_item)) {
                    // For repeater/flexible content, extract text values
                    $sub_text = $this->mxchat_extract_text_from_acf_array($sub_item);
                    if (!empty($sub_text)) {
                        $sub_values[] = $sub_text;
                    }
                } elseif ($sub_item instanceof WP_Post) {
                    // Handle WP_Post objects in arrays
                    $sub_values[] = $sub_item->post_title ?: '';
                } else {
                    $sub_values[] = (string) $sub_item;
                }
            }
            return implode(', ', array_filter($sub_values));
        }
        
        // For other arrays, try to extract meaningful text
        $text_values = array();
        foreach ($value as $key => $val) {
            if (is_string($val) && !empty(trim($val))) {
                $text_values[] = trim($val);
            } elseif ($val instanceof WP_Post) {
                // Handle WP_Post objects in associative arrays
                $text_values[] = $val->post_title ?: '';
            } elseif (is_array($val) && isset($val['post_title'])) {
                $text_values[] = $val['post_title'];
            } elseif (is_array($val) && isset($val['name'])) {
                $text_values[] = $val['name'];
            }
        }
        
        return implode(', ', array_filter($text_values));
    }
    
    // Handle boolean values
    if (is_bool($value)) {
        return $value ? 'Yes' : 'No';
    }
    
    // Handle numeric values
    if (is_numeric($value)) {
        return (string) $value;
    }
    
    // Handle string values
    if (is_string($value)) {
        return trim($value);
    }
    
    // For anything else that we can't handle, return empty string
    // This prevents the "Object could not be converted to string" error
    return '';
}

/**
 * Extract text from complex ACF array structures
 */
private function mxchat_extract_text_from_acf_array($array) {
    if (!is_array($array)) {
        return '';
    }
    
    $text_parts = array();
    
    foreach ($array as $key => $value) {
        if (is_string($value) && !empty(trim($value))) {
            // Skip keys that are likely to be IDs or technical values
            if (!is_numeric($value) || strlen($value) > 10) {
                $text_parts[] = trim($value);
            }
        } elseif ($value instanceof WP_Post) {
            // Handle WP_Post objects
            $text_parts[] = $value->post_title ?: '';
        } elseif (is_array($value)) {
            if (isset($value['post_title'])) {
                $text_parts[] = $value['post_title'];
            } elseif (isset($value['name'])) {
                $text_parts[] = $value['name'];
            } elseif (isset($value['label'])) {
                $text_parts[] = $value['label'];
            }
        } elseif (is_object($value)) {
            // Handle other objects safely
            if (isset($value->post_title)) {
                $text_parts[] = $value->post_title;
            } elseif (isset($value->name)) {
                $text_parts[] = $value->name;
            } elseif (isset($value->display_name)) {
                $text_parts[] = $value->display_name;
            }
        }
    }
    
    return implode(', ', array_filter($text_parts));
}

/**
 * Walk an ACF field value tree and collect attachment IDs for any value that
 * resolves to a PDF in the WordPress media library. Handles the three shapes
 * ACF returns for File/Image/URL fields (array with ID+url, integer attachment ID,
 * plain URL string), and recurses through repeater/group/flexible content.
 *
 * @param mixed $value      The ACF field value (any depth)
 * @param array $out        Accumulator (passed by reference) for attachment IDs
 * @param int   $depth      Recursion guard
 */
private function mxchat_collect_pdf_attachment_ids_from_acf_value($value, &$out, $depth = 0) {
    if ($depth > 6) {
        return; // prevent runaway recursion on circular/very-deep structures
    }

    if (empty($value)) {
        return;
    }

    // Array shapes: ACF File/Image return value=array; repeaters/groups are arrays of arrays
    if (is_array($value)) {
        // Direct File/Image-style array (has 'url' and usually 'ID' + 'mime_type')
        $looks_like_attachment = isset($value['url']) || isset($value['ID']) || isset($value['id']);
        if ($looks_like_attachment) {
            $att_id = 0;
            if (!empty($value['ID']) && is_numeric($value['ID'])) {
                $att_id = (int) $value['ID'];
            } elseif (!empty($value['id']) && is_numeric($value['id'])) {
                $att_id = (int) $value['id'];
            } elseif (!empty($value['url']) && is_string($value['url'])) {
                $att_id = (int) attachment_url_to_postid($value['url']);
            }

            $is_pdf = false;
            if (!empty($value['mime_type']) && $value['mime_type'] === 'application/pdf') {
                $is_pdf = true;
            } elseif (!empty($value['subtype']) && strtolower((string) $value['subtype']) === 'pdf') {
                $is_pdf = true;
            } elseif (!empty($value['url']) && is_string($value['url']) && $this->mxchat_url_looks_like_pdf($value['url'])) {
                $is_pdf = true;
            } elseif ($att_id && get_post_mime_type($att_id) === 'application/pdf') {
                $is_pdf = true;
            }

            if ($is_pdf && $att_id && get_post_mime_type($att_id) === 'application/pdf') {
                $out[] = $att_id;
            }
            // An array node that represents one attachment doesn't contain other
            // attachments inside it — done with this branch.
            return;
        }

        // Recurse: repeater rows, flexible-content layouts, groups, etc.
        foreach ($value as $sub) {
            $this->mxchat_collect_pdf_attachment_ids_from_acf_value($sub, $out, $depth + 1);
        }
        return;
    }

    // Plain numeric attachment ID (ACF File field set to "Return: ID")
    if (is_numeric($value)) {
        $att_id = (int) $value;
        if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
            $out[] = $att_id;
        }
        return;
    }

    // Plain string — URL pointing at a PDF (ACF File field set to "Return: URL", or a custom URL/text field)
    if (is_string($value)) {
        $trimmed = trim($value);
        if ($trimmed !== '' && $this->mxchat_url_looks_like_pdf($trimmed)) {
            $att_id = (int) attachment_url_to_postid($trimmed);
            if ($att_id > 0 && get_post_mime_type($att_id) === 'application/pdf') {
                $out[] = $att_id;
            }
        }
        return;
    }
}

/**
 * Heuristic: does this URL/string look like a PDF reference?
 * Tolerates query strings and fragments (#page=2).
 */
private function mxchat_url_looks_like_pdf($url) {
    if (!is_string($url) || $url === '') {
        return false;
    }
    // Strip query + fragment before checking extension
    $path = preg_replace('/[?#].*$/', '', $url);
    return (bool) preg_match('/\.pdf$/i', $path);
}

/**
 * Extract text from a PDF attachment by ID using the bundled Smalot parser.
 * Reads the file directly from disk via get_attached_file (no HTTP fetch).
 * Result is cached on the attachment as post_meta keyed by file mtime so we
 * only parse the same PDF once unless the file changes on disk.
 *
 * @param int $attachment_id
 * @return string Extracted plain text, or '' on failure.
 */
private function mxchat_extract_pdf_text_by_attachment_id($attachment_id) {
    $attachment_id = (int) $attachment_id;
    if ($attachment_id <= 0) {
        return '';
    }
    if (get_post_mime_type($attachment_id) !== 'application/pdf') {
        return '';
    }

    $pdf_path = get_attached_file($attachment_id);
    if (empty($pdf_path) || !file_exists($pdf_path) || !is_readable($pdf_path)) {
        return '';
    }

    // Raw-file size cap. Parsing very large PDFs can OOM the request; skip with a log entry
    // and let the rest of the ACF content land in the KB. Filterable for users who need it bigger.
    $default_max_bytes = 25 * 1024 * 1024;
    $max_bytes = (int) apply_filters('mxchat_acf_pdf_max_bytes', $default_max_bytes, $attachment_id, $pdf_path);
    if ($max_bytes > 0) {
        $file_size = @filesize($pdf_path);
        if ($file_size !== false && $file_size > $max_bytes) {
            error_log(sprintf(
                '[mxchat] ACF PDF skipped (over size cap): attachment %d "%s" %d bytes > cap %d',
                $attachment_id,
                basename($pdf_path),
                $file_size,
                $max_bytes
            ));
            return '';
        }
    }

    $mtime = @filemtime($pdf_path);
    $cache_meta_key = '_mxchat_acf_pdf_text_v1';
    $cached = get_post_meta($attachment_id, $cache_meta_key, true);
    if (is_array($cached) && isset($cached['mtime'], $cached['text']) && (int) $cached['mtime'] === (int) $mtime) {
        return (string) $cached['text'];
    }

    $text = '';
    try {
        if (function_exists('mxchat_load_pdf_parser')) {
            mxchat_load_pdf_parser();
        }
        if (!class_exists('\\Smalot\\PdfParser\\Parser')) {
            return '';
        }
        $parser = new \Smalot\PdfParser\Parser();
        $pdf = $parser->parseFile($pdf_path);
        $pages = $pdf->getPages();
        $page_texts = array();
        $acf_page_num = 0;
        foreach ($pages as $page) {
            $acf_page_num++;
            $page_text = '';
            try {
                $page_text = $page->getText();
            } catch (\Exception $e) {
                $page_text = '';
            }
            if (!empty($page_text)) {
                $page_text = MxChat_Utils::normalize_pdf_rtl($page_text, 'acf_pdf attachment ' . $attachment_id . ' page ' . $acf_page_num);
                $page_texts[] = $page_text;
            }
        }
        $text = trim(implode("\n\n", $page_texts));
    } catch (\Exception $e) {
        error_log('[mxchat] ACF PDF extraction failed for attachment ' . $attachment_id . ': ' . $e->getMessage());
        return '';
    } catch (\Throwable $e) {
        error_log('[mxchat] ACF PDF extraction error for attachment ' . $attachment_id . ': ' . $e->getMessage());
        return '';
    }

    // Cap per-PDF text to avoid blowing up the embedding payload on enormous PDFs.
    // The chunker downstream will still split this into multiple vectors.
    $max_len = (int) apply_filters('mxchat_acf_pdf_text_max_length', 50000);
    if ($max_len > 0 && strlen($text) > $max_len) {
        $text = substr($text, 0, $max_len);
    }

    update_post_meta($attachment_id, $cache_meta_key, array(
        'mtime' => (int) $mtime,
        'text'  => $text,
    ));

    return $text;
}

/**
 * Handle ACF save - fires after ACF fields are saved
 * This ensures ACF field data is available when syncing to knowledge base
 */
public function mxchat_handle_acf_save($post_id) {
    // Skip if not a valid post
    if (!$post_id || $post_id === 'options') {
        return;
    }

    // Skip autosaves and revisions
    if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
        return;
    }

    $post = get_post($post_id);
    if (!$post) {
        return;
    }

    $post_type = $post->post_type;

    // Check if sync is enabled for this post type
    $should_sync = false;

    if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
        $should_sync = true;
    } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
        $should_sync = true;
    } else if ($post_type === 'product' && class_exists('WooCommerce')) {
        // WooCommerce products - check if WooCommerce integration is enabled
        $options = get_option('mxchat_options', array());
        if (isset($options['enable_woocommerce_integration']) &&
            ($options['enable_woocommerce_integration'] === '1' || $options['enable_woocommerce_integration'] === 'on')) {
            $should_sync = true;
        }
    } else {
        // Check custom post types
        $option_name = 'mxchat_auto_sync_' . $post_type;
        if (get_option($option_name) === '1') {
            $should_sync = true;
        }
    }

    if (!$should_sync) {
        return;
    }

    // Only process published posts
    if ($post->post_status !== 'publish') {
        return;
    }

    // Check if this post has any ACF fields - if not, no need to re-sync
    $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
    if (empty($acf_fields)) {
        return;
    }

    // Use a transient to prevent duplicate processing (post_updated may have already run)
    $transient_key = 'mxchat_acf_synced_' . $post_id;
    if (get_transient($transient_key)) {
        return;
    }
    set_transient($transient_key, true, 60); // Prevent re-processing for 60 seconds

    // Re-run the sync with ACF data now available
    // We pass $update=true since this is effectively an update with ACF data
    $this->mxchat_handle_post_update($post_id, $post, true);
}

public function mxchat_handle_post_update($post_id, $post, $update) {
    // The in-flight-update marker has done its job the moment post_updated runs; drop it
    // before any early return so it can never outlive its own save (a failed $wpdb->update
    // inside wp_insert_post returns after pre_post_update but before the transition).
    unset($this->pending_post_update[$post_id]);

    // Basic validation checks
    if (defined('DOING_AUTOSAVE') && DOING_AUTOSAVE || wp_is_post_revision($post_id)) {
        return;
    }
    
    $post_type = $post->post_type;
    
    // Check if sync is enabled for this post type
    $should_sync = false;
    
    // Check built-in post types first
    if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
        $should_sync = true;
    } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
        $should_sync = true;
    } else {
        // Check custom post types
        $option_name = 'mxchat_auto_sync_' . $post_type;
        if (get_option($option_name) === '1') {
            $should_sync = true;
        }
    }
    
    if (!$should_sync) {
        return;
    }
    
    // Check if we have stored the previous status and URL in our transients
    $previous_status_key = 'mxchat_prev_status_' . $post_id;
    $previous_status = get_transient($previous_status_key);
    
    $previous_url_key = 'mxchat_prev_url_' . $post_id;
    $previous_url = get_transient($previous_url_key);
    
    // If the post was previously published but is now not published, remove from knowledge base
    if ($previous_status === 'publish' && $post->post_status !== 'publish') {
        // Use the stored URL from when it was published, or fall back to current permalink
        $source_url = $previous_url ?: get_permalink($post_id);

        // mxchat_handle_status_transition already deleted for this post earlier in this
        // request (it fires first inside wp_insert_post); skip the redundant round-trip.
        if ($source_url && empty($this->transition_deleted_posts[$post_id])) {
            // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
            MxChat_Utils::delete_chunks_for_url($source_url, 'default');
        }

        // Clean up the transients and exit early
        delete_transient($previous_status_key);
        delete_transient($previous_url_key);
        return;
    }

    // Slug/permalink rename while still published: delete the old vectors before upserting new ones.
    // Without this, md5(old_url) vectors (base + chunks) would be orphaned under the stale URL.
    if ($post->post_status === 'publish' && !empty($previous_url)) {
        $current_url = get_permalink($post_id);
        if ($current_url && $current_url !== $previous_url) {
            MxChat_Utils::delete_chunks_for_url($previous_url, 'default');
        }
    }

    // Store the current status for next time (if this is an update)
    if ($update) {
        set_transient($previous_status_key, $post->post_status, DAY_IN_SECONDS);

        // If the post is currently published, also store its URL
        if ($post->post_status === 'publish') {
            $current_url = get_permalink($post_id);
            set_transient($previous_url_key, $current_url, DAY_IN_SECONDS);
        }
    }
    
    // Only process currently published content for adding/updating.
    // transition_indexed_posts: mxchat_handle_status_transition's arrival edge may have
    // already indexed this post earlier in this request (editor publishes fire
    // transition_post_status first, then post_updated) — skip the duplicate embed.
    // Consume-once: the flag is cleared when honoured, so a LATER save of the same
    // post in one long-running process (WP-CLI scripts, importers) re-indexes normally.
    if ($post->post_status === 'publish') {
        if (!empty($this->transition_indexed_posts[$post_id])) {
            unset($this->transition_indexed_posts[$post_id]);
        } else {
            $this->mxchat_index_published_post($post_id, $post);
        }
    }

    // Clean up the stored previous status if not used above
    if ($previous_status !== 'publish' || $post->post_status === 'publish') {
        delete_transient($previous_status_key);
        delete_transient($previous_url_key);
    }
}

/**
 * Index a published post into the knowledge base: preprocessing filter, content
 * assembly (title/excerpt/body), WooCommerce product enrichment, job_listing meta,
 * ACF fields (+ optional PDF extraction), whitelisted custom meta, embedding and
 * upsert, then tag-based role restriction.
 *
 * Shared by the post_updated auto-sync path (mxchat_handle_post_update) and the
 * transition_post_status arrival edge (mxchat_handle_status_transition), so
 * scheduled publishes (wp_publish_post) and direct status=publish inserts index
 * identically to editor saves (plan 3055e1). Pure extraction of the former
 * publish branch — body indentation retained to keep the diff reviewable.
 */
private function mxchat_index_published_post($post_id, $post) {
        $post_type = $post->post_type;

        // WooCommerce products are owned by the WC-object assembler (plan a3d60c):
        // whenever WooCommerce is active AND the integration is enabled, every
        // product save also fires save_post_product, which queues
        // mxchat_store_product_embedding on shutdown — and that writer runs LAST,
        // overwriting the same md5(permalink) row this path would write. Assembling
        // and embedding the product here was pure duplicate spend (measured: two
        // embedding calls per product save, second one wins). Skip ONLY under the
        // exact conditions the shutdown writer runs — same option read as its own
        // gate — because with the integration off (or WooCommerce inactive) this
        // path is the sole product indexer and must keep working.
        if ($post_type === 'product'
            && class_exists('WooCommerce')
            && isset($this->options['enable_woocommerce_integration'])
            && in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
            return;
        }

        // Get the source URL
        $source_url = get_permalink($post_id);

        // A draft published programmatically (wp_publish_post) can reach this
        // point with an EMPTY post_name — wp_insert_post skips slug generation
        // for draft/pending — and get_permalink() then resolves to the bare
        // site root. A knowledge row keyed to the homepage cites the wrong URL
        // and answers homepage questions with this post's body, so refuse to
        // write it; the post indexes correctly on its next save, once the slug
        // exists. The empty-post_name test is what keeps a legitimate static
        // front page (which has a slug but a root permalink) indexable.
        // (Plan d138c4.)
        if ('' === $post->post_name
            && untrailingslashit($source_url) === untrailingslashit(home_url())) {
            return;
        }

        /**
         * Allow developers to modify post data before processing into the knowledge base.
         * Same filter and signature as the manual bulk-import path
         * (ajax_mxchat_process_selected_content), so a callback registered once covers
         * every indexing route. Purely additive — zero behaviour change when unhooked.
         * Auto-sync runs under the 'default' bot context, matching the rest of this
         * function.
         *
         * @param WP_Post $post   The post about to be indexed.
         * @param string  $bot_id Bot context ('default' on auto-sync).
         */
        $post = apply_filters('mxchat_before_process_post', $post, 'default');
        if (!($post instanceof WP_Post)) {
            $post = get_post($post_id); // defend against a bad callback return
        }

        // Assemble the indexable text via the shared post-kind assembler (a3d60c),
        // reading from the FILTERED post object — not re-fetched by ID, which would
        // discard it. Auto-sync reads content/excerpt in its historical
        // get_post_field() display context, never appended product custom tabs
        // (its product branch is reachable only with the WooCommerce integration
        // off), and gates ACF→PDF extraction behind its own opt-in option —
        // default OFF, because re-parsing every ACF PDF on every editor save is
        // expensive and most sites don't want it (the 25 MB size cap lives in the
        // shared extractor either way).
        $prepared = $this->mxchat_prepare_post_content_for_indexing($post_id, $post, array(
            'read_display'         => true,
            'extract_acf_pdfs'     => get_option('mxchat_auto_sync_acf_pdfs', '0') === '1',
            'include_product_tabs' => false,
        ));
        $final_content = $prepared['content'];

        // Embedding decision — custom-provider-aware. Gating on a cloud API key
        // here silently killed auto-sync on keyless custom-embeddings sites,
        // because generate_embedding() routes custom FIRST and never needs the
        // key (plan cbd5fd). Silent-return shape preserved.
        $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
        if (!$preflight['ok']) {
            return;
        }
        $api_key = $preflight['api_key'];

        // Use the centralized utility function for storage
        $result = MxChat_Utils::submit_content_to_db(
            $final_content,
            $source_url,
            $api_key,
            md5($source_url) // Vector ID for Pinecone
        );
        
        //   After successful storage, apply role restriction based on tags
        if (!is_wp_error($result)) {
            $this->apply_role_restriction_to_post($post_id, $source_url);
        }
}

/**
 * Shared post-fields content assembler (plan a3d60c) — the ONE body behind both
 * post-kind ingestion paths: manual bulk import (ajax_mxchat_process_selected_content)
 * and auto-sync (mxchat_index_published_post). Behavior-preserving extraction; the
 * measured per-caller differences ride $args instead of living as drifting copies:
 *
 *   'read_display'         bool  Auto-sync historically reads content/excerpt via
 *                                get_post_field() in its default 'display' context
 *                                (the post_content / post_excerpt display filters
 *                                fire); bulk import reads the raw properties. Inert
 *                                on a stock install — preserved per-path, not converged.
 *   'extract_acf_pdfs'     bool  Each caller passes its OWN option (bulk:
 *                                mxchat_acf_pdf_extraction; auto-sync:
 *                                mxchat_auto_sync_acf_pdfs) — the two-option design
 *                                is deliberate (plan 11720c). Gates BOTH the PDF-id
 *                                collection walk and the extraction loop; the ids are
 *                                only ever read inside the extraction branch, so
 *                                gating collection is output-identical on every install.
 *   'include_product_tabs' bool  The bulk path has always appended yikes_woo custom
 *                                tabs to product content; the auto-sync product branch
 *                                (reachable only with the WooCommerce integration off)
 *                                never did. Preserved per-path — converging it would be
 *                                a behavior change, recorded on the plan instead.
 *
 * Returns array: 'content' (the assembled indexable text), 'acf_fields_found' and
 * 'pdf_extracted_count' (the bulk path reports both in its AJAX response).
 */
private function mxchat_prepare_post_content_for_indexing($post_id, $post, $args) {
    $read_display         = !empty($args['read_display']);
    $extract_acf_pdfs     = !empty($args['extract_acf_pdfs']);
    $include_product_tabs = !empty($args['include_product_tabs']);

    // Raw post_title, NOT get_the_title(): the_title applies wptexturize +
    // convert_chars and prepends the "Protected:" / "Private:" display chrome.
    // The knowledge base stores facts, not display strings. Entity decode at
    // output time (single-pass, shared helper) — a stored `&amp;` embeds worse
    // than `&` and gets quoted back to visitors (d2c92e).
    $content = $this->mxchat_decode_entities_for_indexing($post->post_title) . "\n\n";

    $raw_excerpt = $read_display ? get_post_field('post_excerpt', $post) : $post->post_excerpt;
    $raw_content = $read_display ? get_post_field('post_content', $post) : $post->post_content;

    // Add short description if it exists (WooCommerce products use post_excerpt for short description)
    // Strip FIRST, then test: an excerpt that is nothing but shortcodes strips to
    // empty, and testing the raw value emitted a bare "Short Description: " label
    // with no value after it. trim() only in the TEST — the emitted value is
    // untouched, so a populated excerpt is byte-identical to before. A
    // whitespace-only excerpt is an empty excerpt and must not produce a labelled
    // line with nothing after it.
    $clean_excerpt = $this->strip_shortcode_tags_preserve_content($raw_excerpt);
    if (trim($clean_excerpt) !== '') {
        $content .= "Short Description: " . $this->mxchat_decode_entities_for_indexing(wp_strip_all_tags($clean_excerpt)) . "\n\n";
    }

    // Main content — remove shortcode tags but preserve content inside them, then
    // strip tags (don't use 'the_content' filter as it may re-add shortcodes).
    $clean_content = $this->strip_shortcode_tags_preserve_content($raw_content);
    $content .= $this->mxchat_decode_entities_for_indexing(wp_strip_all_tags($clean_content));

    // WooCommerce product enrichment (post-fields kind). The WC-object assembler
    // (mxchat_prepare_product_content_for_indexing) owns product rows whenever the
    // integration is on; this branch serves the bulk import (all configurations)
    // and auto-sync with the integration off.
    if (get_post_type($post_id) === 'product' && class_exists('WooCommerce')) {
        $product = wc_get_product($post_id);

        if ($product) {
            $content .= "\n";
            $content .= $this->mxchat_woo_product_summary_lines($product);
        }

        if ($include_product_tabs) {
            $content .= $this->mxchat_woo_custom_tabs_text($post_id);
        }
    }

    // For custom post types like job_listing, include additional fields
    if (get_post_type($post_id) === 'job_listing') {
        // Add job-specific meta if available
        $job_location = get_post_meta($post_id, '_job_location', true);
        if (!empty($job_location)) {
            $content .= "\n\nLocation: " . $job_location;
        }

        // Get job type terms
        $job_types = get_the_terms($post_id, 'job_listing_type');
        if (!empty($job_types) && !is_wp_error($job_types)) {
            $types = array();
            foreach ($job_types as $type) {
                $types[] = $type->name;
            }
            $content .= "\n\nJob Type: " . implode(', ', $types);
        }

        // Get company name if available
        $company_name = get_post_meta($post_id, '_company_name', true);
        if (!empty($company_name)) {
            $content .= "\n\nCompany: " . $company_name;
        }
    }

    // ADD ACF FIELDS SUPPORT
    $acf_fields = $this->mxchat_get_acf_fields_for_post($post_id);
    $pdf_extracted_count = 0;
    if (!empty($acf_fields)) {
        $acf_content_parts = array();
        $pdf_attachment_ids = array();

        foreach ($acf_fields as $field_name => $field_value) {
            $formatted_value = $this->mxchat_format_acf_field_value($field_value, $field_name, $post_id);

            if (!empty($formatted_value)) {
                // Both separators: a hyphenated ACF name should read as words.
                $field_label = ucwords(str_replace(['_', '-'], ' ', $field_name));
                $acf_content_parts[] = $field_label . ": " . $formatted_value;
            }

            // Walk this field's value tree for any PDF attachment references and
            // queue them for extraction — only when this caller's PDF option is on.
            if ($extract_acf_pdfs) {
                $this->mxchat_collect_pdf_attachment_ids_from_acf_value($field_value, $pdf_attachment_ids);
            }
        }

        if (!empty($acf_content_parts)) {
            $content .= "\n\n" . implode("\n", $acf_content_parts);
        }

        // Extract text from each unique PDF found in ACF fields and append as a labeled section
        if ($extract_acf_pdfs && !empty($pdf_attachment_ids)) {
            $pdf_attachment_ids = array_unique(array_filter(array_map('intval', $pdf_attachment_ids)));
            $pdf_sections = array();
            foreach ($pdf_attachment_ids as $att_id) {
                $pdf_text = $this->mxchat_extract_pdf_text_by_attachment_id($att_id);
                if (!empty($pdf_text)) {
                    $pdf_title = get_the_title($att_id);
                    $pdf_url = wp_get_attachment_url($att_id);
                    $header = 'PDF Attachment';
                    if (!empty($pdf_title)) {
                        $header .= ': ' . $pdf_title;
                    }
                    if (!empty($pdf_url)) {
                        $header .= ' (' . $pdf_url . ')';
                    }
                    $pdf_sections[] = $header . "\n" . $pdf_text;
                    $pdf_extracted_count++;
                }
            }
            if (!empty($pdf_sections)) {
                $content .= "\n\nAttached PDFs:\n" . implode("\n\n", $pdf_sections);
            }
        }
    }

    // ADD CUSTOM POST META SUPPORT (whitelisted non-ACF meta fields)
    $custom_meta = $this->mxchat_get_whitelisted_post_meta($post_id);
    if (!empty($custom_meta)) {
        $meta_content_parts = array();

        foreach ($custom_meta as $meta_key => $meta_value) {
            // Convert meta key to readable label
            $meta_label = ucwords(str_replace(array('_', '-'), ' ', $meta_key));
            $meta_content_parts[] = $meta_label . ": " . $meta_value;
        }

        if (!empty($meta_content_parts)) {
            $content .= "\n\n" . implode("\n", $meta_content_parts);
        }
    }

    return array(
        'content'             => $content,
        'acf_fields_found'    => count($acf_fields),
        'pdf_extracted_count' => $pdf_extracted_count,
    );
}

/**
 * Shared WC-object product assembler (plan a3d60c) — the ONE body behind the two
 * WooCommerce-object ingestion paths: the auto-sync product writer
 * (mxchat_store_product_embedding) and the URL/sitemap product import
 * (mxchat_extract_woocommerce_product_content). Assembles from the WC_Product,
 * the authoritative source for product rows (scope decision on the plan).
 */
private function mxchat_prepare_product_content_for_indexing($product) {
    $title = $product->get_name();
    $description = $product->get_description();
    $short_description = $product->get_short_description();

    // Format content consistently
    $content = $title . "\n\n";

    if (!empty($short_description)) {
        $content .= "Short Description: " . wp_strip_all_tags($short_description) . "\n\n";
    }

    if (!empty($description)) {
        $content .= wp_strip_all_tags($description) . "\n\n";
    }

    $content .= $this->mxchat_woo_product_summary_lines($product);
    $content .= $this->mxchat_woo_custom_tabs_text($product->get_id());

    return $content;
}

/**
 * Pricing + SKU + categories lines for a product — shared by both assembler kinds
 * (the post-fields product enrichment and the WC-object assembler).
 */
private function mxchat_woo_product_summary_lines($product) {
    // Add pricing information (base-currency pinned — see mxchat_product_price_lines)
    $lines = $this->mxchat_product_price_lines($product);

    $sku = $product->get_sku();
    if (!empty($sku)) {
        $lines .= "SKU: " . $sku . "\n";
    }

    // Get product categories
    $categories = wp_get_post_terms($product->get_id(), 'product_cat', array('fields' => 'names'));
    if (!empty($categories) && !is_wp_error($categories)) {
        $lines .= "Categories: " . implode(', ', $categories) . "\n";
    }

    return $lines;
}

/**
 * Custom Product Tabs text (supports "Custom Product Tabs for WooCommerce" by
 * Code Parrots) — direct tabs plus applied reusable/saved tabs. The ONE copy of
 * the yikes_woo logic; three sites carried byte-identical clones before a3d60c.
 */
private function mxchat_woo_custom_tabs_text($product_id) {
    $text = '';

    $custom_tabs = get_post_meta($product_id, 'yikes_woo_products_tabs', true);
    if (!empty($custom_tabs) && is_array($custom_tabs)) {
        foreach ($custom_tabs as $tab) {
            $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
            $tab_content = isset($tab['content']) ? $tab['content'] : '';

            if (!empty($tab_title) && !empty($tab_content)) {
                $text .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
            }
        }
    }

    // Also check for reusable/saved tabs applied to this product
    $applied_saved_tabs = get_post_meta($product_id, 'yikes_woo_reusable_products_tabs_applied', true);
    if (!empty($applied_saved_tabs) && is_array($applied_saved_tabs)) {
        $saved_tabs = get_option('yikes_woo_reusable_products_tabs', array());
        if (!empty($saved_tabs) && is_array($saved_tabs)) {
            foreach ($applied_saved_tabs as $saved_tab_id) {
                if (isset($saved_tabs[$saved_tab_id])) {
                    $tab = $saved_tabs[$saved_tab_id];
                    $tab_title = isset($tab['title']) ? $tab['title'] : (isset($tab['tab_title']) ? $tab['tab_title'] : '');
                    $tab_content = isset($tab['content']) ? $tab['content'] : '';

                    if (!empty($tab_title) && !empty($tab_content)) {
                        $text .= "\n" . $tab_title . ": " . wp_strip_all_tags($tab_content) . "\n";
                    }
                }
            }
        }
    }

    return $text;
}

/**
 * Store the post status and URL before update to detect status transitions
 * This runs before the post is actually updated in the database
 */
public function mxchat_store_pre_update_status($post_id, $data) {
    // Core is inside wp_insert_post's update branch, so a post_updated WILL fire later
    // this request and can consume the arrival-edge guard (plan a664f3).
    $this->pending_post_update[$post_id] = true;

    // Get the current post from database (before update)
    $current_post = get_post($post_id);
    
    if ($current_post) {
        // Store the current status temporarily
        $status_key = 'mxchat_prev_status_' . $post_id;
        set_transient($status_key, $current_post->post_status, HOUR_IN_SECONDS);
        
        // If the post is currently published, also store its URL
        if ($current_post->post_status === 'publish') {
            $url_key = 'mxchat_prev_url_' . $post_id;
            $current_url = get_permalink($post_id);
            set_transient($url_key, $current_url, HOUR_IN_SECONDS);
        }
    }
}

/**
 * Whether auto-sync is enabled for a post type (mirrors the checks used by the
 * update/delete handlers; kept as one helper so new call sites cannot drift).
 */
private function mxchat_is_auto_sync_enabled($post_type) {
    if ($post_type === 'post') {
        return get_option('mxchat_auto_sync_posts') === '1';
    }
    if ($post_type === 'page') {
        return get_option('mxchat_auto_sync_pages') === '1';
    }
    return get_option('mxchat_auto_sync_' . $post_type) === '1';
}

/**
 * Remove a post's vectors the moment it leaves 'publish', using the authoritative
 * old status core passes to transition_post_status — no transient involved (plan 816fb1).
 *
 * Covers status changes that never route through wp_update_post (scheduled-expiry
 * plugins and others that flip post_status directly and call wp_transition_post_status),
 * where neither pre_post_update nor post_updated fires and the old detection missed.
 */
public function mxchat_handle_status_transition($new_status, $old_status, $post) {
    if (!($post instanceof WP_Post) || wp_is_post_revision($post->ID)) {
        return;
    }

    // Arrival edge (plan 3055e1): a post BECOMING published is indexed here, because
    // wp_publish_post() — the path scheduled posts take via check_and_publish_future_post —
    // and direct wp_insert_post(status=publish) creates never fire post_updated, so the
    // auto-sync ADD path alone misses them. Editor publishes also pass through here;
    // the transition_indexed_posts guard keeps mxchat_handle_post_update from embedding
    // a second time in the same request.
    if ($new_status === 'publish' && $old_status !== 'publish') {
        if ($this->mxchat_is_auto_sync_enabled($post->post_type)) {
            $this->mxchat_index_published_post($post->ID, $post);

            // Arm the double-fire guard ONLY when a post_updated is actually coming to
            // consume it (plan a664f3). Two publish paths never fire post_updated at all:
            // a direct wp_insert_post(status=publish) create, and wp_publish_post() — the
            // call check_and_publish_future_post() makes for scheduled posts. Arming the
            // guard unconditionally left it set with nothing to consume it, so the NEXT
            // update of that post was swallowed entirely: zero embed calls, no knowledge
            // -base row, silently. Consume-once on this side too, so a guard can never
            // outlive the single save it was armed for.
            if (!empty($this->pending_post_update[$post->ID])) {
                unset($this->pending_post_update[$post->ID]);
                $this->transition_indexed_posts[$post->ID] = true;
            }
        }
        return;
    }

    // Only the publish -> not-publish edge matters here.
    if ($old_status !== 'publish' || $new_status === 'publish') {
        return;
    }
    // Trash is handled by mxchat_handle_post_delete (wp_trash_post) with pre-trash URL
    // resolution; skip to avoid a second network round-trip per trash.
    if ($new_status === 'trash') {
        return;
    }
    if (!$this->mxchat_is_auto_sync_enabled($post->post_type)) {
        return;
    }

    $urls = array();

    // The DB may already hold the new status when this fires, so get_permalink() on the
    // live post could build a draft-style URL whose md5 misses the stored vector IDs.
    // Reconstruct the published permalink from a clone instead.
    $published_clone = clone $post;
    $published_clone->post_status = 'publish';
    $published_url = get_permalink($published_clone);
    if ($published_url) {
        $urls[] = $published_url;
    }

    // Honour the pre-update capture when present (covers a slug change in the same save).
    $previous_url = get_transient('mxchat_prev_url_' . $post->ID);
    if (!empty($previous_url)) {
        $urls[] = $previous_url;
    }

    foreach (array_unique($urls) as $url) {
        MxChat_Utils::delete_chunks_for_url($url, 'default');
    }

    if (!empty($urls)) {
        $this->transition_deleted_posts[$post->ID] = true;
    }
}

/**
 * WP-CLI: remove knowledge-base entries left behind by posts that were unpublished,
 * trashed, or made private before the transition_post_status handler existed.
 *
 * Walks every auto-synced post type's non-published posts, reconstructs each one's
 * published-era permalink, and deletes its vectors (routes to Pinecone or the WP table).
 * Deletion is idempotent, so never-indexed posts are a cheap no-op.
 *
 * ## OPTIONS
 *
 * [--dry-run]
 * : Report what would be removed without deleting anything.
 *
 * ## EXAMPLES
 *
 *     wp mxchat prune-unpublished --dry-run
 *     wp mxchat prune-unpublished
 */
public function cli_prune_unpublished($args, $assoc_args) {
    global $wpdb;
    $dry_run = !empty($assoc_args['dry-run']);
    $table   = $wpdb->prefix . 'mxchat_system_prompt_content';

    $candidate_types = array_merge(array('post', 'page'), array_values(get_post_types(array('_builtin' => false), 'names')));
    $synced_types = array();
    foreach ($candidate_types as $type) {
        if ($this->mxchat_is_auto_sync_enabled($type)) {
            $synced_types[] = $type;
        }
    }
    if (empty($synced_types)) {
        WP_CLI::success('No post types have auto-sync enabled; nothing to prune.');
        return;
    }

    $scanned = 0;
    $pruned  = 0;
    $paged   = 1;
    do {
        $query = new WP_Query(array(
            'post_type'      => $synced_types,
            'post_status'    => array('draft', 'pending', 'private', 'future', 'trash'),
            'posts_per_page' => 100,
            'paged'          => $paged,
            'fields'         => 'ids',
        ));
        foreach ($query->posts as $post_id) {
            $post = get_post($post_id);
            if (!$post) {
                continue;
            }
            $scanned++;

            // Rebuild the permalink the post had while published: publish-status clone,
            // with wp_trash_post's __trashed slug suffix stripped for trashed posts.
            $clone = clone $post;
            $clone->post_status = 'publish';
            if (substr($clone->post_name, -9) === '__trashed') {
                $clone->post_name = substr($clone->post_name, 0, -9);
            }
            $url = get_permalink($clone);
            if (!$url) {
                continue;
            }

            // Local-table row count is exact in WordPress-DB mode; in Pinecone mode it
            // reads 0 but the delete below still routes to Pinecone and is idempotent.
            $local_rows = (int) $wpdb->get_var($wpdb->prepare(
                "SELECT COUNT(*) FROM {$table} WHERE source_url = %s", $url
            ));

            if ($dry_run) {
                if ($local_rows > 0) {
                    WP_CLI::log(sprintf('Would remove %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
                    $pruned += $local_rows;
                }
                continue;
            }

            MxChat_Utils::delete_chunks_for_url($url, 'default');
            if ($local_rows > 0) {
                WP_CLI::log(sprintf('Removed %d row(s): %s (post %d, %s)', $local_rows, $url, $post_id, $post->post_status));
                $pruned += $local_rows;
            }
        }
        $more = $paged < $query->max_num_pages;
        $paged++;
    } while ($more);

    WP_CLI::success(sprintf(
        '%s %d local knowledge row(s) across %d non-published post(s) scanned.%s',
        $dry_run ? 'Would remove' : 'Removed',
        $pruned,
        $scanned,
        ' (Pinecone-mode deletions are not counted locally.)'
    ));
}

/**
 * WP-CLI: repair knowledge-base rows whose PDF text was imported in visual
 * (reversed) order before the RTL normalizer existed. 32bf9e fixed new
 * imports only; this fixes rows already in the table without the customer
 * having to re-source and re-upload the original PDFs (plan d1e6f7).
 *
 * Detection reuses MxChat_Utils::normalize_pdf_rtl() on the stored text: a
 * row is a candidate exactly when the normalizer would change it, so the
 * import-time heuristic and the repair heuristic can never disagree.
 * Repaired rows are RE-EMBEDDED — the stored vector was computed over
 * reversed text and is as broken as the text — so a wet run calls the
 * embedding provider once per repaired row on the site's API key. Runs
 * beyond 25 rows therefore require --yes.
 *
 * Scope notes:
 * - Scans the WordPress knowledge table. Pinecone-mode entries live in
 *   Pinecone, not this table, and are not scanned; if a scanned row's bot
 *   ALSO has Pinecone enabled (hybrid drift), the repaired entry is
 *   re-submitted through the normal import path so the md5-keyed Pinecone
 *   vector is replaced too.
 * - Knowledge rows do not carry a bot id; --bot only selects whose
 *   embedding configuration (model + key) is used for re-embedding.
 * - The mxchat_pdf_rtl_normalize filter is honoured: a site that disabled
 *   normalization gets detections of zero, not surprise rewrites.
 * - The metadata header the PDF importer stores before the text separator
 *   is preserved byte-identical; only the text segment is repaired.
 *
 * ## OPTIONS
 *
 * [--dry-run]
 * : List the rows that would be repaired without changing anything.
 *
 * [--bot=<id>]
 * : Embedding configuration to use for re-embedding. Default: default.
 *
 * [--all-content]
 * : Scan every row containing right-to-left text, not just rows with PDF
 * provenance (a page anchor in the source URL, or pdf content type).
 *
 * [--yes]
 * : Proceed even when more than 25 rows need re-embedding (API cost gate).
 *
 * ## EXAMPLES
 *
 *     wp mxchat rtl-repair --dry-run
 *     wp mxchat rtl-repair
 *     wp mxchat rtl-repair --all-content --yes
 */
public function cli_rtl_repair($args, $assoc_args) {
    global $wpdb;
    $dry_run = !empty($assoc_args['dry-run']);
    $all     = !empty($assoc_args['all-content']);
    $yes     = !empty($assoc_args['yes']);
    $bot_id  = isset($assoc_args['bot']) ? sanitize_key($assoc_args['bot']) : 'default';
    $table   = $wpdb->prefix . 'mxchat_system_prompt_content';

    // Detection pass — no API calls. Walk the table in id batches so a large
    // knowledge base never loads at once.
    $rtl_re     = '/[\x{0590}-\x{05FF}\x{0600}-\x{06FF}\x{0750}-\x{077F}\x{FB50}-\x{FDFF}\x{FE70}-\x{FEFF}]/u';
    $candidates = array();
    $scanned    = 0;
    $last_id    = 0;
    do {
        if ($all) {
            $rows = $wpdb->get_results($wpdb->prepare(
                "SELECT id, article_content, source_url, content_type FROM {$table}
                 WHERE id > %d ORDER BY id ASC LIMIT 200",
                $last_id
            ));
        } else {
            $rows = $wpdb->get_results($wpdb->prepare(
                "SELECT id, article_content, source_url, content_type FROM {$table}
                 WHERE id > %d AND (source_url LIKE %s OR content_type = 'pdf')
                 ORDER BY id ASC LIMIT 200",
                $last_id,
                '%' . $wpdb->esc_like('#page=') . '%'
            ));
        }
        foreach ($rows as $row) {
            $last_id = (int) $row->id;
            $scanned++;
            $content = (string) $row->article_content;
            if (!preg_match($rtl_re, $content)) {
                continue;
            }
            list($header, $text) = $this->mxchat_rtl_repair_split($content);
            $normalized = MxChat_Utils::normalize_pdf_rtl($text, 'rtl-repair row ' . $row->id);
            if (is_string($normalized) && $normalized !== $text) {
                $candidates[] = array(
                    'id'           => (int) $row->id,
                    'source_url'   => (string) $row->source_url,
                    'content_type' => (string) $row->content_type,
                    'new_content'  => $header . $normalized,
                );
            }
        }
    } while (count($rows) === 200);

    WP_CLI::log(sprintf('Scanned %d row(s); %d stored in reversed (visual) order.', $scanned, count($candidates)));
    if (empty($candidates)) {
        WP_CLI::success('No reversed RTL rows found — nothing to repair.');
        return;
    }

    foreach ($candidates as $c) {
        WP_CLI::log(sprintf('%s row %d  %s', $dry_run ? 'Would repair' : 'Will repair', $c['id'], $c['source_url']));
    }
    if ($dry_run) {
        WP_CLI::success(sprintf('Dry run: %d row(s) would be repaired and re-embedded. Run without --dry-run to apply.', count($candidates)));
        return;
    }

    // Cost gate: re-embedding spends the customer's API budget.
    WP_CLI::log(sprintf('Re-embedding will call the embedding provider once per row — %d call(s) on this site\'s API key.', count($candidates)));
    if (count($candidates) > 25 && !$yes) {
        WP_CLI::error(sprintf('%d rows need re-embedding (more than 25). Re-run with --yes to confirm the API cost. No rows were changed.', count($candidates)));
    }

    $bot_options = $this->get_bot_options($bot_id);
    $options     = !empty($bot_options) ? $bot_options : get_option('mxchat_options');
    $preflight   = MxChat_Utils::embedding_preflight($options);
    if (!$preflight['ok']) {
        WP_CLI::error('Embedding configuration problem: ' . $preflight['reason']);
    }
    $api_key = $preflight['api_key'];

    $pinecone_hybrid = $this->mxchat_rtl_repair_pinecone_enabled($bot_id);
    $repaired = 0;
    $failed   = 0;
    foreach ($candidates as $c) {
        $vector = MxChat_Utils::regenerate_embedding($c['new_content'], $api_key, $bot_id);
        if (!is_array($vector)) {
            $failed++;
            $reason = is_wp_error($vector) ? $vector->get_error_message() : 'embedding request failed';
            // Text and vector must stay consistent: never write repaired text
            // beside the stale reversed-text vector.
            WP_CLI::warning(sprintf('Row %d NOT repaired — %s. Row left unchanged.', $c['id'], $reason));
            continue;
        }
        $wpdb->update(
            $table,
            array(
                'article_content'  => $c['new_content'],
                'embedding_vector' => maybe_serialize($vector),
            ),
            array('id' => $c['id']),
            array('%s', '%s'),
            array('%d')
        );
        $repaired++;
        if (class_exists('MxChat_Admin')) {
            MxChat_Admin::mxchat_log_debug('pdf_rtl_repaired', 'Stored KB row restored to logical order and re-embedded', array(
                'row_id'     => $c['id'],
                'source_url' => $c['source_url'],
                'bot'        => $bot_id,
            ));
        }
        // Hybrid drift: the bot indexes into Pinecone but this row sat in the
        // WP table — push the repaired entry through the normal import path so
        // the md5(source_url)-keyed Pinecone vector is replaced as well.
        if ($pinecone_hybrid) {
            MxChat_Utils::submit_content_to_db(
                $c['new_content'],
                $c['source_url'],
                $api_key,
                null,
                $bot_id,
                $c['content_type'] !== '' ? $c['content_type'] : 'pdf'
            );
        }
    }

    WP_CLI::success(sprintf('Repaired + re-embedded %d row(s); %d failed; %d scanned.', $repaired, $failed, $scanned));
}

/**
 * Split a stored KB row into (metadata header incl. separator, text segment).
 * The PDF importer stores wp_json_encode($metadata) . "\n---\n" . $text —
 * repair must touch only the text and keep the header byte-identical.
 */
private function mxchat_rtl_repair_split($content) {
    $sep = "\n---\n";
    $pos = strpos($content, $sep);
    if ($pos !== false && $pos > 0 && $content[0] === '{') {
        $maybe_json = substr($content, 0, $pos);
        if (json_decode($maybe_json) !== null) {
            return array(substr($content, 0, $pos + strlen($sep)), substr($content, $pos + strlen($sep)));
        }
    }
    return array('', $content);
}

/**
 * Mirror of MxChat_Utils::is_pinecone_enabled_for_bot() (private there) for
 * the repair CLI's hybrid-drift check.
 */
private function mxchat_rtl_repair_pinecone_enabled($bot_id) {
    if ($bot_id !== 'default' && class_exists('MxChat_Multi_Bot_Manager')) {
        $cfg = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
        if (!empty($cfg)) {
            return !empty($cfg['use_pinecone']) && !empty($cfg['api_key']) && !empty($cfg['host']);
        }
    }
    $po = get_option('mxchat_pinecone_addon_options');
    return !empty($po['mxchat_use_pinecone']) && $po['mxchat_use_pinecone'] !== '0'
        && !empty($po['mxchat_pinecone_api_key']) && !empty($po['mxchat_pinecone_host']);
}

public function mxchat_handle_post_delete($post_id) {
    // Get post data before it's deleted
    $post = get_post($post_id);

    // Basic validation
    if (!$post || wp_is_post_revision($post_id)) {
        return;
    }

    $post_type = $post->post_type;
    
    // Check if sync is enabled for this post type
    $should_sync = false;
    
    // Check built-in post types first
    if ($post_type === 'post' && get_option('mxchat_auto_sync_posts') === '1') {
        $should_sync = true;
    } else if ($post_type === 'page' && get_option('mxchat_auto_sync_pages') === '1') {
        $should_sync = true;
    } else {
        // Check custom post types
        $option_name = 'mxchat_auto_sync_' . $post_type;
        if (get_option($option_name) === '1') {
            $should_sync = true;
        }
    }
    
    if (!$should_sync) {
        return;
    }

    // Resolve the pre-trash URL. wp_trash_post renames the slug with "__trashed" before firing
    // this hook, so get_permalink() here would return the trashed URL and md5() would miss the
    // real vector IDs stored under the original URL.
    $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
    if (!$source_url) {
        //error_log('MXChat: Failed to resolve source URL for post ' . $post_id);
        return;
    }

    // Use chunk-aware deletion (handles both chunked and non-chunked content)
    $delete_result = MxChat_Utils::delete_chunks_for_url($source_url, 'default');

    if (is_wp_error($delete_result)) {
        //error_log('MXChat: Chunk-aware deletion failed for URL: ' . $source_url . ' - ' . $delete_result->get_error_message());
    }

    delete_transient('mxchat_prev_url_' . $post_id);
    delete_transient('mxchat_prev_status_' . $post_id);
}

/**
 * Resolve the source URL for a post being trashed/deleted.
 *
 * Why: wp_trash_post appends "__trashed" to the slug before the wp_trash_post action fires, so
 * get_permalink() returns a URL whose md5() won't match the vector IDs stored in Pinecone or
 * the source_url rows in the WP DB. Prefer the URL captured by mxchat_store_pre_update_status
 * (runs on pre_post_update, before the rename); fall back to stripping the __trashed suffix.
 */
private function mxchat_resolve_pre_trash_url($post_id) {
    $previous_url = get_transient('mxchat_prev_url_' . $post_id);
    if (!empty($previous_url)) {
        return $previous_url;
    }

    $current = get_permalink($post_id);
    if (!$current) {
        return '';
    }
    return preg_replace('#__trashed(/?)$#', '$1', $current);
}


public function mxchat_handle_product_change($post_id, $post, $update) {
    if ($post->post_type !== 'product') {
        return;
    }

    if ($post->post_status === 'publish') {
        add_action('shutdown', function() use ($post_id) {
            $product = wc_get_product($post_id);
            if ($product) {
                $this->mxchat_store_product_embedding($product);
            }
        });
    }
}

/**
 * Store WooCommerce product embeddings
 */
private function mxchat_store_product_embedding($product) {
    if (!isset($this->options['enable_woocommerce_integration']) ||
        !in_array($this->options['enable_woocommerce_integration'], ['1', 'on'])) {
        return;
    }

    $source_url = get_permalink($product->get_id());
    $product_id = $product->get_id();

    // Build product content via the shared WC-object assembler (a3d60c) — this
    // writer owns product rows whenever the integration is on.
    $content = $this->mxchat_prepare_product_content_for_indexing($product);

    // Embedding decision — custom-provider-aware (plan cbd5fd); silent-return
    // shape preserved.
    $preflight = MxChat_Utils::embedding_preflight(get_option('mxchat_options'));
    if (!$preflight['ok']) {
        //error_log('MxChat Auto-sync: embedding pre-flight failed: ' . $preflight['reason']);
        return;
    }
    $api_key = $preflight['api_key'];

    // Use the centralized utility function for storage
    $result = MxChat_Utils::submit_content_to_db(
        $content,
        $source_url,
        $api_key,
        md5($source_url) // Vector ID for Pinecone
    );

    //   After successful storage, apply role restriction based on tags
    if (!is_wp_error($result)) {
        $this->apply_role_restriction_to_post($product_id, $source_url);
    }

    if (is_wp_error($result)) {
        //error_log('MxChat WooCommerce sync failed for product ' . $product_id . ': ' . $result->get_error_message());
    }
}

public function mxchat_handle_product_delete($post_id) {
    if (get_post_type($post_id) !== 'product') {
        return;
    }

    $source_url = $this->mxchat_resolve_pre_trash_url($post_id);
    if (!$source_url) {
        return;
    }

    // Chunk-aware deletion (routes to Pinecone or WP DB and removes base + all chunks)
    MxChat_Utils::delete_chunks_for_url($source_url, 'default');

    delete_transient('mxchat_prev_url_' . $post_id);
    delete_transient('mxchat_prev_status_' . $post_id);
}

/**
 * Handle individual Pinecone content deletion
 */
public function mxchat_handle_pinecone_prompt_delete() {
    // Check permissions
    if (!current_user_can('manage_options')) {
        wp_die(esc_html__('You do not have sufficient permissions.', 'mxchat'));
    }
    
    // Verify nonce
    if (!isset($_GET['_wpnonce']) || !wp_verify_nonce($_GET['_wpnonce'], 'mxchat_delete_pinecone_prompt_nonce')) {
        wp_die(esc_html__('Security check failed.', 'mxchat'));
    }
    
    $vector_id = isset($_GET['vector_id']) ? sanitize_text_field($_GET['vector_id']) : '';
    
    if (empty($vector_id)) {
        set_transient('mxchat_admin_notice_error', 
            esc_html__('Invalid vector ID.', 'mxchat'), 
            30
        );
        wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
        exit;
    }
    
    // Get Pinecone settings
    $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
    $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
    
    if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
        set_transient('mxchat_admin_notice_error', 
            esc_html__('Pinecone is not properly configured.', 'mxchat'), 
            30
        );
        wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
        exit;
    }
    
    // Delete from Pinecone
    $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
    $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
        $vector_id,
        $pinecone_options['mxchat_pinecone_api_key'],
        $pinecone_options['mxchat_pinecone_host'],
        $pinecone_options['mxchat_pinecone_namespace'] ?? ''
    );
    
    if ($result['success']) {
        // Mirror the removal to the OpenAI Vector Store mapping (plan 15b5c6);
        // a chunk vector id reduces to its base entry there.
        if (class_exists('MxChat_Vectorstore_Manager')) {
            MxChat_Vectorstore_Manager::sync_delete_by_key($vector_id, 'default');
        }
        // No cache clearing needed since we removed caching
        set_transient('mxchat_admin_notice_success',
            esc_html__('Entry deleted successfully from Pinecone.', 'mxchat'),
            30
        );
    } else {
        set_transient('mxchat_admin_notice_error', 
            esc_html__('Failed to delete entry: ', 'mxchat') . $result['message'], 
            30
        );
    }
    
    wp_safe_redirect(admin_url('admin.php?page=mxchat-prompts'));
    exit;
}
/**
 * Handle individual Pinecone content deletion via AJAX
 */
public function ajax_mxchat_delete_pinecone_prompt() {
    // Verify nonce and permissions
    if (!check_ajax_referer('mxchat_delete_pinecone_prompt_nonce', 'nonce', false)) {
        wp_send_json_error('Invalid nonce');
        exit;
    }
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
        exit;
    }
    
    $vector_id = isset($_POST['vector_id']) ? sanitize_text_field($_POST['vector_id']) : '';
    $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';
    
    if (empty($vector_id)) {
        wp_send_json_error('Missing vector ID');
        exit;
    }
    
    // Get bot-specific Pinecone settings
    $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
    $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
    
    $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
    
    if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
        wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
        exit;
    }
    
    // Delete from the correct Pinecone index and namespace
    $result = $pinecone_manager->mxchat_delete_from_pinecone_by_vector_id(
        $vector_id,
        $pinecone_options['mxchat_pinecone_api_key'],
        $pinecone_options['mxchat_pinecone_host'],
        $pinecone_options['mxchat_pinecone_namespace'] ?? ''
    );
    
    if ($result['success']) {
        // Mirror the removal to the OpenAI Vector Store mapping (plan 15b5c6)
        if (class_exists('MxChat_Vectorstore_Manager')) {
            MxChat_Vectorstore_Manager::sync_delete_by_key($vector_id, $bot_id);
        }
        // No cache clearing needed since we removed caching
        wp_send_json_success(array(
            'message' => 'Entry deleted successfully from Pinecone',
            'vector_id' => $vector_id,
            'bot_id' => $bot_id
        ));
    } else {
        MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete from Pinecone: ' . $result['message'], array('vector_id' => $vector_id, 'bot_id' => $bot_id));
        wp_send_json_error('Failed to delete from Pinecone: ' . $result['message']);
    }
    
    exit;
}

/**
 * Handle deletion of all chunks for a given source URL via AJAX
 * Follows the same pattern as ajax_mxchat_delete_pinecone_prompt
 */
public function ajax_mxchat_delete_chunks_by_url() {
    // Verify nonce and permissions
    if (!check_ajax_referer('mxchat_delete_chunks_nonce', 'nonce', false)) {
        wp_send_json_error('Invalid nonce');
        exit;
    }

    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
        exit;
    }

    $source_url = isset($_POST['source_url']) ? esc_url_raw($_POST['source_url']) : '';
    $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
    $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';

    if (empty($source_url)) {
        wp_send_json_error('Missing source URL');
        exit;
    }

    // Generate the base vector ID from the source URL (same as how chunks are created)
    $base_vector_id = md5($source_url);

    if ($data_source === 'pinecone') {
        // Get bot-specific Pinecone settings (same as working delete function)
        $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
        $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);

        $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';

        if (!$use_pinecone || empty($pinecone_options['mxchat_pinecone_api_key'])) {
            wp_send_json_error('Pinecone is not properly configured for bot: ' . $bot_id);
            exit;
        }

        $api_key = $pinecone_options['mxchat_pinecone_api_key'];
        $host = $pinecone_options['mxchat_pinecone_host'];
        $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';

        // Collect all vector IDs to delete
        $vectors_to_delete = array();

        // Add the original single-vector ID (for non-chunked content)
        $vectors_to_delete[] = $base_vector_id;

        // Use Pinecone list API to find all chunk vectors with this prefix
        // NOTE: Pinecone List API is a GET request with query parameters, not POST
        $prefix = $base_vector_id . '_chunk_';

        $query_params = array(
            'prefix' => $prefix,
            'limit' => 100
        );

        if (!empty($namespace)) {
            $query_params['namespace'] = $namespace;
        }

        $list_url = "https://{$host}/vectors/list?" . http_build_query($query_params);

        $list_response = wp_remote_get($list_url, array(
            'headers' => array(
                'Api-Key' => $api_key,
                'accept' => 'application/json'
            ),
            'timeout' => 30
        ));

        if (!is_wp_error($list_response)) {
            $list_body_response = wp_remote_retrieve_body($list_response);
            $list_data = json_decode($list_body_response, true);
            if (!empty($list_data['vectors'])) {
                foreach ($list_data['vectors'] as $vector) {
                    if (isset($vector['id'])) {
                        $vectors_to_delete[] = $vector['id'];
                    }
                }
            }
        }

        if (empty($vectors_to_delete)) {
            // Entry already gone from Pinecone — still clear any mirrored
            // Vector Store file so it can't outlive the entry (plan 15b5c6).
            if (class_exists('MxChat_Vectorstore_Manager')) {
                MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id);
            }
            wp_send_json_success(array(
                'message' => 'No vectors found to delete',
                'source_url' => $source_url
            ));
            exit;
        }

        // Delete all vectors using the same endpoint as the working function
        $delete_url = "https://{$host}/vectors/delete";

        $delete_body = array(
            'ids' => $vectors_to_delete
        );

        if (!empty($namespace)) {
            $delete_body['namespace'] = $namespace;
        }

        $delete_response = wp_remote_post($delete_url, array(
            'headers' => array(
                'Api-Key' => $api_key,
                'accept' => 'application/json',
                'content-type' => 'application/json'
            ),
            'body' => wp_json_encode($delete_body),
            'timeout' => 30
        ));

        if (is_wp_error($delete_response)) {
            MxChat_Admin::mxchat_log_debug('pinecone_error', 'Failed to delete chunks from Pinecone: ' . $delete_response->get_error_message(), array('source_url' => $source_url));
            wp_send_json_error('Failed to delete from Pinecone: ' . $delete_response->get_error_message());
            exit;
        }

        $response_code = wp_remote_retrieve_response_code($delete_response);

        if ($response_code !== 200) {
            MxChat_Admin::mxchat_log_debug('pinecone_error', 'Pinecone API error (HTTP ' . $response_code . ')', array('source_url' => $source_url));
            wp_send_json_error('Pinecone API error (HTTP ' . $response_code . ')');
            exit;
        }

        // Mirror the removal to the OpenAI Vector Store (plan 15b5c6)
        if (class_exists('MxChat_Vectorstore_Manager')) {
            MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id);
        }

        wp_send_json_success(array(
            'message' => 'All chunks deleted successfully from Pinecone',
            'source_url' => $source_url,
            'deleted_count' => count($vectors_to_delete)
        ));

    } else {
        // WordPress database deletion
        global $wpdb;
        $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';

        $result = $wpdb->delete(
            $table_name,
            array('source_url' => $source_url),
            array('%s')
        );

        if ($result === false) {
            MxChat_Admin::mxchat_log_debug('database_error', 'Failed to delete from database: ' . $wpdb->last_error, array('source_url' => $source_url));
            wp_send_json_error('Failed to delete from database: ' . $wpdb->last_error);
            exit;
        }

        // Mirror the removal to the OpenAI Vector Store (plan 15b5c6)
        if (class_exists('MxChat_Vectorstore_Manager')) {
            MxChat_Vectorstore_Manager::sync_delete_entry($source_url, $bot_id);
        }

        wp_send_json_success(array(
            'message' => 'All chunks deleted successfully from database',
            'source_url' => $source_url,
            'deleted_count' => $result
        ));
    }

    exit;
}

/**
 * Handle individual WordPress database content deletion via AJAX
 * Mirrors the Pinecone delete handler but for WordPress database entries
 */
public function ajax_mxchat_delete_wordpress_prompt() {
    // Verify nonce and permissions
    if (!check_ajax_referer('mxchat_delete_wordpress_prompt_nonce', 'nonce', false)) {
        wp_send_json_error('Invalid nonce');
        exit;
    }

    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
        exit;
    }

    $entry_id = isset($_POST['entry_id']) ? intval($_POST['entry_id']) : 0;

    if (empty($entry_id)) {
        wp_send_json_error('Missing entry ID');
        exit;
    }

    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';

    // Capture the identity BEFORE the row disappears — needed to mirror the
    // change into the Vector Store (plan 15b5c6).
    $source_url = $wpdb->get_var($wpdb->prepare(
        "SELECT source_url FROM {$table_name} WHERE id = %d",
        $entry_id
    ));

    // Clear cache for this entry
    wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');

    // Delete from database
    $result = $wpdb->delete(
        $table_name,
        array('id' => $entry_id),
        array('%d')
    );

    if ($result !== false) {
        // Mirror to the Vector Store: if sibling rows remain (this was one
        // chunk of a larger entry) the entry's file is REFRESHED from what's
        // left; if none remain, the file is removed.
        if (!empty($source_url) && class_exists('MxChat_Vectorstore_Manager')) {
            $remaining = (int) $wpdb->get_var($wpdb->prepare(
                "SELECT COUNT(*) FROM {$table_name} WHERE source_url = %s",
                $source_url
            ));
            if ($remaining > 0) {
                MxChat_Vectorstore_Manager::sync_upsert_entry($source_url, '', 'default');
            } else {
                MxChat_Vectorstore_Manager::sync_delete_entry($source_url, 'default');
            }
        }

        wp_send_json_success(array(
            'message' => 'Entry deleted successfully',
            'entry_id' => $entry_id
        ));
    } else {
        wp_send_json_error('Failed to delete entry from database');
    }

    exit;
}

/**
 * Handle bulk deletion of knowledge entries via AJAX
 * Supports both Pinecone and WordPress database entries
 */
public function ajax_mxchat_bulk_delete_knowledge() {
    // Verify nonce and permissions
    if (!check_ajax_referer('mxchat_bulk_delete_knowledge_nonce', 'nonce', false)) {
        wp_send_json_error('Invalid nonce');
        exit;
    }

    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
        exit;
    }

    $entries = isset($_POST['entries']) ? $_POST['entries'] : array();
    $bot_id = isset($_POST['bot_id']) ? sanitize_text_field($_POST['bot_id']) : 'default';

    if (empty($entries) || !is_array($entries)) {
        wp_send_json_error('No entries provided');
        exit;
    }

    // Extend execution time — bulk Pinecone operations can take a while
    if (function_exists('set_time_limit')) {
        set_time_limit(120);
    }

    $success_ids = array();
    $failed_ids = array();
    $errors = array();

    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';

    // Get Pinecone manager for Pinecone deletions
    $pinecone_manager = MxChat_Pinecone_Manager::get_instance();
    $pinecone_options = $pinecone_manager->mxchat_get_bot_pinecone_options($bot_id);
    $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';

    $api_key = $pinecone_options['mxchat_pinecone_api_key'] ?? '';
    $host = $pinecone_options['mxchat_pinecone_host'] ?? '';
    $namespace = $pinecone_options['mxchat_pinecone_namespace'] ?? '';

    // =============================================
    // PHASE 1: Collect all Pinecone vector IDs
    // and separate WordPress entries
    // =============================================
    $pinecone_entry_ids = array();  // entry IDs that are pinecone-sourced
    $wordpress_entries = array();   // entries for WordPress DB deletion
    $all_vector_ids = array();      // all pinecone vector IDs to delete in one batch
    $vs_mirror_urls = array();      // Vector Store mirror: URLs to delete (plan 15b5c6)
    $vs_mirror_keys = array();      // Vector Store mirror: bare vector ids to delete

    foreach ($entries as $entry) {
        $entry_id = sanitize_text_field($entry['id'] ?? '');
        $source = sanitize_text_field($entry['source'] ?? 'wordpress');
        $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
        $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');

        if (empty($entry_id)) {
            continue;
        }

        if ($source === 'pinecone') {
            if (!$use_pinecone || empty($api_key)) {
                $failed_ids[] = $entry_id;
                $errors[] = "Pinecone not configured for entry: $entry_id";
                continue;
            }

            $pinecone_entry_ids[] = $entry_id;

            if ($is_group && !empty($source_url)) {
                // Grouped/chunked entry: collect base ID + chunk IDs via List API
                $base_vector_id = md5($source_url);
                $all_vector_ids[] = $base_vector_id;

                $list_url = 'https://' . $host . '/vectors/list?prefix=' . urlencode($base_vector_id . '_chunk_') . '&limit=100';
                if (!empty($namespace)) {
                    $list_url .= '&namespace=' . rawurlencode($namespace);
                }
                $list_response = wp_remote_get($list_url, array(
                    'headers' => array(
                        'Api-Key' => $api_key,
                        'accept' => 'application/json'
                    ),
                    'timeout' => 30
                ));

                if (!is_wp_error($list_response)) {
                    $list_body = json_decode(wp_remote_retrieve_body($list_response), true);
                    if (!empty($list_body['vectors']) && is_array($list_body['vectors'])) {
                        foreach ($list_body['vectors'] as $vector) {
                            if (isset($vector['id'])) {
                                $all_vector_ids[] = $vector['id'];
                            }
                        }
                    }
                }
            } else {
                // Single entry: the entry_id IS the vector ID
                $all_vector_ids[] = $entry_id;
            }

            if (!empty($source_url)) {
                $vs_mirror_urls[] = $source_url;
            } else {
                $vs_mirror_keys[] = $entry_id;
            }
        } else {
            $wordpress_entries[] = $entry;
        }
    }

    // =============================================
    // PHASE 2: Single batch delete to Pinecone
    // =============================================
    if (!empty($all_vector_ids)) {
        $all_vector_ids = array_values(array_unique($all_vector_ids));
        $pinecone_success = true;
        $batches = array_chunk($all_vector_ids, 100);

        foreach ($batches as $batch) {
            $delete_body = array('ids' => $batch);
            if (!empty($namespace)) {
                $delete_body['namespace'] = $namespace;
            }
            $delete_response = wp_remote_post("https://{$host}/vectors/delete", array(
                'headers' => array(
                    'Api-Key' => $api_key,
                    'accept' => 'application/json',
                    'content-type' => 'application/json'
                ),
                'body' => wp_json_encode($delete_body),
                'timeout' => 60
            ));

            if (is_wp_error($delete_response)) {
                $pinecone_success = false;
                $errors[] = 'Pinecone batch deletion failed: ' . $delete_response->get_error_message();
                MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed: ' . $delete_response->get_error_message());
            } else {
                $response_code = wp_remote_retrieve_response_code($delete_response);
                if ($response_code !== 200) {
                    $pinecone_success = false;
                    $response_body = wp_remote_retrieve_body($delete_response);
                    $errors[] = "Pinecone API error (HTTP $response_code)";
                    MxChat_Admin::mxchat_log_debug('pinecone_error', 'Bulk delete batch failed (HTTP ' . $response_code . ')', array('response' => substr($response_body, 0, 200)));
                }
            }
        }

        // Mark all pinecone entries based on batch result
        foreach ($pinecone_entry_ids as $eid) {
            if ($pinecone_success) {
                $success_ids[] = $eid;
            } else {
                $failed_ids[] = $eid;
            }
        }

        // Mirror the removals to the OpenAI Vector Store (plan 15b5c6)
        if ($pinecone_success && class_exists('MxChat_Vectorstore_Manager')) {
            foreach (array_unique($vs_mirror_urls) as $vs_url) {
                MxChat_Vectorstore_Manager::sync_delete_entry($vs_url, $bot_id);
            }
            foreach (array_unique($vs_mirror_keys) as $vs_key) {
                MxChat_Vectorstore_Manager::sync_delete_by_key($vs_key, $bot_id);
            }
        }
    }

    // =============================================
    // PHASE 3: WordPress database deletions
    // =============================================
    foreach ($wordpress_entries as $entry) {
        $entry_id = sanitize_text_field($entry['id'] ?? '');
        $source_url = isset($entry['sourceUrl']) ? esc_url_raw($entry['sourceUrl']) : '';
        $is_group = isset($entry['isGroup']) && ($entry['isGroup'] === true || $entry['isGroup'] === 'true');

        if (empty($entry_id)) {
            continue;
        }

        try {
            if ($is_group && !empty($source_url)) {
                $result = $wpdb->delete(
                    $table_name,
                    array('source_url' => $source_url),
                    array('%s')
                );
                $row_url = $source_url;
            } else {
                // Identity captured pre-delete for the Vector Store mirror
                $row_url = $wpdb->get_var($wpdb->prepare(
                    "SELECT source_url FROM {$table_name} WHERE id = %d",
                    intval($entry_id)
                ));
                wp_cache_delete('prompt_' . $entry_id, 'mxchat_prompts');
                $result = $wpdb->delete(
                    $table_name,
                    array('id' => intval($entry_id)),
                    array('%d')
                );
            }

            if ($result !== false) {
                $success_ids[] = $entry_id;
                // Mirror to the Vector Store: refresh the entry's file when
                // sibling chunk rows survive, remove it when none do.
                if (!empty($row_url) && class_exists('MxChat_Vectorstore_Manager')) {
                    $remaining = (int) $wpdb->get_var($wpdb->prepare(
                        "SELECT COUNT(*) FROM {$table_name} WHERE source_url = %s",
                        $row_url
                    ));
                    if ($remaining > 0) {
                        MxChat_Vectorstore_Manager::sync_upsert_entry($row_url, '', $bot_id);
                    } else {
                        MxChat_Vectorstore_Manager::sync_delete_entry($row_url, $bot_id);
                    }
                }
            } else {
                $failed_ids[] = $entry_id;
                $errors[] = "Database error for entry: $entry_id";
            }
        } catch (Exception $e) {
            $failed_ids[] = $entry_id;
            $errors[] = $e->getMessage();
        }
    }

    wp_send_json_success(array(
        'success_ids' => $success_ids,
        'failed_ids' => $failed_ids,
        'errors' => $errors,
        'total_processed' => count($success_ids) + count($failed_ids)
    ));

    exit;
}

/**
 *   Get hierarchical roles for dropdown
 */
public function mxchat_get_role_options() {
    return array(
        'public' => __('Public (Everyone)', 'mxchat'),
        'logged_in' => __('Logged In Users', 'mxchat'),
        'subscriber' => __('Subscribers & Above', 'mxchat'),
        'contributor' => __('Contributors & Above', 'mxchat'),
        'author' => __('Authors & Above', 'mxchat'),
        'editor' => __('Editors & Above', 'mxchat'),
        'administrator' => __('Administrators Only', 'mxchat')
    );
}

/**
 *   Check if user has access to content based on role restriction
 */
public function mxchat_user_has_content_access($role_restriction) {
    // Public content is always accessible
    if ($role_restriction === 'public' || empty($role_restriction)) {
        return true;
    }
    
    // Check if user is logged in for logged_in restriction
    if ($role_restriction === 'logged_in') {
        return is_user_logged_in();
    }
    
    // If not logged in, no access to role-restricted content
    if (!is_user_logged_in()) {
        return false;
    }
    
    $user = wp_get_current_user();
    $user_roles = $user->roles;
    
    if (empty($user_roles)) {
        return false;
    }
    
    // Define role hierarchy (higher number = higher access)
    $hierarchy = array(
        'subscriber' => 1,
        'contributor' => 2,
        'author' => 3,
        'editor' => 4,
        'administrator' => 5
    );
    
    // Get required level
    $required_level = isset($hierarchy[$role_restriction]) ? $hierarchy[$role_restriction] : 0;
    
    // Check if user has required level or higher
    foreach ($user_roles as $user_role) {
        $user_level = isset($hierarchy[$user_role]) ? $hierarchy[$user_role] : 0;
        if ($user_level >= $required_level) {
            return true;
        }
    }
    
    return false;
}

/**
 * Handle role restriction updates via AJAX
 *  Removed cache clearing call since we removed caching
 */
public function ajax_mxchat_update_role_restriction() {
    // Verify nonce and permissions
    if (!check_ajax_referer('mxchat_update_role_nonce', 'nonce', false)) {
        wp_send_json_error('Invalid nonce');
        exit;
    }
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
        exit;
    }
    
    $entry_id = isset($_POST['entry_id']) ? sanitize_text_field($_POST['entry_id']) : '';
    $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';
    $data_source = isset($_POST['data_source']) ? sanitize_text_field($_POST['data_source']) : 'wordpress';
    
    if (empty($entry_id)) {
        wp_send_json_error('Invalid entry ID');
        exit;
    }
    
    // Get knowledge manager instance to validate role restriction
    $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
    $valid_roles = array_keys($knowledge_manager->mxchat_get_role_options());
    if (!in_array($role_restriction, $valid_roles)) {
        wp_send_json_error('Invalid role restriction');
        exit;
    }
    
    global $wpdb;
    
    if ($data_source === 'pinecone') {
        // Handle Pinecone role restriction (stored separately in WordPress table)
        $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
        
        // Use REPLACE to insert or update the role restriction
        $result = $wpdb->replace(
            $roles_table,
            array(
                'vector_id' => $entry_id,
                'role_restriction' => $role_restriction,
                'updated_at' => current_time('mysql')
            ),
            array('%s', '%s', '%s')
        );
        
        // No cache clearing needed since we removed caching
        
    } else {
        // Handle WordPress database role restriction (existing functionality)
        $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
        
        $result = $wpdb->update(
            $table_name,
            array('role_restriction' => $role_restriction),
            array('id' => absint($entry_id)),
            array('%s'),
            array('%d')
        );
    }
    
    if ($result === false) {
        wp_send_json_error('Database update failed: ' . $wpdb->last_error);
        exit;
    }

    // Keep the OpenAI Vector Store mirror consistent with the new restriction
    // (plan 15b5c6): non-public pulls the mirrored file, public re-mirrors.
    if (class_exists('MxChat_Vectorstore_Manager')) {
        if ($data_source === 'pinecone') {
            if ($role_restriction !== 'public') {
                MxChat_Vectorstore_Manager::sync_delete_by_key($entry_id, 'default');
            }
            // Public again: Pinecone-mode content isn't held locally, so the
            // entry re-mirrors on its next save/import rather than here.
        } else {
            $row_url = $wpdb->get_var($wpdb->prepare(
                "SELECT source_url FROM {$wpdb->prefix}mxchat_system_prompt_content WHERE id = %d",
                absint($entry_id)
            ));
            if (!empty($row_url)) {
                MxChat_Vectorstore_Manager::handle_role_change($row_url, 'default', $role_restriction);
            }
        }
    }

    wp_send_json_success(array(
        'message' => 'Role restriction updated successfully',
        'role_restriction' => $role_restriction,
        'data_source' => $data_source,
        'entry_id' => $entry_id
    ));
    exit;
}

// ========================================
// ROLE-BASED CONTENT RESTRICTIONS - NEW FUNCTIONS
// Add these to your MxChat_Knowledge_Manager class
// ========================================

/**
 *   Initialize role-based content hooks
 * Add this call to your __construct() or mxchat_init_hooks() method
 */
private function mxchat_init_role_hooks() {
    // AJAX handlers for tag-role mappings
    add_action('wp_ajax_mxchat_add_tag_role_mapping', array($this, 'ajax_add_tag_role_mapping'));
    add_action('wp_ajax_mxchat_delete_tag_role_mapping', array($this, 'ajax_delete_tag_role_mapping'));
    add_action('wp_ajax_mxchat_get_tag_role_mappings', array($this, 'ajax_get_tag_role_mappings'));
    add_action('wp_ajax_mxchat_bulk_update_tag_roles', array($this, 'ajax_bulk_update_tag_roles'));
    
    // Hook to automatically update role restrictions when tags are added/removed
    add_action('set_object_terms', array($this, 'handle_tag_change'), 10, 6);
    
    // Hook to apply role restrictions on auto-sync
    add_action('mxchat_content_stored', array($this, 'apply_role_restriction_after_storage'), 10, 2);
}

/**
 *   Add tag-role mapping via AJAX
 */
public function ajax_add_tag_role_mapping() {
    // Verify nonce and permissions
    check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
        exit;
    }
    
    $tag_input = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
    $role_restriction = isset($_POST['role_restriction']) ? sanitize_text_field($_POST['role_restriction']) : 'public';

    if (empty($tag_input)) {
        wp_send_json_error('Please enter a tag name or slug');
        exit;
    }

    // Validate role restriction
    $valid_roles = array_keys($this->mxchat_get_role_options());
    if (!in_array($role_restriction, $valid_roles)) {
        wp_send_json_error('Invalid role restriction');
        exit;
    }

    // Resolve the tag by slug first, then fall back to its display name, so users can
    // enter either "premium-content" or "Premium Content". (plan b8bcf5 — the field is
    // labeled by name but previously validated by slug only, producing the confusing
    // "Tag does not exist in WordPress" error when a real tag's name was typed.)
    $term = get_term_by('slug', $tag_input, 'post_tag');
    if (!$term) {
        $term = get_term_by('name', $tag_input, 'post_tag');
    }
    if (!$term) {
        wp_send_json_error('No tag with that name or slug exists yet. Create it under Posts → Tags first, then enter its name or slug.');
        exit;
    }

    // Always key the mapping by the RESOLVED slug — apply_role_restriction_to_post()
    // compares against each post's tag slugs, so the stored key must be a slug,
    // never the raw (possibly display-name) input.
    $tag_slug = $term->slug;

    // Get existing mappings
    $mappings = get_option('mxchat_tag_role_mappings', array());
    
    // Check if mapping already exists
    if (isset($mappings[$tag_slug])) {
        wp_send_json_error('Mapping for this tag already exists');
        exit;
    }
    
    // Add new mapping
    $mappings[$tag_slug] = $role_restriction;
    update_option('mxchat_tag_role_mappings', $mappings);
    
    wp_send_json_success(array(
        'message' => 'Tag-role mapping added successfully',
        'tag_slug' => $tag_slug,
        'role_restriction' => $role_restriction
    ));
    exit;
}

/**
 *   Delete tag-role mapping via AJAX
 */
public function ajax_delete_tag_role_mapping() {
    // Verify nonce and permissions
    check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
        exit;
    }
    
    $tag_slug = isset($_POST['tag_slug']) ? sanitize_text_field($_POST['tag_slug']) : '';
    
    if (empty($tag_slug)) {
        wp_send_json_error('Tag slug is required');
        exit;
    }
    
    // Get existing mappings
    $mappings = get_option('mxchat_tag_role_mappings', array());
    
    // Check if mapping exists
    if (!isset($mappings[$tag_slug])) {
        wp_send_json_error('Mapping does not exist');
        exit;
    }
    
    // Remove mapping
    unset($mappings[$tag_slug]);
    update_option('mxchat_tag_role_mappings', $mappings);
    
    wp_send_json_success(array(
        'message' => 'Tag-role mapping deleted successfully',
        'tag_slug' => $tag_slug
    ));
    exit;
}

/**
 *   Get all tag-role mappings via AJAX
 */
public function ajax_get_tag_role_mappings() {
    // Verify nonce and permissions
    check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
        exit;
    }
    
    // Get mappings
    $mappings = get_option('mxchat_tag_role_mappings', array());
    $role_options = $this->mxchat_get_role_options();
    
    $formatted_mappings = array();
    
    foreach ($mappings as $tag_slug => $role_restriction) {
        // Get tag object
        $term = get_term_by('slug', $tag_slug, 'post_tag');
        
        // Count posts with this tag
        $post_count = 0;
        if ($term) {
            $post_count = $term->count;
        }
        
        $formatted_mappings[] = array(
            'tag_slug' => $tag_slug,
            'role_restriction' => $role_restriction,
            'role_label' => $role_options[$role_restriction] ?? $role_restriction,
            'post_count' => $post_count
        );
    }
    
    wp_send_json_success(array(
        'mappings' => $formatted_mappings
    ));
    exit;
}

/**
 *   Bulk update role restrictions for all existing content with mapped tags
 */
public function ajax_bulk_update_tag_roles() {
    // Verify nonce and permissions
    check_ajax_referer('mxchat_prompts_setting_nonce', 'nonce');
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
        exit;
    }
    
    // Get mappings
    $mappings = get_option('mxchat_tag_role_mappings', array());
    
    if (empty($mappings)) {
        wp_send_json_error('No tag-role mappings found');
        exit;
    }
    
    global $wpdb;
    
    // Check if using Pinecone
    $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
    $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
    
    $updated_count = 0;
    $details = array();
    
    foreach ($mappings as $tag_slug => $role_restriction) {
        // Get all posts with this tag
        $posts = get_posts(array(
            'tag' => $tag_slug,
            'post_type' => 'any',
            'posts_per_page' => -1,
            'fields' => 'ids',
            'post_status' => 'publish'
        ));
        
        if (empty($posts)) {
            continue;
        }
        
        $tag_updated = 0;
        
        foreach ($posts as $post_id) {
            $source_url = get_permalink($post_id);
            if (!$source_url) {
                continue;
            }
            
            if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
                // Update Pinecone role restriction
                $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
                $vector_id = md5($source_url);
                
                $result = $wpdb->replace(
                    $roles_table,
                    array(
                        'vector_id' => $vector_id,
                        'role_restriction' => $role_restriction,
                        'updated_at' => current_time('mysql')
                    ),
                    array('%s', '%s', '%s')
                );
            } else {
                // Update WordPress DB
                $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
                
                $result = $wpdb->update(
                    $table_name,
                    array('role_restriction' => $role_restriction),
                    array('source_url' => $source_url),
                    array('%s'),
                    array('%s')
                );
            }
            
            if ($result !== false) {
                $tag_updated++;
                $updated_count++;
            }
        }
        
        if ($tag_updated > 0) {
            $details[] = sprintf(
                'Tag "%s" (%s): %d posts updated',
                $tag_slug,
                $role_restriction,
                $tag_updated
            );
        }
    }
    
    wp_send_json_success(array(
        'message' => 'Bulk update completed',
        'updated_count' => $updated_count,
        'tags_processed' => count($mappings),
        'details' => $details
    ));
    exit;
}

/**
 *   Handle tag changes on posts (when tags are added or removed)
 */
public function handle_tag_change($object_id, $terms, $tt_ids, $taxonomy, $append, $old_tt_ids) {
    // Only process post tags
    if ($taxonomy !== 'post_tag') {
        return;
    }
    
    // Get tag-role mappings
    $mappings = get_option('mxchat_tag_role_mappings', array());
    
    if (empty($mappings)) {
        return;
    }
    
    // Get the post's URL
    $source_url = get_permalink($object_id);
    if (!$source_url) {
        return;
    }
    
    // Determine the highest role restriction based on tags
    $highest_role = 'public';
    $role_hierarchy = array(
        'public' => 0,
        'logged_in' => 1,
        'subscriber' => 2,
        'contributor' => 3,
        'author' => 4,
        'editor' => 5,
        'administrator' => 6
    );
    
    // Get all current tags for the post
    $current_tags = wp_get_post_tags($object_id, array('fields' => 'slugs'));
    
    // Find the highest role restriction among the tags
    foreach ($current_tags as $tag_slug) {
        if (isset($mappings[$tag_slug])) {
            $role = $mappings[$tag_slug];
            if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
                $highest_role = $role;
            }
        }
    }
    
    // Update the role restriction in the database
    global $wpdb;
    
    // Check if using Pinecone
    $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
    $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
    
    if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
        // Update Pinecone role restriction
        $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
        $vector_id = md5($source_url);
        
        $wpdb->replace(
            $roles_table,
            array(
                'vector_id' => $vector_id,
                'role_restriction' => $highest_role,
                'updated_at' => current_time('mysql')
            ),
            array('%s', '%s', '%s')
        );
    } else {
        // Update WordPress DB
        $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';

        $wpdb->update(
            $table_name,
            array('role_restriction' => $highest_role),
            array('source_url' => $source_url),
            array('%s'),
            array('%s')
        );
    }

    // The entry's restriction just changed — keep the OpenAI Vector Store
    // mirror consistent: non-public pulls the file (file_search has no
    // per-role filtering), public re-mirrors it (plan 15b5c6).
    if (class_exists('MxChat_Vectorstore_Manager')) {
        MxChat_Vectorstore_Manager::handle_role_change($source_url, 'default', $highest_role);
    }
}

/**
 *   Apply role restriction after content is stored (for auto-sync)
 */
public function apply_role_restriction_after_storage($post_id, $source_url) {
    // Get tag-role mappings
    $mappings = get_option('mxchat_tag_role_mappings', array());
    
    if (empty($mappings)) {
        return;
    }
    
    // Get all tags for the post
    $post_tags = wp_get_post_tags($post_id, array('fields' => 'slugs'));
    
    if (empty($post_tags)) {
        return;
    }
    
    // Determine the highest role restriction based on tags
    $highest_role = 'public';
    $role_hierarchy = array(
        'public' => 0,
        'logged_in' => 1,
        'subscriber' => 2,
        'contributor' => 3,
        'author' => 4,
        'editor' => 5,
        'administrator' => 6
    );
    
    foreach ($post_tags as $tag_slug) {
        if (isset($mappings[$tag_slug])) {
            $role = $mappings[$tag_slug];
            if (isset($role_hierarchy[$role]) && $role_hierarchy[$role] > $role_hierarchy[$highest_role]) {
                $highest_role = $role;
            }
        }
    }
    
    // If no restricted tags found, return (leave as public)
    if ($highest_role === 'public') {
        return;
    }
    
    // Update the role restriction
    global $wpdb;
    
    // Check if using Pinecone
    $pinecone_options = get_option('mxchat_pinecone_addon_options', array());
    $use_pinecone = ($pinecone_options['mxchat_use_pinecone'] ?? '0') === '1';
    
    if ($use_pinecone && !empty($pinecone_options['mxchat_pinecone_api_key'])) {
        // Update Pinecone role restriction
        $roles_table = $wpdb->prefix . 'mxchat_pinecone_roles';
        $vector_id = md5($source_url);
        
        $wpdb->replace(
            $roles_table,
            array(
                'vector_id' => $vector_id,
                'role_restriction' => $highest_role,
                'updated_at' => current_time('mysql')
            ),
            array('%s', '%s', '%s')
        );
    } else {
        // Update WordPress DB
        $table_name = $wpdb->prefix . 'mxchat_system_prompt_content';

        $wpdb->update(
            $table_name,
            array('role_restriction' => $highest_role),
            array('source_url' => $source_url),
            array('%s'),
            array('%s')
        );
    }

    // The entry's restriction just changed — keep the OpenAI Vector Store
    // mirror consistent: non-public pulls the file (file_search has no
    // per-role filtering), public re-mirrors it (plan 15b5c6).
    if (class_exists('MxChat_Vectorstore_Manager')) {
        MxChat_Vectorstore_Manager::handle_role_change($source_url, 'default', $highest_role);
    }
}


 // ========================================
    // HELPER METHODS
    // ========================================
    
    /**
     * Check if user has required permissions for content processing
     */
    private function mxchat_check_user_permissions() {
        if (!current_user_can('manage_options')) {
            wp_die(esc_html__('You do not have sufficient permissions.', 'mxchat'));
        }
    }
    
    /**
     * Validate nonce for security
     */
    private function mxchat_validate_nonce($nonce_name, $nonce_action) {
        if (!isset($_POST[$nonce_name]) || !wp_verify_nonce($_POST[$nonce_name], $nonce_action)) {
            wp_die(esc_html__('Security check failed.', 'mxchat'));
        }
    }
    
    /**
     * Get embedding API credentials
     */
    private function mxchat_get_embedding_credentials() {
        $embedding_model = $this->options['embedding_model'] ?? 'text-embedding-ada-002';
        
        if (strpos($embedding_model, 'text-embedding-') !== false) {
            return array(
                'type' => 'openai',
                'api_key' => $this->options['api_key'] ?? ''
            );
        } elseif (strpos($embedding_model, 'voyage-') !== false) {
            return array(
                'type' => 'voyage',
                'api_key' => $this->options['voyage_api_key'] ?? ''
            );
        } elseif (strpos($embedding_model, 'gemini-embedding-') !== false) {
            return array(
                'type' => 'gemini',
                'api_key' => $this->options['gemini_api_key'] ?? ''
            );
        }
        
        return array('type' => 'unknown', 'api_key' => '');
    }
    
    /**
     * Log processing errors
     */
    private function mxchat_log_processing_error($operation, $error_message) {
        //error_log("MxChat Knowledge Processing {$operation} Error: " . $error_message);
    }
    
    /**
     * Set admin notice transient
     */
    private function mxchat_set_admin_notice($type, $message) {
        set_transient("mxchat_admin_notice_{$type}", $message, 30);
    }
    
    /**
     * Get Pinecone manager instance for vector operations
     */
    private function mxchat_get_pinecone_manager() {
        return MxChat_Pinecone_Manager::get_instance();
    }
    
    
    // ========================================
// DATABASE QUEUE TABLE MANAGEMENT
// ========================================

/**
 * Create queue table on plugin activation
 * Call this from your plugin activation hook
 */
public function mxchat_create_queue_table() {
    global $wpdb;
    
    $table_name = $wpdb->prefix . 'mxchat_processing_queue';
    $charset_collate = $wpdb->get_charset_collate();
    
    $sql = "CREATE TABLE IF NOT EXISTS $table_name (
        id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
        queue_id varchar(64) NOT NULL,
        item_type varchar(20) NOT NULL,
        item_data longtext NOT NULL,
        status varchar(20) NOT NULL DEFAULT 'pending',
        bot_id varchar(50) NOT NULL DEFAULT 'default',
        priority int(11) NOT NULL DEFAULT 0,
        attempts int(11) NOT NULL DEFAULT 0,
        max_attempts int(11) NOT NULL DEFAULT 3,
        error_message text DEFAULT NULL,
        created_at datetime NOT NULL,
        started_at datetime DEFAULT NULL,
        completed_at datetime DEFAULT NULL,
        PRIMARY KEY  (id),
        KEY queue_id (queue_id),
        KEY status (status),
        KEY item_type (item_type),
        KEY priority (priority)
    ) $charset_collate;";
    
    require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
    dbDelta($sql);
    
    // Also create a meta table for queue metadata
    $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
    
    $meta_sql = "CREATE TABLE IF NOT EXISTS $meta_table (
        id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
        queue_id varchar(64) NOT NULL,
        meta_key varchar(255) NOT NULL,
        meta_value longtext,
        PRIMARY KEY  (id),
        KEY queue_id (queue_id),
        KEY meta_key (meta_key)
    ) $charset_collate;";
    
    dbDelta($meta_sql);
}

/**
 * Add items to the processing queue
 * 
 * @param string $queue_id Unique identifier for this queue batch
 * @param string $item_type Type of item (url, pdf_page)
 * @param array $items Array of items to queue
 * @param string $bot_id Bot ID for processing
 * @return int Number of items queued
 */
private function mxchat_add_to_queue($queue_id, $item_type, $items, $bot_id = 'default') {
    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_processing_queue';
    
    $queued_count = 0;
    $priority = 0;
    
    foreach ($items as $item) {
        $result = $wpdb->insert(
            $table_name,
            array(
                'queue_id' => $queue_id,
                'item_type' => $item_type,
                'item_data' => wp_json_encode($item),
                'status' => 'pending',
                'bot_id' => $bot_id,
                'priority' => $priority,
                'attempts' => 0,
                'max_attempts' => 3,
                'created_at' => current_time('mysql')
            ),
            array('%s', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%s')
        );
        
        if ($result) {
            $queued_count++;
        }
        
        $priority++; // Process in order
    }
    
    return $queued_count;
}

/**
 * Store queue metadata (total counts, source URL, etc.)
 */
private function mxchat_set_queue_meta($queue_id, $meta_key, $meta_value) {
    global $wpdb;
    $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
    
    // Check if meta exists
    $existing = $wpdb->get_var($wpdb->prepare(
        "SELECT id FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
        $queue_id,
        $meta_key
    ));
    
    if ($existing) {
        // Update
        $wpdb->update(
            $meta_table,
            array('meta_value' => maybe_serialize($meta_value)),
            array('queue_id' => $queue_id, 'meta_key' => $meta_key),
            array('%s'),
            array('%s', '%s')
        );
    } else {
        // Insert
        $wpdb->insert(
            $meta_table,
            array(
                'queue_id' => $queue_id,
                'meta_key' => $meta_key,
                'meta_value' => maybe_serialize($meta_value)
            ),
            array('%s', '%s', '%s')
        );
    }
}

/**
 * Get queue metadata
 */
private function mxchat_get_queue_meta($queue_id, $meta_key) {
    global $wpdb;
    $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
    
    $value = $wpdb->get_var($wpdb->prepare(
        "SELECT meta_value FROM $meta_table WHERE queue_id = %s AND meta_key = %s",
        $queue_id,
        $meta_key
    ));
    
    return maybe_unserialize($value);
}

// ========================================
// AJAX QUEUE PROCESSING HANDLERS
// ========================================

/**
 * AJAX: Get next item from queue to process
 */
public function ajax_mxchat_get_next_queue_item() {
    // Verify nonce and permissions
    check_ajax_referer('mxchat_queue_nonce', 'nonce');
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
    }
    
    $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
    
    if (empty($queue_id)) {
        wp_send_json_error('Missing queue ID');
    }
    
    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_processing_queue';
    
    // Get next pending item with retry logic for failed items
    $next_item = $wpdb->get_row($wpdb->prepare(
        "SELECT * FROM $table_name 
        WHERE queue_id = %s 
        AND status IN ('pending', 'failed')
        AND attempts < max_attempts
        ORDER BY priority ASC, id ASC
        LIMIT 1",
        $queue_id
    ));
    
    if (!$next_item) {
        // No more items - queue complete
        wp_send_json_success(array(
            'complete' => true,
            'message' => 'Queue processing complete'
        ));
    }
    
    // Mark item as processing
    $wpdb->update(
        $table_name,
        array(
            'status' => 'processing',
            'started_at' => current_time('mysql'),
            'attempts' => $next_item->attempts + 1
        ),
        array('id' => $next_item->id),
        array('%s', '%s', '%d'),
        array('%d')
    );
    
    wp_send_json_success(array(
        'complete' => false,
        'item' => array(
            'id' => $next_item->id,
            'type' => $next_item->item_type,
            'data' => json_decode($next_item->item_data, true),
            'bot_id' => $next_item->bot_id,
            'attempt' => $next_item->attempts + 1
        )
    ));
}

/**
 * AJAX: Process a single queue item
 */
public function ajax_mxchat_process_queue_item() {
    // Verify nonce and permissions
    check_ajax_referer('mxchat_queue_nonce', 'nonce');
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
    }
    
    $item_id = isset($_POST['item_id']) ? absint($_POST['item_id']) : 0;
    $item_type = isset($_POST['item_type']) ? sanitize_text_field($_POST['item_type']) : '';
    $item_data = isset($_POST['item_data']) ? $_POST['item_data'] : array();
    $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
    
    if (empty($item_id) || empty($item_type)) {
        wp_send_json_error('Missing item data');
    }
    
    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_processing_queue';
    
    // Process based on item type
    try {
        set_time_limit(60); // Give processing 60 seconds
        
        $result = false;
        $error_message = '';
        
        // Read item directly from DB to get queue_id and preserve special chars in item_data
        // (POST round-trip through JS mangles characters like apostrophes in URLs)
        $db_item = $wpdb->get_row($wpdb->prepare(
            "SELECT queue_id, item_data FROM $table_name WHERE id = %d",
            $item_id
        ));
        $item_queue_id = $db_item ? $db_item->queue_id : '';
        if ($db_item && !empty($db_item->item_data)) {
            $db_data = json_decode($db_item->item_data, true);
            if (is_array($db_data)) {
                $item_data = $db_data;
            }
        }

        switch ($item_type) {
            case 'url':
                $result = $this->mxchat_process_queue_url($item_data, $bot_id, $item_queue_id);
                break;

            case 'pdf_page':
                $result = $this->mxchat_process_queue_pdf_page($item_data, $bot_id);
                break;
                
            default:
                throw new Exception('Unknown item type: ' . $item_type);
        }
        
        if (is_wp_error($result)) {
            $error_code = $result->get_error_code();
            // Content errors (empty page, sanitization) are permanent — retrying won't help
            $permanent_codes = array('empty_page', 'empty_after_sanitization', 'no_api_key', 'page_not_found');
            if (in_array($error_code, $permanent_codes)) {
                // Mark as permanently failed — set attempts = max_attempts so it won't be retried
                $current_item = $wpdb->get_row($wpdb->prepare(
                    "SELECT max_attempts FROM $table_name WHERE id = %d", $item_id
                ));
                $wpdb->update(
                    $table_name,
                    array(
                        'status' => 'failed',
                        'error_message' => $result->get_error_message(),
                        'attempts' => $current_item ? $current_item->max_attempts : 3
                    ),
                    array('id' => $item_id),
                    array('%s', '%s', '%d'),
                    array('%d')
                );
                wp_send_json_error(array(
                    'message' => $result->get_error_message(),
                    'permanent_failure' => true,
                    'item_id' => $item_id
                ));
                return;
            }
            throw new Exception($result->get_error_message());
        }

        if ($result === false) {
            throw new Exception('Processing returned false - item may be empty or invalid');
        }
        
        // Mark as completed
        $wpdb->update(
            $table_name,
            array(
                'status' => 'completed',
                'completed_at' => current_time('mysql'),
                'error_message' => null
            ),
            array('id' => $item_id),
            array('%s', '%s', '%s'),
            array('%d')
        );
        
        wp_send_json_success(array(
            'processed' => true,
            'item_id' => $item_id,
            'message' => 'Item processed successfully'
        ));
        
    } catch (Exception $e) {
        $error_message = $e->getMessage();
        
        // Get current attempt count
        $item = $wpdb->get_row($wpdb->prepare(
            "SELECT attempts, max_attempts FROM $table_name WHERE id = %d",
            $item_id
        ));
        
        // Check if we've exhausted retries
        if ($item && $item->attempts >= $item->max_attempts) {
            // Permanently failed
            $wpdb->update(
                $table_name,
                array(
                    'status' => 'failed',
                    'error_message' => $error_message
                ),
                array('id' => $item_id),
                array('%s', '%s'),
                array('%d')
            );
            
            wp_send_json_error(array(
                'message' => 'Item failed after maximum attempts: ' . $error_message,
                'permanent_failure' => true,
                'item_id' => $item_id
            ));
        } else {
            // Mark for retry
            $wpdb->update(
                $table_name,
                array(
                    'status' => 'failed',
                    'error_message' => $error_message
                ),
                array('id' => $item_id),
                array('%s', '%s'),
                array('%d')
            );
            
            wp_send_json_error(array(
                'message' => 'Item processing failed, will retry: ' . $error_message,
                'can_retry' => true,
                'item_id' => $item_id,
                'attempts' => $item ? $item->attempts : 0
            ));
        }
    }
}

/**
 * Process a URL from the queue
 */
private function mxchat_process_queue_url($item_data, $bot_id = 'default', $queue_id = '') {
    $url = isset($item_data['url']) ? $item_data['url'] : '';

    if (empty($url)) {
        return new WP_Error('invalid_url', 'URL is empty');
    }

    // Get bot-specific embedding decision early (needed for both paths) —
    // custom-provider-aware (plan cbd5fd). Error code preserved.
    $bot_options = $this->get_bot_options($bot_id);
    $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');

    $preflight = MxChat_Utils::embedding_preflight($options);
    if (!$preflight['ok']) {
        return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
    }
    $api_key = $preflight['api_key'];

    // Check if this is a WooCommerce product URL and WooCommerce is active
    $is_product_url = (strpos($url, '/product/') !== false || strpos($url, '/shop/') !== false);
    $content_type = $is_product_url ? 'product' : 'url';

    // Try to get WooCommerce product data if it's a product URL
    if ($is_product_url && class_exists('WooCommerce')) {
        $product_content = $this->mxchat_extract_woocommerce_product_content($url);

        if (!empty($product_content)) {
            // Successfully extracted WooCommerce product data with pricing
            $result = MxChat_Utils::submit_content_to_db(
                $product_content,
                $url,
                $api_key,
                null,
                $bot_id,
                'product'
            );
            return $result;
        }
        // If WooCommerce extraction failed, fall through to HTML extraction
    }

    // Fetch URL content (fallback for non-products or when WooCommerce extraction fails)
    $is_likely_pdf = (strtolower(pathinfo(parse_url($url, PHP_URL_PATH) ?: '', PATHINFO_EXTENSION)) === 'pdf');
    $response = wp_remote_get($url, array(
        'timeout' => $is_likely_pdf ? 120 : 30,
        'redirection' => 5,
        'user-agent' => mxchat_ingest_user_agent(),
    ));

    if (is_wp_error($response)) {
        return $response;
    }

    $response_code = wp_remote_retrieve_response_code($response);
    if ($response_code !== 200) {
        return new WP_Error('http_error', 'HTTP ' . $response_code . ' error for: ' . $url);
    }

    // Check if URL is a PDF — expand into per-page queue items using the standard PDF pipeline
    if ($this->mxchat_is_pdf_url($url, $response)) {
        return $this->mxchat_expand_pdf_to_queue($url, $response, $bot_id, $queue_id);
    }

    $html = wp_remote_retrieve_body($response);

    if (empty($html)) {
        return new WP_Error('empty_response', 'Empty response body');
    }

    // Extract and sanitize content
    $content = $this->mxchat_extract_main_content($html);
    $sanitized = $this->mxchat_sanitize_content_for_api($content);

    if (empty($sanitized)) {
        // Not an error - just no content found (maybe a redirect or empty page)
        return false;
    }

    // Submit to database with content_type
    $result = MxChat_Utils::submit_content_to_db(
        $sanitized,
        $url,
        $api_key,
        null,
        $bot_id,
        $content_type
    );

    return $result;
}

/**
 * Expand a PDF URL into per-page queue items using the standard PDF pipeline.
 * Called when a sitemap URL turns out to be a PDF — downloads, parses page count,
 * and adds pdf_page items to the same queue so they process with full progress tracking.
 */
private function mxchat_expand_pdf_to_queue($pdf_url, $response, $bot_id = 'default', $queue_id = '') {
    set_time_limit(120); // PDFs need extra time for download + parsing

    $upload_dir = wp_upload_dir();
    $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
    $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;

    $response_body = wp_remote_retrieve_body($response);
    if (empty($response_body)) {
        return new WP_Error('empty_pdf', 'Empty PDF response for: ' . $pdf_url);
    }

    if (!wp_mkdir_p(dirname($pdf_path))) {
        return new WP_Error('dir_error', 'Failed to create upload directory');
    }

    file_put_contents($pdf_path, $response_body);

    if (!file_exists($pdf_path)) {
        return new WP_Error('save_error', 'Failed to save PDF file');
    }

    try {
        $total_pages = $this->mxchat_validate_and_count_pdf_pages($pdf_path);

        if ($total_pages === false || $total_pages < 1) {
            wp_delete_file($pdf_path);
            return new WP_Error('no_pages', 'PDF has no pages: ' . $pdf_url);
        }

        // Build per-page items identical to mxchat_handle_pdf_for_knowledge_base
        $pages = array();
        for ($i = 1; $i <= $total_pages; $i++) {
            $pages[] = array(
                'pdf_path' => $pdf_path,
                'pdf_url'  => $pdf_url,
                'page_number' => $i,
                'total_pages' => $total_pages
            );
        }

        // Add pdf_page items to the SAME queue so the JS picks them up automatically
        if (!empty($queue_id)) {
            $queued_count = $this->mxchat_add_to_queue($queue_id, 'pdf_page', $pages, $bot_id);
        } else {
            // Fallback: create a new PDF queue (shouldn't happen in sitemap flow)
            $new_queue_id = 'pdf_' . md5($pdf_url . time());
            $queued_count = $this->mxchat_add_to_queue($new_queue_id, 'pdf_page', $pages, $bot_id);
            $this->mxchat_set_queue_meta($new_queue_id, 'source_url', $pdf_url);
            $this->mxchat_set_queue_meta($new_queue_id, 'queue_type', 'pdf');
            $this->mxchat_set_queue_meta($new_queue_id, 'total_items', $total_pages);
            $this->mxchat_set_queue_meta($new_queue_id, 'bot_id', $bot_id);
            $this->mxchat_set_queue_meta($new_queue_id, 'pdf_path', $pdf_path);
            $this->mxchat_set_queue_meta($new_queue_id, 'created_at', current_time('mysql'));
        }

        if ($queued_count === 0) {
            wp_delete_file($pdf_path);
            return new WP_Error('queue_error', 'Failed to add PDF pages to queue');
        }

        // Return true so the original URL item is marked complete
        // The new pdf_page items will be processed in subsequent batches
        return true;

    } catch (Exception $e) {
        if (file_exists($pdf_path)) {
            wp_delete_file($pdf_path);
        }
        return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
    }
}

/**
 * Legacy: Process a PDF URL inline during sitemap queue processing.
 * @deprecated Use mxchat_expand_pdf_to_queue instead — kept for reference only.
 */
private function mxchat_process_pdf_url_inline($pdf_url, $response, $api_key, $bot_id = 'default') {
    set_time_limit(120); // PDFs need more time — downloading + parsing all pages

    $upload_dir = wp_upload_dir();
    $pdf_filename = sanitize_file_name('mxchat_kb_' . md5($pdf_url) . '.pdf');
    $pdf_path = trailingslashit($upload_dir['path']) . $pdf_filename;

    $response_body = wp_remote_retrieve_body($response);
    if (empty($response_body)) {
        return new WP_Error('empty_pdf', 'Empty PDF response');
    }

    if (!wp_mkdir_p(dirname($pdf_path))) {
        return new WP_Error('dir_error', 'Failed to create upload directory');
    }

    file_put_contents($pdf_path, $response_body);

    if (!file_exists($pdf_path)) {
        return new WP_Error('save_error', 'Failed to save PDF file');
    }

    try {
        mxchat_load_pdf_parser();
        $parser = new \Smalot\PdfParser\Parser();
        $pdf = $parser->parseFile($pdf_path);
        $pages = $pdf->getPages();
        $total_pages = count($pages);

        if ($total_pages < 1) {
            wp_delete_file($pdf_path);
            return new WP_Error('no_pages', 'PDF has no pages');
        }

        $processed = 0;
        $skipped_pages = array();

        for ($i = 0; $i < $total_pages; $i++) {
            $page_num = $i + 1;
            $text = $pages[$i]->getText();
            $text = MxChat_Utils::normalize_pdf_rtl($text, 'kb_pdf_import page ' . $page_num);
            if (empty($text)) {
                $skipped_pages[] = 'Page ' . $page_num . ': No text could be extracted — page may contain only images, links, or non-standard encoding';
                continue;
            }

            $sanitized = $this->mxchat_sanitize_content_for_api($text);
            if (empty($sanitized)) {
                $skipped_pages[] = 'Page ' . $page_num . ': Text was extracted but contained only special characters, control codes, or unsupported content';
                continue;
            }

            $metadata = array(
                'document_type' => 'pdf',
                'total_pages'   => $total_pages,
                'current_page'  => $page_num,
                'source_url'    => $pdf_url,
            );

            $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
            $page_url = esc_url($pdf_url . '#page=' . $page_num);

            MxChat_Utils::submit_content_to_db(
                $content_with_metadata,
                $page_url,
                $api_key,
                null,
                $bot_id,
                'pdf'
            );

            $processed++;
        }

        // Clean up the temp PDF file
        wp_delete_file($pdf_path);

        if (!empty($skipped_pages)) {
            error_log('MxChat PDF: Skipped ' . count($skipped_pages) . ' of ' . $total_pages . ' pages: ' . implode('; ', $skipped_pages));
        }

        return $processed > 0 ? true : false;

    } catch (Exception $e) {
        if (file_exists($pdf_path)) {
            wp_delete_file($pdf_path);
        }
        return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
    }
}

/**
 * Extract WooCommerce product content including pricing
 *
 * @param string $url The product URL
 * @return string|false Product content with pricing, or false if not found
 */
private function mxchat_extract_woocommerce_product_content($url) {
    // Try to get product ID from URL
    $product_id = url_to_postid($url);

    // If url_to_postid fails, try to extract from URL pattern
    if (!$product_id) {
        $product_slug = '';

        // Handle pretty permalinks: /product/product-name/
        if (preg_match('/\/product\/([^\/\?]+)/', $url, $matches)) {
            $product_slug = $matches[1];
        }

        if (!empty($product_slug)) {
            $product_post = get_page_by_path($product_slug, OBJECT, 'product');
            if ($product_post) {
                $product_id = $product_post->ID;
            }
        }
    }

    if (!$product_id) {
        return false;
    }

    // Get WooCommerce product object
    $product = wc_get_product($product_id);

    if (!$product) {
        return false;
    }

    // Build product content via the shared WC-object assembler (a3d60c) — same
    // body as the auto-sync product writer, so the two paths can never drift.
    $content = $this->mxchat_prepare_product_content_for_indexing($product);

    return $this->mxchat_sanitize_content_for_api($content);
}

/**
 * Process a PDF page from the queue
 */
private function mxchat_process_queue_pdf_page($item_data, $bot_id = 'default') {
    $pdf_path = isset($item_data['pdf_path']) ? $item_data['pdf_path'] : '';
    $pdf_url = isset($item_data['pdf_url']) ? $item_data['pdf_url'] : '';
    $page_number = isset($item_data['page_number']) ? absint($item_data['page_number']) : 0;
    $total_pages = isset($item_data['total_pages']) ? absint($item_data['total_pages']) : 0;
    
    if (empty($pdf_path) || !file_exists($pdf_path)) {
        return new WP_Error('pdf_not_found', 'PDF file not found: ' . $pdf_path);
    }
    
    if ($page_number < 1) {
        return new WP_Error('invalid_page', 'Invalid page number');
    }
    
    try {
        mxchat_load_pdf_parser();
        $parser = new \Smalot\PdfParser\Parser();
        $pdf = $parser->parseFile($pdf_path);
        $pages = $pdf->getPages();

        if (!isset($pages[$page_number - 1])) {
            return new WP_Error('page_not_found', 'Page ' . $page_number . ' not found in PDF');
        }
        
        $text = $pages[$page_number - 1]->getText();
        $text = MxChat_Utils::normalize_pdf_rtl($text, 'kb_pdf_page page ' . $page_number);

        if (empty($text)) {
            return new WP_Error('empty_page', 'Page ' . $page_number . ': No text could be extracted — page may contain only images, links, or non-standard encoding');
        }

        $sanitized = $this->mxchat_sanitize_content_for_api($text);

        if (empty($sanitized)) {
            return new WP_Error('empty_after_sanitization', 'Page ' . $page_number . ': Text was extracted but contained only special characters, control codes, or unsupported content that was removed during cleanup');
        }
        
        // Create metadata
        $metadata = array(
            'document_type' => 'pdf',
            'total_pages' => $total_pages,
            'current_page' => $page_number,
            'source_url' => $pdf_url
        );
        
        $content_with_metadata = wp_json_encode($metadata) . "\n---\n" . $sanitized;
        $page_url = esc_url($pdf_url . "#page=" . $page_number);
        
        // Get bot-specific embedding decision — custom-provider-aware
        // (plan cbd5fd). Error code preserved.
        $bot_options = $this->get_bot_options($bot_id);
        $options = !empty($bot_options) ? $bot_options : get_option('mxchat_options');

        $preflight = MxChat_Utils::embedding_preflight($options);
        if (!$preflight['ok']) {
            return new WP_Error('no_api_key', $preflight['reason'] . ' (bot: ' . $bot_id . ')');
        }
        $api_key = $preflight['api_key'];

        // Submit to database - UPDATED 2.5.6: Added content_type 'pdf'
        $result = MxChat_Utils::submit_content_to_db(
            $content_with_metadata,
            $page_url,
            $api_key,
            null,
            $bot_id,
            'pdf'
        );

        return $result;
        
    } catch (Exception $e) {
        return new WP_Error('pdf_parse_error', 'Error parsing PDF: ' . $e->getMessage());
    }
}

/**
 * AJAX: Get queue processing status
 */
public function ajax_mxchat_get_queue_status() {
    // Verify nonce and permissions
    check_ajax_referer('mxchat_queue_nonce', 'nonce');
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
    }
    
    $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
    
    if (empty($queue_id)) {
        wp_send_json_error('Missing queue ID');
    }
    
    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_processing_queue';
    
    // Get counts by status
    $counts = $wpdb->get_results($wpdb->prepare(
        "SELECT status, COUNT(*) as count 
        FROM $table_name 
        WHERE queue_id = %s 
        GROUP BY status",
        $queue_id
    ), OBJECT_K);
    
    $total = 0;
    $completed = 0;
    $failed = 0;
    $processing = 0;
    $pending = 0;
    
    foreach ($counts as $status => $data) {
        $count = absint($data->count);
        $total += $count;
        
        switch ($status) {
            case 'completed':
                $completed = $count;
                break;
            case 'failed':
                $failed = $count;
                break;
            case 'processing':
                $processing = $count;
                break;
            case 'pending':
                $pending = $count;
                break;
        }
    }
    
    // Calculate percentage
    $percentage = $total > 0 ? round((($completed + $failed) / $total) * 100) : 0;
    
    // Get failed items details (include all failed items, not just those that exhausted retries)
    $failed_items = array();
    if ($failed > 0) {
        $failed_items = $wpdb->get_results($wpdb->prepare(
            "SELECT item_type, item_data, error_message, attempts
            FROM $table_name
            WHERE queue_id = %s
            AND status = 'failed'
            ORDER BY id DESC
            LIMIT 50",
            $queue_id
        ));
    }
    
    // Get queue metadata
    $source_url = $this->mxchat_get_queue_meta($queue_id, 'source_url');
    $queue_type = $this->mxchat_get_queue_meta($queue_id, 'queue_type');
    
    // Determine if queue is complete
    $is_complete = ($pending === 0 && $processing === 0);
    
    wp_send_json_success(array(
        'queue_id' => $queue_id,
        'queue_type' => $queue_type,
        'source_url' => $source_url,
        'total' => $total,
        'completed' => $completed,
        'failed' => $failed,
        'processing' => $processing,
        'pending' => $pending,
        'percentage' => $percentage,
        'is_complete' => $is_complete,
        'failed_items' => $failed_items,
        'status' => $is_complete ? 'complete' : 'processing'
    ));
}

/**
 * AJAX: Clear completed queue
 */
public function ajax_mxchat_clear_queue() {
    // Verify nonce and permissions
    check_ajax_referer('mxchat_queue_nonce', 'nonce');
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
    }
    
    $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
    
    if (empty($queue_id)) {
        wp_send_json_error('Missing queue ID');
    }
    
    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_processing_queue';
    $meta_table = $wpdb->prefix . 'mxchat_queue_meta';
    
    // Delete queue items
    $wpdb->delete(
        $table_name,
        array('queue_id' => $queue_id),
        array('%s')
    );
    
    // Delete queue metadata
    $wpdb->delete(
        $meta_table,
        array('queue_id' => $queue_id),
        array('%s')
    );
    
    wp_send_json_success(array(
        'message' => 'Queue cleared successfully'
    ));
}

/**
 * AJAX: Retry failed items in queue
 */
public function ajax_mxchat_retry_failed() {
    // Verify nonce and permissions
    check_ajax_referer('mxchat_queue_nonce', 'nonce');
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
    }
    
    $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
    
    if (empty($queue_id)) {
        wp_send_json_error('Missing queue ID');
    }
    
    global $wpdb;
    $table_name = $wpdb->prefix . 'mxchat_processing_queue';
    
    // Reset failed items to pending and reset attempt count
    $updated = $wpdb->update(
        $table_name,
        array(
            'status' => 'pending',
            'attempts' => 0,
            'error_message' => null
        ),
        array(
            'queue_id' => $queue_id,
            'status' => 'failed'
        ),
        array('%s', '%d', '%s'),
        array('%s', '%s')
    );
    
    wp_send_json_success(array(
        'message' => 'Reset ' . $updated . ' failed items for retry',
        'reset_count' => $updated
    ));
}


public function ajax_mxchat_mark_queue_complete() {
    check_ajax_referer('mxchat_queue_nonce', 'nonce');
    
    if (!current_user_can('manage_options')) {
        wp_send_json_error('Unauthorized access');
    }
    
    $queue_id = isset($_POST['queue_id']) ? sanitize_text_field($_POST['queue_id']) : '';
    
    if (empty($queue_id)) {
        wp_send_json_error('Missing queue ID');
    }
    
    // Clear active queue transients
    if (strpos($queue_id, 'sitemap_') === 0) {
        delete_transient('mxchat_active_queue_sitemap');
    } else if (strpos($queue_id, 'pdf_') === 0) {
        delete_transient('mxchat_active_queue_pdf');
    }
    
    wp_send_json_success(array('message' => 'Queue marked as complete'));
}

    
    // ========================================
    // STATIC ACCESS METHODS
    // ========================================
    
    /**
     * Get singleton instance
     */
    public static function get_instance() {
        static $instance = null;
        if ($instance === null) {
            $instance = new self();
        }
        return $instance;
    }
}

// Initialize the Knowledge manager
$mxchat_knowledge_manager = MxChat_Knowledge_Manager::get_instance();