← All changes | includes/class-mxchat-integrator.php +1108 -4893 3.2.3 → 2.4.0 View file →

		@@ -10,40 +10,10 @@
10	10	private $fallbackResponse;
11	11	private $productCardHtml;
12	12	private $word_handler;
13	13	private $last_similarity_analysis = null;
14		- private $current_valid_urls = [];
15		- private $last_vectorstore_error = null;
16		- private $is_streaming = false; // ADDED: Track if current request is streaming
17		- private $streaming_headers_sent = false; // Track if streaming headers have been sent
18	14
19		-/**
20		- * Setup streaming headers - call this right before actually streaming
21		- * This delays header setup to allow actions/forms to return JSON responses
22		- */
23		-private function setup_streaming_headers() {
24		- if ($this->streaming_headers_sent || headers_sent()) {
25		- return false;
26		- }
27	15
28		- // Disable output buffering
29		- while (ob_get_level()) {
30		- ob_end_flush();
31		- }
32		-
33		- // Set headers for SSE
34		- header('Content-Type: text/event-stream');
35		- header('Cache-Control: no-cache');
36		- header('Connection: keep-alive');
37		- header('X-Accel-Buffering: no');
38		-
39		- ob_implicit_flush(true);
40		- flush();
41		-
42		- $this->streaming_headers_sent = true;
43		- return true;
44		-}
45		-
46	16	/**
47	17	* Class constructor
48	18	*/
49	19	public function __construct() {
		@@ -111,28 +81,13 @@
111	81	// Add chat mode checking actions
112	82	add_action('wp_ajax_mxchat_get_current_chat_mode', array($this, 'mxchat_get_current_chat_mode'));
113	83	add_action('wp_ajax_nopriv_mxchat_get_current_chat_mode', array($this, 'mxchat_get_current_chat_mode'));
114	84
115		- // Nonce refresh for page-cache compatibility (WP Rocket, LiteSpeed, etc.)
116		- add_action('wp_ajax_mxchat_refresh_nonce', array($this, 'mxchat_refresh_nonce'));
117		- add_action('wp_ajax_nopriv_mxchat_refresh_nonce', array($this, 'mxchat_refresh_nonce'));
118		-
119		- // Auto-email transcript action
120		- add_action('mxchat_send_delayed_transcript', array($this, 'mxchat_send_delayed_transcript'), 10, 1);
121		-
122	85	add_filter('mxchat_check_actions_only', array($this, 'check_actions_for_addons'), 10, 4);
123	86
124	87
125	88	}
126	89
127		-/**
128		- * Return a fresh nonce so cached pages can replace the stale one.
129		- */
130		-public function mxchat_refresh_nonce() {
131		- nocache_headers();
132		- wp_send_json_success(array('nonce' => wp_create_nonce('mxchat_chat_nonce')));
133		-}
134		-
135	90	// In your core plugin's check_actions_for_addons method:
136	91	public function check_actions_for_addons($default, $message, $user_id, $session_id) {
137	92	//error_log('MxChat Core: check_actions_for_addons called with message: ' . $message);
138	93
		@@ -155,22 +110,8 @@
155	110	wp_die();
156	111	}
157	112
158	113	$session_id = sanitize_text_field($_POST['session_id']);
159		-
160		- // SECURITY FIX: Verify session ownership before retrieving data
161		- // If IP/user changed, signal frontend to reset session instead of blocking
162		- $current_user_identifier = MxChat_User::mxchat_get_user_identifier();
163		-
164		- // Check if this session has an owner recorded
165		- $session_owner = get_option("mxchat_session_owner_{$session_id}");
166		-
167		- // Update session owner if it changed (e.g. IP changed due to network switch)
168		- // The session ID itself is the authentication — if the client has it, they own it
169		- if (!$session_owner || $session_owner !== $current_user_identifier) {
170		- update_option("mxchat_session_owner_{$session_id}", $current_user_identifier, 'no');
171		- }
172		-
173	114	$history = get_option("mxchat_history_{$session_id}", []); // Retrieve stored history
174	115	$chat_mode = get_option("mxchat_mode_{$session_id}", 'ai'); // Get current chat mode
175	116
176	117	if (empty($history)) {
		@@ -187,25 +128,11 @@
187	128	'chat_mode' => $chat_mode
188	129	]);
189	130	wp_die();
190	131	}
191		-private function mxchat_fetch_conversation_history_for_ai($session_id, $session_start_timestamp = 0) {
	132	+
	133	+private function mxchat_fetch_conversation_history_for_ai($session_id) {
192	134	$history = get_option("mxchat_history_{$session_id}", []);
193		-
194		- // Check persistence setting - when OFF, only include messages from current page load
195		- $options = get_option('mxchat_options', []);
196		- $persistence_enabled = isset($options['chat_persistence_toggle']) && $options['chat_persistence_toggle'] === 'on';
197		-
198		- // Filter history when persistence is OFF to match what the user sees
199		- if (!$persistence_enabled && $session_start_timestamp > 0) {
200		- $history = array_filter($history, function($entry) use ($session_start_timestamp) {
201		- // Include messages from this page load onwards
202		- return isset($entry['timestamp']) && $entry['timestamp'] >= $session_start_timestamp;
203		- });
204		- // Re-index array after filtering
205		- $history = array_values($history);
206		- }
207		-
208	135	$formatted_history = [];
209	136
210	137	// Adjusted for code-heavy conversations
211	138	$max_tokens = 120000; // Context window size
		@@ -303,15 +230,8 @@
303	230	'callback' => [$this, 'handle_slack_messages'],
304	231	'permission_callback' => [$this, 'verify_slack_request'],
305	232	]);
306	233
307		- // Telegram webhook endpoint
308		- register_rest_route('mxchat/v1', '/telegram-webhook', [
309		- 'methods' => 'POST',
310		- 'callback' => [$this, 'handle_telegram_webhook'],
311		- 'permission_callback' => [$this, 'verify_telegram_request'],
312		- ]);
313		-
314	234	//error_log(esc_html__('MxChat REST routes registered', 'mxchat'));
315	235	}
316	236
317	237	/**
		@@ -351,11 +271,10 @@
351	271	//error_log(esc_html__('Slack request timestamp too old', 'mxchat'));
352	272	return false;
353	273	}
354	274
355		- // Get raw request body from the WP_REST_Request object
356		- // (php://input may already be consumed by WordPress at this point)
357		- $request_body = $request->get_body();
	275	+ // Get raw request body
	276	+ $request_body = file_get_contents('php://input');
358	277
359	278	// Create the signature base string
360	279	$sig_basestring = "v0:{$timestamp}:{$request_body}";
361	280
		@@ -364,43 +283,8 @@
364	283
365	284	// Compare signatures
366	285	return hash_equals($my_signature, $slack_signature);
367	286	}
368		-
369		-/**
370		- * Verify request is coming from Telegram.
371		- *
372		- * @param WP_REST_Request $request
373		- * @return bool True if valid, false otherwise.
374		- */
375		-public function verify_telegram_request($request) {
376		- $secret_token = $this->options['telegram_webhook_secret'] ?? '';
377		-
378		- //error_log('[MxChat Telegram DEBUG] verify_telegram_request called');
379		- //error_log('[MxChat Telegram DEBUG] Stored secret: ' . (empty($secret_token) ? 'EMPTY' : substr($secret_token, 0, 10) . '...'));
380		-
381		- if (empty($secret_token)) {
382		- // If no secret is configured, allow the request (for initial setup)
383		- //error_log('[MxChat Telegram DEBUG] No secret configured, allowing request');
384		- return true;
385		- }
386		-
387		- // Telegram sends the secret token in the X-Telegram-Bot-Api-Secret-Token header
388		- $request_token = $request->get_header('X-Telegram-Bot-Api-Secret-Token');
389		-
390		- //error_log('[MxChat Telegram DEBUG] Request token: ' . (empty($request_token) ? 'EMPTY' : substr($request_token, 0, 10) . '...'));
391		-
392		- if (empty($request_token)) {
393		- //error_log('[MxChat Telegram DEBUG] Request rejected: No token in header');
394		- return false;
395		- }
396		-
397		- // Timing-safe comparison
398		- $result = hash_equals($secret_token, $request_token);
399		- //error_log('[MxChat Telegram DEBUG] Token comparison result: ' . ($result ? 'MATCH' : 'MISMATCH'));
400		- return $result;
401		-}
402		-
403	287	public function mxchat_stream_events(WP_REST_Request $request) {
404	288	header('Content-Type: text/event-stream');
405	289	header('Cache-Control: no-cache');
406	290	header('Connection: keep-alive');
		@@ -434,9 +318,9 @@
434	318
435	319
436	320
437	321
438		-private function mxchat_save_chat_message($session_id, $role, $message, $originating_page = null, $rag_context = null) {
	322	+private function mxchat_save_chat_message($session_id, $role, $message, $originating_page = null) {
439	323	global $wpdb;
440	324	$table_name = $wpdb->prefix . 'mxchat_chat_transcripts';
441	325	//error_log("[DEBUG] mxchat_save_chat_message -> START for session_id: {$session_id}, role: {$role}");
442	326
		@@ -448,26 +332,14 @@
448	332	$session_id
449	333	));
450	334	$is_new_session = ($existing_messages == 0);
451	335
452		- // Log for debugging
	336	+ // NEW: Log for debugging
453	337	if ($is_new_session) {
454	338	//error_log("[DEBUG] This is a NEW session - first message");
455	339	}
456	340	}
457	341
458		- // SECURITY FIX: Set session ownership for new sessions
459		- if ($is_new_session && $role === 'user') {
460		- $current_user_identifier = MxChat_User::mxchat_get_user_identifier();
461		- $session_owner_key = "mxchat_session_owner_{$session_id}";
462		-
463		- // Only set ownership if not already set
464		- if (!get_option($session_owner_key)) {
465		- update_option($session_owner_key, $current_user_identifier, 'no');
466		- //error_log("[DEBUG] Set session ownership for {$session_id} to {$current_user_identifier}");
467		- }
468		- }
469		-
470	342	// 1) Extract agent name if present
471	343	$agent_name = '';
472	344	if (preg_match('/^Agent: (.*?) - /', $message, $matches)) {
473	345	$agent_name = $matches[1];
		@@ -499,9 +371,9 @@
499	371	$email_option_key = "mxchat_email_{$session_id}";
500	372	$saved_email = get_option($email_option_key);
501	373	//error_log("[DEBUG] mxchat_save_chat_message -> Checking wp_options for email_option_key: {$email_option_key}, found: {$saved_email}");
502	374
503		- // Check for a saved name in wp_options
	375	+ // NEW: Check for a saved name in wp_options
504	376	$name_option_key = "mxchat_name_{$session_id}";
505	377	$saved_name = get_option($name_option_key);
506	378	//error_log("[DEBUG] mxchat_save_chat_message -> Checking wp_options for name_option_key: {$name_option_key}, found: {$saved_name}");
507	379
		@@ -544,9 +416,9 @@
544	416	$insert_data = [
545	417	'user_id' => $user_id,
546	418	'user_identifier'=> $user_identifier,
547	419	'user_email' => $saved_email ?: $user_email,
548		- 'user_name' => $saved_name ?: '', // Add name to insert data
	420	+ 'user_name' => $saved_name ?: '', // NEW: Add name to insert data
549	421	'session_id' => $session_id,
550	422	'role' => $role,
551	423	'message' => $message,
552	424	'timestamp' => current_time('mysql', 1),
		@@ -610,17 +482,9 @@
610	482	$insert_data['originating_page_title'] = $stored_originating['title'] ?? '';
611	483	}
612	484	}
613	485	}
614		-
615		- // Add RAG context if provided (for bot messages)
616		- if ($rag_context !== null && $role === 'bot') {
617		- $rag_context_column_exists = $wpdb->get_var("SHOW COLUMNS FROM $table_name LIKE 'rag_context'");
618		- if ($rag_context_column_exists) {
619		- $insert_data['rag_context'] = is_array($rag_context) ? wp_json_encode($rag_context) : $rag_context;
620		- }
621		- }
622		-
	486	+
623	487	$wpdb->insert($table_name, $insert_data);
624	488	//error_log("[DEBUG] mxchat_save_chat_message -> Inserted message into DB. row_id: {$wpdb->insert_id}, data: " . print_r($insert_data, true));
625	489
626	490	// 9) Send notification email if this is the first user message in a new session
		@@ -631,17 +495,11 @@
631	495	'ip' => $_SERVER['REMOTE_ADDR']
632	496	));
633	497	}
634	498
635		- // 10) Schedule delayed transcript email if enabled and message is from user
636		- if ($wpdb->insert_id && $role === 'user') {
637		- $this->schedule_delayed_transcript_email($session_id);
638		- }
639		-
640	499	//error_log("[DEBUG] mxchat_save_chat_message -> END for session_id: {$session_id}");
641	500	return $message_id;
642	501	}
643		-
644	502	private function send_new_chat_notification($session_id, $user_info = array()) {
645	503	$options = get_option('mxchat_transcripts_options');
646	504
647	505	// Check if notifications are enabled
		@@ -684,200 +542,12 @@
684	542	// Send email
685	543	return wp_mail($to, $subject, $message);
686	544	}
687	545
688		-/**
689		- * Schedule delayed transcript email for a session
690		- * Reschedules if a new user message is received
691		- */
692		-private function schedule_delayed_transcript_email($session_id) {
693		- $options = get_option('mxchat_transcripts_options');
694		-
695		- // Check if auto-email is enabled
696		- if (empty($options['mxchat_auto_email_transcript_enabled'])) {
697		- return;
698		- }
699		-
700		- // Get notification email
701		- $email = !empty($options['mxchat_notification_email']) ?
702		- $options['mxchat_notification_email'] :
703		- get_option('admin_email');
704		-
705		- if (!is_email($email)) {
706		- return;
707		- }
708		-
709		- // Get delay in minutes (default 30)
710		- $delay_minutes = isset($options['mxchat_auto_email_transcript_delay']) ?
711		- intval($options['mxchat_auto_email_transcript_delay']) : 30;
712		-
713		- // Clear any existing scheduled event for this session
714		- $hook = 'mxchat_send_delayed_transcript';
715		- $args = array($session_id);
716		- $timestamp = wp_next_scheduled($hook, $args);
717		-
718		- if ($timestamp) {
719		- wp_unschedule_event($timestamp, $hook, $args);
720		- }
721		-
722		- // Schedule new event
723		- $schedule_time = time() + ($delay_minutes * 60);
724		- wp_schedule_single_event($schedule_time, $hook, $args);
725		-}
726		-
727		-/**
728		- * Check if chat messages contain contact information (email or phone number)
729		- *
730		- * @param array $messages Array of message objects with 'message' property
731		- * @param object|null $session_data Session data object with user_email property
732		- * @return bool True if contact info found, false otherwise
733		- */
734		-private function chat_contains_contact_info($messages, $session_data = null) {
735		- // Check if session already has a stored email
736		- if ($session_data && !empty($session_data->user_email)) {
737		- return true;
738		- }
739		-
740		- // Email regex pattern
741		- $email_pattern = '/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/';
742		-
743		- // Phone number patterns (covers various formats including international, WhatsApp style)
744		- // Matches: +1234567890, (123) 456-7890, 123-456-7890, 123.456.7890, 1234567890, +1 234 567 8900, etc.
745		- $phone_pattern = '/(?:\+?\d{1,3}[-.\s]?)?\(?\d{2,4}\)?[-.\s]?\d{2,4}[-.\s]?\d{2,4}(?:[-.\s]?\d{1,4})?/';
746		-
747		- // Only check user messages (not assistant responses)
748		- foreach ($messages as $msg) {
749		- if ($msg->role !== 'user') {
750		- continue;
751		- }
752		-
753		- $message_text = $msg->message;
754		-
755		- // Check for email
756		- if (preg_match($email_pattern, $message_text)) {
757		- return true;
758		- }
759		-
760		- // Check for phone number (must be at least 7 digits total to avoid false positives)
761		- if (preg_match($phone_pattern, $message_text, $matches)) {
762		- // Count actual digits to avoid matching short numbers
763		- $digits_only = preg_replace('/\D/', '', $matches[0]);
764		- if (strlen($digits_only) >= 7) {
765		- return true;
766		- }
767		- }
768		- }
769		-
770		- return false;
771		-}
772		-
773		-/**
774		- * Send the delayed transcript email with .txt attachment
775		- */
776		-public function mxchat_send_delayed_transcript($session_id) {
777		- global $wpdb;
778		-
779		- $options = get_option('mxchat_transcripts_options');
780		-
781		- // Get notification email
782		- $to = !empty($options['mxchat_notification_email']) ?
783		- $options['mxchat_notification_email'] :
784		- get_option('admin_email');
785		-
786		- if (!is_email($to)) {
787		- return false;
788		- }
789		-
790		- // Get all messages for this session
791		- $table_name = $wpdb->prefix . 'mxchat_chat_transcripts';
792		- $messages = $wpdb->get_results($wpdb->prepare(
793		- "SELECT role, message, timestamp FROM {$table_name}
794		- WHERE session_id = %s
795		- ORDER BY timestamp ASC",
796		- $session_id
797		- ));
798		-
799		- if (empty($messages)) {
800		- return false;
801		- }
802		-
803		- // Get session metadata
804		- $sessions_table = $wpdb->prefix . 'mxchat_sessions';
805		- $session_data = $wpdb->get_row($wpdb->prepare(
806		- "SELECT * FROM {$sessions_table} WHERE session_id = %s",
807		- $session_id
808		- ));
809		-
810		- // Check if contact info is required and if it's present
811		- $require_contact = !empty($options['mxchat_auto_email_transcript_require_contact']);
812		- if ($require_contact && !$this->chat_contains_contact_info($messages, $session_data)) {
813		- // Contact info required but not found - skip sending
814		- return false;
815		- }
816		-
817		- // Build transcript content
818		- $transcript_content = "Chat Transcript\n";
819		- $transcript_content .= "================\n\n";
820		- $transcript_content .= "Session ID: " . $session_id . "\n";
821		-
822		- if ($session_data) {
823		- $transcript_content .= "User: " . ($session_data->user_identifier ?: 'Guest') . "\n";
824		- $transcript_content .= "Email: " . ($session_data->user_email ?: 'Not provided') . "\n";
825		- $transcript_content .= "Started: " . $session_data->created_at . "\n";
826		- }
827		-
828		- $transcript_content .= "\n" . str_repeat("=", 50) . "\n\n";
829		-
830		- // Add messages
831		- foreach ($messages as $msg) {
832		- $role_label = ($msg->role === 'user') ? 'User' : 'Assistant';
833		- $transcript_content .= "[{$msg->timestamp}] {$role_label}:\n";
834		- $transcript_content .= $msg->message . "\n\n";
835		- }
836		-
837		- // Create temporary file for attachment using WP_Filesystem
838		- $upload_dir = wp_upload_dir();
839		- $temp_file = $upload_dir['basedir'] . '/mxchat-transcript-' . $session_id . '.txt';
840		- global $wp_filesystem;
841		- if (empty($wp_filesystem)) {
842		- require_once ABSPATH . 'wp-admin/includes/file.php';
843		- WP_Filesystem();
844		- }
845		- $wp_filesystem->put_contents($temp_file, $transcript_content, FS_CHMOD_FILE);
846		-
847		- // Prepare email
848		- $subject = sprintf('[%s] Chat Transcript - Session %s', get_bloginfo('name'), substr($session_id, 0, 8));
849		-
850		- $message = "Please find attached the full chat transcript.\n\n";
851		- $message .= "Session ID: {$session_id}\n";
852		-
853		- if ($session_data) {
854		- $message .= "User: " . ($session_data->user_identifier ?: 'Guest') . "\n";
855		- $message .= "Email: " . ($session_data->user_email ?: 'Not provided') . "\n";
856		- }
857		-
858		- $message .= "\nView online: " . admin_url('admin.php?page=mxchat-transcripts');
859		-
860		- // Send email with attachment
861		- $attachments = array($temp_file);
862		- $result = wp_mail($to, $subject, $message, '', $attachments);
863		-
864		- // Clean up temporary file
865		- if (file_exists($temp_file)) {
866		- unlink($temp_file);
867		- }
868		-
869		- return $result;
870		-}
871		-
872		-
873		-
874	546	public function mxchat_handle_save_email_and_response() {
875	547	//error_log('[DEBUG] ---------- mxchat_handle_save_email_and_response START ----------');
876	548	//error_log('DEBUG: POST data: ' . print_r($_POST, true));
877	549
878		- nocache_headers();
879		-
880	550	// Validate nonce
881	551	if (!isset($_POST['nonce']) || !wp_verify_nonce($_POST['nonce'], 'mxchat_chat_nonce')) {
882	552	//error_log(esc_html__('[ERROR] Invalid nonce in mxchat_handle_save_email_and_response', 'mxchat'));
883	553	wp_send_json_error(['message' => esc_html__('Invalid nonce.', 'mxchat')]);
		@@ -889,15 +559,15 @@
889	559	$name = isset($_POST['name']) ? sanitize_text_field($_POST['name']) : '';
890	560
891	561	//error_log("[DEBUG] handle_save_email_and_response -> session_id: {$session_id}, email: {$email}, name: {$name}");
892	562
893		- if (empty($session_id) || $session_id === 'null' || empty($email)) {
	563	+ if (empty($session_id) || empty($email)) {
894	564	//error_log("[ERROR] Missing session_id or email: session_id={$session_id}, email={$email}");
895	565	wp_send_json_error(['message' => esc_html__('Session ID or email is missing.', 'mxchat')]);
896	566	wp_die();
897	567	}
898	568
899		- // Validate name if provided (check if name field is enabled and name is required)
	569	+ // NEW: Validate name if provided (check if name field is enabled and name is required)
900	570	$options = get_option('mxchat_options', []);
901	571	$name_field_enabled = isset($options['enable_name_field']) &&
902	572	($options['enable_name_field'] === '1' || $options['enable_name_field'] === 'on');
903	573
		@@ -908,15 +578,15 @@
908	578	}
909	579
910	580	// 1) Always store email in wp_options
911	581	$email_option_key = "mxchat_email_{$session_id}";
912		- update_option($email_option_key, $email, 'no');
	582	+ update_option($email_option_key, $email);
913	583	//error_log("[DEBUG] handle_save_email_and_response -> updated option: {$email_option_key} => {$email}");
914	584
915		- // Store name in wp_options if provided
	585	+ // NEW: Store name in wp_options if provided
916	586	if (!empty($name)) {
917	587	$name_option_key = "mxchat_name_{$session_id}";
918		- update_option($name_option_key, $name, 'no');
	588	+ update_option($name_option_key, $name);
919	589	//error_log("[DEBUG] handle_save_email_and_response -> updated option: {$name_option_key} => {$name}");
920	590	}
921	591
922	592	// 2) (Optional) Also store in DB if a row already exists
		@@ -929,9 +599,9 @@
929	599
930	600	//error_log("[DEBUG] handle_save_email_and_response -> session_count for {$session_id}: {$session_count} (SQL: {$sql})");
931	601
932	602	if ($session_count) {
933		- // Update both user_email and user_name if row(s) exist
	603	+ // NEW: Update both user_email and user_name if row(s) exist
934	604	if (!empty($name)) {
935	605	$update_sql = $wpdb->prepare(
936	606	"UPDATE {$table_name} SET user_email = %s, user_name = %s WHERE session_id = %s",
937	607	$email,
		@@ -960,10 +630,8 @@
960	630
961	631	public function mxchat_check_email_provided() {
962	632	//error_log('[DEBUG] ---------- mxchat_check_email_provided START ----------');
963	633
964		- nocache_headers();
965		-
966	634	if (!isset($_POST['nonce']) || !wp_verify_nonce($_POST['nonce'], 'mxchat_chat_nonce')) {
967	635	//error_log('[ERROR] Invalid nonce in mxchat_check_email_provided');
968	636	wp_send_json_error(['message' => esc_html__('Invalid nonce', 'mxchat')]);
969	637	}
		@@ -968,9 +636,9 @@
968	636	wp_send_json_error(['message' => esc_html__('Invalid nonce', 'mxchat')]);
969	637	}
970	638
971	639	$session_id = isset($_POST['session_id']) ? sanitize_text_field($_POST['session_id']) : '';
972		- if (empty($session_id) || $session_id === 'null') {
	640	+ if (empty($session_id)) {
973	641	//error_log('[ERROR] No session ID provided in mxchat_check_email_provided');
974	642	wp_send_json_error(['message' => esc_html__('No session ID provided', 'mxchat')]);
975	643	}
976	644
		@@ -978,9 +646,9 @@
978	646	if (is_user_logged_in()) {
979	647	$current_user = wp_get_current_user();
980	648	//error_log("[DEBUG] User is logged in as {$current_user->user_email}");
981	649
982		- // Get user's display name for logged in users
	650	+ // NEW: Get user's display name for logged in users
983	651	$user_name = !empty($current_user->display_name) ? $current_user->display_name :
984	652	(!empty($current_user->first_name) ? $current_user->first_name : '');
985	653
986	654	$response_data = ['logged_in' => true, 'email' => $current_user->user_email];
		@@ -990,9 +658,9 @@
990	658
991	659	wp_send_json_success($response_data);
992	660	}
993	661
994		- // Check if name field is required
	662	+ // NEW: Check if name field is required
995	663	$options = get_option('mxchat_options', []);
996	664	$name_field_enabled = isset($options['enable_name_field']) &&
997	665	($options['enable_name_field'] === '1' || $options['enable_name_field'] === 'on');
998	666
		@@ -998,9 +666,9 @@
998	666
999	667	$email_option_key = "mxchat_email_{$session_id}";
1000	668	$stored_email = get_option($email_option_key, '');
1001	669
1002		- // Check for stored name
	670	+ // NEW: Check for stored name
1003	671	$name_option_key = "mxchat_name_{$session_id}";
1004	672	$stored_name = get_option($name_option_key, '');
1005	673
1006	674	//error_log("[DEBUG] mxchat_check_email_provided -> Checking email option: {$email_option_key}, found: {$stored_email}");
		@@ -1005,9 +673,9 @@
1005	673
1006	674	//error_log("[DEBUG] mxchat_check_email_provided -> Checking email option: {$email_option_key}, found: {$stored_email}");
1007	675	//error_log("[DEBUG] mxchat_check_email_provided -> Checking name option: {$name_option_key}, found: {$stored_name}, required: " . ($name_field_enabled ? 'yes' : 'no'));
1008	676
1009		- // Check if we have email and name (if name is required)
	677	+ // NEW: Check if we have email and name (if name is required)
1010	678	$has_required_info = !empty($stored_email);
1011	679
1012	680	if ($name_field_enabled) {
1013	681	$has_required_info = $has_required_info && !empty($stored_name);
		@@ -1027,59 +695,32 @@
1027	695	wp_send_json_error(['message' => esc_html__('No email found', 'mxchat')]);
1028	696	}
1029	697	}
1030	698
1031		-/**
1032		- * Send error response in appropriate format based on streaming mode
1033		- * ADDED: Helper method to consistently handle errors in both streaming and non-streaming modes
1034		- *
1035		- * @param string $error_message The error message to display
1036		- * @param string $error_code Optional error code for debugging
1037		- */
1038		-private function send_error_response($error_message, $error_code = 'api_error') {
1039		- if ($this->is_streaming) {
1040		- echo "data: " . json_encode([
1041		- 'error' => true,
1042		- 'error_message' => $error_message,
1043		- 'error_code' => $error_code,
1044		- 'text' => $error_message,
1045		- 'message' => $error_message
1046		- ]) . "\n\n";
1047		- echo "data: [DONE]\n\n";
1048		- flush();
1049		- } else {
1050		- wp_send_json_error([
1051		- 'error_message' => $error_message,
1052		- 'error_code' => $error_code
1053		- ]);
1054		- }
1055		- wp_die();
1056		-}
1057		-
1058	699	public function mxchat_handle_chat_request() {
1059	700	global $wpdb;
1060	701
1061		- // Debug: Log incoming bot_id
1062		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
1063		- //error_log("=== MXCHAT DEBUG: Starting chat request ===");
1064		- //error_log("MXCHAT DEBUG: Bot ID received: " . $bot_id);
	702	+ // NEW: Check if this is a streaming request
	703	+ $is_streaming = isset($_POST['action']) && $_POST['action'] === 'mxchat_stream_chat';
1065	704
1066		- // Get bot-specific options
1067		- $bot_options = $this->get_bot_options($bot_id);
1068		- $current_options = !empty($bot_options) ? $bot_options : $this->options;
	705	+ // NEW: Set streaming headers if needed
	706	+ if ($is_streaming) {
	707	+ // Disable output buffering
	708	+ while (ob_get_level()) {
	709	+ ob_end_flush(); // Changed from ob_end_clean()
	710	+ }
	711	+
	712	+ // Set headers for SSE
	713	+ header('Content-Type: text/event-stream');
	714	+ header('Cache-Control: no-cache');
	715	+ header('Connection: keep-alive');
	716	+ header('X-Accel-Buffering: no');
	717	+
	718	+ // Add these new lines:
	719	+ ob_implicit_flush(true);
	720	+ flush();
	721	+ }
1069	722
1070		- // Check if this is a streaming request
1071		- // Allow force_streaming_test parameter to bypass the setting check (for admin compatibility testing)
1072		- $force_streaming_test = isset($_POST['force_streaming_test']) && $_POST['force_streaming_test'] === '1' && current_user_can('administrator');
1073		- $is_streaming = isset($_POST['action']) && $_POST['action'] === 'mxchat_stream_chat' &&
1074		- ($force_streaming_test || (isset($current_options['enable_streaming_toggle']) && $current_options['enable_streaming_toggle'] === 'on'));
1075		-
1076		- // ADDED: Store streaming state in class property for use in private methods
1077		- $this->is_streaming = $is_streaming;
1078		-
1079		- // NOTE: Streaming headers are now set later via setup_streaming_headers()
1080		- // This allows actions/forms to return JSON responses without header conflicts
1081		-
1082	723	// Check if MX Chat Moderation is active
1083	724	if (class_exists('MX_Chat_Moderation')) {
1084	725	// Get user email and IP
1085	726	$user_email = '';
		@@ -1144,31 +785,13 @@
1144	785
1145	786	// Rest of your existing code...
1146	787	$session_id = isset($_POST['session_id']) ? sanitize_text_field($_POST['session_id']) : '';
1147	788
1148		- // Treat the literal strings 'null' / 'undefined' as missing too. Browser edge cases
1149		- // (Safari ITP, private mode, cross-origin iframes with partitioned storage) can cause
1150		- // the frontend FormData.append() to stringify a null session_id into the literal
1151		- // "null", which would otherwise pass empty() and pollute the transcripts table with
1152		- // ghost sessions that group every visitor's first message under one row.
1153		- if ($session_id === 'null' || $session_id === 'undefined') {
1154		- $session_id = '';
1155		- }
1156		-
1157	789	if (empty($session_id)) {
1158	790	wp_send_json_error(esc_html__('Session ID is missing.', 'mxchat'));
1159	791	wp_die();
1160	792	}
1161	793
1162		- // Update session owner if it changed (e.g. IP changed due to network switch)
1163		- // The session ID itself is the authentication — if the client has it, they own it
1164		- $current_user_identifier = MxChat_User::mxchat_get_user_identifier();
1165		- $session_owner = get_option("mxchat_session_owner_{$session_id}");
1166		-
1167		- if (!$session_owner || $session_owner !== $current_user_identifier) {
1168		- update_option("mxchat_session_owner_{$session_id}", $current_user_identifier, 'no');
1169		- }
1170		-
1171	794	// Validate and sanitize the incoming message
1172	795	if (empty($_POST['message'])) {
1173	796	wp_send_json_error(esc_html__('No message received.', 'mxchat'));
1174	797	wp_die();
		@@ -1174,191 +797,209 @@
1174	797	wp_die();
1175	798	}
1176	799
1177	800
1178		- // Track originating page for first message in session
1179		- $table_name = $wpdb->prefix . 'mxchat_chat_transcripts';
	801	+ // NEW: Track originating page for first message in session
	802	+$table_name = $wpdb->prefix . 'mxchat_chat_transcripts';
1180	803
1181		- // Check if originating page columns exist
1182		- $columns_exist = $wpdb->get_var("SHOW COLUMNS FROM $table_name LIKE 'originating_page_url'");
	804	+// Check if originating page columns exist
	805	+$columns_exist = $wpdb->get_var("SHOW COLUMNS FROM $table_name LIKE 'originating_page_url'");
1183	806
1184		- if ($columns_exist) {
1185		- // Check if this session already has messages
1186		- $message_count = $wpdb->get_var($wpdb->prepare(
1187		- "SELECT COUNT(*) FROM $table_name WHERE session_id = %s",
1188		- $session_id
1189		- ));
	807	+if ($columns_exist) {
	808	+ // Check if this session already has messages
	809	+ $message_count = $wpdb->get_var($wpdb->prepare(
	810	+ "SELECT COUNT(*) FROM $table_name WHERE session_id = %s",
	811	+ $session_id
	812	+ ));
	813	+
	814	+ // If this is the first message in the session
	815	+ if ($message_count == 0) {
	816	+ // Get originating page from JavaScript (preferred) or HTTP_REFERER (fallback)
	817	+ $originating_url = '';
	818	+ $originating_title = '';
1190	819
1191		- // If this is the first message in the session
1192		- if ($message_count == 0) {
1193		- // Get originating page from JavaScript (preferred) or HTTP_REFERER (fallback)
1194		- $originating_url = '';
1195		- $originating_title = '';
	820	+ // Try to get from POST data first (sent by JavaScript)
	821	+ if (isset($_POST['current_page_url'])) {
	822	+ $originating_url = esc_url_raw($_POST['current_page_url']);
	823	+ $originating_title = isset($_POST['current_page_title'])
	824	+ ? sanitize_text_field($_POST['current_page_title'])
	825	+ : '';
	826	+ }
	827	+ // Fallback to HTTP_REFERER if not provided by JavaScript
	828	+ else if (isset($_SERVER['HTTP_REFERER'])) {
	829	+ $originating_url = esc_url_raw($_SERVER['HTTP_REFERER']);
	830	+ }
	831	+
	832	+ // Generate title if we have URL but no title
	833	+ if ($originating_url && empty($originating_title)) {
	834	+ $parsed_url = parse_url($originating_url);
	835	+ $path = isset($parsed_url['path']) ? trim($parsed_url['path'], '/') : '';
1196	836
1197		- // Try to get from POST data first (sent by JavaScript)
1198		- if (isset($_POST['current_page_url'])) {
1199		- $originating_url = esc_url_raw($_POST['current_page_url']);
1200		- $originating_title = isset($_POST['current_page_title'])
1201		- ? sanitize_text_field($_POST['current_page_title'])
1202		- : '';
	837	+ if (empty($path) || $path === 'index.php' || $path === 'index.html') {
	838	+ $originating_title = 'Homepage';
	839	+ } else {
	840	+ // Clean up the path to make a readable title
	841	+ $originating_title = str_replace(['-', '_', '/', '.php', '.html'], ' ', $path);
	842	+ $originating_title = ucwords(trim($originating_title));
1203	843	}
1204		- // Fallback to HTTP_REFERER if not provided by JavaScript
1205		- else if (isset($_SERVER['HTTP_REFERER'])) {
1206		- $originating_url = esc_url_raw($_SERVER['HTTP_REFERER']);
1207		- }
1208		-
1209		- // Generate title if we have URL but no title
1210		- if ($originating_url && empty($originating_title)) {
1211		- $parsed_url = parse_url($originating_url);
1212		- $path = isset($parsed_url['path']) ? trim($parsed_url['path'], '/') : '';
1213		-
1214		- if (empty($path) || $path === 'index.php' || $path === 'index.html') {
1215		- $originating_title = 'Homepage';
1216		- } else {
1217		- // Clean up the path to make a readable title
1218		- $originating_title = str_replace(['-', '_', '/', '.php', '.html'], ' ', $path);
1219		- $originating_title = ucwords(trim($originating_title));
1220		- }
1221		- }
1222		-
1223		- // Store for later use when saving the message
1224		- $this->pending_originating_page = [
1225		- 'url' => $originating_url,
1226		- 'title' => $originating_title
1227		- ];
1228	844	}
	845	+
	846	+ // Store for later use when saving the message
	847	+ $this->pending_originating_page = [
	848	+ 'url' => $originating_url,
	849	+ 'title' => $originating_title
	850	+ ];
1229	851	}
	852	+}
	853	+
	854	+
	855	+
	856	+ // NEW: Get page context if provided
	857	+ $page_context = null;
	858	+ if (isset($_POST['page_context']) && !empty($_POST['page_context'])) {
	859	+ $page_context_raw = stripslashes($_POST['page_context']);
	860	+ $page_context = json_decode($page_context_raw, true);
1230	861
1231		-
1232		-
1233		- // Get page context if provided
1234		- $page_context = null;
1235		- if (isset($_POST['page_context']) && !empty($_POST['page_context'])) {
1236		- $page_context_raw = stripslashes($_POST['page_context']);
1237		- $page_context = json_decode($page_context_raw, true);
	862	+ // Validate page context structure
	863	+ if (is_array($page_context) &&
	864	+ isset($page_context['url']) &&
	865	+ isset($page_context['title']) &&
	866	+ isset($page_context['content'])) {
1238	867
1239		- // Validate page context structure
1240		- if (is_array($page_context) &&
1241		- isset($page_context['url']) &&
1242		- isset($page_context['title']) &&
1243		- isset($page_context['content'])) {
1244		-
1245		- // Sanitize page context
1246		- $page_context['url'] = esc_url_raw($page_context['url']);
1247		- $page_context['title'] = sanitize_text_field($page_context['title']);
1248		- $page_context['content'] = wp_kses_post($page_context['content']);
1249		- } else {
1250		- $page_context = null;
1251		- }
	868	+ // Sanitize page context
	869	+ $page_context['url'] = esc_url_raw($page_context['url']);
	870	+ $page_context['title'] = sanitize_text_field($page_context['title']);
	871	+ $page_context['content'] = wp_kses_post($page_context['content']);
	872	+ } else {
	873	+ $page_context = null;
1252	874	}
	875	+ }
1253	876
1254		- // Modify the message sanitization to preserve PHP tags in code blocks
1255		- $allowed_tags = [
1256		- 'pre' => [],
1257		- 'code' => ['class' => true],
1258		- 'span' => ['class' => true],
1259		- 'div' => ['class' => true],
1260		- ];
	877	+ // Modify the message sanitization to preserve PHP tags in code blocks
	878	+ $allowed_tags = [
	879	+ 'pre' => [],
	880	+ 'code' => ['class' => true],
	881	+ 'span' => ['class' => true],
	882	+ 'div' => ['class' => true],
	883	+ ];
1261	884
1262		- // First preserve code blocks
1263		- $message = preg_replace_callback('/<pre><code.*?>.*?<\/code><\/pre>/s', function($matches) {
1264		- return htmlspecialchars_decode($matches[0]);
1265		- }, $_POST['message']);
	885	+ // First preserve code blocks
	886	+ $message = preg_replace_callback('/<pre><code.*?>.*?<\/code><\/pre>/s', function($matches) {
	887	+ return htmlspecialchars_decode($matches[0]);
	888	+ }, $_POST['message']);
1266	889
1267		- // Then apply sanitization
1268		- $message = wp_kses($message, $allowed_tags);
	890	+ // Then apply sanitization
	891	+ $message = wp_kses($message, $allowed_tags);
1269	892
1270		- // Preserve code blocks from markdown conversion
1271		- $message = preg_replace('/```(\w+)?\s*([\s\S]+?)```/s', '<pre><code class="$1">$2</code></pre>', $message);
1272		- $message = apply_filters('mxchat_filter_message', $message, 'prompt', $session_id);
	893	+ // Preserve code blocks from markdown conversion
	894	+ $message = preg_replace('/```(\w+)?\s*([\s\S]+?)```/s', '<pre><code class="$1">$2</code></pre>', $message);
	895	+ $message = apply_filters('mxchat_filter_message', $message, 'prompt', $session_id);
1273	896
1274		- // ===== SIMPLIFIED TESTING PANEL INITIALIZATION =====
1275		- // Always initialize testing data for admins (no toggle needed)
1276		- $testing_data = null;
1277		- if (current_user_can('administrator')) {
1278		- // For vision messages, use the original user message for the query display
1279		- $query_for_testing = $message;
1280		- if (isset($_POST['vision_processed']) && $_POST['vision_processed'] && isset($_POST['original_user_message'])) {
1281		- $query_for_testing = sanitize_textarea_field($_POST['original_user_message']);
1282		- }
1283		-
1284		- $testing_data = [
1285		- 'query' => $query_for_testing,
1286		- 'timestamp' => time(),
1287		- 'top_matches' => [],
1288		- 'action_matches' => [], // Initialize action matches array
1289		- 'page_context' => $page_context, // Include page context in testing data
1290		- 'is_vision' => isset($_POST['vision_processed']) && $_POST['vision_processed'],
1291		- 'bot_id' => $bot_id // Include bot ID in testing data
1292		- ];
1293		-
1294		- // Get similarity threshold from bot options or default options
1295		- $similarity_threshold = isset($current_options['similarity_threshold'])
1296		- ? ((int) $current_options['similarity_threshold']) / 100
1297		- : 0.35;
1298		-
1299		- $testing_data['similarity_threshold'] = $similarity_threshold;
1300		-
1301		- // Determine knowledge base type using bot-specific config
1302		- $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
1303		- $use_pinecone = isset($bot_pinecone_config['use_pinecone']) ? $bot_pinecone_config['use_pinecone'] : false;
1304		- $testing_data['knowledge_base_type'] = $use_pinecone ? 'Pinecone' : 'WordPress Database';
	897	+// ===== SIMPLIFIED TESTING PANEL INITIALIZATION =====
	898	+ // Always initialize testing data for admins (no toggle needed)
	899	+ $testing_data = null;
	900	+ if (current_user_can('administrator')) {
	901	+ // For vision messages, use the original user message for the query display
	902	+ $query_for_testing = $message;
	903	+ if (isset($_POST['vision_processed']) && $_POST['vision_processed'] && isset($_POST['original_user_message'])) {
	904	+ $query_for_testing = sanitize_textarea_field($_POST['original_user_message']);
1305	905	}
1306		- // ===== END SIMPLIFIED TESTING INITIALIZATION =====
	906	+
	907	+ $testing_data = [
	908	+ 'query' => $query_for_testing,
	909	+ 'timestamp' => time(),
	910	+ 'top_matches' => [],
	911	+ 'action_matches' => [], // NEW: Initialize action matches array
	912	+ 'page_context' => $page_context, // NEW: Include page context in testing data
	913	+ 'is_vision' => isset($_POST['vision_processed']) && $_POST['vision_processed']
	914	+ ];
	915	+
	916	+ // Get similarity threshold
	917	+ $similarity_threshold = isset($this->options['similarity_threshold'])
	918	+ ? ((int) $this->options['similarity_threshold']) / 100
	919	+ : 0.75;
	920	+
	921	+ $testing_data['similarity_threshold'] = $similarity_threshold;
	922	+
	923	+ // Determine knowledge base type
	924	+ $addon_options = get_option('mxchat_pinecone_addon_options', array());
	925	+ $use_pinecone = (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1');
	926	+ $testing_data['knowledge_base_type'] = $use_pinecone ? 'Pinecone' : 'WordPress Database';
	927	+ }
	928	+ // ===== END SIMPLIFIED TESTING INITIALIZATION =====
1307	929
1308		- // Add debug before and after:
1309		- //error_log('MxChat Core: About to call mxchat_pre_process_message filter with message: ' . $message);
1310		- $pre_processed_result = apply_filters('mxchat_pre_process_message', $message, $user_id, $session_id);
1311		- //error_log('MxChat Core: Filter returned: ' . (is_array($pre_processed_result) ? 'array' : $pre_processed_result));
	930	+// Add debug before and after:
	931	+//error_log('MxChat Core: About to call mxchat_pre_process_message filter with message: ' . $message);
	932	+$pre_processed_result = apply_filters('mxchat_pre_process_message', $message, $user_id, $session_id);
	933	+//error_log('MxChat Core: Filter returned: ' . (is_array($pre_processed_result) ? 'array' : $pre_processed_result));
1312	934
1313	935
1314		- // If the pre-processing returned a result (not the original message), use it directly
1315		- if (is_array($pre_processed_result) && isset($pre_processed_result['text'])) {
1316		- // Save the AI response
1317		- $this->mxchat_save_chat_message($session_id, 'bot', $pre_processed_result['text']);
1318		-
1319		- // Save HTML content if provided
1320		- if (!empty($pre_processed_result['html'])) {
1321		- $this->mxchat_save_chat_message($session_id, 'bot', $pre_processed_result['html']);
1322		- }
1323		-
1324		- // Add testing data if admin
1325		- $response_data = [
1326		- 'text' => $pre_processed_result['text'],
1327		- 'html' => $pre_processed_result['html'] ?? '',
1328		- 'session_id' => $session_id
1329		- ];
1330		-
1331		- if ($testing_data !== null) {
1332		- $response_data['testing_data'] = $testing_data;
1333		- }
1334		-
1335		- wp_send_json($response_data);
1336		- wp_die();
	936	+ // If the pre-processing returned a result (not the original message), use it directly
	937	+ if (is_array($pre_processed_result) && isset($pre_processed_result['text'])) {
	938	+ // Save the AI response
	939	+ $this->mxchat_save_chat_message($session_id, 'bot', $pre_processed_result['text']);
	940	+
	941	+ // Save HTML content if provided
	942	+ if (!empty($pre_processed_result['html'])) {
	943	+ $this->mxchat_save_chat_message($session_id, 'bot', $pre_processed_result['html']);
1337	944	}
	945	+
	946	+ // Add testing data if admin
	947	+ $response_data = [
	948	+ 'text' => $pre_processed_result['text'],
	949	+ 'html' => $pre_processed_result['html'] ?? '',
	950	+ 'session_id' => $session_id
	951	+ ];
	952	+
	953	+ if ($testing_data !== null) {
	954	+ $response_data['testing_data'] = $testing_data;
	955	+ }
	956	+
	957	+ wp_send_json($response_data);
	958	+ wp_die();
	959	+ }
1338	960
1339		- // Save the user's message - handle vision processed messages differently
1340		- if (isset($_POST['vision_processed']) && $_POST['vision_processed'] && isset($_POST['original_user_message'])) {
1341		- // For vision messages, save the original user message with image indicator
1342		- $original_message = sanitize_textarea_field($_POST['original_user_message']);
1343		- if (isset($_POST['vision_images_count']) && $_POST['vision_images_count'] > 0) {
1344		- $image_count = intval($_POST['vision_images_count']);
1345		- $original_message .= " [{$image_count} image(s)]";
1346		- }
1347		- $this->mxchat_save_chat_message($session_id, 'user', $original_message);
1348		- } else {
1349		- // Regular message - save as normal
1350		- $this->mxchat_save_chat_message($session_id, 'user', $message);
	961	+ // Save the user's message - handle vision processed messages differently
	962	+ if (isset($_POST['vision_processed']) && $_POST['vision_processed'] && isset($_POST['original_user_message'])) {
	963	+ // For vision messages, save the original user message with image indicator
	964	+ $original_message = sanitize_textarea_field($_POST['original_user_message']);
	965	+ if (isset($_POST['vision_images_count']) && $_POST['vision_images_count'] > 0) {
	966	+ $image_count = intval($_POST['vision_images_count']);
	967	+ $original_message .= " [{$image_count} image(s)]";
1351	968	}
	969	+ $this->mxchat_save_chat_message($session_id, 'user', $original_message);
	970	+ } else {
	971	+ // Regular message - save as normal
	972	+ $this->mxchat_save_chat_message($session_id, 'user', $message);
	973	+ }
1352	974
	975	+
	976	+if (is_email($message)) {
	977	+ // Add the email to Loops
	978	+ $this->add_email_to_loops($message);
1353	979
1354		- if (is_email($message)) {
1355		- // Add the email to Loops
1356		- $this->add_email_to_loops($message);
	980	+ // Get the user's success message instruction
	981	+ $user_success_message = $this->options['email_capture_response'] ?? __('Thank you for providing your email! You\'ve been added to our list.', 'mxchat');
	982	+
	983	+ // Set instruction for AI using the user's success message
	984	+ $this->current_action_instruction = $user_success_message;
	985	+
	986	+ // Clear the email capture transient since we got the email
	987	+ delete_transient('mxchat_email_capture_' . $user_id);
	988	+ }
	989	+
	990	+ // NEW: Check if we're in an email capture flow but user hasn't provided email yet
	991	+ elseif (get_transient('mxchat_email_capture_' . $user_id)) {
	992	+ // Check if the message contains an email (not the whole message being an email)
	993	+ if (preg_match('/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/', $message, $matches)) {
	994	+ $extracted_email = $matches[0];
1357	995
1358		- // Get the user's success message instruction using current_options
1359		- $user_success_message = $current_options['email_capture_response'] ?? __('Thank you for providing your email! You\'ve been added to our list.', 'mxchat');
	996	+ // Add the extracted email to Loops
	997	+ $this->add_email_to_loops($extracted_email);
1360	998
	999	+ // Get the user's success message instruction
	1000	+ $user_success_message = $this->options['email_capture_response'] ?? __('Thank you for providing your email! You\'ve been added to our list.', 'mxchat');
	1001	+
1361	1002	// Set instruction for AI using the user's success message
1362	1003	$this->current_action_instruction = $user_success_message;
1363	1004
1364	1005	// Clear the email capture transient since we got the email
		@@ -1363,692 +1004,468 @@
1363	1004
1364	1005	// Clear the email capture transient since we got the email
1365	1006	delete_transient('mxchat_email_capture_' . $user_id);
1366	1007	}
1367		-
1368		- // Check if we're in an email capture flow but user hasn't provided email yet
1369		- elseif (get_transient('mxchat_email_capture_' . $user_id)) {
1370		- // Check if the message contains an email (not the whole message being an email)
1371		- if (preg_match('/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/', $message, $matches)) {
1372		- $extracted_email = $matches[0];
1373		-
1374		- // Add the extracted email to Loops
1375		- $this->add_email_to_loops($extracted_email);
1376		-
1377		- // Get the user's success message instruction using current_options
1378		- $user_success_message = $current_options['email_capture_response'] ?? __('Thank you for providing your email! You\'ve been added to our list.', 'mxchat');
1379		-
1380		- // Set instruction for AI using the user's success message
1381		- $this->current_action_instruction = $user_success_message;
1382		-
1383		- // Clear the email capture transient since we got the email
1384		- delete_transient('mxchat_email_capture_' . $user_id);
1385		- }
1386		- // If no email found but we're in capture mode, remind them
1387		- else {
1388		- // Get the original instruction to remind them using current_options
1389		- $original_instruction = $current_options['triggered_phrase_response'] ?? __("Please provide your email address.", 'mxchat');
1390		- $this->current_action_instruction = $original_instruction;
1391		- }
	1008	+ // If no email found but we're in capture mode, remind them
	1009	+ else {
	1010	+ // Get the original instruction to remind them
	1011	+ $original_instruction = $this->options['triggered_phrase_response'] ?? __("Please provide your email address.", 'mxchat');
	1012	+ $this->current_action_instruction = $original_instruction;
1392	1013	}
	1014	+ }
1393	1015
1394		- $intent_info = '';
	1016	+ $intent_info = '';
1395	1017
1396		- // Check chat mode
1397		- $chat_mode = get_option("mxchat_mode_{$session_id}", 'ai');
	1018	+ // Check chat mode
	1019	+ $chat_mode = get_option("mxchat_mode_{$session_id}", 'ai');
1398	1020
1399		- // Handle agent mode
1400	1021	// Handle agent mode
1401		- if ($chat_mode === 'agent') {
1402		- // First, check for switch intent before doing anything else
1403		- $intent_matched = $this->mxchat_check_intent_and_invoke_callback($message, $user_id, $session_id);
	1022	+// Handle agent mode
	1023	+ if ($chat_mode === 'agent') {
	1024	+ // First, check for switch intent before doing anything else
	1025	+ $intent_matched = $this->mxchat_check_intent_and_invoke_callback($message, $user_id, $session_id);
1404	1026
1405		- // Capture action analysis for testing panel after intent check
1406		- if ($testing_data !== null && isset($this->last_action_analysis) && !empty($this->last_action_analysis)) {
1407		- $testing_data['action_matches'] = $this->last_action_analysis;
	1027	+ // NEW: Capture action analysis for testing panel after intent check
	1028	+ if ($testing_data !== null && isset($this->last_action_analysis) && !empty($this->last_action_analysis)) {
	1029	+ $testing_data['action_matches'] = $this->last_action_analysis;
	1030	+ }
	1031	+
	1032	+ // Around line 506, in the agent mode handling section:
	1033	+ if ($intent_matched && !empty($this->fallbackResponse['text'])) {
	1034	+ // Update chat mode first
	1035	+ update_option("mxchat_mode_{$session_id}", 'ai');
	1036	+
	1037	+ // Clear any existing PDF context to start fresh
	1038	+ $this->clear_pdf_transients($session_id);
	1039	+
	1040	+ // Prepare clean switch response with explicit chat_mode
	1041	+ $response_data = [
	1042	+ 'text' => $this->fallbackResponse['text'],
	1043	+ 'html' => $this->fallbackResponse['html'] ?? '',
	1044	+ 'session_id' => $session_id,
	1045	+ 'chat_mode' => 'ai' // EXPLICITLY SET THIS
	1046	+ ];
	1047	+
	1048	+ if ($testing_data !== null) {
	1049	+ $response_data['testing_data'] = $testing_data;
1408	1050	}
1409		-
1410		- // Around line 506, in the agent mode handling section:
1411		- if ($intent_matched && !empty($this->fallbackResponse['text'])) {
1412		- // Update chat mode first
1413		- update_option("mxchat_mode_{$session_id}", 'ai');
1414		-
1415		- // Clear any existing PDF context to start fresh
1416		- $this->clear_pdf_transients($session_id);
1417		-
1418		- // Prepare clean switch response with explicit chat_mode
1419		- $response_data = [
1420		- 'text' => $this->fallbackResponse['text'],
1421		- 'html' => $this->fallbackResponse['html'] ?? '',
1422		- 'session_id' => $session_id,
1423		- 'chat_mode' => 'ai' // EXPLICITLY SET THIS
	1051	+
	1052	+ // Save the mode switch message
	1053	+ $this->mxchat_save_chat_message($session_id, 'system', esc_html__('Switched to AI chat mode', 'mxchat'));
	1054	+ $this->mxchat_save_chat_message($session_id, 'bot', $this->fallbackResponse['text']);
	1055	+
	1056	+ // Send response and exit
	1057	+ wp_send_json($response_data);
	1058	+ wp_die();
	1059	+ } elseif (!$intent_matched) {
	1060	+ // No intent matched, handle live agent message
	1061	+ try {
	1062	+ $this->mxchat_send_user_message_to_agent($message, $user_id, $session_id);
	1063	+
	1064	+ $agent_response = [
	1065	+ 'status' => 'waiting_for_agent',
	1066	+ 'message' => esc_html__('Message sent to live agent.', 'mxchat')
1424	1067	];
1425		-
	1068	+
1426	1069	if ($testing_data !== null) {
1427		- $response_data['testing_data'] = $testing_data;
	1070	+ $agent_response['testing_data'] = $testing_data;
1428	1071	}
1429		-
1430		- // Save the mode switch message
1431		- $this->mxchat_save_chat_message($session_id, 'system', esc_html__('Switched to AI chat mode', 'mxchat'));
1432		- $this->mxchat_save_chat_message($session_id, 'bot', $this->fallbackResponse['text']);
1433		-
1434		- // Send response and exit
1435		- wp_send_json($response_data);
1436		- wp_die();
1437		- } elseif (!$intent_matched) {
1438		- // No intent matched, handle live agent message
1439		- try {
1440		- $this->mxchat_send_user_message_to_agent($message, $user_id, $session_id);
1441	1072
1442		- $agent_response = [
1443		- 'status' => 'waiting_for_agent',
1444		- 'message' => esc_html__('Message sent to live agent.', 'mxchat')
1445		- ];
1446		-
1447		- if ($testing_data !== null) {
1448		- $agent_response['testing_data'] = $testing_data;
1449		- }
1450		-
1451		- wp_send_json_success($agent_response);
1452		- } catch (\Exception $e) {
1453		- wp_send_json_error(esc_html__('Failed to send message to agent', 'mxchat'));
1454		- }
1455		- wp_die();
	1073	+ wp_send_json_success($agent_response);
	1074	+ } catch (\Exception $e) {
	1075	+ wp_send_json_error(esc_html__('Failed to send message to agent', 'mxchat'));
1456	1076	}
	1077	+ wp_die();
1457	1078	}
	1079	+ }
1458	1080
1459		- // Step 1: Check for new PDF URL in the message
1460		- if (!isset($_POST['vision_processed']) && preg_match('/https?:\/\/[^\s"]+/i', $message, $matches)) {
1461		- $new_pdf_url = $matches[0];
	1081	+ // Step 1: Check for new PDF URL in the message
	1082	+ if (!isset($_POST['vision_processed']) && preg_match('/https?:\/\/[^\s"]+/i', $message, $matches)) {
	1083	+ $new_pdf_url = $matches[0];
1462	1084
1463		- // Check if this is likely a PDF-related request
1464		- $pdf_keywords = ['pdf', 'document', 'read', 'analyze'];
1465		- $is_pdf_request = false;
	1085	+ // Check if this is likely a PDF-related request
	1086	+ $pdf_keywords = ['pdf', 'document', 'read', 'analyze'];
	1087	+ $is_pdf_request = false;
1466	1088
1467		- foreach ($pdf_keywords as $keyword) {
1468		- if (stripos($message, $keyword) !== false) {
1469		- $is_pdf_request = true;
1470		- break;
1471		- }
	1089	+ foreach ($pdf_keywords as $keyword) {
	1090	+ if (stripos($message, $keyword) !== false) {
	1091	+ $is_pdf_request = true;
	1092	+ break;
1472	1093	}
	1094	+ }
1473	1095
1474		- // If it looks like a PDF request or we're waiting for a PDF URL
1475		- if ($is_pdf_request || get_transient('mxchat_waiting_for_pdf_url_' . $session_id)) {
1476		- // Validate HTTPS
1477		- if (wp_http_validate_url($new_pdf_url) && parse_url($new_pdf_url, PHP_URL_SCHEME) === 'https') {
1478		- // Extract filename from URL
1479		- $pdf_filename = basename(parse_url($new_pdf_url, PHP_URL_PATH));
	1096	+ // If it looks like a PDF request or we're waiting for a PDF URL
	1097	+ if ($is_pdf_request || get_transient('mxchat_waiting_for_pdf_url_' . $session_id)) {
	1098	+ // Validate HTTPS
	1099	+ if (wp_http_validate_url($new_pdf_url) && parse_url($new_pdf_url, PHP_URL_SCHEME) === 'https') {
	1100	+ // Extract filename from URL
	1101	+ $pdf_filename = basename(parse_url($new_pdf_url, PHP_URL_PATH));
1480	1102
1481		- // Clear previous PDF transients
1482		- $this->clear_pdf_transients($session_id);
	1103	+ // Clear previous PDF transients
	1104	+ $this->clear_pdf_transients($session_id);
1483	1105
1484		- // Process new PDF using current_options
1485		- $max_pages = $current_options['pdf_max_pages'] ?? 69;
1486		- $embeddings = $this->fetch_and_split_pdf_pages($new_pdf_url, $max_pages);
	1106	+ // Process new PDF
	1107	+ $max_pages = $this->options['pdf_max_pages'] ?? 69;
	1108	+ $embeddings = $this->fetch_and_split_pdf_pages($new_pdf_url, $max_pages);
1487	1109
1488		- if ($embeddings === 'too_many_pages') {
1489		- $error_text = sprintf(
1490		- $current_options['pdf_intent_error_text'] ??
1491		- esc_html__("The provided PDF exceeds the maximum allowed limit of %d pages. Please provide a smaller document.", 'mxchat'),
1492		- $max_pages
1493		- );
1494		- $this->fallbackResponse['text'] = $error_text;
1495		- } elseif ($embeddings) {
1496		- // Store new PDF information
1497		- $pdf_filename = basename(parse_url($new_pdf_url, PHP_URL_PATH));
	1110	+ if ($embeddings === 'too_many_pages') {
	1111	+ $error_text = sprintf(
	1112	+ $this->options['pdf_intent_error_text'] ??
	1113	+ esc_html__("The provided PDF exceeds the maximum allowed limit of %d pages. Please provide a smaller document.", 'mxchat'),
	1114	+ $max_pages
	1115	+ );
	1116	+ $this->fallbackResponse['text'] = $error_text;
	1117	+ } elseif ($embeddings) {
	1118	+ // Store new PDF information
	1119	+ $pdf_filename = basename(parse_url($new_pdf_url, PHP_URL_PATH));
1498	1120
1499		- // If the filename is generic, create a more descriptive one
1500		- if (in_array($pdf_filename, ['results_download.php', 'download.php', 'view.php', 'pdf.php']) ||
1501		- strpos($pdf_filename, '.php') !== false) {
1502		- $pdf_filename = 'Document_' . date('Y-m-d_H-i') . '.pdf';
1503		- }
	1121	+ // If the filename is generic, create a more descriptive one
	1122	+ if (in_array($pdf_filename, ['results_download.php', 'download.php', 'view.php', 'pdf.php']) ||
	1123	+ strpos($pdf_filename, '.php') !== false) {
	1124	+ $pdf_filename = 'Document_' . date('Y-m-d_H-i') . '.pdf';
	1125	+ }
1504	1126
1505		- set_transient('mxchat_pdf_url_' . $session_id, $new_pdf_url, HOUR_IN_SECONDS);
1506		- set_transient('mxchat_pdf_filename_' . $session_id, $pdf_filename, HOUR_IN_SECONDS);
1507		- set_transient('mxchat_pdf_embeddings_' . $session_id, $embeddings, HOUR_IN_SECONDS);
1508		- set_transient('mxchat_include_pdf_in_context_' . $session_id, true, HOUR_IN_SECONDS);
	1127	+ set_transient('mxchat_pdf_url_' . $session_id, $new_pdf_url, HOUR_IN_SECONDS);
	1128	+ set_transient('mxchat_pdf_filename_' . $session_id, $pdf_filename, HOUR_IN_SECONDS);
	1129	+ set_transient('mxchat_pdf_embeddings_' . $session_id, $embeddings, HOUR_IN_SECONDS);
	1130	+ set_transient('mxchat_include_pdf_in_context_' . $session_id, true, HOUR_IN_SECONDS);
1509	1131
1510		- $success_text = $current_options['pdf_intent_success_text'] ??
1511		- esc_html__("I've processed the new PDF '{$pdf_filename}'. What questions do you have about it?", 'mxchat');
	1132	+ $success_text = $this->options['pdf_intent_success_text'] ??
	1133	+ esc_html__("I've processed the new PDF '{$pdf_filename}'. What questions do you have about it?", 'mxchat');
1512	1134
1513		- $pdf_response = [
1514		- 'success' => true,
1515		- 'message' => $success_text,
1516		- 'data' => [
1517		- 'filename' => $pdf_filename
1518		- ]
1519		- ];
1520		-
1521		- if ($testing_data !== null) {
1522		- $pdf_response['testing_data'] = $testing_data;
1523		- }
1524		-
1525		- wp_send_json($pdf_response);
1526		- wp_die();
1527		- } else {
1528		- $error_text = $current_options['pdf_intent_error_text'] ??
1529		- esc_html__("Sorry, I couldn't process the PDF. Please ensure it's a valid file.", 'mxchat');
1530		- $this->fallbackResponse['text'] = $error_text;
1531		- }
1532		-
1533		- $pdf_error_response = [
1534		- 'success' => false,
1535		- 'message' => $this->fallbackResponse['text']
	1135	+ $pdf_response = [
	1136	+ 'success' => true,
	1137	+ 'message' => $success_text,
	1138	+ 'data' => [
	1139	+ 'filename' => $pdf_filename
	1140	+ ]
1536	1141	];
1537	1142
1538	1143	if ($testing_data !== null) {
1539		- $pdf_error_response['testing_data'] = $testing_data;
	1144	+ $pdf_response['testing_data'] = $testing_data;
1540	1145	}
1541	1146
1542		- wp_send_json($pdf_error_response);
	1147	+ wp_send_json($pdf_response);
1543	1148	wp_die();
	1149	+ } else {
	1150	+ $error_text = $this->options['pdf_intent_error_text'] ??
	1151	+ esc_html__("Sorry, I couldn't process the PDF. Please ensure it's a valid file.", 'mxchat');
	1152	+ $this->fallbackResponse['text'] = $error_text;
1544	1153	}
1545		- }
1546		- }
1547	1154
1548		-
1549		- // Step 2: Detect intent and handle intent-based responses
1550		- $intent_result = $this->mxchat_check_intent_and_invoke_callback($message, $user_id, $session_id);
1551		-
1552		- // Capture action analysis for testing panel after intent check
1553		- if ($testing_data !== null && isset($this->last_action_analysis) && !empty($this->last_action_analysis)) {
1554		- $testing_data['action_matches'] = $this->last_action_analysis;
1555		- }
1556		-
1557		- // Step 3: Handle the intent result appropriately
1558		- if ($intent_result !== false) {
1559		- // Intent was matched - ALWAYS send as JSON response, never streaming
1560		-
1561		- if (is_array($intent_result) && (isset($intent_result['text']) || isset($intent_result['html']))) {
1562		- // Intent returned a direct response array
1563		- $response_data = [
1564		- 'text' => $intent_result['text'] ?? '',
1565		- 'html' => $intent_result['html'] ?? '',
1566		- 'session_id' => $session_id
	1155	+ $pdf_error_response = [
	1156	+ 'success' => false,
	1157	+ 'message' => $this->fallbackResponse['text']
1567	1158	];
1568		-
1569		- // IMPORTANT: Include chat_mode if present (for WhatsApp, Slack, etc.)
1570		- if (isset($intent_result['chat_mode'])) {
1571		- $response_data['chat_mode'] = $intent_result['chat_mode'];
1572		- }
1573		-
	1159	+
1574	1160	if ($testing_data !== null) {
1575		- $response_data['testing_data'] = $testing_data;
	1161	+ $pdf_error_response['testing_data'] = $testing_data;
1576	1162	}
1577	1163
1578		- wp_send_json($response_data);
	1164	+ wp_send_json($pdf_error_response);
1579	1165	wp_die();
1580		- } else if ($intent_result === true && (!empty($this->fallbackResponse['text']) || !empty($this->fallbackResponse['html']))) {
1581		- // Intent returned true and set fallbackResponse
1582		-
1583		- // SAVE TO TRANSCRIPT
1584		- if (!empty($this->fallbackResponse['text'])) {
1585		- $this->mxchat_save_chat_message($session_id, 'bot', $this->fallbackResponse['text']);
1586		- }
1587		- // Save action HTML (product cards, featured products, etc.) so it renders in transcripts
1588		- if (!empty($this->fallbackResponse['html'])) {
1589		- $this->mxchat_save_chat_message($session_id, 'bot', $this->fallbackResponse['html']);
1590		- }
1591		-
1592		- $response_data = [
1593		- 'text' => $this->fallbackResponse['text'] ?? '',
1594		- 'html' => $this->fallbackResponse['html'] ?? '',
1595		- 'session_id' => $session_id
1596		- ];
1597		-
1598		- if (isset($this->fallbackResponse['chat_mode'])) {
1599		- $response_data['chat_mode'] = $this->fallbackResponse['chat_mode'];
1600		- }
1601		-
1602		- if ($testing_data !== null) {
1603		- $response_data['testing_data'] = $testing_data;
1604		- }
1605		-
1606		- wp_send_json($response_data);
1607		- wp_die();
1608	1166	}
1609	1167	}
	1168	+ }
1610	1169
1611		- // If we get here, no intent matched OR the intent didn't provide a usable response
1612		-
1613		- // Step 4: Generate AI response
1614		- // Get session start timestamp - when persistence is OFF, only include messages from this page load
1615		- $session_start_timestamp = isset($_POST['session_start_timestamp']) ? intval($_POST['session_start_timestamp']) : 0;
1616		- $conversation_history = $this->mxchat_fetch_conversation_history_for_ai($session_id, $session_start_timestamp);
1617		- $this->mxchat_increment_chat_count();
	1170	+ // Check if there's an active recommendation flow session
	1171	+ $flow_state = get_option("mxchat_sr_flow_state_{$session_id}", array());
	1172	+ if (!empty($flow_state) && isset($flow_state['flow_id'])) {
	1173	+ // Create a dummy intent object that matches the original intent
	1174	+ $dummy_intent = new stdClass();
	1175	+ $dummy_intent->intent_label = 'Recommendation Flow ' . $flow_state['flow_id'];
	1176	+ $dummy_intent->phrases = ''; // Empty phrases to avoid matching the original trigger
1618	1177
1619		- // Generate embedding for the user's query - USE BOT-SPECIFIC API KEY
1620		- $api_key = $current_options['api_key'] ?? $this->options['api_key'];
1621		- $user_message_embedding = $this->mxchat_generate_embedding($message, $api_key);
	1178	+ // Call the recommendation flow handler directly
	1179	+ $response_data = apply_filters('mxchat_sr_recommendation_flow', false, $message, $user_id, $session_id, $dummy_intent);
1622	1180
1623		- // Check if the embedding generation returned an error
1624		- if (is_array($user_message_embedding) && isset($user_message_embedding['error'])) {
1625		- $error_message = $user_message_embedding['error'];
1626		- $error_code = $user_message_embedding['error_code'] ?? 'embedding_error';
1627		-
1628		- // FIXED: Send error in appropriate format based on streaming mode
1629		- if ($is_streaming) {
1630		- echo "data: " . json_encode([
1631		- 'error' => true,
1632		- 'error_message' => $error_message,
1633		- 'error_code' => $error_code,
1634		- 'text' => $error_message,
1635		- 'message' => $error_message
1636		- ]) . "\n\n";
1637		- echo "data: [DONE]\n\n";
1638		- flush();
1639		- } else {
1640		- wp_send_json_error([
1641		- 'error_message' => $error_message,
1642		- 'error_code' => $error_code
1643		- ]);
	1181	+ // If the handler returned a response, send it
	1182	+ if (is_array($response_data) && (isset($response_data['text']) || isset($response_data['html']))) {
	1183	+ // Save the bot's response to the chat history
	1184	+ if (!empty($response_data['text'])) {
	1185	+ $this->mxchat_save_chat_message($session_id, 'bot', $response_data['text']);
1644	1186	}
	1187	+ if (!empty($response_data['html'])) {
	1188	+ $this->mxchat_save_chat_message($session_id, 'bot', $response_data['html']);
	1189	+ }
	1190	+
	1191	+ if ($testing_data !== null) {
	1192	+ $response_data['testing_data'] = $testing_data;
	1193	+ }
	1194	+
	1195	+ // Send the response
	1196	+ wp_send_json($response_data);
1645	1197	wp_die();
1646	1198	}
	1199	+ }
1647	1200
1648		- // Check if the embedding is valid
1649		- if (!is_array($user_message_embedding) || empty($user_message_embedding)) {
1650		- $error_message = esc_html__('Unable to process your message. The embedding service is not responding correctly.', 'mxchat');
	1201	+ // Step 2: Detect intent and handle intent-based responses
	1202	+ $intent_result = $this->mxchat_check_intent_and_invoke_callback($message, $user_id, $session_id);
1651	1203
1652		- // FIXED: Send error in appropriate format based on streaming mode
	1204	+ // NEW: Capture action analysis for testing panel after intent check
	1205	+ if ($testing_data !== null && isset($this->last_action_analysis) && !empty($this->last_action_analysis)) {
	1206	+ $testing_data['action_matches'] = $this->last_action_analysis;
	1207	+ }
	1208	+
	1209	+ // Step 3: Handle the intent result appropriately
	1210	+ if ($intent_result !== false) {
	1211	+ // Intent was matched - ALWAYS send as JSON response, never streaming
	1212	+
	1213	+ if (is_array($intent_result) && (isset($intent_result['text']) || isset($intent_result['html']))) {
	1214	+ // Intent returned a direct response array
	1215	+ $response_data = [
	1216	+ 'text' => $intent_result['text'] ?? '',
	1217	+ 'html' => $intent_result['html'] ?? '',
	1218	+ 'session_id' => $session_id
	1219	+ ];
	1220	+
	1221	+ if ($testing_data !== null) {
	1222	+ $response_data['testing_data'] = $testing_data;
	1223	+ }
	1224	+
	1225	+ // Clear streaming headers if they were set
1653	1226	if ($is_streaming) {
1654		- echo "data: " . json_encode([
1655		- 'error' => true,
1656		- 'error_message' => $error_message,
1657		- 'error_code' => 'invalid_embedding',
1658		- 'text' => $error_message,
1659		- 'message' => $error_message
1660		- ]) . "\n\n";
1661		- echo "data: [DONE]\n\n";
1662		- flush();
1663		- } else {
1664		- wp_send_json_error([
1665		- 'error_message' => $error_message,
1666		- 'error_code' => 'invalid_embedding'
1667		- ]);
	1227	+ header_remove('Content-Type');
	1228	+ header_remove('Cache-Control');
	1229	+ header_remove('Connection');
	1230	+ header_remove('X-Accel-Buffering');
	1231	+ header('Content-Type: application/json');
1668	1232	}
	1233	+
	1234	+ wp_send_json($response_data);
1669	1235	wp_die();
1670		- }
1671		-
1672		- // Build context with both knowledge base and PDF content if available
1673		- $context_content = "User asked: '{$message}'\n\n";
1674		-
1675		- // Add action instruction if present (add this right after the above line)
1676		- if (!empty($this->current_action_instruction)) {
1677		- $context_content .= "===== SPECIAL INSTRUCTION =====\n";
1678		- $context_content .= "IMPORTANT: " . $this->current_action_instruction . "\n";
1679		- $context_content .= "Respond naturally and conversationally while following this instruction.\n";
1680		- $context_content .= "===== END SPECIAL INSTRUCTION =====\n\n";
	1236	+ } else if ($intent_result === true && (!empty($this->fallbackResponse['text']) || !empty($this->fallbackResponse['html']))) {
	1237	+ // Intent returned true and set fallbackResponse
1681	1238
1682		- // Clear the instruction after using it
1683		- $this->current_action_instruction = null;
1684		- }
1685		-
1686		-
1687		- // Add page context if available and contextual awareness is enabled using current_options
1688		- if ($page_context && isset($current_options['contextual_awareness_toggle']) && $current_options['contextual_awareness_toggle'] === 'on') {
1689		- $context_content .= "===== CURRENT PAGE CONTEXT =====\n";
1690		- $context_content .= "Page URL: " . $page_context['url'] . "\n";
1691		- $context_content .= "Page Title: " . $page_context['title'] . "\n";
1692		- $context_content .= "Page Content: " . $page_context['content'] . "\n";
1693		- $context_content .= "===== END CURRENT PAGE CONTEXT =====\n\n";
1694		- }
1695		-
1696		- // Get relevant content from knowledge base - PASS BOT_ID and MESSAGE for Vector Store
1697		- $relevant_content = $this->mxchat_find_relevant_content($user_message_embedding, $bot_id, $message);
1698		-
1699		- // NEW: Also extract URLs from system instructions (only if citation links enabled)
1700		- // Use fresh options to ensure we get the latest setting value
1701		- $fresh_options = get_option('mxchat_options', []);
1702		- $citation_links_enabled = isset($fresh_options['citation_links_toggle']) ? ($fresh_options['citation_links_toggle'] === 'on') : true;
1703		-
1704		- $system_instructions = $this->get_system_instructions($bot_id, $session_id);
1705		- if ($citation_links_enabled && !empty($system_instructions)) {
1706		- preg_match_all(
1707		- '#\bhttps?://[^\s<>"\']+#i',
1708		- $system_instructions,
1709		- $system_instruction_urls
1710		- );
1711		-
1712		- if (!empty($system_instruction_urls[0])) {
1713		- // Merge with existing valid URLs
1714		- $this->current_valid_urls = array_merge(
1715		- $this->current_valid_urls,
1716		- $system_instruction_urls[0]
1717		- );
1718		- // Remove duplicates
1719		- $this->current_valid_urls = array_unique($this->current_valid_urls);
1720		-
1721		- //error_log("Added " . count($system_instruction_urls[0]) . " URLs from system instructions");
	1239	+ // SAVE TO TRANSCRIPT FIRST
	1240	+ if (!empty($this->fallbackResponse['text'])) {
	1241	+ $this->mxchat_save_chat_message($session_id, 'bot', $this->fallbackResponse['text']);
1722	1242	}
1723		- }
1724		-
1725		-// ===== CAPTURE REAL SIMILARITY DATA FOR ADMINS =====
1726		-if ($testing_data !== null && $this->last_similarity_analysis !== null) {
1727		- // Update testing data with the REAL similarity analysis
1728		- $testing_data['top_matches'] = $this->last_similarity_analysis['top_matches'];
1729		- $testing_data['total_documents_checked'] = $this->last_similarity_analysis['total_checked'] ?? 0;
1730		- $testing_data['knowledge_base_type'] = $this->last_similarity_analysis['knowledge_base_type'];
1731		- $testing_data['sources_used'] = $this->last_similarity_analysis['sources_used'] ?? 0;
1732		- $testing_data['total_chunks_used'] = $this->last_similarity_analysis['total_chunks_used'] ?? 0;
1733		-}
1734		-// ===== END SIMILARITY DATA CAPTURE =====
1735		-
1736		-// NEW: Add valid URLs to testing data for admin panel display (AFTER similarity data)
1737		-if ($testing_data !== null && !empty($this->current_valid_urls)) {
1738		- $testing_data['approved_urls'] = array_values($this->current_valid_urls);
1739		- //error_log("Added " . count($this->current_valid_urls) . " approved URLs to testing data");
1740		-}
1741		-
1742		- if (!empty($relevant_content)) {
1743		- $context_content .= "===== OFFICIAL KNOWLEDGE DATABASE CONTENT =====\n" . $relevant_content . "\n===== END OF OFFICIAL KNOWLEDGE DATABASE CONTENT =====\n\n";
1744		- } else {
1745		- $context_content .= "===== NO RELEVANT CONTENT FOUND IN KNOWLEDGE DATABASE =====\n";
1746		- }
1747		-
1748		- // NEW: Add approved URLs list to context for AI (only if citation links enabled)
1749		- if ($citation_links_enabled && !empty($this->current_valid_urls)) {
1750		- $context_content .= "===== APPROVED URLS FOR CITATIONS =====\n";
1751		- $context_content .= "You may ONLY use these exact URLs in your response:\n";
1752		- foreach ($this->current_valid_urls as $url) {
1753		- $context_content .= "- " . $url . "\n";
	1243	+ if (!empty($this->fallbackResponse['html'])) {
	1244	+ $this->mxchat_save_chat_message($session_id, 'bot', $this->fallbackResponse['html']);
1754	1245	}
1755		- $context_content .= "\nCRITICAL: Do NOT create, modify, extend, or invent any other URLs. ";
1756		- $context_content .= "===== END APPROVED URLS =====\n\n";
1757		- }
1758		-
1759		- // Check for and include PDF content
1760		- $pdf_url = get_transient('mxchat_pdf_url_' . $session_id);
1761		- $pdf_embeddings = get_transient('mxchat_pdf_embeddings_' . $session_id);
1762		- $pdf_filename = get_transient('mxchat_pdf_filename_' . $session_id);
1763		- if ($pdf_url && $pdf_embeddings && get_transient('mxchat_include_pdf_in_context_' . $session_id)) {
1764		- $relevant_pdf_pages = $this->find_relevant_pdf_pages($user_message_embedding, $pdf_embeddings);
1765		- if (!empty($relevant_pdf_pages)) {
1766		- $context_content .= "Relevant content from PDF document '{$pdf_filename}':\n";
1767		- foreach ($relevant_pdf_pages as $page_data) {
1768		- $context_content .= "Page {$page_data['page_number']} of '{$pdf_filename}': {$page_data['text']}\n";
1769		- }
1770		- $context_content .= "\n";
	1246	+
	1247	+ $response_data = [
	1248	+ 'text' => $this->fallbackResponse['text'] ?? '',
	1249	+ 'html' => $this->fallbackResponse['html'] ?? '',
	1250	+ 'session_id' => $session_id
	1251	+ ];
	1252	+
	1253	+ if (isset($this->fallbackResponse['chat_mode'])) {
	1254	+ $response_data['chat_mode'] = $this->fallbackResponse['chat_mode'];
1771	1255	}
1772		- }
1773		-
1774		- // Check for and include Word content
1775		- $word_url = get_transient('mxchat_word_url_' . $session_id);
1776		- $word_embeddings = get_transient('mxchat_word_embeddings_' . $session_id);
1777		- $word_filename = get_transient('mxchat_word_filename_' . $session_id);
1778		- if ($word_url && $word_embeddings && get_transient('mxchat_include_word_in_context_' . $session_id)) {
1779		- $relevant_word_chunks = $this->word_handler->mxchat_find_relevant_word_chunks($user_message_embedding, $word_embeddings);
1780		- if (!empty($relevant_word_chunks)) {
1781		- $context_content .= "Relevant content from Word document '{$word_filename}':\n";
1782		- foreach ($relevant_word_chunks as $chunk_data) {
1783		- $context_content .= "Section {$chunk_data['chunk_number']} of '{$word_filename}': {$chunk_data['text']}\n";
1784		- }
1785		- $context_content .= "\n";
	1256	+
	1257	+ if ($testing_data !== null) {
	1258	+ $response_data['testing_data'] = $testing_data;
1786	1259	}
	1260	+
	1261	+ // Clear streaming headers if they were set
	1262	+ if ($is_streaming) {
	1263	+ header_remove('Content-Type');
	1264	+ header_remove('Cache-Control');
	1265	+ header_remove('Connection');
	1266	+ header_remove('X-Accel-Buffering');
	1267	+ header('Content-Type: application/json');
	1268	+ }
	1269	+
	1270	+ wp_send_json($response_data);
	1271	+ wp_die();
1787	1272	}
1788		-
1789		- $context_content = apply_filters('mxchat_prepare_context', $context_content, $session_id);
	1273	+ }
1790	1274
1791		- // Extract model from current options for bot-specific model support
1792		- $selected_model = isset($current_options['model']) ? $current_options['model'] : 'gpt-5.1-chat-latest';
	1275	+ // If we get here, no intent matched OR the intent didn't provide a usable response
	1276	+
	1277	+ // Step 4: Generate AI response
	1278	+ $conversation_history = $this->mxchat_fetch_conversation_history_for_ai($session_id);
	1279	+ $this->mxchat_increment_chat_count();
	1280	+
	1281	+ // Generate embedding for the user's query
	1282	+ $user_message_embedding = $this->mxchat_generate_embedding($message, $this->options['api_key']);
	1283	+
	1284	+ // Check if the embedding generation returned an error
	1285	+ if (is_array($user_message_embedding) && isset($user_message_embedding['error'])) {
	1286	+ $error_message = $user_message_embedding['error'];
	1287	+ $error_code = $user_message_embedding['error_code'] ?? 'embedding_error';
1793	1288
1794		- $response = $this->mxchat_generate_response(
1795		- $context_content,
1796		- $current_options['api_key'] ?? $this->options['api_key'],
1797		- $current_options['xai_api_key'] ?? $this->options['xai_api_key'],
1798		- $current_options['claude_api_key'] ?? $this->options['claude_api_key'],
1799		- $current_options['deepseek_api_key'] ?? $this->options['deepseek_api_key'],
1800		- $current_options['gemini_api_key'] ?? $this->options['gemini_api_key'],
1801		- $current_options['openrouter_api_key'] ?? $this->options['openrouter_api_key'],
1802		- $conversation_history,
1803		- $is_streaming,
1804		- $session_id,
1805		- $testing_data,
1806		- $selected_model
1807		- );
1808		-
1809		- // Handle streaming vs non-streaming responses
1810		- if ($is_streaming) {
1811		- // Check if streaming actually happened or if it fell back to regular response
1812		- if ($response === true) {
1813		- wp_die();
1814		- }
1815		- // If we get here, streaming fell back to regular response, continue
1816		- // But if there's an error, we need to send it as SSE format since headers are already set
1817		- if (is_array($response) && isset($response['error'])) {
1818		- $error_message = $response['error'];
1819		- $error_code = $response['error_code'] ?? 'api_error';
1820		- // Send error in SSE format that the client JS can handle
1821		- echo "data: " . json_encode([
1822		- 'error' => true,
1823		- 'error_message' => $error_message,
1824		- 'error_code' => $error_code,
1825		- 'text' => $error_message, // Also include as text for fallback handling
1826		- 'message' => $error_message
1827		- ]) . "\n\n";
1828		- echo "data: [DONE]\n\n";
1829		- flush();
1830		- wp_die();
1831		- }
1832		- }
	1289	+ wp_send_json_error([
	1290	+ 'error_message' => $error_message,
	1291	+ 'error_code' => $error_code
	1292	+ ]);
	1293	+ wp_die();
	1294	+ }
	1295	+
	1296	+ // Check if the embedding is valid
	1297	+ if (!is_array($user_message_embedding) || empty($user_message_embedding)) {
	1298	+ wp_send_json_error([
	1299	+ 'error_message' => esc_html__('Unable to process your message. The embedding service is not responding correctly.', 'mxchat'),
	1300	+ 'error_code' => 'invalid_embedding'
	1301	+ ]);
	1302	+ wp_die();
	1303	+ }
1833	1304
1834		- // Check if the response is an error array (non-streaming mode)
1835		- if (is_array($response) && isset($response['error'])) {
1836		- wp_send_json_error([
1837		- 'error_message' => $response['error'],
1838		- 'error_code' => $response['error_code'] ?? 'api_error'
1839		- ]);
1840		- wp_die();
1841		- }
	1305	+ // Build context with both knowledge base and PDF content if available
	1306	+ $context_content = "User asked: '{$message}'\n\n";
	1307	+
	1308	+ // NEW: Add action instruction if present (add this right after the above line)
	1309	+ if (!empty($this->current_action_instruction)) {
	1310	+ $context_content .= "===== SPECIAL INSTRUCTION =====\n";
	1311	+ $context_content .= "IMPORTANT: " . $this->current_action_instruction . "\n";
	1312	+ $context_content .= "Respond naturally and conversationally while following this instruction.\n";
	1313	+ $context_content .= "===== END SPECIAL INSTRUCTION =====\n\n";
1842	1314
1843		- // DEBUG: Check what we have
1844		- //error_log("=== BEFORE URL VALIDATION ===");
1845		- //error_log("current_valid_urls is empty? " . (empty($this->current_valid_urls) ? 'YES' : 'NO'));
1846		- //error_log("current_valid_urls count: " . count($this->current_valid_urls));
1847		- //error_log("current_valid_urls content: " . print_r($this->current_valid_urls, true));
1848		-
1849		- // If we get here, the response is valid text - now validate URLs
1850		- if (!empty($this->current_valid_urls)) {
1851		- //error_log("CALLING validate_and_clean_urls");
1852		- $response = $this->validate_and_clean_urls($response, $this->current_valid_urls);
1853		- } else {
1854		- //error_log("SKIPPING validation - current_valid_urls is empty");
1855		- }
1856		- // ===== END URL VALIDATION =====
	1315	+ // Clear the instruction after using it
	1316	+ $this->current_action_instruction = null;
	1317	+ }
1857	1318
1858		- // Prepare RAG context data for storage (only include documents used for context)
1859		- $rag_context_for_storage = null;
1860		- $has_rag_data = $this->last_similarity_analysis !== null && !empty($this->last_similarity_analysis['top_matches']);
1861		- $has_action_data = isset($this->last_action_analysis) && !empty($this->last_action_analysis);
1862	1319
1863		- if ($has_rag_data || $has_action_data) {
1864		- $rag_context_for_storage = [];
	1320	+ // NEW: Add page context if available and contextual awareness is enabled
	1321	+ if ($page_context && isset($this->options['contextual_awareness_toggle']) && $this->options['contextual_awareness_toggle'] === 'on') {
	1322	+ $context_content .= "===== CURRENT PAGE CONTEXT =====\n";
	1323	+ $context_content .= "Page URL: " . $page_context['url'] . "\n";
	1324	+ $context_content .= "Page Title: " . $page_context['title'] . "\n";
	1325	+ $context_content .= "Page Content: " . $page_context['content'] . "\n";
	1326	+ $context_content .= "===== END CURRENT PAGE CONTEXT =====\n\n";
	1327	+ }
1865	1328
1866		- // Add RAG/source data if available
1867		- if ($has_rag_data) {
1868		- $rag_context_for_storage['top_matches'] = $this->last_similarity_analysis['top_matches'];
1869		- $rag_context_for_storage['approved_urls'] = $this->current_valid_urls ?? [];
1870		- $rag_context_for_storage['similarity_threshold'] = $this->last_similarity_analysis['threshold_used'] ?? 0.35;
1871		- $rag_context_for_storage['knowledge_base_type'] = $this->last_similarity_analysis['knowledge_base_type'] ?? 'WordPress Database';
1872		- $rag_context_for_storage['total_documents_checked'] = $this->last_similarity_analysis['total_checked'] ?? 0;
1873		- $rag_context_for_storage['sources_used'] = $this->last_similarity_analysis['sources_used'] ?? 0;
1874		- $rag_context_for_storage['total_chunks_used'] = $this->last_similarity_analysis['total_chunks_used'] ?? 0;
1875		- }
	1329	+ // Get relevant content from knowledge base - THIS IS WHERE THE SIMILARITY ANALYSIS HAPPENS
	1330	+ $relevant_content = $this->mxchat_find_relevant_content($user_message_embedding);
	1331	+
	1332	+ // ===== CAPTURE REAL SIMILARITY DATA FOR ADMINS =====
	1333	+ if ($testing_data !== null && $this->last_similarity_analysis !== null) {
	1334	+ // Update testing data with the REAL similarity analysis
	1335	+ $testing_data['top_matches'] = $this->last_similarity_analysis['top_matches'];
	1336	+ $testing_data['total_documents_checked'] = $this->last_similarity_analysis['total_checked'] ?? 0;
	1337	+ $testing_data['knowledge_base_type'] = $this->last_similarity_analysis['knowledge_base_type'];
	1338	+ }
	1339	+ // ===== END SIMILARITY DATA CAPTURE =====
	1340	+
	1341	+ if (!empty($relevant_content)) {
	1342	+ $context_content .= "===== OFFICIAL KNOWLEDGE DATABASE CONTENT =====\n" . $relevant_content . "\n===== END OF OFFICIAL KNOWLEDGE DATABASE CONTENT =====\n\n";
	1343	+ } else {
	1344	+ $context_content .= "===== NO RELEVANT CONTENT FOUND IN KNOWLEDGE DATABASE =====\n";
	1345	+ }
1876	1346
1877		- // Add action analysis data if available
1878		- if ($has_action_data) {
1879		- $rag_context_for_storage['action_analysis'] = $this->last_action_analysis;
	1347	+ // Check for and include PDF content
	1348	+ $pdf_url = get_transient('mxchat_pdf_url_' . $session_id);
	1349	+ $pdf_embeddings = get_transient('mxchat_pdf_embeddings_' . $session_id);
	1350	+ $pdf_filename = get_transient('mxchat_pdf_filename_' . $session_id);
	1351	+ if ($pdf_url && $pdf_embeddings && get_transient('mxchat_include_pdf_in_context_' . $session_id)) {
	1352	+ $relevant_pdf_pages = $this->find_relevant_pdf_pages($user_message_embedding, $pdf_embeddings);
	1353	+ if (!empty($relevant_pdf_pages)) {
	1354	+ $context_content .= "Relevant content from PDF document '{$pdf_filename}':\n";
	1355	+ foreach ($relevant_pdf_pages as $page_data) {
	1356	+ $context_content .= "Page {$page_data['page_number']} of '{$pdf_filename}': {$page_data['text']}\n";
1880	1357	}
	1358	+ $context_content .= "\n";
1881	1359	}
	1360	+ }
1882	1361
1883		- // Save the cleaned response with RAG context
1884		- $this->mxchat_save_chat_message($session_id, 'bot', $response, null, $rag_context_for_storage);
1885		-
1886		- // Step 5: Save additional content if available
1887		- if (!empty($this->productCardHtml)) {
1888		- $this->mxchat_save_chat_message($session_id, 'bot', $this->productCardHtml);
	1362	+ // Check for and include Word content
	1363	+ $word_url = get_transient('mxchat_word_url_' . $session_id);
	1364	+ $word_embeddings = get_transient('mxchat_word_embeddings_' . $session_id);
	1365	+ $word_filename = get_transient('mxchat_word_filename_' . $session_id);
	1366	+ if ($word_url && $word_embeddings && get_transient('mxchat_include_word_in_context_' . $session_id)) {
	1367	+ $relevant_word_chunks = $this->word_handler->mxchat_find_relevant_word_chunks($user_message_embedding, $word_embeddings);
	1368	+ if (!empty($relevant_word_chunks)) {
	1369	+ $context_content .= "Relevant content from Word document '{$word_filename}':\n";
	1370	+ foreach ($relevant_word_chunks as $chunk_data) {
	1371	+ $context_content .= "Section {$chunk_data['chunk_number']} of '{$word_filename}': {$chunk_data['text']}\n";
	1372	+ }
	1373	+ $context_content .= "\n";
1889	1374	}
	1375	+ }
	1376	+
	1377	+ $context_content = apply_filters('mxchat_prepare_context', $context_content, $session_id);
1890	1378
1891		- if (!empty($this->fallbackResponse['html'])) {
1892		- $this->mxchat_save_chat_message($session_id, 'bot', $this->fallbackResponse['html']);
	1379	+ // Generate response
	1380	+ $response = $this->mxchat_generate_response(
	1381	+ $context_content,
	1382	+ $this->options['api_key'],
	1383	+ $this->options['xai_api_key'],
	1384	+ $this->options['claude_api_key'],
	1385	+ $this->options['deepseek_api_key'],
	1386	+ $this->options['gemini_api_key'],
	1387	+ $conversation_history,
	1388	+ $is_streaming,
	1389	+ $session_id,
	1390	+ $testing_data
	1391	+ );
	1392	+
	1393	+ // Handle streaming vs non-streaming responses
	1394	+ if ($is_streaming) {
	1395	+ // Check if streaming actually happened or if it fell back to regular response
	1396	+ if ($response === true) {
	1397	+ wp_die();
1893	1398	}
1894		-
1895		- // Step 6: Return the response
1896		- // DEBUG: Check if newlines exist in the response
1897		- //error_log("=== MXCHAT NON-STREAMING RESPONSE DEBUG ===");
1898		- //error_log("Response has newlines: " . (strpos($response, "\n") !== false ? 'YES' : 'NO'));
1899		- //error_log("Response first 500 chars: " . substr($response, 0, 500));
1900		-
1901		- $response_data = [
1902		- 'text' => $response,
1903		- 'html' => !empty($this->productCardHtml) ? $this->productCardHtml : ($this->fallbackResponse['html'] ?? ''),
1904		- 'session_id' => $session_id
1905		- ];
1906		-
1907		- // Include vectorstore error info for admin debugging (only visible to admins via testing_data)
1908		- if (!empty($this->last_vectorstore_error) && $testing_data !== null) {
1909		- $testing_data['vectorstore_error'] = $this->last_vectorstore_error;
1910		- }
1911		-
1912		- // Also pass it as a top-level field so JS can show a better error message to admins
1913		- if (!empty($this->last_vectorstore_error) && current_user_can('manage_options')) {
1914		- $response_data['vectorstore_error'] = $this->last_vectorstore_error;
1915		- }
1916		-
1917		- // Always add testing data for admins (no toggle needed)
1918		- if ($testing_data !== null) {
1919		- $response_data['testing_data'] = $testing_data;
1920		- }
1921		-
1922		- wp_send_json($response_data);
	1399	+ // If we get here, streaming fell back to regular response, continue
	1400	+ }
	1401	+
	1402	+ // Check if the response is an error array
	1403	+ if (is_array($response) && isset($response['error'])) {
	1404	+ wp_send_json_error([
	1405	+ 'error_message' => $response['error'],
	1406	+ 'error_code' => $response['error_code'] ?? 'api_error'
	1407	+ ]);
1923	1408	wp_die();
1924		-}
1925		-
1926		-/**
1927		- * Get bot-specific options for multi-bot functionality
1928		- * Falls back to default options if bot_id is 'default' or multi-bot add-on is not active
1929		- */
1930		-// Also debug the bot options retrieval
1931		-private function get_bot_options($bot_id = 'default') {
1932		- //error_log("MXCHAT DEBUG: get_bot_options called for bot: " . $bot_id);
1933		-
1934		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
1935		- //error_log("MXCHAT DEBUG: Using default options (no multi-bot or bot is 'default')");
1936		- return array();
1937	1409	}
1938	1410
1939		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
1940		-
1941		- if (!empty($bot_options)) {
1942		- //error_log("MXCHAT DEBUG: Got bot-specific options from filter");
1943		- if (isset($bot_options['similarity_threshold'])) {
1944		- //error_log(" - similarity_threshold: " . $bot_options['similarity_threshold']);
1945		- }
	1411	+ // If we get here, the response is valid text
	1412	+ $this->mxchat_save_chat_message($session_id, 'bot', $response);
	1413	+
	1414	+ // Step 5: Save additional content if available
	1415	+ if (!empty($this->productCardHtml)) {
	1416	+ $this->mxchat_save_chat_message($session_id, 'bot', $this->productCardHtml);
1946	1417	}
1947		-
1948		- return is_array($bot_options) ? $bot_options : array();
1949		-}
1950	1418
1951		-/**
1952		- * Get bot-specific Pinecone configuration
1953		- * Used in the knowledge retrieval functions
1954		- */
1955		-// Also add debugging to your get_bot_pinecone_config function
1956		-private function get_bot_pinecone_config($bot_id = 'default') {
1957		- //error_log("MXCHAT DEBUG: get_bot_pinecone_config called for bot: " . $bot_id);
1958		-
1959		- // If default bot or multi-bot add-on not active, use default Pinecone config
1960		- if ($bot_id === 'default' || !class_exists('MxChat_Multi_Bot_Manager')) {
1961		- //error_log("MXCHAT DEBUG: Using default Pinecone config (no multi-bot or bot is 'default')");
1962		- $addon_options = get_option('mxchat_pinecone_addon_options', array());
1963		- $config = array(
1964		- 'use_pinecone' => (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1'),
1965		- 'api_key' => $addon_options['mxchat_pinecone_api_key'] ?? '',
1966		- 'host' => $addon_options['mxchat_pinecone_host'] ?? '',
1967		- 'namespace' => $addon_options['mxchat_pinecone_namespace'] ?? ''
1968		- );
1969		- //error_log("MXCHAT DEBUG: Default config - use_pinecone: " . ($config['use_pinecone'] ? 'true' : 'false'));
1970		- return $config;
	1419	+ if (!empty($this->fallbackResponse['html'])) {
	1420	+ $this->mxchat_save_chat_message($session_id, 'bot', $this->fallbackResponse['html']);
1971	1421	}
1972		-
1973		- //error_log("MXCHAT DEBUG: Calling filter 'mxchat_get_bot_pinecone_config' for bot: " . $bot_id);
1974		-
1975		- // Hook for multi-bot add-on to provide bot-specific Pinecone config
1976		- $bot_pinecone_config = apply_filters('mxchat_get_bot_pinecone_config', array(), $bot_id);
1977		-
1978		- if (!empty($bot_pinecone_config)) {
1979		- //error_log("MXCHAT DEBUG: Got bot-specific config from filter");
1980		- //error_log(" - use_pinecone: " . (isset($bot_pinecone_config['use_pinecone']) ? ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false') : 'not set'));
1981		- //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'not set'));
1982		- //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'not set'));
1983		- } else {
1984		- //error_log("MXCHAT DEBUG: Filter returned empty config!");
	1422	+
	1423	+ // Step 6: Return the response
	1424	+ $response_data = [
	1425	+ 'text' => $response,
	1426	+ 'html' => !empty($this->productCardHtml) ? $this->productCardHtml : ($this->fallbackResponse['html'] ?? ''),
	1427	+ 'session_id' => $session_id
	1428	+ ];
	1429	+
	1430	+ // Always add testing data for admins (no toggle needed)
	1431	+ if ($testing_data !== null) {
	1432	+ $response_data['testing_data'] = $testing_data;
1985	1433	}
1986		-
1987		- return is_array($bot_pinecone_config) ? $bot_pinecone_config : array();
	1434	+
	1435	+ wp_send_json($response_data);
	1436	+ wp_die();
1988	1437	}
1989	1438
1990		-
1991	1439	// Updated function to check intents and invoke the callback function
1992	1440	private function mxchat_check_intent_and_invoke_callback($message, $user_id, $session_id) {
1993	1441	global $wpdb;
1994	1442	$chat_mode = get_option("mxchat_mode_{$session_id}", 'ai');
1995	1443
1996		- // Get the current bot_id
1997		- $current_bot_id = $this->get_current_bot_id($session_id);
1998		-
1999	1444	// Generate the user embedding
2000	1445	$user_embedding = $this->mxchat_generate_embedding($message, $this->options['api_key']);
2001		-
	1446	+
2002	1447	// Check if embedding generation returned an error
2003	1448	if (is_array($user_embedding) && isset($user_embedding['error'])) {
2004	1449	$error_message = $user_embedding['error'];
2005	1450	$error_code = $user_embedding['error_code'] ?? 'embedding_error';
2006		-
2007		- // FIXED: Send error in appropriate format based on streaming mode
2008		- if ($this->is_streaming) {
2009		- echo "data: " . json_encode([
2010		- 'error' => true,
2011		- 'error_message' => $error_message,
2012		- 'error_code' => $error_code,
2013		- 'text' => $error_message,
2014		- 'message' => $error_message
2015		- ]) . "\n\n";
2016		- echo "data: [DONE]\n\n";
2017		- flush();
2018		- } else {
2019		- wp_send_json_error([
2020		- 'error_message' => $error_message,
2021		- 'error_code' => $error_code
2022		- ]);
2023		- }
	1451	+
	1452	+ wp_send_json_error([
	1453	+ 'error_message' => $error_message,
	1454	+ 'error_code' => $error_code
	1455	+ ]);
2024	1456	wp_die();
2025	1457	}
2026		-
	1458	+
2027	1459	// Check if embedding is valid
2028	1460	if (!is_array($user_embedding) || empty($user_embedding)) {
2029		- $error_message = esc_html__('Unable to process your message. The embedding service is not responding correctly.', 'mxchat');
2030		-
2031		- // FIXED: Send error in appropriate format based on streaming mode
2032		- if ($this->is_streaming) {
2033		- echo "data: " . json_encode([
2034		- 'error' => true,
2035		- 'error_message' => $error_message,
2036		- 'error_code' => 'invalid_embedding',
2037		- 'text' => $error_message,
2038		- 'message' => $error_message
2039		- ]) . "\n\n";
2040		- echo "data: [DONE]\n\n";
2041		- flush();
2042		- } else {
2043		- wp_send_json_error([
2044		- 'error_message' => $error_message,
2045		- 'error_code' => 'invalid_embedding'
2046		- ]);
2047		- }
	1461	+ wp_send_json_error([
	1462	+ 'error_message' => esc_html__('Unable to process your message. The embedding service is not responding correctly.', 'mxchat'),
	1463	+ 'error_code' => 'invalid_embedding'
	1464	+ ]);
2048	1465	wp_die();
2049	1466	}
2050		-
	1467	+
2051	1468	// Fetch intents from the database
2052	1469	$table_name = $wpdb->prefix . 'mxchat_intents';
2053	1470	if ($chat_mode === 'agent') {
2054	1471	$query = $wpdb->prepare(
		@@ -2058,29 +1475,19 @@
2058	1475	$intents = $wpdb->get_results($query);
2059	1476	} else {
2060	1477	$intents = $wpdb->get_results("SELECT * FROM $table_name WHERE enabled = 1 OR enabled IS NULL");
2061	1478	}
2062		-
	1479	+
2063	1480	if (empty($intents)) {
2064	1481	return false;
2065	1482	}
2066		-
2067		- // Prefetch individual phrase vectors from wp_mxchat_intent_phrases (grouped by intent_id)
2068		- $phrases_table = $wpdb->prefix . 'mxchat_intent_phrases';
2069		- $phrases_by_intent = [];
2070		- if ($wpdb->get_var("SHOW TABLES LIKE '$phrases_table'") === $phrases_table) {
2071		- $all_phrases = $wpdb->get_results("SELECT intent_id, phrase, embedding_vector FROM $phrases_table");
2072		- foreach ($all_phrases as $p) {
2073		- $phrases_by_intent[$p->intent_id][] = $p;
2074		- }
2075		- }
2076		-
	1483	+
2077	1484	$highest_similarity = -INF;
2078	1485	$matched_intent = null;
2079		-
2080		- // Array to store action analysis for testing panel
	1486	+
	1487	+ // NEW: Array to store action analysis for testing panel
2081	1488	$action_analysis = [];
2082		-
	1489	+
2083	1490	foreach ($intents as $intent) {
2084	1491	// Additional check for enabled state
2085	1492	$is_enabled = isset($intent->enabled) ? (bool)$intent->enabled : true;
2086	1493	if (!$is_enabled) {
		@@ -2085,57 +1492,22 @@
2085	1492	$is_enabled = isset($intent->enabled) ? (bool)$intent->enabled : true;
2086	1493	if (!$is_enabled) {
2087	1494	continue;
2088	1495	}
2089		-
2090		- // Check if this action is enabled for the current bot
2091		- if (!$this->is_action_enabled_for_bot($intent, $current_bot_id)) {
2092		- continue;
2093		- }
2094		-
2095		- $best_similarity = -INF;
2096		- $matched_phrase_text = '';
2097		-
2098		- // Check legacy embedding vector (existing behavior)
	1496	+
2099	1497	$intent_embedding_serialized = $intent->embedding_vector;
2100	1498	$intent_embedding = $intent_embedding_serialized
2101	1499	? unserialize($intent_embedding_serialized, ['allowed_classes' => false])
2102	1500	: null;
2103		-
2104		- if (is_array($intent_embedding) && !empty($intent_embedding)) {
2105		- $legacy_similarity = $this->mxchat_calculate_cosine_similarity($user_embedding, $intent_embedding);
2106		- if ($legacy_similarity > $best_similarity) {
2107		- $best_similarity = $legacy_similarity;
2108		- $matched_phrase_text = 'legacy';
2109		- }
2110		- }
2111		-
2112		- // Check individual phrase vectors
2113		- if (isset($phrases_by_intent[$intent->id])) {
2114		- foreach ($phrases_by_intent[$intent->id] as $phrase_row) {
2115		- $phrase_embedding = $phrase_row->embedding_vector
2116		- ? unserialize($phrase_row->embedding_vector, ['allowed_classes' => false])
2117		- : null;
2118		- if (!is_array($phrase_embedding)) {
2119		- continue;
2120		- }
2121		- $phrase_similarity = $this->mxchat_calculate_cosine_similarity($user_embedding, $phrase_embedding);
2122		- if ($phrase_similarity > $best_similarity) {
2123		- $best_similarity = $phrase_similarity;
2124		- $matched_phrase_text = $phrase_row->phrase;
2125		- }
2126		- }
2127		- }
2128		-
2129		- // Skip if no valid embedding was found at all
2130		- if ($best_similarity === -INF) {
	1501	+
	1502	+ if (!is_array($intent_embedding)) {
2131	1503	continue;
2132	1504	}
2133		-
2134		- $similarity = $best_similarity;
	1505	+
	1506	+ $similarity = $this->mxchat_calculate_cosine_similarity($user_embedding, $intent_embedding);
2135	1507	$intent_threshold = isset($intent->similarity_threshold) ? $intent->similarity_threshold : 0.85;
2136		-
2137		- // Store action analysis data for testing panel
	1508	+
	1509	+ // NEW: Store action analysis data for testing panel
2138	1510	$action_analysis[] = [
2139	1511	'intent_label' => $intent->intent_label,
2140	1512	'callback_function' => $intent->callback_function,
2141	1513	'similarity' => round($similarity, 4),
		@@ -2142,12 +1514,11 @@
2142	1514	'similarity_percentage' => round($similarity * 100, 2),
2143	1515	'threshold' => $intent_threshold,
2144	1516	'threshold_percentage' => round($intent_threshold * 100, 2),
2145	1517	'above_threshold' => $similarity >= $intent_threshold,
2146		- 'matched_phrase' => $matched_phrase_text,
2147	1518	'triggered' => false // Will be updated below if this intent is triggered
2148	1519	];
2149		-
	1520	+
2150	1521	if ($similarity >= $intent_threshold && $similarity > $highest_similarity) {
2151	1522	$highest_similarity = $similarity;
2152	1523	$matched_intent = $intent;
2153	1524	}
		@@ -2152,9 +1523,9 @@
2152	1523	$matched_intent = $intent;
2153	1524	}
2154	1525	}
2155	1526
2156		- // Mark the triggered action if any
	1527	+ // NEW: Mark the triggered action if any
2157	1528	if ($matched_intent) {
2158	1529	foreach ($action_analysis as &$action) {
2159	1530	if ($action['intent_label'] === $matched_intent->intent_label) {
2160	1531	$action['triggered'] = true;
		@@ -2162,9 +1533,9 @@
2162	1533	}
2163	1534	}
2164	1535	}
2165	1536
2166		- // Sort actions by similarity (highest first) and store for testing panel
	1537	+ // NEW: Sort actions by similarity (highest first) and store for testing panel
2167	1538	usort($action_analysis, function($a, $b) {
2168	1539	return $b['similarity'] <=> $a['similarity'];
2169	1540	});
2170	1541
		@@ -2171,73 +1542,47 @@
2171	1542	// Store action analysis for testing panel capture
2172	1543	$this->last_action_analysis = $action_analysis;
2173	1544
2174	1545	// Around line 715 in your mxchat_check_intent_and_invoke_callback function
2175		- if ($matched_intent) {
2176		- // If the callback is a method on this instance (core callback), call it directly
2177		- if (method_exists($this, $matched_intent->callback_function)) {
2178		- $callback_result = call_user_func(
2179		- [$this, $matched_intent->callback_function],
2180		- $message,
2181		- $user_id,
2182		- $session_id,
2183		- $matched_intent,
2184		- $user_context ?? null
2185		- );
	1546	+if ($matched_intent) {
	1547	+ // If the callback is a method on this instance (core callback), call it directly
	1548	+ if (method_exists($this, $matched_intent->callback_function)) {
	1549	+ $callback_result = call_user_func(
	1550	+ [$this, $matched_intent->callback_function],
	1551	+ $message,
	1552	+ $user_id,
	1553	+ $session_id,
	1554	+ $matched_intent,
	1555	+ $user_context ?? null
	1556	+ );
	1557	+ } else {
	1558	+ // Otherwise, use apply_filters for add-on callbacks
	1559	+ $callback_result = apply_filters(
	1560	+ $matched_intent->callback_function,
	1561	+ false,
	1562	+ $message,
	1563	+ $user_id,
	1564	+ $session_id,
	1565	+ $matched_intent
	1566	+ );
	1567	+ }
	1568	+
	1569	+ // Handle the callback result properly
	1570	+ if ($callback_result !== false) {
	1571	+ // If callback returned an array with chat_mode, use it directly
	1572	+ if (is_array($callback_result) && isset($callback_result['chat_mode'])) {
	1573	+ $this->fallbackResponse = $callback_result;
	1574	+ return $callback_result; // Return the full array
2186	1575	} else {
2187		- // Otherwise, use apply_filters for add-on callbacks
2188		- $callback_result = apply_filters(
2189		- $matched_intent->callback_function,
2190		- false,
2191		- $message,
2192		- $user_id,
2193		- $session_id,
2194		- $matched_intent
2195		- );
	1576	+ $this->fallbackResponse = $callback_result;
	1577	+ return true;
2196	1578	}
2197		-
2198		- // Handle the callback result properly
2199		- if ($callback_result !== false) {
2200		- // If callback returned an array with chat_mode, use it directly
2201		- if (is_array($callback_result) && isset($callback_result['chat_mode'])) {
2202		- $this->fallbackResponse = $callback_result;
2203		- return $callback_result; // Return the full array
2204		- } else {
2205		- $this->fallbackResponse = $callback_result;
2206		- return true;
2207		- }
2208		- }
2209	1579	}
	1580	+}
2210	1581
2211	1582	return false;
2212	1583	}
2213	1584
2214		-/**
2215		- * Check if an action is enabled for a specific bot
2216		- */
2217		-private function is_action_enabled_for_bot($intent, $bot_id) {
2218		- // If enabled_bots column doesn't exist or is null, assume it's enabled for all bots (backward compatibility)
2219		- if (!isset($intent->enabled_bots) || empty($intent->enabled_bots)) {
2220		- return true;
2221		- }
2222		-
2223		- $enabled_bots = json_decode($intent->enabled_bots, true);
2224		-
2225		- // If JSON decode fails or returns empty array, assume enabled for all (backward compatibility)
2226		- if (!is_array($enabled_bots) || empty($enabled_bots)) {
2227		- return true;
2228		- }
2229		-
2230		- // Admin testing tab uses bot_id "testing" — treat it as "default" so all
2231		- // default-bot actions are testable from the admin panel
2232		- if ($bot_id === 'testing') {
2233		- $bot_id = 'default';
2234		- }
2235		-
2236		- // Check if the current bot is in the enabled bots list
2237		- return in_array($bot_id, $enabled_bots);
2238		-}
2239		-
2240	1585	// Helper function to clear PDF and Word document related transients
2241	1586	private function clear_pdf_transients($session_id) {
2242	1587	// PDF transients
2243	1588	delete_transient('mxchat_pdf_url_' . $session_id);
		@@ -2272,17 +1617,17 @@
2272	1617
2273	1618	public function mxchat_generate_image($message, $user_id, $session_id) {
2274	1619	//error_log("Starting image generation for message: " . $message);
2275	1620
2276		- // Prepare a prompt for OpenAI image generation
	1621	+ // Prepare a prompt for DALL-E
2277	1622	$prompt = esc_html__('Create an image of ', 'mxchat') . sanitize_text_field($message);
2278		-
	1623	+
2279	1624	// Use the existing OpenAI API key
2280	1625	$openai_api_key = sanitize_text_field($this->options['api_key']);
2281		-
2282		- // Call OpenAI GPT Image to generate an image
2283		- $image_response = $this->mxchat_generate_openai_image($prompt, $openai_api_key);
2284	1626
	1627	+ // Call DALL-E to generate an image
	1628	+ $image_response = $this->mxchat_generate_dalle_image($prompt, $openai_api_key);
	1629	+
2285	1630	// Check if the response contains an image URL
2286	1631	if (isset($image_response['imageUrl'])) {
2287	1632	$image_url = esc_url_raw($image_response['imageUrl']);
2288	1633
		@@ -2325,103 +1670,24 @@
2325	1670	// Return the response directly instead of relying on the property
2326	1671	return $this->fallbackResponse;
2327	1672	}
2328	1673	}
2329		-
2330		-public function mxchat_generate_gemini_image($message, $user_id, $session_id) {
2331		- $prompt = esc_html__('Create an image of ', 'mxchat') . sanitize_text_field($message);
2332		-
2333		- $gemini_api_key = sanitize_text_field($this->options['gemini_api_key'] ?? '');
2334		- if (empty($gemini_api_key)) {
2335		- $response_text = esc_html__("Gemini API key is not configured.", 'mxchat');
2336		- $this->mxchat_save_chat_message($session_id, 'bot', $response_text);
2337		- return ['text' => $response_text, 'html' => '', 'images' => []];
2338		- }
2339		-
2340		- $image_response = $this->mxchat_generate_imagen_image($prompt, $gemini_api_key);
2341		-
2342		- if (isset($image_response['imageUrl'])) {
2343		- $image_url = esc_url_raw($image_response['imageUrl']);
2344		-
2345		- $response_html = '<img src="' . esc_url($image_url) . '" alt="' . esc_attr__('Generated Image', 'mxchat') . '" class="mxchat-generated-image" />';
2346		- $response_text = esc_html__('Here is the image I generated:', 'mxchat');
2347		-
2348		- $this->mxchat_save_chat_message($session_id, 'bot', $response_text);
2349		- $this->mxchat_save_chat_message($session_id, 'bot', $response_html);
2350		-
2351		- $this->fallbackResponse = [
2352		- 'text' => $response_text,
2353		- 'html' => $response_html,
2354		- 'images' => [$image_url]
2355		- ];
2356		-
2357		- return $this->fallbackResponse;
2358		- } else {
2359		- $response_text = esc_html__("I'm sorry, but I couldn't generate an image based on your request.", 'mxchat');
2360		-
2361		- $this->mxchat_save_chat_message($session_id, 'bot', $response_text);
2362		-
2363		- $this->fallbackResponse = [
2364		- 'text' => $response_text,
2365		- 'html' => '',
2366		- 'images' => []
2367		- ];
2368		-
2369		- return $this->fallbackResponse;
2370		- }
2371		-}
2372		-
2373		-private function mxchat_save_generated_image($base64_data, $mime_type = 'image/png', $prefix = 'mxchat-generated') {
2374		- $extension = ($mime_type === 'image/jpeg') ? 'jpg' : 'png';
2375		- $filename = sanitize_file_name($prefix . '-' . wp_generate_uuid4() . '.' . $extension);
2376		- $decoded = base64_decode($base64_data);
2377		-
2378		- if ($decoded === false) {
2379		- return new \WP_Error('decode_failed', esc_html__('Failed to decode image data.', 'mxchat'));
2380		- }
2381		-
2382		- $upload = wp_upload_bits($filename, null, $decoded);
2383		-
2384		- if (!empty($upload['error'])) {
2385		- return new \WP_Error('upload_failed', $upload['error']);
2386		- }
2387		-
2388		- $attach_id = wp_insert_attachment([
2389		- 'post_mime_type' => $mime_type,
2390		- 'post_title' => $prefix,
2391		- 'post_content' => '',
2392		- 'post_status' => 'inherit',
2393		- ], $upload['file']);
2394		-
2395		- if (is_wp_error($attach_id)) {
2396		- return $attach_id;
2397		- }
2398		-
2399		- require_once ABSPATH . 'wp-admin/includes/image.php';
2400		- $metadata = wp_generate_attachment_metadata($attach_id, $upload['file']);
2401		- wp_update_attachment_metadata($attach_id, $metadata);
2402		-
2403		- return esc_url_raw(wp_get_attachment_url($attach_id));
2404		-}
2405		-
2406		-private function mxchat_generate_openai_image($prompt, $api_key, $model = 'gpt-image-1', $timeout = 60) {
	1674	+private function mxchat_generate_dalle_image($prompt, $api_key, $model = 'dall-e-3', $timeout = 60) {
2407	1675	$api_url = 'https://api.openai.com/v1/images/generations';
2408	1676	$body = json_encode([
2409		- 'prompt' => sanitize_text_field($prompt),
2410		- 'n' => 1,
2411		- 'size' => '1024x1024',
2412		- 'quality' => 'medium',
2413		- 'output_format' => 'png',
2414		- 'model' => sanitize_text_field($model),
	1677	+ 'prompt' => sanitize_text_field($prompt),
	1678	+ 'n' => 1,
	1679	+ 'size' => '1024x1024',
	1680	+ 'model' => sanitize_text_field($model),
2415	1681	]);
2416	1682
2417	1683	$args = [
2418		- 'body' => $body,
	1684	+ 'body' => $body,
2419	1685	'headers' => [
2420		- 'Content-Type' => 'application/json',
	1686	+ 'Content-Type' => 'application/json',
2421	1687	'Authorization' => 'Bearer ' . sanitize_text_field($api_key),
2422	1688	],
2423		- 'method' => 'POST',
	1689	+ 'method' => 'POST',
2424	1690	'timeout' => absint($timeout),
2425	1691	];
2426	1692
2427	1693	$response = wp_remote_post($api_url, $args);
		@@ -2426,67 +1692,22 @@
2426	1692
2427	1693	$response = wp_remote_post($api_url, $args);
2428	1694
2429	1695	if (is_wp_error($response)) {
	1696	+ //error_log("DALL-E request failed: " . $response->get_error_message());
2430	1697	return ['error' => esc_html__('Error generating image: ', 'mxchat') . $response->get_error_message()];
2431	1698	}
2432	1699
2433	1700	$response_body = json_decode(wp_remote_retrieve_body($response), true);
2434	1701
2435		- $b64 = $response_body['data'][0]['b64_json'] ?? $response_body['data'][0]['b64'] ?? null;
2436		- if ($b64) {
2437		- $saved_url = $this->mxchat_save_generated_image($b64, 'image/png', 'mxchat-openai');
2438		- if (is_wp_error($saved_url)) {
2439		- return ['error' => $saved_url->get_error_message()];
2440		- }
2441		- return ['imageUrl' => $saved_url];
	1702	+ if (isset($response_body['data'][0]['url'])) {
	1703	+ return ['imageUrl' => esc_url_raw($response_body['data'][0]['url'])];
2442	1704	} else {
	1705	+ //error_log("DALL-E response error: " . wp_remote_retrieve_body($response));
2443	1706	return ['error' => esc_html__('Failed to generate image.', 'mxchat')];
2444	1707	}
2445	1708	}
2446	1709
2447		-private function mxchat_generate_imagen_image($prompt, $api_key, $timeout = 60) {
2448		- $api_url = 'https://generativelanguage.googleapis.com/v1beta/models/imagen-4.0-generate-001:predict';
2449		-
2450		- $body = json_encode([
2451		- 'instances' => [['prompt' => sanitize_text_field($prompt)]],
2452		- 'parameters' => [
2453		- 'sampleCount' => 1,
2454		- 'aspectRatio' => '1:1',
2455		- ],
2456		- ]);
2457		-
2458		- $args = [
2459		- 'body' => $body,
2460		- 'headers' => [
2461		- 'Content-Type' => 'application/json',
2462		- 'x-goog-api-key' => sanitize_text_field($api_key),
2463		- ],
2464		- 'method' => 'POST',
2465		- 'timeout' => absint($timeout),
2466		- ];
2467		-
2468		- $response = wp_remote_post($api_url, $args);
2469		-
2470		- if (is_wp_error($response)) {
2471		- return ['error' => esc_html__('Error generating image: ', 'mxchat') . $response->get_error_message()];
2472		- }
2473		-
2474		- $response_body = json_decode(wp_remote_retrieve_body($response), true);
2475		-
2476		- $b64 = $response_body['predictions'][0]['bytesBase64Encoded'] ?? $response_body['predictions'][0]['imageBytes'] ?? null;
2477		- if ($b64) {
2478		- $mime = $response_body['predictions'][0]['mimeType'] ?? 'image/png';
2479		- $saved_url = $this->mxchat_save_generated_image($b64, $mime, 'mxchat-gemini');
2480		- if (is_wp_error($saved_url)) {
2481		- return ['error' => $saved_url->get_error_message()];
2482		- }
2483		- return ['imageUrl' => $saved_url];
2484		- } else {
2485		- return ['error' => esc_html__('Failed to generate image.', 'mxchat')];
2486		- }
2487		-}
2488		-
2489	1710	/**
2490	1711	* Handle web search requests.
2491	1712	*
2492	1713	* Sends the refined search query to the Brave Search API and uses the
		@@ -2535,10 +1756,10 @@
2535	1756	$transient_key = 'mxchat_search_' . md5($refined_search_query);
2536	1757	$results = get_transient($transient_key);
2537	1758
2538	1759	if (false === $results) {
2539		- // SECURITY FIX: Changed to wp_safe_remote_get
2540		- $response = wp_safe_remote_get(
	1760	+ // Fetch new results from the Brave Search API
	1761	+ $response = wp_remote_get(
2541	1762	$api_url,
2542	1763	array(
2543	1764	'headers' => array(
2544	1765	'Accept' => 'application/json',
		@@ -2677,10 +1898,9 @@
2677	1898	],
2678	1899	'timeout' => 10,
2679	1900	];
2680	1901
2681		- // SECURITY FIX: Changed to wp_safe_remote_get
2682		- $response = wp_safe_remote_get($api_url, $args);
	1902	+ $response = wp_remote_get($api_url, $args);
2683	1903
2684	1904	if (is_wp_error($response)) {
2685	1905	return array(
2686	1906	'text' => __("I'm sorry, I couldn't retrieve any images based on your request.", 'mxchat'),
		@@ -2753,9 +1973,9 @@
2753	1973	$system_prompt = esc_html__("Interpret the user's request to provide only the essential keywords or phrases for image searching. Remove conversational language, politeness, or extra context. Return a concise search query that doesn't lose any of the original meaning.", 'mxchat');
2754	1974
2755	1975	// Get options and determine the selected model
2756	1976	$options = $this->options ?? get_option('mxchat_options');
2757		- $selected_model = isset($options['model']) ? $options['model'] : 'gpt-5.1-chat-latest';
	1977	+ $selected_model = isset($options['model']) ? $options['model'] : 'gpt-4o';
2758	1978
2759	1979	// Extract model prefix to determine the provider
2760	1980	$model_parts = explode('-', $selected_model);
2761	1981	$provider = strtolower($model_parts[0]);
		@@ -2803,9 +2023,9 @@
2803	2023
2804	2024	/**
2805	2025	* Interpret query using OpenAI models
2806	2026	*/
2807		-private function interpret_query_with_openai($user_query, $system_prompt, $api_key, $model = 'gpt-5.1-chat-latest') {
	2027	+private function interpret_query_with_openai($user_query, $system_prompt, $api_key, $model = 'gpt-4o') {
2808	2028	$url = 'https://api.openai.com/v1/chat/completions';
2809	2029	$args = [
2810	2030	'headers' => [
2811	2031	'Authorization' => 'Bearer ' . $api_key,
		@@ -2876,13 +2096,13 @@
2876	2096	/**
2877	2097	* Interpret query using Gemini models
2878	2098	*/
2879	2099	private function interpret_query_with_gemini($user_query, $system_prompt, $api_key, $model) {
2880		- // Use v1beta for preview models, v1 for stable models
2881		- $api_version = (strpos($model, 'preview') !== false || strpos($model, 'exp') !== false) ? 'v1beta' : 'v1';
2882		-
2883		- $url = "https://generativelanguage.googleapis.com/{$api_version}/models/{$model}:generateContent?key=" . urlencode($api_key);
	2100	+ // Strip "gemini-" prefix for the API
	2101	+ $model_version = str_replace('gemini-', '', $model);
2884	2102
	2103	+ $url = "https://generativelanguage.googleapis.com/v1/models/$model_version:generateContent?key=" . urlencode($api_key);
	2104	+
2885	2105	$args = [
2886	2106	'headers' => [
2887	2107	'Content-Type' => 'application/json',
2888	2108	],
		@@ -3078,9 +2298,9 @@
3078	2298	}
3079	2299
3080	2300
3081	2301	/**
3082		- * Enhanced fetch_and_split_pdf_pages with SSRF protection
	2302	+ * Enhanced fetch_and_split_pdf_pages with detailed debugging
3083	2303	*/
3084	2304	private function fetch_and_split_pdf_pages($pdf_source, $max_pages) {
3085	2305	// CLEAR DEBUG LOGGING
3086	2306	//error_log("=== MXCHAT PDF PROCESSING START ===");
		@@ -3135,19 +2355,10 @@
3135	2355	// (I'll include the key parts with debug logging)
3136	2356
3137	2357	if (filter_var($pdf_source, FILTER_VALIDATE_URL)) {
3138	2358	//error_log("Downloading PDF from URL...");
3139		-
3140		- // SECURITY FIX: Validate URL before processing
3141		- if (!$this->mxchat_is_safe_pdf_url($pdf_source)) {
3142		- //error_log("❌ SECURITY: Blocked unsafe PDF URL");
3143		- return false;
3144		- }
3145		-
3146	2359	$temp_file = wp_tempnam($pdf_source);
3147		-
3148		- // SECURITY FIX: Changed from wp_remote_get to wp_safe_remote_get
3149		- $response = wp_safe_remote_get($pdf_source, [
	2360	+ $response = wp_remote_get($pdf_source, [
3150	2361	'timeout' => 60,
3151	2362	'headers' => ['User-Agent' => 'MxChat PDF Processor']
3152	2363	]);
3153	2364
		@@ -3156,14 +2367,9 @@
3156	2367	//error_log("❌ BASIC PROCESSING: Failed to download PDF: " . $error_message);
3157	2368	return false;
3158	2369	}
3159	2370
3160		- global $wp_filesystem;
3161		- if (empty($wp_filesystem)) {
3162		- require_once ABSPATH . 'wp-admin/includes/file.php';
3163		- WP_Filesystem();
3164		- }
3165		- $wp_filesystem->put_contents($temp_file, wp_remote_retrieve_body($response), FS_CHMOD_FILE);
	2371	+ file_put_contents($temp_file, wp_remote_retrieve_body($response));
3166	2372	//error_log("✅ PDF downloaded successfully");
3167	2373	} else {
3168	2374	$temp_file = $pdf_source;
3169	2375	//error_log("Using local PDF file: " . $temp_file);
		@@ -3170,9 +2376,8 @@
3170	2376	}
3171	2377
3172	2378	// Parse PDF
3173	2379	//error_log("Parsing PDF with basic parser...");
3174		- mxchat_load_pdf_parser();
3175	2380	$parser = new \Smalot\PdfParser\Parser();
3176	2381	$pdf = $parser->parseFile($temp_file);
3177	2382	$pages = $pdf->getPages();
3178	2383
		@@ -3235,33 +2440,8 @@
3235	2440	return false;
3236	2441	}
3237	2442	}
3238	2443
3239		-
3240		-/**
3241		- * Validate PDF URL for security
3242		- * Prevents SSRF attacks by blocking dangerous URLs
3243		- */
3244		-
3245		-private function mxchat_is_safe_pdf_url($url) {
3246		- // Use WordPress core function for comprehensive validation
3247		- // This blocks localhost, private IPs, and reserved IP ranges
3248		- $validated_url = wp_http_validate_url($url);
3249		-
3250		- if ($validated_url === false) {
3251		- return false;
3252		- }
3253		-
3254		- // Additional check: only allow HTTP/HTTPS schemes
3255		- $parsed = parse_url($url);
3256		- if (!isset($parsed['scheme']) || !in_array($parsed['scheme'], ['http', 'https'], true)) {
3257		- return false;
3258		- }
3259		-
3260		- return true;
3261		-}
3262		-
3263		-
3264	2444	private function mxchat_clean_text($text) {
3265	2445	// Remove excessive whitespace
3266	2446	$text = preg_replace('/\s+/', ' ', $text);
3267	2447
		@@ -3300,10 +2480,9 @@
3300	2480	}
3301	2481
3302	2482	return [];
3303	2483	}
3304		-
3305		-
	2484	+// Add this to your class
3306	2485	public function handle_pdf_upload() {
3307	2486	check_ajax_referer('mxchat_chat_nonce', 'nonce');
3308	2487
3309	2488	if (!isset($_FILES['pdf_file']) || !isset($_POST['session_id'])) {
		@@ -3310,29 +2489,12 @@
3310	2489	wp_send_json_error(esc_html__('Missing required parameters.', 'mxchat'));
3311	2490	return;
3312	2491	}
3313	2492
3314		- // SECURITY FIX: Check if PDF uploads are enabled in settings
3315		- $options = get_option('mxchat_options', array());
3316		- $show_pdf_button = isset($options['show_pdf_upload_button']) ? $options['show_pdf_upload_button'] : 'on';
3317		-
3318		- if ($show_pdf_button !== 'on') {
3319		- wp_send_json_error(esc_html__('PDF uploads are currently disabled.', 'mxchat'));
3320		- return;
3321		- }
3322		-
3323	2493	$file = $_FILES['pdf_file'];
3324	2494	$session_id = sanitize_text_field($_POST['session_id']);
3325	2495	$original_filename = sanitize_text_field($file['name']);
3326	2496
3327		- // Update session owner if it changed (e.g. IP changed due to network switch)
3328		- $current_user_identifier = MxChat_User::mxchat_get_user_identifier();
3329		- $session_owner = get_option("mxchat_session_owner_{$session_id}");
3330		-
3331		- if (!$session_owner || $session_owner !== $current_user_identifier) {
3332		- update_option("mxchat_session_owner_{$session_id}", $current_user_identifier, 'no');
3333		- }
3334		-
3335	2497	$file_type = wp_check_filetype($file['name'], ['pdf' => 'application/pdf']);
3336	2498	if ($file_type['type'] !== 'application/pdf') {
3337	2499	wp_send_json_error(esc_html__('Invalid file type. Only PDF files are allowed.', 'mxchat'));
3338	2500	return;
		@@ -3338,12 +2500,9 @@
3338	2500	return;
3339	2501	}
3340	2502
3341	2503	$upload_dir = wp_upload_dir();
3342		-
3343		- // SECURITY FIX: Generate random filename without exposing session_id
3344		- $random_string = wp_generate_password(20, false, false); // 20 char alphanumeric string
3345		- $pdf_filename = 'mxchat_' . $random_string . '_' . time() . '.pdf';
	2504	+ $pdf_filename = 'mxchat_' . $session_id . '_' . time() . '.pdf';
3346	2505	$pdf_path = $upload_dir['path'] . '/' . $pdf_filename;
3347	2506
3348	2507	if (!move_uploaded_file($file['tmp_name'], $pdf_path)) {
3349	2508	wp_send_json_error(esc_html__('Failed to upload file.', 'mxchat'));
		@@ -3374,9 +2533,8 @@
3374	2533	return;
3375	2534	}
3376	2535
3377	2536	if (!empty($embeddings)) {
3378		- // Store the mapping between session and the random filename
3379	2537	set_transient('mxchat_pdf_url_' . $session_id, $pdf_path, HOUR_IN_SECONDS);
3380	2538	set_transient('mxchat_pdf_filename_' . $session_id, $original_filename, HOUR_IN_SECONDS);
3381	2539	set_transient('mxchat_pdf_embeddings_' . $session_id, $embeddings, HOUR_IN_SECONDS);
3382	2540	set_transient('mxchat_include_pdf_in_context_' . $session_id, true, HOUR_IN_SECONDS);
		@@ -3420,8 +2578,10 @@
3420	2578	wp_die();
3421	2579	}
3422	2580
3423	2581
	2582	+
	2583	+
3424	2584	function mxchat_fetch_new_messages() {
3425	2585	$session_id = sanitize_text_field($_POST['session_id']);
3426	2586	$last_seen_id = sanitize_text_field($_POST['last_seen_id']);
3427	2587	$persistence_enabled = $_POST['persistence_enabled'] === 'true';
		@@ -3434,31 +2594,14 @@
3434	2594	}
3435	2595
3436	2596	$history = get_option("mxchat_history_{$session_id}", []);
3437	2597
3438		- //error_log("MxChat WhatsApp DEBUG: Fetch new messages for session {$session_id}");
3439		- //error_log("MxChat WhatsApp DEBUG: last_seen_id = " . var_export($last_seen_id, true));
3440		- //error_log("MxChat WhatsApp DEBUG: History count = " . count($history));
3441		- //error_log("MxChat WhatsApp DEBUG: Full history = " . print_r($history, true));
3442		-
3443	2598	$new_messages = array_filter($history, function ($message) use ($last_seen_id, $persistence_enabled, $initial_timestamp) {
3444		- //error_log("MxChat WhatsApp DEBUG: Checking message - ID: " . ($message['id'] ?? 'NO_ID') . ", Role: " . ($message['role'] ?? 'NO_ROLE'));
3445		-
3446	2599	// If persistence is enabled, show all new messages
3447	2600	if ($persistence_enabled) {
3448		- $has_id = !empty($message['id']);
3449		- $is_agent = $message['role'] === 'agent';
3450		-
3451		- // If last_seen_id is empty, 'NaN', or invalid, show all agent messages
3452		- if (empty($last_seen_id) || $last_seen_id === 'NaN' || $last_seen_id === 'undefined') {
3453		- $is_newer = true;
3454		- } else {
3455		- $is_newer = strcmp($message['id'] ?? '', $last_seen_id) > 0;
3456		- }
3457		-
3458		- //error_log("MxChat WhatsApp DEBUG: has_id={$has_id}, is_newer={$is_newer}, is_agent={$is_agent}");
3459		-
3460		- return $has_id && $is_newer && $is_agent;
	2601	+ return !empty($message['id']) &&
	2602	+ strcmp($message['id'], $last_seen_id) > 0 &&
	2603	+ $message['role'] === 'agent';
3461	2604	}
3462	2605
3463	2606	// If persistence is disabled, only show messages after initial timestamp
3464	2607	return !empty($message['id']) &&
		@@ -3465,16 +2608,12 @@
3465	2608	$message['role'] === 'agent' &&
3466	2609	$message['timestamp'] > $initial_timestamp;
3467	2610	});
3468	2611
3469		- //error_log("MxChat WhatsApp DEBUG: Filtered messages count = " . count($new_messages));
	2612	+ //error_log(esc_html__("New agent messages fetched for session $session_id. Last seen ID: $last_seen_id", 'mxchat'));
3470	2613
3471		- // Include current chat mode so frontend can detect agent→AI transitions
3472		- $chat_mode = get_option("mxchat_mode_{$session_id}", 'ai');
3473		-
3474	2614	wp_send_json_success([
3475		- 'new_messages' => array_values($new_messages),
3476		- 'chat_mode' => $chat_mode
	2615	+ 'new_messages' => array_values($new_messages)
3477	2616	]);
3478	2617	wp_die();
3479	2618	}
3480	2619	public function mxchat_live_agent_handover($message, $user_id, $session_id) {
		@@ -3759,393 +2898,9 @@
3759	2898
3760	2899	//error_log("[DEBUG] Generated channel name: {$channel_name}");
3761	2900	return $channel_name;
3762	2901	}
3763		-
3764		-/**
3765		- * Telegram Live Agent Handover
3766		- * Creates a forum topic in the Telegram group and notifies agents
3767		- */
3768		-public function mxchat_telegram_live_agent_handover($message, $user_id, $session_id) {
3769		- // Check if Telegram agents are available
3770		- $telegram_available = $this->options['telegram_status'] ?? 'off';
3771		- if ($telegram_available !== 'on') {
3772		- $away_message = $this->options['telegram_away_message'] ?? 'Sorry, live agents are currently unavailable. I can continue helping you as an AI assistant.';
3773		- $this->fallbackResponse = [
3774		- 'text' => $away_message,
3775		- 'html' => '',
3776		- 'images' => [],
3777		- 'chat_mode' => 'ai'
3778		- ];
3779		- wp_send_json([
3780		- 'text' => $away_message,
3781		- 'html' => '',
3782		- 'chat_mode' => 'ai',
3783		- 'session_id' => $session_id
3784		- ]);
3785		- wp_die();
3786		- }
3787		-
3788		- $telegram_bot_token = $this->options['telegram_bot_token'] ?? '';
3789		- $telegram_group_id = $this->options['telegram_group_id'] ?? '';
3790		-
3791		- if (empty($telegram_bot_token) || empty($telegram_group_id)) {
3792		- return false;
3793		- }
3794		-
3795		- // Check if topic already exists for this session
3796		- $topic_id = get_option("mxchat_telegram_topic_{$session_id}", '');
3797		-
3798		- if (empty($topic_id)) {
3799		- // Generate topic name
3800		- $topic_name = $this->generate_telegram_topic_name($session_id);
3801		-
3802		- // Random icon color (Telegram forum topic colors)
3803		- $icon_colors = [0x6FB9F0, 0xFFD67E, 0xCB86DB, 0x8EEE98, 0xFF93B2, 0xFB6F5F];
3804		- $icon_color = $icon_colors[array_rand($icon_colors)];
3805		-
3806		- // Create forum topic
3807		- $response = wp_remote_post("https://api.telegram.org/bot{$telegram_bot_token}/createForumTopic", [
3808		- 'headers' => ['Content-Type' => 'application/json'],
3809		- 'body' => json_encode([
3810		- 'chat_id' => $telegram_group_id,
3811		- 'name' => $topic_name,
3812		- 'icon_color' => $icon_color
3813		- ])
3814		- ]);
3815		-
3816		- if (!is_wp_error($response)) {
3817		- $response_body = wp_remote_retrieve_body($response);
3818		- $response_data = json_decode($response_body, true);
3819		-
3820		- if (isset($response_data['ok']) && $response_data['ok']) {
3821		- $topic_id = $response_data['result']['message_thread_id'];
3822		- update_option("mxchat_telegram_topic_{$session_id}", $topic_id);
3823		- update_option("mxchat_telegram_group_{$session_id}", $telegram_group_id);
3824		- }
3825		- }
3826		-
3827		- if (empty($topic_id)) {
3828		- return false; // Failed to create topic
3829		- }
3830		- }
3831		-
3832		- // Get recent chat history
3833		- $history = get_option("mxchat_history_{$session_id}", []);
3834		- $recent_history = array_slice($history, -5);
3835		-
3836		- // Format conversation context for Telegram (HTML format)
3837		- $conversation_context = "";
3838		- if (!empty($recent_history)) {
3839		- $conversation_context = "<b>Recent Conversation:</b>\n";
3840		- foreach ($recent_history as $hist_message) {
3841		- $role_display = $hist_message['role'] === 'user' ? '👤 User' : '🤖 AI';
3842		- $escaped_content = htmlspecialchars($hist_message['content'], ENT_QUOTES, 'UTF-8');
3843		- $conversation_context .= "{$role_display}: {$escaped_content}\n";
3844		- }
3845		- $conversation_context .= "\n";
3846		- }
3847		-
3848		- // Get user info
3849		- $user_email = get_option("mxchat_email_{$session_id}", 'Not provided');
3850		- $user_name = get_option("mxchat_name_{$session_id}", 'Anonymous');
3851		-
3852		- // Update session mode
3853		- update_option("mxchat_mode_{$session_id}", 'agent');
3854		-
3855		- // Send initial message to topic
3856		- $escaped_message = htmlspecialchars($message, ENT_QUOTES, 'UTF-8');
3857		- $topic_message = "🔔 <b>New Live Agent Request</b>\n\n";
3858		- $topic_message .= "<b>Session ID:</b> <code>{$session_id}</code>\n";
3859		- $topic_message .= "<b>User:</b> {$user_name}\n";
3860		- $topic_message .= "<b>Email:</b> {$user_email}\n\n";
3861		-
3862		- if (!empty($conversation_context)) {
3863		- $topic_message .= $conversation_context;
3864		- }
3865		-
3866		- $topic_message .= "<b>Current Message:</b>\n{$escaped_message}\n\n";
3867		- $topic_message .= "<i>Reply in this topic - messages will be sent to the user</i>\n";
3868		- $topic_message .= "<i>Type #close, #end, #disconnect, or #done to end the session</i>";
3869		-
3870		- wp_remote_post("https://api.telegram.org/bot{$telegram_bot_token}/sendMessage", [
3871		- 'headers' => ['Content-Type' => 'application/json'],
3872		- 'body' => json_encode([
3873		- 'chat_id' => $telegram_group_id,
3874		- 'message_thread_id' => $topic_id,
3875		- 'text' => $topic_message,
3876		- 'parse_mode' => 'HTML'
3877		- ])
3878		- ]);
3879		-
3880		- $success_message = $this->options['telegram_notification_message'] ?? "I've notified a support agent. Please allow a moment for them to respond.";
3881		- $this->mxchat_save_chat_message($session_id, 'bot', $success_message);
3882		-
3883		- $this->fallbackResponse = [
3884		- 'text' => $success_message,
3885		- 'html' => '',
3886		- 'images' => [],
3887		- 'chat_mode' => 'agent'
3888		- ];
3889		-
3890		- wp_send_json([
3891		- 'success' => true,
3892		- 'text' => $success_message,
3893		- 'html' => '',
3894		- 'chat_mode' => 'agent',
3895		- 'session_id' => $session_id,
3896		- 'fallbackResponse' => $this->fallbackResponse
3897		- ]);
3898		- wp_die();
3899		-}
3900		-
3901		-/**
3902		- * Generate topic name for Telegram forum
3903		- */
3904		-private function generate_telegram_topic_name($session_id) {
3905		- $name = null;
3906		- $email = null;
3907		-
3908		- // Check logged in user
3909		- if (is_user_logged_in()) {
3910		- $current_user = wp_get_current_user();
3911		- if (!empty($current_user->display_name)) {
3912		- $name = $current_user->display_name;
3913		- }
3914		- if (!empty($current_user->user_email)) {
3915		- $email = $current_user->user_email;
3916		- }
3917		- }
3918		-
3919		- // Check session data
3920		- if (empty($name)) {
3921		- $name = get_option("mxchat_name_{$session_id}");
3922		- }
3923		- if (empty($email)) {
3924		- $email = get_option("mxchat_email_{$session_id}");
3925		- }
3926		-
3927		- // Generate topic name
3928		- $session_suffix = substr($session_id, -6);
3929		-
3930		- if (!empty($name)) {
3931		- // Clean name for topic (max 128 chars in Telegram)
3932		- $clean_name = preg_replace('/[^\p{L}\p{N}\s\-]/u', '', $name);
3933		- $clean_name = trim($clean_name);
3934		- if (strlen($clean_name) > 50) {
3935		- $clean_name = substr($clean_name, 0, 50);
3936		- }
3937		- return "Chat - {$clean_name} ({$session_suffix})";
3938		- } elseif (!empty($email)) {
3939		- // Use email prefix
3940		- $email_prefix = explode('@', $email)[0];
3941		- if (strlen($email_prefix) > 30) {
3942		- $email_prefix = substr($email_prefix, 0, 30);
3943		- }
3944		- return "Chat - {$email_prefix} ({$session_suffix})";
3945		- }
3946		-
3947		- return "Chat - {$session_suffix}";
3948		-}
3949		-
3950		-/**
3951		- * Send user message to Telegram agent
3952		- */
3953		-public function mxchat_send_user_message_to_telegram_agent($message, $user_id, $session_id) {
3954		- $telegram_bot_token = $this->options['telegram_bot_token'] ?? '';
3955		- $topic_id = get_option("mxchat_telegram_topic_{$session_id}", '');
3956		- $group_id = get_option("mxchat_telegram_group_{$session_id}", '');
3957		-
3958		- if (empty($telegram_bot_token) || empty($topic_id) || empty($group_id)) {
3959		- return false;
3960		- }
3961		-
3962		- $escaped_message = htmlspecialchars($message, ENT_QUOTES, 'UTF-8');
3963		- $user_message = "👤 <b>User:</b> {$escaped_message}";
3964		-
3965		- $response = wp_remote_post("https://api.telegram.org/bot{$telegram_bot_token}/sendMessage", [
3966		- 'headers' => ['Content-Type' => 'application/json'],
3967		- 'body' => json_encode([
3968		- 'chat_id' => $group_id,
3969		- 'message_thread_id' => $topic_id,
3970		- 'text' => $user_message,
3971		- 'parse_mode' => 'HTML'
3972		- ])
3973		- ]);
3974		-
3975		- return !is_wp_error($response);
3976		-}
3977		-
3978		-/**
3979		- * Handle incoming Telegram webhook
3980		- */
3981		-public function handle_telegram_webhook(WP_REST_Request $request) {
3982		- $body = $request->get_body();
3983		- $data = json_decode($body, true);
3984		-
3985		- //error_log('[MxChat Telegram DEBUG] Webhook received: ' . $body);
3986		-
3987		- // Handle message events from forum topics
3988		- if (isset($data['message'])) {
3989		- $message_data = $data['message'];
3990		-
3991		- // Skip if not from a forum topic
3992		- if (!isset($message_data['message_thread_id'])) {
3993		- //error_log('[MxChat Telegram DEBUG] Skipped: No message_thread_id (not a forum topic message)');
3994		- return new WP_REST_Response(['ok' => true]);
3995		- }
3996		-
3997		- // Skip bot messages
3998		- if (isset($message_data['from']['is_bot']) && $message_data['from']['is_bot']) {
3999		- //error_log('[MxChat Telegram DEBUG] Skipped: Message from bot');
4000		- return new WP_REST_Response(['ok' => true]);
4001		- }
4002		-
4003		- $chat_id = $message_data['chat']['id'] ?? '';
4004		- $topic_id = $message_data['message_thread_id'];
4005		- $message_text = $message_data['text'] ?? '';
4006		- $message_id = $message_data['message_id'] ?? '';
4007		- $from = $message_data['from'] ?? [];
4008		- $agent_name = trim(($from['first_name'] ?? '') . ' ' . ($from['last_name'] ?? ''));
4009		- if (empty($agent_name)) {
4010		- $agent_name = $from['username'] ?? 'Agent';
4011		- }
4012		-
4013		- //error_log("[MxChat Telegram DEBUG] Parsed: chat_id={$chat_id}, topic_id={$topic_id}, agent={$agent_name}, text={$message_text}");
4014		-
4015		- // Skip empty messages
4016		- if (empty($message_text)) {
4017		- //error_log('[MxChat Telegram DEBUG] Skipped: Empty message text');
4018		- return new WP_REST_Response(['ok' => true]);
4019		- }
4020		-
4021		- // Find session ID by topic ID - cast to string for comparison
4022		- global $wpdb;
4023		- $topic_id_str = strval($topic_id);
4024		- $session_option = $wpdb->get_var(
4025		- $wpdb->prepare(
4026		- "SELECT option_name FROM {$wpdb->options}
4027		- WHERE option_name LIKE %s
4028		- AND option_value = %s",
4029		- 'mxchat_telegram_topic_%',
4030		- $topic_id_str
4031		- )
4032		- );
4033		-
4034		- //error_log("[MxChat Telegram DEBUG] Looking for topic_id={$topic_id_str} in options, found: " . ($session_option ?: 'NULL'));
4035		-
4036		- if ($session_option) {
4037		- $session_id = str_replace('mxchat_telegram_topic_', '', $session_option);
4038		- //error_log("[MxChat Telegram DEBUG] Session ID: {$session_id}");
4039		-
4040		- // Verify the group ID matches
4041		- $stored_group_id = get_option("mxchat_telegram_group_{$session_id}", '');
4042		- //error_log("[MxChat Telegram DEBUG] Stored group_id={$stored_group_id}, received chat_id={$chat_id}");
4043		-
4044		- if (strval($stored_group_id) != strval($chat_id)) {
4045		- //error_log('[MxChat Telegram DEBUG] Skipped: Group ID mismatch');
4046		- return new WP_REST_Response(['ok' => true]);
4047		- }
4048		-
4049		- // Check for closure commands
4050		- $lower_text = strtolower(trim($message_text));
4051		- if (in_array($lower_text, ['#close', '#end', '#disconnect', '#done'])) {
4052		- //error_log("[MxChat Telegram DEBUG] Closure command received: {$lower_text}");
4053		- // End the live agent session
4054		- update_option("mxchat_mode_{$session_id}", 'ai');
4055		-
4056		- // Save disconnect message
4057		- $disconnect_message = "Live agent session ended. You're now chatting with the AI assistant.";
4058		- $this->mxchat_save_chat_message($session_id, 'bot', $disconnect_message);
4059		-
4060		- // Notify in Telegram
4061		- $telegram_bot_token = $this->options['telegram_bot_token'] ?? '';
4062		- if (!empty($telegram_bot_token)) {
4063		- wp_remote_post("https://api.telegram.org/bot{$telegram_bot_token}/sendMessage", [
4064		- 'headers' => ['Content-Type' => 'application/json'],
4065		- 'body' => json_encode([
4066		- 'chat_id' => $chat_id,
4067		- 'message_thread_id' => $topic_id,
4068		- 'text' => "✅ Session closed. User returned to AI chatbot.",
4069		- 'parse_mode' => 'HTML'
4070		- ])
4071		- ]);
4072		-
4073		- // Optionally close the topic
4074		- wp_remote_post("https://api.telegram.org/bot{$telegram_bot_token}/closeForumTopic", [
4075		- 'headers' => ['Content-Type' => 'application/json'],
4076		- 'body' => json_encode([
4077		- 'chat_id' => $chat_id,
4078		- 'message_thread_id' => $topic_id
4079		- ])
4080		- ]);
4081		- }
4082		-
4083		- return new WP_REST_Response(['ok' => true]);
4084		- }
4085		-
4086		- // Deduplicate messages
4087		- $message_key = md5($session_id . $message_id . $message_text);
4088		- $processed_messages = get_transient('mxchat_telegram_messages_' . $session_id) ?: [];
4089		-
4090		- if (in_array($message_key, $processed_messages)) {
4091		- //error_log('[MxChat Telegram DEBUG] Skipped: Duplicate message');
4092		- return new WP_REST_Response(['ok' => true]);
4093		- }
4094		-
4095		- $processed_messages[] = $message_key;
4096		- if (count($processed_messages) > 50) {
4097		- $processed_messages = array_slice($processed_messages, -50);
4098		- }
4099		- set_transient('mxchat_telegram_messages_' . $session_id, $processed_messages, HOUR_IN_SECONDS);
4100		-
4101		- // Save the agent message - format with agent name prefix for proper parsing
4102		- $formatted_message = "Agent: {$agent_name} - {$message_text}";
4103		- //error_log("[MxChat Telegram DEBUG] Saving agent message: {$formatted_message}");
4104		-
4105		- $this->mxchat_save_chat_message($session_id, 'agent', $formatted_message);
4106		-
4107		- // Verify the message was saved to history
4108		- $history = get_option("mxchat_history_{$session_id}", []);
4109		- $last_message = end($history);
4110		- //error_log("[MxChat Telegram DEBUG] History after save - count: " . count($history) . ", last message role: " . ($last_message['role'] ?? 'none'));
4111		-
4112		- // Send confirmation back to Telegram
4113		- $telegram_bot_token = $this->options['telegram_bot_token'] ?? '';
4114		- if (!empty($telegram_bot_token)) {
4115		- $confirm_key = 'mxchat_telegram_confirm_' . $message_key;
4116		- if (!get_transient($confirm_key)) {
4117		- wp_remote_post("https://api.telegram.org/bot{$telegram_bot_token}/sendMessage", [
4118		- 'headers' => ['Content-Type' => 'application/json'],
4119		- 'body' => json_encode([
4120		- 'chat_id' => $chat_id,
4121		- 'message_thread_id' => $topic_id,
4122		- 'text' => "✅ <i>Message sent to user</i>",
4123		- 'parse_mode' => 'HTML',
4124		- 'reply_to_message_id' => $message_id
4125		- ])
4126		- ]);
4127		- set_transient($confirm_key, true, 300);
4128		- }
4129		- }
4130		- } else {
4131		- //error_log("[MxChat Telegram DEBUG] No session found for topic_id={$topic_id}");
4132		- }
4133		- } else {
4134		- //error_log('[MxChat Telegram DEBUG] No message in webhook data');
4135		- }
4136		-
4137		- return new WP_REST_Response(['ok' => true]);
4138		-}
4139		-
4140	2902	public function mxchat_send_user_message_to_agent($message, $user_id, $session_id) {
4141		- // Check if this is a Telegram agent session
4142		- $telegram_topic_id = get_option("mxchat_telegram_topic_{$session_id}", '');
4143		- if (!empty($telegram_topic_id)) {
4144		- return $this->mxchat_send_user_message_to_telegram_agent($message, $user_id, $session_id);
4145		- }
4146		-
4147		- // Otherwise, try Slack
4148	2903	$slack_bot_token = $this->options['live_agent_bot_token'] ?? '';
4149	2904	$channel_id = get_option("mxchat_channel_{$session_id}", '');
4150	2905
4151	2906	if (empty($slack_bot_token) || empty($channel_id)) {
		@@ -4371,33 +3126,33 @@
4371	3126
4372	3127	$channel_id = $event['channel'];
4373	3128	$message_text = $event['text'] ?? '';
4374	3129	$message_ts = $event['ts'] ?? '';
4375		-
	3130	+
4376	3131	// Find session ID by looking for matching channel
4377	3132	global $wpdb;
4378	3133	$session_option = $wpdb->get_var(
4379	3134	$wpdb->prepare(
4380		- "SELECT option_name FROM {$wpdb->options}
4381		- WHERE option_name LIKE 'mxchat_channel_%'
	3135	+ "SELECT option_name FROM {$wpdb->options}
	3136	+ WHERE option_name LIKE 'mxchat_channel_%'
4382	3137	AND option_value = %s",
4383	3138	$channel_id
4384	3139	)
4385	3140	);
4386		-
	3141	+
4387	3142	if ($session_option) {
4388	3143	$session_id = str_replace('mxchat_channel_', '', $session_option);
4389		-
	3144	+
4390	3145	// Create a unique key for this specific message
4391	3146	$message_key = md5($session_id . $message_ts . $message_text);
4392	3147	$processed_messages = get_transient('mxchat_processed_messages_' . $session_id) ?: [];
4393		-
	3148	+
4394	3149	// Check if we've already processed this exact message
4395	3150	if (in_array($message_key, $processed_messages)) {
4396	3151	//error_log("Duplicate message detected for session $session_id");
4397	3152	return new WP_REST_Response(['ok' => true]);
4398	3153	}
4399		-
	3154	+
4400	3155	// Add to processed messages
4401	3156	$processed_messages[] = $message_key;
4402	3157	// Keep only last 50 messages per session
4403	3158	if (count($processed_messages) > 50) {
		@@ -4403,46 +3158,14 @@
4403	3158	if (count($processed_messages) > 50) {
4404	3159	$processed_messages = array_slice($processed_messages, -50);
4405	3160	}
4406	3161	set_transient('mxchat_processed_messages_' . $session_id, $processed_messages, HOUR_IN_SECONDS);
4407		-
4408		- $slack_bot_token = $this->options['live_agent_bot_token'] ?? '';
4409		-
4410		- // Handle agent ending the chat — transfer back to AI
4411		- // Format: "!endchat" or "!endchat <custom message to user>"
4412		- if (preg_match('/^!endchat\b/i', trim($message_text))) {
4413		- update_option("mxchat_mode_{$session_id}", 'ai');
4414		-
4415		- // Extract custom message after !endchat, or use empty string
4416		- $custom_message = trim(preg_replace('/^!endchat\s*/i', '', trim($message_text)));
4417		-
4418		- // Send the agent's custom farewell message if provided
4419		- if (!empty($custom_message)) {
4420		- $this->mxchat_save_chat_message($session_id, 'agent', $custom_message);
4421		- }
4422		-
4423		- // Confirm in Slack channel
4424		- if (!empty($slack_bot_token)) {
4425		- wp_remote_post('https://slack.com/api/chat.postMessage', [
4426		- 'headers' => [
4427		- 'Content-Type' => 'application/json',
4428		- 'Authorization' => 'Bearer ' . $slack_bot_token
4429		- ],
4430		- 'body' => json_encode([
4431		- 'channel' => $channel_id,
4432		- 'text' => "✅ *Chat ended.* User has been transferred back to AI mode.",
4433		- 'mrkdwn' => true
4434		- ])
4435		- ]);
4436		- }
4437		-
4438		- return new WP_REST_Response(['ok' => true]);
4439		- }
4440		-
	3162	+
4441	3163	// Save the agent message
4442	3164	$this->mxchat_save_chat_message($session_id, 'agent', $message_text);
4443		-
	3165	+
4444	3166	// Send confirmation back to Slack (only once)
	3167	+ $slack_bot_token = $this->options['live_agent_bot_token'] ?? '';
4445	3168	if (!empty($slack_bot_token)) {
4446	3169	// Use a transient to prevent duplicate confirmations
4447	3170	$confirm_key = 'mxchat_confirm_' . $message_key;
4448	3171	if (!get_transient($confirm_key)) {
		@@ -4452,9 +3175,9 @@
4452	3175	'Authorization' => 'Bearer ' . $slack_bot_token
4453	3176	],
4454	3177	'body' => json_encode([
4455	3178	'channel' => $channel_id,
4456		- 'text' => "✅ _Message sent to user_",
	3179	+ 'text' => "✅ _Message sent to user_",
4457	3180	'thread_ts' => $event['ts'] // Reply in thread
4458	3181	])
4459	3182	]);
4460	3183	// Set transient to prevent duplicate confirmations
		@@ -4692,564 +3415,251 @@
4692	3415	];
4693	3416	}
4694	3417	}
4695	3418
	3419	+private function mxchat_find_relevant_content($user_embedding) {
	3420	+ //error_log('MXChat Vector Search: Starting content search...');
4696	3421
4697		-private function mxchat_find_relevant_content($user_embedding, $bot_id = 'default', $user_query = '') {
4698		- //error_log("MXCHAT DEBUG: find_relevant_content called with bot_id: " . $bot_id);
	3422	+ // Retrieve the add-on settings from the database.
	3423	+ $addon_options = get_option('mxchat_pinecone_addon_options', array());
4699	3424
4700		- // Check for OpenAI Vector Store first (takes priority when enabled)
4701		- $bot_vectorstore_config = $this->get_bot_vectorstore_config($bot_id);
	3425	+ // Determine whether Pinecone is enabled.
	3426	+ $use_pinecone = (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1') ? 1 : 0;
4702	3427
4703		- if ($bot_vectorstore_config['use_vectorstore']) {
4704		- // Get current model to verify it's an OpenAI model
4705		- $bot_options = $this->get_bot_options($bot_id);
4706		- $mxchat_options = get_option('mxchat_options', array());
4707		- $current_options = !empty($bot_options) ? $bot_options : $mxchat_options;
4708		- $selected_model = $current_options['model'] ?? 'gpt-5.1-chat-latest';
	3428	+ //error_log('Pinecone enabled flag: ' . $use_pinecone);
4709	3429
4710		- if ($this->is_openai_chat_model($selected_model)) {
4711		- //error_log("MXCHAT DEBUG: Using OpenAI Vector Store for knowledge retrieval");
4712		- return $this->find_relevant_content_openai_vectorstore($user_query, $bot_id, $bot_vectorstore_config);
4713		- } else {
4714		- //error_log("MXCHAT DEBUG: Vector Store enabled but model is not OpenAI (" . $selected_model . "), skipping Vector Store");
4715		- }
4716		- }
4717		-
4718		- // Get bot-specific Pinecone configuration
4719		- $bot_pinecone_config = $this->get_bot_pinecone_config($bot_id);
4720		-
4721		- // Debug: Log the Pinecone configuration
4722		- //error_log("MXCHAT DEBUG: Pinecone config for bot '$bot_id':");
4723		- //error_log(" - use_pinecone: " . ($bot_pinecone_config['use_pinecone'] ? 'true' : 'false'));
4724		- //error_log(" - api_key: " . (empty($bot_pinecone_config['api_key']) ? 'EMPTY' : 'SET (hidden)'));
4725		- //error_log(" - host: " . ($bot_pinecone_config['host'] ?? 'NOT SET'));
4726		- //error_log(" - namespace: " . ($bot_pinecone_config['namespace'] ?? 'NOT SET'));
4727		-
4728		- // Determine whether to use Pinecone based on bot configuration
4729		- $use_pinecone = isset($bot_pinecone_config['use_pinecone']) ? $bot_pinecone_config['use_pinecone'] : false;
4730		-
4731		- //error_log("MXCHAT DEBUG: Using " . ($use_pinecone ? "Pinecone" : "WordPress Database") . " for knowledge retrieval");
4732		-
4733		- if ($use_pinecone) {
4734		- return $this->find_relevant_content_pinecone($user_embedding, $bot_id, $bot_pinecone_config);
	3430	+ if ($use_pinecone === 1) {
	3431	+ //error_log('MXChat Vector Search: Using Pinecone database');
	3432	+ return $this->find_relevant_content_pinecone($user_embedding);
4735	3433	} else {
4736		- return $this->find_relevant_content_wordpress($user_embedding, $bot_id);
	3434	+ //error_log('MXChat Vector Search: Using WordPress database');
	3435	+ return $this->find_relevant_content_wordpress($user_embedding);
4737	3436	}
4738	3437	}
4739	3438
4740		-private function find_relevant_content_wordpress($user_embedding, $bot_id = 'default') {
	3439	+private function find_relevant_content_wordpress($user_embedding) {
4741	3440	global $wpdb;
4742	3441	$system_prompt_table = $wpdb->prefix . 'mxchat_system_prompt_content';
	3442	+ $cache_key = 'mxchat_system_prompt_embeddings';
	3443	+ $batch_size = 500;
	3444	+
4743	3445	// Initialize similarity analysis storage
4744	3446	$this->last_similarity_analysis = [
4745	3447	'knowledge_base_type' => 'WordPress Database',
4746		- 'bot_id' => $bot_id,
4747	3448	'top_matches' => [],
4748	3449	'threshold_used' => 0,
4749	3450	'total_checked' => 0
4750	3451	];
4751	3452
4752		- // NEW: Initialize valid URLs array
4753		- $valid_urls = [];
	3453	+ // Retrieve embeddings from cache or database
	3454	+ $embeddings = wp_cache_get($cache_key, 'mxchat_system_prompts');
	3455	+ if ($embeddings === false) {
	3456	+ // Cache miss - load embeddings from database WITH CONTENT and ROLE RESTRICTION for testing
	3457	+ $embeddings = [];
	3458	+ $offset = 0;
4754	3459
4755		- // Get bot-specific options for similarity threshold
4756		- $bot_options = $this->get_bot_options($bot_id);
4757		- $current_options = !empty($bot_options) ? $bot_options : $this->options;
	3460	+ do {
	3461	+ $query = $wpdb->prepare(
	3462	+ "SELECT id, embedding_vector, article_content, source_url, role_restriction
	3463	+ FROM {$system_prompt_table}
	3464	+ LIMIT %d OFFSET %d",
	3465	+ $batch_size,
	3466	+ $offset
	3467	+ );
4758	3468
4759		- // Get knowledge manager instance for role checking
4760		- $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
	3469	+ $batch = $wpdb->get_results($query);
	3470	+ if (empty($batch)) {
	3471	+ break;
	3472	+ }
4761	3473
4762		- // Get base similarity threshold from bot options or default options
4763		- $similarity_threshold = isset($current_options['similarity_threshold'])
4764		- ? ((int) $current_options['similarity_threshold']) / 100
4765		- : 0.35;
4766		- $this->last_similarity_analysis['threshold_used'] = $similarity_threshold;
	3474	+ $embeddings = array_merge($embeddings, $batch);
	3475	+ $offset += $batch_size;
	3476	+ unset($batch);
	3477	+ } while (true);
4767	3478
4768		- // Precompute bot_filter once, outside the streaming loop
4769		- $bot_filter = '';
4770		- if ($bot_id !== 'default') {
4771		- $column_exists = $wpdb->get_var("SHOW COLUMNS FROM {$system_prompt_table} LIKE 'bot_metadata'");
4772		- if ($column_exists) {
4773		- $bot_filter = $wpdb->prepare(" AND (bot_metadata = %s OR bot_metadata IS NULL OR bot_metadata = '')", $bot_id);
	3479	+ if (empty($embeddings)) {
	3480	+ return '';
4774	3481	}
	3482	+
	3483	+ // Cache embeddings for future use (but note: this now includes content and role restrictions)
	3484	+ wp_cache_set($cache_key, $embeddings, 'mxchat_system_prompts', 3600);
4775	3485	}
4776	3486
4777		- // ===== STREAMING TOP-K PASS =====
4778		- // Stream rows in small batches, compute cosine similarity per row, and keep only:
4779		- // - top 10 by raw similarity (for the testing/debug display panel)
4780		- // - candidates above threshold with access (capped) for context assembly
4781		- // This bounds peak memory regardless of knowledge base size and avoids loading
4782		- // article_content for every row. article_content is fetched in Phase 2 for winners only.
4783		- $batch_size = 250;
4784		- $max_candidates = 200; // safety cap, well above rag_sources_limit * max_chunks_per_source
4785		- $top_display = [];
4786		- $candidates = [];
4787		- $total_checked = 0;
4788		- $offset = 0;
	3487	+ // NEW: Get knowledge manager instance for role checking
	3488	+ $knowledge_manager = MxChat_Knowledge_Manager::get_instance();
4789	3489
4790		- do {
4791		- $batch = $wpdb->get_results($wpdb->prepare(
4792		- "SELECT id, embedding_vector, source_url, role_restriction
4793		- FROM {$system_prompt_table}
4794		- WHERE 1=1 {$bot_filter}
4795		- LIMIT %d OFFSET %d",
4796		- $batch_size,
4797		- $offset
4798		- ));
4799		-
4800		- if (empty($batch)) {
4801		- break;
4802		- }
4803		-
4804		- foreach ($batch as $row) {
4805		- $database_embedding = $row->embedding_vector
4806		- ? unserialize($row->embedding_vector, ['allowed_classes' => false])
4807		- : null;
4808		-
4809		- if (!is_array($database_embedding) || !is_array($user_embedding)) {
4810		- unset($database_embedding);
4811		- continue;
4812		- }
4813		-
	3490	+ // Get configuration options
	3491	+ $main_options = get_option('mxchat_options', []);
	3492	+
	3493	+ // Get base similarity threshold (default 75%)
	3494	+ $similarity_threshold = isset($main_options['similarity_threshold'])
	3495	+ ? ((int) $main_options['similarity_threshold']) / 100
	3496	+ : 0.75;
	3497	+
	3498	+ $this->last_similarity_analysis['threshold_used'] = $similarity_threshold;
	3499	+
	3500	+ // Calculate similarities and build results array
	3501	+ $all_similarities = [];
	3502	+ $relevant_results = [];
	3503	+
	3504	+ foreach ($embeddings as $embedding) {
	3505	+ $database_embedding = $embedding->embedding_vector
	3506	+ ? unserialize($embedding->embedding_vector, ['allowed_classes' => false])
	3507	+ : null;
	3508	+
	3509	+ if (is_array($database_embedding) && is_array($user_embedding)) {
4814	3510	$similarity = $this->mxchat_calculate_cosine_similarity($user_embedding, $database_embedding);
4815		- unset($database_embedding);
4816		-
4817		- $role_restriction = $row->role_restriction ?? 'public';
	3511	+
	3512	+ // NEW: Check role access
	3513	+ $role_restriction = $embedding->role_restriction ?? 'public';
4818	3514	$has_access = $knowledge_manager->mxchat_user_has_content_access($role_restriction);
4819		- $source_url = $row->source_url ?? '';
4820		-
4821		- // Maintain top 10 display buffer (insert-if-beats-worst)
4822		- if (count($top_display) < 10) {
4823		- $top_display[] = [
4824		- 'id' => $row->id,
4825		- 'similarity' => $similarity,
4826		- 'source_url' => $source_url,
4827		- 'role_restriction' => $role_restriction,
4828		- 'has_access' => $has_access,
4829		- ];
4830		- usort($top_display, function ($a, $b) {
4831		- return $b['similarity'] <=> $a['similarity'];
4832		- });
4833		- } elseif ($similarity > $top_display[9]['similarity']) {
4834		- $top_display[9] = [
4835		- 'id' => $row->id,
4836		- 'similarity' => $similarity,
4837		- 'source_url' => $source_url,
4838		- 'role_restriction' => $role_restriction,
4839		- 'has_access' => $has_access,
4840		- ];
4841		- usort($top_display, function ($a, $b) {
4842		- return $b['similarity'] <=> $a['similarity'];
4843		- });
	3515	+
	3516	+ // Store ALL similarities for testing (top 10)
	3517	+ $source_display = '';
	3518	+ if (!empty($embedding->source_url) && $embedding->source_url !== '#') {
	3519	+ $source_display = $embedding->source_url;
	3520	+ } else {
	3521	+ $content_preview = strip_tags($embedding->article_content ?? '');
	3522	+ $content_preview = preg_replace('/\s+/', ' ', $content_preview);
	3523	+ $source_display = substr(trim($content_preview), 0, 50) . '...';
4844	3524	}
4845		-
4846		- // Track candidates for context assembly (above threshold + has access)
	3525	+
	3526	+ $all_similarities[] = [
	3527	+ 'document_id' => $embedding->id,
	3528	+ 'similarity' => $similarity,
	3529	+ 'similarity_percentage' => round($similarity * 100, 2),
	3530	+ 'above_threshold' => $similarity >= $similarity_threshold,
	3531	+ 'source_display' => $source_display,
	3532	+ 'content_preview' => substr(strip_tags($embedding->article_content ?? ''), 0, 100) . '...',
	3533	+ 'used_for_context' => false, // Initialize as false, we'll update this later
	3534	+ 'role_restriction' => $role_restriction, // NEW: Include role info for testing
	3535	+ 'has_access' => $has_access, // NEW: Include access info for testing
	3536	+ 'filtered_out' => !$has_access // NEW: Mark if filtered out by role
	3537	+ ];
	3538	+
	3539	+ // Only consider results above threshold AND with access for actual content retrieval
4847	3540	if ($similarity >= $similarity_threshold && $has_access) {
4848		- $candidates[] = [
4849		- 'id' => $row->id,
4850		- 'similarity' => $similarity,
4851		- 'source_url' => $source_url,
	3541	+ $relevant_results[] = [
	3542	+ 'id' => $embedding->id,
	3543	+ 'similarity' => $similarity
4852	3544	];
4853	3545	}
4854		-
4855		- $total_checked++;
4856	3546	}
4857		-
4858		- unset($batch);
4859		-
4860		- // Trim candidates periodically to cap memory during long scans
4861		- if (count($candidates) > $max_candidates) {
4862		- usort($candidates, function ($a, $b) {
4863		- return $b['similarity'] <=> $a['similarity'];
4864		- });
4865		- $candidates = array_slice($candidates, 0, $max_candidates);
4866		- }
4867		-
4868		- $offset += $batch_size;
4869		- } while (true);
4870		-
4871		- if ($total_checked === 0) {
4872		- $this->current_valid_urls = [];
4873		- return '';
	3547	+
	3548	+ unset($database_embedding);
4874	3549	}
4875	3550
4876		- // Final candidates sort (best first)
4877		- if (count($candidates) > 1) {
4878		- usort($candidates, function ($a, $b) {
4879		- return $b['similarity'] <=> $a['similarity'];
4880		- });
4881		- }
4882		-
4883		- // ===== PHASE 2: FETCH ARTICLE CONTENT ONLY FOR WINNERS =====
4884		- // Gather unique IDs we actually need (top_display + candidates) and pull
4885		- // article_content in bounded IN() batches. This avoids loading content for
4886		- // every row during the similarity scan.
4887		- $needed_ids = [];
4888		- foreach ($top_display as $item) {
4889		- $needed_ids[$item['id']] = true;
4890		- }
4891		- foreach ($candidates as $item) {
4892		- $needed_ids[$item['id']] = true;
4893		- }
4894		- $needed_ids = array_keys($needed_ids);
4895		-
4896		- $content_map = [];
4897		- if (!empty($needed_ids)) {
4898		- foreach (array_chunk($needed_ids, 250) as $chunk_ids) {
4899		- $placeholders = implode(',', array_fill(0, count($chunk_ids), '%d'));
4900		- $rows = $wpdb->get_results($wpdb->prepare(
4901		- "SELECT id, article_content FROM {$system_prompt_table} WHERE id IN ($placeholders)",
4902		- ...$chunk_ids
4903		- ));
4904		- foreach ($rows as $r) {
4905		- $content_map[$r->id] = $r->article_content;
4906		- }
4907		- unset($rows);
4908		- }
4909		- }
4910		-
4911		- // Build the all_similarities display array from the top 10
4912		- $all_similarities = [];
4913		- foreach ($top_display as $item) {
4914		- $article_content_for_parse = $content_map[$item['id']] ?? '';
4915		- $parsed_for_display = MxChat_Chunker::parse_stored_chunk($article_content_for_parse);
4916		- $is_chunk = $parsed_for_display['is_chunked'];
4917		- $chunk_meta = $parsed_for_display['metadata'];
4918		-
4919		- if (!empty($item['source_url']) && $item['source_url'] !== '#') {
4920		- $source_display = $item['source_url'];
4921		- } else {
4922		- $content_preview = strip_tags($article_content_for_parse);
4923		- $content_preview = preg_replace('/\s+/', ' ', $content_preview);
4924		- $source_display = substr(trim($content_preview), 0, 50) . '...';
4925		- }
4926		-
4927		- $all_similarities[] = [
4928		- 'document_id' => $item['id'],
4929		- 'similarity' => $item['similarity'],
4930		- 'similarity_percentage' => round($item['similarity'] * 100, 2),
4931		- 'above_threshold' => $item['similarity'] >= $similarity_threshold,
4932		- 'source_display' => $source_display,
4933		- 'content_preview' => substr(strip_tags($parsed_for_display['text'] ?? ''), 0, 100) . '...',
4934		- 'used_for_context' => false,
4935		- 'role_restriction' => $item['role_restriction'],
4936		- 'has_access' => $item['has_access'],
4937		- 'filtered_out' => !$item['has_access'],
4938		- 'is_chunk' => $is_chunk,
4939		- 'chunk_index' => $is_chunk ? ($chunk_meta['chunk_index'] ?? 0) : null,
4940		- 'total_chunks' => $is_chunk ? ($chunk_meta['total_chunks'] ?? 1) : null
4941		- ];
4942		- }
4943		-
4944		- // Build url_groups from candidates for chunk reassembly
4945		- $url_groups = array();
4946		- foreach ($candidates as $cand) {
4947		- $article_content = $content_map[$cand['id']] ?? '';
4948		- $parsed = MxChat_Chunker::parse_stored_chunk($article_content);
4949		- $is_chunked = $parsed['is_chunked'];
4950		- $chunk_index = $parsed['metadata']['chunk_index'] ?? 0;
4951		- $text_content = $parsed['text'];
4952		-
4953		- $source_url = $cand['source_url'];
4954		- $group_key = !empty($source_url) ? $source_url : '_manual_' . $cand['id'];
4955		-
4956		- if (!isset($url_groups[$group_key])) {
4957		- $url_groups[$group_key] = array(
4958		- 'source_url' => $source_url,
4959		- 'best_score' => 0,
4960		- 'is_chunked' => $is_chunked,
4961		- 'chunks' => array(),
4962		- 'single_text' => '',
4963		- 'single_id' => null
4964		- );
4965		- }
4966		-
4967		- if ($cand['similarity'] > $url_groups[$group_key]['best_score']) {
4968		- $url_groups[$group_key]['best_score'] = $cand['similarity'];
4969		- }
4970		-
4971		- if ($is_chunked) {
4972		- $url_groups[$group_key]['is_chunked'] = true;
4973		- $url_groups[$group_key]['chunks'][] = array(
4974		- 'id' => $cand['id'],
4975		- 'score' => $cand['similarity'],
4976		- 'chunk_index' => $chunk_index,
4977		- 'text' => $text_content
4978		- );
4979		- } else {
4980		- $url_groups[$group_key]['single_text'] = $text_content;
4981		- $url_groups[$group_key]['single_id'] = $cand['id'];
4982		- }
4983		- }
4984		-
4985	3551	// Sort ALL similarities for testing display (highest first)
4986	3552	usort($all_similarities, function ($a, $b) {
4987	3553	return $b['similarity'] <=> $a['similarity'];
4988	3554	});
4989		-
4990		- // Sort URL groups by best score (highest first)
4991		- uasort($url_groups, function($a, $b) {
4992		- return $b['best_score'] <=> $a['best_score'];
	3555	+
	3556	+ // Sort relevant results by similarity (highest first)
	3557	+ usort($relevant_results, function ($a, $b) {
	3558	+ return $b['similarity'] <=> $a['similarity'];
4993	3559	});
4994		-
4995		- // Get RAG sources limit from options (default 6, min 3, max 10)
4996		- $rag_sources_limit = isset($current_options['rag_sources_limit']) ? intval($current_options['rag_sources_limit']) : 3;
4997		- if ($rag_sources_limit < 3) $rag_sources_limit = 3;
4998		- if ($rag_sources_limit > 10) $rag_sources_limit = 10;
4999		-
5000		- // Take top N unique URLs based on user setting
5001		- $top_urls = array_slice($url_groups, 0, $rag_sources_limit, true);
5002		-
5003		- // Track which document IDs are used for context
	3560	+
	3561	+ // Get top 5 results for actual content (standard approach)
	3562	+ $top_results = array_slice($relevant_results, 0, 5);
	3563	+
	3564	+ // NOW mark which documents are actually used for context
5004	3565	$used_document_ids = [];
5005		- foreach ($top_urls as $group) {
5006		- if ($group['is_chunked']) {
5007		- foreach ($group['chunks'] as $chunk) {
5008		- $used_document_ids[] = $chunk['id'];
5009		- }
5010		- } elseif ($group['single_id']) {
5011		- $used_document_ids[] = $group['single_id'];
5012		- }
	3566	+ foreach ($top_results as $result) {
	3567	+ $used_document_ids[] = $result['id'];
5013	3568	}
5014		-
	3569	+
5015	3570	// Update the all_similarities array to mark which were actually used
5016	3571	foreach ($all_similarities as &$similarity_item) {
5017	3572	$similarity_item['used_for_context'] = in_array($similarity_item['document_id'], $used_document_ids);
5018	3573	}
5019		-
5020		- // Store top 10 for testing panel
	3574	+
	3575	+ // Store top 10 for testing panel (now with correct used_for_context flags and role info)
5021	3576	$this->last_similarity_analysis['top_matches'] = array_slice($all_similarities, 0, 10);
5022		- $this->last_similarity_analysis['total_checked'] = $total_checked;
5023		-
	3577	+ $this->last_similarity_analysis['total_checked'] = count($embeddings);
	3578	+
	3579	+ //error_log("MxChat Testing: Stored " . count($this->last_similarity_analysis['top_matches']) . " top matches for testing");
	3580	+
5024	3581	// Initialize final content
5025	3582	$content = '';
5026		- $matches_used = 0;
5027		- $total_chunks_used = 0;
5028		- $max_total_chunks = isset($current_options['rag_chunks_limit']) ? intval($current_options['rag_chunks_limit']) : 15;
5029		- if ($max_total_chunks < 8) $max_total_chunks = 8;
5030		- if ($max_total_chunks > 20) $max_total_chunks = 20;
5031		- $max_chunks_per_source = 5; // Cap per individual source to limit token usage
5032		-
5033		- // Check if citation links are enabled (default to 'on' for backwards compatibility)
5034		- // Use fresh options to ensure we get the latest setting value
5035		- $fresh_options = get_option('mxchat_options', []);
5036		- $citation_links_enabled = isset($fresh_options['citation_links_toggle']) ? ($fresh_options['citation_links_toggle'] === 'on') : true;
5037		-
5038		- // Build content from top sources
5039		- foreach ($top_urls as $group_key => $group) {
5040		- $source_url = $group['source_url']; // Use actual source_url, not the group key
5041		-
5042		- // Stop if we've hit the total chunk limit
5043		- if ($total_chunks_used >= $max_total_chunks) {
5044		- break;
	3583	+
	3584	+ // Track document IDs to avoid duplicates
	3585	+ $added_document_ids = [];
	3586	+
	3587	+ // Fetch and format content for each selected result
	3588	+ foreach ($top_results as $index => $result) {
	3589	+ if (in_array($result['id'], $added_document_ids)) {
	3590	+ continue;
5045	3591	}
5046		-
5047		- $full_text = '';
5048		- $chunks_in_this_source = 1; // Default for non-chunked content
5049		-
5050		- if ($group['is_chunked']) {
5051		- // Calculate how many chunks we can still use (respect both total and per-source caps)
5052		- $chunks_remaining = min($max_chunks_per_source, $max_total_chunks - $total_chunks_used);
5053		-
5054		- // Fetch chunks for this URL with limit
5055		- $full_text = $this->reassemble_chunks_from_wordpress($source_url, $chunks_remaining, $chunks_in_this_source);
5056		-
5057		- // If fetching all chunks fails, fall back to matched chunks
5058		- if (empty($full_text)) {
5059		- // Sort matched chunks by index and concatenate
5060		- usort($group['chunks'], function($a, $b) {
5061		- return $a['chunk_index'] <=> $b['chunk_index'];
5062		- });
5063		-
5064		- $chunk_texts = array();
5065		- $chunks_in_this_source = 0;
5066		- foreach ($group['chunks'] as $chunk) {
5067		- if ($total_chunks_used + $chunks_in_this_source >= $max_total_chunks) {
5068		- break;
5069		- }
5070		- $chunk_texts[] = $chunk['text'];
5071		- $chunks_in_this_source++;
	3592	+
	3593	+ $chunk_content = $this->fetch_content_with_product_links($result['id']);
	3594	+ $added_document_ids[] = $result['id'];
	3595	+
	3596	+ $content .= "## Reference " . ($index + 1) . " ##\n";
	3597	+ $content .= $chunk_content . "\n\n";
	3598	+
	3599	+ // PDF surrounding pages logic (unchanged)
	3600	+ if (strpos($chunk_content, '{"document_type":"pdf"') !== false) {
	3601	+ $surrounding_content = $wpdb->get_results($wpdb->prepare(
	3602	+ "SELECT id, article_content, role_restriction FROM {$system_prompt_table}
	3603	+ WHERE id IN (
	3604	+ (SELECT id FROM {$system_prompt_table} WHERE id < %d ORDER BY id DESC LIMIT 1),
	3605	+ (SELECT id FROM {$system_prompt_table} WHERE id > %d ORDER BY id ASC LIMIT 1)
	3606	+ )",
	3607	+ $result['id'],
	3608	+ $result['id']
	3609	+ ));
	3610	+
	3611	+ // NEW: Check role access for surrounding content too
	3612	+ if (!empty($surrounding_content[0])) {
	3613	+ $surrounding_role = $surrounding_content[0]->role_restriction ?? 'public';
	3614	+ if ($knowledge_manager->mxchat_user_has_content_access($surrounding_role)) {
	3615	+ $content .= "## Related Content ##\n";
	3616	+ $content .= $surrounding_content[0]->article_content . "\n\n";
	3617	+ $added_document_ids[] = $surrounding_content[0]->id;
5072	3618	}
5073		- $full_text = implode("\n\n", $chunk_texts);
5074	3619	}
5075		- } else {
5076		- $full_text = $group['single_text'];
5077		- $chunks_in_this_source = 1;
5078		- }
5079		-
5080		- if (!empty($full_text)) {
5081		- // Strip URLs from content if citation links are disabled
5082		- if (!$citation_links_enabled) {
5083		- $full_text = preg_replace('#\bhttps?://[^\s<>"\']+#i', '', $full_text);
5084		- $full_text = preg_replace('/\s+/', ' ', trim($full_text)); // Clean up extra spaces
5085		- }
5086		-
5087		- // Use numbered reference for URL-based entries, plain info label for manual entries
5088		- // Manual entries are stored with an internal mxchat:// placeholder URL — never expose them as citations
5089		- if (!empty($source_url) && $source_url !== '#' && strpos($source_url, 'mxchat://') !== 0) {
5090		- $matches_used++;
5091		- $content .= "## Reference " . $matches_used . " ##\n";
5092		- $content .= $full_text . "\n\n";
5093		-
5094		- // Only include citation URLs if citation links are enabled
5095		- if ($citation_links_enabled) {
5096		- $valid_urls[] = $source_url;
5097		- $content .= "URL: " . $source_url . "\n\n";
	3620	+
	3621	+ if (!empty($surrounding_content[1])) {
	3622	+ $surrounding_role = $surrounding_content[1]->role_restriction ?? 'public';
	3623	+ if ($knowledge_manager->mxchat_user_has_content_access($surrounding_role)) {
	3624	+ $content .= "## Related Content ##\n";
	3625	+ $content .= $surrounding_content[1]->article_content . "\n\n";
	3626	+ $added_document_ids[] = $surrounding_content[1]->id;
5098	3627	}
5099		- } else {
5100		- // Manual entry — no reference number, no citation
5101		- $content .= "## Information ##\n";
5102		- $content .= $full_text . "\n\n";
5103	3628	}
5104		-
5105		- // Extract any URLs from the text content itself (only if citation links enabled)
5106		- if ($citation_links_enabled) {
5107		- preg_match_all(
5108		- '#\bhttps?://[^\s<>"\']+#i',
5109		- $full_text,
5110		- $content_urls
5111		- );
5112		- if (!empty($content_urls[0])) {
5113		- $valid_urls = array_merge($valid_urls, $content_urls[0]);
5114		- }
5115		- }
5116		-
5117		- $total_chunks_used += $chunks_in_this_source;
5118	3629	}
5119	3630	}
5120		-
5121		- // NEW: Store unique valid URLs for validation
5122		- $this->current_valid_urls = array_unique($valid_urls);
5123		-
5124		- // Store sources and chunks counts for testing/transcript display
5125		- $this->last_similarity_analysis['sources_used'] = $matches_used;
5126		- $this->last_similarity_analysis['total_chunks_used'] = $total_chunks_used;
5127		-
5128		- // Allow add-ons to act on similarity results (e.g. WooCommerce product card display)
5129		- do_action('mxchat_similarity_results', $this->last_similarity_analysis['top_matches'], $bot_id);
5130		-
	3631	+
5131	3632	// Add response guidelines
5132		- if (empty($top_urls)) {
	3633	+ if (empty($top_results)) {
5133	3634	$content = "No reference information was found for this query.\n\n";
5134	3635	} else {
5135		- // Build response guidelines based on citation links setting
5136	3636	$content .= "\n## Response Guidelines ##\n" .
5137	3637	"You are an AI Chatbot. Answer naturally and helpfully using only the information from the references above. " .
5138	3638	"Be conversational and friendly, but never mention your knowledge base or training data. " .
5139	3639	"If you don't have specific information or are uncertain about any details, it's always " .
5140	3640	"better to honestly say you don't know rather than making up or guessing at answers. " .
5141		- "When information is incomplete, let them know you are unsure.\n\n";
5142		-
5143		- // Only add hyperlink instructions if citation links are enabled
5144		- if ($citation_links_enabled) {
5145		- $content .= "CRITICAL: When creating hyperlinks, always use proper markdown format with descriptive text: " .
5146		- "[descriptive text](url). NEVER use empty brackets like [](url). The text in brackets must describe what the link is about. " .
5147		- "Only cite references that have a URL. Do not cite or add source labels to Information sections that have no URL.";
5148		- } else {
5149		- $content .= "IMPORTANT: Do not include any citation links, source URLs, or hyperlinks in your responses. " .
5150		- "Simply provide helpful answers based on the reference information without citing sources.";
5151		- }
	3641	+ "When information is incomplete, let them know you are unsure.";
5152	3642	}
5153	3643
5154	3644	return trim($content);
5155	3645	}
5156	3646
5157		-/**
5158		- * Fetch and reassemble chunks for a URL from WordPress database
5159		- *
5160		- * @param string $source_url The source URL to fetch chunks for
5161		- * @param int $max_chunks Maximum number of chunks to return (0 = unlimited)
5162		- * @param int &$chunk_count Reference to store the actual number of chunks returned
5163		- * @return string Reassembled content from chunks
5164		- */
5165		-private function reassemble_chunks_from_wordpress($source_url, $max_chunks = 0, &$chunk_count = 0) {
5166		- global $wpdb;
5167		- $table = $wpdb->prefix . 'mxchat_system_prompt_content';
5168		-
5169		- // Fetch all rows with this source_url
5170		- $rows = $wpdb->get_results($wpdb->prepare(
5171		- "SELECT article_content FROM {$table}
5172		- WHERE source_url = %s
5173		- ORDER BY id ASC",
5174		- $source_url
5175		- ));
5176		-
5177		- if (empty($rows)) {
5178		- $chunk_count = 0;
5179		- return '';
5180		- }
5181		-
5182		- // Parse and sort chunks by index
5183		- $chunks = array();
5184		- foreach ($rows as $row) {
5185		- $parsed = MxChat_Chunker::parse_stored_chunk($row->article_content);
5186		-
5187		- if ($parsed['is_chunked']) {
5188		- $chunk_index = $parsed['metadata']['chunk_index'] ?? 0;
5189		- $chunks[$chunk_index] = $parsed['text'];
5190		- } else {
5191		- // Non-chunked content - just return it
5192		- $chunks[] = $parsed['text'];
5193		- }
5194		- }
5195		-
5196		- // Sort by chunk index
5197		- ksort($chunks);
5198		-
5199		- // Apply chunk limit if specified
5200		- if ($max_chunks > 0 && count($chunks) > $max_chunks) {
5201		- $chunks = array_slice($chunks, 0, $max_chunks, true);
5202		- }
5203		-
5204		- // Store actual chunk count
5205		- $chunk_count = count($chunks);
5206		-
5207		- // Reassemble content
5208		- return implode("\n\n", $chunks);
5209		-}
5210		-
5211		-private function find_relevant_content_pinecone($user_embedding, $bot_id = 'default', $bot_config = null) {
5212		- global $wpdb;
	3647	+private function find_relevant_content_pinecone($user_embedding) {
	3648	+ global $wpdb; // For single role lookups
	3649	+ $options = get_option('mxchat_pinecone_addon_options', array());
	3650	+ $api_key = $options['mxchat_pinecone_api_key'] ?? '';
	3651	+ $host = $options['mxchat_pinecone_host'] ?? '';
5213	3652
5214		- //error_log("MXCHAT DEBUG: find_relevant_content_pinecone called");
5215		- //error_log(" - bot_id: " . $bot_id);
5216		- //error_log(" - user_embedding is array: " . (is_array($user_embedding) ? 'yes' : 'no'));
5217		- //error_log(" - user_embedding count: " . (is_array($user_embedding) ? count($user_embedding) : 'N/A'));
5218		-
5219		- // Use bot-specific config or fall back to default
5220		- if ($bot_config === null) {
5221		- $bot_config = $this->get_bot_pinecone_config($bot_id);
5222		- }
5223		-
5224		- $api_key = $bot_config['api_key'] ?? '';
5225		- $host = $bot_config['host'] ?? '';
5226		- $namespace = $bot_config['namespace'] ?? '';
5227		-
5228		- //error_log("MXCHAT DEBUG: Pinecone query parameters:");
5229		- //error_log(" - API Key: " . (empty($api_key) ? 'EMPTY - ERROR!' : 'Present (length: ' . strlen($api_key) . ')'));
5230		- //error_log(" - Host: " . (empty($host) ? 'EMPTY - ERROR!' : $host));
5231		- //error_log(" - Namespace: " . (empty($namespace) ? 'EMPTY (will use default)' : $namespace));
5232		-
5233	3653	// Initialize similarity analysis storage
5234	3654	$this->last_similarity_analysis = [
5235	3655	'knowledge_base_type' => 'Pinecone',
5236		- 'bot_id' => $bot_id,
5237		- 'namespace' => $namespace,
5238	3656	'top_matches' => [],
5239	3657	'threshold_used' => 0,
5240	3658	'total_checked' => 0
5241	3659	];
5242	3660
5243		- // NEW: Initialize valid URLs array
5244		- $valid_urls = [];
5245		-
5246	3661	if (empty($host) || empty($api_key)) {
5247		- //error_log("MXCHAT DEBUG ERROR: Missing Pinecone host or API key!");
5248		- //error_log(" - Host empty: " . (empty($host) ? 'YES' : 'NO'));
5249		- //error_log(" - API key empty: " . (empty($api_key) ? 'YES' : 'NO'));
5250		- // Store empty array for valid URLs since we can't proceed
5251		- $this->current_valid_urls = [];
5252	3662	return '';
5253	3663	}
5254	3664
5255	3665	// Get knowledge manager instance for role checking
		@@ -5254,37 +3664,26 @@
5254	3664
5255	3665	// Get knowledge manager instance for role checking
5256	3666	$knowledge_manager = MxChat_Knowledge_Manager::get_instance();
5257	3667
5258		- // Get the similarity threshold from the bot options or main options
5259		- $bot_options = $this->get_bot_options($bot_id);
5260		- $current_options = !empty($bot_options) ? $bot_options : get_option('mxchat_options', []);
	3668	+ // Get the similarity threshold from the main options
	3669	+ $main_options = get_option('mxchat_options', []);
	3670	+ $similarity_threshold = isset($main_options['similarity_threshold'])
	3671	+ ? ((int) $main_options['similarity_threshold']) / 100
	3672	+ : 0.75;
5261	3673
5262		- $similarity_threshold = isset($current_options['similarity_threshold'])
5263		- ? ((int) $current_options['similarity_threshold']) / 100
5264		- : 0.35;
5265		-
5266	3674	$this->last_similarity_analysis['threshold_used'] = $similarity_threshold;
5267	3675
5268		- // Prepare the query request for Pinecone
	3676	+ // Prepare the query request for Pinecone (request more for testing)
5269	3677	$api_endpoint = "https://{$host}/query";
5270	3678
5271	3679	$request_body = array(
5272	3680	'vector' => $user_embedding,
5273		- 'topK' => 50, // Increased for chunked content grouping - need more candidates to find top N unique URLs
	3681	+ 'topK' => 20, // Request more to get good testing data
5274	3682	'includeMetadata' => true,
5275	3683	'includeValues' => true
5276	3684	);
5277	3685
5278		- // Add namespace if specified for this bot
5279		- if (!empty($namespace)) {
5280		- $request_body['namespace'] = $namespace;
5281		- }
5282		-
5283		- //error_log("MXCHAT DEBUG: About to call Pinecone API");
5284		- //error_log(" - Endpoint: " . $api_endpoint);
5285		- //error_log(" - Namespace in request: " . (!empty($namespace) ? $namespace : 'NOT SET'));
5286		-
5287	3686	$response = wp_remote_post($api_endpoint, array(
5288	3687	'headers' => array(
5289	3688	'Api-Key' => $api_key,
5290	3689	'accept' => 'application/json',
		@@ -5294,242 +3693,62 @@
5294	3693	'timeout' => 30
5295	3694	));
5296	3695
5297	3696	if (is_wp_error($response)) {
5298		- //error_log("MXCHAT DEBUG ERROR: WP Error in Pinecone request: " . $response->get_error_message());
5299		- // Store empty array for valid URLs
5300		- $this->current_valid_urls = [];
5301	3697	return '';
5302	3698	}
5303	3699
5304	3700	$response_code = wp_remote_retrieve_response_code($response);
5305		- //error_log("MXCHAT DEBUG: Pinecone response code: " . $response_code);
5306		-
5307	3701	if ($response_code !== 200) {
5308		- $response_body = wp_remote_retrieve_body($response);
5309		- //error_log("MXCHAT DEBUG ERROR: Pinecone API error response: " . substr($response_body, 0, 500));
5310		- // Store empty array for valid URLs
5311		- $this->current_valid_urls = [];
5312	3702	return '';
5313	3703	}
5314	3704
5315		- // ADD DETAILED DEBUG SECTION HERE
5316		- $response_body = wp_remote_retrieve_body($response);
5317		- //error_log("MXCHAT DEBUG: Raw Pinecone response length: " . strlen($response_body));
5318		-
5319		- $results = json_decode($response_body, true);
5320		-
5321		- if (json_last_error() !== JSON_ERROR_NONE) {
5322		- //error_log("MXCHAT DEBUG ERROR: JSON decode error: " . json_last_error_msg());
5323		- //error_log("MXCHAT DEBUG: First 500 chars of response: " . substr($response_body, 0, 500));
5324		- // Store empty array for valid URLs
5325		- $this->current_valid_urls = [];
5326		- return '';
5327		- }
5328		-
5329		- //error_log("MXCHAT DEBUG: Pinecone response structure:");
5330		- //error_log(" - Has 'matches' key: " . (isset($results['matches']) ? 'yes' : 'no'));
5331		- //error_log(" - Has 'namespace' key: " . (isset($results['namespace']) ? 'yes (' . $results['namespace'] . ')' : 'no'));
5332		-
	3705	+ $results = json_decode(wp_remote_retrieve_body($response), true);
5333	3706	if (empty($results['matches'])) {
5334		- //error_log("MXCHAT DEBUG: No matches found in Pinecone response");
5335		- //error_log("MXCHAT DEBUG: Response keys: " . implode(', ', array_keys($results)));
5336		- // Store empty array for valid URLs
5337		- $this->current_valid_urls = [];
5338	3707	return '';
5339	3708	}
5340	3709
5341		- //error_log("MXCHAT DEBUG: Found " . count($results['matches']) . " matches in Pinecone");
5342		-
5343		- // Log first match details for debugging
5344		- if (!empty($results['matches'][0])) {
5345		- $first_match = $results['matches'][0];
5346		- //error_log("MXCHAT DEBUG: First match details:");
5347		- //error_log(" - Score: " . ($first_match['score'] ?? 'no score'));
5348		- //error_log(" - Has metadata: " . (isset($first_match['metadata']) ? 'yes' : 'no'));
5349		- if (isset($first_match['metadata'])) {
5350		- //error_log(" - Metadata keys: " . implode(', ', array_keys($first_match['metadata'])));
5351		- }
5352		- }
5353		-
5354	3710	// Initialize the final content
5355	3711	$content = '';
5356	3712	$matches_used = 0;
5357	3713	$matches_used_for_context = [];
5358		- $total_chunks_used = 0;
5359		- $max_total_chunks = isset($current_options['rag_chunks_limit']) ? intval($current_options['rag_chunks_limit']) : 15;
5360		- if ($max_total_chunks < 8) $max_total_chunks = 8;
5361		- if ($max_total_chunks > 20) $max_total_chunks = 20;
5362		- $max_chunks_per_source = 5; // Cap per individual source to limit token usage
5363		-
5364		- // Check if citation links are enabled (default to 'on' for backwards compatibility)
5365		- // Use fresh options to ensure we get the latest setting value
5366		- $fresh_options = get_option('mxchat_options', []);
5367		- $citation_links_enabled = isset($fresh_options['citation_links_toggle']) ? ($fresh_options['citation_links_toggle'] === 'on') : true;
5368		-
5369		- // NEW CHUNKING LOGIC: Group results by source_url for chunk reassembly
5370		- $url_groups = array();
5371		-
	3714	+
	3715	+ // Process each match for actual content generation (lazy role checking)
5372	3716	foreach ($results['matches'] as $index => $match) {
5373	3717	// Skip if similarity is below threshold
5374	3718	if ($match['score'] < $similarity_threshold) {
5375	3719	continue;
5376	3720	}
5377		-
5378		- $metadata = $match['metadata'] ?? array();
5379		- $source_url = $metadata['source_url'] ?? '';
5380		- $match_id = $match['id'] ?? '';
5381		-
5382		- // LAZY ROLE CHECK: Only check role for content we're actually considering
5383		- $role_restriction = $this->get_single_vector_role($match_id, $metadata);
5384		- $has_access = $knowledge_manager->mxchat_user_has_content_access($role_restriction);
5385		-
5386		- // Skip if user doesn't have access
5387		- if (!$has_access) {
5388		- continue;
5389		- }
5390		-
5391		- // Use a unique key for manual entries without a source URL
5392		- $group_key = !empty($source_url) ? $source_url : '_manual_' . $match_id;
5393		-
5394		- // Group by source URL (or unique key for manual entries)
5395		- if (!isset($url_groups[$group_key])) {
5396		- $url_groups[$group_key] = array(
5397		- 'source_url' => $source_url,
5398		- 'best_score' => 0,
5399		- 'is_chunked' => isset($metadata['is_chunked']) && $metadata['is_chunked'],
5400		- 'chunks' => array(),
5401		- 'single_text' => ''
5402		- );
5403		- }
5404		-
5405		- // Track best score for this group
5406		- if ($match['score'] > $url_groups[$group_key]['best_score']) {
5407		- $url_groups[$group_key]['best_score'] = $match['score'];
5408		- }
5409		-
5410		- // Store chunk info or single text
5411		- if ($url_groups[$group_key]['is_chunked']) {
5412		- $url_groups[$group_key]['chunks'][] = array(
5413		- 'id' => $match_id,
5414		- 'score' => $match['score'],
5415		- 'chunk_index' => $metadata['chunk_index'] ?? 0,
5416		- 'text' => $metadata['text'] ?? ''
5417		- );
5418		- } else {
5419		- // Non-chunked content - just store the text
5420		- $url_groups[$group_key]['single_text'] = $metadata['text'] ?? '';
5421		- $url_groups[$group_key]['single_id'] = $match_id;
5422		- }
5423		- }
5424		-
5425		- // Sort URL groups by best score (highest first)
5426		- uasort($url_groups, function($a, $b) {
5427		- return $b['best_score'] <=> $a['best_score'];
5428		- });
5429		-
5430		- // Get RAG sources limit from options (default 6, min 3, max 10)
5431		- $rag_sources_limit = isset($current_options['rag_sources_limit']) ? intval($current_options['rag_sources_limit']) : 3;
5432		- if ($rag_sources_limit < 3) $rag_sources_limit = 3;
5433		- if ($rag_sources_limit > 10) $rag_sources_limit = 10;
5434		-
5435		- // Take top N unique URLs based on user setting
5436		- $top_urls = array_slice($url_groups, 0, $rag_sources_limit, true);
5437		-
5438		- // Track which match IDs are actually used for context
5439		- foreach ($top_urls as $group) {
5440		- if ($group['is_chunked']) {
5441		- foreach ($group['chunks'] as $chunk) {
5442		- $matches_used_for_context[] = $chunk['id'];
5443		- }
5444		- } elseif (!empty($group['single_id'])) {
5445		- $matches_used_for_context[] = $group['single_id'];
5446		- }
5447		- }
5448		-
5449		- // Build content from top sources
5450		- foreach ($top_urls as $group_key => $group) {
5451		- $source_url = $group['source_url']; // Use actual source_url, not the group key
5452		-
5453		- // Stop if we've hit the total chunk limit
5454		- if ($total_chunks_used >= $max_total_chunks) {
	3721	+
	3722	+ // Limit to top 5 matches above threshold
	3723	+ if ($matches_used >= 5) {
5455	3724	break;
5456	3725	}
5457		-
5458		- $full_text = '';
5459		- $chunks_in_this_source = 1; // Default for non-chunked content
5460		-
5461		- if ($group['is_chunked']) {
5462		- // Calculate how many chunks we can still use (respect both total and per-source caps)
5463		- $chunks_remaining = min($max_chunks_per_source, $max_total_chunks - $total_chunks_used);
5464		-
5465		- // Fetch chunks for this URL with limit
5466		- $full_text = $this->reassemble_chunks_from_pinecone($source_url, $bot_config, $chunks_remaining, $chunks_in_this_source);
5467		-
5468		- // If fetching all chunks fails, fall back to matched chunks
5469		- if (empty($full_text)) {
5470		- // Sort matched chunks by index and concatenate
5471		- usort($group['chunks'], function($a, $b) {
5472		- return $a['chunk_index'] <=> $b['chunk_index'];
5473		- });
5474		-
5475		- $chunk_texts = array();
5476		- $chunks_in_this_source = 0;
5477		- foreach ($group['chunks'] as $chunk) {
5478		- if ($total_chunks_used + $chunks_in_this_source >= $max_total_chunks) {
5479		- break;
5480		- }
5481		- $chunk_texts[] = $chunk['text'];
5482		- $chunks_in_this_source++;
5483		- }
5484		- $full_text = implode("\n\n", $chunk_texts);
	3726	+
	3727	+ if (!empty($match['metadata']['text'])) {
	3728	+ // LAZY ROLE CHECK: Only check role for content we're actually considering
	3729	+ $match_id = $match['id'] ?? '';
	3730	+ $role_restriction = $this->get_single_vector_role($match_id, $match['metadata']);
	3731	+ $has_access = $knowledge_manager->mxchat_user_has_content_access($role_restriction);
	3732	+
	3733	+ // Skip if user doesn't have access
	3734	+ if (!$has_access) {
	3735	+ continue;
5485	3736	}
5486		- } else {
5487		- $full_text = $group['single_text'];
5488		- $chunks_in_this_source = 1;
5489		- }
5490		-
5491		- if (!empty($full_text)) {
5492		- // Strip URLs from content if citation links are disabled
5493		- if (!$citation_links_enabled) {
5494		- $full_text = preg_replace('#\bhttps?://[^\s<>"\']+#i', '', $full_text);
5495		- $full_text = preg_replace('/\s+/', ' ', trim($full_text)); // Clean up extra spaces
	3737	+
	3738	+ // User has access - add to content
	3739	+ $content .= "## Reference " . ($matches_used + 1) . " ##\n";
	3740	+ $content .= $match['metadata']['text'] . "\n\n";
	3741	+
	3742	+ if (!empty($match['metadata']['source_url'])) {
	3743	+ $content .= "URL: " . $match['metadata']['source_url'] . "\n\n";
5496	3744	}
5497		-
5498		- // Use numbered reference for URL-based entries, plain info label for manual entries
5499		- // Manual entries are stored with an internal mxchat:// placeholder URL — never expose them as citations
5500		- if (!empty($source_url) && $source_url !== '#' && strpos($source_url, 'mxchat://') !== 0) {
5501		- $matches_used++;
5502		- $content .= "## Reference " . $matches_used . " ##\n";
5503		- $content .= $full_text . "\n\n";
5504		-
5505		- // Only include citation URLs if citation links are enabled
5506		- if ($citation_links_enabled) {
5507		- $valid_urls[] = $source_url;
5508		- $content .= "URL: " . $source_url . "\n\n";
5509		- }
5510		- } else {
5511		- // Manual entry — no reference number, no citation
5512		- $content .= "## Information ##\n";
5513		- $content .= $full_text . "\n\n";
5514		- }
5515		-
5516		- // Extract any URLs from the text content itself (only if citation links enabled)
5517		- if ($citation_links_enabled) {
5518		- preg_match_all(
5519		- '#\bhttps?://[^\s<>"\']+#i',
5520		- $full_text,
5521		- $content_urls
5522		- );
5523		- if (!empty($content_urls[0])) {
5524		- $valid_urls = array_merge($valid_urls, $content_urls[0]);
5525		- }
5526		- }
5527		-
5528		- $total_chunks_used += $chunks_in_this_source;
	3745	+
	3746	+ $matches_used_for_context[] = $match['id'] ?? $index;
	3747	+ $matches_used++;
5529	3748	}
5530	3749	}
5531		-
	3750	+
5532	3751	// Process ALL matches for testing data (top 10) - with role checking for testing display
5533	3752	$all_matches = [];
5534	3753	foreach ($results['matches'] as $index => $match) {
5535	3754	if ($index >= 10) break; // Limit to top 10 for testing
		@@ -5549,19 +3768,9 @@
5549	3768	$source_display = substr(trim($content_preview), 0, 50) . '...';
5550	3769	}
5551	3770
5552	3771	$match_id_for_display = $match['id'] ?? $index;
5553		-
5554		- // Check for chunk metadata in Pinecone
5555		- $is_chunk = isset($match['metadata']['is_chunked']) && $match['metadata']['is_chunked'];
5556		- $chunk_index = isset($match['metadata']['chunk_index']) ? intval($match['metadata']['chunk_index']) : null;
5557		- $total_chunks = isset($match['metadata']['total_chunks']) ? intval($match['metadata']['total_chunks']) : null;
5558		-
5559		- // Also detect chunk from vector ID pattern: {hash}_chunk_{index}
5560		- if (!$is_chunk && MxChat_Chunker::is_chunk_vector_id($match_id_for_display)) {
5561		- $is_chunk = true;
5562		- }
5563		-
	3772	+
5564	3773	$all_matches[] = [
5565	3774	'document_id' => $match_id_for_display,
5566	3775	'similarity' => $match['score'],
5567	3776	'similarity_percentage' => round($match['score'] * 100, 2),
		@@ -5570,12 +3779,9 @@
5570	3779	'content_preview' => substr(strip_tags($match['metadata']['text'] ?? ''), 0, 100) . '...',
5571	3780	'used_for_context' => in_array($match_id_for_display, $matches_used_for_context),
5572	3781	'role_restriction' => $role_restriction,
5573	3782	'has_access' => $has_access,
5574		- 'filtered_out' => !$has_access,
5575		- 'is_chunk' => $is_chunk,
5576		- 'chunk_index' => $chunk_index,
5577		- 'total_chunks' => $total_chunks
	3783	+ 'filtered_out' => !$has_access
5578	3784	];
5579	3785	}
5580	3786
5581	3787	// Store for testing panel
		@@ -5580,40 +3786,23 @@
5580	3786
5581	3787	// Store for testing panel
5582	3788	$this->last_similarity_analysis['top_matches'] = $all_matches;
5583	3789	$this->last_similarity_analysis['total_checked'] = count($results['matches']);
5584		- $this->last_similarity_analysis['sources_used'] = $matches_used;
5585		- $this->last_similarity_analysis['total_chunks_used'] = $total_chunks_used;
5586		-
5587		- // NEW: Store unique valid URLs for validation
5588		- $this->current_valid_urls = array_unique($valid_urls);
5589		-
5590		- // Allow add-ons to act on similarity results (e.g. WooCommerce product card display)
5591		- do_action('mxchat_similarity_results', $this->last_similarity_analysis['top_matches'], $bot_id);
5592		-
	3790	+
	3791	+ //error_log("MxChat Testing: Stored " . count($this->last_similarity_analysis['top_matches']) . " Pinecone matches for testing");
	3792	+
5593	3793	// Add response guidelines
5594	3794	if ($matches_used === 0) {
5595	3795	$content = "No reference information was found for this query.\n\n";
5596	3796	} else {
5597		- // Build response guidelines based on citation links setting
5598		- $content .= "\n## Response Guidelines ##\n" .
5599		- "You are an AI Chatbot. Answer naturally and helpfully using only the information from the references above. " .
5600		- "Be conversational and friendly, but never mention your knowledge base or training data. " .
5601		- "If you don't have specific information or are uncertain about any details, it's always " .
5602		- "better to honestly say you don't know rather than making up or guessing at answers. " .
5603		- "When information is incomplete, let them know you are unsure.\n\n";
5604		-
5605		- // Only add hyperlink instructions if citation links are enabled
5606		- if ($citation_links_enabled) {
5607		- $content .= "CRITICAL: When creating hyperlinks, always use proper markdown format with descriptive text: " .
5608		- "[descriptive text](url). NEVER use empty brackets like [](url). The text in brackets must describe what the link is about. " .
5609		- "Only cite references that have a URL. Do not cite or add source labels to Information sections that have no URL.";
5610		- } else {
5611		- $content .= "IMPORTANT: Do not include any citation links, source URLs, or hyperlinks in your responses. " .
5612		- "Simply provide helpful answers based on the reference information without citing sources.";
5613		- }
	3797	+ $content .= "\n## Response Guidelines ##\n" .
	3798	+ "You are an AI Chatbot. Answer naturally and helpfully using only the information from the references above. " .
	3799	+ "Be conversational and friendly, but never mention your knowledge base or training data. " .
	3800	+ "If you don't have specific information or are uncertain about any details, it's always " .
	3801	+ "better to honestly say you don't know rather than making up or guessing at answers. " .
	3802	+ "When information is incomplete, let them know you are unsure.";
5614	3803	}
5615		-
	3804	+
5616	3805	return trim($content);
5617	3806	}
5618	3807
5619	3808	/**
		@@ -5653,518 +3842,12 @@
5653	3842	}
5654	3843
5655	3844	// Cache individual role for 1 hour
5656	3845	wp_cache_set($cache_key, $role_restriction, 'mxchat_vector_roles', 3600);
5657		-
	3846	+
5658	3847	return $role_restriction;
5659	3848	}
5660	3849
5661		-/**
5662		- * Fetch and reassemble all chunks for a URL from Pinecone
5663		- *
5664		- * @param string $source_url The source URL to fetch chunks for
5665		- * @param array $bot_config Bot-specific Pinecone configuration
5666		- * @return string Reassembled content from all chunks
5667		- */
5668		-private function reassemble_chunks_from_pinecone($source_url, $bot_config, $max_chunks = 0, &$chunk_count = 0) {
5669		- $api_key = $bot_config['api_key'] ?? '';
5670		- $host = $bot_config['host'] ?? '';
5671		- $namespace = $bot_config['namespace'] ?? '';
5672		-
5673		- if (empty($host) || empty($api_key)) {
5674		- $chunk_count = 0;
5675		- return '';
5676		- }
5677		-
5678		- $base_hash = md5($source_url);
5679		-
5680		- // Use Pinecone list API to find all chunk vectors with this prefix
5681		- $list_url = "https://{$host}/vectors/list";
5682		-
5683		- // Limit to max_chunks if specified, otherwise fetch up to 100
5684		- $fetch_limit = ($max_chunks > 0 && $max_chunks < 100) ? $max_chunks : 100;
5685		-
5686		- $list_body = array(
5687		- 'prefix' => $base_hash . '_chunk_',
5688		- 'limit' => $fetch_limit
5689		- );
5690		-
5691		- if (!empty($namespace)) {
5692		- $list_body['namespace'] = $namespace;
5693		- }
5694		-
5695		- $list_response = wp_remote_post($list_url, array(
5696		- 'headers' => array(
5697		- 'Api-Key' => $api_key,
5698		- 'accept' => 'application/json',
5699		- 'content-type' => 'application/json'
5700		- ),
5701		- 'body' => wp_json_encode($list_body),
5702		- 'timeout' => 30
5703		- ));
5704		-
5705		- if (is_wp_error($list_response)) {
5706		- //error_log('[MXCHAT-CHUNK] List API error: ' . $list_response->get_error_message());
5707		- return '';
5708		- }
5709		-
5710		- $list_data = json_decode(wp_remote_retrieve_body($list_response), true);
5711		-
5712		- if (empty($list_data['vectors'])) {
5713		- //error_log('[MXCHAT-CHUNK] No chunk vectors found for URL: ' . $source_url);
5714		- return '';
5715		- }
5716		-
5717		- // Extract vector IDs
5718		- $vector_ids = array();
5719		- foreach ($list_data['vectors'] as $vector) {
5720		- if (isset($vector['id'])) {
5721		- $vector_ids[] = $vector['id'];
5722		- }
5723		- }
5724		-
5725		- if (empty($vector_ids)) {
5726		- return '';
5727		- }
5728		-
5729		- // Fetch all chunk content
5730		- $fetch_url = "https://{$host}/vectors/fetch";
5731		-
5732		- $fetch_body = array(
5733		- 'ids' => $vector_ids
5734		- );
5735		-
5736		- if (!empty($namespace)) {
5737		- $fetch_body['namespace'] = $namespace;
5738		- }
5739		-
5740		- $fetch_response = wp_remote_post($fetch_url, array(
5741		- 'headers' => array(
5742		- 'Api-Key' => $api_key,
5743		- 'accept' => 'application/json',
5744		- 'content-type' => 'application/json'
5745		- ),
5746		- 'body' => wp_json_encode($fetch_body),
5747		- 'timeout' => 30
5748		- ));
5749		-
5750		- if (is_wp_error($fetch_response)) {
5751		- //error_log('[MXCHAT-CHUNK] Fetch API error: ' . $fetch_response->get_error_message());
5752		- return '';
5753		- }
5754		-
5755		- $fetch_data = json_decode(wp_remote_retrieve_body($fetch_response), true);
5756		-
5757		- if (empty($fetch_data['vectors'])) {
5758		- return '';
5759		- }
5760		-
5761		- // Sort chunks by index and reassemble
5762		- $chunks = array();
5763		- foreach ($fetch_data['vectors'] as $id => $vector) {
5764		- $metadata = $vector['metadata'] ?? array();
5765		- $chunk_index = $metadata['chunk_index'] ?? 0;
5766		- $text = $metadata['text'] ?? '';
5767		-
5768		- // Store chunk with its index
5769		- $chunks[$chunk_index] = $text;
5770		- }
5771		-
5772		- // Sort by chunk index
5773		- ksort($chunks);
5774		-
5775		- // Apply chunk limit if specified
5776		- if ($max_chunks > 0 && count($chunks) > $max_chunks) {
5777		- $chunks = array_slice($chunks, 0, $max_chunks, true);
5778		- }
5779		-
5780		- // Store actual chunk count
5781		- $chunk_count = count($chunks);
5782		-
5783		- // Reassemble content
5784		- return implode("\n\n", $chunks);
5785		-}
5786		-
5787		-/**
5788		- * Search for relevant content using OpenAI Vector Store (File Search)
5789		- *
5790		- * @param string $user_query The user's query text
5791		- * @param string $bot_id The bot ID
5792		- * @param array $vectorstore_config Vector Store configuration
5793		- * @return string Formatted context string with references
5794		- */
5795		-private function find_relevant_content_openai_vectorstore($user_query, $bot_id = 'default', $vectorstore_config = array()) {
5796		- //error_log("MXCHAT DEBUG: find_relevant_content_openai_vectorstore called");
5797		- //error_log(" - bot_id: " . $bot_id);
5798		- //error_log(" - user_query length: " . strlen($user_query));
5799		-
5800		- // Get OpenAI API key
5801		- $mxchat_options = get_option('mxchat_options', array());
5802		- $api_key = $mxchat_options['api_key'] ?? '';
5803		-
5804		- // Reset vectorstore error tracking
5805		- $this->last_vectorstore_error = null;
5806		-
5807		- if (empty($api_key)) {
5808		- //error_log("MXCHAT DEBUG ERROR: OpenAI API key not configured");
5809		- $this->last_vectorstore_error = 'Vector Store search failed: OpenAI API key is not configured.';
5810		- $this->current_valid_urls = [];
5811		- return '';
5812		- }
5813		-
5814		- // Get Vector Store configuration
5815		- if (empty($vectorstore_config)) {
5816		- $vectorstore_config = $this->get_bot_vectorstore_config($bot_id);
5817		- }
5818		-
5819		- $vectorstore_ids_string = $vectorstore_config['vectorstore_ids'] ?? '';
5820		- $max_results = $vectorstore_config['max_results'] ?? 5;
5821		-
5822		- if (empty($vectorstore_ids_string)) {
5823		- //error_log("MXCHAT DEBUG ERROR: No Vector Store IDs configured");
5824		- $this->last_vectorstore_error = 'Vector Store search failed: No Vector Store IDs are configured for this bot.';
5825		- $this->current_valid_urls = [];
5826		- return '';
5827		- }
5828		-
5829		- // Parse Vector Store IDs
5830		- $vectorstore_ids = array_map('trim', explode(',', $vectorstore_ids_string));
5831		- $vectorstore_ids = array_filter($vectorstore_ids); // Remove empty values
5832		-
5833		- //error_log("MXCHAT DEBUG: Vector Store IDs: " . implode(', ', $vectorstore_ids));
5834		- //error_log("MXCHAT DEBUG: Max results: " . $max_results);
5835		-
5836		- // Initialize similarity analysis storage
5837		- $this->last_similarity_analysis = [
5838		- 'knowledge_base_type' => 'OpenAI Vector Store',
5839		- 'bot_id' => $bot_id,
5840		- 'vectorstore_ids' => $vectorstore_ids,
5841		- 'top_matches' => [],
5842		- 'threshold_used' => 0,
5843		- 'total_checked' => 0
5844		- ];
5845		-
5846		- $valid_urls = [];
5847		-
5848		- // Get the selected model
5849		- $bot_options = $this->get_bot_options($bot_id);
5850		- $current_options = !empty($bot_options) ? $bot_options : $mxchat_options;
5851		- $selected_model = $current_options['model'] ?? 'gpt-5.1-chat-latest';
5852		-
5853		- // Verify it's an OpenAI model
5854		- if (!$this->is_openai_chat_model($selected_model)) {
5855		- //error_log("MXCHAT DEBUG ERROR: Vector Store search requires OpenAI model. Current: " . $selected_model);
5856		- $this->last_vectorstore_error = 'Vector Store search requires an OpenAI model. Current model: ' . $selected_model;
5857		- $this->current_valid_urls = [];
5858		- return '';
5859		- }
5860		-
5861		- // Use OpenAI Responses API with file_search tool
5862		- $request_body = array(
5863		- 'model' => $selected_model,
5864		- 'input' => $user_query,
5865		- 'tools' => array(
5866		- array(
5867		- 'type' => 'file_search',
5868		- 'vector_store_ids' => $vectorstore_ids,
5869		- 'max_num_results' => intval($max_results)
5870		- )
5871		- ),
5872		- 'include' => array('output[*].file_search_call.search_results')
5873		- );
5874		-
5875		- //error_log("MXCHAT VECTORSTORE: ========== REQUEST START ==========");
5876		- //error_log("MXCHAT VECTORSTORE: Model: " . $selected_model);
5877		- //error_log("MXCHAT VECTORSTORE: Query: " . substr($user_query, 0, 200));
5878		- //error_log("MXCHAT VECTORSTORE: Vector Store IDs: " . implode(', ', $vectorstore_ids));
5879		- //error_log("MXCHAT VECTORSTORE: Max Results: " . $max_results);
5880		- //error_log("MXCHAT VECTORSTORE: Request body: " . wp_json_encode($request_body));
5881		-
5882		- $response = wp_remote_post('https://api.openai.com/v1/responses', array(
5883		- 'headers' => array(
5884		- 'Authorization' => 'Bearer ' . $api_key,
5885		- 'Content-Type' => 'application/json'
5886		- ),
5887		- 'body' => wp_json_encode($request_body),
5888		- 'timeout' => 60
5889		- ));
5890		-
5891		- if (is_wp_error($response)) {
5892		- //error_log("MXCHAT VECTORSTORE ERROR: WP Error: " . $response->get_error_message());
5893		- $this->last_vectorstore_error = 'Vector Store API request failed: ' . $response->get_error_message();
5894		- $this->current_valid_urls = [];
5895		- return '';
5896		- }
5897		-
5898		- $response_code = wp_remote_retrieve_response_code($response);
5899		- //error_log("MXCHAT VECTORSTORE: Response code: " . $response_code);
5900		-
5901		- $response_body = wp_remote_retrieve_body($response);
5902		- //error_log("MXCHAT VECTORSTORE: Raw response (first 2000 chars): " . substr($response_body, 0, 2000));
5903		-
5904		- if ($response_code !== 200) {
5905		- //error_log("MXCHAT VECTORSTORE ERROR: API error response: " . $response_body);
5906		- $api_error_detail = '';
5907		- $decoded_error = json_decode($response_body, true);
5908		- if (isset($decoded_error['error']['message'])) {
5909		- $api_error_detail = $decoded_error['error']['message'];
5910		- }
5911		- $this->last_vectorstore_error = 'Vector Store API returned HTTP ' . $response_code . ($api_error_detail ? ': ' . $api_error_detail : '');
5912		- $this->current_valid_urls = [];
5913		- return '';
5914		- }
5915		- $result = json_decode($response_body, true);
5916		-
5917		- if (json_last_error() !== JSON_ERROR_NONE) {
5918		- //error_log("MXCHAT VECTORSTORE ERROR: JSON decode error: " . json_last_error_msg());
5919		- $this->last_vectorstore_error = 'Vector Store response could not be parsed: ' . json_last_error_msg();
5920		- $this->current_valid_urls = [];
5921		- return '';
5922		- }
5923		-
5924		- // Debug: Log the structure of the result
5925		- //error_log("MXCHAT VECTORSTORE: Result keys: " . implode(', ', array_keys($result)));
5926		- if (isset($result['output'])) {
5927		- //error_log("MXCHAT VECTORSTORE: Output count: " . count($result['output']));
5928		- foreach ($result['output'] as $idx => $out) {
5929		- //error_log("MXCHAT VECTORSTORE: Output[$idx] type: " . ($out['type'] ?? 'unknown'));
5930		- //error_log("MXCHAT VECTORSTORE: Output[$idx] keys: " . implode(', ', array_keys($out)));
5931		- }
5932		- } else {
5933		- //error_log("MXCHAT VECTORSTORE: No 'output' key in result!");
5934		- }
5935		-
5936		- // Extract file search results from the response
5937		- $content = '';
5938		- $matches_used = 0;
5939		- $all_matches = [];
5940		-
5941		- // The Responses API returns output array with tool results
5942		- if (isset($result['output']) && is_array($result['output'])) {
5943		- foreach ($result['output'] as $output_item) {
5944		- // Look for file_search_call results
5945		- if (isset($output_item['type']) && $output_item['type'] === 'file_search_call') {
5946		- //error_log("MXCHAT VECTORSTORE: Found file_search_call output item");
5947		- //error_log("MXCHAT VECTORSTORE: file_search_call keys: " . implode(', ', array_keys($output_item)));
5948		-
5949		- // Check for search_results in the output item directly
5950		- $search_results = $output_item['search_results'] ?? $output_item['results'] ?? [];
5951		- //error_log("MXCHAT VECTORSTORE: Search results count: " . count($search_results));
5952		-
5953		- if (empty($search_results)) {
5954		- //error_log("MXCHAT VECTORSTORE: No search results found in file_search_call");
5955		- //error_log("MXCHAT VECTORSTORE: file_search_call content: " . wp_json_encode($output_item));
5956		- }
5957		-
5958		- foreach ($search_results as $index => $search_result) {
5959		- $filename = $search_result['filename'] ?? '';
5960		- $score = $search_result['score'] ?? 0;
5961		- $text_content = '';
5962		-
5963		- // Extract text content from the result
5964		- // The text can be directly on the result OR nested under content array
5965		- if (isset($search_result['text']) && !empty($search_result['text'])) {
5966		- // Direct text field (OpenAI's actual format)
5967		- $text_content = $search_result['text'];
5968		- //error_log("MXCHAT VECTORSTORE: Found text directly on result[$index], length: " . strlen($text_content));
5969		- } elseif (isset($search_result['content']) && is_array($search_result['content'])) {
5970		- // Nested content array format
5971		- foreach ($search_result['content'] as $content_item) {
5972		- if (isset($content_item['text'])) {
5973		- $text_content .= $content_item['text'] . "\n";
5974		- }
5975		- }
5976		- //error_log("MXCHAT VECTORSTORE: Found text in content array for result[$index], length: " . strlen($text_content));
5977		- } else {
5978		- //error_log("MXCHAT VECTORSTORE: No text found for result[$index]. Keys: " . implode(', ', array_keys($search_result)));
5979		- }
5980		-
5981		- if (!empty($text_content)) {
5982		- $content .= "## Reference " . ($matches_used + 1) . " ##\n";
5983		- $content .= trim($text_content) . "\n\n";
5984		-
5985		- if (!empty($filename)) {
5986		- $content .= "Source: " . $filename . "\n\n";
5987		- }
5988		-
5989		- // Extract URLs from content
5990		- preg_match_all(
5991		- '#\bhttps?://[^\s<>"\']+#i',
5992		- $text_content,
5993		- $content_urls
5994		- );
5995		- if (!empty($content_urls[0])) {
5996		- $valid_urls = array_merge($valid_urls, $content_urls[0]);
5997		- }
5998		-
5999		- $matches_used++;
6000		- }
6001		-
6002		- // Store for similarity analysis
6003		- $all_matches[] = [
6004		- 'document_id' => $filename ?: ('result_' . $index),
6005		- 'similarity' => $score,
6006		- 'similarity_percentage' => round($score * 100, 2),
6007		- 'above_threshold' => true,
6008		- 'source_display' => $filename,
6009		- 'content_preview' => substr(strip_tags($text_content), 0, 100) . '...',
6010		- 'used_for_context' => true,
6011		- 'role_restriction' => 'public',
6012		- 'has_access' => true,
6013		- 'filtered_out' => false
6014		- ];
6015		- }
6016		- }
6017		-
6018		- // Also check for message content with annotations (citations)
6019		- if (isset($output_item['type']) && $output_item['type'] === 'message') {
6020		- if (isset($output_item['content']) && is_array($output_item['content'])) {
6021		- foreach ($output_item['content'] as $content_block) {
6022		- if (isset($content_block['annotations']) && is_array($content_block['annotations'])) {
6023		- foreach ($content_block['annotations'] as $annotation) {
6024		- if (isset($annotation['filename'])) {
6025		- $filename = $annotation['filename'];
6026		- $score = $annotation['score'] ?? 0;
6027		- $text_content = '';
6028		-
6029		- if (isset($annotation['content']) && is_array($annotation['content'])) {
6030		- foreach ($annotation['content'] as $ann_content) {
6031		- if (isset($ann_content['text'])) {
6032		- $text_content .= $ann_content['text'] . "\n";
6033		- }
6034		- }
6035		- }
6036		-
6037		- if (!empty($text_content) && $matches_used < $max_results) {
6038		- $content .= "## Reference " . ($matches_used + 1) . " ##\n";
6039		- $content .= trim($text_content) . "\n\n";
6040		- $content .= "Source: " . $filename . "\n\n";
6041		-
6042		- preg_match_all(
6043		- '#\bhttps?://[^\s<>"\']+#i',
6044		- $text_content,
6045		- $content_urls
6046		- );
6047		- if (!empty($content_urls[0])) {
6048		- $valid_urls = array_merge($valid_urls, $content_urls[0]);
6049		- }
6050		-
6051		- $matches_used++;
6052		-
6053		- $all_matches[] = [
6054		- 'document_id' => $filename,
6055		- 'similarity' => $score,
6056		- 'similarity_percentage' => round($score * 100, 2),
6057		- 'above_threshold' => true,
6058		- 'source_display' => $filename,
6059		- 'content_preview' => substr(strip_tags($text_content), 0, 100) . '...',
6060		- 'used_for_context' => true,
6061		- 'role_restriction' => 'public',
6062		- 'has_access' => true,
6063		- 'filtered_out' => false
6064		- ];
6065		- }
6066		- }
6067		- }
6068		- }
6069		- }
6070		- }
6071		- }
6072		- }
6073		- }
6074		-
6075		- // Store for testing panel
6076		- $this->last_similarity_analysis['top_matches'] = $all_matches;
6077		- $this->last_similarity_analysis['total_checked'] = count($all_matches);
6078		-
6079		- // Store unique valid URLs for validation
6080		- $this->current_valid_urls = array_unique($valid_urls);
6081		-
6082		- // Allow add-ons to act on similarity results (e.g. WooCommerce product card display)
6083		- do_action('mxchat_similarity_results', $this->last_similarity_analysis['top_matches'], $bot_id);
6084		-
6085		- //error_log("MXCHAT VECTORSTORE: ========== SEARCH COMPLETE ==========");
6086		- //error_log("MXCHAT VECTORSTORE: Matches used: " . $matches_used);
6087		- //error_log("MXCHAT VECTORSTORE: All matches count: " . count($all_matches));
6088		- //error_log("MXCHAT VECTORSTORE: Content length: " . strlen($content));
6089		- if ($matches_used > 0) {
6090		- //error_log("MXCHAT VECTORSTORE: Content preview: " . substr($content, 0, 500));
6091		- }
6092		-
6093		- // Check if citation links are enabled
6094		- $citation_links_enabled = ($mxchat_options['citation_links_toggle'] ?? 'on') === 'on';
6095		-
6096		- // Add response guidelines
6097		- if ($matches_used === 0) {
6098		- //error_log("MXCHAT VECTORSTORE: No matches found - returning empty reference message");
6099		- $content = "No reference information was found for this query.\n\n";
6100		- } else {
6101		- // Build response guidelines based on citation links setting
6102		- $content .= "\n## Response Guidelines ##\n" .
6103		- "You are an AI Chatbot. Answer naturally and helpfully using only the information from the references above. " .
6104		- "Be conversational and friendly, but never mention your knowledge base or training data. " .
6105		- "If you don't have specific information or are uncertain about any details, it's always " .
6106		- "better to honestly say you don't know rather than making up or guessing at answers. " .
6107		- "When information is incomplete, let them know you are unsure.\n\n";
6108		-
6109		- // Only add hyperlink instructions if citation links are enabled
6110		- if ($citation_links_enabled) {
6111		- $content .= "CRITICAL: When creating hyperlinks, always use proper markdown format with descriptive text: " .
6112		- "[descriptive text](url). NEVER use empty brackets like [](url). The text in brackets must describe what the link is about.";
6113		- } else {
6114		- $content .= "IMPORTANT: Do not include any citation links, source URLs, or hyperlinks in your responses. " .
6115		- "Simply provide helpful answers based on the reference information without citing sources.";
6116		- }
6117		- }
6118		-
6119		- //error_log("MXCHAT DEBUG: Vector Store search complete. Matches used: " . $matches_used);
6120		-
6121		- return trim($content);
6122		-}
6123		-
6124		-/**
6125		- * Check if the given model is an OpenAI chat model
6126		- *
6127		- * @param string $model The model ID
6128		- * @return bool True if it's an OpenAI model
6129		- */
6130		-private function is_openai_chat_model($model) {
6131		- $openai_prefixes = array('gpt-', 'o1-', 'o3-');
6132		- foreach ($openai_prefixes as $prefix) {
6133		- if (strpos($model, $prefix) === 0) {
6134		- return true;
6135		- }
6136		- }
6137		- return false;
6138		-}
6139		-
6140		-/**
6141		- * Get bot-specific Vector Store configuration
6142		- *
6143		- * @param string $bot_id The bot ID
6144		- * @return array Configuration array
6145		- */
6146		-private function get_bot_vectorstore_config($bot_id = 'default') {
6147		- $vectorstore_options = get_option('mxchat_openai_vectorstore_options', array());
6148		-
6149		- // Default global settings
6150		- $default_config = array(
6151		- 'use_vectorstore' => ($vectorstore_options['mxchat_use_openai_vectorstore'] ?? '0') === '1',
6152		- 'vectorstore_ids' => $vectorstore_options['mxchat_vectorstore_ids'] ?? '',
6153		- 'max_results' => $vectorstore_options['mxchat_vectorstore_max_results'] ?? 5
6154		- );
6155		-
6156		- // Allow multi-bot plugin to override with bot-specific settings
6157		- $bot_config = apply_filters('mxchat_get_bot_vectorstore_config', $default_config, $bot_id);
6158		-
6159		- // Preserve max_results from global settings if not set in bot config
6160		- if (!isset($bot_config['max_results'])) {
6161		- $bot_config['max_results'] = $default_config['max_results'];
6162		- }
6163		-
6164		- return $bot_config;
6165		-}
6166		-
6167	3850	private function mxchat_find_relevant_products($user_embedding) {
6168	3851	//error_log('MXChat Vector Search: Starting product search...');
6169	3852
6170	3853	// Retrieve the add-on settings from the database
		@@ -6185,75 +3868,73 @@
6185	3868	}
6186	3869	private function find_relevant_products_wordpress($user_embedding) {
6187	3870	global $wpdb;
6188	3871	$system_prompt_table = $wpdb->prefix . 'mxchat_system_prompt_content';
	3872	+ $cache_key = 'mxchat_system_prompt_embeddings';
	3873	+ $batch_size = 500;
6189	3874
6190		- if (!is_array($user_embedding)) {
6191		- return '';
6192		- }
	3875	+ // Original WordPress database search logic
	3876	+ // [Previous implementation remains the same]
	3877	+ $embeddings = wp_cache_get($cache_key, 'mxchat_system_prompts');
	3878	+ if ($embeddings === false) {
	3879	+ $embeddings = [];
	3880	+ $offset = 0;
6193	3881
6194		- // Streaming top-K pass: scan rows in small batches, keep only the top 3
6195		- // results above the similarity threshold. Peak memory is bounded by
6196		- // $batch_size embedding rows plus a 3-element top list.
6197		- $batch_size = 250;
6198		- $similarity_threshold = 0.85;
6199		- $top_k = 3;
6200		- $top_results = [];
6201		- $offset = 0;
	3882	+ do {
	3883	+ $query = $wpdb->prepare(
	3884	+ "SELECT id, embedding_vector
	3885	+ FROM {$system_prompt_table}
	3886	+ LIMIT %d OFFSET %d",
	3887	+ $batch_size,
	3888	+ $offset
	3889	+ );
6202	3890
6203		- do {
6204		- $batch = $wpdb->get_results($wpdb->prepare(
6205		- "SELECT id, embedding_vector
6206		- FROM {$system_prompt_table}
6207		- LIMIT %d OFFSET %d",
6208		- $batch_size,
6209		- $offset
6210		- ));
	3891	+ $batch = $wpdb->get_results($query);
	3892	+ if (empty($batch)) {
	3893	+ break;
	3894	+ }
6211	3895
6212		- if (empty($batch)) {
6213		- break;
6214		- }
	3896	+ $embeddings = array_merge($embeddings, $batch);
	3897	+ $offset += $batch_size;
6215	3898
6216		- foreach ($batch as $row) {
6217		- $database_embedding = $row->embedding_vector
6218		- ? unserialize($row->embedding_vector, ['allowed_classes' => false])
6219		- : null;
	3899	+ unset($batch);
6220	3900
6221		- if (!is_array($database_embedding)) {
6222		- unset($database_embedding);
6223		- continue;
6224		- }
	3901	+ } while (true);
6225	3902
	3903	+ if (empty($embeddings)) {
	3904	+ return '';
	3905	+ }
	3906	+ wp_cache_set($cache_key, $embeddings, 'mxchat_system_prompts', 3600);
	3907	+ }
	3908	+
	3909	+ $relevant_results = [];
	3910	+ foreach ($embeddings as $embedding) {
	3911	+ $database_embedding = $embedding->embedding_vector
	3912	+ ? unserialize($embedding->embedding_vector, ['allowed_classes' => false])
	3913	+ : null;
	3914	+ if (is_array($database_embedding) && is_array($user_embedding)) {
6226	3915	$similarity = $this->mxchat_calculate_cosine_similarity($user_embedding, $database_embedding);
6227		- unset($database_embedding);
6228		-
6229		- if ($similarity < $similarity_threshold) {
6230		- continue;
6231		- }
6232		-
6233		- // Insert into bounded top-K (kept sorted descending)
6234		- if (count($top_results) < $top_k) {
6235		- $top_results[] = ['id' => $row->id, 'similarity' => $similarity];
6236		- usort($top_results, function ($a, $b) {
6237		- return $b['similarity'] <=> $a['similarity'];
6238		- });
6239		- } elseif ($similarity > $top_results[$top_k - 1]['similarity']) {
6240		- $top_results[$top_k - 1] = ['id' => $row->id, 'similarity' => $similarity];
6241		- usort($top_results, function ($a, $b) {
6242		- return $b['similarity'] <=> $a['similarity'];
6243		- });
6244		- }
	3916	+ $relevant_results[] = [
	3917	+ 'id' => $embedding->id,
	3918	+ 'similarity' => $similarity
	3919	+ ];
6245	3920	}
	3921	+ unset($database_embedding);
	3922	+ }
6246	3923
6247		- unset($batch);
6248		- $offset += $batch_size;
6249		- } while (true);
	3924	+ // Use fixed threshold for products
	3925	+ $similarity_threshold = 0.85;
6250	3926
6251		- if (empty($top_results)) {
6252		- return '';
6253		- }
	3927	+ $relevant_results = array_filter($relevant_results, function ($result) use ($similarity_threshold) {
	3928	+ return $result['similarity'] >= $similarity_threshold;
	3929	+ });
	3930	+ usort($relevant_results, function ($a, $b) {
	3931	+ return $b['similarity'] <=> $a['similarity'];
	3932	+ });
6254	3933
	3934	+ $top_results = array_slice($relevant_results, 0, 5);
6255	3935	$content = '';
	3936	+
6256	3937	foreach ($top_results as $result) {
6257	3938	$chunk_content = $this->fetch_content_with_product_links($result['id']);
6258	3939	$content .= $chunk_content . "\n\n";
6259	3940	}
		@@ -6259,10 +3940,8 @@
6259	3940	}
6260	3941
6261	3942	return trim($content);
6262	3943	}
6263		-
6264		-
6265	3944	private function find_relevant_products_pinecone($user_embedding) {
6266	3945	//error_log('Starting Pinecone product search...');
6267	3946
6268	3947	$options = get_option('mxchat_pinecone_addon_options', array());
		@@ -6337,10 +4016,8 @@
6337	4016	}
6338	4017
6339	4018	return trim($content);
6340	4019	}
6341		-
6342		-
6343	4020	private function fetch_content_with_product_links($most_relevant_id) {
6344	4021	global $wpdb;
6345	4022	$system_prompt_table = $wpdb->prefix . 'mxchat_system_prompt_content';
6346	4023
		@@ -6360,85 +4037,12 @@
6360	4037	return null;
6361	4038	}
6362	4039
6363	4040	/**
6364		- * Get system instructions for a specific bot or default
6365		- * Checks for multi-bot add-on and uses bot-specific instructions if available
6366		- * Automatically strips URLs if citation links are disabled
6367		- * Replaces {visitor_name} placeholder with actual visitor name if available
6368		- *
6369		- * @param string $bot_id The bot ID to get instructions for
6370		- * @param string $session_id Optional session ID to lookup visitor name
	4041	+ * Modified streaming functions to include testing data
6371	4042	*/
6372		-private function get_system_instructions($bot_id = 'default', $session_id = '') {
6373		- $instructions = '';
6374	4043
6375		- // Check if multi-bot add-on is active
6376		- if (class_exists('MxChat_Multi_Bot_Core_Manager') && $bot_id !== 'default') {
6377		- // Get bot-specific options from multi-bot add-on
6378		- $bot_options = apply_filters('mxchat_get_bot_options', array(), $bot_id);
6379		-
6380		- // If bot has custom system instructions, use those
6381		- if (!empty($bot_options['system_prompt_instructions'])) {
6382		- $instructions = $bot_options['system_prompt_instructions'];
6383		- }
6384		- }
6385		-
6386		- // Fall back to default system instructions
6387		- if (empty($instructions)) {
6388		- $instructions = isset($this->options['system_prompt_instructions']) ? $this->options['system_prompt_instructions'] : '';
6389		- }
6390		-
6391		- // Check if citation links are disabled - if so, strip URLs from instructions
6392		- $fresh_options = get_option('mxchat_options', []);
6393		- $citation_links_enabled = isset($fresh_options['citation_links_toggle']) ? ($fresh_options['citation_links_toggle'] === 'on') : true;
6394		-
6395		- if (!$citation_links_enabled && !empty($instructions)) {
6396		- $instructions = preg_replace('#\bhttps?://[^\s<>"\']+#i', '', $instructions);
6397		- $instructions = preg_replace('/\s+/', ' ', trim($instructions)); // Clean up extra spaces
6398		- }
6399		-
6400		- // Replace {visitor_name} placeholder with actual visitor name if available
6401		- if (!empty($instructions) && !empty($session_id) && stripos($instructions, '{visitor_name}') !== false) {
6402		- $name_option_key = "mxchat_name_{$session_id}";
6403		- $visitor_name = get_option($name_option_key, '');
6404		-
6405		- if (!empty($visitor_name)) {
6406		- $instructions = str_ireplace('{visitor_name}', sanitize_text_field($visitor_name), $instructions);
6407		- } else {
6408		- // Remove placeholder if no name is available
6409		- $instructions = str_ireplace('{visitor_name}', '', $instructions);
6410		- $instructions = preg_replace('/\s{2,}/', ' ', trim($instructions)); // Clean up extra spaces
6411		- }
6412		- }
6413		-
6414		- // Allow developers to filter system instructions and process shortcodes
6415		- $instructions = apply_filters('mxchat_system_instructions', $instructions, $bot_id, $session_id);
6416		- $instructions = do_shortcode($instructions);
6417		-
6418		- return $instructions;
6419		-}
6420		-/**
6421		- * Get the current bot ID from session or request context
6422		- */
6423		-private function get_current_bot_id($session_id = '') {
6424		- // First, check if bot_id is passed in the current request
6425		- if (isset($_POST['bot_id']) && !empty($_POST['bot_id'])) {
6426		- return sanitize_key($_POST['bot_id']);
6427		- }
6428		-
6429		- // If not in POST, try to get it from session data
6430		- if (!empty($session_id)) {
6431		- $bot_id = get_option("mxchat_session_bot_{$session_id}", '');
6432		- if (!empty($bot_id)) {
6433		- return $bot_id;
6434		- }
6435		- }
6436		-
6437		- // Fall back to default
6438		- return 'default';
6439		-}
6440		-private function mxchat_generate_response($relevant_content, $api_key, $xai_api_key, $claude_api_key, $deepseek_api_key, $gemini_api_key, $openrouter_api_key, $conversation_history, $streaming = false, $session_id = '', $testing_data = null, $selected_model = 'gpt-5.1-chat-latest') {
	4044	+private function mxchat_generate_response($relevant_content, $api_key, $xai_api_key, $claude_api_key, $deepseek_api_key, $gemini_api_key, $conversation_history, $streaming = false, $session_id = '', $testing_data = null) {
6441	4045	try {
6442	4046	if (!$relevant_content) {
6443	4047	$error_response = [
6444	4048	'error' => esc_html__("I couldn't find relevant information on that topic.", 'mxchat'),
		@@ -6444,74 +4048,25 @@
6444	4048	'error' => esc_html__("I couldn't find relevant information on that topic.", 'mxchat'),
6445	4049	'error_code' => 'no_relevant_content'
6446	4050	];
6447	4051
	4052	+ // Add testing data to error response if available
6448	4053	if ($testing_data !== null) {
6449	4054	$error_response['testing_data'] = $testing_data;
	4055	+ //error_log("MxChat Testing: Added testing data to no_relevant_content error");
6450	4056	}
6451	4057
6452	4058	return $error_response;
6453	4059	}
6454	4060
	4061	+ // Ensure conversation_history is an array
6455	4062	if (!is_array($conversation_history)) {
6456	4063	$conversation_history = array();
6457	4064	}
6458	4065
6459		- // Check if this is an OpenRouter model
6460		- if ($selected_model === 'openrouter') {
6461		- // Get the actual OpenRouter model from options
6462		- $openrouter_selected_model = $this->options['openrouter_selected_model'] ?? '';
6463		-
6464		- if (empty($openrouter_selected_model)) {
6465		- $error_response = [
6466		- 'error' => esc_html__('No OpenRouter model selected. Please select a model in settings.', 'mxchat'),
6467		- 'error_code' => 'no_openrouter_model_selected'
6468		- ];
6469		- if ($testing_data !== null) {
6470		- $error_response['testing_data'] = $testing_data;
6471		- }
6472		- return $error_response;
6473		- }
6474		-
6475		- if (empty($openrouter_api_key)) {
6476		- $error_response = [
6477		- 'error' => esc_html__('OpenRouter API key is not configured', 'mxchat'),
6478		- 'error_code' => 'missing_openrouter_api_key'
6479		- ];
6480		- if ($testing_data !== null) {
6481		- $error_response['testing_data'] = $testing_data;
6482		- }
6483		- return $error_response;
6484		- }
6485		-
6486		- if ($streaming) {
6487		- return $this->mxchat_generate_response_openrouter_stream(
6488		- $openrouter_selected_model,
6489		- $openrouter_api_key,
6490		- $conversation_history,
6491		- $relevant_content,
6492		- $session_id,
6493		- $testing_data
6494		- );
6495		- } else {
6496		- $response = $this->mxchat_generate_response_openrouter(
6497		- $openrouter_selected_model,
6498		- $openrouter_api_key,
6499		- $conversation_history,
6500		- $relevant_content
6501		- );
6502		- }
6503		-
6504		- if (is_array($response) && isset($response['error'])) {
6505		- if ($testing_data !== null) {
6506		- $response['testing_data'] = $testing_data;
6507		- }
6508		- return $response;
6509		- }
6510		-
6511		- return $response;
6512		- }
6513		-
	4066	+ // Get selected model with default fallback
	4067	+ $selected_model = isset($this->options['model']) ? $this->options['model'] : 'gpt-4o';
	4068	+
6514	4069	// Extract model prefix to determine the provider
6515	4070	$model_parts = explode('-', $selected_model);
6516	4071	$provider = strtolower($model_parts[0]);
6517	4072
		@@ -6553,9 +4108,9 @@
6553	4108	$claude_api_key,
6554	4109	$conversation_history,
6555	4110	$relevant_content,
6556	4111	$session_id,
6557		- $testing_data
	4112	+ $testing_data // Pass testing data
6558	4113	);
6559	4114	} else {
6560	4115	$response = $this->mxchat_generate_response_claude(
6561	4116	$selected_model,
		@@ -6583,9 +4138,9 @@
6583	4138	$xai_api_key,
6584	4139	$conversation_history,
6585	4140	$relevant_content,
6586	4141	$session_id,
6587		- $testing_data
	4142	+ $testing_data // Pass testing data
6588	4143	);
6589	4144	} else {
6590	4145	$response = $this->mxchat_generate_response_xai(
6591	4146	$selected_model,
		@@ -6613,9 +4168,9 @@
6613	4168	$deepseek_api_key,
6614	4169	$conversation_history,
6615	4170	$relevant_content,
6616	4171	$session_id,
6617		- $testing_data
	4172	+ $testing_data // Pass testing data
6618	4173	);
6619	4174	} else {
6620	4175	$response = $this->mxchat_generate_response_deepseek(
6621	4176	$selected_model,
		@@ -6637,27 +4192,9 @@
6637	4192	$error_response['testing_data'] = $testing_data;
6638	4193	}
6639	4194	return $error_response;
6640	4195	}
6641		-
6642		- // Check if web search is enabled for this OpenAI model
6643		- $web_search_enabled = isset($this->options['enable_web_search']) && $this->options['enable_web_search'] === 'on';
6644		- // Models that don't support web search
6645		- $unsupported_web_search_models = array('gpt-4.1-nano');
6646		- $model_supports_web_search = !in_array($selected_model, $unsupported_web_search_models);
6647		-
6648		- if ($web_search_enabled && $model_supports_web_search) {
6649		- // Use Responses API (required for some models, or when web search is enabled)
6650		- return $this->mxchat_generate_response_openai_web_search(
6651		- $selected_model,
6652		- $api_key,
6653		- $conversation_history,
6654		- $relevant_content,
6655		- $session_id,
6656		- $testing_data,
6657		- $streaming
6658		- );
6659		- } elseif ($streaming) {
	4196	+ if ($streaming) {
6660	4197	return $this->mxchat_generate_response_openai_stream(
6661	4198	$selected_model,
6662	4199	$api_key,
6663	4200	$conversation_history,
		@@ -6662,9 +4199,9 @@
6662	4199	$api_key,
6663	4200	$conversation_history,
6664	4201	$relevant_content,
6665	4202	$session_id,
6666		- $testing_data
	4203	+ $testing_data // Pass testing data
6667	4204	);
6668	4205	} else {
6669	4206	$response = $this->mxchat_generate_response_openai(
6670	4207	$selected_model,
		@@ -6675,8 +4212,9 @@
6675	4212	}
6676	4213	break;
6677	4214
6678	4215	default:
	4216	+ // Default to OpenAI for custom models or unrecognized prefixes
6679	4217	if (empty($api_key)) {
6680	4218	$error_response = [
6681	4219	'error' => esc_html__('OpenAI API key is not configured', 'mxchat'),
6682	4220	'error_code' => 'missing_openai_api_key'
		@@ -6685,25 +4223,9 @@
6685	4223	$error_response['testing_data'] = $testing_data;
6686	4224	}
6687	4225	return $error_response;
6688	4226	}
6689		-
6690		- // Check if web search is enabled (default case also handles OpenAI models)
6691		- $web_search_enabled = isset($this->options['enable_web_search']) && $this->options['enable_web_search'] === 'on';
6692		- $unsupported_web_search_models = array('gpt-4.1-nano');
6693		- $model_supports_web_search = !in_array($selected_model, $unsupported_web_search_models);
6694		-
6695		- if ($web_search_enabled && $model_supports_web_search) {
6696		- return $this->mxchat_generate_response_openai_web_search(
6697		- $selected_model,
6698		- $api_key,
6699		- $conversation_history,
6700		- $relevant_content,
6701		- $session_id,
6702		- $testing_data,
6703		- $streaming
6704		- );
6705		- } elseif ($streaming) {
	4227	+ if ($streaming) {
6706	4228	return $this->mxchat_generate_response_openai_stream(
6707	4229	$selected_model,
6708	4230	$api_key,
6709	4231	$conversation_history,
		@@ -6708,9 +4230,9 @@
6708	4230	$api_key,
6709	4231	$conversation_history,
6710	4232	$relevant_content,
6711	4233	$session_id,
6712		- $testing_data
	4234	+ $testing_data // Pass testing data
6713	4235	);
6714	4236	} else {
6715	4237	$response = $this->mxchat_generate_response_openai(
6716	4238	$selected_model,
		@@ -6721,18 +4243,24 @@
6721	4243	}
6722	4244	break;
6723	4245	}
6724	4246
	4247	+ // Check if the response is an error array from the provider-specific function
6725	4248	if (is_array($response) && isset($response['error'])) {
	4249	+ // Add testing data to error response if available
6726	4250	if ($testing_data !== null) {
6727	4251	$response['testing_data'] = $testing_data;
	4252	+ //error_log("MxChat Testing: Added testing data to provider error response");
6728	4253	}
6729		- return $response;
	4254	+ return $response; // Pass through the error with testing data
6730	4255	}
6731	4256
	4257	+ // For successful non-streaming responses, we don't add testing data here
	4258	+ // because it will be added in the main handler
6732	4259	return $response;
6733	4260
6734	4261	} catch (Exception $e) {
	4262	+ //error_log('MXChat Error: ' . $e->getMessage());
6735	4263	$error_response = [
6736	4264	'error' => sprintf(esc_html__('An error occurred: %s', 'mxchat'), esc_html($e->getMessage())),
6737	4265	'error_code' => 'system_exception',
6738	4266	'exception_details' => $e->getMessage()
		@@ -6737,230 +4265,23 @@
6737	4265	'error_code' => 'system_exception',
6738	4266	'exception_details' => $e->getMessage()
6739	4267	];
6740	4268
	4269	+ // Add testing data to exception response if available
6741	4270	if ($testing_data !== null) {
6742	4271	$error_response['testing_data'] = $testing_data;
	4272	+ //error_log("MxChat Testing: Added testing data to exception response");
6743	4273	}
6744	4274
6745	4275	return $error_response;
6746	4276	}
6747	4277	}
6748		-private function mxchat_generate_response_openrouter_stream($selected_model, $openrouter_api_key, $conversation_history, $relevant_content, $session_id, $testing_data = null) {
6749		- try {
6750		- $bot_id = $this->get_current_bot_id($session_id);
6751		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
6752		-
6753		- if (!is_array($conversation_history)) {
6754		- $conversation_history = array();
6755		- }
6756	4278
6757		- $formatted_conversation = array();
6758		-
6759		- $formatted_conversation[] = array(
6760		- 'role' => 'system',
6761		- 'content' => $system_prompt_instructions . " " . $relevant_content
6762		- );
6763		-
6764		- foreach ($conversation_history as $message) {
6765		- if (is_array($message) && isset($message['role']) && isset($message['content'])) {
6766		- $role = $message['role'];
6767		- if ($role === 'bot' || $role === 'agent') {
6768		- $role = 'assistant';
6769		- }
6770		- if (!in_array($role, ['system', 'assistant', 'user'])) {
6771		- $role = 'user';
6772		- }
6773		- $formatted_conversation[] = array(
6774		- 'role' => $role,
6775		- 'content' => $message['content']
6776		- );
6777		- }
6778		- }
6779		-
6780		- if (headers_sent() || !function_exists('curl_init')) {
6781		- $regular_response = $this->mxchat_generate_response_openrouter(
6782		- $selected_model,
6783		- $openrouter_api_key,
6784		- $conversation_history,
6785		- $relevant_content
6786		- );
6787		-
6788		- // Save bot response to transcript
6789		- if (!empty($regular_response) && !empty($session_id)) {
6790		- $this->mxchat_save_chat_message($session_id, 'bot', $regular_response);
6791		- }
6792		-
6793		- $response_data = [
6794		- 'text' => $regular_response,
6795		- 'html' => '',
6796		- 'session_id' => $session_id
6797		- ];
6798		-
6799		- if ($testing_data !== null) {
6800		- $response_data['testing_data'] = $testing_data;
6801		- }
6802		-
6803		- header('Content-Type: application/json');
6804		- echo json_encode($response_data);
6805		- return true;
6806		- }
6807		-
6808		- $body = json_encode([
6809		- 'model' => $selected_model,
6810		- 'messages' => $formatted_conversation,
6811		- 'temperature' => 1,
6812		- 'stream' => true
6813		- ]);
6814		-
6815		- // Setup streaming headers now that we know we're actually streaming
6816		- $this->setup_streaming_headers();
6817		-
6818		- $ch = curl_init();
6819		- curl_setopt($ch, CURLOPT_URL, 'https://openrouter.ai/api/v1/chat/completions');
6820		- curl_setopt($ch, CURLOPT_RETURNTRANSFER, false);
6821		- curl_setopt($ch, CURLOPT_POST, true);
6822		- curl_setopt($ch, CURLOPT_POSTFIELDS, $body);
6823		- curl_setopt($ch, CURLOPT_HTTPHEADER, array(
6824		- 'Content-Type: application/json',
6825		- 'Authorization: Bearer ' . $openrouter_api_key,
6826		- 'HTTP-Referer: ' . home_url(),
6827		- 'X-Title: ' . get_bloginfo('name')
6828		- ));
6829		- curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, true);
6830		- curl_setopt($ch, CURLOPT_TIMEOUT, 60);
6831		-
6832		- $full_response = '';
6833		- $stream_started = false;
6834		- $buffer = '';
6835		-
6836		- curl_setopt($ch, CURLOPT_WRITEFUNCTION, function($ch, $data) use (&$full_response, &$stream_started, &$buffer, $testing_data) {
6837		- if (!$stream_started && $testing_data !== null) {
6838		- echo "data: " . json_encode(['testing_data' => $testing_data]) . "\n\n";
6839		- flush();
6840		- $stream_started = true;
6841		- }
6842		-
6843		- $buffer .= $data;
6844		- $lines = explode("\n", $buffer);
6845		- $buffer = array_pop($lines);
6846		-
6847		- foreach ($lines as $line) {
6848		- if (trim($line) === '') {
6849		- continue;
6850		- }
6851		-
6852		- if (strpos($line, 'data: ') !== 0) {
6853		- continue;
6854		- }
6855		-
6856		- $json_str = substr($line, 6);
6857		-
6858		- if (trim($json_str) === '[DONE]') {
6859		- echo "data: [DONE]\n\n";
6860		- flush();
6861		- continue;
6862		- }
6863		-
6864		- $json = json_decode(trim($json_str), true);
6865		- if ($json && isset($json['choices'][0]['delta']['content'])) {
6866		- $content = $json['choices'][0]['delta']['content'];
6867		- $full_response .= $content;
6868		-
6869		- echo "data: " . json_encode(['content' => $content]) . "\n\n";
6870		- flush();
6871		- }
6872		- }
6873		-
6874		- return strlen($data);
6875		- });
6876		-
6877		- $response = curl_exec($ch);
6878		- $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
6879		-
6880		- if (curl_errno($ch) || $http_code !== 200) {
6881		- curl_close($ch);
6882		-
6883		- $regular_response = $this->mxchat_generate_response_openrouter(
6884		- $selected_model,
6885		- $openrouter_api_key,
6886		- $conversation_history,
6887		- $relevant_content
6888		- );
6889		-
6890		- $response_data = [
6891		- 'text' => $regular_response,
6892		- 'html' => '',
6893		- 'session_id' => $session_id
6894		- ];
6895		-
6896		- if ($testing_data !== null) {
6897		- $response_data['testing_data'] = $testing_data;
6898		- }
6899		-
6900		- header('Content-Type: application/json');
6901		- echo json_encode($response_data);
6902		- return true;
6903		- }
6904		-
6905		- curl_close($ch);
6906		-
6907		- if (!empty($full_response) && !empty($session_id)) {
6908		- // Prepare RAG context for streaming response
6909		- $rag_context_for_storage = null;
6910		- $has_rag_data = $this->last_similarity_analysis !== null && !empty($this->last_similarity_analysis['top_matches']);
6911		- $has_action_data = isset($this->last_action_analysis) && !empty($this->last_action_analysis);
6912		-
6913		- if ($has_rag_data || $has_action_data) {
6914		- $rag_context_for_storage = [];
6915		-
6916		- if ($has_rag_data) {
6917		- $rag_context_for_storage['top_matches'] = $this->last_similarity_analysis['top_matches'];
6918		- $rag_context_for_storage['approved_urls'] = $this->current_valid_urls ?? [];
6919		- $rag_context_for_storage['similarity_threshold'] = $this->last_similarity_analysis['threshold_used'] ?? 0.35;
6920		- $rag_context_for_storage['knowledge_base_type'] = $this->last_similarity_analysis['knowledge_base_type'] ?? 'WordPress Database';
6921		- $rag_context_for_storage['total_documents_checked'] = $this->last_similarity_analysis['total_checked'] ?? 0;
6922		- }
6923		-
6924		- if ($has_action_data) {
6925		- $rag_context_for_storage['action_analysis'] = $this->last_action_analysis;
6926		- }
6927		- }
6928		- $this->mxchat_save_chat_message($session_id, 'bot', $full_response, null, $rag_context_for_storage);
6929		- }
6930		-
6931		- return true;
6932		-
6933		- } catch (Exception $e) {
6934		- $regular_response = $this->mxchat_generate_response_openrouter(
6935		- $selected_model,
6936		- $openrouter_api_key,
6937		- $conversation_history,
6938		- $relevant_content
6939		- );
6940		-
6941		- $response_data = [
6942		- 'text' => $regular_response,
6943		- 'html' => '',
6944		- 'session_id' => $session_id
6945		- ];
6946		-
6947		- if ($testing_data !== null) {
6948		- $response_data['testing_data'] = $testing_data;
6949		- }
6950		-
6951		- header('Content-Type: application/json');
6952		- echo json_encode($response_data);
6953		- return true;
6954		- }
6955		-}
6956	4279	private function mxchat_generate_response_openai_stream($selected_model, $api_key, $conversation_history, $relevant_content, $session_id, $testing_data = null) {
6957	4280	try {
6958		- $bot_id = $this->get_current_bot_id($session_id);
	4281	+ // Get system prompt instructions from options
	4282	+ $system_prompt_instructions = isset($this->options['system_prompt_instructions']) ? $this->options['system_prompt_instructions'] : '';
6959	4283
6960		- // Get system prompt instructions using centralized function
6961		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
6962		-
6963	4284	// Ensure conversation_history is an array
6964	4285	if (!is_array($conversation_history)) {
6965	4286	$conversation_history = array();
6966	4287	}
		@@ -6991,8 +4312,9 @@
6991	4312
6992	4313	// Check if we can actually stream
6993	4314	if (headers_sent() || !function_exists('curl_init')) {
6994	4315	// Fallback to regular response with testing data
	4316	+ //error_log("MxChat: OpenAI streaming not possible, falling back to regular response");
6995	4317	$regular_response = $this->mxchat_generate_response_openai(
6996	4318	$selected_model,
6997	4319	$api_key,
6998	4320	$conversation_history,
		@@ -6998,13 +4320,8 @@
6998	4320	$conversation_history,
6999	4321	$relevant_content
7000	4322	);
7001	4323
7002		- // Save bot response to transcript
7003		- if (!empty($regular_response) && !empty($session_id)) {
7004		- $this->mxchat_save_chat_message($session_id, 'bot', $regular_response);
7005		- }
7006		-
7007	4324	$response_data = [
7008	4325	'text' => $regular_response,
7009	4326	'html' => '',
7010	4327	'session_id' => $session_id
		@@ -7011,8 +4328,9 @@
7011	4328	];
7012	4329
7013	4330	if ($testing_data !== null) {
7014	4331	$response_data['testing_data'] = $testing_data;
	4332	+ //error_log("MxChat Testing: Added testing data to OpenAI fallback response");
7015	4333	}
7016	4334
7017	4335	header('Content-Type: application/json');
7018	4336	echo json_encode($response_data);
		@@ -7018,45 +4336,16 @@
7018	4336	echo json_encode($response_data);
7019	4337	return true;
7020	4338	}
7021	4339
7022		- // Check if this is a GPT-5 model (supports reasoning_effort parameter)
7023		- $is_gpt5_model = (
7024		- strpos($selected_model, 'gpt-5') === 0 ||
7025		- $selected_model === 'gpt-5.2' ||
7026		- $selected_model === 'gpt-5.1-2025-11-13' ||
7027		- $selected_model === 'gpt-5' ||
7028		- $selected_model === 'gpt-5-mini' ||
7029		- $selected_model === 'gpt-5-nano'
7030		- );
7031		-
7032		- // Build request body with optimal settings for fast streaming
7033		- $request_body = [
	4340	+ // Prepare the request body with stream: true
	4341	+ $body = json_encode([
7034	4342	'model' => $selected_model,
7035	4343	'messages' => $formatted_conversation,
7036	4344	'temperature' => 1,
7037	4345	'stream' => true
7038		- ];
	4346	+ ]);
7039	4347
7040		- // Add reasoning_effort only for GPT-5 models that support it
7041		- // These chat models don't support reasoning_effort parameter
7042		- $no_reasoning_models = array('gpt-5.2', 'gpt-5.1-chat-latest', 'gpt-5.3-chat-latest', 'gpt-5.4-mini', 'gpt-5.4-nano');
7043		- if ($is_gpt5_model && !in_array($selected_model, $no_reasoning_models, true)) {
7044		- // GPT-5.1 uses 'low' instead of 'minimal'
7045		- if ($selected_model === 'gpt-5.1-2025-11-13') {
7046		- $request_body['reasoning_effort'] = 'low';
7047		- } elseif ($selected_model === 'gpt-5.4') {
7048		- $request_body['reasoning_effort'] = 'none';
7049		- } else {
7050		- $request_body['reasoning_effort'] = 'minimal';
7051		- }
7052		- }
7053		-
7054		- $body = json_encode($request_body);
7055		-
7056		- // Setup streaming headers now that we know we're actually streaming
7057		- $this->setup_streaming_headers();
7058		-
7059	4348	// Use cURL for streaming support
7060	4349	$ch = curl_init();
7061	4350	curl_setopt($ch, CURLOPT_URL, 'https://api.openai.com/v1/chat/completions');
7062	4351	curl_setopt($ch, CURLOPT_RETURNTRANSFER, false);
		@@ -7070,54 +4359,39 @@
7070	4359	curl_setopt($ch, CURLOPT_TIMEOUT, 60);
7071	4360
7072	4361	$full_response = ''; // Accumulate full response for saving
7073	4362	$stream_started = false;
7074		- $buffer = ''; // CRITICAL: Add persistent buffer for incomplete chunks
7075	4363
7076	4364	// Buffer control for real-time streaming
7077		- curl_setopt($ch, CURLOPT_WRITEFUNCTION, function($ch, $data) use (&$full_response, &$stream_started, &$buffer, $testing_data) {
	4365	+ curl_setopt($ch, CURLOPT_WRITEFUNCTION, function($ch, $data) use (&$full_response, &$stream_started, $testing_data) {
7078	4366	// Send testing data as the first event if available
7079	4367	if (!$stream_started && $testing_data !== null) {
7080	4368	echo "data: " . json_encode(['testing_data' => $testing_data]) . "\n\n";
7081	4369	flush();
7082	4370	$stream_started = true;
	4371	+ //error_log("MxChat Testing: Sent testing data in OpenAI stream");
7083	4372	}
7084	4373
7085		- // CRITICAL FIX: Append new data to buffer
7086		- $buffer .= $data;
	4374	+ // Process each chunk of data
	4375	+ $lines = explode("\n", $data);
7087	4376
7088		- // Process complete lines only
7089		- $lines = explode("\n", $buffer);
7090		-
7091		- // CRITICAL FIX: Keep the last incomplete line in the buffer
7092		- // The last element might be incomplete, so keep it in buffer
7093		- $buffer = array_pop($lines);
7094		-
7095	4377	foreach ($lines as $line) {
7096		- // Skip empty lines
7097		- if (trim($line) === '') {
	4378	+ if (trim($line) === '' || strpos($line, 'data: ') !== 0) {
7098	4379	continue;
7099	4380	}
7100	4381
7101		- // Only process lines that start with "data: "
7102		- if (strpos($line, 'data: ') !== 0) {
7103		- continue;
7104		- }
7105		-
7106	4382	$json_str = substr($line, 6); // Remove 'data: ' prefix
7107	4383
7108		- if (trim($json_str) === '[DONE]') {
	4384	+ if ($json_str === '[DONE]') {
7109	4385	echo "data: [DONE]\n\n";
7110	4386	flush();
7111	4387	continue;
7112	4388	}
7113	4389
7114		- // Try to decode JSON
7115		- $json = json_decode(trim($json_str), true);
7116		- if ($json && isset($json['choices'][0]['delta']['content'])) {
	4390	+ $json = json_decode($json_str, true);
	4391	+ if (isset($json['choices'][0]['delta']['content'])) {
7117	4392	$content = $json['choices'][0]['delta']['content'];
7118		- $full_response .= $content; // Accumulate the full response
7119		-
	4393	+ $full_response .= $content; // Accumulate
7120	4394	// Send as SSE format
7121	4395	echo "data: " . json_encode(['content' => $content]) . "\n\n";
7122	4396	flush();
7123	4397	}
		@@ -7129,12 +4403,12 @@
7129	4403	$response = curl_exec($ch);
7130	4404	$http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
7131	4405
7132	4406	if (curl_errno($ch) || $http_code !== 200) {
7133		- $curl_error = curl_error($ch);
7134	4407	curl_close($ch);
7135		-
	4408	+
7136	4409	// Fallback to regular response
	4410	+ //error_log("MxChat: OpenAI streaming failed, falling back");
7137	4411	$regular_response = $this->mxchat_generate_response_openai(
7138	4412	$selected_model,
7139	4413	$api_key,
7140	4414	$conversation_history,
		@@ -7139,34 +4413,20 @@
7139	4413	$api_key,
7140	4414	$conversation_history,
7141	4415	$relevant_content
7142	4416	);
7143		-
7144		- // FIXED: Check if regular response returned an error
7145		- if (is_array($regular_response) && isset($regular_response['error'])) {
7146		- // Send error in SSE format since we're in streaming mode
7147		- echo "data: " . json_encode([
7148		- 'error' => true,
7149		- 'error_message' => $regular_response['error'],
7150		- 'error_code' => $regular_response['error_code'] ?? 'api_error',
7151		- 'text' => $regular_response['error'],
7152		- 'message' => $regular_response['error']
7153		- ]) . "\n\n";
7154		- echo "data: [DONE]\n\n";
7155		- flush();
7156		- return true;
7157		- }
7158		-
	4417	+
7159	4418	$response_data = [
7160	4419	'text' => $regular_response,
7161	4420	'html' => '',
7162	4421	'session_id' => $session_id
7163	4422	];
7164		-
	4423	+
7165	4424	if ($testing_data !== null) {
7166	4425	$response_data['testing_data'] = $testing_data;
	4426	+ //error_log("MxChat Testing: Added testing data to OpenAI error fallback");
7167	4427	}
7168		-
	4428	+
7169	4429	header('Content-Type: application/json');
7170	4430	echo json_encode($response_data);
7171	4431	return true;
7172	4432	}
		@@ -7171,37 +4431,19 @@
7171	4431	return true;
7172	4432	}
7173	4433
7174	4434	curl_close($ch);
7175		-
	4435	+
7176	4436	// Save the complete response to maintain chat persistence
7177	4437	if (!empty($full_response) && !empty($session_id)) {
7178		- // Prepare RAG context for streaming response
7179		- $rag_context_for_storage = null;
7180		- $has_rag_data = $this->last_similarity_analysis !== null && !empty($this->last_similarity_analysis['top_matches']);
7181		- $has_action_data = isset($this->last_action_analysis) && !empty($this->last_action_analysis);
7182		-
7183		- if ($has_rag_data || $has_action_data) {
7184		- $rag_context_for_storage = [];
7185		-
7186		- if ($has_rag_data) {
7187		- $rag_context_for_storage['top_matches'] = $this->last_similarity_analysis['top_matches'];
7188		- $rag_context_for_storage['approved_urls'] = $this->current_valid_urls ?? [];
7189		- $rag_context_for_storage['similarity_threshold'] = $this->last_similarity_analysis['threshold_used'] ?? 0.35;
7190		- $rag_context_for_storage['knowledge_base_type'] = $this->last_similarity_analysis['knowledge_base_type'] ?? 'WordPress Database';
7191		- $rag_context_for_storage['total_documents_checked'] = $this->last_similarity_analysis['total_checked'] ?? 0;
7192		- }
7193		-
7194		- if ($has_action_data) {
7195		- $rag_context_for_storage['action_analysis'] = $this->last_action_analysis;
7196		- }
7197		- }
7198		- $this->mxchat_save_chat_message($session_id, 'bot', $full_response, null, $rag_context_for_storage);
	4438	+ $this->mxchat_save_chat_message($session_id, 'bot', $full_response);
7199	4439	}
7200		-
	4440	+
7201	4441	return true; // Indicate streaming completed successfully
7202		-
	4442	+
7203	4443	} catch (Exception $e) {
	4444	+ //error_log("MxChat OpenAI streaming exception: " . $e->getMessage());
	4445	+
7204	4446	// Fallback to regular response
7205	4447	$regular_response = $this->mxchat_generate_response_openai(
7206	4448	$selected_model,
7207	4449	$api_key,
		@@ -7207,401 +4449,30 @@
7207	4449	$api_key,
7208	4450	$conversation_history,
7209	4451	$relevant_content
7210	4452	);
7211		-
7212		- // FIXED: Check if regular response returned an error
7213		- if (is_array($regular_response) && isset($regular_response['error'])) {
7214		- // Send error in SSE format since we're in streaming mode
7215		- echo "data: " . json_encode([
7216		- 'error' => true,
7217		- 'error_message' => $regular_response['error'],
7218		- 'error_code' => $regular_response['error_code'] ?? 'api_error',
7219		- 'text' => $regular_response['error'],
7220		- 'message' => $regular_response['error']
7221		- ]) . "\n\n";
7222		- echo "data: [DONE]\n\n";
7223		- flush();
7224		- return true;
7225		- }
7226		-
	4453	+
7227	4454	$response_data = [
7228	4455	'text' => $regular_response,
7229	4456	'html' => '',
7230	4457	'session_id' => $session_id
7231	4458	];
7232		-
	4459	+
7233	4460	if ($testing_data !== null) {
7234	4461	$response_data['testing_data'] = $testing_data;
	4462	+ //error_log("MxChat Testing: Added testing data to OpenAI exception fallback");
7235	4463	}
7236		-
	4464	+
7237	4465	header('Content-Type: application/json');
7238	4466	echo json_encode($response_data);
7239	4467	return true;
7240	4468	}
7241	4469	}
7242		-
7243		-/**
7244		- * Generate response using OpenAI Responses API with web search tool
7245		- * This uses the newer Responses API which supports web search functionality
7246		- */
7247		-private function mxchat_generate_response_openai_web_search($selected_model, $api_key, $conversation_history, $relevant_content, $session_id, $testing_data = null, $streaming = false) {
	4470	+private function mxchat_generate_response_claude_stream($selected_model, $claude_api_key, $conversation_history, $relevant_content, $session_id, $testing_data = null) {
7248	4471	try {
7249		- $bot_id = $this->get_current_bot_id($session_id);
7250		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
	4472	+ // Get system prompt instructions from options
	4473	+ $system_prompt_instructions = isset($this->options['system_prompt_instructions']) ? $this->options['system_prompt_instructions'] : '';
7251	4474
7252		- if (!is_array($conversation_history)) {
7253		- $conversation_history = array();
7254		- }
7255		-
7256		- // Build the input for Responses API
7257		- // The Responses API uses a different format - we need to construct the input properly
7258		- $input_parts = [];
7259		-
7260		- // Add system instructions as context
7261		- $system_context = $system_prompt_instructions . "\n\n" . $relevant_content;
7262		-
7263		- // Build conversation as input items for Responses API
7264		- foreach ($conversation_history as $message) {
7265		- if (is_array($message) && isset($message['role']) && isset($message['content'])) {
7266		- $role = $message['role'];
7267		- if ($role === 'bot' || $role === 'agent') {
7268		- $role = 'assistant';
7269		- }
7270		- if (!in_array($role, ['assistant', 'user'])) {
7271		- $role = 'user';
7272		- }
7273		- $input_parts[] = [
7274		- 'type' => 'message',
7275		- 'role' => $role,
7276		- 'content' => $message['content']
7277		- ];
7278		- }
7279		- }
7280		-
7281		- // Build request body for Responses API
7282		- $request_body = [
7283		- 'model' => $selected_model,
7284		- 'input' => $input_parts,
7285		- 'instructions' => $system_context,
7286		- 'stream' => $streaming
7287		- ];
7288		-
7289		- // Only add web search tool if web search is enabled in settings
7290		- $web_search_enabled = isset($this->options['enable_web_search']) && $this->options['enable_web_search'] === 'on';
7291		- if ($web_search_enabled) {
7292		- $request_body['tools'] = [
7293		- ['type' => 'web_search']
7294		- ];
7295		- }
7296		-
7297		- // Add reasoning effort for supported models
7298		- $is_gpt5_model = strpos($selected_model, 'gpt-5') === 0;
7299		- $no_reasoning_web = array('gpt-5.2', 'gpt-5.3-chat-latest', 'gpt-5.4-mini', 'gpt-5.4-nano');
7300		- if ($is_gpt5_model && !in_array($selected_model, $no_reasoning_web, true)) {
7301		- if ($selected_model === 'gpt-5.1-2025-11-13') {
7302		- $request_body['reasoning'] = ['effort' => 'low'];
7303		- } elseif ($selected_model === 'gpt-5.4') {
7304		- $request_body['reasoning'] = ['effort' => 'low'];
7305		- }
7306		- }
7307		-
7308		- //error_log("MXCHAT WEB SEARCH: Request body: " . json_encode($request_body));
7309		-
7310		- if ($streaming) {
7311		- return $this->mxchat_web_search_streaming_response($request_body, $api_key, $session_id, $testing_data);
7312		- } else {
7313		- return $this->mxchat_web_search_non_streaming_response($request_body, $api_key, $session_id, $testing_data);
7314		- }
7315		-
7316		- } catch (Exception $e) {
7317		- //error_log("MXCHAT WEB SEARCH ERROR: " . $e->getMessage());
7318		- return [
7319		- 'error' => sprintf(esc_html__('Web search error: %s', 'mxchat'), esc_html($e->getMessage())),
7320		- 'error_code' => 'web_search_exception'
7321		- ];
7322		- }
7323		-}
7324		-
7325		-/**
7326		- * Handle non-streaming web search response
7327		- */
7328		-private function mxchat_web_search_non_streaming_response($request_body, $api_key, $session_id, $testing_data) {
7329		- $request_body['stream'] = false;
7330		-
7331		- $response = wp_remote_post('https://api.openai.com/v1/responses', array(
7332		- 'headers' => array(
7333		- 'Authorization' => 'Bearer ' . $api_key,
7334		- 'Content-Type' => 'application/json'
7335		- ),
7336		- 'body' => json_encode($request_body),
7337		- 'timeout' => 90
7338		- ));
7339		-
7340		- if (is_wp_error($response)) {
7341		- //error_log("MXCHAT WEB SEARCH ERROR: WP Error: " . $response->get_error_message());
7342		- return [
7343		- 'error' => esc_html__('Failed to connect to OpenAI web search API', 'mxchat'),
7344		- 'error_code' => 'web_search_connection_error'
7345		- ];
7346		- }
7347		-
7348		- $response_code = wp_remote_retrieve_response_code($response);
7349		- $response_body = wp_remote_retrieve_body($response);
7350		-
7351		- //error_log("MXCHAT WEB SEARCH: Response code: " . $response_code);
7352		- //error_log("MXCHAT WEB SEARCH: Response body (first 2000): " . substr($response_body, 0, 2000));
7353		-
7354		- if ($response_code !== 200) {
7355		- $error_data = json_decode($response_body, true);
7356		- $error_message = $error_data['error']['message'] ?? 'Unknown API error';
7357		- return [
7358		- 'error' => sprintf(esc_html__('OpenAI API error: %s', 'mxchat'), esc_html($error_message)),
7359		- 'error_code' => 'web_search_api_error'
7360		- ];
7361		- }
7362		-
7363		- $result = json_decode($response_body, true);
7364		-
7365		- if (json_last_error() !== JSON_ERROR_NONE) {
7366		- return [
7367		- 'error' => esc_html__('Invalid response from OpenAI', 'mxchat'),
7368		- 'error_code' => 'web_search_json_error'
7369		- ];
7370		- }
7371		-
7372		- // Extract the response text and citations from Responses API format
7373		- $output_text = '';
7374		- $citations = [];
7375		-
7376		- if (isset($result['output'])) {
7377		- foreach ($result['output'] as $output_item) {
7378		- if ($output_item['type'] === 'message' && isset($output_item['content'])) {
7379		- foreach ($output_item['content'] as $content_item) {
7380		- if ($content_item['type'] === 'output_text') {
7381		- $output_text .= $content_item['text'];
7382		-
7383		- // Extract citations/annotations
7384		- if (isset($content_item['annotations'])) {
7385		- foreach ($content_item['annotations'] as $annotation) {
7386		- if ($annotation['type'] === 'url_citation') {
7387		- $citations[] = [
7388		- 'url' => $annotation['url'],
7389		- 'title' => $annotation['title'] ?? ''
7390		- ];
7391		- }
7392		- }
7393		- }
7394		- }
7395		- }
7396		- }
7397		- }
7398		- }
7399		-
7400		- // If we have citations, append them to the response
7401		- if (!empty($citations)) {
7402		- $output_text .= "\n\n**Sources:**\n";
7403		- $seen_urls = [];
7404		- foreach ($citations as $citation) {
7405		- if (!in_array($citation['url'], $seen_urls)) {
7406		- $seen_urls[] = $citation['url'];
7407		- $title = !empty($citation['title']) ? $citation['title'] : $citation['url'];
7408		- $output_text .= "- [" . $title . "](" . $citation['url'] . ")\n";
7409		- }
7410		- }
7411		- }
7412		-
7413		- // Transcript save is handled by the main handler (mxchat_handle_chat_request)
7414		- // which includes rag_context for the "sources" link in transcripts.
7415		-
7416		- return $output_text;
7417		-}
7418		-
7419		-/**
7420		- * Handle streaming web search response using Responses API
7421		- */
7422		-private function mxchat_web_search_streaming_response($request_body, $api_key, $session_id, $testing_data) {
7423		- $request_body['stream'] = true;
7424		-
7425		- // Check if we can stream
7426		- if (headers_sent() || !function_exists('curl_init')) {
7427		- // Fallback to non-streaming
7428		- return $this->mxchat_web_search_non_streaming_response($request_body, $api_key, $session_id, $testing_data);
7429		- }
7430		-
7431		- // Setup streaming headers
7432		- $this->setup_streaming_headers();
7433		-
7434		- $ch = curl_init();
7435		- curl_setopt($ch, CURLOPT_URL, 'https://api.openai.com/v1/responses');
7436		- curl_setopt($ch, CURLOPT_RETURNTRANSFER, false);
7437		- curl_setopt($ch, CURLOPT_POST, true);
7438		- curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($request_body));
7439		- curl_setopt($ch, CURLOPT_HTTPHEADER, array(
7440		- 'Content-Type: application/json',
7441		- 'Authorization: Bearer ' . $api_key
7442		- ));
7443		- curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, true);
7444		- curl_setopt($ch, CURLOPT_TIMEOUT, 120);
7445		-
7446		- $full_response = '';
7447		- $stream_started = false;
7448		- $buffer = '';
7449		- $citations = [];
7450		-
7451		- curl_setopt($ch, CURLOPT_WRITEFUNCTION, function($ch, $data) use (&$full_response, &$stream_started, &$buffer, &$citations, $testing_data) {
7452		- // Send testing data as first event if available
7453		- if (!$stream_started && $testing_data !== null) {
7454		- echo "data: " . json_encode(['testing_data' => $testing_data]) . "\n\n";
7455		- flush();
7456		- $stream_started = true;
7457		- }
7458		-
7459		- $buffer .= $data;
7460		- $lines = explode("\n", $buffer);
7461		- $buffer = array_pop($lines);
7462		-
7463		- foreach ($lines as $line) {
7464		- if (trim($line) === '') continue;
7465		- if (strpos($line, 'data: ') !== 0) continue;
7466		-
7467		- $json_str = substr($line, 6);
7468		-
7469		- if (trim($json_str) === '[DONE]') {
7470		- // Append citations if we have any
7471		- if (!empty($citations)) {
7472		- $citation_text = "\n\n**Sources:**\n";
7473		- $seen_urls = [];
7474		- foreach ($citations as $citation) {
7475		- if (!in_array($citation['url'], $seen_urls)) {
7476		- $seen_urls[] = $citation['url'];
7477		- $title = !empty($citation['title']) ? $citation['title'] : $citation['url'];
7478		- $citation_text .= "- [" . $title . "](" . $citation['url'] . ")\n";
7479		- }
7480		- }
7481		- echo "data: " . json_encode(['content' => $citation_text]) . "\n\n";
7482		- $full_response .= $citation_text;
7483		- flush();
7484		- }
7485		- echo "data: [DONE]\n\n";
7486		- flush();
7487		- continue;
7488		- }
7489		-
7490		- $json = json_decode(trim($json_str), true);
7491		- if (!$json) continue;
7492		-
7493		- // Handle Responses API streaming events
7494		- // The format is different from Chat Completions
7495		- if (isset($json['type'])) {
7496		- switch ($json['type']) {
7497		- case 'response.output_text.delta':
7498		- // Text content delta
7499		- if (isset($json['delta'])) {
7500		- $content = $json['delta'];
7501		- $full_response .= $content;
7502		- echo "data: " . json_encode(['content' => $content]) . "\n\n";
7503		- flush();
7504		- }
7505		- break;
7506		-
7507		- case 'response.output_item.done':
7508		- // Check for citations in completed items
7509		- if (isset($json['item']['content'])) {
7510		- foreach ($json['item']['content'] as $content_item) {
7511		- if (isset($content_item['annotations'])) {
7512		- foreach ($content_item['annotations'] as $annotation) {
7513		- if ($annotation['type'] === 'url_citation') {
7514		- $citations[] = [
7515		- 'url' => $annotation['url'],
7516		- 'title' => $annotation['title'] ?? ''
7517		- ];
7518		- }
7519		- }
7520		- }
7521		- }
7522		- }
7523		- break;
7524		- }
7525		- }
7526		- }
7527		-
7528		- return strlen($data);
7529		- });
7530		-
7531		- $response = curl_exec($ch);
7532		- $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
7533		-
7534		- if (curl_errno($ch) || $http_code !== 200) {
7535		- $curl_error = curl_error($ch);
7536		- curl_close($ch);
7537		-
7538		- //error_log("MXCHAT WEB SEARCH STREAM ERROR: HTTP $http_code, cURL error: $curl_error");
7539		-
7540		- // Fallback to non-streaming
7541		- $fallback_response = $this->mxchat_web_search_non_streaming_response($request_body, $api_key, $session_id, $testing_data);
7542		-
7543		- if (is_array($fallback_response) && isset($fallback_response['error'])) {
7544		- echo "data: " . json_encode([
7545		- 'error' => true,
7546		- 'error_message' => $fallback_response['error'],
7547		- 'error_code' => $fallback_response['error_code'] ?? 'web_search_error'
7548		- ]) . "\n\n";
7549		- echo "data: [DONE]\n\n";
7550		- flush();
7551		- return true;
7552		- }
7553		-
7554		- $response_data = [
7555		- 'text' => $fallback_response,
7556		- 'html' => '',
7557		- 'session_id' => $session_id
7558		- ];
7559		- if ($testing_data !== null) {
7560		- $response_data['testing_data'] = $testing_data;
7561		- }
7562		- header('Content-Type: application/json');
7563		- echo json_encode($response_data);
7564		- return true;
7565		- }
7566		-
7567		- curl_close($ch);
7568		-
7569		- // Save the complete response with RAG context so the "sources" link
7570		- // appears in transcripts — mirrors the pattern used by Claude/OpenAI streaming.
7571		- if (!empty($full_response) && !empty($session_id)) {
7572		- $rag_context_for_storage = null;
7573		- $has_rag_data = $this->last_similarity_analysis !== null && !empty($this->last_similarity_analysis['top_matches']);
7574		- $has_action_data = isset($this->last_action_analysis) && !empty($this->last_action_analysis);
7575		-
7576		- if ($has_rag_data || $has_action_data) {
7577		- $rag_context_for_storage = [];
7578		-
7579		- if ($has_rag_data) {
7580		- $rag_context_for_storage['top_matches'] = $this->last_similarity_analysis['top_matches'];
7581		- $rag_context_for_storage['approved_urls'] = $this->current_valid_urls ?? [];
7582		- $rag_context_for_storage['similarity_threshold'] = $this->last_similarity_analysis['threshold_used'] ?? 0.35;
7583		- $rag_context_for_storage['knowledge_base_type'] = $this->last_similarity_analysis['knowledge_base_type'] ?? 'WordPress Database';
7584		- $rag_context_for_storage['total_documents_checked'] = $this->last_similarity_analysis['total_checked'] ?? 0;
7585		- }
7586		-
7587		- if ($has_action_data) {
7588		- $rag_context_for_storage['action_analysis'] = $this->last_action_analysis;
7589		- }
7590		- }
7591		- $this->mxchat_save_chat_message($session_id, 'bot', $full_response, null, $rag_context_for_storage);
7592		- }
7593		-
7594		- return true;
7595		-}
7596		-
7597		-private function mxchat_generate_response_claude_stream($selected_model, $claude_api_key, $conversation_history, $relevant_content, $session_id, $testing_data = null) {
7598		- try {
7599		- // Get bot ID from session or request
7600		- $bot_id = $this->get_current_bot_id($session_id);
7601		-
7602		- // Get system prompt instructions using centralized function
7603		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
7604	4475	// Ensure conversation_history is an array
7605	4476	if (!is_array($conversation_history)) {
7606	4477	$conversation_history = array();
7607	4478	}
		@@ -7653,13 +4524,8 @@
7653	4524	array_slice($conversation_history, 0, -1), // Remove the added content
7654	4525	$relevant_content
7655	4526	);
7656	4527
7657		- // Save bot response to transcript
7658		- if (!empty($regular_response) && !empty($session_id)) {
7659		- $this->mxchat_save_chat_message($session_id, 'bot', $regular_response);
7660		- }
7661		-
7662	4528	// Return as JSON with testing data
7663	4529	$response_data = [
7664	4530	'text' => $regular_response,
7665	4531	'html' => '',
		@@ -7678,11 +4544,8 @@
7678	4544	echo json_encode($response_data);
7679	4545	return true; // Indicate we handled the response
7680	4546	}
7681	4547
7682		- // Setup streaming headers now that we know we're actually streaming
7683		- $this->setup_streaming_headers();
7684		-
7685	4548	// Use cURL for streaming support
7686	4549	$ch = curl_init();
7687	4550	curl_setopt($ch, CURLOPT_URL, 'https://api.anthropic.com/v1/messages');
7688	4551	curl_setopt($ch, CURLOPT_RETURNTRANSFER, false);
		@@ -7697,12 +4560,11 @@
7697	4560	curl_setopt($ch, CURLOPT_TIMEOUT, 60);
7698	4561
7699	4562	$full_response = ''; // Accumulate full response for saving
7700	4563	$stream_started = false;
7701		- $buffer = ''; // CRITICAL: Add persistent buffer for incomplete chunks
7702	4564
7703	4565	// Buffer control for real-time streaming
7704		- curl_setopt($ch, CURLOPT_WRITEFUNCTION, function($ch, $data) use (&$full_response, &$stream_started, &$buffer, $testing_data) {
	4566	+ curl_setopt($ch, CURLOPT_WRITEFUNCTION, function($ch, $data) use (&$full_response, &$stream_started, $testing_data) {
7705	4567	// Send testing data as the first event if available
7706	4568	if (!$stream_started && $testing_data !== null) {
7707	4569	echo "data: " . json_encode(['testing_data' => $testing_data]) . "\n\n";
7708	4570	flush();
		@@ -7709,17 +4571,10 @@
7709	4571	$stream_started = true;
7710	4572	//error_log("MxChat Testing: Sent testing data in Claude stream");
7711	4573	}
7712	4574
7713		- // CRITICAL FIX: Append new data to buffer
7714		- $buffer .= $data;
7715		-
7716		- // Process complete lines only
7717		- $lines = explode("\n", $buffer);
7718		-
7719		- // CRITICAL FIX: Keep the last incomplete line in the buffer
7720		- // The last element might be incomplete, so keep it in buffer
7721		- $buffer = array_pop($lines);
	4575	+ // Process each chunk of data
	4576	+ $lines = explode("\n", $data);
7722	4577
7723	4578	foreach ($lines as $line) {
7724	4579	if (trim($line) === '') {
7725	4580	continue;
		@@ -7733,9 +4588,9 @@
7733	4588
7734	4589	if (strpos($line, 'data: ') === 0) {
7735	4590	$json_str = substr($line, 6); // Remove 'data: ' prefix
7736	4591
7737		- $json = json_decode(trim($json_str), true);
	4592	+ $json = json_decode($json_str, true);
7738	4593	if (json_last_error() !== JSON_ERROR_NONE) {
7739	4594	continue;
7740	4595	}
7741	4596
		@@ -7787,35 +4642,20 @@
7787	4642	$claude_api_key,
7788	4643	array_slice($conversation_history, 0, -1), // Remove the added content
7789	4644	$relevant_content
7790	4645	);
7791		-
7792		- // FIXED: Check if regular response returned an error
7793		- if (is_array($regular_response) && isset($regular_response['error'])) {
7794		- // Send error in SSE format since we're in streaming mode
7795		- echo "data: " . json_encode([
7796		- 'error' => true,
7797		- 'error_message' => $regular_response['error'],
7798		- 'error_code' => $regular_response['error_code'] ?? 'api_error',
7799		- 'text' => $regular_response['error'],
7800		- 'message' => $regular_response['error']
7801		- ]) . "\n\n";
7802		- echo "data: [DONE]\n\n";
7803		- flush();
7804		- return true;
7805		- }
7806		-
	4646	+
7807	4647	$response_data = [
7808	4648	'text' => $regular_response,
7809	4649	'html' => '',
7810	4650	'session_id' => $session_id
7811	4651	];
7812		-
	4652	+
7813	4653	if ($testing_data !== null) {
7814	4654	$response_data['testing_data'] = $testing_data;
7815	4655	//error_log("MxChat Testing: Added testing data to Claude error fallback");
7816	4656	}
7817		-
	4657	+
7818	4658	header('Content-Type: application/json');
7819	4659	echo json_encode($response_data);
7820	4660	return true;
7821	4661	}
		@@ -7821,29 +4661,9 @@
7821	4661	}
7822	4662
7823	4663	// Save the complete response to maintain chat persistence
7824	4664	if (!empty($full_response) && !empty($session_id)) {
7825		- // Prepare RAG context for streaming response
7826		- $rag_context_for_storage = null;
7827		- $has_rag_data = $this->last_similarity_analysis !== null && !empty($this->last_similarity_analysis['top_matches']);
7828		- $has_action_data = isset($this->last_action_analysis) && !empty($this->last_action_analysis);
7829		-
7830		- if ($has_rag_data || $has_action_data) {
7831		- $rag_context_for_storage = [];
7832		-
7833		- if ($has_rag_data) {
7834		- $rag_context_for_storage['top_matches'] = $this->last_similarity_analysis['top_matches'];
7835		- $rag_context_for_storage['approved_urls'] = $this->current_valid_urls ?? [];
7836		- $rag_context_for_storage['similarity_threshold'] = $this->last_similarity_analysis['threshold_used'] ?? 0.35;
7837		- $rag_context_for_storage['knowledge_base_type'] = $this->last_similarity_analysis['knowledge_base_type'] ?? 'WordPress Database';
7838		- $rag_context_for_storage['total_documents_checked'] = $this->last_similarity_analysis['total_checked'] ?? 0;
7839		- }
7840		-
7841		- if ($has_action_data) {
7842		- $rag_context_for_storage['action_analysis'] = $this->last_action_analysis;
7843		- }
7844		- }
7845		- $this->mxchat_save_chat_message($session_id, 'bot', $full_response, null, $rag_context_for_storage);
	4665	+ $this->mxchat_save_chat_message($session_id, 'bot', $full_response);
7846	4666	}
7847	4667
7848	4668	return true; // Indicate streaming completed successfully
7849	4669
		@@ -7848,9 +4668,9 @@
7848	4668	return true; // Indicate streaming completed successfully
7849	4669
7850	4670	} catch (Exception $e) {
7851	4671	//error_log("MxChat Claude streaming exception: " . $e->getMessage());
7852		-
	4672	+
7853	4673	// Fallback to regular response on exception
7854	4674	$regular_response = $this->mxchat_generate_response_claude(
7855	4675	$selected_model,
7856	4676	$claude_api_key,
		@@ -7856,35 +4676,20 @@
7856	4676	$claude_api_key,
7857	4677	$conversation_history,
7858	4678	$relevant_content
7859	4679	);
7860		-
7861		- // FIXED: Check if regular response returned an error
7862		- if (is_array($regular_response) && isset($regular_response['error'])) {
7863		- // Send error in SSE format since we're in streaming mode
7864		- echo "data: " . json_encode([
7865		- 'error' => true,
7866		- 'error_message' => $regular_response['error'],
7867		- 'error_code' => $regular_response['error_code'] ?? 'api_error',
7868		- 'text' => $regular_response['error'],
7869		- 'message' => $regular_response['error']
7870		- ]) . "\n\n";
7871		- echo "data: [DONE]\n\n";
7872		- flush();
7873		- return true;
7874		- }
7875		-
	4680	+
7876	4681	$response_data = [
7877	4682	'text' => $regular_response,
7878	4683	'html' => '',
7879	4684	'session_id' => $session_id
7880	4685	];
7881		-
	4686	+
7882	4687	if ($testing_data !== null) {
7883	4688	$response_data['testing_data'] = $testing_data;
7884	4689	//error_log("MxChat Testing: Added testing data to Claude exception fallback");
7885	4690	}
7886		-
	4691	+
7887	4692	header('Content-Type: application/json');
7888	4693	echo json_encode($response_data);
7889	4694	return true;
7890	4695	}
		@@ -7890,14 +4695,11 @@
7890	4695	}
7891	4696	}
7892	4697	private function mxchat_generate_response_xai_stream($selected_model, $xai_api_key, $conversation_history, $relevant_content, $session_id, $testing_data = null) {
7893	4698	try {
7894		- // Get bot ID from session or request
7895		- $bot_id = $this->get_current_bot_id($session_id);
	4699	+ // Get system prompt instructions from options
	4700	+ $system_prompt_instructions = isset($this->options['system_prompt_instructions']) ? $this->options['system_prompt_instructions'] : '';
7896	4701
7897		- // Get system prompt instructions using centralized function
7898		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
7899		-
7900	4702	// Ensure conversation_history is an array
7901	4703	if (!is_array($conversation_history)) {
7902	4704	$conversation_history = array();
7903	4705	}
		@@ -7936,13 +4738,8 @@
7936	4738	$conversation_history,
7937	4739	$relevant_content
7938	4740	);
7939	4741
7940		- // Save bot response to transcript
7941		- if (!empty($regular_response) && !empty($session_id)) {
7942		- $this->mxchat_save_chat_message($session_id, 'bot', $regular_response);
7943		- }
7944		-
7945	4742	$response_data = [
7946	4743	'text' => $regular_response,
7947	4744	'html' => '',
7948	4745	'session_id' => $session_id
		@@ -7965,11 +4762,8 @@
7965	4762	'temperature' => 0.8,
7966	4763	'stream' => true
7967	4764	]);
7968	4765
7969		- // Setup streaming headers now that we know we're actually streaming
7970		- $this->setup_streaming_headers();
7971		-
7972	4766	// Use cURL for streaming support
7973	4767	$ch = curl_init();
7974	4768	curl_setopt($ch, CURLOPT_URL, 'https://api.x.ai/v1/chat/completions');
7975	4769	curl_setopt($ch, CURLOPT_RETURNTRANSFER, false);
		@@ -7983,12 +4777,11 @@
7983	4777	curl_setopt($ch, CURLOPT_TIMEOUT, 60);
7984	4778
7985	4779	$full_response = ''; // Accumulate full response for saving
7986	4780	$stream_started = false;
7987		- $buffer = ''; // CRITICAL: Add persistent buffer for incomplete chunks
7988	4781
7989	4782	// Buffer control for real-time streaming
7990		- curl_setopt($ch, CURLOPT_WRITEFUNCTION, function($ch, $data) use (&$full_response, &$stream_started, &$buffer, $testing_data) {
	4783	+ curl_setopt($ch, CURLOPT_WRITEFUNCTION, function($ch, $data) use (&$full_response, &$stream_started, $testing_data) {
7991	4784	// Send testing data as the first event if available
7992	4785	if (!$stream_started && $testing_data !== null) {
7993	4786	echo "data: " . json_encode(['testing_data' => $testing_data]) . "\n\n";
7994	4787	flush();
		@@ -7995,40 +4788,26 @@
7995	4788	$stream_started = true;
7996	4789	//error_log("MxChat Testing: Sent testing data in X.AI stream");
7997	4790	}
7998	4791
7999		- // CRITICAL FIX: Append new data to buffer
8000		- $buffer .= $data;
	4792	+ // Process each chunk of data
	4793	+ $lines = explode("\n", $data);
8001	4794
8002		- // Process complete lines only
8003		- $lines = explode("\n", $buffer);
8004		-
8005		- // CRITICAL FIX: Keep the last incomplete line in the buffer
8006		- // The last element might be incomplete, so keep it in buffer
8007		- $buffer = array_pop($lines);
8008		-
8009	4795	foreach ($lines as $line) {
8010		- // Skip empty lines
8011		- if (trim($line) === '') {
	4796	+ if (trim($line) === '' || strpos($line, 'data: ') !== 0) {
8012	4797	continue;
8013	4798	}
8014	4799
8015		- // Only process lines that start with "data: "
8016		- if (strpos($line, 'data: ') !== 0) {
8017		- continue;
8018		- }
8019		-
8020	4800	$json_str = substr($line, 6); // Remove 'data: ' prefix
8021	4801
8022		- if (trim($json_str) === '[DONE]') {
	4802	+ if ($json_str === '[DONE]') {
8023	4803	echo "data: [DONE]\n\n";
8024	4804	flush();
8025	4805	continue;
8026	4806	}
8027	4807
8028		- // Try to decode JSON
8029		- $json = json_decode(trim($json_str), true);
8030		- if ($json && isset($json['choices'][0]['delta']['content'])) {
	4808	+ $json = json_decode($json_str, true);
	4809	+ if (isset($json['choices'][0]['delta']['content'])) {
8031	4810	$content = $json['choices'][0]['delta']['content'];
8032	4811	$full_response .= $content; // Accumulate
8033	4812	// Send as SSE format
8034	4813	echo "data: " . json_encode(['content' => $content]) . "\n\n";
		@@ -8070,39 +4849,19 @@
8070	4849	return true;
8071	4850	}
8072	4851
8073	4852	curl_close($ch);
8074		-
	4853	+
8075	4854	// Save the complete response to maintain chat persistence
8076	4855	if (!empty($full_response) && !empty($session_id)) {
8077		- // Prepare RAG context for streaming response
8078		- $rag_context_for_storage = null;
8079		- $has_rag_data = $this->last_similarity_analysis !== null && !empty($this->last_similarity_analysis['top_matches']);
8080		- $has_action_data = isset($this->last_action_analysis) && !empty($this->last_action_analysis);
8081		-
8082		- if ($has_rag_data || $has_action_data) {
8083		- $rag_context_for_storage = [];
8084		-
8085		- if ($has_rag_data) {
8086		- $rag_context_for_storage['top_matches'] = $this->last_similarity_analysis['top_matches'];
8087		- $rag_context_for_storage['approved_urls'] = $this->current_valid_urls ?? [];
8088		- $rag_context_for_storage['similarity_threshold'] = $this->last_similarity_analysis['threshold_used'] ?? 0.35;
8089		- $rag_context_for_storage['knowledge_base_type'] = $this->last_similarity_analysis['knowledge_base_type'] ?? 'WordPress Database';
8090		- $rag_context_for_storage['total_documents_checked'] = $this->last_similarity_analysis['total_checked'] ?? 0;
8091		- }
8092		-
8093		- if ($has_action_data) {
8094		- $rag_context_for_storage['action_analysis'] = $this->last_action_analysis;
8095		- }
8096		- }
8097		- $this->mxchat_save_chat_message($session_id, 'bot', $full_response, null, $rag_context_for_storage);
	4856	+ $this->mxchat_save_chat_message($session_id, 'bot', $full_response);
8098	4857	}
8099		-
	4858	+
8100	4859	return true; // Indicate streaming completed successfully
8101		-
	4860	+
8102	4861	} catch (Exception $e) {
8103	4862	//error_log("MxChat X.AI streaming exception: " . $e->getMessage());
8104		-
	4863	+
8105	4864	// Fallback to regular response
8106	4865	$regular_response = $this->mxchat_generate_response_xai(
8107	4866	$selected_model,
8108	4867	$xai_api_key,
		@@ -8127,14 +4886,11 @@
8127	4886	}
8128	4887	}
8129	4888	private function mxchat_generate_response_deepseek_stream($selected_model, $deepseek_api_key, $conversation_history, $relevant_content, $session_id, $testing_data = null) {
8130	4889	try {
8131		- // Get bot ID from session or request
8132		- $bot_id = $this->get_current_bot_id($session_id);
	4890	+ // Get system prompt instructions from options
	4891	+ $system_prompt_instructions = isset($this->options['system_prompt_instructions']) ? $this->options['system_prompt_instructions'] : '';
8133	4892
8134		- // Get system prompt instructions using centralized function
8135		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
8136		-
8137	4893	// Ensure conversation_history is an array
8138	4894	if (!is_array($conversation_history)) {
8139	4895	$conversation_history = array();
8140	4896	}
		@@ -8173,13 +4929,8 @@
8173	4929	$conversation_history,
8174	4930	$relevant_content
8175	4931	);
8176	4932
8177		- // Save bot response to transcript
8178		- if (!empty($regular_response) && !empty($session_id)) {
8179		- $this->mxchat_save_chat_message($session_id, 'bot', $regular_response);
8180		- }
8181		-
8182	4933	$response_data = [
8183	4934	'text' => $regular_response,
8184	4935	'html' => '',
8185	4936	'session_id' => $session_id
		@@ -8202,11 +4953,8 @@
8202	4953	'temperature' => 0.8,
8203	4954	'stream' => true
8204	4955	]);
8205	4956
8206		- // Setup streaming headers now that we know we're actually streaming
8207		- $this->setup_streaming_headers();
8208		-
8209	4957	// Use cURL for streaming support
8210	4958	$ch = curl_init();
8211	4959	curl_setopt($ch, CURLOPT_URL, 'https://api.deepseek.com/v1/chat/completions');
8212	4960	curl_setopt($ch, CURLOPT_RETURNTRANSFER, false);
		@@ -8220,12 +4968,11 @@
8220	4968	curl_setopt($ch, CURLOPT_TIMEOUT, 60);
8221	4969
8222	4970	$full_response = ''; // Accumulate full response for saving
8223	4971	$stream_started = false;
8224		- $buffer = ''; // CRITICAL: Add persistent buffer for incomplete chunks
8225	4972
8226	4973	// Buffer control for real-time streaming
8227		- curl_setopt($ch, CURLOPT_WRITEFUNCTION, function($ch, $data) use (&$full_response, &$stream_started, &$buffer, $testing_data) {
	4974	+ curl_setopt($ch, CURLOPT_WRITEFUNCTION, function($ch, $data) use (&$full_response, &$stream_started, $testing_data) {
8228	4975	// Send testing data as the first event if available
8229	4976	if (!$stream_started && $testing_data !== null) {
8230	4977	echo "data: " . json_encode(['testing_data' => $testing_data]) . "\n\n";
8231	4978	flush();
		@@ -8232,43 +4979,28 @@
8232	4979	$stream_started = true;
8233	4980	//error_log("MxChat Testing: Sent testing data in DeepSeek stream");
8234	4981	}
8235	4982
8236		- // CRITICAL FIX: Append new data to buffer
8237		- $buffer .= $data;
	4983	+ // Process each chunk of data
	4984	+ $lines = explode("\n", $data);
8238	4985
8239		- // Process complete lines only
8240		- $lines = explode("\n", $buffer);
8241		-
8242		- // CRITICAL FIX: Keep the last incomplete line in the buffer
8243		- // The last element might be incomplete, so keep it in buffer
8244		- $buffer = array_pop($lines);
8245		-
8246	4986	foreach ($lines as $line) {
8247		- // Skip empty lines
8248		- if (trim($line) === '') {
	4987	+ if (trim($line) === '' || strpos($line, 'data: ') !== 0) {
8249	4988	continue;
8250	4989	}
8251	4990
8252		- // Only process lines that start with "data: "
8253		- if (strpos($line, 'data: ') !== 0) {
8254		- continue;
8255		- }
8256		-
8257	4991	$json_str = substr($line, 6); // Remove 'data: ' prefix
8258	4992
8259		- if (trim($json_str) === '[DONE]') {
	4993	+ if ($json_str === '[DONE]') {
8260	4994	echo "data: [DONE]\n\n";
8261	4995	flush();
8262	4996	continue;
8263	4997	}
8264	4998
8265		- // Try to decode JSON
8266		- $json = json_decode(trim($json_str), true);
8267		- if ($json && isset($json['choices'][0]['delta']['content'])) {
	4999	+ $json = json_decode($json_str, true);
	5000	+ if (isset($json['choices'][0]['delta']['content'])) {
8268	5001	$content = $json['choices'][0]['delta']['content'];
8269		- $full_response .= $content; // Accumulate the full response
8270		-
	5002	+ $full_response .= $content; // Accumulate
8271	5003	// Send as SSE format
8272	5004	echo "data: " . json_encode(['content' => $content]) . "\n\n";
8273	5005	flush();
8274	5006	}
		@@ -8321,39 +5053,19 @@
8321	5053	return true;
8322	5054	}
8323	5055
8324	5056	curl_close($ch);
8325		-
	5057	+
8326	5058	// Save the complete response to maintain chat persistence
8327	5059	if (!empty($full_response) && !empty($session_id)) {
8328		- // Prepare RAG context for streaming response
8329		- $rag_context_for_storage = null;
8330		- $has_rag_data = $this->last_similarity_analysis !== null && !empty($this->last_similarity_analysis['top_matches']);
8331		- $has_action_data = isset($this->last_action_analysis) && !empty($this->last_action_analysis);
8332		-
8333		- if ($has_rag_data || $has_action_data) {
8334		- $rag_context_for_storage = [];
8335		-
8336		- if ($has_rag_data) {
8337		- $rag_context_for_storage['top_matches'] = $this->last_similarity_analysis['top_matches'];
8338		- $rag_context_for_storage['approved_urls'] = $this->current_valid_urls ?? [];
8339		- $rag_context_for_storage['similarity_threshold'] = $this->last_similarity_analysis['threshold_used'] ?? 0.35;
8340		- $rag_context_for_storage['knowledge_base_type'] = $this->last_similarity_analysis['knowledge_base_type'] ?? 'WordPress Database';
8341		- $rag_context_for_storage['total_documents_checked'] = $this->last_similarity_analysis['total_checked'] ?? 0;
8342		- }
8343		-
8344		- if ($has_action_data) {
8345		- $rag_context_for_storage['action_analysis'] = $this->last_action_analysis;
8346		- }
8347		- }
8348		- $this->mxchat_save_chat_message($session_id, 'bot', $full_response, null, $rag_context_for_storage);
	5060	+ $this->mxchat_save_chat_message($session_id, 'bot', $full_response);
8349	5061	}
8350		-
	5062	+
8351	5063	return true; // Indicate streaming completed successfully
8352		-
	5064	+
8353	5065	} catch (Exception $e) {
8354	5066	//error_log("MxChat DeepSeek streaming exception: " . $e->getMessage());
8355		-
	5067	+
8356	5068	// Fallback to regular response
8357	5069	$regular_response = $this->mxchat_generate_response_deepseek(
8358	5070	$selected_model,
8359	5071	$deepseek_api_key,
		@@ -8387,120 +5099,12 @@
8387	5099	return true;
8388	5100	}
8389	5101	}
8390	5102
	5103	+private function mxchat_generate_response_claude($selected_model, $claude_api_key, $conversation_history, $relevant_content) {
	5104	+ // Get system prompt instructions from options
	5105	+ $system_prompt_instructions = isset($this->options['system_prompt_instructions']) ? $this->options['system_prompt_instructions'] : '';
8391	5106
8392		-private function mxchat_generate_response_openrouter($selected_model, $openrouter_api_key, $conversation_history, $relevant_content) {
8393		- try {
8394		- if (!is_array($conversation_history)) {
8395		- $conversation_history = array();
8396		- }
8397		-
8398		- $bot_id = $this->get_current_bot_id('');
8399		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
8400		-
8401		- $formatted_conversation = array();
8402		-
8403		- $formatted_conversation[] = array(
8404		- 'role' => 'system',
8405		- 'content' => $system_prompt_instructions . " " . $relevant_content
8406		- );
8407		-
8408		- foreach ($conversation_history as $message) {
8409		- if (is_array($message) && isset($message['role']) && isset($message['content'])) {
8410		- $role = $message['role'];
8411		-
8412		- if ($role === 'bot' || $role === 'agent') {
8413		- $role = 'assistant';
8414		- }
8415		- if (!in_array($role, ['system', 'assistant', 'user'])) {
8416		- $role = 'user';
8417		- }
8418		-
8419		- $formatted_conversation[] = array(
8420		- 'role' => $role,
8421		- 'content' => $message['content']
8422		- );
8423		- }
8424		- }
8425		-
8426		- $body = json_encode([
8427		- 'model' => $selected_model,
8428		- 'messages' => $formatted_conversation,
8429		- 'temperature' => 1,
8430		- ]);
8431		-
8432		- $args = [
8433		- 'body' => $body,
8434		- 'headers' => [
8435		- 'Content-Type' => 'application/json',
8436		- 'Authorization' => 'Bearer ' . $openrouter_api_key,
8437		- 'HTTP-Referer' => home_url(),
8438		- 'X-Title' => get_bloginfo('name'),
8439		- ],
8440		- 'timeout' => 60,
8441		- 'redirection' => 5,
8442		- 'blocking' => true,
8443		- 'httpversion' => '1.0',
8444		- 'sslverify' => true,
8445		- ];
8446		-
8447		- $response = wp_remote_post('https://openrouter.ai/api/v1/chat/completions', $args);
8448		-
8449		- if (is_wp_error($response)) {
8450		- $error_message = $response->get_error_message();
8451		- return [
8452		- 'error' => esc_html__('Connection error when contacting OpenRouter: ', 'mxchat') . esc_html($error_message),
8453		- 'error_code' => 'openrouter_connection_error',
8454		- 'provider' => 'openrouter'
8455		- ];
8456		- }
8457		-
8458		- $status_code = wp_remote_retrieve_response_code($response);
8459		- if ($status_code !== 200) {
8460		- $response_body = wp_remote_retrieve_body($response);
8461		- $decoded_response = json_decode($response_body, true);
8462		-
8463		- $error_message = isset($decoded_response['error']['message'])
8464		- ? $decoded_response['error']['message']
8465		- : 'HTTP Error ' . $status_code;
8466		-
8467		- return [
8468		- 'error' => esc_html__('OpenRouter API error: ', 'mxchat') . esc_html($error_message),
8469		- 'error_code' => 'openrouter_api_error',
8470		- 'provider' => 'openrouter',
8471		- 'status_code' => $status_code
8472		- ];
8473		- }
8474		-
8475		- $response_body = wp_remote_retrieve_body($response);
8476		- $decoded_response = json_decode($response_body, true);
8477		-
8478		- if (isset($decoded_response['choices'][0]['message']['content'])) {
8479		- return trim($decoded_response['choices'][0]['message']['content']);
8480		- } else {
8481		- return [
8482		- 'error' => esc_html__('Unexpected response format from OpenRouter.', 'mxchat'),
8483		- 'error_code' => 'openrouter_response_format_error',
8484		- 'provider' => 'openrouter'
8485		- ];
8486		- }
8487		- } catch (Exception $e) {
8488		- return [
8489		- 'error' => esc_html__('System error when processing OpenRouter request: ', 'mxchat') . esc_html($e->getMessage()),
8490		- 'error_code' => 'openrouter_exception',
8491		- 'provider' => 'openrouter'
8492		- ];
8493		- }
8494		-}
8495		-private function mxchat_generate_response_claude($selected_model, $claude_api_key, $conversation_history, $relevant_content) {
8496		-
8497		- // Get bot ID from session or request
8498		- $bot_id = $this->get_current_bot_id($session_id);
8499		-
8500		- // Get system prompt instructions using centralized function
8501		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
8502		-
8503	5107	// Clean and validate conversation history
8504	5108	foreach ($conversation_history as &$message) {
8505	5109	// Convert bot and agent roles to assistant
8506	5110	if ($message['role'] === 'bot' || $message['role'] === 'agent') {
		@@ -8602,14 +5206,11 @@
8602	5206	if (!is_array($conversation_history)) {
8603	5207	$conversation_history = array();
8604	5208	}
8605	5209
8606		- // Get bot ID from session or request
8607		- $bot_id = $this->get_current_bot_id('');
8608		-
8609		- // Get system prompt instructions using centralized function
8610		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
8611		-
	5210	+ // Get system prompt instructions from options
	5211	+ $system_prompt_instructions = isset($this->options['system_prompt_instructions']) ? $this->options['system_prompt_instructions'] : '';
	5212	+
8612	5213	// Create a new array for the formatted conversation
8613	5214	$formatted_conversation = array();
8614	5215
8615	5216	// Add system message first
		@@ -8637,42 +5238,15 @@
8637	5238	);
8638	5239	}
8639	5240	}
8640	5241
8641		- // Check if this is a GPT-5 model (supports reasoning_effort parameter)
8642		- $is_gpt5_model = (
8643		- strpos($selected_model, 'gpt-5') === 0 ||
8644		- $selected_model === 'gpt-5.2' ||
8645		- $selected_model === 'gpt-5.1-2025-11-13' ||
8646		- $selected_model === 'gpt-5' ||
8647		- $selected_model === 'gpt-5-mini' ||
8648		- $selected_model === 'gpt-5-nano'
8649		- );
8650		-
8651		- // Build request body with optimal settings for fast responses
8652		- $request_body = [
	5242	+ $body = json_encode([
8653	5243	'model' => $selected_model,
8654	5244	'messages' => $formatted_conversation,
8655	5245	'temperature' => 1,
8656	5246	'stream' => false
8657		- ];
	5247	+ ]);
8658	5248
8659		- // Add reasoning_effort only for GPT-5 models that support it
8660		- // These chat models don't support reasoning_effort parameter
8661		- $no_reasoning_models = array('gpt-5.2', 'gpt-5.1-chat-latest', 'gpt-5.3-chat-latest', 'gpt-5.4-mini', 'gpt-5.4-nano');
8662		- if ($is_gpt5_model && !in_array($selected_model, $no_reasoning_models, true)) {
8663		- // GPT-5.1 uses 'low' instead of 'minimal'
8664		- if ($selected_model === 'gpt-5.1-2025-11-13') {
8665		- $request_body['reasoning_effort'] = 'low';
8666		- } elseif ($selected_model === 'gpt-5.4') {
8667		- $request_body['reasoning_effort'] = 'none';
8668		- } else {
8669		- $request_body['reasoning_effort'] = 'minimal';
8670		- }
8671		- }
8672		-
8673		- $body = json_encode($request_body);
8674		-
8675	5249	$args = [
8676	5250	'body' => $body,
8677	5251	'headers' => [
8678	5252	'Content-Type' => 'application/json',
		@@ -8688,8 +5262,9 @@
8688	5262	$response = wp_remote_post('https://api.openai.com/v1/chat/completions', $args);
8689	5263
8690	5264	if (is_wp_error($response)) {
8691	5265	$error_message = $response->get_error_message();
	5266	+ //error_log('OpenAI API Error: ' . $error_message);
8692	5267	return [
8693	5268	'error' => esc_html__('Connection error when contacting OpenAI: ', 'mxchat') . esc_html($error_message),
8694	5269	'error_code' => 'openai_connection_error',
8695	5270	'provider' => 'openai'
		@@ -8708,8 +5283,10 @@
8708	5283	$error_type = isset($decoded_response['error']['type'])
8709	5284	? $decoded_response['error']['type']
8710	5285	: 'unknown';
8711	5286
	5287	+ //error_log('OpenAI API HTTP Error: ' . $status_code . ' - ' . $error_message);
	5288	+
8712	5289	// Handle specific error types
8713	5290	switch ($error_type) {
8714	5291	case 'invalid_request_error':
8715	5292	if (strpos($error_message, 'API key') !== false) {
		@@ -8757,8 +5334,9 @@
8757	5334
8758	5335	if (isset($decoded_response['choices'][0]['message']['content'])) {
8759	5336	return trim($decoded_response['choices'][0]['message']['content']);
8760	5337	} else {
	5338	+ //error_log('OpenAI API Response Format Error: ' . print_r($decoded_response, true));
8761	5339	return [
8762	5340	'error' => esc_html__('Unexpected response format from OpenAI.', 'mxchat'),
8763	5341	'error_code' => 'openai_response_format_error',
8764	5342	'provider' => 'openai'
		@@ -8764,8 +5342,9 @@
8764	5342	'provider' => 'openai'
8765	5343	];
8766	5344	}
8767	5345	} catch (Exception $e) {
	5346	+ //error_log('OpenAI Exception: ' . $e->getMessage());
8768	5347	return [
8769	5348	'error' => esc_html__('System error when processing OpenAI request: ', 'mxchat') . esc_html($e->getMessage()),
8770	5349	'error_code' => 'openai_exception',
8771	5350	'provider' => 'openai'
		@@ -8771,17 +5350,13 @@
8771	5350	'provider' => 'openai'
8772	5351	];
8773	5352	}
8774	5353	}
8775		-
8776	5354	private function mxchat_generate_response_xai($selected_model, $xai_api_key, $conversation_history, $relevant_content) {
8777	5355	try {
8778		- // Get bot ID from session or request
8779		- $bot_id = $this->get_current_bot_id($session_id);
8780		-
8781		- // Get system prompt instructions using centralized function
8782		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
8783		-
	5356	+ // Get system prompt instructions from options
	5357	+ $system_prompt_instructions = isset($this->options['system_prompt_instructions']) ? $this->options['system_prompt_instructions'] : '';
	5358	+
8784	5359	// Add system prompt to relevant content
8785	5360	$content_with_instructions = $system_prompt_instructions . " " . $relevant_content;
8786	5361
8787	5362	// Prepend system instructions to the conversation history
		@@ -8973,14 +5548,11 @@
8973	5548	if (!is_array($conversation_history)) {
8974	5549	$conversation_history = array();
8975	5550	}
8976	5551
8977		- // Get bot ID from session or request
8978		- $bot_id = $this->get_current_bot_id($session_id);
8979		-
8980		- // Get system prompt instructions using centralized function
8981		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
8982		-
	5552	+ // Get system prompt instructions from options
	5553	+ $system_prompt_instructions = isset($this->options['system_prompt_instructions']) ? $this->options['system_prompt_instructions'] : '';
	5554	+
8983	5555	// Create a new array for the formatted conversation
8984	5556	$formatted_conversation = array();
8985	5557
8986	5558	// Add system message first
		@@ -9132,14 +5704,11 @@
9132	5704	];
9133	5705	}
9134	5706	}
9135	5707	private function mxchat_generate_response_gemini($selected_model, $gemini_api_key, $conversation_history, $relevant_content) {
9136		- // Get bot ID from session or request
9137		- $bot_id = $this->get_current_bot_id($session_id);
9138		-
9139		- // Get system prompt instructions using centralized function
9140		- $system_prompt_instructions = $this->get_system_instructions($bot_id, $session_id);
9141		-
	5708	+ // Get system prompt instructions from options
	5709	+ $system_prompt_instructions = isset($this->options['system_prompt_instructions']) ? $this->options['system_prompt_instructions'] : '';
	5710	+
9142	5711	// Add system prompt to relevant content
9143	5712	$content_with_instructions = $system_prompt_instructions . " " . $relevant_content;
9144	5713
9145	5714	// Format messages for Gemini API
		@@ -9234,11 +5803,9 @@
9234	5803	]
9235	5804	]);
9236	5805
9237	5806	// Prepare the API endpoint
9238		- // Use v1beta for preview models (Gemini 3, experimental), v1 for stable models
9239		- $api_version = (strpos($selected_model, 'preview') !== false || strpos($selected_model, 'exp') !== false) ? 'v1beta' : 'v1';
9240		- $api_endpoint = 'https://generativelanguage.googleapis.com/' . $api_version . '/models/' . $selected_model . ':generateContent?key=' . $gemini_api_key;
	5807	+ $api_endpoint = 'https://generativelanguage.googleapis.com/v1/models/' . $selected_model . ':generateContent?key=' . $gemini_api_key;
9241	5808
9242	5809	// Set up the API request
9243	5810	$args = [
9244	5811	'body' => $body,
		@@ -9277,12 +5844,11 @@
9277	5844	return "Sorry, I couldn't process that request. The response format was unexpected.";
9278	5845	}
9279	5846	}
9280	5847
9281		-
9282	5848	public function test_streaming_request() {
9283	5849	$options = get_option('mxchat_options', []);
9284		- $model = $options['model'] ?? 'gpt-5.1-chat-latest';
	5850	+ $model = $options['model'] ?? 'gpt-4o';
9285	5851
9286	5852	// Detect provider from model prefix
9287	5853	$provider = strtolower(explode('-', $model)[0]);
9288	5854
		@@ -9408,8 +5974,9 @@
9408	5974
9409	5975	return true;
9410	5976	}
9411	5977
	5978	+
9412	5979	public function mxchat_dismiss_pre_chat_message() {
9413	5980	// Get and sanitize the user identifier
9414	5981	$user_id = $this->mxchat_get_user_identifier();
9415	5982	$user_id = sanitize_key($user_id);
		@@ -9463,56 +6030,36 @@
9463	6030
9464	6031	return $dotProduct / ($normA * $normB);
9465	6032	}
9466	6033
9467		-
9468	6034	public function mxchat_enqueue_scripts_styles() {
9469		- // Fetch options from the database first to check loading strategy
9470		- $this->options = get_option('mxchat_options');
9471		- $loading_strategy = isset($this->options['script_loading_strategy']) ? $this->options['script_loading_strategy'] : 'default';
9472		-
9473		- // Always enqueue CSS immediately
	6035	+ // Define version numbers for the styles and scripts
	6036	+ $chat_style_version = '2.4.0';
	6037	+ $chat_script_version = '2.4.0';
	6038	+ // Enqueue the script
	6039	+ wp_enqueue_script(
	6040	+ 'mxchat-chat-js',
	6041	+ plugin_dir_url(__FILE__) . '../js/chat-script.js',
	6042	+ array('jquery'),
	6043	+ $chat_script_version,
	6044	+ true
	6045	+ );
	6046	+ // Enqueue the CSS
9474	6047	wp_enqueue_style(
9475	6048	'mxchat-chat-css',
9476	6049	plugin_dir_url(__FILE__) . '../css/chat-style.css',
9477	6050	array(),
9478		- MXCHAT_VERSION
	6051	+ $chat_style_version
9479	6052	);
9480		-
9481		- // Handle script loading based on strategy
9482		- if ($loading_strategy === 'default' || $loading_strategy === 'defer') {
9483		- // Enqueue the script normally
9484		- wp_enqueue_script(
9485		- 'mxchat-chat-js',
9486		- plugin_dir_url(__FILE__) . '../js/chat-script.js',
9487		- array('jquery'),
9488		- MXCHAT_VERSION,
9489		- true
9490		- );
9491		-
9492		- // Add defer attribute if strategy is 'defer'
9493		- if ($loading_strategy === 'defer') {
9494		- wp_script_add_data('mxchat-chat-js', 'strategy', 'defer');
9495		- }
9496		- } else {
9497		- // For delay or interaction-based loading, we'll use a custom loader
9498		- // Don't enqueue the main script - we'll load it dynamically
9499		- add_action('wp_footer', array($this, 'mxchat_output_delayed_script_loader'), 99);
9500		- }
9501		-
	6053	+ // Fetch options from the database
	6054	+ $this->options = get_option('mxchat_options');
9502	6055	$prompts_options = get_option('mxchat_prompts_options', array());
9503		-
9504		- // Check if AI theme is active - if so, skip inline colors in JavaScript
9505		- $theme_options = get_option('mxchat_theme_options', array());
9506		- $ai_theme_active = !empty($theme_options['active_ai_theme_css']);
9507		- $has_bot_theme_assignments = !empty($theme_options['bot_theme_assignments']);
9508		- $skip_inline_colors = $ai_theme_active || $has_bot_theme_assignments;
9509		-
	6056	+
9510	6057	// Prepare settings for JavaScript
9511	6058	$style_settings = array(
9512	6059	'ajax_url' => admin_url('admin-ajax.php'),
9513	6060	'nonce' => wp_create_nonce('mxchat_chat_nonce'),
9514		- 'model' => isset($this->options['model']) ? $this->options['model'] : 'gpt-5.1-chat-latest',
	6061	+ 'model' => isset($this->options['model']) ? $this->options['model'] : 'gpt-4o',
9515	6062	'enable_streaming_toggle' => isset($this->options['enable_streaming_toggle']) ? $this->options['enable_streaming_toggle'] : 'on',
9516	6063	'contextual_awareness_toggle' => isset($this->options['contextual_awareness_toggle']) ? $this->options['contextual_awareness_toggle'] : 'off',
9517	6064	'link_target_toggle' => $this->options['link_target_toggle'] ?? 'off',
9518	6065	'rate_limit_message' => $this->options['rate_limit_message'] ?? 'Rate limit exceeded. Please try again later.',
		@@ -9539,134 +6086,15 @@
9539	6086	'use_pinecone' => $prompts_options['mxchat_use_pinecone'] ?? '0',
9540	6087	'email_collection_enabled' => $this->options['enable_email_block'] ?? 'off', // FIXED
9541	6088	'initial_email_state' => null, // Also fixed this undefined variable
9542	6089	'skip_email_check' => true,
9543		- 'pinecone_enabled' => isset($prompts_options['mxchat_use_pinecone']) && $prompts_options['mxchat_use_pinecone'] === '1',
9544		- 'skip_inline_colors' => $skip_inline_colors,
9545		- 'bot_theme_assignments' => $theme_options['bot_theme_assignments'] ?? array()
	6090	+ 'pinecone_enabled' => isset($prompts_options['mxchat_use_pinecone']) && $prompts_options['mxchat_use_pinecone'] === '1'
9546	6091	);
9547		-
9548		- // For normal/defer loading, use wp_localize_script
9549		- // For delayed loading, we store settings in a transient to be output inline
9550		- if ($loading_strategy === 'default' || $loading_strategy === 'defer') {
9551		- wp_localize_script('mxchat-chat-js', 'mxchatChat', $style_settings);
9552		- } else {
9553		- // Store settings for the delayed loader to use
9554		- set_transient('mxchat_delayed_settings_' . get_current_user_id(), $style_settings, 60);
9555		- }
	6092	+ // Pass the settings to the script
	6093	+ wp_localize_script('mxchat-chat-js', 'mxchatChat', $style_settings);
9556	6094	}
9557	6095
9558		-/**
9559		- * Output the delayed script loader for performance optimization
9560		- */
9561		-public function mxchat_output_delayed_script_loader() {
9562		- $this->options = get_option('mxchat_options');
9563		- $loading_strategy = isset($this->options['script_loading_strategy']) ? $this->options['script_loading_strategy'] : 'default';
9564		- $script_url = plugin_dir_url(__FILE__) . '../js/chat-script.js?ver=' . MXCHAT_VERSION;
9565	6096
9566		- // Get the stored settings
9567		- $prompts_options = get_option('mxchat_prompts_options', array());
9568		- $theme_options = get_option('mxchat_theme_options', array());
9569		- $ai_theme_active = !empty($theme_options['active_ai_theme_css']);
9570		- $has_bot_theme_assignments = !empty($theme_options['bot_theme_assignments']);
9571		- $skip_inline_colors = $ai_theme_active || $has_bot_theme_assignments;
9572		-
9573		- $style_settings = array(
9574		- 'ajax_url' => admin_url('admin-ajax.php'),
9575		- 'nonce' => wp_create_nonce('mxchat_chat_nonce'),
9576		- 'model' => isset($this->options['model']) ? $this->options['model'] : 'gpt-5.1-chat-latest',
9577		- 'enable_streaming_toggle' => isset($this->options['enable_streaming_toggle']) ? $this->options['enable_streaming_toggle'] : 'on',
9578		- 'contextual_awareness_toggle' => isset($this->options['contextual_awareness_toggle']) ? $this->options['contextual_awareness_toggle'] : 'off',
9579		- 'link_target_toggle' => $this->options['link_target_toggle'] ?? 'off',
9580		- 'rate_limit_message' => $this->options['rate_limit_message'] ?? 'Rate limit exceeded. Please try again later.',
9581		- 'complianz_toggle' => isset($this->options['complianz_toggle']) && $this->options['complianz_toggle'] === 'on',
9582		- 'user_message_bg_color' => $this->options['user_message_bg_color'] ?? '#fff',
9583		- 'user_message_font_color' => $this->options['user_message_font_color'] ?? '#212121',
9584		- 'bot_message_bg_color' => $this->options['bot_message_bg_color'] ?? '#212121',
9585		- 'bot_message_font_color' => $this->options['bot_message_font_color'] ?? '#fff',
9586		- 'top_bar_bg_color' => $this->options['top_bar_bg_color'] ?? '#212121',
9587		- 'send_button_font_color' => $this->options['send_button_font_color'] ?? '#212121',
9588		- 'close_button_color' => $this->options['close_button_color'] ?? '#fff',
9589		- 'chatbot_background_color' => $this->options['chatbot_background_color'] ?? '#212121',
9590		- 'chatbot_bg_color' => $this->options['chatbot_bg_color'] ?? '#fff',
9591		- 'icon_color' => $this->options['icon_color'] ?? '#fff',
9592		- 'chat_input_font_color' => $this->options['chat_input_font_color'] ?? '#212121',
9593		- 'chat_persistence_toggle' => $this->options['chat_persistence_toggle'] ?? 'off',
9594		- 'appendWidgetToBody' => $this->options['append_to_body'] ?? 'off',
9595		- 'live_agent_message_bg_color' => $this->options['live_agent_message_bg_color'] ?? '#ffffff',
9596		- 'live_agent_message_font_color' => $this->options['live_agent_message_font_color'] ?? '#333333',
9597		- 'chat_toolbar_toggle' => $this->options['chat_toolbar_toggle'] ?? 'off',
9598		- 'mode_indicator_bg_color' => $this->options['mode_indicator_bg_color'] ?? '#767676',
9599		- 'mode_indicator_font_color' => $this->options['mode_indicator_font_color'] ?? '#ffffff',
9600		- 'toolbar_icon_color' => $this->options['toolbar_icon_color'] ?? '#212121',
9601		- 'use_pinecone' => $prompts_options['mxchat_use_pinecone'] ?? '0',
9602		- 'email_collection_enabled' => $this->options['enable_email_block'] ?? 'off',
9603		- 'initial_email_state' => null,
9604		- 'skip_email_check' => true,
9605		- 'pinecone_enabled' => isset($prompts_options['mxchat_use_pinecone']) && $prompts_options['mxchat_use_pinecone'] === '1',
9606		- 'skip_inline_colors' => $skip_inline_colors,
9607		- 'bot_theme_assignments' => $theme_options['bot_theme_assignments'] ?? array()
9608		- );
9609		-
9610		- // Determine delay time based on strategy
9611		- $delay_ms = 0;
9612		- switch ($loading_strategy) {
9613		- case 'delay_1s':
9614		- $delay_ms = 1000;
9615		- break;
9616		- case 'delay_3s':
9617		- $delay_ms = 3000;
9618		- break;
9619		- case 'delay_5s':
9620		- $delay_ms = 5000;
9621		- break;
9622		- }
9623		-
9624		- ?>
9625		- <script type="text/javascript">
9626		- (function() {
9627		- var mxchatLoaded = false;
9628		- var mxchatChat = <?php echo wp_json_encode($style_settings); ?>;
9629		- window.mxchatChat = mxchatChat;
9630		-
9631		- function loadMxChatScript() {
9632		- if (mxchatLoaded) return;
9633		- mxchatLoaded = true;
9634		-
9635		- function appendChatScript() {
9636		- var script = document.createElement('script');
9637		- script.src = <?php echo wp_json_encode($script_url); ?>;
9638		- script.type = 'text/javascript';
9639		- document.body.appendChild(script);
9640		- }
9641		-
9642		- if (typeof jQuery !== 'undefined') {
9643		- appendChatScript();
9644		- } else {
9645		- var jq = document.createElement('script');
9646		- jq.src = <?php echo wp_json_encode(includes_url('js/jquery/jquery.min.js')); ?>;
9647		- jq.onload = appendChatScript;
9648		- document.body.appendChild(jq);
9649		- }
9650		- }
9651		-
9652		- <?php if ($loading_strategy === 'on_interaction'): ?>
9653		- // Load on user interaction
9654		- var events = ['scroll', 'mousemove', 'touchstart', 'keydown', 'click'];
9655		- events.forEach(function(evt) {
9656		- window.addEventListener(evt, loadMxChatScript, {once: true, passive: true});
9657		- });
9658		- // Fallback: load after 8 seconds if no interaction
9659		- setTimeout(loadMxChatScript, 8000);
9660		- <?php else: ?>
9661		- // Load after specified delay
9662		- setTimeout(loadMxChatScript, <?php echo intval($delay_ms); ?>);
9663		- <?php endif; ?>
9664		- })();
9665		- </script>
9666		- <?php
9667		-}
9668		-
9669	6097	/**
9670	6098	* Setup the cron jobs for rate limits with guard against multiple calls
9671	6099	*/
9672	6100	public function setup_rate_limit_cron_jobs() {
		@@ -9804,9 +6232,9 @@
9804	6232	update_option('mxchat_next_rate_limit_check', time() + $check_interval);
9805	6233	}
9806	6234	}
9807	6235	/**
9808		- * Enhanced rate limit check that includes fallback cleanup and bot-specific rate limits
	6236	+ * Enhanced rate limit check that includes fallback cleanup
9809	6237	*/
9810	6238	public function check_rate_limit() {
9811	6239	// Check if we need to run fallback cleanup
9812	6240	$use_fallback = get_option('mxchat_use_fallback_rate_limits', false);
		@@ -9816,18 +6244,11 @@
9816	6244	$this->mxchat_reset_rate_limits();
9817	6245	update_option('mxchat_next_rate_limit_check', time() + 3600); // Next hour
9818	6246	}
9819	6247
9820		- // Get bot ID from current request context
9821		- $bot_id = isset($_POST['bot_id']) ? sanitize_key($_POST['bot_id']) : 'default';
	6248	+ // Continue with your existing rate limit logic...
	6249	+ $all_options = get_option('mxchat_options', []);
9822	6250
9823		- // Get bot-specific options (includes rate limits if overridden)
9824		- $bot_options = $this->get_bot_options($bot_id);
9825		- $current_options = !empty($bot_options) ? $bot_options : $this->options;
9826		-
9827		- // Use bot-specific rate limits if available, otherwise fall back to default
9828		- $rate_limits_source = isset($current_options['rate_limits']) ? $current_options['rate_limits'] : get_option('mxchat_options', [])['rate_limits'] ?? [];
9829		-
9830	6251	// Determine user role or if logged out
9831	6252	if (is_user_logged_in()) {
9832	6253	$user = wp_get_current_user();
9833	6254	$user_id = $user->ID;
		@@ -9847,13 +6268,13 @@
9847	6268	$user_id = $this->get_client_ip();
9848	6269	}
9849	6270
9850	6271	// Check if rate limits are configured for this role
9851		- if (!isset($rate_limits_source[$role])) {
	6272	+ if (!isset($all_options['rate_limits'][$role])) {
9852	6273	return true; // No limit set for this role
9853	6274	}
9854	6275
9855		- $limit = $rate_limits_source[$role]['limit'];
	6276	+ $limit = $all_options['rate_limits'][$role]['limit'];
9856	6277
9857	6278	// If unlimited, return true immediately
9858	6279	if ($limit === 'unlimited') {
9859	6280	return true;
		@@ -9858,16 +6279,13 @@
9858	6279	if ($limit === 'unlimited') {
9859	6280	return true;
9860	6281	}
9861	6282
9862		- // Get the option name for this user/role with safer naming (include bot_id for bot-specific limits)
	6283	+ // Get the option name for this user/role with safer naming
9863	6284	$safe_role = preg_replace('/[^a-zA-Z0-9_]/', '_', $role);
9864	6285	$safe_user_id = preg_replace('/[^a-zA-Z0-9_]/', '_', $user_id);
9865		- $safe_bot_id = preg_replace('/[^a-zA-Z0-9_]/', '_', $bot_id);
	6286	+ $option_name = 'mxchat_chat_limit_' . $safe_role . '_' . $safe_user_id;
9866	6287
9867		- // Include bot_id in option name so each bot has separate rate limits
9868		- $option_name = 'mxchat_chat_limit_' . $safe_bot_id . '_' . $safe_role . '_' . $safe_user_id;
9869		-
9870	6288	// Get the counter data
9871	6289	$limit_data = get_option($option_name, ['count' => 0, 'timestamp' => time()]);
9872	6290
9873	6291	// If first request or counter reset needed, set the initial timestamp
		@@ -9876,10 +6294,10 @@
9876	6294	update_option($option_name, $limit_data);
9877	6295	}
9878	6296
9879	6297	// Get the timeframe
9880		- $timeframe = isset($rate_limits_source[$role]['timeframe']) ?
9881		- $rate_limits_source[$role]['timeframe'] : 'daily';
	6298	+ $timeframe = isset($all_options['rate_limits'][$role]['timeframe']) ?
	6299	+ $all_options['rate_limits'][$role]['timeframe'] : 'daily';
9882	6300
9883	6301	// Check if the counter needs to be reset based on timeframe
9884	6302	$current_time = time();
9885	6303	$timestamp = $limit_data['timestamp'];
		@@ -9908,10 +6326,10 @@
9908	6326
9909	6327	// Check if user has exceeded their limit
9910	6328	if ($limit_data['count'] >= intval($limit)) {
9911	6329	// Get the custom message for this role
9912		- $message = !empty($rate_limits_source[$role]['message'])
9913		- ? $rate_limits_source[$role]['message']
	6330	+ $message = !empty($all_options['rate_limits'][$role]['message'])
	6331	+ ? $all_options['rate_limits'][$role]['message']
9914	6332	: __('Rate limit exceeded. Please try again later.', 'mxchat');
9915	6333
9916	6334	// Add timeframe information to the message if placeholders exist
9917	6335	$timeframe_label = '';
		@@ -10201,11 +6619,8 @@
10201	6619
10202	6620	/**
10203	6621	* AJAX handler to get system information for testing panel
10204	6622	*/
10205		-/**
10206		- * AJAX handler to get system information for testing panel
10207		- */
10208	6623	public function mxchat_get_system_info() {
10209	6624	// Verify nonce for security
10210	6625	if (!wp_verify_nonce($_POST['nonce'], 'mxchat_test_nonce')) {
10211	6626	wp_send_json_error(['message' => 'Invalid nonce']);
		@@ -10223,24 +6638,10 @@
10223	6638	? $this->options['system_prompt_instructions']
10224	6639	: 'No system prompt configured';
10225	6640
10226	6641	// Get selected model
10227		- $selected_model = isset($this->options['model']) ? $this->options['model'] : 'gpt-5.1-chat-latest';
	6642	+ $selected_model = isset($this->options['model']) ? $this->options['model'] : 'gpt-4o';
10228	6643
10229		- // Check if OpenRouter is being used
10230		- $is_openrouter = ($selected_model === 'openrouter');
10231		- $openrouter_model = '';
10232		-
10233		- if ($is_openrouter) {
10234		- // Get the actual OpenRouter model that's selected
10235		- $openrouter_model = isset($this->options['openrouter_selected_model'])
10236		- ? $this->options['openrouter_selected_model']
10237		- : 'No OpenRouter model selected';
10238		-
10239		- // Update selected_model display to show both
10240		- $selected_model = 'OpenRouter: ' . $openrouter_model;
10241		- }
10242		-
10243	6644	// Get API key status (just check if they exist, don't expose the keys)
10244	6645	$api_status = [];
10245	6646	$api_status['openai'] = !empty($this->options['api_key']);
10246	6647	$api_status['claude'] = !empty($this->options['claude_api_key']);
		@@ -10246,15 +6647,12 @@
10246	6647	$api_status['claude'] = !empty($this->options['claude_api_key']);
10247	6648	$api_status['gemini'] = !empty($this->options['gemini_api_key']);
10248	6649	$api_status['xai'] = !empty($this->options['xai_api_key']);
10249	6650	$api_status['deepseek'] = !empty($this->options['deepseek_api_key']);
10250		- $api_status['openrouter'] = !empty($this->options['openrouter_api_key']);
10251	6651
10252	6652	wp_send_json_success([
10253	6653	'system_prompt' => $system_prompt,
10254	6654	'selected_model' => $selected_model,
10255		- 'is_openrouter' => $is_openrouter,
10256		- 'openrouter_model' => $openrouter_model,
10257	6655	'api_status' => $api_status
10258	6656	]);
10259	6657	}
10260	6658
		@@ -10273,12 +6671,12 @@
10273	6671	wp_send_json_error(['message' => 'Unauthorized']);
10274	6672	return;
10275	6673	}
10276	6674
10277		- // Get similarity threshold from main options (default 35%)
	6675	+ // Get similarity threshold from main options (default 75%)
10278	6676	$similarity_threshold = isset($this->options['similarity_threshold'])
10279	6677	? ((int) $this->options['similarity_threshold']) / 100
10280		- : 0.35;
	6678	+ : 0.75;
10281	6679
10282	6680	wp_send_json_success([
10283	6681	'threshold' => $similarity_threshold,
10284	6682	'threshold_percentage' => ($similarity_threshold * 100) . '%'
		@@ -10293,42 +6691,24 @@
10293	6691	if (!wp_verify_nonce($_POST['nonce'], 'mxchat_test_nonce')) {
10294	6692	wp_send_json_error(['message' => 'Invalid nonce']);
10295	6693	return;
10296	6694	}
10297		-
	6695	+
10298	6696	// Only allow admin users
10299	6697	if (!current_user_can('administrator')) {
10300	6698	wp_send_json_error(['message' => 'Unauthorized']);
10301	6699	return;
10302	6700	}
10303		-
10304		- // Check OpenAI Vector Store first (takes priority)
10305		- $vectorstore_options = get_option('mxchat_openai_vectorstore_options', array());
10306		- $use_vectorstore = (isset($vectorstore_options['mxchat_use_openai_vectorstore']) && $vectorstore_options['mxchat_use_openai_vectorstore'] === '1');
10307		-
10308		- if ($use_vectorstore) {
10309		- $vectorstore_ids = $vectorstore_options['mxchat_openai_vectorstore_ids'] ?? '';
10310		- $id_count = !empty($vectorstore_ids) ? count(array_filter(array_map('trim', explode(',', $vectorstore_ids)))) : 0;
10311		-
10312		- $kb_info = [
10313		- 'type' => 'OpenAI Vector Store',
10314		- 'status' => 'Active',
10315		- 'documents' => $id_count > 0 ? $id_count . ' vector store' . ($id_count > 1 ? 's' : '') . ' configured' : 'No vector stores configured'
10316		- ];
10317		-
10318		- wp_send_json_success($kb_info);
10319		- return;
10320		- }
10321		-
	6701	+
10322	6702	// Check Pinecone vs WordPress
10323	6703	$addon_options = get_option('mxchat_pinecone_addon_options', array());
10324	6704	$use_pinecone = (isset($addon_options['mxchat_use_pinecone']) && $addon_options['mxchat_use_pinecone'] === '1');
10325		-
	6705	+
10326	6706	$kb_info = [
10327	6707	'type' => $use_pinecone ? 'Pinecone' : 'WordPress Database',
10328	6708	'status' => 'Active'
10329	6709	];
10330		-
	6710	+
10331	6711	// Get document count
10332	6712	if ($use_pinecone) {
10333	6713	$kb_info['documents'] = 'Connected to Pinecone';
10334	6714	$kb_info['api_configured'] = !empty($addon_options['mxchat_pinecone_api_key']);
		@@ -10338,9 +6718,9 @@
10338	6718	$table_name = $wpdb->prefix . 'mxchat_system_prompt_content';
10339	6719	$count = $wpdb->get_var("SELECT COUNT(*) FROM {$table_name}");
10340	6720	$kb_info['documents'] = $count ? $count . ' documents' : 'No documents';
10341	6721	}
10342		-
	6722	+
10343	6723	wp_send_json_success($kb_info);
10344	6724	}
10345	6725
10346	6726	/**
		@@ -10422,13 +6802,9 @@
10422	6802	// Clear any other session-specific transients
10423	6803	delete_transient("mxchat_waiting_for_pdf_url_{$session_id}");
10424	6804	delete_transient("mxchat_include_pdf_in_context_{$session_id}");
10425	6805	delete_transient("mxchat_include_word_in_context_{$session_id}");
10426		-
10427		- // Clear form addon state (pending forms and submitted forms)
10428		- delete_option("mxchat_pending_form_{$session_id}");
10429		- delete_option("mxchat_submitted_forms_{$session_id}");
10430		-
	6806	+
10431	6807	//error_log("MxChat: Cleared all data for session: {$session_id}");
10432	6808	}
10433	6809
10434	6810	/**
		@@ -10463,15 +6839,15 @@
10463	6839	$testing_data = [
10464	6840	'query' => $message,
10465	6841	'timestamp' => time(),
10466	6842	'top_matches' => [],
10467		- 'action_matches' => [] // Add action matches
	6843	+ 'action_matches' => [] // NEW: Add action matches
10468	6844	];
10469	6845
10470	6846	// Get similarity threshold
10471	6847	$similarity_threshold = isset($this->options['similarity_threshold'])
10472	6848	? ((int) $this->options['similarity_threshold']) / 100
10473		- : 0.35;
	6849	+ : 0.75;
10474	6850
10475	6851	$testing_data['similarity_threshold'] = $similarity_threshold;
10476	6852
10477	6853	// Use the real similarity analysis if available
		@@ -10486,9 +6862,9 @@
10486	6862
10487	6863	$testing_data['knowledge_base_type'] = $use_pinecone ? 'Pinecone' : 'WordPress Database';
10488	6864	}
10489	6865
10490		- // Include action analysis if available
	6866	+ // NEW: Include action analysis if available
10491	6867	if (isset($this->last_action_analysis) && !empty($this->last_action_analysis)) {
10492	6868	$testing_data['action_matches'] = $this->last_action_analysis;
10493	6869
10494	6870	// Clear it after capturing to avoid stale data
		@@ -10499,9 +6875,9 @@
10499	6875	}
10500	6876
10501	6877
10502	6878	/**
10503		- * Track URL clicks from chatbot responses
	6879	+ * NEW: Track URL clicks from chatbot responses
10504	6880	*/
10505	6881	public function mxchat_track_url_click() {
10506	6882	// Verify nonce for security
10507	6883	if (!isset($_POST['nonce']) || !wp_verify_nonce($_POST['nonce'], 'mxchat_chat_nonce')) {
		@@ -10538,9 +6914,9 @@
10538	6914	wp_die();
10539	6915	}
10540	6916
10541	6917	/**
10542		- * Get URL click analytics for a session
	6918	+ * NEW: Get URL click analytics for a session
10543	6919	*/
10544	6920	public function mxchat_get_url_clicks($session_id) {
10545	6921	global $wpdb;
10546	6922	$table_name = $wpdb->prefix . 'mxchat_url_clicks';
		@@ -10552,9 +6928,9 @@
10552	6928
10553	6929	return $clicks;
10554	6930	}
10555	6931	/**
10556		- * Track the originating page where chat was started
	6932	+ * NEW: Track the originating page where chat was started
10557	6933	*/
10558	6934	public function mxchat_track_originating_page() {
10559	6935	// Verify nonce
10560	6936	if (!isset($_POST['nonce']) || !wp_verify_nonce($_POST['nonce'], 'mxchat_chat_nonce')) {
		@@ -10603,169 +6979,8 @@
10603	6979	wp_send_json_success(['message' => 'Originating page tracked']);
10604	6980	wp_die();
10605	6981	}
10606	6982
10607		-/**
10608		- * Validate and clean URLs from AI response
10609		- * Removes any URLs that aren't in the knowledge base
10610		- *
10611		- * @param string $response_text The AI-generated response
10612		- * @param array $valid_urls Array of URLs from the knowledge base
10613		- * @return string Cleaned response with invalid URLs removed/flagged
10614		- */
10615		-private function validate_and_clean_urls($response_text, $valid_urls) {
10616		- // DEBUG: Log what we're working with
10617		- //error_log("=== MxChat URL Validation Debug ===");
10618		- //error_log("Valid URLs count: " . count($valid_urls));
10619		- //error_log("Valid URLs: " . print_r($valid_urls, true));
10620		- //error_log("Response text length: " . strlen($response_text));
10621		- //error_log("Response text preview: " . substr($response_text, 0, 500));
10622		-
10623		- // If no valid URLs provided or empty response, return as-is
10624		- if (empty($valid_urls) || empty($response_text)) {
10625		- //error_log("Validation skipped - empty valid_urls or response");
10626		- return $response_text;
10627		- }
10628		-
10629		- // Extract all URLs from the AI response
10630		- // This regex matches http:// and https:// URLs
10631		- preg_match_all(
10632		- '#\bhttps?://[^\s<>"\')\]]+#i',
10633		- $response_text,
10634		- $matches
10635		- );
10636		-
10637		- // If no URLs found in response, return as-is
10638		- if (empty($matches[0])) {
10639		- //error_log("No URLs found in response");
10640		- return $response_text;
10641		- }
10642		-
10643		- $found_urls = $matches[0];
10644		- $cleaned_response = $response_text;
10645		- $removed_count = 0;
10646		-
10647		- // Normalize valid URLs for comparison (remove trailing slashes, fragments, etc.)
10648		- $normalized_valid_urls = array_map(function($url) {
10649		- // Remove trailing slash
10650		- $url = rtrim($url, '/');
10651		- // Remove URL fragments (#section)
10652		- $url = preg_replace('/#.*$/', '', $url);
10653		- // Remove trailing punctuation that might have been captured
10654		- $url = rtrim($url, '.,;:!?');
10655		- return $url;
10656		- }, $valid_urls);
10657		-
10658		- //error_log("Normalized valid URLs: " . print_r($normalized_valid_urls, true));
10659		-
10660		- foreach ($found_urls as $found_url) {
10661		- // Clean up the found URL (remove trailing punctuation that might have been captured)
10662		- $clean_found_url = rtrim($found_url, '.,;:!?)');
10663		-
10664		- // DEBUG: Log each URL being checked
10665		- //error_log("Checking found URL: " . $found_url);
10666		-
10667		- // Normalize for comparison
10668		- $normalized_found = rtrim($clean_found_url, '/');
10669		- $normalized_found = preg_replace('/#.*$/', '', $normalized_found);
10670		-
10671		- //error_log("Normalized found URL: " . $normalized_found);
10672		-
10673		- // Check if this URL exists in our valid URLs list
10674		- $is_valid = false;
10675		-
10676		- //error_log("Starting validation checks for: " . $normalized_found);
10677		-
10678		- // First, try exact match
10679		- if (in_array($normalized_found, $normalized_valid_urls)) {
10680		- $is_valid = true;
10681		- //error_log("EXACT MATCH FOUND");
10682		- } else {
10683		- //error_log("No exact match, checking variations...");
10684		- // If no exact match, check if it's a variation (with query params, etc.)
10685		- foreach ($normalized_valid_urls as $valid_url) {
10686		- //error_log(" Comparing against valid URL: " . $valid_url);
10687		-
10688		- // Check if the found URL starts with a valid URL (handles query params)
10689		- if (strpos($normalized_found, $valid_url) === 0) {
10690		- // Check what comes after the valid URL
10691		- $remainder = substr($normalized_found, strlen($valid_url));
10692		-
10693		- // Only valid if:
10694		- // 1. Exact match (remainder is empty)
10695		- // 2. Query params (starts with ?)
10696		- // 3. Fragment (starts with #)
10697		- if (empty($remainder) || $remainder[0] === '?' || $remainder[0] === '#') {
10698		- $is_valid = true;
10699		- //error_log(" MATCH: Found URL is valid variation of base URL");
10700		- break;
10701		- } else {
10702		- //error_log(" NOT A MATCH: Found URL extends path beyond valid URL (remainder: " . $remainder . ")");
10703		- }
10704		- }
10705		- // Also check the reverse (in case valid URL has query params)
10706		- if (strpos($valid_url, $normalized_found) === 0) {
10707		- $is_valid = true;
10708		- //error_log(" MATCH: Valid URL starts with found URL");
10709		- break;
10710		- }
10711		- }
10712		-
10713		- if (!$is_valid) {
10714		- //error_log("NO MATCH FOUND - URL should be removed");
10715		- }
10716		- }
10717		-
10718		- // If URL is not valid, remove it from the response
10719		- if (!$is_valid) {
10720		- // Log the removal for debugging
10721		- //error_log("MxChat: Removed hallucinated URL: " . $found_url);
10722		- //error_log("MxChat: Valid URLs were: " . implode(', ', array_slice($normalized_valid_urls, 0, 5)));
10723		-
10724		- $removed_count++;
10725		-
10726		- // Check if URL is part of a markdown link: [text](url)
10727		- $markdown_pattern = '/\[([^\]]+)\]\(' . preg_quote($found_url, '/') . '\)/';
10728		- if (preg_match($markdown_pattern, $cleaned_response)) {
10729		- //error_log("Found markdown link, removing but keeping text");
10730		- // Remove the markdown link but keep the text
10731		- $cleaned_response = preg_replace($markdown_pattern, '$1', $cleaned_response);
10732		- }
10733		- // Check if URL is part of an HTML link: <a href="url">text</a>
10734		- else if (preg_match('/<a[^>]*href=["\']' . preg_quote($found_url, '/') . '["\'][^>]*>(.*?)<\/a>/i', $cleaned_response, $link_match)) {
10735		- //error_log("Found HTML link, removing but keeping text");
10736		- // Remove the HTML link but keep the text
10737		- $link_text = $link_match[1];
10738		- $cleaned_response = preg_replace(
10739		- '/<a[^>]*href=["\']' . preg_quote($found_url, '/') . '["\'][^>]*>.*?<\/a>/i',
10740		- $link_text,
10741		- $cleaned_response
10742		- );
10743		- }
10744		- // Otherwise just remove the bare URL
10745		- else {
10746		- //error_log("Removing bare URL");
10747		- $cleaned_response = str_replace($found_url, '', $cleaned_response);
10748		- }
10749		- }
10750		- }
10751		-
10752		- // Log summary if any URLs were removed
10753		- if ($removed_count > 0) {
10754		- //error_log("MxChat: URL Validation Summary - Removed {$removed_count} hallucinated URL(s)");
10755		- } else {
10756		- //error_log("MxChat: URL Validation Summary - No URLs removed, all were valid");
10757		- }
10758		-
10759		- // Clean up any double spaces or awkward punctuation left behind
10760		- // IMPORTANT: Only collapse horizontal whitespace (spaces/tabs), preserve newlines for markdown formatting
10761		- $cleaned_response = preg_replace('/[^\S\n]+/', ' ', $cleaned_response); // Collapse spaces/tabs but NOT newlines
10762		- $cleaned_response = preg_replace('/[^\S\n]+([.,;:!?])/', '$1', $cleaned_response); // Same for punctuation cleanup
10763		-
10764		- //error_log("Final cleaned response: " . $cleaned_response);
10765		-
10766		- return trim($cleaned_response);
10767		-}
10768	6983
10769	6984	/**
10770	6985	* AJAX handler to get current chat mode for a session
10771	6986	*/