PluginProbe
wpForo Forum / 3.2.1
wpForo Forum v3.2.1
3.2.1 3.2.0 3.1.7 3.1.6 3.1.5 3.1.4 3.1.2 3.1.1 3.1.0 3.0.9 3.0.8 3.0.7 trunk 1.0.0 1.0.1 1.0.2 1.1.0 1.1.1 1.1.2 1.2.0 1.3.0 1.3.1 1.4.0 1.4.1 1.4.10 All 141 releases
wpforo / admin / pages / tabs / ai-features-tab-rag-indexing.php

ai-features-tab-rag-indexing.php in wpForo Forum 3.2.1, at admin/pages/tabs/ai-features-tab-rag-indexing.php

894 lines 44.3 KB
No matching file
Up and down to move Enter to open Esc to close
Raw Download Zip
1 <?php
2 /**
3 * AI Features - AI Content Indexing Tab
4 *
5 * @package wpForo
6 * @subpackage Admin
7 */
8
9 if ( ! defined( 'ABSPATH' ) ) {
10 exit;
11 }
12
13 /**
14 * Render AI Content Indexing tab content
15 *
16 * @param bool $is_connected Whether tenant is connected to AI service
17 * @param array $status Tenant status data from API
18 */
19 function wpforo_ai_render_rag_indexing_tab( $is_connected, $status ) {
20 if ( ! $is_connected ) {
21 ?>
22 <div class="wpforo-ai-box wpforo-ai-not-connected-notice">
23 <div class="wpforo-ai-box-body">
24 <div class="wpforo-ai-status-badge status-warning">
25 <span class="dashicons dashicons-warning"></span>
26 <?php _e( 'Not Connected', 'wpforo' ); ?>
27 </div>
28 <p><?php _e( 'Please connect to wpForo AI API first in the Overview tab to enable AI Content Indexing.', 'wpforo' ); ?></p>
29 <a href="<?php echo esc_url( admin_url( 'admin.php?page=wpforo-ai&tab=overview' ) ); ?>" class="button button-primary">
30 <?php _e( 'Go to Overview', 'wpforo' ); ?>
31 </a>
32 </div>
33 </div>
34 <?php
35 return;
36 }
37
38 // Get all boards and determine current board
39 $boards = WPF()->board->get_boards();
40 $current_boardid = isset( $_GET['boardid'] ) ? intval( $_GET['boardid'] ) : 0;
41
42 // Validate boardid exists
43 $valid_boardids = array_column( $boards, 'boardid' );
44 if ( ! in_array( $current_boardid, $valid_boardids ) ) {
45 $current_boardid = 0; // Default to main board
46 }
47
48 // Switch to the selected board context
49 if ( $current_boardid > 0 ) {
50 WPF()->change_board( $current_boardid );
51 }
52
53 // Render board sub-tabs if multiple boards exist
54 if ( count( $boards ) > 1 ) :
55 // Find current board title (from settings, not label)
56 $current_board_title = __( 'Board', 'wpforo' );
57 foreach ( $boards as $board ) {
58 if ( isset( $board['boardid'] ) && (int) $board['boardid'] === $current_boardid ) {
59 $current_board_title = isset( $board['settings']['title'] ) ? esc_html( $board['settings']['title'] ) : ( isset( $board['title'] ) ? esc_html( $board['title'] ) : $current_board_title );
60 break;
61 }
62 }
63 ?>
64 <div class="wpforo-ai-board-tabs">
65 <div class="wpforo-ai-board-current">
66 <?php echo esc_html__( 'Board:', 'wpforo' ); ?> <strong><?php echo $current_board_title; ?></strong>
67 </div>
68 <div class="wpforo-ai-board-tabs-list">
69 <?php foreach ( $boards as $board ) :
70 $board_id = isset( $board['boardid'] ) ? (int) $board['boardid'] : 0;
71 $board_label = isset( $board['title'] ) ? esc_html( $board['title'] ) : __( 'Board', 'wpforo' ) . ' ' . $board_id;
72 $tab_url = add_query_arg( array(
73 'page' => 'wpforo-ai',
74 'tab' => 'rag_indexing',
75 'boardid' => $board_id,
76 ), admin_url( 'admin.php' ) );
77 $active_class = ( $current_boardid === $board_id ) ? 'active' : '';
78 ?>
79 <a href="<?php echo esc_url( $tab_url ); ?>" class="wpforo-ai-board-tab <?php echo esc_attr( $active_class ); ?>">
80 <span class="dashicons dashicons-category"></span>
81 <?php echo $board_label; ?>
82 </a>
83 <?php endforeach; ?>
84 </div>
85 </div>
86 <?php
87 endif;
88
89 // Use VectorStorageManager abstraction for all statistics
90 $storage_manager = WPF()->vector_storage->for_board( $current_boardid );
91 $storage_mode = $storage_manager->get_storage_mode();
92
93 // Get unified indexing stats (works for both local and cloud)
94 $indexing_stats = $storage_manager->get_indexing_stats();
95
96 $total_indexed = (int) ( $indexing_stats['total_indexed'] ?? 0 );
97 $total_topics = (int) ( $indexing_stats['total_topics'] ?? 0 );
98 $indexing_progress = (int) ( $indexing_stats['indexing_progress'] ?? 0 );
99 $is_indexing = (bool) ( $indexing_stats['is_indexing'] ?? false );
100 $last_indexed_at = $indexing_stats['last_indexed_at'] ?? null;
101 $storage_size_mb = $indexing_stats['storage_size_mb'] ?? null;
102
103 // Check for pending WP Cron jobs
104 $pending_jobs_info = $storage_manager->get_pending_cron_jobs();
105 $has_pending_cron_jobs = $pending_jobs_info['has_pending_jobs'];
106 $pending_topics_count_early = $pending_jobs_info['pending_topics'];
107
108 // Only show the spinner when actually indexing (backend is processing or cron batch is running).
109 // Queued topics waiting for their scheduled time should NOT trigger the spinner.
110 $is_processing = $is_indexing || $pending_jobs_info['is_actively_processing'];
111
112 // Get storage recommendation
113 $recommendation = $storage_manager->get_storage_recommendation();
114
115 // Local stats for display (only relevant in local mode)
116 $local_stats = $storage_manager->is_local_mode() ? [
117 'total_embeddings' => $total_indexed,
118 'total_topics' => $total_topics,
119 'storage_size_mb' => $storage_size_mb,
120 ] : [];
121
122 ?>
123 <div class="wpforo-ai-rag-indexing">
124
125 <!-- RAG Status Box -->
126 <div class="wpforo-ai-box wpforo-ai-rag-status-box">
127 <div class="wpforo-ai-box-header">
128 <h2><?php _e( 'Indexing Status', 'wpforo' ); ?></h2>
129 <div class="wpforo-ai-header-actions">
130 <button type="button" class="button button-small wpforo-ai-refresh-rag-status" data-loading-text="<?php esc_attr_e( 'Refreshing...', 'wpforo' ); ?>">
131 <span class="dashicons dashicons-update"></span>
132 <?php _e( 'Refresh Status', 'wpforo' ); ?>
133 </button>
134 </div>
135 </div>
136 <div class="wpforo-ai-box-body">
137 <div class="wpforo-ai-rag-stats">
138 <div class="rag-stat-item">
139 <div class="stat-icon">
140 <span class="dashicons dashicons-format-chat"></span>
141 </div>
142 <div class="stat-info">
143 <div class="stat-value" id="rag-total-topics"><?php echo number_format( $total_topics ); ?><?php if ( $pending_topics_count_early > 0 ) : ?> <small class="rag-queued-count">| <?php echo number_format( $pending_topics_count_early ); ?> <?php _e( 'queued...', 'wpforo' ); ?></small><?php endif; ?></div>
144 <div class="stat-label"><?php _e( 'Total Threads Indexed', 'wpforo' ); ?></div>
145 </div>
146 </div>
147
148 <div class="rag-stat-item">
149 <div class="stat-icon">
150 <span class="dashicons dashicons-<?php echo $is_processing ? 'update-alt wpforo-rag-status-spin' : 'saved'; ?>"></span>
151 </div>
152 <div class="stat-info">
153 <div class="stat-value <?php echo $is_processing ? 'status-active' : 'status-idle'; ?>" id="rag-indexing-status">
154 <?php
155 if ( $is_processing ) {
156 _e( 'Indexing...', 'wpforo' );
157 } else {
158 _e( 'Idle', 'wpforo' );
159 }
160 ?>
161 </div>
162 <div class="stat-label"><?php _e( 'Current Status', 'wpforo' ); ?></div>
163 </div>
164 </div>
165
166 <?php if ( $last_indexed_at ) : ?>
167 <div class="rag-stat-item">
168 <div class="stat-icon">
169 <span class="dashicons dashicons-clock"></span>
170 </div>
171 <div class="stat-info">
172 <div class="stat-value"><?php echo esc_html( wpforo_ai_format_date( $last_indexed_at ) ); ?></div>
173 <div class="stat-label"><?php _e( 'Last Indexed', 'wpforo' ); ?></div>
174 </div>
175 </div>
176 <?php endif; ?>
177 </div>
178 </div>
179 </div>
180
181 <!-- Storage Settings Box (Business+ plan required for cloud storage) -->
182 <?php
183 // Cloud storage requires Business plan or higher
184 $wpf_ai_cloud_storage_available = isset( WPF()->ai_client ) && WPF()->ai_client->is_feature_available( 'vector_db_cloud_storage' );
185 if ( $wpf_ai_cloud_storage_available ) :
186 ?>
187 <div class="wpforo-ai-box wpforo-ai-storage-settings-box">
188 <div class="wpforo-ai-box-header">
189 <h2><?php _e( 'Storage Settings', 'wpforo' ); ?></h2>
190 </div>
191 <div class="wpforo-ai-box-body">
192 <div class="wpforo-ai-storage-options">
193 <div class="wpforo-ai-storage-toggle">
194 <label class="wpforo-ai-toggle-label">
195 <span class="wpforo-ai-toggle-text"><?php _e( 'Storage Mode', 'wpforo' ); ?></span>
196 <div class="wpforo-ai-toggle-switch">
197 <input type="radio" name="wpforo_ai_storage_mode" value="local" id="storage-mode-local" <?php checked( $storage_mode, 'local' ); ?>>
198 <label for="storage-mode-local" class="wpforo-ai-storage-option <?php echo $storage_mode === 'local' ? 'active' : ''; ?>">
199 <span class="dashicons dashicons-database"></span>
200 <?php _e( 'Local (WordPress)', 'wpforo' ); ?>
201 </label>
202 <input type="radio" name="wpforo_ai_storage_mode" value="cloud" id="storage-mode-cloud" <?php checked( $storage_mode, 'cloud' ); ?>>
203 <label for="storage-mode-cloud" class="wpforo-ai-storage-option <?php echo $storage_mode === 'cloud' ? 'active' : ''; ?>">
204 <span class="dashicons dashicons-cloud"></span>
205 <?php _e( 'Cloud (gVectors on AWS)', 'wpforo' ); ?>
206 </label>
207 </div>
208 </label>
209 </div>
210
211 <div class="wpforo-ai-storage-info">
212 <?php if ( $storage_mode === 'local' ) : ?>
213 <div class="wpforo-ai-storage-recommendation status-<?php echo esc_attr( $recommendation['status'] ); ?>">
214 <span class="dashicons dashicons-<?php echo esc_attr( $recommendation['icon'] ); ?>"></span>
215 <span><?php echo esc_html( $recommendation['message'] ); ?></span>
216 </div>
217 <?php if ( ! empty( $local_stats ) ) : ?>
218 <div class="wpforo-ai-storage-stats">
219 <span class="stat-item">
220 <strong id="local-total-embeddings"><?php echo number_format( $local_stats['total_embeddings'] ); ?></strong>
221 <?php _e( 'vectors', 'wpforo' ); ?>
222 </span>
223 <span class="stat-item">
224 <strong id="local-total-topics"><?php echo number_format( $local_stats['total_topics'] ); ?></strong>
225 <?php _e( 'topics', 'wpforo' ); ?>
226 </span>
227 <span class="stat-item">
228 <strong id="local-storage-size"><?php echo $local_stats['total_embeddings'] > 0 ? $local_stats['storage_size_mb'] : '0'; ?> MB</strong>
229 <?php _e( 'storage', 'wpforo' ); ?>
230 </span>
231 </div>
232 <?php endif; ?>
233 <?php else : ?>
234 <div class="wpforo-ai-storage-recommendation status-info">
235 <span class="dashicons dashicons-cloud"></span>
236 <span><?php _e( 'Vectors are stored in gVectors AI Services on AWS for optimal performance with large datasets.', 'wpforo' ); ?></span>
237 </div>
238 <?php endif; ?>
239 </div>
240
241 <p class="description">
242 <?php _e( 'Local storage keeps all data within your WordPress database. Recommended for forums with up to 100,000 posts.', 'wpforo' ); ?>
243 <br>
244 <?php _e( 'Cloud storage (gVectors AI Services on AWS Cloud) provides faster search performance for large forums but requires sharing data with our API.', 'wpforo' ); ?>
245 </p>
246 </div>
247 </div>
248 </div>
249 <?php endif; // End Storage Settings plan check ?>
250
251 <!-- Manual Content Indexing Box -->
252 <div class="wpforo-ai-box wpforo-ai-manual-ingest-box">
253 <div class="wpforo-ai-box-header">
254 <h2><?php _e( 'Forum Content Indexing', 'wpforo' ); ?></h2>
255 <div class="wpforo-ai-header-actions">
256 <?php
257 // Get auto-indexing option (board-specific)
258 $auto_indexing_enabled = (bool) wpforo_get_option( 'ai_auto_indexing_enabled', 0 );
259
260 // Get image indexing option (board-specific, Professional+ only)
261 $image_indexing_enabled = (bool) wpforo_get_option( 'ai_image_indexing_enabled', 0 );
262
263 // Get document indexing option (board-specific, Professional+ only)
264 $document_indexing_enabled = (bool) wpforo_get_option( 'ai_document_indexing_enabled', 0 );
265
266 // Check if user has eligible plan for image/document indexing
267 $plan = isset( $status['subscription']['plan'] ) ? strtolower( $status['subscription']['plan'] ) : '';
268 $has_image_plan = in_array( $plan, [ 'professional', 'business', 'enterprise' ], true );
269 $has_document_plan = in_array( $plan, [ 'professional', 'business', 'enterprise' ], true );
270 ?>
271 <label class="wpforo-ai-auto-index-toggle" title="<?php esc_attr_e( 'Include images in indexing (Professional/Business/Enterprise only). Posts with images will consume +1 credit.', 'wpforo' ); ?>">
272 <span class="wpforo-ai-auto-index-label">
273 <?php _e( 'Index Images', 'wpforo' ); ?>
274 <?php if ( ! $has_image_plan ) : ?>
275 <span class="wpforo-ai-plan-badge" title="<?php esc_attr_e( 'Requires Professional, Business or Enterprise plan', 'wpforo' ); ?>">[ Professional+ Plan ]</span>
276 <?php endif; ?>
277 </span>
278 <div class="wpforo-ai-switch">
279 <input type="checkbox" id="wpforo-ai-image-indexing" name="ai_image_indexing_enabled" value="1" <?php checked( $image_indexing_enabled ); ?> data-board-id="<?php echo esc_attr( $current_boardid ); ?>" <?php disabled( ! $has_image_plan ); ?>>
280 <span class="wpforo-ai-switch-slider"></span>
281 </div>
282 </label>
283 <label class="wpforo-ai-auto-index-toggle" title="<?php esc_attr_e( 'Include document attachments (PDF, DOCX, etc.) in indexing (Professional+ only). Credit cost: 1 per document page.', 'wpforo' ); ?>">
284 <span class="wpforo-ai-auto-index-label">
285 <?php _e( 'Index Documents', 'wpforo' ); ?>
286 <?php if ( ! $has_document_plan ) : ?>
287 <span class="wpforo-ai-plan-badge" title="<?php esc_attr_e( 'Requires Professional, Business or Enterprise plan', 'wpforo' ); ?>">[ Professional+ Plan ]</span>
288 <?php endif; ?>
289 </span>
290 <div class="wpforo-ai-switch">
291 <input type="checkbox" id="wpforo-ai-document-indexing" name="ai_document_indexing_enabled" value="1" <?php checked( $document_indexing_enabled ); ?> data-board-id="<?php echo esc_attr( $current_boardid ); ?>" <?php disabled( ! $has_document_plan ); ?>>
292 <span class="wpforo-ai-switch-slider"></span>
293 </div>
294 </label>
295 <label class="wpforo-ai-auto-index-toggle" title="<?php esc_attr_e( 'Automatically index new and approved topics', 'wpforo' ); ?>">
296 <span class="wpforo-ai-auto-index-label"><?php _e( 'Automatically index all new approved topics', 'wpforo' ); ?></span>
297 <div class="wpforo-ai-switch">
298 <input type="checkbox" id="wpforo-ai-auto-indexing" name="ai_auto_indexing_enabled" value="1" <?php checked( $auto_indexing_enabled ); ?> data-board-id="<?php echo esc_attr( $current_boardid ); ?>">
299 <span class="wpforo-ai-switch-slider"></span>
300 </div>
301 </label>
302 </div>
303 </div>
304 <div class="wpforo-ai-box-body">
305 <?php
306 // Get total topics count (transient-cached, board+storage specific)
307 $_ttc_cache_key = 'wpforo_ai_ttc_' . $current_boardid . '_' . $storage_mode;
308 $total_topics_count = get_transient( $_ttc_cache_key );
309 if ( false === $total_topics_count ) {
310 $total_topics_count = (int) WPF()->db->get_var(
311 "SELECT COUNT(*) FROM `" . WPF()->tables->topics . "`"
312 );
313 set_transient( $_ttc_cache_key, $total_topics_count, 10 * MINUTE_IN_SECONDS );
314 }
315 $total_topics_count = (int) $total_topics_count;
316
317 // Get available credits from status
318 $subscription = isset( $status['subscription'] ) && is_array( $status['subscription'] ) ? $status['subscription'] : [];
319 $credits_remaining = isset( $subscription['credits_remaining'] ) ? (int) $subscription['credits_remaining'] : 0;
320
321 // Calculate remaining topics to index (not already indexed)
322 // Note: $total_topics = unique topics with embeddings, $total_indexed = total embeddings (can be > topics due to chunking)
323 $remaining_to_index = max( 0, $total_topics_count - $total_topics );
324
325 // Calculate how many topics can be indexed with available credits
326 # $credits_needed = $total_topics_count; // Max credits if indexing from scratch
327 $can_index_count = min( $remaining_to_index, $credits_remaining );
328 $has_enough_credits = $credits_remaining >= $remaining_to_index;
329 $all_indexed = $remaining_to_index === 0;
330 ?>
331
332 <!-- Index All Section -->
333 <div class="wpforo-ai-index-all-section">
334 <div class="wpforo-ai-index-all-content">
335 <div class="wpforo-ai-index-all-info">
336 <h3>
337 <span class="dashicons dashicons-database-view"></span>
338 <?php _e( 'Full Content Indexing', 'wpforo' ); ?>
339 </h3>
340 <p class="description">
341 <?php _e( 'Index all forum topics in one click. This will queue all topics for AI features. Only new or modified topics will consume credits - unchanged topics are skipped automatically.', 'wpforo' ); ?>
342 </p>
343 <p class="description" style="color: #2e7d32;">
344 <?php _e( 'All new topics and replies are automatically indexed. If a topic or post is unapproved, the indexing process will only start when a moderator has approved them.', 'wpforo' ); ?>
345 </p>
346 </div>
347
348 <div class="wpforo-ai-index-all-stats">
349 <div class="stat-item">
350 <span class="stat-label"><?php _e( 'Total Topics:', 'wpforo' ); ?></span>
351 <strong class="stat-value" id="index-total-topics-count"><?php echo number_format( $total_topics_count ); ?></strong>
352 </div>
353 <div class="stat-item">
354 <span class="stat-label"><?php _e( 'Total Indexed:', 'wpforo' ); ?></span>
355 <strong class="stat-value stat-indexed" id="index-total-indexed"><?php echo number_format( $total_topics ); ?></strong>
356 </div>
357 <div class="stat-item">
358 <span class="stat-label"><?php _e( 'Remaining to Index:', 'wpforo' ); ?></span>
359 <strong class="stat-value" id="index-remaining">
360 <?php echo number_format( $remaining_to_index ); ?>
361 </strong>
362 </div>
363 <div class="stat-item">
364 <span class="stat-label"><?php _e( 'Credits Available:', 'wpforo' ); ?></span>
365 <strong class="stat-value" id="index-credits-available">
366 <?php echo number_format( $credits_remaining ); ?>
367 </strong>
368 </div>
369 </div>
370
371 </div>
372
373 <?php
374 // Use already-computed pending jobs info from earlier
375 $has_pending_jobs = $has_pending_cron_jobs;
376 $pending_topics_count = $pending_topics_count_early;
377 ?>
378
379 <?php if ( $has_pending_jobs ) : ?>
380 <div class="wpforo-ai-credit-notice wpforo-ai-notice-info">
381 <span class="dashicons dashicons-backup"></span>
382 <p>
383 <?php
384 printf(
385 __( 'Background processing in progress: %s topics are queued for indexing.', 'wpforo' ),
386 '<strong>' . number_format( $pending_topics_count ) . '</strong>'
387 );
388 ?>
389 </p>
390 </div>
391 <?php endif; ?>
392
393 <div class="wpforo-ai-index-all-action">
394 <?php
395 // For local storage mode, always render both buttons for JavaScript toggle
396 // For cloud mode, use server-side rendering
397 $can_index = $total_topics_count > 0 && $credits_remaining > 0;
398 ?>
399 <?php if ( $can_index ) : ?>
400 <!-- Reindex button: hidden when indexing is in progress -->
401 <button type="button" class="button button-primary button-hero wpforo-ai-reindex-all" style="<?php echo $has_pending_jobs ? 'display:none;' : ''; ?>" data-confirm="<?php esc_attr_e( 'This will index all forum topics. Continue?', 'wpforo' ); ?>" data-nonce="<?php echo esc_attr( wp_create_nonce( 'wpforo_ai_reindex_all' ) ); ?>">
402 <span class="dashicons dashicons-update"></span>
403 <?php
404 if ( $all_indexed ) {
405 printf(
406 __( 'Re-Index All %s Topics', 'wpforo' ),
407 number_format( $total_topics_count )
408 );
409 } elseif ( $has_enough_credits ) {
410 printf(
411 __( 'Index Remaining %s Topics', 'wpforo' ),
412 number_format( $remaining_to_index )
413 );
414 } else {
415 printf(
416 __( 'Index %s Topics (Using All Credits)', 'wpforo' ),
417 number_format( $can_index_count )
418 );
419 }
420 ?>
421 </button>
422 <!-- Stop button: hidden when indexing is NOT in progress -->
423 <button type="button" class="button button-secondary button-hero wpforo-ai-stop-indexing" style="<?php echo ! $has_pending_jobs ? 'display:none;' : ''; ?>" data-confirm="<?php esc_attr_e( 'This will stop all pending indexing jobs. Continue?', 'wpforo' ); ?>" data-nonce="<?php echo esc_attr( wp_create_nonce( 'wpforo_ai_stop_indexing' ) ); ?>">
424 <span class="dashicons dashicons-controls-pause"></span>
425 <?php _e( 'Stop Indexing', 'wpforo' ); ?>
426 </button>
427 <?php else : ?>
428 <button type="button" class="button button-primary button-hero" disabled>
429 <span class="dashicons dashicons-update"></span>
430 <?php _e( 'Index All Topics', 'wpforo' ); ?>
431 </button>
432 <?php endif; ?>
433 </div>
434
435
436 <!-- Indexing breakdown loaded via AJAX (cached 1 day) -->
437 <div id="wpforo-ai-indexing-breakdown-container"
438 data-loading-text="<?php esc_attr_e( 'Loading...', 'wpforo' ); ?>"
439 data-excluded-text="<?php esc_attr_e( '%s topics are excluded from indexing', 'wpforo' ); ?>"
440 data-intro-text="<?php esc_attr_e( 'The following topics are automatically excluded from AI indexing:', 'wpforo' ); ?>"
441 data-private-text="<?php esc_attr_e( 'private topics - these are only visible to their authors', 'wpforo' ); ?>"
442 data-unapproved-text="<?php esc_attr_e( 'unapproved topics - these will be indexed once approved by moderators', 'wpforo' ); ?>"
443 data-note-text="<?php esc_attr_e( 'Private topics are never indexed to protect user privacy. Unapproved topics will be automatically indexed when approved.', 'wpforo' ); ?>">
444 </div>
445 </div>
446
447 <div class="wpforo-ai-section-divider">
448 <span><?php _e( 'Or select specific content to index:', 'wpforo' ); ?></span>
449 </div>
450
451 <div class="wpforo-ai-ingest-grid">
452 <!-- Forum Selection Column -->
453 <div class="wpforo-ai-ingest-column">
454 <h3><?php _e( 'Index Topics by Forums', 'wpforo' ); ?></h3>
455
456 <form method="post" action="" class="wpforo-ai-forum-ingest-form">
457 <?php wp_nonce_field( 'wpforo_ai_forum_ingest' ); ?>
458 <input type="hidden" name="wpforo_ai_action" value="forum_ingest">
459
460 <!-- Date Range Filter (optional) -->
461 <div class="form-row wpforo-ai-date-range wpforo-ai-forum-date-range">
462 <label><?php _e( 'Date Range (optional)', 'wpforo' ); ?></label>
463 <div class="wpforo-ai-date-inputs">
464 <input type="date" id="forum-date-from" name="date_from" class="regular-text" placeholder="<?php esc_attr_e( 'From', 'wpforo' ); ?>">
465 <span class="wpforo-ai-date-separator"><?php _e( 'to', 'wpforo' ); ?></span>
466 <input type="date" id="forum-date-to" name="date_to" class="regular-text" placeholder="<?php esc_attr_e( 'To', 'wpforo' ); ?>">
467 </div>
468 </div>
469
470 <p class="description">
471 <?php _e( 'Select one or more forums to index their topics. Use date range to filter by topic creation date.', 'wpforo' ); ?>
472 <br>
473 <em><?php _e( 'Up to 10,000 not-yet-indexed topics are queued per click (newest first). Private and unapproved topics are always skipped. Click again after processing completes to continue with the remaining topics.', 'wpforo' ); ?></em>
474 </p>
475
476 <?php
477 // Get all forums and sort hierarchically
478 // (get_forums() orders by `order` field only, not by parent hierarchy)
479 $all_forums = WPF()->forum->get_forums();
480
481 // Sort forums hierarchically: parents first, then children under each parent
482 $forums_by_parent = [];
483 foreach ( $all_forums as $forum ) {
484 $parent = (int) ( $forum['parentid'] ?? 0 );
485 if ( ! isset( $forums_by_parent[ $parent ] ) ) {
486 $forums_by_parent[ $parent ] = [];
487 }
488 $forums_by_parent[ $parent ][] = $forum;
489 }
490 // Sort children by order within each parent group
491 foreach ( $forums_by_parent as &$children ) {
492 usort( $children, function( $a, $b ) {
493 return (int) ( $a['order'] ?? 0 ) - (int) ( $b['order'] ?? 0 );
494 } );
495 }
496 unset( $children );
497 // Build flat list by walking the tree
498 $sorted_forums = [];
499 $add_forums_recursive = function( $parentid ) use ( &$add_forums_recursive, &$sorted_forums, &$forums_by_parent ) {
500 if ( ! isset( $forums_by_parent[ $parentid ] ) ) {
501 return;
502 }
503 foreach ( $forums_by_parent[ $parentid ] as $forum ) {
504 $sorted_forums[] = $forum;
505 $add_forums_recursive( (int) $forum['forumid'] );
506 }
507 };
508 $add_forums_recursive( 0 );
509 $all_forums = $sorted_forums;
510 // Get indexed counts from AI backend
511 $indexed_counts = wpforo_ai_get_indexed_counts_by_forum();
512
513 // Get all forum topic counts in a single GROUP BY query
514 // instead of calling WPF()->topic->get_count() per forum (N+1 problem)
515 // Cache for 5 minutes to avoid heavy GROUP BY on large forums
516 $_ftc_cache_key = 'wpforo_ai_ftc_' . $current_boardid;
517 $forum_topic_counts = get_transient( $_ftc_cache_key );
518 if ( false === $forum_topic_counts ) {
519 $forum_topic_counts = [];
520 $_ftc_rows = WPF()->db->get_results(
521 "SELECT `forumid`, COUNT(*) as `cnt` FROM `" . WPF()->tables->topics . "` GROUP BY `forumid`",
522 ARRAY_A
523 );
524 if ( $_ftc_rows ) {
525 foreach ( $_ftc_rows as $_ftc_row ) {
526 $forum_topic_counts[ (int) $_ftc_row['forumid'] ] = (int) $_ftc_row['cnt'];
527 }
528 }
529 set_transient( $_ftc_cache_key, $forum_topic_counts, 5 * MINUTE_IN_SECONDS );
530 }
531
532 if ( ! empty( $all_forums ) ) :
533 ?>
534 <div class="wpforo-ai-forum-checklist">
535 <?php
536 foreach ( $all_forums as $forum ) {
537 $forum_id = isset( $forum['forumid'] ) ? (int) $forum['forumid'] : 0;
538 $forum_title = isset( $forum['title'] ) ? esc_html( $forum['title'] ) : '';
539 $parent_id = isset( $forum['parentid'] ) ? (int) $forum['parentid'] : 0;
540 $is_cat = isset( $forum['is_cat'] ) ? (int) $forum['is_cat'] : 0;
541
542 // Get topic count for this forum (from pre-fetched GROUP BY)
543 $topic_count = isset( $forum_topic_counts[ $forum_id ] ) ? $forum_topic_counts[ $forum_id ] : 0;
544
545 // Get indexed count for this forum
546 $indexed_count = isset( $indexed_counts[ $forum_id ] ) ? $indexed_counts[ $forum_id ] : 0;
547
548 // Add class for child forums
549 $item_class = $parent_id > 0 ? 'wpforo-ai-forum-child' : 'wpforo-ai-forum-parent';
550 if ( $is_cat ) {
551 $item_class .= ' wpforo-ai-forum-category';
552 }
553 ?>
554 <label class="wpforo-ai-forum-checkbox-item <?php echo esc_attr( $item_class ); ?>">
555 <input
556 type="checkbox"
557 name="forum_ids[]"
558 value="<?php echo esc_attr( $forum_id ); ?>"
559 data-parent-id="<?php echo esc_attr( $parent_id ); ?>"
560 data-forum-id="<?php echo esc_attr( $forum_id ); ?>"
561 data-is-category="<?php echo esc_attr( $is_cat ); ?>"
562 >
563 <?php echo $forum_title; ?>
564 <?php if ( ! $is_cat || $parent_id > 0 ) : ?>
565 <span class="wpforo-ai-forum-info">
566 (<?php echo number_format_i18n( $topic_count ); ?> topics / <?php echo number_format_i18n( $indexed_count ); ?> indexed)
567 </span>
568 <?php endif; ?>
569 </label>
570 <?php
571 }
572 ?>
573 </div>
574
575 <div class="wpforo-ai-forum-actions">
576 <button type="button" class="button button-small wpforo-ai-select-all-forums">
577 <?php _e( 'Select All', 'wpforo' ); ?>
578 </button>
579 <button type="button" class="button button-small wpforo-ai-deselect-all-forums">
580 <?php _e( 'Deselect All', 'wpforo' ); ?>
581 </button>
582 </div>
583
584 <div class="form-actions">
585 <button type="submit" name="wpforo_ai_action" value="clear_forum_index" class="button button-large wpforo-ai-clear-forum-btn" onclick="return confirm('<?php echo esc_js( __( 'This will permanently delete all indexed content from the selected forums. Topics will need to be re-indexed to appear in AI search results. Continue?', 'wpforo' ) ); ?>');">
586 <span class="dashicons dashicons-trash"></span>
587 <?php _e( 'Clear Selected Forum Index', 'wpforo' ); ?>
588 </button>
589 <button type="submit" class="button button-primary button-large" onclick="return confirm('<?php echo esc_js( __( 'This will queue up to 10,000 not-yet-indexed topics from the selected forums for background indexing. Private and unapproved topics are automatically excluded. Continue?', 'wpforo' ) ); ?>');">
590 <span class="dashicons dashicons-upload"></span>
591 <?php _e( 'Index Selected Forums', 'wpforo' ); ?>
592 </button>
593 </div>
594 <?php else : ?>
595 <p class="wpforo-ai-no-forums"><?php _e( 'No forums found.', 'wpforo' ); ?></p>
596 <?php endif; ?>
597 </form>
598 </div>
599
600 <!-- Custom Indexing Column -->
601 <div class="wpforo-ai-ingest-column wpforo-ai-filtered-index">
602 <h3><?php _e( 'Custom Indexing', 'wpforo' ); ?></h3>
603 <p class="description">
604 <?php _e( 'Index topics using custom filters. Combine date range with other filters.', 'wpforo' ); ?>
605 </p>
606
607 <form method="post" action="" class="wpforo-ai-filtered-ingest-form">
608 <?php wp_nonce_field( 'wpforo_ai_filtered_ingest' ); ?>
609 <input type="hidden" name="wpforo_ai_action" value="filtered_ingest">
610
611 <!-- Date Range Filter -->
612 <div class="form-row wpforo-ai-date-range">
613 <label><?php _e( 'Date Range', 'wpforo' ); ?></label>
614 <div class="wpforo-ai-date-inputs">
615 <input type="date" id="filter-date-from" name="date_from" class="regular-text" placeholder="<?php esc_attr_e( 'From', 'wpforo' ); ?>">
616 <span class="wpforo-ai-date-separator"><?php _e( 'to', 'wpforo' ); ?></span>
617 <input type="date" id="filter-date-to" name="date_to" class="regular-text" placeholder="<?php esc_attr_e( 'To', 'wpforo' ); ?>">
618 </div>
619 <p class="description">
620 <?php _e( 'Filter by topic creation date. Can be combined with any other filter.', 'wpforo' ); ?>
621 </p>
622 </div>
623
624 <!-- Topic Tags Filter -->
625 <div class="form-row">
626 <label for="filter-tags">
627 <?php _e( 'Topic Tags', 'wpforo' ); ?>
628 </label>
629 <input type="text" id="filter-tags" name="topic_tags" class="regular-text wpforo-ai-tags-input" placeholder="<?php esc_attr_e( 'Start typing to search tags...', 'wpforo' ); ?>" autocomplete="off">
630 <p class="description">
631 <?php _e( 'Type to search and select tags (comma-separated). Cannot be combined with User IDs filter.', 'wpforo' ); ?>
632 </p>
633 </div>
634
635 <!-- Topic User IDs Filter -->
636 <div class="form-row">
637 <label for="filter-userids">
638 <?php _e( 'Topic Author User IDs', 'wpforo' ); ?>
639 </label>
640 <input type="text" id="filter-userids" name="user_ids" class="regular-text" placeholder="<?php esc_attr_e( 'e.g., 1, 5, 12', 'wpforo' ); ?>">
641 <p class="description">
642 <?php _e( 'Comma-separated user IDs. Cannot be combined with Tags filter.', 'wpforo' ); ?>
643 </p>
644 </div>
645
646 <hr class="wpforo-ai-filter-divider">
647
648 <!-- Specific Topics Filter (independent) -->
649 <div class="form-row">
650 <label for="filter-topics">
651 <?php _e( 'Specific Topic IDs or URLs', 'wpforo' ); ?>
652 </label>
653 <textarea id="filter-topics" name="topic_inputs" class="large-text" rows="3" placeholder="<?php esc_attr_e( "e.g., 123, 456\nhttps://yourforum.com/topic/sample-topic/", 'wpforo' ); ?>"></textarea>
654 <p class="description">
655 <?php _e( 'Comma or newline separated. This filter works independently - ignores other filters above.', 'wpforo' ); ?>
656 </p>
657 </div>
658
659 <div class="form-actions">
660 <button type="submit" class="button button-primary button-large">
661 <span class="dashicons dashicons-upload"></span>
662 <?php _e( 'Index Filtered Topics', 'wpforo' ); ?>
663 </button>
664 </div>
665 </form>
666 </div>
667 </div>
668
669 <!-- Quick Actions -->
670 <div class="wpforo-ai-quick-actions">
671 <button type="button" class="button wpforo-ai-reindex-all" data-confirm="<?php esc_attr_e( 'This will re-index all topics. Continue?', 'wpforo' ); ?>" data-nonce="<?php echo esc_attr( wp_create_nonce( 'wpforo_ai_reindex_all' ) ); ?>">
672 <span class="dashicons dashicons-update"></span>
673 <?php _e( 'Re-Index All Topics', 'wpforo' ); ?>
674 </button>
675 <?php
676 // Show Re-Index Topic Images button for Professional+ plans
677 $has_image_reindex_plan = in_array( $plan, [ 'professional', 'business', 'enterprise' ], true );
678 if ( $has_image_reindex_plan ) :
679 ?>
680 <button type="button" class="button wpforo-ai-reindex-images" data-confirm="<?php esc_attr_e( 'This will re-index all topics that contain images. Topics without images will not be affected. Continue?', 'wpforo' ); ?>" data-nonce="<?php echo esc_attr( wp_create_nonce( 'wpforo_ai_reindex_images' ) ); ?>">
681 <span class="dashicons dashicons-format-image"></span>
682 <?php _e( 'Re-Index Topic Images', 'wpforo' ); ?>
683 </button>
684 <?php endif; ?>
685 <button type="button" class="button wpforo-ai-cleanup-session" data-scope="forum" data-confirm="<?php esc_attr_e( 'Reset stuck forum indexing session? This clears pending queues, cron jobs and cached state for BOTH local and cloud modes. Already-indexed topics are NOT affected.', 'wpforo' ); ?>" title="<?php esc_attr_e( 'Clears pending indexing queues, WP-Cron jobs, session locks and status caches. Use this if indexing is stuck on "Indexing..." but nothing is progressing. Works for both local and cloud storage modes. Does not delete any indexed data.', 'wpforo' ); ?>">
686 <span class="dashicons dashicons-update-alt"></span>
687 <?php _e( 'Cleanup Indexing Session', 'wpforo' ); ?>
688 </button>
689 <button type="button" class="button button-link-delete wpforo-ai-clear-database" data-confirm="<?php esc_attr_e( 'Are you sure? This will delete ALL indexed data and cannot be undone!', 'wpforo' ); ?>" data-nonce="<?php echo esc_attr( wp_create_nonce( 'wpforo_ai_clear_database' ) ); ?>">
690 <span class="dashicons dashicons-trash"></span>
691 <?php _e( 'Clear Forum Index', 'wpforo' ); ?>
692 </button>
693 </div>
694 </div>
695 </div>
696
697 <!-- Chunking Configuration Box -->
698 <div class="wpforo-ai-box wpforo-ai-chunking-config-box">
699 <div class="wpforo-ai-box-header">
700 <h2><?php _e( 'Chunking Configuration', 'wpforo' ); ?></h2>
701 </div>
702 <div class="wpforo-ai-box-body">
703 <p class="wpforo-ai-description">
704 <?php _e( 'Configure how forum content is split into chunks for vector indexing. These settings affect search quality and cost.', 'wpforo' ); ?>
705 </p>
706
707 <form method="post" action="" class="wpforo-ai-chunking-form">
708 <?php wp_nonce_field( 'wpforo_ai_save_chunking_config' ); ?>
709 <input type="hidden" name="wpforo_ai_action" value="save_chunking_config">
710
711 <div class="wpforo-ai-chunking-settings">
712 <?php
713 // Get saved values
714 $saved_chunk_size = (int) wpforo_get_option( 'ai_chunk_size', 512 );
715 $saved_overlap_percent = (int) wpforo_get_option( 'ai_overlap_percent', 20 );
716 $saved_pagination_size = (int) wpforo_get_option( 'ai_pagination_size', 20 );
717 ?>
718
719 <div class="chunking-setting-row">
720 <div class="setting-label">
721 <label for="wpforo-ai-chunk-size">
722 <?php _e( 'Chunk Size (tokens)', 'wpforo' ); ?>
723 </label>
724 <p class="description">
725 <?php _e( 'Number of tokens per chunk. 1 token ≈ 4 characters. Smaller chunks give more precise search results. Larger chunks provide more context but may include less relevant content. Default: 512 (~2000 characters)', 'wpforo' ); ?>
726 </p>
727 </div>
728 <div class="setting-control">
729 <input type="number" id="wpforo-ai-chunk-size" name="chunk_size" class="regular-text" value="<?php echo esc_attr( $saved_chunk_size ); ?>" min="100" max="1024">
730 <span class="input-hint"><?php _e( 'Range: 100-1024 tokens', 'wpforo' ); ?></span>
731 </div>
732 </div>
733
734 <div class="chunking-setting-row">
735 <div class="setting-label">
736 <label for="wpforo-ai-overlap-percent">
737 <?php _e( 'Overlap Percentage (%)', 'wpforo' ); ?>
738 </label>
739 <p class="description">
740 <?php _e( 'Percentage of overlap between consecutive chunks. Helps maintain context across chunk boundaries. Default: 20%', 'wpforo' ); ?>
741 </p>
742 </div>
743 <div class="setting-control">
744 <input type="number" id="wpforo-ai-overlap-percent" name="overlap_percent" class="regular-text" value="<?php echo esc_attr( $saved_overlap_percent ); ?>" min="5" max="50">
745 <span class="input-hint"><?php _e( 'Range: 5-50%', 'wpforo' ); ?></span>
746 </div>
747 </div>
748
749 <div class="chunking-setting-row">
750 <div class="setting-label">
751 <label for="wpforo-ai-pagination-size">
752 <?php _e( 'Topics Per Batch', 'wpforo' ); ?>
753 </label>
754 <p class="description">
755 <?php _e( 'Number of topics to process per batch during indexing. Lower values are more reliable but slower. Higher values are faster but may timeout on slower servers. Default: 10', 'wpforo' ); ?>
756 </p>
757 </div>
758 <div class="setting-control">
759 <input type="number" id="wpforo-ai-pagination-size" name="pagination_size" class="regular-text" value="<?php echo esc_attr( $saved_pagination_size ); ?>" min="1" max="50">
760 <span class="input-hint"><?php _e( 'Range: 1-50', 'wpforo' ); ?></span>
761 </div>
762 </div>
763
764 <div class="chunking-info-notice">
765 <span class="dashicons dashicons-info"></span>
766 <div class="notice-content">
767 <strong><?php _e( 'How this affects indexing:', 'wpforo' ); ?></strong>
768 <ul>
769 <li><?php _e( 'Larger chunk size = better context preservation, fewer vectors.', 'wpforo' ); ?></li>
770 <li><?php _e( 'Higher overlap = better search quality at chunk boundaries.', 'wpforo' ); ?></li>
771 <li><?php _e( 'More topics per step = faster re-indexing but uses more memory. Reduce if you experience memory issues.', 'wpforo' ); ?></li>
772 <li><?php _e( 'These settings will be used for all future indexing operations.', 'wpforo' ); ?></li>
773 </ul>
774 </div>
775 </div>
776
777 <div class="chunking-actions">
778 <button type="submit" class="button button-primary button-large">
779 <span class="dashicons dashicons-saved"></span>
780 <?php _e( 'Save Configuration', 'wpforo' ); ?>
781 </button>
782 </div>
783 </div>
784 </form>
785 </div>
786 </div>
787
788 <!-- Search Test Box (only show if topics are indexed) -->
789 <?php if ( $total_indexed > 0 ) : ?>
790 <div class="wpforo-ai-box wpforo-ai-search-test-box">
791 <div class="wpforo-ai-box-header">
792 <h2><?php _e( 'Test Semantic Search', 'wpforo' ); ?></h2>
793 </div>
794 <div class="wpforo-ai-box-body">
795 <p class="wpforo-ai-description">
796 <?php _e( 'Test the AI semantic search with your indexed topics. Results include metadata for generating topic and post URLs.', 'wpforo' ); ?>
797 </p>
798
799 <form id="wpforo-ai-search-test-form" class="wpforo-ai-search-test-form">
800 <?php wp_nonce_field( 'wpforo_ai_features_nonce' ); ?>
801 <div class="form-row">
802 <label for="search-query">
803 <?php _e( 'Search Query', 'wpforo' ); ?>
804 </label>
805 <input type="text" id="search-query" name="query" class="large-text" placeholder="<?php esc_attr_e( 'e.g., how to reset password', 'wpforo' ); ?>" required>
806 <p class="description">
807 <?php _e( 'Enter a natural language search query to test semantic search.', 'wpforo' ); ?>
808 </p>
809 </div>
810
811 <div class="form-row">
812 <label for="search-limit">
813 <?php _e( 'Number of Results', 'wpforo' ); ?>
814 </label>
815 <input type="number" id="search-limit" name="limit" class="small-text" value="1" min="1" max="20">
816 <p class="description">
817 <?php _e( 'Maximum number of search results to return (1-20).', 'wpforo' ); ?>
818 </p>
819 </div>
820
821 <div class="form-actions">
822 <button type="submit" class="button button-primary" id="search-test-btn">
823 <span class="dashicons dashicons-search"></span>
824 <?php _e( 'Test Search', 'wpforo' ); ?>
825 </button>
826 <span class="spinner" style="float:none; margin: 0 10px;"></span>
827 </div>
828 </form>
829
830 <!-- Search Results Display -->
831 <div id="search-test-results" class="wpforo-ai-search-results" style="display:none;">
832 <h3><?php _e( 'Search Results', 'wpforo' ); ?></h3>
833 <div id="search-results-content"></div>
834 </div>
835 </div>
836 </div>
837 <?php endif; ?>
838
839
840 <!-- Indexing Queue Info (if indexing) -->
841 <?php if ( $is_indexing ) : ?>
842 <div class="wpforo-ai-box wpforo-ai-queue-info-box">
843 <div class="wpforo-ai-box-header">
844 <h2><?php _e( 'Indexing Queue', 'wpforo' ); ?></h2>
845 </div>
846 <div class="wpforo-ai-box-body">
847 <div class="queue-info" id="wpforo-ai-queue-info">
848 <div class="queue-stat">
849 <span class="dashicons dashicons-hourglass"></span>
850 <span id="queue-pending">-</span> <?php _e( 'pending', 'wpforo' ); ?>
851 </div>
852 <div class="queue-stat">
853 <span class="dashicons dashicons-yes-alt"></span>
854 <span id="queue-completed">-</span> <?php _e( 'completed', 'wpforo' ); ?>
855 </div>
856 <div class="queue-stat">
857 <span class="dashicons dashicons-warning"></span>
858 <span id="queue-failed">-</span> <?php _e( 'failed', 'wpforo' ); ?>
859 </div>
860 </div>
861 <p class="description">
862 <?php _e( 'The queue updates automatically every 30 seconds while indexing is in progress.', 'wpforo' ); ?>
863 </p>
864 </div>
865 </div>
866 <?php endif; ?>
867
868 </div>
869
870 <!-- AI Content Indexing Tab Specific JavaScript -->
871 <script type="text/javascript">
872 jQuery(document).ready(function($) {
873 // RAG-specific functionality (polling and auto-refresh)
874 if (typeof window.WpForoAI !== 'undefined') {
875 // Start polling if indexing or processing is active
876 <?php if ( $is_processing ) : ?>
877 if (typeof WpForoAI.startRAGStatusPolling === 'function') {
878 WpForoAI.startRAGStatusPolling();
879 }
880 <?php endif; ?>
881 }
882
883 // Auto-refresh page only when actively processing (cron is due or batch running).
884 // Queued topics scheduled for future (e.g., 1h/24h auto-indexing) should NOT trigger refresh.
885 <?php if ( $pending_jobs_info['is_actively_processing'] ) : ?>
886 setTimeout(function() {
887 window.location.reload();
888 }, 30000); // Refresh after 30 seconds
889 <?php endif; ?>
890 });
891 </script>
892 <?php
893 }
894