| 1 |
<?php |
| 2 |
|
| 3 |
|
| 4 |
if (!defined('ABSPATH')) { |
| 5 |
exit; |
| 6 |
} |
| 7 |
|
| 8 |
/** |
| 9 |
* N-Gram based filtering for spell checker optimization. |
| 10 |
* |
| 11 |
* This class provides N-gram extraction, similarity computation, and caching |
| 12 |
* to reduce Levenshtein distance calculations from 100-300 calls to <50 calls |
| 13 |
* on large sites by pre-filtering candidates based on character overlap. |
| 14 |
* |
| 15 |
* Architecture: Database-backed N-gram cache for scalability. |
| 16 |
* - Pre-computes N-grams for all existing pages (background process) |
| 17 |
* - Computes N-grams for 404 URL only in real-time (~0.1ms) |
| 18 |
* - Uses Dice coefficient similarity to filter candidates (50-100ms) |
| 19 |
* - Reduces Levenshtein calls by 5-10x on large sites |
| 20 |
*/ |
| 21 |
class ABJ_404_Solution_NGramFilter { |
| 22 |
|
| 23 |
/** Maximum entries to load from N-gram cache to prevent memory exhaustion. |
| 24 |
* JSON decode of N-gram data is memory-intensive; 1000 entries is safe for 128MB limit. */ |
| 25 |
const CACHE_LOAD_LIMIT = 1000; |
| 26 |
|
| 27 |
/** Cache TTL for coverage ratio transient (seconds). |
| 28 |
* Prevents per-request COUNT(*) queries on large sites under 404 bursts. */ |
| 29 |
const COVERAGE_RATIO_CACHE_TTL = 300; // 5 minutes |
| 30 |
|
| 31 |
/** TTL for coverage version transient (seconds). |
| 32 |
* Version persists across ratio TTL cycles; 1 day is sufficient. */ |
| 33 |
const COVERAGE_VERSION_TTL = 86400; // 1 day |
| 34 |
|
| 35 |
/** Transient key for coverage ratio cache version. |
| 36 |
* Version is a timestamp; cached ratios with older timestamps are stale. */ |
| 37 |
const COVERAGE_VERSION_KEY = 'abj404_ngram_coverage_version'; |
| 38 |
|
| 39 |
/** Transient key for coverage ratio cache data. */ |
| 40 |
const COVERAGE_RATIO_KEY = 'abj404_ngram_coverage_ratio'; |
| 41 |
|
| 42 |
/** @var self|null */ |
| 43 |
private static $instance = null; |
| 44 |
|
| 45 |
/** @var ABJ_404_Solution_DataAccess */ |
| 46 |
private $dao; |
| 47 |
|
| 48 |
/** @var ABJ_404_Solution_Logging */ |
| 49 |
private $logger; |
| 50 |
|
| 51 |
/** @var ABJ_404_Solution_Functions */ |
| 52 |
private $f; |
| 53 |
|
| 54 |
/** @var int|null Per-request memoized N-gram cache count */ |
| 55 |
private $ngramCountMemo = null; |
| 56 |
|
| 57 |
/** @var array<string, mixed>|null Per-request memoized coverage ratio data */ |
| 58 |
private $coverageRatioMemo = null; |
| 59 |
|
| 60 |
/** |
| 61 |
* Invalidate coverage ratio caches (transient and per-request memos). |
| 62 |
* Call this whenever N-gram or permalink counts change, including after |
| 63 |
* TRUNCATE operations during cache rebuilds. |
| 64 |
* |
| 65 |
* Uses timestamp-based versioning: sets version to current time(). |
| 66 |
* Cached ratios with older timestamps are stale. This approach is: |
| 67 |
* - Overflow-safe: no accumulating counter |
| 68 |
* - Race-safe: concurrent invalidations both write current time |
| 69 |
*/ |
| 70 |
/** @return void */ |
| 71 |
public function invalidateCoverageCaches() { |
| 72 |
// Set version to current timestamp (race-safe: concurrent writes both invalidate) |
| 73 |
set_transient(self::COVERAGE_VERSION_KEY, time(), self::COVERAGE_VERSION_TTL); |
| 74 |
|
| 75 |
// Also delete the ratio transient to force immediate recompute |
| 76 |
delete_transient(self::COVERAGE_RATIO_KEY); |
| 77 |
|
| 78 |
// Clear per-request memos |
| 79 |
$this->ngramCountMemo = null; |
| 80 |
$this->coverageRatioMemo = null; |
| 81 |
} |
| 82 |
|
| 83 |
/** |
| 84 |
* Check if the N-gram cache is initialized (multisite-aware). |
| 85 |
* |
| 86 |
* On multisite, checks both get_site_option() (network activation) and |
| 87 |
* get_option() (per-site activation) since we can't reliably determine |
| 88 |
* activation mode on frontend requests where is_plugin_active_for_network() |
| 89 |
* isn't available. |
| 90 |
* |
| 91 |
* @return bool True if cache is initialized |
| 92 |
*/ |
| 93 |
public function isCacheInitialized() { |
| 94 |
$optionName = 'abj404_ngram_cache_initialized'; |
| 95 |
|
| 96 |
if (is_multisite()) { |
| 97 |
// Check site option first (network activation stores here) |
| 98 |
// Then fall back to per-site option (per-site activation stores here) |
| 99 |
// This handles both activation modes without requiring admin functions |
| 100 |
$siteValue = get_site_option($optionName); |
| 101 |
if ($siteValue === '1') { |
| 102 |
return true; |
| 103 |
} |
| 104 |
// Fall through to check per-site option |
| 105 |
} |
| 106 |
|
| 107 |
return get_option($optionName) === '1'; |
| 108 |
} |
| 109 |
|
| 110 |
/** |
| 111 |
* Constructor with dependency injection. |
| 112 |
* |
| 113 |
* @param ABJ_404_Solution_DataAccess|null $dataAccess Data access layer |
| 114 |
* @param ABJ_404_Solution_Logging|null $logging Logging service |
| 115 |
* @param ABJ_404_Solution_Functions|null $functions String utilities |
| 116 |
*/ |
| 117 |
public function __construct($dataAccess = null, $logging = null, $functions = null) { |
| 118 |
// Use injected dependencies or fall back to getInstance() for backward compatibility |
| 119 |
$this->dao = $dataAccess !== null ? $dataAccess : abj_service('data_access'); |
| 120 |
$this->logger = $logging !== null ? $logging : abj_service('logging'); |
| 121 |
$this->f = $functions !== null ? $functions : abj_service('functions'); |
| 122 |
} |
| 123 |
|
| 124 |
/** @return self */ |
| 125 |
public static function getInstance() { |
| 126 |
if (self::$instance == null) { |
| 127 |
self::$instance = new ABJ_404_Solution_NGramFilter(); |
| 128 |
} |
| 129 |
|
| 130 |
return self::$instance; |
| 131 |
} |
| 132 |
|
| 133 |
/** |
| 134 |
* Extract N-grams from a URL string. |
| 135 |
* |
| 136 |
* Generates both bigrams (n=2) and trigrams (n=3) for optimal accuracy. |
| 137 |
* Research shows using both provides better typo detection than either alone. |
| 138 |
* |
| 139 |
* @param string $url The URL to extract N-grams from |
| 140 |
* @param array<int, int> $ngramSizes Array of N-gram sizes to extract (default: [2, 3]) |
| 141 |
* @return array{bi: array<int, string>, tri: array<int, string>} |
| 142 |
* |
| 143 |
* Example: |
| 144 |
* Input: "product" |
| 145 |
* Output: [ |
| 146 |
* 'bi' => ['pr', 'ro', 'od', 'du', 'uc', 'ct'], |
| 147 |
* 'tri' => ['pro', 'rod', 'odu', 'duc', 'uct'] |
| 148 |
* ] |
| 149 |
*/ |
| 150 |
public function extractNGrams($url, $ngramSizes = [2, 3]) { |
| 151 |
if (empty($url)) { |
| 152 |
return ['bi' => [], 'tri' => []]; |
| 153 |
} |
| 154 |
|
| 155 |
// Normalize: lowercase and use mbstring for UTF-8 support |
| 156 |
$url = $this->f->strtolower($url); |
| 157 |
|
| 158 |
// Limit URL length to prevent excessive N-gram generation |
| 159 |
// Most real URLs are < 200 chars. Limiting to 500 prevents: |
| 160 |
// - Memory exhaustion (4000+ N-grams for 2083 char URLs) |
| 161 |
// - Slow JSON encoding/decoding |
| 162 |
// - Database bloat |
| 163 |
$maxLength = 500; |
| 164 |
$originalLength = $this->f->strlen($url); |
| 165 |
if ($originalLength > $maxLength) { |
| 166 |
$this->logger->infoMessage("WARNING: URL too long for N-gram extraction: {$originalLength} chars, truncating to {$maxLength}. URL: " . $this->f->substr($url, 0, 100) . "..."); |
| 167 |
$url = $this->f->substr($url, 0, $maxLength); |
| 168 |
} |
| 169 |
|
| 170 |
$result = []; |
| 171 |
$length = $this->f->strlen($url); |
| 172 |
|
| 173 |
foreach ($ngramSizes as $n) { |
| 174 |
$ngrams = []; |
| 175 |
|
| 176 |
// Extract all N-grams of size n |
| 177 |
for ($i = 0; $i <= $length - $n; $i++) { |
| 178 |
$ngram = $this->f->substr($url, $i, $n); |
| 179 |
// Use array keys for automatic deduplication |
| 180 |
$ngrams[$ngram] = true; |
| 181 |
} |
| 182 |
|
| 183 |
// Store under 'bi' for n=2, 'tri' for n=3 |
| 184 |
$key = ($n == 2) ? 'bi' : 'tri'; |
| 185 |
// Convert keys to strings to prevent PHP from converting numeric strings to integers |
| 186 |
$result[$key] = array_map('strval', array_keys($ngrams)); |
| 187 |
} |
| 188 |
|
| 189 |
/** @var array{bi: array<int, string>, tri: array<int, string>} $result */ |
| 190 |
return $result; |
| 191 |
} |
| 192 |
|
| 193 |
/** |
| 194 |
* Compute Dice coefficient similarity between two N-gram sets. |
| 195 |
* |
| 196 |
* Dice coefficient: 2 * |intersection| / (|set1| + |set2|) |
| 197 |
* Range: 0.0 (no overlap) to 1.0 (identical) |
| 198 |
* |
| 199 |
* Threshold correlation: |
| 200 |
* - 0.4 = ~30% edit distance (recommended) |
| 201 |
* - 0.5 = ~20% edit distance |
| 202 |
* - 0.6 = ~10% edit distance |
| 203 |
* |
| 204 |
* @param array{bi?: array<int, string>, tri?: array<int, string>} $ngrams1 First N-gram set |
| 205 |
* @param array{bi?: array<int, string>, tri?: array<int, string>} $ngrams2 Second N-gram set |
| 206 |
* @return float Similarity score between 0.0 and 1.0 |
| 207 |
*/ |
| 208 |
public function diceCoefficient($ngrams1, $ngrams2) { |
| 209 |
// Combine bigrams and trigrams for similarity computation |
| 210 |
$set1 = array_merge( |
| 211 |
isset($ngrams1['bi']) ? $ngrams1['bi'] : [], |
| 212 |
isset($ngrams1['tri']) ? $ngrams1['tri'] : [] |
| 213 |
); |
| 214 |
$set2 = array_merge( |
| 215 |
isset($ngrams2['bi']) ? $ngrams2['bi'] : [], |
| 216 |
isset($ngrams2['tri']) ? $ngrams2['tri'] : [] |
| 217 |
); |
| 218 |
|
| 219 |
// Handle empty sets |
| 220 |
if (empty($set1) || empty($set2)) { |
| 221 |
return 0.0; |
| 222 |
} |
| 223 |
|
| 224 |
// Convert to associative arrays for fast lookup |
| 225 |
$set1 = array_flip($set1); |
| 226 |
$set2 = array_flip($set2); |
| 227 |
|
| 228 |
// Count intersection |
| 229 |
$intersection = count(array_intersect_key($set1, $set2)); |
| 230 |
|
| 231 |
// Dice coefficient: 2 * |intersection| / (|set1| + |set2|) |
| 232 |
// Defensive: guard denominator even though empty() check above should guarantee > 0. |
| 233 |
$denominator = count($set1) + count($set2); |
| 234 |
$dice = ($denominator > 0) ? (2.0 * $intersection) / $denominator : 0.0; |
| 235 |
|
| 236 |
return $dice; |
| 237 |
} |
| 238 |
|
| 239 |
/** |
| 240 |
* Store N-grams for a page in the database. |
| 241 |
* |
| 242 |
* @param int $pageId The page/post ID |
| 243 |
* @param string $url Original URL |
| 244 |
* @param string $urlNormalized Normalized URL for matching |
| 245 |
* @param array<string, mixed> $ngrams N-gram data |
| 246 |
* @param string $type Entity type: 'post', 'page', 'category', 'tag' (default: 'post') |
| 247 |
* @param bool $skipInvalidation Skip cache invalidation (for bulk operations) |
| 248 |
* @return bool Success status |
| 249 |
*/ |
| 250 |
public function storeNGrams($pageId, $url, $urlNormalized, $ngrams, $type = 'post', $skipInvalidation = false) { |
| 251 |
// Input validation |
| 252 |
if (!is_numeric($pageId) || $pageId <= 0) { |
| 253 |
$this->logger->errorMessage("Invalid page ID for N-gram storage: " . var_export($pageId, true)); |
| 254 |
return false; |
| 255 |
} |
| 256 |
|
| 257 |
if (!is_array($ngrams) || !isset($ngrams['bi']) || !isset($ngrams['tri'])) { |
| 258 |
$this->logger->errorMessage("Invalid N-gram structure for page ID {$pageId}"); |
| 259 |
return false; |
| 260 |
} |
| 261 |
|
| 262 |
if (!is_array($ngrams['bi']) || !is_array($ngrams['tri'])) { |
| 263 |
$this->logger->errorMessage("Invalid N-gram array types for page ID {$pageId}"); |
| 264 |
return false; |
| 265 |
} |
| 266 |
|
| 267 |
$ngramJson = json_encode($ngrams); |
| 268 |
if ($ngramJson === false) { |
| 269 |
$this->logger->errorMessage("Failed to JSON encode N-grams for page ID {$pageId}"); |
| 270 |
return false; |
| 271 |
} |
| 272 |
|
| 273 |
$ngramCount = count($ngrams['bi']) + count($ngrams['tri']); |
| 274 |
|
| 275 |
$table = $this->dao->getPrefixedTableName('abj404_ngram_cache'); |
| 276 |
|
| 277 |
// Use REPLACE to handle updates (REPLACE = DELETE + INSERT). |
| 278 |
// Routed through DAO for centralized timeout/retry/recovery. |
| 279 |
$queryResult = $this->dao->queryAndGetResults( |
| 280 |
"REPLACE INTO {$table} (id, type, url, url_normalized, ngrams, ngram_count, last_updated) |
| 281 |
VALUES (%d, %s, %s, %s, %s, %d, %s)", |
| 282 |
['query_params' => [ |
| 283 |
(int)$pageId, |
| 284 |
$type, |
| 285 |
$url, |
| 286 |
$urlNormalized, |
| 287 |
$ngramJson, |
| 288 |
$ngramCount, |
| 289 |
current_time('mysql'), |
| 290 |
]] |
| 291 |
); |
| 292 |
|
| 293 |
$lastError = isset($queryResult['last_error']) && is_string($queryResult['last_error']) ? $queryResult['last_error'] : ''; |
| 294 |
if ($lastError !== '') { |
| 295 |
global $wpdb; |
| 296 |
$dbName = isset($wpdb->dbname) && is_string($wpdb->dbname) ? $wpdb->dbname : ''; |
| 297 |
// Enhanced error message with multisite context and table details |
| 298 |
$errorContext = sprintf( |
| 299 |
"Failed to store N-grams for page ID %d: %s, Table: %s, Prefix: %s, DB: %s", |
| 300 |
$pageId, |
| 301 |
$lastError, |
| 302 |
$table, |
| 303 |
$this->dao->getLowercasePrefix(), |
| 304 |
$dbName |
| 305 |
); |
| 306 |
|
| 307 |
// Add multisite context if applicable |
| 308 |
if (is_multisite()) { |
| 309 |
$errorContext .= sprintf(", Blog ID: %d", get_current_blog_id()); |
| 310 |
} |
| 311 |
|
| 312 |
if (!$this->dao->classifyAndHandleInfrastructureError($lastError)) { |
| 313 |
$this->logger->errorMessage($errorContext); |
| 314 |
} |
| 315 |
return false; |
| 316 |
} |
| 317 |
|
| 318 |
// Invalidate coverage ratio caches since N-gram count changed |
| 319 |
// (skip during bulk operations for efficiency) |
| 320 |
if (!$skipInvalidation) { |
| 321 |
$this->invalidateCoverageCaches(); |
| 322 |
} |
| 323 |
|
| 324 |
return true; |
| 325 |
} |
| 326 |
|
| 327 |
/** |
| 328 |
* Get N-grams for a specific page. |
| 329 |
* |
| 330 |
* @param int $pageId The page/post ID |
| 331 |
* @param string $type Entity type: 'post', 'page', 'category', 'tag' (default: 'post') |
| 332 |
* @return array{bi: array<int, string>, tri: array<int, string>}|null N-gram data or null if not found |
| 333 |
*/ |
| 334 |
public function getNGramsForPage($pageId, $type = 'post') { |
| 335 |
$table = $this->dao->getPrefixedTableName('abj404_ngram_cache'); |
| 336 |
|
| 337 |
$queryResult = $this->dao->queryAndGetResults( |
| 338 |
"SELECT ngrams FROM {$table} WHERE id = %d AND type = %s", |
| 339 |
['query_params' => [$pageId, $type]] |
| 340 |
); |
| 341 |
|
| 342 |
$rows = isset($queryResult['rows']) && is_array($queryResult['rows']) ? $queryResult['rows'] : []; |
| 343 |
$first = $rows[0] ?? null; |
| 344 |
if (!is_array($first) || !isset($first['ngrams']) || !is_string($first['ngrams'])) { |
| 345 |
return null; |
| 346 |
} |
| 347 |
|
| 348 |
$decoded = json_decode($first['ngrams'], true); |
| 349 |
if (!is_array($decoded) || !isset($decoded['bi'], $decoded['tri'])) { |
| 350 |
return null; |
| 351 |
} |
| 352 |
/** @var array{bi: array<int, string>, tri: array<int, string>} $decoded */ |
| 353 |
return $decoded; |
| 354 |
} |
| 355 |
|
| 356 |
/** |
| 357 |
* Get all cached N-grams for similarity queries. |
| 358 |
* |
| 359 |
* DEPRECATED: This method loads all entries into memory and should not be used |
| 360 |
* on large sites. Use findSimilarPagesEfficient() instead for sites with > 1000 pages. |
| 361 |
* |
| 362 |
* @deprecated Use database-side filtering for large sites |
| 363 |
* @return array<int, array<string, mixed>> Array of cached entries with id, url, url_normalized, and ngrams |
| 364 |
*/ |
| 365 |
public function getAllCachedNGrams() { |
| 366 |
$table = $this->dao->getPrefixedTableName('abj404_ngram_cache'); |
| 367 |
|
| 368 |
// Check cache size first - abort if too large |
| 369 |
$count = $this->dao->queryScalarInt("SELECT COUNT(*) AS c FROM {$table}"); |
| 370 |
if ($count > 10000) { |
| 371 |
$this->logger->errorMessage("CRITICAL: N-gram cache has {$count} entries. Cannot load into memory. Feature disabled for this request."); |
| 372 |
return []; |
| 373 |
} |
| 374 |
|
| 375 |
if ($count > 5000) { |
| 376 |
$this->logger->infoMessage("WARNING: N-gram cache has {$count} entries. This may cause memory issues."); |
| 377 |
} |
| 378 |
|
| 379 |
$listResult = $this->dao->queryAndGetResults( |
| 380 |
"SELECT id, url, url_normalized, ngrams, ngram_count FROM {$table}" |
| 381 |
); |
| 382 |
$results = isset($listResult['rows']) && is_array($listResult['rows']) ? $listResult['rows'] : []; |
| 383 |
|
| 384 |
if (empty($results)) { |
| 385 |
return []; |
| 386 |
} |
| 387 |
|
| 388 |
// Decode JSON for each entry and ensure array format |
| 389 |
$output = []; |
| 390 |
foreach ($results as $row) { |
| 391 |
if (is_object($row)) { |
| 392 |
$row = (array) $row; |
| 393 |
} |
| 394 |
if (!is_array($row)) { |
| 395 |
continue; |
| 396 |
} |
| 397 |
$ngramsRaw = isset($row['ngrams']) && is_string($row['ngrams']) ? $row['ngrams'] : ''; |
| 398 |
$row['ngrams'] = json_decode($ngramsRaw, true); |
| 399 |
$output[] = $row; |
| 400 |
} |
| 401 |
|
| 402 |
return $output; |
| 403 |
} |
| 404 |
|
| 405 |
/** |
| 406 |
* Get cached N-grams efficiently with database-side filtering. |
| 407 |
* |
| 408 |
* Uses two-range query strategy to avoid filesort from ORDER BY ABS(). |
| 409 |
* Splits query into below-target (DESC) and above-target (ASC), then merges |
| 410 |
* results by proximity to target in PHP. |
| 411 |
* |
| 412 |
* @param int $minNgramCount Minimum N-gram count (for filtering dissimilar pages) |
| 413 |
* @param int $maxNgramCount Maximum N-gram count |
| 414 |
* @param int $limit Maximum number of results to return |
| 415 |
* @param int|null $targetNgramCount The query's actual N-gram count for proximity ordering |
| 416 |
* @return array<int, array<string, mixed>> Array of cached entries |
| 417 |
*/ |
| 418 |
public function getCachedNGramsFiltered($minNgramCount, $maxNgramCount, $limit = 1000, $targetNgramCount = null) { |
| 419 |
$table = $this->dao->getPrefixedTableName('abj404_ngram_cache'); |
| 420 |
|
| 421 |
// Clamp target to valid range; fall back to midpoint if not provided |
| 422 |
$orderTarget = ($targetNgramCount !== null) |
| 423 |
? max($minNgramCount, min($maxNgramCount, (int)$targetNgramCount)) |
| 424 |
: (int)(($minNgramCount + $maxNgramCount) / 2); |
| 425 |
|
| 426 |
$halfLimit = (int)ceil($limit / 2); |
| 427 |
|
| 428 |
// Query 1: ngram_count <= target, ORDER BY ngram_count DESC |
| 429 |
// Uses idx_ngram_count for both range scan and sort (no filesort) |
| 430 |
$belowResult = $this->dao->queryAndGetResults( |
| 431 |
"SELECT id, url, url_normalized, ngrams, ngram_count |
| 432 |
FROM {$table} |
| 433 |
WHERE ngram_count >= %d AND ngram_count <= %d |
| 434 |
ORDER BY ngram_count DESC |
| 435 |
LIMIT %d", |
| 436 |
['query_params' => [$minNgramCount, $orderTarget, $halfLimit]] |
| 437 |
); |
| 438 |
$resultsBelow = isset($belowResult['rows']) && is_array($belowResult['rows']) ? $belowResult['rows'] : []; |
| 439 |
|
| 440 |
// Query 2: above target - adjust limit based on below results to handle skewed distributions |
| 441 |
// If below side returned fewer than halfLimit, give the remainder to above side |
| 442 |
$belowCount = count($resultsBelow); |
| 443 |
$aboveLimit = $limit - $belowCount; |
| 444 |
|
| 445 |
$aboveResult = $this->dao->queryAndGetResults( |
| 446 |
"SELECT id, url, url_normalized, ngrams, ngram_count |
| 447 |
FROM {$table} |
| 448 |
WHERE ngram_count > %d AND ngram_count <= %d |
| 449 |
ORDER BY ngram_count ASC |
| 450 |
LIMIT %d", |
| 451 |
['query_params' => [$orderTarget, $maxNgramCount, $aboveLimit]] |
| 452 |
); |
| 453 |
$resultsAbove = isset($aboveResult['rows']) && is_array($aboveResult['rows']) ? $aboveResult['rows'] : []; |
| 454 |
|
| 455 |
// If we didn't get enough results, fetch additional from whichever side hit its limit |
| 456 |
$aboveCount = count($resultsAbove); |
| 457 |
$totalFetched = $belowCount + $aboveCount; |
| 458 |
|
| 459 |
if ($totalFetched < $limit && $belowCount === $halfLimit) { |
| 460 |
// Below hit its limit, might have more rows - fetch additional |
| 461 |
$additionalNeeded = $limit - $totalFetched; |
| 462 |
$extraBelowResult = $this->dao->queryAndGetResults( |
| 463 |
"SELECT id, url, url_normalized, ngrams, ngram_count |
| 464 |
FROM {$table} |
| 465 |
WHERE ngram_count >= %d AND ngram_count <= %d |
| 466 |
ORDER BY ngram_count DESC |
| 467 |
LIMIT %d OFFSET %d", |
| 468 |
['query_params' => [$minNgramCount, $orderTarget, $additionalNeeded, $belowCount]] |
| 469 |
); |
| 470 |
$extraBelow = isset($extraBelowResult['rows']) && is_array($extraBelowResult['rows']) ? $extraBelowResult['rows'] : []; |
| 471 |
$resultsBelow = array_merge($resultsBelow, $extraBelow); |
| 472 |
$totalFetched = count($resultsBelow) + $aboveCount; |
| 473 |
} |
| 474 |
|
| 475 |
if ($totalFetched < $limit && $aboveCount === $aboveLimit) { |
| 476 |
// Above hit its limit, might have more rows - fetch additional |
| 477 |
$additionalNeeded = $limit - $totalFetched; |
| 478 |
$extraAboveResult = $this->dao->queryAndGetResults( |
| 479 |
"SELECT id, url, url_normalized, ngrams, ngram_count |
| 480 |
FROM {$table} |
| 481 |
WHERE ngram_count > %d AND ngram_count <= %d |
| 482 |
ORDER BY ngram_count ASC |
| 483 |
LIMIT %d OFFSET %d", |
| 484 |
['query_params' => [$orderTarget, $maxNgramCount, $additionalNeeded, $aboveCount]] |
| 485 |
); |
| 486 |
$extraAbove = isset($extraAboveResult['rows']) && is_array($extraAboveResult['rows']) ? $extraAboveResult['rows'] : []; |
| 487 |
$resultsAbove = array_merge($resultsAbove, $extraAbove); |
| 488 |
} |
| 489 |
|
| 490 |
// Merge results by proximity to target |
| 491 |
$merged = $this->mergeByProximity($resultsBelow, $resultsAbove, $orderTarget, $limit); |
| 492 |
|
| 493 |
// Decode JSON for each entry, filtering out corrupt entries |
| 494 |
$validResults = []; |
| 495 |
foreach ($merged as $row) { |
| 496 |
if (!is_array($row)) { |
| 497 |
continue; |
| 498 |
} |
| 499 |
$ngramsJson = isset($row['ngrams']) && is_string($row['ngrams']) ? $row['ngrams'] : ''; |
| 500 |
$decoded = json_decode($ngramsJson, true); |
| 501 |
if ($decoded === null && json_last_error() !== JSON_ERROR_NONE) { |
| 502 |
$rowId = isset($row['id']) ? $row['id'] : 0; |
| 503 |
$this->logger->errorMessage(sprintf( |
| 504 |
"Corrupt N-gram JSON for page ID %s: %s", |
| 505 |
(is_scalar($rowId) ? (string)$rowId : '0'), |
| 506 |
json_last_error_msg() |
| 507 |
)); |
| 508 |
continue; // Skip corrupt entry |
| 509 |
} |
| 510 |
$row['ngrams'] = $decoded; |
| 511 |
$validResults[] = $row; |
| 512 |
} |
| 513 |
|
| 514 |
return $validResults; |
| 515 |
} |
| 516 |
|
| 517 |
/** |
| 518 |
* Merge two arrays sorted by proximity to target, interleaving results. |
| 519 |
* |
| 520 |
* Both input arrays must be pre-sorted by proximity to the target: |
| 521 |
* - $below: ngram_count <= target, ordered DESC by ngram_count (closest first) |
| 522 |
* - $above: ngram_count > target, ordered ASC by ngram_count (closest first) |
| 523 |
* |
| 524 |
* @param array<int, mixed> $below Results with ngram_count <= target |
| 525 |
* @param array<int, mixed> $above Results with ngram_count > target |
| 526 |
* @param int $targetNgramCount The target N-gram count |
| 527 |
* @param int $limit Maximum results to return |
| 528 |
* @return array<int, mixed> Merged results ordered by proximity to target |
| 529 |
*/ |
| 530 |
private function mergeByProximity($below, $above, $targetNgramCount, $limit) { |
| 531 |
$result = []; |
| 532 |
$i = 0; |
| 533 |
$j = 0; |
| 534 |
$belowCount = count($below); |
| 535 |
$aboveCount = count($above); |
| 536 |
|
| 537 |
while (count($result) < $limit && ($i < $belowCount || $j < $aboveCount)) { |
| 538 |
// Calculate distances (use PHP_INT_MAX as sentinel for exhausted arrays) |
| 539 |
$belowEntry = $below[$i] ?? null; |
| 540 |
$aboveEntry = $above[$j] ?? null; |
| 541 |
$belowNgramRaw = (is_array($belowEntry) && isset($belowEntry['ngram_count'])) ? $belowEntry['ngram_count'] : 0; |
| 542 |
$belowNgramCount = is_scalar($belowNgramRaw) ? (int)$belowNgramRaw : 0; |
| 543 |
$aboveNgramRaw = (is_array($aboveEntry) && isset($aboveEntry['ngram_count'])) ? $aboveEntry['ngram_count'] : 0; |
| 544 |
$aboveNgramCount = is_scalar($aboveNgramRaw) ? (int)$aboveNgramRaw : 0; |
| 545 |
$distBelow = ($i < $belowCount) |
| 546 |
? abs($belowNgramCount - $targetNgramCount) |
| 547 |
: PHP_INT_MAX; |
| 548 |
$distAbove = ($j < $aboveCount) |
| 549 |
? abs($aboveNgramCount - $targetNgramCount) |
| 550 |
: PHP_INT_MAX; |
| 551 |
|
| 552 |
// Pick the entry closer to target; prefer below on tie (includes exact matches) |
| 553 |
if ($distBelow <= $distAbove) { |
| 554 |
$result[] = $below[$i]; |
| 555 |
$i++; |
| 556 |
} else { |
| 557 |
$result[] = $above[$j]; |
| 558 |
$j++; |
| 559 |
} |
| 560 |
} |
| 561 |
|
| 562 |
return $result; |
| 563 |
} |
| 564 |
|
| 565 |
/** |
| 566 |
* Invalidate (delete) N-grams for a specific page. |
| 567 |
* Call this when a page is updated or deleted. |
| 568 |
* |
| 569 |
* @param int $pageId The page/post ID |
| 570 |
* @param string $type Entity type: 'post', 'page', 'category', 'tag' (default: 'post') |
| 571 |
* @return bool Success status |
| 572 |
*/ |
| 573 |
public function invalidatePage($pageId, $type = 'post') { |
| 574 |
$table = $this->dao->getPrefixedTableName('abj404_ngram_cache'); |
| 575 |
$queryResult = $this->dao->queryAndGetResults( |
| 576 |
"DELETE FROM {$table} WHERE id = %d AND type = %s", |
| 577 |
['query_params' => [(int)$pageId, $type]] |
| 578 |
); |
| 579 |
|
| 580 |
$lastError = isset($queryResult['last_error']) && is_string($queryResult['last_error']) ? $queryResult['last_error'] : ''; |
| 581 |
$success = $lastError === ''; |
| 582 |
if ($success) { |
| 583 |
// Invalidate coverage ratio caches since N-gram count changed |
| 584 |
$this->invalidateCoverageCaches(); |
| 585 |
} |
| 586 |
|
| 587 |
return $success; |
| 588 |
} |
| 589 |
|
| 590 |
/** |
| 591 |
* Update N-grams for specific pages (incremental update). |
| 592 |
* |
| 593 |
* This method updates N-grams for specific page IDs, useful when |
| 594 |
* individual pages are added or updated in the permalink cache. |
| 595 |
* |
| 596 |
* @param array<int, int> $pageIds Array of page IDs to update |
| 597 |
* @return array{processed: int, success: int, failed: int} |
| 598 |
*/ |
| 599 |
public function updateNGramsForPages($pageIds) { |
| 600 |
if (empty($pageIds) || !is_array($pageIds)) { |
| 601 |
return ['processed' => 0, 'success' => 0, 'failed' => 0]; |
| 602 |
} |
| 603 |
|
| 604 |
$permalinkCacheTable = $this->dao->getPrefixedTableName('abj404_permalink_cache'); |
| 605 |
|
| 606 |
// Prepare IN clause for page IDs |
| 607 |
$placeholders = implode(',', array_fill(0, count($pageIds), '%d')); |
| 608 |
$pageResult = $this->dao->queryAndGetResults( |
| 609 |
"SELECT id, url FROM {$permalinkCacheTable} WHERE id IN ({$placeholders})", |
| 610 |
['query_params' => array_values($pageIds)] |
| 611 |
); |
| 612 |
$pages = isset($pageResult['rows']) && is_array($pageResult['rows']) ? $pageResult['rows'] : []; |
| 613 |
|
| 614 |
if (empty($pages)) { |
| 615 |
return ['processed' => 0, 'success' => 0, 'failed' => 0]; |
| 616 |
} |
| 617 |
|
| 618 |
$stats = ['processed' => 0, 'success' => 0, 'failed' => 0]; |
| 619 |
|
| 620 |
foreach ($pages as $page) { |
| 621 |
if (is_object($page)) { |
| 622 |
$page = (array) $page; |
| 623 |
} |
| 624 |
|
| 625 |
$pageId = $page['id']; |
| 626 |
$url = $page['url']; |
| 627 |
|
| 628 |
// Normalize URL for matching (lowercase, trim) |
| 629 |
$urlNormalized = $this->f->strtolower(trim($url)); |
| 630 |
|
| 631 |
// Extract N-grams |
| 632 |
$ngrams = $this->extractNGrams($urlNormalized); |
| 633 |
|
| 634 |
// Store in database |
| 635 |
$success = $this->storeNGrams($pageId, $url, $urlNormalized, $ngrams); |
| 636 |
|
| 637 |
$stats['processed']++; |
| 638 |
if ($success) { |
| 639 |
$stats['success']++; |
| 640 |
} else { |
| 641 |
$stats['failed']++; |
| 642 |
} |
| 643 |
} |
| 644 |
|
| 645 |
$this->logger->debugMessage(sprintf( |
| 646 |
"Incremental N-gram update: %d pages, %d success, %d failed", |
| 647 |
$stats['processed'], |
| 648 |
$stats['success'], |
| 649 |
$stats['failed'] |
| 650 |
)); |
| 651 |
|
| 652 |
return $stats; |
| 653 |
} |
| 654 |
|
| 655 |
/** |
| 656 |
* Rebuild the N-gram cache for all pages (background process). |
| 657 |
* |
| 658 |
* This method processes pages in batches to avoid memory issues and timeouts. |
| 659 |
* Should be called during permalink cache updates or as a scheduled task. |
| 660 |
* |
| 661 |
* @param int $batchSize Number of pages to process per batch (default: 100) |
| 662 |
* @param int $offset Starting offset for pagination (default: 0) |
| 663 |
* @return array{processed: int, success: int, failed: int} |
| 664 |
*/ |
| 665 |
public function rebuildCache($batchSize = 100, $offset = 0) { |
| 666 |
$permalinkCacheTable = $this->dao->getPrefixedTableName('abj404_permalink_cache'); |
| 667 |
|
| 668 |
// Get a batch of pages from permalink cache |
| 669 |
$batchResult = $this->dao->queryAndGetResults( |
| 670 |
"SELECT id, url FROM {$permalinkCacheTable} LIMIT %d OFFSET %d", |
| 671 |
['query_params' => [$batchSize, $offset]] |
| 672 |
); |
| 673 |
$pages = isset($batchResult['rows']) && is_array($batchResult['rows']) ? $batchResult['rows'] : []; |
| 674 |
|
| 675 |
$stats = [ |
| 676 |
'processed' => 0, |
| 677 |
'success' => 0, |
| 678 |
'failed' => 0 |
| 679 |
]; |
| 680 |
|
| 681 |
foreach ($pages as $page) { |
| 682 |
// Handle both object and array results (defensive coding for test environments) |
| 683 |
if (is_object($page)) { |
| 684 |
$page = (array) $page; |
| 685 |
} |
| 686 |
|
| 687 |
$pageId = $page['id']; |
| 688 |
$url = $page['url']; |
| 689 |
|
| 690 |
// Normalize URL for matching (lowercase, trim) |
| 691 |
$urlNormalized = $this->f->strtolower(trim($url)); |
| 692 |
|
| 693 |
// Extract N-grams |
| 694 |
$ngrams = $this->extractNGrams($urlNormalized); |
| 695 |
|
| 696 |
// Store in database (skip per-item invalidation for bulk efficiency) |
| 697 |
$success = $this->storeNGrams($pageId, $url, $urlNormalized, $ngrams, 'post', true); |
| 698 |
|
| 699 |
$stats['processed']++; |
| 700 |
if ($success) { |
| 701 |
$stats['success']++; |
| 702 |
} else { |
| 703 |
$stats['failed']++; |
| 704 |
} |
| 705 |
} |
| 706 |
|
| 707 |
// Invalidate coverage caches once at end of batch (not per-item) |
| 708 |
if ($stats['success'] > 0) { |
| 709 |
$this->invalidateCoverageCaches(); |
| 710 |
} |
| 711 |
|
| 712 |
// Log only every 1000 pages to reduce log verbosity |
| 713 |
if ($offset % 1000 == 0) { |
| 714 |
$this->logger->debugMessage(sprintf( |
| 715 |
"N-gram cache rebuild batch (offset %d): %d processed, %d success, %d failed", |
| 716 |
$offset, |
| 717 |
$stats['processed'], |
| 718 |
$stats['success'], |
| 719 |
$stats['failed'] |
| 720 |
)); |
| 721 |
} |
| 722 |
|
| 723 |
return $stats; |
| 724 |
} |
| 725 |
|
| 726 |
/** |
| 727 |
* Find pages similar to a 404 URL using N-gram filtering. |
| 728 |
* |
| 729 |
* This is the main method called by SpellChecker to reduce candidates. |
| 730 |
* |
| 731 |
* Process: |
| 732 |
* 1. Extract N-grams for the 404 URL (~0.1ms) |
| 733 |
* 2. Load filtered cached N-grams from DB (database-side filtering) |
| 734 |
* 3. Compute Dice similarity for each (~0.05ms each) |
| 735 |
* 4. Filter by minimum similarity threshold (removes 80-90%) |
| 736 |
* 5. Sort by similarity (best matches first) |
| 737 |
* 6. Return top N candidates |
| 738 |
* |
| 739 |
* @param string $url404 The 404 URL to find matches for |
| 740 |
* @param float $minSimilarity Minimum Dice coefficient (default: 0.4) |
| 741 |
* @param int $maxCandidates Maximum candidates to return (default: 100) |
| 742 |
* @return array<int, float> Associative array [id => similarity_score] sorted by score (descending) |
| 743 |
*/ |
| 744 |
public function findSimilarPages($url404, $minSimilarity = 0.4, $maxCandidates = 100) { |
| 745 |
global $wpdb; |
| 746 |
|
| 747 |
// Start timing for performance tracking |
| 748 |
$startTime = microtime(true); |
| 749 |
|
| 750 |
// Step 1: Extract N-grams for the 404 URL |
| 751 |
$url404Normalized = $this->f->strtolower(trim($url404)); |
| 752 |
$queryNGrams = $this->extractNGrams($url404Normalized); |
| 753 |
$queryCombinedCount = count($queryNGrams['bi']) + count($queryNGrams['tri']); |
| 754 |
|
| 755 |
// Early return if search term is too short for N-gram filtering |
| 756 |
if ($queryCombinedCount == 0) { |
| 757 |
$this->logger->debugMessage("Search term too short for N-gram filtering: '{$url404}'"); |
| 758 |
return []; |
| 759 |
} |
| 760 |
|
| 761 |
// Check cache size to determine strategy (use memoized count) |
| 762 |
$totalCount = $this->getCacheCount(); |
| 763 |
|
| 764 |
if ($totalCount == 0) { |
| 765 |
$this->logger->debugMessage("N-gram cache is empty."); |
| 766 |
|
| 767 |
// Schedule background rebuild if not already initialized/scheduled |
| 768 |
// This ensures automatic recovery from empty cache state. |
| 769 |
// Use the multisite-aware getter so network-activated installs read |
| 770 |
// get_site_option (where the flag is stored) rather than get_option |
| 771 |
// (which would be empty on the frontend 404 dispatch path and cause |
| 772 |
// duplicate rebuild scheduling). |
| 773 |
if (!$this->isCacheInitialized()) { |
| 774 |
try { |
| 775 |
$dbUpgrades = abj_service('database_upgrades'); |
| 776 |
$dbUpgrades->scheduleNGramCacheRebuild(); |
| 777 |
$this->logger->infoMessage("Empty N-gram cache detected during 404 request. Scheduled background rebuild."); |
| 778 |
} catch (Exception $e) { |
| 779 |
$this->logger->errorMessage("Failed to schedule N-gram cache rebuild: " . $e->getMessage()); |
| 780 |
} |
| 781 |
} else { |
| 782 |
$this->logger->debugMessage("N-gram cache rebuild already initialized or scheduled."); |
| 783 |
} |
| 784 |
|
| 785 |
return []; |
| 786 |
} |
| 787 |
|
| 788 |
// Step 2: Load cached N-grams with smart filtering |
| 789 |
// Calculate N-gram count range for filtering (40% tolerance) |
| 790 |
$minCount = max(1, (int)($queryCombinedCount * 0.4)); |
| 791 |
$maxCount = (int)($queryCombinedCount * 2.5); |
| 792 |
|
| 793 |
// Use efficient database-side filtering for large caches |
| 794 |
if ($totalCount > self::CACHE_LOAD_LIMIT) { |
| 795 |
$this->logger->debugMessage("Using database-side filtering for {$totalCount} entries"); |
| 796 |
$cachedPages = $this->getCachedNGramsFiltered($minCount, $maxCount, self::CACHE_LOAD_LIMIT, $queryCombinedCount); |
| 797 |
} else { |
| 798 |
// For small caches, load all (legacy behavior) |
| 799 |
$cachedPages = $this->getAllCachedNGrams(); |
| 800 |
} |
| 801 |
|
| 802 |
if (empty($cachedPages)) { |
| 803 |
$this->logger->debugMessage("No matching candidates after filtering."); |
| 804 |
return []; |
| 805 |
} |
| 806 |
|
| 807 |
// Step 3: Compute similarity for each page |
| 808 |
$similarities = []; |
| 809 |
foreach ($cachedPages as $page) { |
| 810 |
if (!is_array($page)) { |
| 811 |
continue; |
| 812 |
} |
| 813 |
$pageId = isset($page['id']) ? $page['id'] : null; |
| 814 |
$pageNGrams = isset($page['ngrams']) ? $page['ngrams'] : null; |
| 815 |
|
| 816 |
// Quick optimization: Skip if N-gram counts are too different |
| 817 |
// (This is redundant for filtered queries but kept for unfiltered path) |
| 818 |
$pageNgramCountRaw = isset($page['ngram_count']) ? $page['ngram_count'] : 0; |
| 819 |
$pageCombinedCount = is_scalar($pageNgramCountRaw) ? (int)$pageNgramCountRaw : 0; |
| 820 |
$denominator = max(1, $queryCombinedCount, $pageCombinedCount); |
| 821 |
$countRatio = min($queryCombinedCount, $pageCombinedCount) / $denominator; |
| 822 |
if ($countRatio < 0.4) { |
| 823 |
continue; |
| 824 |
} |
| 825 |
|
| 826 |
// Compute Dice coefficient |
| 827 |
/** @var array{bi?: array<int, string>, tri?: array<int, string>} $pageNGramsTyped */ |
| 828 |
$pageNGramsTyped = is_array($pageNGrams) ? $pageNGrams : array(); |
| 829 |
$similarity = $this->diceCoefficient($queryNGrams, $pageNGramsTyped); |
| 830 |
|
| 831 |
// Step 4: Filter by minimum similarity |
| 832 |
if ($similarity >= $minSimilarity) { |
| 833 |
$similarities[$pageId] = $similarity; |
| 834 |
} |
| 835 |
} |
| 836 |
|
| 837 |
// Step 5: Sort by similarity (descending) |
| 838 |
arsort($similarities); |
| 839 |
|
| 840 |
// Step 6: Limit to top N candidates |
| 841 |
if (count($similarities) > $maxCandidates) { |
| 842 |
$similarities = array_slice($similarities, 0, $maxCandidates, true); |
| 843 |
} |
| 844 |
|
| 845 |
$endTime = microtime(true); |
| 846 |
$duration = ($endTime - $startTime) * 1000; // Convert to milliseconds |
| 847 |
|
| 848 |
$this->logger->debugMessage(sprintf( |
| 849 |
"N-gram filtering: %d total, %d examined → %d candidates (≥%.2f similarity) in %.2fms", |
| 850 |
$totalCount, |
| 851 |
count($cachedPages), |
| 852 |
count($similarities), |
| 853 |
$minSimilarity, |
| 854 |
$duration |
| 855 |
)); |
| 856 |
|
| 857 |
// Track usage stats |
| 858 |
$this->trackNGramUsage($totalCount, count($cachedPages), count($similarities), $duration); |
| 859 |
|
| 860 |
return $similarities; |
| 861 |
} |
| 862 |
|
| 863 |
/** |
| 864 |
* Check if the N-gram cache is populated. |
| 865 |
* |
| 866 |
* @return bool True if cache has entries, false otherwise |
| 867 |
*/ |
| 868 |
public function isCachePopulated() { |
| 869 |
return $this->getCacheCount() > 0; |
| 870 |
} |
| 871 |
|
| 872 |
/** |
| 873 |
* Get cache entry count (memoized per-request). |
| 874 |
* |
| 875 |
* Use this instead of getCacheStats() when only the count is needed, |
| 876 |
* especially in hot code paths like 404 request handling. |
| 877 |
* |
| 878 |
* @return int Number of entries in the N-gram cache |
| 879 |
*/ |
| 880 |
public function getCacheCount() { |
| 881 |
// Return memoized value if available |
| 882 |
if ($this->ngramCountMemo !== null) { |
| 883 |
return $this->ngramCountMemo; |
| 884 |
} |
| 885 |
|
| 886 |
// Check if coverage ratio memo has the count |
| 887 |
if ($this->coverageRatioMemo !== null && isset($this->coverageRatioMemo['ngram_count'])) { |
| 888 |
$ngramCountVal = $this->coverageRatioMemo['ngram_count']; |
| 889 |
$this->ngramCountMemo = is_scalar($ngramCountVal) ? (int)$ngramCountVal : 0; |
| 890 |
return $this->ngramCountMemo; |
| 891 |
} |
| 892 |
|
| 893 |
global $wpdb; |
| 894 |
$table = $this->dao->getPrefixedTableName('abj404_ngram_cache'); |
| 895 |
if (!isset($wpdb) || !is_object($wpdb) || !is_callable([$wpdb, 'get_var'])) { |
| 896 |
// In test environments or very early bootstrap, wpdb may not exist. |
| 897 |
// Treat as "no cache" rather than fatal. |
| 898 |
$this->ngramCountMemo = 0; |
| 899 |
return $this->ngramCountMemo; |
| 900 |
} |
| 901 |
|
| 902 |
$this->ngramCountMemo = $this->dao->queryScalarInt("SELECT COUNT(*) AS c FROM {$table}"); |
| 903 |
return $this->ngramCountMemo; |
| 904 |
} |
| 905 |
|
| 906 |
/** |
| 907 |
* Get cache coverage ratio (ngram entries / permalink entries). |
| 908 |
* |
| 909 |
* Used to detect stale or incomplete caches. A ratio < 1.0 indicates |
| 910 |
* some permalink entries are not in the N-gram cache. |
| 911 |
* |
| 912 |
* Results are memoized per-request and cached in a transient for 5 minutes. |
| 913 |
* Uses version-based validation to avoid expensive COUNT(*) queries on every |
| 914 |
* request - versions are bumped by invalidateCoverageCaches() when data changes. |
| 915 |
* |
| 916 |
* @return float Coverage ratio (0.0 to 1.0+), or 1.0 if permalink cache is empty |
| 917 |
*/ |
| 918 |
public function getCacheCoverageRatio() { |
| 919 |
// Fast path: return memoized value if available (already validated this request) |
| 920 |
if ($this->coverageRatioMemo !== null) { |
| 921 |
$ratioVal = isset($this->coverageRatioMemo['ratio']) ? $this->coverageRatioMemo['ratio'] : 0; |
| 922 |
return is_scalar($ratioVal) ? (float)$ratioVal : 0.0; |
| 923 |
} |
| 924 |
|
| 925 |
// Get current version (cheap scalar read, no COUNT queries) |
| 926 |
$versionTransient = get_transient(self::COVERAGE_VERSION_KEY); |
| 927 |
$currentVersion = is_scalar($versionTransient) ? (int)$versionTransient : 0; |
| 928 |
|
| 929 |
// Check transient with version-based validation |
| 930 |
$cached = get_transient(self::COVERAGE_RATIO_KEY); |
| 931 |
if ($cached !== false && is_array($cached) |
| 932 |
&& isset($cached['ratio'], $cached['version']) |
| 933 |
&& (int)$cached['version'] === $currentVersion) { |
| 934 |
// Valid: version matches, trust the cached ratio without COUNT queries |
| 935 |
$this->coverageRatioMemo = $cached; |
| 936 |
if (isset($cached['ngram_count'])) { |
| 937 |
$this->ngramCountMemo = (int)$cached['ngram_count']; |
| 938 |
} |
| 939 |
return (float)$cached['ratio']; |
| 940 |
} |
| 941 |
|
| 942 |
// Transient miss or version mismatch - compute fresh ratio |
| 943 |
$ngramTable = $this->dao->getPrefixedTableName('abj404_ngram_cache'); |
| 944 |
$permalinkTable = $this->dao->getPrefixedTableName('abj404_permalink_cache'); |
| 945 |
|
| 946 |
// Get both counts (required for ratio computation) |
| 947 |
$ngramCount = $this->dao->queryScalarInt("SELECT COUNT(*) AS c FROM {$ngramTable}"); |
| 948 |
$permalinkCount = $this->dao->queryScalarInt("SELECT COUNT(*) AS c FROM {$permalinkTable}"); |
| 949 |
|
| 950 |
// Memoize ngram count to avoid redundant queries elsewhere |
| 951 |
$this->ngramCountMemo = $ngramCount; |
| 952 |
|
| 953 |
if ($permalinkCount === 0) { |
| 954 |
// Empty permalink cache with existing N-grams = stale state (during rebuild) |
| 955 |
// Return 0.0 to skip prefiltering until both caches are populated |
| 956 |
$ratio = ($ngramCount === 0) ? 1.0 : 0.0; |
| 957 |
} else { |
| 958 |
$ratio = $ngramCount / $permalinkCount; |
| 959 |
} |
| 960 |
|
| 961 |
// Memoize for this request (include version for cache storage) |
| 962 |
$this->coverageRatioMemo = [ |
| 963 |
'ratio' => $ratio, |
| 964 |
'ngram_count' => $ngramCount, |
| 965 |
'permalink_count' => $permalinkCount, |
| 966 |
'version' => $currentVersion |
| 967 |
]; |
| 968 |
|
| 969 |
// @cache-write-audit: opt-out — self-validating cache. The cached |
| 970 |
// payload carries the coverage version key (line 941 reads |
| 971 |
// $cached['version']); any mutation to the underlying tables calls |
| 972 |
// invalidateCoverageCaches() which bumps COVERAGE_VERSION_KEY, so a |
| 973 |
// stale (or query-error-poisoned) entry is invalidated by the next |
| 974 |
// mutation rather than by an explicit last_error/timed_out check. |
| 975 |
// Reference fixes: 6315bcb8, c8fba7ee, 2a0a2dd6. |
| 976 |
set_transient(self::COVERAGE_RATIO_KEY, $this->coverageRatioMemo, self::COVERAGE_RATIO_CACHE_TTL); |
| 977 |
|
| 978 |
return $ratio; |
| 979 |
} |
| 980 |
|
| 981 |
/** |
| 982 |
* Get cache statistics for admin display. |
| 983 |
* |
| 984 |
* @return array<string, mixed> Statistics including total_entries, posts_entries, etc. |
| 985 |
*/ |
| 986 |
public function getCacheStats() { |
| 987 |
$table = $this->dao->getPrefixedTableName('abj404_ngram_cache'); |
| 988 |
|
| 989 |
$totalEntries = $this->dao->queryScalarInt("SELECT COUNT(*) AS c FROM {$table}"); |
| 990 |
$postsEntries = $this->dao->queryScalarInt( |
| 991 |
"SELECT COUNT(*) AS c FROM {$table} WHERE type = %s", |
| 992 |
['query_params' => ['post']] |
| 993 |
); |
| 994 |
$categoryEntries = $this->dao->queryScalarInt( |
| 995 |
"SELECT COUNT(*) AS c FROM {$table} WHERE type = %s", |
| 996 |
['query_params' => ['category']] |
| 997 |
); |
| 998 |
$tagEntries = $this->dao->queryScalarInt( |
| 999 |
"SELECT COUNT(*) AS c FROM {$table} WHERE type = %s", |
| 1000 |
['query_params' => ['tag']] |
| 1001 |
); |
| 1002 |
$lastUpdatedResult = $this->dao->queryAndGetResults( |
| 1003 |
"SELECT MAX(last_updated) AS m FROM {$table}" |
| 1004 |
); |
| 1005 |
$lastUpdatedRows = isset($lastUpdatedResult['rows']) && is_array($lastUpdatedResult['rows']) ? $lastUpdatedResult['rows'] : []; |
| 1006 |
$lastUpdatedFirst = $lastUpdatedRows[0] ?? null; |
| 1007 |
$lastUpdated = is_array($lastUpdatedFirst) && isset($lastUpdatedFirst['m']) ? $lastUpdatedFirst['m'] : null; |
| 1008 |
|
| 1009 |
return [ |
| 1010 |
'total_entries' => $totalEntries, |
| 1011 |
'posts_entries' => $postsEntries, |
| 1012 |
'category_entries' => $categoryEntries, |
| 1013 |
'tag_entries' => $tagEntries, |
| 1014 |
'last_updated' => $lastUpdated, |
| 1015 |
]; |
| 1016 |
} |
| 1017 |
|
| 1018 |
/** |
| 1019 |
* Track N-gram usage statistics. |
| 1020 |
* |
| 1021 |
* @param int $totalInCache Total entries in cache |
| 1022 |
* @param int $examined Number of entries examined |
| 1023 |
* @param int $candidates Number of candidates returned |
| 1024 |
* @param float $duration Time taken in milliseconds |
| 1025 |
*/ |
| 1026 |
/** |
| 1027 |
* @param int $totalInCache |
| 1028 |
* @param int $examined |
| 1029 |
* @param int $candidates |
| 1030 |
* @param float $duration |
| 1031 |
* @return void |
| 1032 |
*/ |
| 1033 |
private function trackNGramUsage($totalInCache, $examined, $candidates, $duration) { |
| 1034 |
// Get current stats |
| 1035 |
$defaultStats = array( |
| 1036 |
'total_queries' => 0, |
| 1037 |
'total_entries_examined' => 0, |
| 1038 |
'total_candidates_returned' => 0, |
| 1039 |
'total_duration_ms' => 0, |
| 1040 |
'avg_reduction_percent' => 0, |
| 1041 |
'last_reset' => time() |
| 1042 |
); |
| 1043 |
$statsRaw = get_option('abj404_ngram_usage_stats', $defaultStats); |
| 1044 |
/** @var array<string, mixed> $stats */ |
| 1045 |
$stats = is_array($statsRaw) ? $statsRaw : $defaultStats; |
| 1046 |
|
| 1047 |
// Update stats |
| 1048 |
$stats['total_queries'] = (isset($stats['total_queries']) && is_numeric($stats['total_queries'])) ? (int)$stats['total_queries'] + 1 : 1; |
| 1049 |
$stats['total_entries_examined'] = (isset($stats['total_entries_examined']) && is_numeric($stats['total_entries_examined'])) ? (int)$stats['total_entries_examined'] + $examined : $examined; |
| 1050 |
$stats['total_candidates_returned'] = (isset($stats['total_candidates_returned']) && is_numeric($stats['total_candidates_returned'])) ? (int)$stats['total_candidates_returned'] + $candidates : $candidates; |
| 1051 |
$stats['total_duration_ms'] = (isset($stats['total_duration_ms']) && is_numeric($stats['total_duration_ms'])) ? (float)$stats['total_duration_ms'] + $duration : $duration; |
| 1052 |
|
| 1053 |
// Calculate average reduction (how much ngrams reduced the search space) |
| 1054 |
if ($totalInCache > 0) { |
| 1055 |
$reductionPercent = (($totalInCache - $examined) / $totalInCache) * 100; |
| 1056 |
$prevAvgReduction = (isset($stats['avg_reduction_percent']) && is_numeric($stats['avg_reduction_percent'])) ? (float)$stats['avg_reduction_percent'] : 0; |
| 1057 |
$totalQueries = (int)$stats['total_queries']; |
| 1058 |
$stats['avg_reduction_percent'] = (($prevAvgReduction * ($totalQueries - 1)) + $reductionPercent) / $totalQueries; |
| 1059 |
} |
| 1060 |
|
| 1061 |
// Reset stats monthly to avoid unbounded growth |
| 1062 |
$monthAgo = time() - (30 * 24 * 60 * 60); |
| 1063 |
$lastReset = (isset($stats['last_reset']) && is_numeric($stats['last_reset'])) ? (int)$stats['last_reset'] : 0; |
| 1064 |
if ($lastReset < $monthAgo) { |
| 1065 |
$stats = [ |
| 1066 |
'total_queries' => 1, |
| 1067 |
'total_entries_examined' => $examined, |
| 1068 |
'total_candidates_returned' => $candidates, |
| 1069 |
'total_duration_ms' => $duration, |
| 1070 |
'avg_reduction_percent' => ($totalInCache > 0) ? (($totalInCache - $examined) / $totalInCache) * 100 : 0, |
| 1071 |
'last_reset' => time() |
| 1072 |
]; |
| 1073 |
} |
| 1074 |
|
| 1075 |
update_option('abj404_ngram_usage_stats', $stats); |
| 1076 |
} |
| 1077 |
|
| 1078 |
/** |
| 1079 |
* Get N-gram usage statistics. |
| 1080 |
* |
| 1081 |
* @return array<string, mixed> Usage statistics |
| 1082 |
*/ |
| 1083 |
public function getUsageStats() { |
| 1084 |
$defaultStats = array( |
| 1085 |
'total_queries' => 0, |
| 1086 |
'total_entries_examined' => 0, |
| 1087 |
'total_candidates_returned' => 0, |
| 1088 |
'total_duration_ms' => 0, |
| 1089 |
'avg_reduction_percent' => 0, |
| 1090 |
'last_reset' => time() |
| 1091 |
); |
| 1092 |
$statsRaw = get_option('abj404_ngram_usage_stats', $defaultStats); |
| 1093 |
/** @var array<string, mixed> $stats */ |
| 1094 |
$stats = is_array($statsRaw) ? $statsRaw : $defaultStats; |
| 1095 |
|
| 1096 |
$totalQueries = (isset($stats['total_queries']) && is_numeric($stats['total_queries'])) ? (int)$stats['total_queries'] : 0; |
| 1097 |
$totalExamined = (isset($stats['total_entries_examined']) && is_numeric($stats['total_entries_examined'])) ? (float)$stats['total_entries_examined'] : 0; |
| 1098 |
$totalCandidates = (isset($stats['total_candidates_returned']) && is_numeric($stats['total_candidates_returned'])) ? (float)$stats['total_candidates_returned'] : 0; |
| 1099 |
$totalDuration = (isset($stats['total_duration_ms']) && is_numeric($stats['total_duration_ms'])) ? (float)$stats['total_duration_ms'] : 0; |
| 1100 |
|
| 1101 |
// Calculate averages |
| 1102 |
if ($totalQueries > 0) { |
| 1103 |
$stats['avg_examined_per_query'] = round($totalExamined / $totalQueries, 1); |
| 1104 |
$stats['avg_candidates_per_query'] = round($totalCandidates / $totalQueries, 1); |
| 1105 |
$stats['avg_duration_ms'] = round($totalDuration / $totalQueries, 2); |
| 1106 |
} else { |
| 1107 |
$stats['avg_examined_per_query'] = 0; |
| 1108 |
$stats['avg_candidates_per_query'] = 0; |
| 1109 |
$stats['avg_duration_ms'] = 0; |
| 1110 |
} |
| 1111 |
|
| 1112 |
return $stats; |
| 1113 |
} |
| 1114 |
} |
| 1115 |
|
