PluginProbe
404 Solution / 4.3.0
404 Solution v4.3.0
4.3.5 4.3.4 4.3.3 4.3.2 4.3.1 4.3.0 4.2.0 4.1.19 4.1.18 4.1.17 4.1.16 4.1.15 4.1.13 4.1.12 4.1.11 4.1.10 4.1.9 4.1.8 4.1.7 4.1.6 4.1.5 4.1.4 4.1.3 trunk 2.30.0 All 109 releases
404-solution / includes / ngram / NGramCacheRepository.php

NGramCacheRepository.php in 404 Solution 4.3.0, at includes/ngram/NGramCacheRepository.php

476 lines 18.3 KB
No matching file
Up and down to move Enter to open Esc to close
Raw Download Zip
1 <?php
2
3 if (!defined('ABSPATH')) {
4 exit;
5 }
6
7 /**
8 * Repository for the abj404_ngram_cache table.
9 *
10 * Owns reads (per-page lookup, full-table load with guard, two-range filtered
11 * load, count, stats) and writes (REPLACE on store, DELETE on invalidate)
12 * for stored N-gram entries. After every write, calls the coverage policy
13 * (lazily resolved to avoid construction-time circular dependency, since
14 * the policy can also call back into repository-shaped helpers) so the
15 * coverage transient is invalidated in one place.
16 *
17 * The two-range query strategy in getCachedNGramsFiltered() avoids filesort
18 * from ORDER BY ABS() by splitting into ASC and DESC halves and merging via
19 * the proximity-merge primitive.
20 */
21 class ABJ_404_Solution_NGramCacheRepository {
22
23 /** Maximum entries to load from N-gram cache to prevent memory exhaustion.
24 * JSON decode of N-gram data is memory-intensive; 1000 entries is safe for 128MB limit. */
25 const CACHE_LOAD_LIMIT = 1000;
26
27 /** @var ABJ_404_Solution_DatabaseCore */
28 private $dbCore;
29
30 /** @var ABJ_404_Solution_Logging */
31 private $logger;
32
33 /** @var ABJ_404_Solution_NGramSimilarity */
34 private $similarity;
35
36 /** @var callable|null Lazily resolves the NGramCoveragePolicy for write-path invalidation. */
37 private $coveragePolicyResolver;
38
39 /** @var int|null Per-request memoized cache count */
40 private $cacheCountMemo = null;
41
42 /**
43 * @param ABJ_404_Solution_DatabaseCore|null $dbCore
44 * @param ABJ_404_Solution_Logging|null $logging
45 * @param ABJ_404_Solution_NGramSimilarity|null $similarity
46 * @param callable|null $coveragePolicyResolver Called on write to fetch the coverage policy. Lazy to break ctor cycle.
47 */
48 public function __construct($dbCore = null, $logging = null, $similarity = null, $coveragePolicyResolver = null) {
49 $this->dbCore = $dbCore !== null ? $dbCore : abj_service('db_core');
50 $this->logger = $logging !== null ? $logging : abj_service('logging');
51 $this->similarity = $similarity instanceof ABJ_404_Solution_NGramSimilarity
52 ? $similarity
53 : new ABJ_404_Solution_NGramSimilarity();
54 $this->coveragePolicyResolver = $coveragePolicyResolver;
55 }
56
57 /**
58 * Store N-grams for a page.
59 *
60 * @param int $pageId
61 * @param string $url
62 * @param string $urlNormalized
63 * @param array<string, mixed> $ngrams
64 * @param string $type Entity type: 'post', 'page', 'category', 'tag'
65 * @param bool $skipInvalidation Skip coverage invalidation (for bulk operations)
66 * @return bool
67 */
68 public function storeNGrams($pageId, $url, $urlNormalized, $ngrams, $type = 'post', $skipInvalidation = false) {
69 if (!is_numeric($pageId) || $pageId <= 0) {
70 $this->logger->errorMessage("Invalid page ID for N-gram storage: " . var_export($pageId, true));
71 return false;
72 }
73
74 if (!is_array($ngrams) || !isset($ngrams['bi']) || !isset($ngrams['tri'])) {
75 $this->logger->errorMessage("Invalid N-gram structure for page ID {$pageId}");
76 return false;
77 }
78
79 if (!is_array($ngrams['bi']) || !is_array($ngrams['tri'])) {
80 $this->logger->errorMessage("Invalid N-gram array types for page ID {$pageId}");
81 return false;
82 }
83
84 $ngramJson = json_encode($ngrams);
85 if ($ngramJson === false) {
86 $this->logger->errorMessage("Failed to JSON encode N-grams for page ID {$pageId}");
87 return false;
88 }
89
90 $ngramCount = count($ngrams['bi']) + count($ngrams['tri']);
91 $table = $this->dbCore->tableNameResolver()->getPrefixedTableName('abj404_ngram_cache');
92
93 // REPLACE = DELETE + INSERT. Routed through DAO for timeout/retry/recovery.
94 $queryResult = $this->dbCore->queryAndGetResults(
95 "REPLACE INTO {$table} (id, type, url, url_normalized, ngrams, ngram_count, last_updated)
96 VALUES (%d, %s, %s, %s, %s, %d, %d)",
97 ['query_params' => [
98 (int)$pageId,
99 $type,
100 $url,
101 $urlNormalized,
102 $ngramJson,
103 $ngramCount,
104 abj_clock()->wpNow(),
105 ]]
106 );
107
108 $lastError = isset($queryResult['last_error']) && is_string($queryResult['last_error']) ? $queryResult['last_error'] : '';
109 if ($lastError !== '') {
110 global $wpdb;
111 $dbName = isset($wpdb->dbname) && is_string($wpdb->dbname) ? $wpdb->dbname : '';
112 $errorContext = sprintf(
113 "Failed to store N-grams for page ID %d: %s, Table: %s, Prefix: %s, DB: %s",
114 $pageId,
115 $lastError,
116 $table,
117 $this->dbCore->tableNameResolver()->getLowercasePrefix(),
118 $dbName
119 );
120
121 if (is_multisite()) {
122 $errorContext .= sprintf(", Blog ID: %d", get_current_blog_id());
123 }
124
125 if (!$this->dbCore->errorClassifier()->classifyAndHandleInfrastructureError($lastError)) {
126 $this->logger->errorMessage($errorContext);
127 }
128 return false;
129 }
130
131 $this->cacheCountMemo = null;
132 if (!$skipInvalidation) {
133 $this->invalidateCoverageCaches();
134 }
135
136 return true;
137 }
138
139 /**
140 * Get N-grams for a specific page.
141 *
142 * @param int $pageId
143 * @param string $type
144 * @return array{bi: array<int, string>, tri: array<int, string>}|null
145 */
146 public function getNGramsForPage($pageId, $type = 'post') {
147 $table = $this->dbCore->tableNameResolver()->getPrefixedTableName('abj404_ngram_cache');
148
149 $queryResult = $this->dbCore->queryAndGetResults(
150 "SELECT ngrams FROM {$table} WHERE id = %d AND type = %s",
151 ['query_params' => [$pageId, $type]]
152 );
153
154 $rows = isset($queryResult['rows']) && is_array($queryResult['rows']) ? $queryResult['rows'] : [];
155 $first = $rows[0] ?? null;
156 if (!is_array($first) || !isset($first['ngrams']) || !is_string($first['ngrams'])) {
157 return null;
158 }
159
160 $decoded = json_decode($first['ngrams'], true);
161 if (!is_array($decoded) || !isset($decoded['bi'], $decoded['tri'])) {
162 return null;
163 }
164 /** @var array{bi: array<int, string>, tri: array<int, string>} $decoded */
165 return $decoded;
166 }
167
168 /**
169 * Get all cached N-grams for similarity queries.
170 *
171 * Loads entire table into memory; only safe for small caches. Above
172 * CACHE_LOAD_LIMIT (1000) callers should use getCachedNGramsFiltered().
173 *
174 * @param string|null $type When non-null, restrict the load to a single
175 * entity type ('post', 'category', 'tag', ...). Null preserves the
176 * historical all-types scan (the posts path is unaffected).
177 * @return array<int, array<string, mixed>>
178 */
179 public function getAllCachedNGrams($type = null) {
180 $table = $this->dbCore->tableNameResolver()->getPrefixedTableName('abj404_ngram_cache');
181
182 $count = ($type !== null)
183 ? $this->getCacheCountForType((string)$type)
184 : $this->dbCore->queryScalarInt("SELECT COUNT(*) AS c FROM {$table}");
185 if ($count > 10000) {
186 $this->logger->errorMessage("CRITICAL: N-gram cache has {$count} entries. Cannot load into memory. Feature disabled for this request.");
187 return [];
188 }
189
190 if ($count > 5000) {
191 $this->logger->infoMessage("WARNING: N-gram cache has {$count} entries. This may cause memory issues.");
192 }
193
194 if ($type !== null) {
195 $listResult = $this->dbCore->queryAndGetResults(
196 "SELECT id, url, url_normalized, ngrams, ngram_count FROM {$table} WHERE type = %s",
197 ['query_params' => [(string)$type]]
198 );
199 } else {
200 $listResult = $this->dbCore->queryAndGetResults(
201 "SELECT id, url, url_normalized, ngrams, ngram_count FROM {$table}"
202 );
203 }
204 $results = isset($listResult['rows']) && is_array($listResult['rows']) ? $listResult['rows'] : [];
205
206 if (empty($results)) {
207 return [];
208 }
209
210 $output = [];
211 foreach ($results as $row) {
212 if (is_object($row)) {
213 $row = (array) $row;
214 }
215 if (!is_array($row)) {
216 continue;
217 }
218 $ngramsRaw = isset($row['ngrams']) && is_string($row['ngrams']) ? $row['ngrams'] : '';
219 $row['ngrams'] = json_decode($ngramsRaw, true);
220 $output[] = $row;
221 }
222
223 return $output;
224 }
225
226 /**
227 * Get cached N-grams with database-side range filtering.
228 *
229 * Two-range strategy avoids filesort from ORDER BY ABS(): splits the
230 * query into below-target (DESC) and above-target (ASC), then merges
231 * by proximity to target in PHP.
232 *
233 * @param int $minNgramCount
234 * @param int $maxNgramCount
235 * @param int $limit
236 * @param int|null $targetNgramCount
237 * @param string|null $type When non-null, restrict to a single entity type
238 * ('post', 'category', 'tag', ...). Null preserves the all-types scan.
239 * @return array<int, array<string, mixed>>
240 */
241 public function getCachedNGramsFiltered($minNgramCount, $maxNgramCount, $limit = 1000, $targetNgramCount = null, $type = null) {
242 $table = $this->dbCore->tableNameResolver()->getPrefixedTableName('abj404_ngram_cache');
243
244 $orderTarget = ($targetNgramCount !== null)
245 ? max($minNgramCount, min($maxNgramCount, (int)$targetNgramCount))
246 : (int)(($minNgramCount + $maxNgramCount) / 2);
247
248 $halfLimit = (int)ceil($limit / 2);
249
250 $resultsBelow = $this->fetchBelowTarget($table, $minNgramCount, $orderTarget, $halfLimit, 0, $type);
251 $belowCount = count($resultsBelow);
252 $aboveLimit = $limit - $belowCount;
253 $resultsAbove = $this->fetchAboveTarget($table, $orderTarget, $maxNgramCount, $aboveLimit, 0, $type);
254 $aboveCount = count($resultsAbove);
255
256 $totalFetched = $belowCount + $aboveCount;
257 // Balance for skewed distributions: if one side hit its cap and the
258 // other has headroom, pull more from the saturated side.
259 if ($totalFetched < $limit && $belowCount === $halfLimit) {
260 $extra = $this->fetchBelowTarget($table, $minNgramCount, $orderTarget, $limit - $totalFetched, $belowCount, $type);
261 $resultsBelow = array_merge($resultsBelow, $extra);
262 $totalFetched = count($resultsBelow) + $aboveCount;
263 }
264 if ($totalFetched < $limit && $aboveCount === $aboveLimit) {
265 $extra = $this->fetchAboveTarget($table, $orderTarget, $maxNgramCount, $limit - $totalFetched, $aboveCount, $type);
266 $resultsAbove = array_merge($resultsAbove, $extra);
267 }
268
269 $merged = $this->similarity->mergeByProximity($resultsBelow, $resultsAbove, $orderTarget, $limit);
270 return $this->decodeNGramRows($merged);
271 }
272
273 /**
274 * @param string $table
275 * @param int $minNgramCount
276 * @param int $orderTarget
277 * @param int $limit
278 * @param int $offset
279 * @param string|null $type Optional single-type restriction.
280 * @return array<int, mixed>
281 */
282 private function fetchBelowTarget($table, $minNgramCount, $orderTarget, $limit, $offset = 0, $type = null) {
283 $typeClause = ($type !== null) ? " AND type = %s" : '';
284 $params = ($type !== null)
285 ? [$minNgramCount, $orderTarget, (string)$type, $limit, $offset]
286 : [$minNgramCount, $orderTarget, $limit, $offset];
287 $result = $this->dbCore->queryAndGetResults(
288 "SELECT id, url, url_normalized, ngrams, ngram_count
289 FROM {$table}
290 WHERE ngram_count >= %d AND ngram_count <= %d{$typeClause}
291 ORDER BY ngram_count DESC
292 LIMIT %d OFFSET %d",
293 ['query_params' => $params]
294 );
295 return isset($result['rows']) && is_array($result['rows']) ? $result['rows'] : [];
296 }
297
298 /**
299 * @param string $table
300 * @param int $orderTarget
301 * @param int $maxNgramCount
302 * @param int $limit
303 * @param int $offset
304 * @param string|null $type Optional single-type restriction.
305 * @return array<int, mixed>
306 */
307 private function fetchAboveTarget($table, $orderTarget, $maxNgramCount, $limit, $offset = 0, $type = null) {
308 $typeClause = ($type !== null) ? " AND type = %s" : '';
309 $params = ($type !== null)
310 ? [$orderTarget, $maxNgramCount, (string)$type, $limit, $offset]
311 : [$orderTarget, $maxNgramCount, $limit, $offset];
312 $result = $this->dbCore->queryAndGetResults(
313 "SELECT id, url, url_normalized, ngrams, ngram_count
314 FROM {$table}
315 WHERE ngram_count > %d AND ngram_count <= %d{$typeClause}
316 ORDER BY ngram_count ASC
317 LIMIT %d OFFSET %d",
318 ['query_params' => $params]
319 );
320 return isset($result['rows']) && is_array($result['rows']) ? $result['rows'] : [];
321 }
322
323 /**
324 * @param array<int, mixed> $rows
325 * @return array<int, array<string, mixed>>
326 */
327 private function decodeNGramRows(array $rows) {
328 $validResults = [];
329 foreach ($rows as $row) {
330 if (!is_array($row)) {
331 continue;
332 }
333 $ngramsJson = isset($row['ngrams']) && is_string($row['ngrams']) ? $row['ngrams'] : '';
334 $decoded = json_decode($ngramsJson, true);
335 if ($decoded === null && json_last_error() !== JSON_ERROR_NONE) {
336 $rowId = isset($row['id']) ? $row['id'] : 0;
337 $this->logger->errorMessage(sprintf(
338 "Corrupt N-gram JSON for page ID %s: %s",
339 (is_scalar($rowId) ? (string)$rowId : '0'),
340 json_last_error_msg()
341 ));
342 continue;
343 }
344 $row['ngrams'] = $decoded;
345 $validResults[] = $row;
346 }
347 return $validResults;
348 }
349
350 /**
351 * Invalidate (delete) N-grams for a specific page.
352 *
353 * @param int $pageId
354 * @param string $type
355 * @return bool
356 */
357 public function invalidatePage($pageId, $type = 'post') {
358 $table = $this->dbCore->tableNameResolver()->getPrefixedTableName('abj404_ngram_cache');
359 $queryResult = $this->dbCore->queryAndGetResults(
360 "DELETE FROM {$table} WHERE id = %d AND type = %s",
361 ['query_params' => [(int)$pageId, $type]]
362 );
363
364 $lastError = isset($queryResult['last_error']) && is_string($queryResult['last_error']) ? $queryResult['last_error'] : '';
365 $success = $lastError === '';
366 if ($success) {
367 $this->cacheCountMemo = null;
368 $this->invalidateCoverageCaches();
369 }
370
371 return $success;
372 }
373
374 /**
375 * Get cache entry count (memoized per-request).
376 *
377 * Use this instead of getCacheStats() when only the count is needed.
378 *
379 * @return int
380 */
381 public function getCacheCount() {
382 if ($this->cacheCountMemo !== null) {
383 return $this->cacheCountMemo;
384 }
385
386 global $wpdb;
387 $table = $this->dbCore->tableNameResolver()->getPrefixedTableName('abj404_ngram_cache');
388 if (!isset($wpdb) || !is_object($wpdb) || !is_callable([$wpdb, 'get_var'])) {
389 // Test environments / very early bootstrap: treat as no cache.
390 $this->cacheCountMemo = 0;
391 return $this->cacheCountMemo;
392 }
393
394 $this->cacheCountMemo = $this->dbCore->queryScalarInt("SELECT COUNT(*) AS c FROM {$table}");
395 return $this->cacheCountMemo;
396 }
397
398 /**
399 * Count cache entries of a single type ('post', 'category', 'tag', ...).
400 *
401 * Used by the type-scoped term prefilter to make its load-limit decision
402 * and to feed the term coverage policy's readiness gate. Not memoized:
403 * the term path calls this at most twice per request (count + ratio),
404 * both against the indexed `type` column.
405 *
406 * @param string $type
407 * @return int
408 */
409 public function getCacheCountForType(string $type): int {
410 $table = $this->dbCore->tableNameResolver()->getPrefixedTableName('abj404_ngram_cache');
411 return $this->dbCore->queryScalarInt(
412 "SELECT COUNT(*) AS c FROM {$table} WHERE type = %s",
413 ['query_params' => [$type]]
414 );
415 }
416
417 /**
418 * Reset per-request memoization.
419 *
420 * Used after bulk operations that bypass storeNGrams() (e.g. TRUNCATE
421 * during rebuild) so the next getCacheCount() reads fresh state.
422 *
423 * @return void
424 */
425 public function resetMemo() {
426 $this->cacheCountMemo = null;
427 }
428
429 /**
430 * Get cache statistics for admin display.
431 *
432 * @return array<string, mixed>
433 */
434 public function getCacheStats() {
435 $table = $this->dbCore->tableNameResolver()->getPrefixedTableName('abj404_ngram_cache');
436
437 $totalEntries = $this->dbCore->queryScalarInt("SELECT COUNT(*) AS c FROM {$table}");
438 $postsEntries = $this->getCacheCountForType('post');
439 $categoryEntries = $this->getCacheCountForType('category');
440 $tagEntries = $this->getCacheCountForType('tag');
441 $lastUpdatedResult = $this->dbCore->queryAndGetResults(
442 "SELECT MAX(last_updated) AS m FROM {$table}"
443 );
444 $lastUpdatedRows = isset($lastUpdatedResult['rows']) && is_array($lastUpdatedResult['rows']) ? $lastUpdatedResult['rows'] : [];
445 $lastUpdatedFirst = $lastUpdatedRows[0] ?? null;
446 $lastUpdated = is_array($lastUpdatedFirst) && isset($lastUpdatedFirst['m']) ? $lastUpdatedFirst['m'] : null;
447
448 return [
449 'total_entries' => $totalEntries,
450 'posts_entries' => $postsEntries,
451 'category_entries' => $categoryEntries,
452 'tag_entries' => $tagEntries,
453 'last_updated' => $lastUpdated,
454 ];
455 }
456
457 /**
458 * Invalidate coverage caches by resolving the policy lazily.
459 *
460 * Lazy resolution avoids constructor-time coupling between the
461 * repository and coverage policy services.
462 *
463 * @return void
464 */
465 private function invalidateCoverageCaches() {
466 $resolver = $this->coveragePolicyResolver;
467 if (!is_callable($resolver)) {
468 return;
469 }
470 $policy = $resolver();
471 if ($policy instanceof ABJ_404_Solution_NGramCoveragePolicy) {
472 $policy->invalidateCoverageCaches();
473 }
474 }
475 }
476