PluginProbe
404 Solution / trunk
404 Solution vtrunk
4.3.5 4.3.4 4.3.3 4.3.2 4.3.1 4.3.0 4.2.0 4.1.19 4.1.18 4.1.17 4.1.16 4.1.15 4.1.13 4.1.12 4.1.11 4.1.10 4.1.9 4.1.8 4.1.7 4.1.6 4.1.5 4.1.4 4.1.3 trunk 2.30.0 All 109 releases
404-solution / includes / ngram / NGramFilter.php

NGramFilter.php in 404 Solution trunk, at includes/ngram/NGramFilter.php

482 lines 18.5 KB
No matching file
Up and down to move Enter to open Esc to close
Raw Download Zip
1 <?php
2
3
4 if (!defined('ABSPATH')) {
5 exit;
6 }
7
8 // Backward-compat: legacy tests/code do `require_once NGramFilter.php`
9 // and expect every collaborator class to be available.
10 require_once __DIR__ . '/NGramExtractor.php';
11 require_once __DIR__ . '/NGramSimilarity.php';
12 require_once __DIR__ . '/NGramCacheRepository.php';
13 require_once __DIR__ . '/NGramCoveragePolicy.php';
14 require_once __DIR__ . '/NGramRebuilder.php';
15 require_once __DIR__ . '/NGramUsageTelemetry.php';
16 require_once __DIR__ . '/NGramFilterCollaboratorResolver.php';
17
18 /**
19 * Candidate selection orchestrator: maps a 404 URL to a ranked subset of
20 * cached pages worth Levenshtein-comparing.
21 *
22 * After i804 this class composes single-responsibility collaborators rather
23 * than holding their logic inline. The orchestration shape here is:
24 *
25 * 1. Extractor: char N-grams from the 404 URL.
26 * 2. CoveragePolicy: gates whether the cache is trustworthy this request.
27 * 3. CacheRepository: range-filtered candidate load (or full load for
28 * small caches).
29 * 4. Similarity: Dice coefficient per candidate.
30 * 5. Sort + cap.
31 * 6. UsageTelemetry: rolling counters for admin diagnostics.
32 *
33 * Service name remains `ngram_filter` for backward compatibility; production
34 * callers that only need a specific collaborator (e.g. ContentRepository
35 * invalidating coverage caches) wire to the new services directly.
36 */
37 class ABJ_404_Solution_NGramFilter {
38
39 // Constants forwarded from the new collaborators for backward compat
40 // with callers (and tests) that reference NGramFilter::CONST_NAME.
41 const COVERAGE_RATIO_CACHE_TTL = ABJ_404_Solution_NGramCoveragePolicy::COVERAGE_RATIO_CACHE_TTL;
42 const COVERAGE_VERSION_TTL = ABJ_404_Solution_NGramCoveragePolicy::COVERAGE_VERSION_TTL;
43 const COVERAGE_VERSION_KEY = ABJ_404_Solution_NGramCoveragePolicy::COVERAGE_VERSION_KEY;
44 const COVERAGE_RATIO_KEY = ABJ_404_Solution_NGramCoveragePolicy::COVERAGE_RATIO_KEY;
45 const CACHE_LOAD_LIMIT = ABJ_404_Solution_NGramCacheRepository::CACHE_LOAD_LIMIT;
46
47 /** @var self|null */
48 private static $instance = null;
49 /**
50 * Test seam: install or clear the cached singleton instance without
51 * private-field reflection. Pass null to reset between tests; pass a
52 * configured instance (or double) to install it. Mirrors the setInstance()
53 * contract on DataAccess / PluginLogic (M105 singleton-reset seam).
54 *
55 * @param self|null $instance
56 * @return void
57 */
58 public static function setInstance($instance) {
59 self::$instance = $instance;
60 }
61
62
63 /** @var ABJ_404_Solution_Logging */
64 private $logger;
65
66 /** @var ABJ_404_Solution_Functions */
67 private $f;
68
69 /** @var ABJ_404_Solution_NGramExtractor */
70 private $extractor;
71
72 /** @var ABJ_404_Solution_NGramSimilarity */
73 private $similarity;
74
75 /** @var ABJ_404_Solution_NGramCacheRepository */
76 private $repo;
77
78 /** @var ABJ_404_Solution_NGramCoveragePolicy */
79 private $coveragePolicy;
80
81 /** @var ABJ_404_Solution_NGramUsageTelemetry */
82 private $telemetry;
83
84 /**
85 * Constructor with dependency injection.
86 *
87 * Legacy three-arg form (dbCore, logging, functions) is preserved so
88 * pre-i804 ServiceContainer wiring and test fixtures don't break; in
89 * that mode the collaborators are resolved from abj_service(). Tests
90 * needing isolation pass all six explicitly.
91 *
92 * @param ABJ_404_Solution_DatabaseCore|null $dbCore Legacy; unused if all collaborators are passed.
93 * @param ABJ_404_Solution_Logging|null $logging
94 * @param ABJ_404_Solution_Functions|null $functions
95 * @param ABJ_404_Solution_NGramExtractor|null $extractor
96 * @param ABJ_404_Solution_NGramSimilarity|null $similarity
97 * @param ABJ_404_Solution_NGramCacheRepository|null $repo
98 * @param ABJ_404_Solution_NGramCoveragePolicy|null $coveragePolicy
99 * @param ABJ_404_Solution_NGramUsageTelemetry|null $telemetry
100 */
101 public function __construct(
102 $dbCore = null,
103 $logging = null,
104 $functions = null,
105 $extractor = null,
106 $similarity = null,
107 $repo = null,
108 $coveragePolicy = null,
109 $telemetry = null
110 ) {
111 $loggingResolved = $logging !== null ? $logging : abj_service('logging');
112 $functionsResolved = $functions !== null ? $functions : abj_service('functions');
113 if (!$loggingResolved instanceof ABJ_404_Solution_Logging) {
114 throw new RuntimeException('NGramFilter requires a Logging instance.');
115 }
116 if (!$functionsResolved instanceof ABJ_404_Solution_Functions) {
117 throw new RuntimeException('NGramFilter requires a Functions instance.');
118 }
119 $this->logger = $loggingResolved;
120 $this->f = $functionsResolved;
121
122 // Collaborators: prefer the explicit arg, then the container-registered
123 // instance, then a direct instantiation for unit-test contexts that
124 // load NGramFilter standalone without bootstrap. The new collaborators
125 // are pure composition over $dbCore/$logger/$f already, so direct
126 // construction is equivalent.
127 $this->extractor = ABJ_404_Solution_NGramFilterCollaboratorResolver::resolveExtractor($extractor, $this->f, $this->logger);
128 $this->similarity = ABJ_404_Solution_NGramFilterCollaboratorResolver::resolveSimilarity($similarity);
129 $this->repo = ABJ_404_Solution_NGramFilterCollaboratorResolver::resolveRepo($repo, $dbCore, $this->logger, $this->similarity);
130 $this->coveragePolicy = ABJ_404_Solution_NGramFilterCollaboratorResolver::resolveCoveragePolicy($coveragePolicy, $dbCore);
131 $this->telemetry = ABJ_404_Solution_NGramFilterCollaboratorResolver::resolveTelemetry($telemetry);
132 }
133
134 /** @return self */
135 public static function getInstance() {
136 if (self::$instance == null) {
137 self::$instance = new ABJ_404_Solution_NGramFilter();
138 }
139 return self::$instance;
140 }
141
142 /**
143 * Find pages similar to a 404 URL using N-gram filtering.
144 *
145 * Main entry point called by SpellLevenshteinEngine. Returns associative
146 * array of [pageId => similarity_score] sorted descending.
147 *
148 * @param string $url404
149 * @param float $minSimilarity Minimum Dice coefficient (default 0.4)
150 * @param int $maxCandidates Maximum candidates to return (default 100)
151 * @return array<int, float>
152 */
153 public function findSimilarPages($url404, $minSimilarity = 0.4, $maxCandidates = 100) {
154 return $this->findSimilar($url404, $minSimilarity, $maxCandidates, null);
155 }
156
157 /**
158 * Find taxonomy terms (categories or tags) similar to a 404 URL using the
159 * same N-gram filtering posts use, but restricted to a single cache type
160 * so post ids never collide with term ids.
161 *
162 * Returns [termId => similarity_score] sorted descending. The caller
163 * (SpellSuggestionScorer / CategoryTagMatchingEngine) is responsible for
164 * only invoking this when the term coverage policy reports the type's
165 * cache is trustworthy; on a cold cache it falls back to the full
166 * getPublishedCategories()/getPublishedTags() scan.
167 *
168 * @param string $url404
169 * @param string $type Cache type to restrict to: 'category' or 'tag'.
170 * @param float $minSimilarity Minimum Dice coefficient.
171 * @param int $maxCandidates Maximum candidates to return.
172 * @return array<int, float>
173 */
174 public function findSimilarTermIds($url404, $type, $minSimilarity = 0.4, $maxCandidates = 100) {
175 return $this->findSimilar($url404, $minSimilarity, $maxCandidates, (string)$type);
176 }
177
178 /**
179 * Shared candidate-selection pipeline for both the all-types post path
180 * (type=null) and the single-type term path (type='category'|'tag').
181 *
182 * @param string $url404
183 * @param float $minSimilarity
184 * @param int $maxCandidates
185 * @param string|null $type Null = historical all-types scan (posts).
186 * @return array<int, float>
187 */
188 private function findSimilar($url404, $minSimilarity, $maxCandidates, $type) {
189 $startTime = abj_clock()->nowFloat();
190
191 $url404Normalized = $this->f->strtolower(trim($url404));
192 $queryNGrams = $this->extractor->extractNGrams($url404Normalized);
193 $queryCombinedCount = count($queryNGrams['bi']) + count($queryNGrams['tri']);
194
195 if ($queryCombinedCount == 0) {
196 $this->logger->debugMessage("Search term too short for N-gram filtering: '{$url404}'");
197 return [];
198 }
199
200 $totalCount = ($type !== null) ? $this->repo->getCacheCountForType($type) : $this->repo->getCacheCount();
201
202 if ($totalCount == 0) {
203 $this->logger->debugMessage("N-gram cache is empty" . ($type !== null ? " for type '{$type}'." : "."));
204
205 // Only the all-types (post) path schedules a rebuild on an empty
206 // cache. The term path is gated upstream by the coverage policy
207 // and falls back to a full taxonomy scan when cold, so it must not
208 // trigger rebuild scheduling here.
209 if ($type === null) {
210 $this->scheduleRebuildIfNeeded();
211 }
212
213 return [];
214 }
215
216 // 40% tolerance window around query's ngram count
217 $minCount = max(1, (int)($queryCombinedCount * 0.4));
218 $maxCount = (int)($queryCombinedCount * 2.5);
219
220 if ($totalCount > ABJ_404_Solution_NGramCacheRepository::CACHE_LOAD_LIMIT) {
221 $this->logger->debugMessage("Using database-side filtering for {$totalCount} entries");
222 $cachedPages = $this->repo->getCachedNGramsFiltered($minCount, $maxCount, ABJ_404_Solution_NGramCacheRepository::CACHE_LOAD_LIMIT, $queryCombinedCount, $type);
223 } else {
224 $cachedPages = $this->repo->getAllCachedNGrams($type);
225 }
226
227 if (empty($cachedPages)) {
228 $this->logger->debugMessage("No matching candidates after filtering.");
229 return [];
230 }
231
232 $similarities = $this->scoreCachedRows($cachedPages, $queryNGrams, $queryCombinedCount, $minSimilarity);
233
234 arsort($similarities);
235
236 if (count($similarities) > $maxCandidates) {
237 $similarities = array_slice($similarities, 0, $maxCandidates, true);
238 }
239
240 $duration = (abj_clock()->nowFloat() - $startTime) * 1000;
241
242 $this->logger->debugMessage(sprintf(
243 "N-gram filtering: %d total, %d examined -> %d candidates (>=%.2f similarity) in %.2fms",
244 $totalCount,
245 count($cachedPages),
246 count($similarities),
247 $minSimilarity,
248 $duration
249 ));
250
251 $this->telemetry->trackNGramUsage($totalCount, count($cachedPages), count($similarities), $duration);
252
253 return $similarities;
254 }
255
256 /**
257 * Per-row Dice scoring loop shared by the post and term paths.
258 *
259 * @param array<int, mixed> $cachedPages Rows from the cache repository (ngrams already decoded).
260 * @param array{bi: array<int, string>, tri: array<int, string>} $queryNGrams
261 * @param int $queryCombinedCount
262 * @param float $minSimilarity
263 * @return array<int, float> [id => similarity]
264 */
265 private function scoreCachedRows(array $cachedPages, array $queryNGrams, $queryCombinedCount, $minSimilarity) {
266 $similarities = [];
267 foreach ($cachedPages as $page) {
268 if (!is_array($page)) {
269 continue;
270 }
271 $pageId = isset($page['id']) ? $page['id'] : null;
272 $pageNGrams = isset($page['ngrams']) ? $page['ngrams'] : null;
273
274 // Quick optimization: skip if N-gram counts differ too much.
275 // Redundant for filtered queries; preserved for unfiltered path.
276 $pageNgramCountRaw = isset($page['ngram_count']) ? $page['ngram_count'] : 0;
277 $pageCombinedCount = is_scalar($pageNgramCountRaw) ? (int)$pageNgramCountRaw : 0;
278 $commonUpperBound = min($queryCombinedCount, $pageCombinedCount);
279 $diceUpperBound = (2 * $commonUpperBound) / max(1, $queryCombinedCount + $pageCombinedCount);
280 if ($diceUpperBound < $minSimilarity) {
281 continue;
282 }
283
284 /** @var array{bi?: array<int, string>, tri?: array<int, string>} $pageNGramsTyped */
285 $pageNGramsTyped = is_array($pageNGrams) ? $pageNGrams : array();
286 $sim = $this->similarity->diceCoefficient($queryNGrams, $pageNGramsTyped);
287
288 if ($sim >= $minSimilarity) {
289 $similarities[$pageId] = $sim;
290 }
291 }
292 return $similarities;
293 }
294
295 /**
296 * Schedule a background N-gram rebuild when the all-types cache is empty
297 * and not already initialized. Multisite-aware init check so
298 * network-activated installs read get_site_option on frontend dispatch.
299 *
300 * @return void
301 */
302 private function scheduleRebuildIfNeeded() {
303 if (!$this->coveragePolicy->isCacheInitialized()) {
304 try {
305 $dbUpgrades = abj_service('database_upgrades');
306 $dbUpgrades->components()->nGramUpgrade()->scheduleNGramCacheRebuild();
307 $this->logger->infoMessage("Empty N-gram cache detected during 404 request. Scheduled background rebuild.");
308 } catch (Exception $e) {
309 $this->logger->errorMessage("Failed to schedule N-gram cache rebuild: " . $e->getMessage());
310 }
311 } else {
312 $this->logger->debugMessage("N-gram cache rebuild already initialized or scheduled.");
313 }
314 }
315
316 /**
317 * Convenience predicate for SpellLevenshteinEngine.
318 *
319 * @return bool
320 */
321 public function isCachePopulated() {
322 return $this->repo->getCacheCount() > 0;
323 }
324
325 // Legacy compatibility methods for external/test callers that still
326 // subclass or hold the historical filter service.
327
328 /**
329 * @param string $url
330 * @param array<int, int> $ngramSizes
331 * @return array{bi: array<int, string>, tri: array<int, string>}
332 */
333 public function extractNGrams($url, $ngramSizes = [2, 3]) {
334 return $this->extractor->extractNGrams($url, $ngramSizes);
335 }
336
337 /**
338 * @param array{bi?: array<int, string>, tri?: array<int, string>} $ngrams1
339 * @param array{bi?: array<int, string>, tri?: array<int, string>} $ngrams2
340 * @return float
341 */
342 public function diceCoefficient($ngrams1, $ngrams2) {
343 return $this->similarity->diceCoefficient($ngrams1, $ngrams2);
344 }
345
346 /**
347 * @param int $pageId
348 * @param string $url
349 * @param string $urlNormalized
350 * @param array<string, mixed> $ngrams
351 * @param string $type
352 * @param bool $skipInvalidation
353 * @return bool
354 */
355 public function storeNGrams($pageId, $url, $urlNormalized, $ngrams, $type = 'post', $skipInvalidation = false) {
356 return $this->repo->storeNGrams($pageId, $url, $urlNormalized, $ngrams, $type, $skipInvalidation);
357 }
358
359 /**
360 * @param int $pageId
361 * @param string $type
362 * @return array{bi: array<int, string>, tri: array<int, string>}|null
363 */
364 public function getNGramsForPage($pageId, $type = 'post') {
365 return $this->repo->getNGramsForPage($pageId, $type);
366 }
367
368 /** @return array<int, array<string, mixed>> */
369 public function getAllCachedNGrams() {
370 return $this->repo->getAllCachedNGrams();
371 }
372
373 /**
374 * @param int $minNgramCount
375 * @param int $maxNgramCount
376 * @param int $limit
377 * @param int|null $targetNgramCount
378 * @return array<int, array<string, mixed>>
379 */
380 public function getCachedNGramsFiltered($minNgramCount, $maxNgramCount, $limit = 1000, $targetNgramCount = null) {
381 return $this->repo->getCachedNGramsFiltered($minNgramCount, $maxNgramCount, $limit, $targetNgramCount);
382 }
383
384 /**
385 * @param int $pageId
386 * @param string $type
387 * @return bool
388 */
389 public function invalidatePage($pageId, $type = 'post') {
390 return $this->repo->invalidatePage($pageId, $type);
391 }
392
393 /** @return int */
394 public function getCacheCount() {
395 return $this->repo->getCacheCount();
396 }
397
398 /**
399 * @param string $type
400 * @return int
401 */
402 public function getCacheCountForType($type) {
403 return $this->repo->getCacheCountForType((string)$type);
404 }
405
406 /** @return array<string, mixed> */
407 public function getCacheStats() {
408 return $this->repo->getCacheStats();
409 }
410
411 /** @return void */
412 public function invalidateCoverageCaches() {
413 $this->coveragePolicy->invalidateCoverageCaches();
414 $this->repo->resetMemo();
415 }
416
417 /** @return bool */
418 public function isCacheInitialized() {
419 return $this->coveragePolicy->isCacheInitialized();
420 }
421
422 /** @return float */
423 public function getCacheCoverageRatio() {
424 return $this->coveragePolicy->getCacheCoverageRatio();
425 }
426
427 /**
428 * @param int $batchSize
429 * @param int $offset
430 * @return array{processed: int, success: int, failed: int}
431 */
432 public function rebuildCache($batchSize = 100, $offset = 0) {
433 return $this->rebuilder()->rebuildCache($batchSize, $offset);
434 }
435
436 /**
437 * @param array<int, int> $pageIds
438 * @return array{processed: int, success: int, failed: int}
439 */
440 public function updateNGramsForPages($pageIds) {
441 return $this->rebuilder()->updateNGramsForPages($pageIds);
442 }
443
444 /** @return array<string, mixed> */
445 public function getUsageStats() {
446 return $this->telemetry->getUsageStats();
447 }
448
449 /**
450 * Lazy rebuilder accessor. Rebuilder isn't part of the constructor
451 * collaborator list (it's an orchestration-time helper rather than a
452 * findSimilarPages dependency); resolved through the container when a
453 * facade caller needs it.
454 *
455 * @return ABJ_404_Solution_NGramRebuilder
456 */
457 private function rebuilder() {
458 if (class_exists('ABJ_404_Solution_ServiceContainer')) {
459 $container = ABJ_404_Solution_ServiceContainer::getInstance();
460 if ($container->has('ngram_rebuilder')) {
461 $service = $container->get('ngram_rebuilder');
462 if ($service instanceof ABJ_404_Solution_NGramRebuilder) {
463 return $service;
464 }
465 }
466 }
467 // Container miss (unit tests that haven't registered the service):
468 // build a transient rebuilder from already-resolved collaborators so
469 // the facade still works without container plumbing.
470 return new ABJ_404_Solution_NGramRebuilder(
471 new ABJ_404_Solution_NGramRebuilderDependencies(
472 null,
473 $this->logger,
474 $this->f,
475 $this->extractor,
476 $this->repo,
477 $this->coveragePolicy
478 )
479 );
480 }
481 }
482