PluginProbe
404 Solution / 4.2.0
404 Solution v4.2.0
4.3.5 4.3.4 4.3.3 4.3.2 4.3.1 4.3.0 4.2.0 4.1.19 4.1.18 4.1.17 4.1.16 4.1.15 4.1.13 4.1.12 4.1.11 4.1.10 4.1.9 4.1.8 4.1.7 4.1.6 4.1.5 4.1.4 4.1.3 trunk 2.30.0 All 109 releases
404-solution / includes / SpellLevenshteinEngine.php

SpellLevenshteinEngine.php in 404 Solution 4.2.0, at includes/SpellLevenshteinEngine.php

544 lines 17.8 KB
No matching file
Up and down to move Enter to open Esc to close
Raw Download Zip
1 <?php
2
3 if (!defined('ABSPATH')) {
4 exit;
5 }
6
7 /**
8 * Levenshtein distance engine and candidate pre-filtering for the
9 * spell-checking subsystem.
10 *
11 * Extracted from SpellCheckerTrait_LevenshteinEngine as a standalone class
12 * with explicit dependency injection.
13 */
14 class ABJ_404_Solution_SpellLevenshteinEngine {
15
16 const MAX_DIST = 2083;
17
18 const MAX_LIKELY_DISTANCE = 300;
19
20 const NGRAM_PREFILTER_THRESHOLD = 0.3;
21
22 const NGRAM_PREFILTER_MAX_CANDIDATES = 500;
23
24 const NGRAM_MIN_CACHE_ENTRIES = 50;
25
26 const NGRAM_SECONDARY_THRESHOLD = 0.4;
27
28 const NGRAM_SECONDARY_MAX_CANDIDATES = 100;
29
30 const NGRAM_MIN_COVERAGE_RATIO = 0.8;
31
32 const NGRAM_SECONDARY_MIN_CANDIDATES = 50;
33
34 /** @var ABJ_404_Solution_Functions */
35 private $f;
36
37 /** @var ABJ_404_Solution_PluginLogic */
38 private $logic;
39
40 /** @var ABJ_404_Solution_Logging */
41 private $logger;
42
43 /** @var ABJ_404_Solution_ContentRepository */
44 private $contentRepository;
45
46 /** @var ABJ_404_Solution_NGramFilter */
47 private $ngramFilter;
48
49 /** @var ABJ_404_Solution_SpellURLMatcher */
50 private $urlMatcher;
51
52 /** @var array<int, string> */
53 private array $separatingCharacters;
54
55 private bool $enablePerformanceCounters = false;
56
57 private bool $skipNgramGate4 = false;
58
59 private int $levenshteinCallCount = 0;
60
61 private int $totalPagesConsidered = 0;
62
63 /** @var ABJ_404_Solution_PublishedPostsProvider|null */
64 private ?ABJ_404_Solution_PublishedPostsProvider $publishedPostsProvider = null;
65
66 /**
67 * @param ABJ_404_Solution_Functions $functions
68 * @param ABJ_404_Solution_PluginLogic $logic
69 * @param ABJ_404_Solution_Logging $logger
70 * @param ABJ_404_Solution_ContentRepository $contentRepository
71 * @param ABJ_404_Solution_NGramFilter $ngramFilter
72 * @param ABJ_404_Solution_SpellURLMatcher $urlMatcher
73 * @param array<int, string> $separatingCharacters
74 */
75 public function __construct($functions, $logic, $logger, $contentRepository, $ngramFilter, $urlMatcher, array $separatingCharacters) {
76 $this->f = $functions;
77 $this->logic = $logic;
78 $this->logger = $logger;
79 $this->contentRepository = $contentRepository;
80 $this->ngramFilter = $ngramFilter;
81 $this->urlMatcher = $urlMatcher;
82 $this->separatingCharacters = $separatingCharacters;
83 }
84
85 /** @param ABJ_404_Solution_PublishedPostsProvider|null $provider */
86 public function setPublishedPostsProvider(?ABJ_404_Solution_PublishedPostsProvider $provider): void {
87 $this->publishedPostsProvider = $provider;
88 }
89
90 public function enablePerformanceCounters(bool $enable = true): void {
91 $this->enablePerformanceCounters = $enable;
92 if ($enable) {
93 $this->resetPerformanceCounters();
94 }
95 }
96
97 public function setSkipNgramGate4(bool $skip = true): void {
98 $this->skipNgramGate4 = $skip;
99 }
100
101 public function resetPerformanceCounters(): void {
102 $this->levenshteinCallCount = 0;
103 $this->totalPagesConsidered = 0;
104 }
105
106 /**
107 * @return array{levenshtein_calls: int, pages_considered: int, efficiency_percent: float}
108 */
109 public function getPerformanceCounters(): array {
110 $efficiency = 0;
111 if ($this->totalPagesConsidered > 0) {
112 $efficiency = ($this->levenshteinCallCount / $this->totalPagesConsidered) * 100;
113 }
114
115 return [
116 'levenshtein_calls' => $this->levenshteinCallCount,
117 'pages_considered' => $this->totalPagesConsidered,
118 'efficiency_percent' => round($efficiency, 2)
119 ];
120 }
121
122 /**
123 * @param string $requestedURLCleaned
124 * @param string $fullURLspaces
125 * @param string $rowType
126 * @param array<int, array<string, mixed>>|null $rows
127 * @return array<int|string, mixed>
128 */
129 function getLikelyMatchIDs(string $requestedURLCleaned, string $fullURLspaces, string $rowType, ?array $rows = null) {
130
131 $options = $this->logic->getOptions();
132 $suggestMaxLikely = isset($options['suggest_max']) && is_scalar($options['suggest_max']) ? $options['suggest_max'] : 5;
133 $onlyNeedThisManyPages = min(5 * absint($suggestMaxLikely), 100);
134
135 $ngramPrefilterResult = $this->tryApplyNgramPrefilter($rowType, $rows, $requestedURLCleaned);
136 if ($ngramPrefilterResult === 'early_return') {
137 return array();
138 }
139 $ngramPrefilterApplied = ($ngramPrefilterResult === 'applied');
140
141 $minDistances = array();
142 $maxDistances = array();
143 for ($currentDistanceIndex = 0; $currentDistanceIndex <= self::MAX_DIST; $currentDistanceIndex++) {
144 $maxDistances[$currentDistanceIndex] = array();
145 $minDistances[$currentDistanceIndex] = array();
146 }
147
148 $requestedURLCleanedLength = $this->f->strlen($requestedURLCleaned);
149 $fullURLspacesLength = $this->f->strlen($fullURLspaces);
150
151 $userRequestedURLWords = explode(" ", (empty($fullURLspaces) ? $requestedURLCleaned : $fullURLspaces));
152 $idsWithWordsInCommon = array();
153 $observedPermalinksById = array();
154 $wasntReadyCount = 0;
155
156 if ($this->publishedPostsProvider === null) {
157 return array();
158 }
159 if (!$ngramPrefilterApplied) {
160 $this->publishedPostsProvider->resetBatch();
161 }
162 if ($rows != null) {
163 $this->publishedPostsProvider->useThisData($rows);
164 }
165 $currentBatch = $this->publishedPostsProvider->getNextBatch($requestedURLCleanedLength);
166
167 $row = array_pop($currentBatch);
168 while ($row != null) {
169 $row = (array)$row;
170
171 if ($this->enablePerformanceCounters) {
172 $this->totalPagesConsidered++;
173 }
174
175 $id = null;
176 $the_permalink = null;
177 $urlParts = null;
178 if ($rowType == 'pages') {
179 $id = $row['id'];
180
181 } else if ($rowType == 'tags') {
182 $id = array_key_exists('term_id', $row) ? $row['term_id'] : null;
183
184 } else if ($rowType == 'categories') {
185 $id = array_key_exists('term_id', $row) ? $row['term_id'] : null;
186
187 } else if ($rowType == 'image') {
188 $id = $row['id'];
189
190 } else {
191 throw new \Exception("Unknown row type ... " . esc_html($rowType)); // allow-raw-error: assertion, should never reach user
192 }
193
194 if ($id === null) {
195 $row = array_pop($currentBatch);
196 continue;
197 }
198 $idInt = is_scalar($id) ? (int)$id : 0;
199
200 if (array_key_exists('url', $row)) {
201 $the_permalink = isset($row['url']) && is_string($row['url']) ? $row['url'] : '';
202 $the_permalink = $this->f->normalizeUrlString($the_permalink);
203 $urlParts = parse_url($the_permalink);
204
205 if (is_bool($urlParts)) {
206 $this->contentRepository->removeFromPermalinkCache($idInt);
207 }
208 }
209 if (!array_key_exists('url', $row) || (isset($urlParts) && is_bool($urlParts))) {
210 $wasntReadyCount++;
211 $the_permalink = $this->urlMatcher->getPermalink($idInt, $rowType);
212 $the_permalink = $this->f->normalizeUrlString($the_permalink);
213 $urlParts = parse_url($the_permalink);
214 }
215
216 abj_service('request_context')->debug_info = 'Likely match IDs processing permalink: ' .
217 $the_permalink . ', $wasntReadyCount: ' . $wasntReadyCount;
218
219 if (!is_array($urlParts) || !array_key_exists('path', $urlParts)) {
220 continue;
221 }
222 if (is_string($the_permalink)) {
223 $observedPermalinksById[$idInt] = $the_permalink;
224 }
225 $existingPageURL = $this->logic->removeHomeDirectory($urlParts['path']);
226 $urlParts = null;
227
228 $existingPageURLSpaces = $this->f->str_replace($this->separatingCharacters, " ", $existingPageURL);
229
230 $existingPageURLCleaned = $this->urlMatcher->getLastURLPart($existingPageURLSpaces);
231 $existingPageURLSpaces = null;
232
233 $minDist = abs($this->f->strlen($existingPageURLCleaned) - $requestedURLCleanedLength);
234 if ($fullURLspaces != '') {
235 $minDist = min($minDist, abs($fullURLspacesLength - $requestedURLCleanedLength));
236 }
237 $maxDist = $this->f->strlen($existingPageURLCleaned);
238 if ($fullURLspaces != '') {
239 $maxDist = min($maxDist, $fullURLspacesLength);
240 }
241
242 $existingPageURLCleanedWords = explode(" ", $existingPageURLCleaned);
243 $wordsInCommon = array_intersect($userRequestedURLWords, $existingPageURLCleanedWords);
244 $wordsInCommon = array_merge(array_unique($wordsInCommon, SORT_REGULAR), array());
245 if (count($wordsInCommon) > 0) {
246 array_push($idsWithWordsInCommon, $id);
247 $lengthOfTheLongestWordInCommon = max(array_map(array($this->f,'strlen'), $wordsInCommon));
248 $maxDist = $maxDist - $lengthOfTheLongestWordInCommon;
249 }
250
251 if (isset($minDistances[$minDist])) {
252 array_push($minDistances[$minDist], $id);
253 } else {
254 $minDistances[$minDist] = [$id];
255 }
256
257 if ($maxDist < 0) {
258 $this->logger->errorMessage("maxDist is less than 0 (" . $maxDist .
259 ") for '" . $existingPageURLCleaned . "', wordsInCommon: " .
260 json_encode($wordsInCommon) . ", ");
261 $maxDist = 0;
262 } else if ($maxDist > self::MAX_DIST) {
263 $maxDist = self::MAX_DIST;
264 }
265
266 if (is_array($maxDistances[$maxDist])) {
267 array_push($maxDistances[$maxDist], $id);
268 }
269
270 $row = array_pop($currentBatch);
271 if ($row == null) {
272 $maxAcceptableDistance = $this->getMaxAcceptableDistance($maxDistances, $onlyNeedThisManyPages);
273
274 $currentBatch = $this->publishedPostsProvider->getNextBatch(
275 $requestedURLCleanedLength, 1000, $maxAcceptableDistance);
276 $row = array_pop($currentBatch);
277 }
278 }
279 abj_service('request_context')->debug_info = '';
280
281 if ($wasntReadyCount > 0) {
282 $this->logger->infoMessage("The permalink cache wasn't ready for " . $wasntReadyCount . " IDs.");
283 }
284
285 $candidateIds = $this->pruneAndPrioritizeCandidates(
286 $maxDistances, $minDistances, $onlyNeedThisManyPages,
287 $idsWithWordsInCommon, $ngramPrefilterApplied, $requestedURLCleaned
288 );
289
290 return $this->batchLookupPermalinks(
291 array_values(array_unique($candidateIds)), $rowType, $observedPermalinksById
292 );
293 }
294
295 /**
296 * @param string $rowType
297 * @param array<int, array<string, mixed>>|null $rows
298 * @param string $requestedURLCleaned
299 * @return string 'applied' if prefilter was used, 'early_return' if no matches exist, 'skipped' otherwise
300 */
301 private function tryApplyNgramPrefilter(string $rowType, ?array $rows, string $requestedURLCleaned): string {
302 if ($rowType != 'pages' || $rows !== null) {
303 return 'skipped';
304 }
305
306 $cacheCount = $this->ngramFilter->getCacheCount();
307
308 if ($cacheCount < self::NGRAM_MIN_CACHE_ENTRIES) {
309 $this->logger->debugMessage(sprintf(
310 "N-gram prefilter skipped (gate 1: min entries): count=%d (need %d)",
311 $cacheCount, self::NGRAM_MIN_CACHE_ENTRIES
312 ));
313 return 'skipped';
314 }
315 if (!$this->ngramFilter->isCacheInitialized()) {
316 $this->logger->debugMessage(sprintf(
317 "N-gram prefilter skipped (gate 2: not initialized): count=%d", $cacheCount
318 ));
319 return 'skipped';
320 }
321 $coverageRatio = $this->ngramFilter->getCacheCoverageRatio();
322 if ($coverageRatio < self::NGRAM_MIN_COVERAGE_RATIO) {
323 $this->logger->debugMessage(sprintf(
324 "N-gram prefilter skipped (gate 3: low coverage): ratio=%.2f (need %.2f)",
325 $coverageRatio, self::NGRAM_MIN_COVERAGE_RATIO
326 ));
327 return 'skipped';
328 }
329
330 $similarPages = $this->ngramFilter->findSimilarPages(
331 $requestedURLCleaned, self::NGRAM_PREFILTER_THRESHOLD, self::NGRAM_PREFILTER_MAX_CANDIDATES
332 );
333
334 if (!empty($similarPages) && $this->publishedPostsProvider !== null) {
335 $candidateIds = array_keys($similarPages);
336 $this->publishedPostsProvider->resetBatch();
337 $this->publishedPostsProvider->restrictToIds($candidateIds);
338 $this->logger->debugMessage(sprintf(
339 "N-gram prefilter: Restricted to %d candidates (cache has %d entries, coverage=%.2f)",
340 count($candidateIds), $cacheCount, $coverageRatio
341 ));
342 return 'applied';
343 }
344
345 if ($this->skipNgramGate4) {
346 $this->logger->debugMessage(
347 "N-gram prefilter: zero candidates at Dice >= 0.3 \xe2\x80\x94 skipNgramGate4 is set, falling through to full scan"
348 );
349 return 'skipped';
350 }
351
352 $this->logger->debugMessage(
353 "N-gram prefilter: zero candidates at Dice >= 0.3 \xe2\x80\x94 no similar pages exist, returning early"
354 );
355 return 'early_return';
356 }
357
358 /**
359 * @param array<int, array<int, mixed>> $maxDistances
360 * @param array<int, array<int, mixed>> $minDistances
361 * @param int $onlyNeedThisManyPages
362 * @param array<int, mixed> $idsWithWordsInCommon
363 * @param bool $ngramPrefilterApplied
364 * @param string $requestedURLCleaned
365 * @return array<int, mixed>
366 */
367 private function pruneAndPrioritizeCandidates(
368 array $maxDistances, array $minDistances, int $onlyNeedThisManyPages,
369 array $idsWithWordsInCommon, bool $ngramPrefilterApplied, string $requestedURLCleaned
370 ): array {
371 $pagesSeenSoFar = 0;
372 $maxDistFound = self::MAX_LIKELY_DISTANCE;
373 for ($currentDistanceIndex = 0; $currentDistanceIndex <= self::MAX_LIKELY_DISTANCE; $currentDistanceIndex++) {
374 $pagesSeenSoFar += sizeof($maxDistances[$currentDistanceIndex]);
375 if ($pagesSeenSoFar >= $onlyNeedThisManyPages) {
376 $maxDistFound = $currentDistanceIndex;
377 break;
378 }
379 }
380
381 $listOfIDsToReturn = array();
382 for ($currentDistanceIndex = 0; $currentDistanceIndex <= $maxDistFound; $currentDistanceIndex++) {
383 $listOfMinDistanceIDs = $minDistances[$currentDistanceIndex];
384 $listOfIDsToReturn = array_merge($listOfIDsToReturn, $listOfMinDistanceIDs);
385 }
386
387 $idsWithWords = array_intersect($listOfIDsToReturn, $idsWithWordsInCommon);
388 $idsWithoutWords = array_diff($listOfIDsToReturn, $idsWithWordsInCommon);
389 $listOfIDsToReturn = array_merge($idsWithWords, $idsWithoutWords);
390
391 $beforeNGramCount = count($listOfIDsToReturn);
392 if (!$ngramPrefilterApplied
393 && $beforeNGramCount > self::NGRAM_SECONDARY_MIN_CANDIDATES
394 && $this->ngramFilter->getCacheCount() >= self::NGRAM_MIN_CACHE_ENTRIES
395 && $this->ngramFilter->isCacheInitialized()
396 && $this->ngramFilter->getCacheCoverageRatio() >= self::NGRAM_MIN_COVERAGE_RATIO) {
397 $similarPages = $this->ngramFilter->findSimilarPages(
398 $requestedURLCleaned,
399 self::NGRAM_SECONDARY_THRESHOLD,
400 min($beforeNGramCount, self::NGRAM_SECONDARY_MAX_CANDIDATES)
401 );
402 if (!empty($similarPages)) {
403 $ngramFilteredIDs = array_keys($similarPages);
404 $listOfIDsToReturn = array_intersect($listOfIDsToReturn, $ngramFilteredIDs);
405 usort($listOfIDsToReturn, function($a, $b) use ($similarPages) {
406 $simA = isset($similarPages[$a]) ? $similarPages[$a] : 0;
407 $simB = isset($similarPages[$b]) ? $similarPages[$b] : 0;
408 return $simB <=> $simA;
409 });
410 $this->logger->debugMessage(sprintf(
411 "N-gram filter (secondary): %d to %d candidates (%.1f%% reduction)",
412 $beforeNGramCount, count($listOfIDsToReturn),
413 100 * (1 - count($listOfIDsToReturn) / max(1, $beforeNGramCount))
414 ));
415 }
416 }
417
418 if (count($listOfIDsToReturn) > 300 && count($idsWithWordsInCommon) >= $onlyNeedThisManyPages) {
419 $maybeOKguesses = array_intersect($listOfIDsToReturn, $idsWithWordsInCommon);
420 return (count($maybeOKguesses) >= $onlyNeedThisManyPages)
421 ? $maybeOKguesses : $idsWithWordsInCommon;
422 }
423 return $listOfIDsToReturn;
424 }
425
426 /**
427 * @param array<int, mixed> $ids
428 * @param string $rowType
429 * @param array<int, string> $observedPermalinksById
430 * @return array<int|string, string>
431 */
432 private function batchLookupPermalinks(array $ids, string $rowType, array $observedPermalinksById = array()): array {
433 if (empty($ids)) {
434 return [];
435 }
436 $result = [];
437 if ($rowType === 'pages' || $rowType === 'image') {
438 $intIds = array_map(function($v) { return is_scalar($v) ? (int)$v : 0; }, $ids);
439 $rows = $this->contentRepository->getPermalinksByIds($intIds);
440 foreach ($rows as $row) {
441 $row = (array)$row;
442 if (isset($row['id'], $row['url']) && is_string($row['url'])) {
443 $result[(int)$row['id']] = $this->f->normalizeUrlString($row['url']);
444 }
445 }
446 foreach ($intIds as $id) {
447 if (!isset($result[$id]) && isset($observedPermalinksById[$id])) {
448 $result[$id] = $this->f->normalizeUrlString($observedPermalinksById[$id]);
449 }
450 }
451 } else {
452 foreach ($ids as $id) {
453 $idInt = is_scalar($id) ? (int)$id : 0;
454 $permalink = $this->urlMatcher->getPermalink($idInt, $rowType);
455 if (is_string($permalink) && $permalink !== '') {
456 $result[$id] = $permalink;
457 }
458 }
459 }
460 return $result;
461 }
462
463 /**
464 * @param array<int, array<int, mixed>> $maxDistances
465 * @param int $onlyNeedThisManyPages
466 * @return int
467 */
468 function getMaxAcceptableDistance(array $maxDistances, int $onlyNeedThisManyPages): int {
469 $pagesSeenSoFar = 0;
470 $currentDistanceIndex = 0;
471 $maxDistFound = self::MAX_LIKELY_DISTANCE;
472 for ($currentDistanceIndex = 0; $currentDistanceIndex <= self::MAX_LIKELY_DISTANCE; $currentDistanceIndex++) {
473 $pagesSeenSoFar += sizeof($maxDistances[$currentDistanceIndex]);
474
475 if ($pagesSeenSoFar >= $onlyNeedThisManyPages) {
476 $maxDistFound = $currentDistanceIndex;
477 break;
478 }
479 }
480
481 $acceptableDistance = (int)($maxDistFound * 1.1);
482 return $acceptableDistance;
483 }
484
485 /**
486 * @param string $str1
487 * @param string $str2
488 * @return int
489 * @throws Exception
490 */
491 function customLevenshtein($str1, $str2) {
492 if ($this->enablePerformanceCounters) {
493 $this->levenshteinCallCount++;
494 }
495 abj_service('request_context')->debug_info = 'customLevenshtein. str1: ' . esc_html($str1) . ', str2: ' . esc_html($str2);
496
497 $RowLen = $this->f->strlen($str1);
498 $ColLen = $this->f->strlen($str2);
499 $cost = 0;
500
501 if (max($RowLen, $ColLen) > ABJ404_MAX_URL_LENGTH) {
502 throw new \Exception("Maximum string length in customLevenshtein is " . // allow-raw-error: assertion, should never reach user
503 ABJ404_MAX_URL_LENGTH . ". Yours is " . max($RowLen, $ColLen) . ".");
504 }
505
506 if (strlen($str1) <= 255 && strlen($str2) <= 255) {
507 return levenshtein($str1, $str2);
508 }
509
510 if ($RowLen == 0) {
511 return $ColLen;
512 } else if ($ColLen == 0) {
513 return $RowLen;
514 }
515
516 $chars1 = mb_str_split($str1, 1, 'UTF-8');
517 $chars2 = mb_str_split($str2, 1, 'UTF-8');
518
519 $v0 = array_fill(0, $RowLen + 1, 0);
520 $v1 = array_fill(0, $RowLen + 1, 0);
521
522 for ($RowIdx = 1; $RowIdx <= $RowLen; $RowIdx++) {
523 $v0[$RowIdx] = $RowIdx;
524 }
525
526 for ($ColIdx = 1; $ColIdx <= $ColLen; $ColIdx++) {
527 $v1[0] = $ColIdx;
528
529 for ($RowIdx = 1; $RowIdx <= $RowLen; $RowIdx++) {
530 $cost = ($chars1[$RowIdx - 1] === $chars2[$ColIdx - 1]) ? 0 : 1;
531 $v1[$RowIdx] = min($v0[$RowIdx] + 1, $v1[$RowIdx - 1] + 1, $v0[$RowIdx - 1] + $cost);
532 }
533
534 $vTmp = $v0;
535 $v0 = $v1;
536 $v1 = $vTmp;
537 }
538
539 abj_service('request_context')->debug_info = 'Cleared after customLevenshtein.';
540 return $v0[$RowLen];
541 }
542
543 }
544