404-solution / includes / ngram / NGramCacheRebuildBatchRunner.php

NGramCacheRebuildBatchRunner.php in 404 Solution trunk, at includes/ngram/NGramCacheRebuildBatchRunner.php

496 lines 22.5 KB

Raw Download Zip

1	<?php
2
3	if (!defined('ABSPATH')) {
4	exit;
5	}
6
7	require_once __DIR__ . '/NGramRebuildProgressState.php';
8	require_once __DIR__ . '/NGramRescheduleFailureReport.php';
9	require_once __DIR__ . '/NGramRebuildDrain.php';
10	require_once __DIR__ . '/NGramRebuildRetryPolicy.php';
11
12	/**
13	* Runs the N-gram cache rebuild: one bounded chunk of batches per WP-Cron
14	* tick, advancing the rebuild cursor and rescheduling the chain until the
15	* whole content set is covered.
16	*
17	* Reached only from the cron callback
18	* (ABJ_404_Solution_DatabaseUpgradeNGram::rebuildNGramCacheAsync). Deciding
19	* WHETHER a rebuild needs to start or resume is a separate concern with
20	* different callers -- the 404 request path, the daily reconciler, the admin
21	* rebuild button, the activation initializer -- and lives in
22	* {@see ABJ_404_Solution_NGramCacheRebuildScheduler}.
23	*
24	* Multisite-aware: a network-activated install drains one site at a time,
25	* tracking the last site id it finished and the per-site cursor through the
26	* network option store, so a large network converges across ticks without ever
27	* holding more than one site's batch in memory. The site id is the cursor
28	* rather than a count of sites done, because a count is a POSITION in a list
29	* other requests can change: delete a site earlier in the network and every
30	* later position slides down one, so the next tick steps over a site that
31	* nothing revisits. See {@see ABJ_404_Solution_NetworkSitesRepository}.
32	*
33	* Lock acquisition is owned by the orchestrator (DatabaseUpgradeNGram). This
34	* collaborator assumes the 'ngram_rebuild' SyncUtils lock is already held when
35	* its methods are called.
36	*/
37	class ABJ_404_Solution_NGramCacheRebuildBatchRunner {
38
39	/**
40	* WP-Cron hook this runner reschedules itself on. The canonical
41	* definition lives on the cron adapter that owns hook names
42	* (ABJ_404_Solution_CronScheduler::HOOK_REBUILD_NGRAM_CACHE); the literal
43	* is repeated here so the Pattern 8 coordination-key audit can see the
44	* key in the file that enqueues it.
45	*/
46	const REBUILD_CRON_HOOK = 'abj404_rebuild_ngram_cache_hook';
47
48	/** @var ABJ_404_Solution_DatabaseCore */
49	private $dbCore;
50
51	/**
52	* Owns the rebuild cursor and the invariant that it never passes rows that
53	* did not rebuild. This runner decides WHICH set to drain and what to do
54	* about a drain that fell short; it does not touch the cursor itself.
55	*
56	* @var ABJ_404_Solution_NGramRebuildDrain
57	*/
58	private $drain;
59
60	/** @var ABJ_404_Solution_Logging */
61	private $logger;
62
63	/** @var ABJ_404_Solution_NGramNetworkOptionStore */
64	private $optionStore;
65
66	/** @var ABJ_404_Solution_CronScheduler */
67	private $cronScheduler;
68
69	/** @var ABJ_404_Solution_NGramRebuildProgressState */
70	private $progress;
71
72	/** @var ABJ_404_Solution_NGramRescheduleFailureReport */
73	private $rescheduleFailureReport;
74
75	/**
76	* Owns the whole answer to "when, and whether, does the chain come back
77	* after a failed tick". This runner decides WHAT happened; it does not
78	* decide what a failure is worth waiting for.
79	*
80	* @var ABJ_404_Solution_NGramRebuildRetryPolicy
81	*/
82	private $retryPolicy;
83
84	/** @var ABJ_404_Solution_NetworkSitesRepository\|null Built on first use. */
85	private $networkSites = null;
86
87	/**
88	* @param ABJ_404_Solution_NGramRebuildRuntime $runtime Database, logging and
89	* cron: the platform a rebuild tick runs against.
90	* @param mixed $rebuilder Object exposing rebuildCache().
91	* @param ABJ_404_Solution_NGramNetworkOptionStore $optionStore
92	* @param ABJ_404_Solution_NGramRebuildRetryPolicy\|null $retryPolicy Defaults to
93	* the shipped policy; supplied by tests that pin the jitter window.
94	* @throws InvalidArgumentException When $rebuilder cannot rebuild.
95	*/
96	public function __construct(
97	ABJ_404_Solution_NGramRebuildRuntime $runtime,
98	$rebuilder,
99	ABJ_404_Solution_NGramNetworkOptionStore $optionStore,
100	?ABJ_404_Solution_NGramRebuildRetryPolicy $retryPolicy = null
101	) {
102	// The types are DECLARED rather than only documented, for the same
103	// reason the rebuilder is checked below: positional objects whose types
104	// nothing verifies means transposing two of them is legal PHP, and the
105	// mistake then surfaces as a fatal on some later cron tick instead of
106	// at the wiring site. $rebuilder is the one that cannot be declared --
107	// it is anything exposing rebuildCache() -- so it keeps the explicit
108	// check that a declaration would otherwise have given it.
109	//
110	// Reject a rebuilder that cannot rebuild HERE, where the wiring mistake
111	// actually is. Accepting anything and only checking three dispatch
112	// layers down turned a container misconfiguration into a batch failure
113	// that recurred on every cron tick and read like a data problem.
114	if (!is_object($rebuilder) \|\| !method_exists($rebuilder, 'rebuildCache')) {
115	throw new InvalidArgumentException(
116	'NGramCacheRebuildBatchRunner requires a rebuilder exposing rebuildCache(); got ' .
117	(is_object($rebuilder) ? get_class($rebuilder) : gettype($rebuilder)) . '.'
118	);
119	}
120	// Unpacked into fields rather than held as a runtime and dereferenced
121	// at each use: the runtime is how these three ARRIVE together, not a
122	// thing this class needs to keep. Holding both would be two references
123	// to the same collaborator that could be read inconsistently.
124	$this->dbCore = $runtime->dbCore();
125	$this->logger = $runtime->logger();
126	$this->cronScheduler = $runtime->cronScheduler();
127	$this->optionStore = $optionStore;
128	$this->progress = new ABJ_404_Solution_NGramRebuildProgressState($optionStore);
129	$this->drain = new ABJ_404_Solution_NGramRebuildDrain(
130	array($rebuilder, 'rebuildCache'), $this->progress);
131	$this->rescheduleFailureReport = new ABJ_404_Solution_NGramRescheduleFailureReport(
132	$runtime, $this->progress);
133	$this->retryPolicy = $retryPolicy instanceof ABJ_404_Solution_NGramRebuildRetryPolicy
134	? $retryPolicy
135	: new ABJ_404_Solution_NGramRebuildRetryPolicy();
136	}
137
138	/**
139	* Rows currently in the permalink cache -- the set a rebuild walks.
140	*
141	* Delegated to the repository that owns permalink-cache reads rather than
142	* issuing the COUNT here: an orchestrator that also writes its own SQL is
143	* two layers in one method, and this exact count already had an
144	* authoritative implementation.
145	*
146	* @return int
147	*/
148	private function countPermalinkCacheRows(): int {
149	$repository = new ABJ_404_Solution_PermalinkCacheRepository($this->dbCore);
150	return $repository->getPermalinkCacheCount();
151	}
152
153	/**
154	* The network's site list, read one site at a time by immutable id.
155	*
156	* Delegated for the same reason the row count above is: an orchestrator
157	* that also writes its own SQL is two layers in one method. Memoized
158	* because a single cron tick asks it twice and it holds no per-site state.
159	*
160	* @return ABJ_404_Solution_NetworkSitesRepository
161	*/
162	private function networkSites(): ABJ_404_Solution_NetworkSitesRepository {
163	if ($this->networkSites === null) {
164	$this->networkSites = new ABJ_404_Solution_NetworkSitesRepository($this->dbCore);
165	}
166	return $this->networkSites;
167	}
168
169	/**
170	* Reschedule the chain, reporting a refusal instead of dropping it.
171	*
172	* Every reschedule in this class goes through here. The single-site path
173	* used to check the return value and the multisite paths did not, so a
174	* WP-Cron refusal stranded a multisite rebuild silently -- the one failure
175	* mode where nothing else re-arms the chain.
176	*
177	* The offset the report needs is read from the cursor rather than passed
178	* alongside the delay: as adjacent int parameters the two were transposable
179	* at every call site, and a transposition would silently change both when
180	* the chain resumes and where it resumes from.
181	*
182	* @param int $delaySeconds
183	* @param array<int, mixed> $args
184	* @param float $progress Percent complete, for the report.
185	* @return void
186	*/
187	private function rescheduleChain(int $delaySeconds, array $args, float $progress): void {
188	// Resolved to an absolute second here, then used TWICE: once to ask for
189	// the event and once to say what was asked for. The report used to
190	// rebuild both this timestamp and these args from what it could reach,
191	// and got both wrong on the multisite path -- see
192	// ABJ_404_Solution_NGramRescheduleFailureReport::report().
193	$timestamp = $this->cronScheduler->timestampAfter($delaySeconds);
194	if ($this->cronScheduler->scheduleSingleAt(self::REBUILD_CRON_HOOK, $timestamp, $args) === false) {
195	$this->rescheduleFailureReport->report(array(
196	'hookName' => self::REBUILD_CRON_HOOK,
197	'offset' => $this->progress->cursor(),
198	'progressPercent' => $progress,
199	'args' => $args,
200	'requestedTimestamp' => $timestamp,
201	));
202	}
203	}
204
205	/**
206	* Arm the chain's next link at the cadence this tick earned, or report why
207	* it is not being armed at all.
208	*
209	* Every reschedule that follows a drain or a probe goes through here, which
210	* is the point: the delay used to be the literal 10 written out at four
211	* separate call sites, so there was no single place that could be taught to
212	* back off, and nothing anywhere asked whether the failure was one that
213	* retrying could fix.
214	*
215	* Two inputs, deliberately separate. The CADENCE comes from the consecutive
216	* failure count, so a tick that succeeded runs at the base interval however
217	* bad the last hour was. Whether to re-arm AT ALL comes from the failure
218	* text, so a statement the server has already rejected on its own terms is
219	* surfaced once instead of re-run until the failure budget runs out.
220	*
221	* A chain that is not re-armed is not a dead rebuild: the cache stays
222	* honestly uninitialized, and the next 404, daily reconcile or admin
223	* rebuild re-arms it through
224	* {@see ABJ_404_Solution_NGramCacheRebuildScheduler}.
225	*
226	* @param string $failureText What stopped this tick, or '' when nothing did.
227	* Callers holding a drain outcome read it through
228	* {@see ABJ_404_Solution_NGramRebuildDrain::failureTextOf()} rather
229	* than testing the raw field, so every branch here and in the
230	* callers agrees on what the absence of a failure looks like.
231	* @param array<int, mixed> $args Cron arguments for the next link.
232	* @param float $progress Percent complete, for the refusal report.
233	* @return void
234	*/
235	private function rescheduleNextLink(string $failureText, array $args, float $progress): void {
236	if ($failureText !== '' && !$this->retryPolicy->isWorthRetrying($failureText)) {
237	$this->logger->errorMessage(
238	'N-gram cache rebuild stopped, because retrying cannot fix this: ' . $failureText
239	. ' The cache is left uninitialized so a later rebuild can resume it once the '
240	. 'underlying problem is corrected.'
241	);
242	return;
243	}
244
245	$this->rescheduleChain(
246	$this->retryPolicy->secondsUntilNextAttempt($this->progress->consecutiveFailures()),
247	$args,
248	$progress
249	);
250	}
251
252	/**
253	* Record that this tick failed, and report when failures stop being
254	* transient.
255	*
256	* @param string $context Human-readable description of what failed.
257	* @return void
258	*/
259	private function recordBatchFailure(string $context): void {
260	$failures = $this->progress->recordFailure();
261	$this->logger->errorMessage(
262	$context . " (consecutive failure {$failures} of "
263	. ABJ_404_Solution_NGramRebuildProgressState::MAX_CONSECUTIVE_FAILURES . ')'
264	);
265	}
266
267	/**
268	* Log whatever stopped a drain short, if anything did.
269	*
270	* The drain reports its failure rather than logging it, so the consecutive
271	* failure ledger and the message format stay in one place here -- the same
272	* place the network walk's own failures go through. A drain stops at its
273	* first failure, so there is at most one to report per call.
274	*
275	* @param array{failureContext?: mixed} $outcome
276	* @return void
277	*/
278	private function reportDrainFailure(array $outcome): void {
279	$failureText = ABJ_404_Solution_NGramRebuildDrain::failureTextOf($outcome);
280	if ($failureText !== '') {
281	$this->recordBatchFailure($failureText);
282	}
283	}
284
285	/**
286	* WP-Cron callback: process one chunk of rebuild batches and
287	* reschedule for the next chunk until the entire content set is
288	* covered. Multisite-aware: drains one site at a time before
289	* moving on.
290	*
291	* @return void
292	*/
293	public function runAsyncBatch() {
294	if ($this->progress->hasExhaustedRetries()) {
295	$this->logger->errorMessage(
296	'N-gram cache rebuild stopped: '
297	. ABJ_404_Solution_NGramRebuildProgressState::MAX_CONSECUTIVE_FAILURES
298	. ' consecutive batch failures. The cache is left uninitialized so a later '
299	. 'rebuild can resume it; clear '
300	. ABJ_404_Solution_NGramRebuildProgressState::OPTION_CONSECUTIVE_FAILURES
301	. ' to retry sooner.'
302	);
303	return;
304	}
305
306	if ($this->optionStore->isNetworkActivated()) {
307	$this->progress->useNetworkCursor();
308	$this->runMultisiteBatch();
309	return;
310	}
311	$this->progress->useSingleSiteCursor();
312	$this->runSingleSiteBatch();
313	}
314
315	/**
316	* Per-batch worker for multisite: walk to the site this tick owns, drain one
317	* chunk of batches of it, then either retire it or come back to it on the
318	* next tick.
319	*/
320	private function runMultisiteBatch(): void {
321	if (!$this->progress->networkWalkStarted()) {
322	$liveCount = $this->networkSites()->countSites();
323	if ($liveCount === null) {
324	// The count reports only that it could not be read, not WHY, so
325	// there is no driver text to classify here and this failure is
326	// always treated as retryable. That is the safe direction: the
327	// consecutive-failure budget still stops the chain, it just
328	// costs a few backed-off probes to get there instead of one.
329	$failure = 'N-gram rebuild could not count the sites in this network; leaving the '
330	. 'walk unstarted.';
331	$this->recordBatchFailure($failure);
332	$this->rescheduleNextLink($failure, array(), 0.0);
333	return;
334	}
335	$this->progress->beginNetworkWalk($liveCount);
336	}
337
338	// The stored total is a snapshot from when the walk began, so it is
339	// reported as an approximation and nothing decides anything from it.
340	$totalSites = $this->progress->totalSites(0);
341	$completedSites = $this->progress->sitesCompleted();
342	$lastSiteId = $this->progress->lastCompletedSiteId();
343
344	// The walk is keyed on the last site id it FINISHED, never on how many
345	// sites it has finished. A count is a position in a list, and positions
346	// are assigned at read time: deleting a site earlier in the network
347	// slides every later site down one, so the next position steps over the
348	// site in the gap and nothing afterwards ever revisits it. An id cannot
349	// move. Completion follows from the same read -- the network has ended
350	// when no site has an id past the cursor -- rather than from comparing
351	// a count against a total read at some other moment.
352	$nextSite = $this->networkSites()->nextSiteAfter($lastSiteId);
353
354	if ($nextSite->isUnreadable()) {
355	// Could not ASK the network, which is not the same as the network
356	// having ENDED. The walk keeps its cursor either way (whether the
357	// retry policy comes back for it or not); recording completion
358	// here is what published a network as rebuilt after draining none
359	// of it.
360	$failure = sprintf(
361	'N-gram rebuild could not read the site after %d in this network (%s); '
362	. 'holding the walk where it is rather than recording the network as finished.',
363	$lastSiteId,
364	$nextSite->reason()
365	);
366	$this->recordBatchFailure($failure);
367	// The reason carries the driver's own text, so a site table that
368	// does not exist is distinguishable here from one that is briefly
369	// unreachable, and only the second earns another probe.
370	$this->rescheduleNextLink($failure, array(), 0.0);
371	return;
372	}
373
374	if ($nextSite->isEndOfNetwork()) {
375	$this->progress->markNetworkComplete();
376	$this->logger->infoMessage("N-gram cache rebuild complete for all sites in network!");
377	return;
378	}
379
380	$currentSiteId = $nextSite->siteId();
381
382	// Everything from here to the matching restore runs against another
383	// site's tables. A throw anywhere in that span -- the row count, the
384	// option store, the logger, the scheduler -- used to escape with the
385	// switch still in effect, leaving the REST of this cron request reading
386	// and writing the wrong site. finally makes the unwind unconditional.
387	switch_to_blog($currentSiteId);
388	try {
389	$sitePages = $this->countPermalinkCacheRows();
390
391	if ($sitePages == 0) {
392	$this->progress->advanceToNextSite($currentSiteId);
393	$this->logger->infoMessage(sprintf(
394	"Site %d has no pages. Moving to next site. Progress: %d of ~%d sites completed.",
395	$currentSiteId, $completedSites + 1, $totalSites
396	));
397	// Deliberately immediate, and deliberately NOT through the
398	// retry policy: nothing failed here, this tick simply had no
399	// work, and a network of empty sites should be walked off in
400	// one burst rather than one cadence interval per empty site.
401	$this->rescheduleChain(0, array(), 100.0);
402	return;
403	}
404
405	$this->logger->infoMessage(sprintf(
406	"Processing N-gram cache for site %d (site %d of ~%d): Offset %d of %d pages",
407	$currentSiteId, $completedSites + 1, $totalSites,
408	$this->progress->cursor(), $sitePages
409	));
410
411	$outcome = $this->drain->drain($sitePages, "site {$currentSiteId}");
412	$this->reportDrainFailure($outcome);
413
414	$this->logger->infoMessage(sprintf(
415	"Site %d progress: %d%% complete (%d/%d pages), %d success, %d failed",
416	$currentSiteId, $outcome['percent'], $outcome['offset'], $sitePages,
417	$outcome['success'], $outcome['rowsFailed']
418	));
419
420	if (ABJ_404_Solution_NGramRebuildDrain::isClean($outcome, $sitePages)) {
421	$this->progress->advanceToNextSite($currentSiteId);
422	$this->progress->clearFailures();
423	$this->logger->infoMessage(sprintf(
424	"Site %d complete! Progress: %d of ~%d sites completed.",
425	$currentSiteId, $completedSites + 1, $totalSites
426	));
427	} else if (ABJ_404_Solution_NGramRebuildDrain::failureTextOf($outcome) === '') {
428	// Rebuilt every row it touched, just not the last of them: a
429	// success that has not finished yet, and success is what puts
430	// the retry curve back to the base cadence.
431	$this->progress->clearFailures();
432	}
433
434	$this->rescheduleNextLink(
435	ABJ_404_Solution_NGramRebuildDrain::failureTextOf($outcome),
436	array(),
437	$outcome['percent']
438	);
439	} finally {
440	restore_current_blog();
441	}
442	}
443
444	/**
445	* Per-batch worker for single-site: drain one chunk of batches,
446	* then either complete (mark initialized) or reschedule for the next chunk.
447	*/
448	private function runSingleSiteBatch(): void {
449	$totalPages = $this->countPermalinkCacheRows();
450
451	if ($totalPages == 0) {
452	$this->logger->debugMessage("No pages to process. Setting initialized flag.");
453	$this->progress->markComplete();
454	return;
455	}
456
457	$this->logger->infoMessage(sprintf(
458	"Async N-gram rebuild: Processing batch at offset %d of %d total pages",
459	$this->progress->cursor(), $totalPages
460	));
461
462	$outcome = $this->drain->drain($totalPages, 'this site');
463	$this->reportDrainFailure($outcome);
464
465	$this->logger->infoMessage(sprintf(
466	"Async N-gram rebuild progress: %d%% complete (%d/%d pages), %d success, %d failed",
467	$outcome['percent'], $outcome['offset'], $totalPages,
468	$outcome['success'], $outcome['rowsFailed']
469	));
470
471	// A run that failed is NOT complete, however far the cursor got
472	// beforehand. Marking it initialized here is what published an empty or
473	// partial cache as fully built, with nothing left to re-arm a rebuild.
474	if (!ABJ_404_Solution_NGramRebuildDrain::isClean($outcome, $totalPages)) {
475	if (ABJ_404_Solution_NGramRebuildDrain::failureTextOf($outcome) === '') {
476	// Same rule as the multisite path: a tick that rebuilt every
477	// row it touched is a success, and success resets the backoff.
478	// Without this the single-site chain kept an old outage's
479	// pacing for the whole remainder of a healthy rebuild.
480	$this->progress->clearFailures();
481	}
482	$this->rescheduleNextLink(
483	ABJ_404_Solution_NGramRebuildDrain::failureTextOf($outcome),
484	[$outcome['offset']],
485	$outcome['percent']
486	);
487	return;
488	}
489
490	$this->progress->markComplete();
491	$this->logger->infoMessage("N-gram cache rebuild complete! Total: {$outcome['processed']} processed, "
492	. "{$outcome['success']} success, {$outcome['rowsFailed']} failed.");
493	}
494
495	}
496