404-solution / includes / database / upgrades / DatabaseUpgradeRedirectsDenormBackfill.php

DatabaseUpgradeRedirectsDenormBackfill.php in 404 Solution trunk, at includes/database/upgrades/DatabaseUpgradeRedirectsDenormBackfill.php

329 lines 15.3 KB

Raw Download Zip

1	<?php
2
3	if (!defined('ABSPATH')) {
4	exit;
5	}
6
7	// allow-no-test-found: exercised by RedirectsDenormBackfillIntegrationTest
8
9	/**
10	* Chunked, time-budgeted backfill of the four denormalized derived columns on
11	* the redirects table: logshits, last_used, dest_for_view, published_status.
12	*
13	* Denorm Step 3a (i458 / i459). These columns roll the per-redirect values that
14	* the staged view_done pipeline used to compute at build time directly onto the
15	* redirects row, so admin reads can drop the staged build entirely (Step 3b).
16	* This component owns only the one-time INITIAL population of those columns for
17	* rows that pre-date the column add. Real-time freshness (Step 3c) and the
18	* nightly full reconcile (Step 3d) are separate components.
19	*
20	* The not-yet-backfilled sentinel is `dest_for_view IS NULL`: the column add
21	* leaves every existing row NULL, and a processed row is always set to a
22	* non-NULL value (empty string at minimum), so a single `WHERE dest_for_view
23	* IS NULL` predicate is both the chunk selector and the completion probe. This
24	* mirrors the `canonical_url IS NULL` backlog pattern in
25	* {@see ABJ_404_Solution_DatabaseUpgradeCanonicalUrlBackfill}.
26	*
27	* Each chunk resolves dest_for_view + published_status with the same per-type
28	* logic the staged pipeline used (stages S4-S8: posts, terms, home, external,
29	* 404-displayed) and rolls up logshits + last_used from the wp_abj404_logs_hits
30	* rollup by canonical URL (NOT raw logsv2: report.md Finding 2). The whole run is
31	* bounded by row count
32	* (REDIRECTS_DENORM_BACKFILL_CHUNK_SIZE) and wall clock
33	* (REDIRECTS_DENORM_BACKFILL_TIME_BUDGET_SEC) so a large site converges across
34	* successive daily cron ticks without ever blocking a request.
35	*
36	* The two narrow LEFT(<source>, 191) sort-key columns (dest_sort_key,
37	* url_sort_key) that derive from these are drained by the sibling
38	* {@see ABJ_404_Solution_DatabaseUpgradeRedirectsSortKeyBackfill}; the full
39	* deferred pass below runs the main backfill then delegates both sort-key drains
40	* there under one shared time budget.
41	*
42	* Reached by {@see ABJ_404_Solution_DatabaseUpgradeDailyMaintenance} (daily
43	* cron). Never run synchronously during activation: reads must never wait on it.
44	*/
45	class ABJ_404_Solution_DatabaseUpgradeRedirectsDenormBackfill extends ABJ_404_Solution_DatabaseUpgradeComponent {
46
47	/**
48	* Resolve the four derived columns for any redirect rows still carrying the
49	* dest_for_view IS NULL sentinel, one chunk at a time.
50	*
51	* Idempotent: once every row is resolved the chunk selector matches zero
52	* rows and the function returns immediately. Reprocessing a row recomputes
53	* the same values from the same sources, so an interrupted run resumes
54	* cleanly on the next invocation.
55	*
56	* Skips silently (returns 0) when:
57	* - $wpdb is unavailable,
58	* - the redirects table is missing (degraded site state),
59	* - the dest_for_view column is missing (column add has not happened yet,
60	* e.g. immediately after upgrade before verifyColumns ran).
61	*
62	* @param ?float $deadlineFloat Absolute wall-clock deadline (abj_clock
63	* nowFloat seconds) to stop by. When null, the method uses its own
64	* REDIRECTS_DENORM_BACKFILL_TIME_BUDGET_SEC budget. A shared deadline lets
65	* {@see runDeferredDenormBackfillPass()} bound the whole three-drain pass
66	* by a single budget instead of one budget per drain.
67	* @return int Number of redirect rows resolved in this invocation.
68	*/
69	public function backfillRedirectsDenormColumns(?float $deadlineFloat = null): int {
70	global $wpdb;
71	if (!isset($wpdb)) {
72	return 0;
73	}
74	$redirectsTable = $this->dbCore->doTableNameReplacements('{wp_abj404_redirects}');
75
76	// SHOW TABLES existence probe, same shape as the canonical-url backfill.
77	// Routing through queryAndGetResults would log a benign "table doesn't
78	// exist" error on freshly-installed sites before the create-tables flow
79	// has run.
80	// DAO-bypass-approved: schema existence probe, see comment above.
81	$found = $wpdb->get_var("SHOW TABLES LIKE '" . esc_sql($redirectsTable) . "'");
82	if ($found !== $redirectsTable) {
83	return 0;
84	}
85	if ($this->columnExists($redirectsTable, 'dest_for_view') !== true) {
86	return 0;
87	}
88
89	$chunkSize = (int)$this->getRedirectsDenormBackfillChunkSize();
90	if ($chunkSize < 1) {
91	$chunkSize = 1;
92	}
93	$start = abj_clock()->nowFloat();
94	$deadline = $deadlineFloat ?? ($start + (float)$this->getRedirectsDenormBackfillTimeBudgetSec());
95	$totalResolved = 0;
96
97	while (abj_clock()->nowFloat() < $deadline) {
98	$ids = $this->fetchNextBackfillChunkIds($redirectsTable, $chunkSize);
99	if ($ids === null) {
100	// Read error already warned about; stop so we don't spin.
101	return $totalResolved;
102	}
103	if (empty($ids)) {
104	break;
105	}
106	if (!$this->resolveDenormColumnsForIds($redirectsTable, $ids)) {
107	return $totalResolved;
108	}
109	$this->writeCursorOption(
110	ABJ_404_Solution_DatabaseUpgradeRuntimeState::REDIRECTS_DENORM_BACKFILL_CURSOR_OPTION,
111	(int)max($ids)
112	);
113	$totalResolved += count($ids);
114	if (count($ids) < $chunkSize) {
115	break;
116	}
117	}
118
119	if ($totalResolved > 0) {
120	$this->logger->infoMessage(sprintf(
121	"backfillRedirectsDenormColumns: resolved %d redirect rows in %.2fs.",
122	$totalResolved,
123	abj_clock()->nowFloat() - $start
124	));
125	}
126
127	return $totalResolved;
128	}
129
130	/**
131	* Run the full deferred denorm backfill pass (main derived columns + both
132	* narrow sort keys) under ONE shared time budget.
133	*
134	* Why this exists: browser-triggered admin AJAX runs the three drains
135	* back-to-back after the table has rendered. With a per-call budget the
136	* combined pass could consume up to 3x REDIRECTS_DENORM_BACKFILL_TIME_BUDGET_SEC.
137	* A single shared deadline caps the post-load request at one budget;
138	* whatever backlog remains drains on the next browser poll or daily cron.
139	* The daily-maintenance path deliberately keeps the per-call budgets (it is
140	* true cron, never request-blocking, so faster nightly convergence is
141	* preferred there).
142	*
143	* @param ?float $timeBudgetSec Wall-clock budget for the shared pass. When
144	* null, uses REDIRECTS_DENORM_BACKFILL_TIME_BUDGET_SEC.
145	* @return void
146	*/
147	public function runDeferredDenormBackfillPass(?float $timeBudgetSec = null): void {
148	$budget = $timeBudgetSec ?? (float)$this->getRedirectsDenormBackfillTimeBudgetSec();
149	$deadline = abj_clock()->nowFloat() + $budget;
150	$this->backfillRedirectsDenormColumns($deadline);
151	$sortKey = $this->upgrades()->redirectsSortKeyBackfillUpgrade();
152	$sortKey->backfillRedirectsDestSortKey($deadline);
153	$sortKey->backfillRedirectsUrlSortKey($deadline);
154	}
155
156	/**
157	* Read the next chunk of redirect ids that still need backfilling.
158	*
159	* @param string $redirectsTable
160	* @param int $chunkSize
161	* @return array<int, int>\|null List of ids (possibly empty), or null on a query error.
162	*/
163	private function fetchNextBackfillChunkIds(string $redirectsTable, int $chunkSize): ?array {
164	$cursorOption = ABJ_404_Solution_DatabaseUpgradeRuntimeState::REDIRECTS_DENORM_BACKFILL_CURSOR_OPTION;
165	$cursor = $this->readCursorOption($cursorOption);
166	$ids = $this->queryBackfillChunkIds($redirectsTable, $chunkSize, $cursor);
167	if ($ids === null) {
168	return null;
169	}
170	if (empty($ids) && $cursor > 0) {
171	$this->writeCursorOption($cursorOption, 0);
172	$ids = $this->queryBackfillChunkIds($redirectsTable, $chunkSize, 0);
173	if ($ids === null) {
174	return null;
175	}
176	}
177	return $ids;
178	}
179
180	/**
181	* @param string $redirectsTable
182	* @param int $chunkSize
183	* @param int $afterId
184	* @return array<int, int>\|null
185	*/
186	private function queryBackfillChunkIds(string $redirectsTable, int $chunkSize, int $afterId): ?array {
187	$result = $this->dbCore->queryAndGetResults(
188	"SELECT id FROM " . $redirectsTable .
189	" WHERE id > " . (int)$afterId . " AND dest_for_view IS NULL ORDER BY id ASC LIMIT " . $chunkSize
190	);
191	$lastError = isset($result['last_error']) && is_string($result['last_error']) ? $result['last_error'] : '';
192	if ($lastError !== '') {
193	$this->logger->warn("backfillRedirectsDenormColumns: stopping after read error: " . $lastError);
194	return null;
195	}
196	$rows = is_array($result['rows'] ?? null) ? $result['rows'] : array();
197	$ids = array();
198	foreach ($rows as $row) {
199	if (is_array($row) && isset($row['id']) && is_numeric($row['id'])) {
200	$ids[] = (int)$row['id'];
201	}
202	}
203	return $ids;
204	}
205
206	/**
207	* Populate the four derived columns for an explicit list of redirect ids.
208	*
209	* dest_for_view + published_status are resolved per redirect type, mirroring
210	* staged-build stages S4-S8. Any row whose type matches none of those stages
211	* is caught by a final UPDATE so the chunk always drains (no row keeps the
212	* dest_for_view IS NULL sentinel). logshits + last_used are rolled up from
213	* the wp_abj404_logs_hits rollup by canonical URL (NOT raw logsv2: report.md
214	* Finding 2) when that rollup table exists.
215	*
216	* @param string $redirectsTable
217	* @param array<int, int> $ids
218	* @return bool True if the chunk resolved cleanly, false if a write errored.
219	*/
220	private function resolveDenormColumnsForIds(string $redirectsTable, array $ids): bool {
221	// Delegate the per-chunk write (per-type dest/published statements plus
222	// the logs_hits rollup) to the shared resolver, the single source of
223	// truth the Step 3d nightly reconcile also uses so the two bulk-write
224	// paths can never drift. $recompute = false: the catch-all guards on the
225	// dest_for_view IS NULL sentinel, which keeps the chunk draining and the
226	// backlog probe converging.
227	return ABJ_404_Solution_RedirectsDenormChunkResolver::resolveChunk(
228	$this->dbCore,
229	$this->logger,
230	$redirectsTable,
231	$ids,
232	false
233	);
234	}
235
236	/**
237	* The four denormalized derived columns added to the redirects table in
238	* Denorm Step 3a (i459), keyed by column name with the exact column DDL
239	* fragment used in ADD COLUMN. Single source of truth shared by the
240	* targeted online-DDL add and the chunked backfill's column-exists
241	* guards, both of which live in this component. Must stay in sync with
242	* createRedirectsTable.sql.
243	*
244	* @var array<string, string>
245	*/
246	private const REDIRECTS_DENORM_COLUMN_DDL = array(
247	'logshits' => '`logshits` BIGINT(20) NOT NULL DEFAULT 0',
248	'last_used' => '`last_used` BIGINT(20) DEFAULT NULL',
249	'dest_for_view' => '`dest_for_view` VARCHAR(2048) DEFAULT NULL',
250	'dest_sort_key' => '`dest_sort_key` VARCHAR(191) DEFAULT NULL',
251	'url_sort_key' => '`url_sort_key` VARCHAR(191) DEFAULT NULL',
252	'published_status' => '`published_status` TINYINT(4) DEFAULT NULL',
253	);
254
255	/**
256	* Add the four denormalized derived columns (logshits, last_used,
257	* dest_for_view, published_status) to the redirects table with online
258	* DDL when supported.
259	*
260	* Sibling of
261	* {@see ABJ_404_Solution_DatabaseUpgradeCanonicalUrlBackfill::ensureRedirectsCanonicalUrlColumn()}:
262	* a small
263	* idempotent helper that runs ahead of the generic verifyColumns() flow
264	* so the column adds can use ALGORITHM=INPLACE, LOCK=NONE on InnoDB 5.6
265	* or newer (no table lock during the rewrite; 21K-row redirects tables
266	* add in seconds). Only the columns actually missing are added, so
267	* re-running this on a fully-migrated table is a no-op (each column is
268	* SHOW COLUMNS-guarded per defensive philosophy #1/#7).
269	*
270	* On engines that don't support online DDL for ADD COLUMN the explicit
271	* ALGORITHM clause causes ER_ALTER_OPERATION_NOT_SUPPORTED; we then fall
272	* back to a bare ALTER, which is what verifyColumns() also runs as the
273	* safety net. The derived columns carry sensible defaults (logshits 0;
274	* the rest NULL) so existing rows are valid immediately;
275	* backfillRedirectsDenormColumns() populates the real values across
276	* later cron ticks without ever blocking activation.
277	*
278	* @param string $redirectsTable Fully-qualified redirects table name.
279	* @return void
280	*/
281	public function ensureRedirectsDenormColumns(string $redirectsTable): void {
282	$missingClauses = array();
283	foreach (self::REDIRECTS_DENORM_COLUMN_DDL as $columnName => $columnDdl) {
284	if ($this->columnExists($redirectsTable, $columnName) === false) {
285	// Definitely absent. An unreadable probe (null) is not absence,
286	// and adding on it would rewrite a table we cannot introspect.
287	$missingClauses[] = 'ADD COLUMN ' . $columnDdl;
288	}
289	}
290	if (empty($missingClauses)) {
291	return;
292	}
293
294	$addClause = implode(', ', $missingClauses);
295	$inplaceQuery = "ALTER TABLE " . $redirectsTable . " " . $addClause .
296	", ALGORITHM=INPLACE, LOCK=NONE";
297	$result = $this->dbCore->queryAndGetResults($inplaceQuery,
298	array('log_too_slow' => false, 'log_errors' => false));
299	if (empty($result['last_error'])) {
300	$this->logger->infoMessage("Added denorm columns to {$redirectsTable} " .
301	"(ALGORITHM=INPLACE, LOCK=NONE): " . $addClause);
302	return;
303	}
304	$lastError = isset($result['last_error']) && is_scalar($result['last_error'])
305	? (string)$result['last_error'] : '';
306	if ($this->schemaChangeWasAlreadyApplied($lastError)) {
307	// Another request added at least one of these between the
308	// columnExists() probes above and this ALTER, which rejects the
309	// whole statement. The bare fallback carries the same clause list
310	// and meets the same answer; the next tick re-probes and asks for
311	// only whatever is still genuinely missing.
312	$this->logger->infoMessage("Denorm columns on {$redirectsTable} were added by another " .
313	"process while this one was adding them: " . $addClause);
314	return;
315	}
316	// Engine didn't support online DDL for ADD COLUMN, fall back to a
317	// bare ALTER, same as verifyColumns() would run. On modern InnoDB the
318	// bare ALTER is itself implicitly INSTANT/INPLACE for ADD COLUMN with
319	// a default, so this branch only runs on legacy engines.
320	$bareQuery = "ALTER TABLE " . $redirectsTable . " " . $addClause;
321	$bare = $this->dbCore->queryAndGetResults($bareQuery,
322	array('log_too_slow' => false));
323	if (empty($bare['last_error'])) {
324	$this->logger->infoMessage("Added denorm columns to {$redirectsTable} " .
325	"(bare ALTER fallback): " . $addClause);
326	}
327	}
328	}
329