thinkrank / includes / seo / class-instant-indexing-reconciler.php

class-instant-indexing-reconciler.php in ThinkRank AI SEO – AI SEO Plugin for WordPress: Schema, XML Sitemaps, Meta Tags, Search Console & Local SEO 2.9.0, at includes/seo/class-instant-indexing-reconciler.php

517 lines 18.6 KB

Raw Download Zip

1	<?php
2	/**
3	* Instant Indexing Reconciler Class
4	*
5	* Scheduled reconciliation between what has actually been published and what
6	* IndexNow has been told about, plus retries for the gaps it finds.
7	*
8	* @package ThinkRank
9	* @subpackage SEO
10	* @since 1.31.0
11	*/
12
13	declare(strict_types=1);
14
15	namespace ThinkRank\SEO;
16
17	// Prevent direct access
18	if (!defined('ABSPATH')) {
19	exit;
20	}
21
22	/**
23	* Instant Indexing Reconciler Class
24	*
25	* The automatic submission path is best-effort: it hangs off
26	* transition_post_status and defers the outbound call to a single WP-Cron event.
27	* Several ordinary situations therefore leave a published URL newer than
28	* anything IndexNow was told about — a direct `$wpdb->update()` fires no hooks
29	* at all, two saves inside the 15-second dedupe window collapse into one
30	* submission, a missed or dropped cron event disappears without trace, a failed
31	* submission is never retried, and anything edited while the feature was off is
32	* simply never sent.
33	*
34	* Rather than chase each of those individually, this reconciler compares state:
35	* every published URL's post_modified against the newest successful submission
36	* for that URL. That is mechanism-agnostic, so it closes gaps caused by causes
37	* nobody has thought of yet.
38	*
39	* Work is bounded. A cron run walks a batch of posts from a stored cursor and
40	* wraps around, so a large site reconciles over successive runs instead of
41	* trying to hold every permalink in memory at once.
42	*
43	* @since 1.31.0
44	*/
45	class Instant_Indexing_Reconciler {
46
47	/**
48	* Recurring reconciliation event.
49	*/
50	public const CRON_HOOK = 'thinkrank_instant_indexing_reconcile';
51
52	/**
53	* Option holding the batch cursor (offset into the ordered post list).
54	*/
55	private const CURSOR_OPTION = 'thinkrank_instant_indexing_reconcile_cursor';
56
57	/**
58	* Posts examined per cron run.
59	*/
60	public const BATCH_SIZE = 200;
61
62	/**
63	* Default ceiling on posts examined when building a report on demand.
64	*/
65	public const REPORT_LIMIT = 500;
66
67	/**
68	* How long to leave a failing URL alone before trying it again.
69	*/
70	public const RETRY_COOLDOWN = 6 * HOUR_IN_SECONDS;
71
72	/**
73	* Window over which repeated failures are counted for the park rule.
74	*/
75	public const FAILURE_WINDOW = 7 * DAY_IN_SECONDS;
76
77	/**
78	* Consecutive recent failures after which a URL stops being retried.
79	*
80	* Without this a permanently broken URL — 404, wrong host, revoked key —
81	* would consume the retry budget on every run forever and crowd out URLs
82	* that could actually succeed.
83	*/
84	public const MAX_FAILURES_BEFORE_PARK = 5;
85
86	/**
87	* Coverage states.
88	*/
89	public const STATE_CURRENT = 'current';
90	public const STATE_STALE = 'stale';
91	public const STATE_FAILED = 'failed';
92	public const STATE_NEVER = 'never_submitted';
93
94	/**
95	* Instant Indexing Manager instance.
96	*
97	* @var Instant_Indexing_Manager\|null
98	*/
99	private ?Instant_Indexing_Manager $manager = null;
100
101	/**
102	* Settings option name, shared with the manager.
103	*
104	* @var string
105	*/
106	private string $option_name = 'thinkrank_instant_indexing_settings';
107
108	/**
109	* Register the cron listener and make sure the event exists.
110	*
111	* @since 1.31.0
112	* @return void
113	*/
114	public function init(): void {
115	add_action(self::CRON_HOOK, [$this, 'reconcile']);
116
117	if (!wp_next_scheduled(self::CRON_HOOK)) {
118	// Offset the first run so a freshly activated site isn't reconciling
119	// during the activation request itself.
120	wp_schedule_event(time() + HOUR_IN_SECONDS, 'daily', self::CRON_HOOK);
121	}
122	}
123
124	/**
125	* Decide which coverage bucket a URL falls into.
126	*
127	* Pure: every input is passed in, so the rules can be tested without a
128	* database. All timestamps are site-local `Y-m-d H:i:s` strings, matching
129	* both `post_modified` and the log table's `created_at`; lexicographic
130	* comparison is correct for that format.
131	*
132	* @since 1.31.0
133	*
134	* @param string $post_modified Post's last modification time.
135	* @param string\|null $last_success Newest successful submission, or null.
136	* @param string\|null $last_attempt Newest submission of any status, or null.
137	* @return string One of the STATE_* constants.
138	*/
139	public static function classify(string $post_modified, ?string $last_success, ?string $last_attempt): string {
140	// Nothing was ever sent for this URL.
141	if (null === $last_attempt \|\| '' === $last_attempt) {
142	return self::STATE_NEVER;
143	}
144
145	// A success at or after the last edit means search engines know about
146	// the current content — even if a later attempt happened to fail.
147	if (null !== $last_success && '' !== $last_success && $last_success >= $post_modified) {
148	return self::STATE_CURRENT;
149	}
150
151	// The most recent attempt failed (the newest row is not the successful
152	// one), and nothing since the edit succeeded.
153	if ($last_success !== $last_attempt) {
154	return self::STATE_FAILED;
155	}
156
157	// Submitted successfully, but the post has been edited since.
158	return self::STATE_STALE;
159	}
160
161	/**
162	* Whether a URL in a given state should be resubmitted on this run.
163	*
164	* Pure, for the same reason as classify().
165	*
166	* @since 1.31.0
167	*
168	* @param string $state A STATE_* constant.
169	* @param string\|null $last_attempt Newest submission of any status, or null.
170	* @param int $recent_failures Failures inside FAILURE_WINDOW.
171	* @param int $now Current timestamp.
172	* @return bool True when the URL should be retried now.
173	*/
174	public static function should_retry(string $state, ?string $last_attempt, int $recent_failures, int $now): bool {
175	if (self::STATE_CURRENT === $state) {
176	return false;
177	}
178
179	if (self::STATE_FAILED === $state) {
180	// Park a URL that keeps failing so it can't monopolise the budget.
181	if ($recent_failures >= self::MAX_FAILURES_BEFORE_PARK) {
182	return false;
183	}
184
185	// Back off between attempts rather than hammering on every run.
186	if (null !== $last_attempt && '' !== $last_attempt) {
187	$attempted_at = strtotime($last_attempt);
188	if (false !== $attempted_at && ($now - $attempted_at) < self::RETRY_COOLDOWN) {
189	return false;
190	}
191	}
192	}
193
194	return true;
195	}
196
197	/**
198	* Build a coverage report over the published, auto-submitted content.
199	*
200	* @since 1.31.0
201	*
202	* @param int $limit Maximum posts to examine.
203	* @param int $offset Offset into the ordered post list.
204	* @param int $examples Maximum example URLs retained per bucket.
205	* @return array{
206	* counts: array<string,int>, total: int, examined: int, offset: int,
207	* urls: array<string,array<int,array<string,string>>>, truncated: bool,
208	* enabled: bool, generated_at: string
209	* }
210	*/
211	public function build_report(int $limit = self::REPORT_LIMIT, int $offset = 0, int $examples = 25): array {
212	$limit = max(1, $limit);
213	$offset = max(0, $offset);
214
215	$counts = [
216	self::STATE_CURRENT => 0,
217	self::STATE_STALE => 0,
218	self::STATE_FAILED => 0,
219	self::STATE_NEVER => 0,
220	];
221	$urls = [
222	self::STATE_CURRENT => [],
223	self::STATE_STALE => [],
224	self::STATE_FAILED => [],
225	self::STATE_NEVER => [],
226	];
227
228	$total = $this->count_tracked_posts();
229	$rows = $this->get_tracked_posts($limit, $offset);
230	$log = $this->get_submission_index();
231
232	foreach ($rows as $row) {
233	$url = get_permalink((int) $row['ID']);
234	if (!$url) {
235	continue;
236	}
237
238	$entry = $log[$url] ?? null;
239	$state = self::classify(
240	(string) $row['post_modified'],
241	$entry['last_success'] ?? null,
242	$entry['last_attempt'] ?? null
243	);
244
245	$counts[$state]++;
246
247	if (count($urls[$state]) < $examples) {
248	$urls[$state][] = [
249	'url' => $url,
250	'post_id' => (int) $row['ID'],
251	'post_modified' => (string) $row['post_modified'],
252	'last_success' => $entry['last_success'] ?? '',
253	'last_attempt' => $entry['last_attempt'] ?? '',
254	];
255	}
256	}
257
258	return [
259	'counts' => $counts,
260	'total' => $total,
261	'examined' => count($rows),
262	'offset' => $offset,
263	'urls' => $urls,
264	// Signals that the counts describe a window, not the whole site, so
265	// the UI never presents a partial pass as a full audit.
266	'truncated' => ($offset + count($rows)) < $total,
267	'enabled' => $this->is_enabled(),
268	'generated_at' => current_time('mysql'),
269	];
270	}
271
272	/**
273	* Scheduled reconciliation pass: find gaps in one batch and resubmit them.
274	*
275	* @since 1.31.0
276	*
277	* @param bool $dry_run When true, classify and select but submit nothing.
278	* @return array Summary of the pass.
279	*/
280	public function reconcile(bool $dry_run = false): array {
281	$summary = [
282	'ran' => false,
283	'reason' => '',
284	'examined' => 0,
285	'counts' => [],
286	'retried' => 0,
287	'retried_urls' => [],
288	'submission' => null,
289	'next_offset' => 0,
290	];
291
292	if (!$this->is_enabled()) {
293	$summary['reason'] = 'Instant Indexing is disabled';
294	return $summary;
295	}
296
297	$total = $this->count_tracked_posts();
298	if (0 === $total) {
299	$summary['ran'] = true;
300	$summary['reason'] = 'No published content to reconcile';
301	return $summary;
302	}
303
304	// Walk the site in batches across successive runs, wrapping at the end.
305	$offset = (int) get_option(self::CURSOR_OPTION, 0);
306	if ($offset >= $total) {
307	$offset = 0;
308	}
309
310	$report = $this->build_report(self::BATCH_SIZE, $offset, self::BATCH_SIZE);
311	$log = $this->get_submission_index();
312	$now = time();
313
314	$retry = [];
315	foreach ([self::STATE_NEVER, self::STATE_STALE, self::STATE_FAILED] as $state) {
316	foreach ($report['urls'][$state] as $item) {
317	$entry = $log[$item['url']] ?? null;
318
319	if (!self::should_retry($state, $entry['last_attempt'] ?? null, (int) ($entry['recent_failures'] ?? 0), $now)) {
320	continue;
321	}
322
323	$retry[] = $item['url'];
324	}
325	}
326
327	// Respect the same per-submission ceiling every other path uses; the
328	// remainder is picked up by the next run.
329	$retry = array_slice(array_values(array_unique($retry)), 0, Instant_Indexing_Manager::MAX_URLS_PER_SUBMISSION);
330
331	$summary['ran'] = true;
332	$summary['examined'] = $report['examined'];
333	$summary['counts'] = $report['counts'];
334	$summary['retried'] = count($retry);
335	$summary['retried_urls'] = $retry;
336
337	if (!$dry_run) {
338	if (!empty($retry)) {
339	$summary['submission'] = $this->get_manager()->submit_urls($retry);
340	}
341
342	$next = $offset + $report['examined'];
343	if ($next >= $total \|\| 0 === $report['examined']) {
344	$next = 0;
345	}
346	update_option(self::CURSOR_OPTION, $next, false);
347	$summary['next_offset'] = $next;
348	}
349
350	return $summary;
351	}
352
353	/**
354	* Newest successful / newest overall submission per URL, plus recent failures.
355	*
356	* One aggregate query rather than a lookup per URL — the log has no usable
357	* equality index on the full 2048-char url column, so per-URL queries would
358	* scan the table once per post.
359	*
360	* @since 1.31.0
361	* @return array<string,array{last_success:?string,last_attempt:?string,recent_failures:int}>
362	*/
363	private function get_submission_index(): array {
364	global $wpdb;
365
366	$table = $wpdb->prefix . 'thinkrank_instant_indexing_logs';
367	$window_start = gmdate('Y-m-d H:i:s', (int) (current_time('timestamp') - self::FAILURE_WINDOW)); // phpcs:ignore WordPress.DateTime.CurrentTimeTimestamp.Requested -- log rows are written with current_time('mysql'), so the window must be computed in the same site-local frame.
368
369	// Aggregate over a plugin-owned log table. The only interpolated value is
370	// the table name, built from $wpdb->prefix; the date bound is prepared.
371	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber, PluginCheck.Security.DirectDB.UnescapedDBParameter
372	$rows = $wpdb->get_results(
373	$wpdb->prepare(
374	"SELECT url,
375	MAX(CASE WHEN status = 'success' THEN created_at END) AS last_success,
376	MAX(created_at) AS last_attempt,
377	SUM(CASE WHEN status <> 'success' AND created_at >= %s THEN 1 ELSE 0 END) AS recent_failures
378	FROM `{$table}`
379	GROUP BY url",
380	$window_start
381	),
382	ARRAY_A
383	);
384	// phpcs:enable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber, PluginCheck.Security.DirectDB.UnescapedDBParameter
385
386	$index = [];
387	foreach ((array) $rows as $row) {
388	$index[(string) $row['url']] = [
389	'last_success' => null !== $row['last_success'] ? (string) $row['last_success'] : null,
390	'last_attempt' => null !== $row['last_attempt'] ? (string) $row['last_attempt'] : null,
391	'recent_failures' => (int) $row['recent_failures'],
392	];
393	}
394
395	return $index;
396	}
397
398	/**
399	* Published posts of the auto-submitted types, ordered stably for cursoring.
400	*
401	* @since 1.31.0
402	*
403	* @param int $limit Maximum rows.
404	* @param int $offset Offset into the ordered list.
405	* @return array<int,array{ID:string,post_modified:string}>
406	*/
407	private function get_tracked_posts(int $limit, int $offset): array {
408	global $wpdb;
409
410	$types = $this->get_tracked_post_types();
411	if (empty($types)) {
412	return [];
413	}
414
415	$placeholders = implode(', ', array_fill(0, count($types), '%s'));
416	$params = array_merge($types, [$limit, $offset]);
417
418	// ID + timestamp only, over the indexed post_type/post_status pair;
419	// WP_Query would hydrate every post object for no benefit. The only
420	// interpolation is the %s placeholder list, built from a count.
421	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber, PluginCheck.Security.DirectDB.UnescapedDBParameter
422	$rows = $wpdb->get_results(
423	$wpdb->prepare(
424	"SELECT ID, post_modified
425	FROM {$wpdb->posts}
426	WHERE post_status = 'publish'
427	AND post_type IN ({$placeholders})
428	ORDER BY ID ASC
429	LIMIT %d OFFSET %d",
430	$params
431	),
432	ARRAY_A
433	);
434	// phpcs:enable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber, PluginCheck.Security.DirectDB.UnescapedDBParameter
435
436	$rows = (array) $rows;
437
438	// Prime the cache in one query so the get_permalink() calls that follow
439	// don't each fetch their post row separately.
440	if (!empty($rows)) {
441	_prime_post_caches(array_map(static fn($r) => (int) $r['ID'], $rows), false, false);
442	}
443
444	return $rows;
445	}
446
447	/**
448	* Total published posts in the auto-submitted types.
449	*
450	* @since 1.31.0
451	* @return int
452	*/
453	private function count_tracked_posts(): int {
454	global $wpdb;
455
456	$types = $this->get_tracked_post_types();
457	if (empty($types)) {
458	return 0;
459	}
460
461	$placeholders = implode(', ', array_fill(0, count($types), '%s'));
462
463	// Indexed COUNT over post_type/post_status; the only interpolation is the
464	// %s placeholder list, built from a count.
465	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, PluginCheck.Security.DirectDB.UnescapedDBParameter
466	return (int) $wpdb->get_var(
467	$wpdb->prepare(
468	"SELECT COUNT(*)
469	FROM {$wpdb->posts}
470	WHERE post_status = 'publish'
471	AND post_type IN ({$placeholders})",
472	$types
473	)
474	);
475	// phpcs:enable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, PluginCheck.Security.DirectDB.UnescapedDBParameter
476	}
477
478	/**
479	* Post types configured for automatic submission.
480	*
481	* @since 1.31.0
482	* @return array<int,string>
483	*/
484	private function get_tracked_post_types(): array {
485	$settings = get_option($this->option_name, []);
486	$types = $settings['auto_submit_post_types'] ?? [];
487
488	return array_values(array_filter(array_map('strval', (array) $types)));
489	}
490
491	/**
492	* Whether Instant Indexing is switched on.
493	*
494	* @since 1.31.0
495	* @return bool
496	*/
497	private function is_enabled(): bool {
498	$settings = get_option($this->option_name, []);
499
500	return !empty($settings['enabled']);
501	}
502
503	/**
504	* Manager used for the outbound submission, built on first use.
505	*
506	* @since 1.31.0
507	* @return Instant_Indexing_Manager
508	*/
509	private function get_manager(): Instant_Indexing_Manager {
510	if (null === $this->manager) {
511	$this->manager = new Instant_Indexing_Manager();
512	}
513
514	return $this->manager;
515	}
516	}
517