thinkrank / includes / seo / class-instant-indexing-reconciler.php

class-instant-indexing-reconciler.php in ThinkRank AI SEO – AI SEO Plugin for WordPress: Schema, XML Sitemaps, Meta Tags, Search Console & Local SEO 2.14.1, at includes/seo/class-instant-indexing-reconciler.php

527 lines 19.1 KB

Raw Download Zip

1	<?php
2	/**
3	* Instant Indexing Reconciler Class
4	*
5	* Scheduled reconciliation between what has actually been published and what
6	* IndexNow has been told about, plus retries for the gaps it finds.
7	*
8	* @package ThinkRank
9	* @subpackage SEO
10	* @since 1.31.0
11	*/
12
13	declare(strict_types=1);
14
15	namespace ThinkRank\SEO;
16
17	// Prevent direct access
18	if (!defined('ABSPATH')) {
19	exit;
20	}
21
22	/**
23	* Instant Indexing Reconciler Class
24	*
25	* The automatic submission path is best-effort: it hangs off
26	* transition_post_status and defers the outbound call to a single WP-Cron event.
27	* Several ordinary situations therefore leave a published URL newer than
28	* anything IndexNow was told about — a direct `$wpdb->update()` fires no hooks
29	* at all, two saves inside the 15-second dedupe window collapse into one
30	* submission, a missed or dropped cron event disappears without trace, a failed
31	* submission is never retried, and anything edited while the feature was off is
32	* simply never sent.
33	*
34	* Rather than chase each of those individually, this reconciler compares state:
35	* every published URL's post_modified against the newest successful submission
36	* for that URL. That is mechanism-agnostic, so it closes gaps caused by causes
37	* nobody has thought of yet.
38	*
39	* Work is bounded. A cron run walks a batch of posts from a stored cursor and
40	* wraps around, so a large site reconciles over successive runs instead of
41	* trying to hold every permalink in memory at once.
42	*
43	* @since 1.31.0
44	*/
45	class Instant_Indexing_Reconciler {
46
47	/**
48	* Recurring reconciliation event.
49	*/
50	public const CRON_HOOK = 'thinkrank_instant_indexing_reconcile';
51
52	/**
53	* Option holding the batch cursor (offset into the ordered post list).
54	*/
55	private const CURSOR_OPTION = 'thinkrank_instant_indexing_reconcile_cursor';
56
57	/**
58	* Posts examined per cron run.
59	*/
60	public const BATCH_SIZE = 200;
61
62	/**
63	* Default ceiling on posts examined when building a report on demand.
64	*/
65	public const REPORT_LIMIT = 500;
66
67	/**
68	* How long to leave a failing URL alone before trying it again.
69	*/
70	public const RETRY_COOLDOWN = 6 * HOUR_IN_SECONDS;
71
72	/**
73	* Window over which repeated failures are counted for the park rule.
74	*/
75	public const FAILURE_WINDOW = 7 * DAY_IN_SECONDS;
76
77	/**
78	* Consecutive recent failures after which a URL stops being retried.
79	*
80	* Without this a permanently broken URL — 404, wrong host, revoked key —
81	* would consume the retry budget on every run forever and crowd out URLs
82	* that could actually succeed.
83	*/
84	public const MAX_FAILURES_BEFORE_PARK = 5;
85
86	/**
87	* Coverage states.
88	*/
89	public const STATE_CURRENT = 'current';
90	public const STATE_STALE = 'stale';
91	public const STATE_FAILED = 'failed';
92	public const STATE_NEVER = 'never_submitted';
93
94	/**
95	* Instant Indexing Manager instance.
96	*
97	* @var Instant_Indexing_Manager\|null
98	*/
99	private ?Instant_Indexing_Manager $manager = null;
100
101	/**
102	* Settings option name, shared with the manager.
103	*
104	* @var string
105	*/
106	private string $option_name = 'thinkrank_instant_indexing_settings';
107
108	/**
109	* Register the cron listener and make sure the event exists.
110	*
111	* @since 1.31.0
112	* @return void
113	*/
114	public function init(): void {
115	add_action(self::CRON_HOOK, [$this, 'reconcile']);
116
117	if (!wp_next_scheduled(self::CRON_HOOK)) {
118	// Offset the first run so a freshly activated site isn't reconciling
119	// during the activation request itself.
120	wp_schedule_event(time() + HOUR_IN_SECONDS, 'daily', self::CRON_HOOK);
121	}
122	}
123
124	/**
125	* Decide which coverage bucket a URL falls into.
126	*
127	* Pure: every input is passed in, so the rules can be tested without a
128	* database. All timestamps are site-local `Y-m-d H:i:s` strings, matching
129	* both `post_modified` and the log table's `created_at`; lexicographic
130	* comparison is correct for that format.
131	*
132	* @since 1.31.0
133	*
134	* @param string $post_modified Post's last modification time.
135	* @param string\|null $last_success Newest successful submission, or null.
136	* @param string\|null $last_attempt Newest submission of any status, or null.
137	* @return string One of the STATE_* constants.
138	*/
139	public static function classify(string $post_modified, ?string $last_success, ?string $last_attempt): string {
140	// Nothing was ever sent for this URL.
141	if (null === $last_attempt \|\| '' === $last_attempt) {
142	return self::STATE_NEVER;
143	}
144
145	// A success at or after the last edit means search engines know about
146	// the current content — even if a later attempt happened to fail.
147	if (null !== $last_success && '' !== $last_success && $last_success >= $post_modified) {
148	return self::STATE_CURRENT;
149	}
150
151	// The most recent attempt failed (the newest row is not the successful
152	// one), and nothing since the edit succeeded.
153	if ($last_success !== $last_attempt) {
154	return self::STATE_FAILED;
155	}
156
157	// Submitted successfully, but the post has been edited since.
158	return self::STATE_STALE;
159	}
160
161	/**
162	* Whether a URL in a given state should be resubmitted on this run.
163	*
164	* Pure, for the same reason as classify().
165	*
166	* @since 1.31.0
167	*
168	* @param string $state A STATE_* constant.
169	* @param string\|null $last_attempt Newest submission of any status, or null.
170	* @param int $recent_failures Failures inside FAILURE_WINDOW.
171	* @param int $now Current timestamp.
172	* @return bool True when the URL should be retried now.
173	*/
174	public static function should_retry(string $state, ?string $last_attempt, int $recent_failures, int $now): bool {
175	if (self::STATE_CURRENT === $state) {
176	return false;
177	}
178
179	if (self::STATE_FAILED === $state) {
180	// Park a URL that keeps failing so it can't monopolise the budget.
181	if ($recent_failures >= self::MAX_FAILURES_BEFORE_PARK) {
182	return false;
183	}
184
185	// Back off between attempts rather than hammering on every run.
186	if (null !== $last_attempt && '' !== $last_attempt) {
187	$attempted_at = strtotime($last_attempt);
188	if (false !== $attempted_at && ($now - $attempted_at) < self::RETRY_COOLDOWN) {
189	return false;
190	}
191	}
192	}
193
194	return true;
195	}
196
197	/**
198	* Build a coverage report over the published, auto-submitted content.
199	*
200	* @since 1.31.0
201	*
202	* @param int $limit Maximum posts to examine.
203	* @param int $offset Offset into the ordered post list.
204	* @param int $examples Maximum example URLs retained per bucket.
205	* @return array{
206	* counts: array<string,int>, total: int, examined: int, offset: int,
207	* urls: array<string,array<int,array<string,string>>>, truncated: bool,
208	* enabled: bool, generated_at: string
209	* }
210	*/
211	public function build_report(int $limit = self::REPORT_LIMIT, int $offset = 0, int $examples = 25): array {
212	$limit = max(1, $limit);
213	$offset = max(0, $offset);
214
215	$counts = [
216	self::STATE_CURRENT => 0,
217	self::STATE_STALE => 0,
218	self::STATE_FAILED => 0,
219	self::STATE_NEVER => 0,
220	];
221	$urls = [
222	self::STATE_CURRENT => [],
223	self::STATE_STALE => [],
224	self::STATE_FAILED => [],
225	self::STATE_NEVER => [],
226	];
227
228	$total = $this->count_tracked_posts();
229	$rows = $this->get_tracked_posts($limit, $offset);
230	$log = $this->get_submission_index();
231
232	foreach ($rows as $row) {
233	// A post that is not an indexable destination is not owed a
234	// submission. Counting it as "never submitted" made reconcile()
235	// send every noindexed, password-protected and redirected post the
236	// automatic path had rightly skipped (#911).
237	$post = get_post((int) $row['ID']);
238	if ($post instanceof \WP_Post && !Indexability::is_indexable_post($post)) {
239	continue;
240	}
241
242	$url = get_permalink((int) $row['ID']);
243	if (!$url) {
244	continue;
245	}
246
247	$entry = $log[$url] ?? null;
248	$state = self::classify(
249	(string) $row['post_modified'],
250	$entry['last_success'] ?? null,
251	$entry['last_attempt'] ?? null
252	);
253
254	$counts[$state]++;
255
256	if (count($urls[$state]) < $examples) {
257	$urls[$state][] = [
258	'url' => $url,
259	'post_id' => (int) $row['ID'],
260	'post_modified' => (string) $row['post_modified'],
261	'last_success' => $entry['last_success'] ?? '',
262	'last_attempt' => $entry['last_attempt'] ?? '',
263	];
264	}
265	}
266
267	return [
268	'counts' => $counts,
269	'total' => $total,
270	'examined' => count($rows),
271	'offset' => $offset,
272	'urls' => $urls,
273	// Signals that the counts describe a window, not the whole site, so
274	// the UI never presents a partial pass as a full audit.
275	'truncated' => ($offset + count($rows)) < $total,
276	'enabled' => $this->is_enabled(),
277	'generated_at' => current_time('mysql'),
278	];
279	}
280
281	/**
282	* Scheduled reconciliation pass: find gaps in one batch and resubmit them.
283	*
284	* @since 1.31.0
285	*
286	* @param bool $dry_run When true, classify and select but submit nothing.
287	* @return array Summary of the pass.
288	*/
289	public function reconcile(bool $dry_run = false): array {
290	$summary = [
291	'ran' => false,
292	'reason' => '',
293	'examined' => 0,
294	'counts' => [],
295	'retried' => 0,
296	'retried_urls' => [],
297	'submission' => null,
298	'next_offset' => 0,
299	];
300
301	if (!$this->is_enabled()) {
302	$summary['reason'] = 'Instant Indexing is disabled';
303	return $summary;
304	}
305
306	$total = $this->count_tracked_posts();
307	if (0 === $total) {
308	$summary['ran'] = true;
309	$summary['reason'] = 'No published content to reconcile';
310	return $summary;
311	}
312
313	// Walk the site in batches across successive runs, wrapping at the end.
314	$offset = (int) get_option(self::CURSOR_OPTION, 0);
315	if ($offset >= $total) {
316	$offset = 0;
317	}
318
319	$report = $this->build_report(self::BATCH_SIZE, $offset, self::BATCH_SIZE);
320	$log = $this->get_submission_index();
321	$now = time();
322
323	$retry = [];
324	foreach ([self::STATE_NEVER, self::STATE_STALE, self::STATE_FAILED] as $state) {
325	foreach ($report['urls'][$state] as $item) {
326	$entry = $log[$item['url']] ?? null;
327
328	if (!self::should_retry($state, $entry['last_attempt'] ?? null, (int) ($entry['recent_failures'] ?? 0), $now)) {
329	continue;
330	}
331
332	$retry[] = $item['url'];
333	}
334	}
335
336	// Respect the same per-submission ceiling every other path uses; the
337	// remainder is picked up by the next run.
338	$retry = array_slice(array_values(array_unique($retry)), 0, Instant_Indexing_Manager::MAX_URLS_PER_SUBMISSION);
339
340	$summary['ran'] = true;
341	$summary['examined'] = $report['examined'];
342	$summary['counts'] = $report['counts'];
343	$summary['retried'] = count($retry);
344	$summary['retried_urls'] = $retry;
345
346	if (!$dry_run) {
347	if (!empty($retry)) {
348	$summary['submission'] = $this->get_manager()->submit_urls($retry);
349	}
350
351	$next = $offset + $report['examined'];
352	if ($next >= $total \|\| 0 === $report['examined']) {
353	$next = 0;
354	}
355	update_option(self::CURSOR_OPTION, $next, false);
356	$summary['next_offset'] = $next;
357	}
358
359	return $summary;
360	}
361
362	/**
363	* Newest successful / newest overall submission per URL, plus recent failures.
364	*
365	* One aggregate query rather than a lookup per URL — the log has no usable
366	* equality index on the full 2048-char url column, so per-URL queries would
367	* scan the table once per post.
368	*
369	* @since 1.31.0
370	* @return array<string,array{last_success:?string,last_attempt:?string,recent_failures:int}>
371	*/
372	private function get_submission_index(): array {
373	global $wpdb;
374
375	$table = $wpdb->prefix . 'thinkrank_instant_indexing_logs';
376	$window_start = gmdate('Y-m-d H:i:s', (int) (current_time('timestamp') - self::FAILURE_WINDOW)); // phpcs:ignore WordPress.DateTime.CurrentTimeTimestamp.Requested -- log rows are written with current_time('mysql'), so the window must be computed in the same site-local frame.
377
378	// Aggregate over a plugin-owned log table. The only interpolated value is
379	// the table name, built from $wpdb->prefix; the date bound is prepared.
380	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber, PluginCheck.Security.DirectDB.UnescapedDBParameter
381	$rows = $wpdb->get_results(
382	$wpdb->prepare(
383	"SELECT url,
384	MAX(CASE WHEN status = 'success' THEN created_at END) AS last_success,
385	MAX(created_at) AS last_attempt,
386	SUM(CASE WHEN status <> 'success' AND created_at >= %s THEN 1 ELSE 0 END) AS recent_failures
387	FROM `{$table}`
388	GROUP BY url",
389	$window_start
390	),
391	ARRAY_A
392	);
393	// phpcs:enable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber, PluginCheck.Security.DirectDB.UnescapedDBParameter
394
395	$index = [];
396	foreach ((array) $rows as $row) {
397	$index[(string) $row['url']] = [
398	'last_success' => null !== $row['last_success'] ? (string) $row['last_success'] : null,
399	'last_attempt' => null !== $row['last_attempt'] ? (string) $row['last_attempt'] : null,
400	'recent_failures' => (int) $row['recent_failures'],
401	];
402	}
403
404	return $index;
405	}
406
407	/**
408	* Published posts of the auto-submitted types, ordered stably for cursoring.
409	*
410	* @since 1.31.0
411	*
412	* @param int $limit Maximum rows.
413	* @param int $offset Offset into the ordered list.
414	* @return array<int,array{ID:string,post_modified:string}>
415	*/
416	private function get_tracked_posts(int $limit, int $offset): array {
417	global $wpdb;
418
419	$types = $this->get_tracked_post_types();
420	if (empty($types)) {
421	return [];
422	}
423
424	$placeholders = implode(', ', array_fill(0, count($types), '%s'));
425	$params = array_merge($types, [$limit, $offset]);
426
427	// ID + timestamp only, over the indexed post_type/post_status pair;
428	// WP_Query would hydrate every post object for no benefit. The only
429	// interpolation is the %s placeholder list, built from a count.
430	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber, PluginCheck.Security.DirectDB.UnescapedDBParameter
431	$rows = $wpdb->get_results(
432	$wpdb->prepare(
433	"SELECT ID, post_modified
434	FROM {$wpdb->posts}
435	WHERE post_status = 'publish'
436	AND post_type IN ({$placeholders})
437	ORDER BY ID ASC
438	LIMIT %d OFFSET %d",
439	$params
440	),
441	ARRAY_A
442	);
443	// phpcs:enable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber, PluginCheck.Security.DirectDB.UnescapedDBParameter
444
445	$rows = (array) $rows;
446
447	// Prime the cache in one query so the get_permalink() calls that follow
448	// don't each fetch their post row separately. Meta too: build_report()
449	// reads each post's robots override to decide indexability.
450	if (!empty($rows)) {
451	_prime_post_caches(array_map(static fn($r) => (int) $r['ID'], $rows), false, true);
452	}
453
454	return $rows;
455	}
456
457	/**
458	* Total published posts in the auto-submitted types.
459	*
460	* @since 1.31.0
461	* @return int
462	*/
463	private function count_tracked_posts(): int {
464	global $wpdb;
465
466	$types = $this->get_tracked_post_types();
467	if (empty($types)) {
468	return 0;
469	}
470
471	$placeholders = implode(', ', array_fill(0, count($types), '%s'));
472
473	// Indexed COUNT over post_type/post_status; the only interpolation is the
474	// %s placeholder list, built from a count.
475	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, PluginCheck.Security.DirectDB.UnescapedDBParameter
476	return (int) $wpdb->get_var(
477	$wpdb->prepare(
478	"SELECT COUNT(*)
479	FROM {$wpdb->posts}
480	WHERE post_status = 'publish'
481	AND post_type IN ({$placeholders})",
482	$types
483	)
484	);
485	// phpcs:enable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.UnfinishedPrepare, PluginCheck.Security.DirectDB.UnescapedDBParameter
486	}
487
488	/**
489	* Post types configured for automatic submission.
490	*
491	* @since 1.31.0
492	* @return array<int,string>
493	*/
494	private function get_tracked_post_types(): array {
495	$settings = get_option($this->option_name, []);
496	$types = $settings['auto_submit_post_types'] ?? [];
497
498	return array_values(array_filter(array_map('strval', (array) $types)));
499	}
500
501	/**
502	* Whether Instant Indexing is switched on.
503	*
504	* @since 1.31.0
505	* @return bool
506	*/
507	private function is_enabled(): bool {
508	$settings = get_option($this->option_name, []);
509
510	return !empty($settings['enabled']);
511	}
512
513	/**
514	* Manager used for the outbound submission, built on first use.
515	*
516	* @since 1.31.0
517	* @return Instant_Indexing_Manager
518	*/
519	private function get_manager(): Instant_Indexing_Manager {
520	if (null === $this->manager) {
521	$this->manager = new Instant_Indexing_Manager();
522	}
523
524	return $this->manager;
525	}
526	}
527