xspeed / includes / class-preloader.php

class-preloader.php in xSpeed Cache: AI-Powered Performance Hub with MCP, Caching & CDN 1.3.7, at includes/class-preloader.php

818 lines 27.6 KB

Raw Download Zip

1	<?php
2	/**
3	* Preloader — sitemap-driven cache warmer.
4	*
5	* On `start()`:
6	* 1. Fetches the configured sitemap (or auto-detects /wp-sitemap.xml).
7	* 2. Recursively follows nested sitemap indexes.
8	* 3. Filters URLs against the Cache module's excluded_urls list.
9	* 4. Queues the result in a transient.
10	* 5. Schedules the next WP-Cron tick.
11	*
12	* Each `tick()` processes up to `batch_size` URLs from the queue via
13	* `wp_remote_get()` (short timeout, sslverify off for local dev tolerance,
14	* a UA that flags itself so site owners can spot crawler traffic in
15	* access logs). Cache::should_cache() picks up the GET → writes the cache
16	* file on miss. The next visitor sees a HIT.
17	*
18	* State (transient `xspeed_preloader_state`):
19	* { running, started_at, finished_at, queue, processed, total,
20	* last_url, errors[] }
21	*
22	* Configuration comes from PreloaderModule's per-module option
23	* (xspeed_module_preloader) via Settings_Manager.
24	*
25	* @package XSpeed
26	*/
27
28	declare(strict_types=1);
29
30	namespace XSpeed;
31
32	defined( 'ABSPATH' ) \|\| exit;
33
34	final class Preloader {
35
36	public const STATE_KEY = 'xspeed_preloader_state';
37	public const STATE_TTL = 86400; // 24h — long enough for slow crawls.
38	public const CRON_HOOK = 'xspeed_preloader_tick';
39	/**
40	* User-agent for every request the preloader makes.
41	*
42	* Deliberately contains no substring from the 7G/8G bad-bot lists. The
43	* previous value, "xSpeed-Preloader/1.0", matched the `loader` token in
44	* the alphabetical slice `(linkscan\|linkwalker\|loader\|lwp-download\|...)`
45	* — a match on "Preloader" — so nginx ports of 8G answered every warm
46	* with 403 and newly published posts were never warmed. Upstream 8G
47	* v1.5 has since dropped `loader`, but forks and vendored copies (xCloud
48	* among them) still ship the older slice, so the name has to stay clear
49	* of it. "Warmer" matches nothing in either list. (#481)
50	*
51	* Read through user_agent() rather than using this constant directly, so
52	* the `xspeed_preloader_user_agent` filter applies.
53	*/
54	public const USER_AGENT = 'xSpeed-Warmer/1.0 (+cache warmer; admin-initiated)';
55	public const REQUEST_TIMEOUT = 8;
56
57	/**
58	* Default cap on NEW remote images resolved per warmed page.
59	*
60	* A crawl warms the cache; it is not a licence to hit third-party hosts
61	* hundreds of times for one page.
62	*
63	* The cap counts only images whose dimensions are not already known, and
64	* results persist between runs — so each crawl advances through a heavily
65	* embedded page rather than re-picking the same first N. That is what
66	* makes a cap safe here: without the skip it would strand everything past
67	* the limit permanently, because images appear in the same DOM order
68	* every time.
69	*
70	* 20 is a starting point, not a measurement. Sites that embed more can
71	* raise it via `xspeed_preloader_remote_dimension_limit`.
72	*/
73	private const REMOTE_DIMENSION_LIMIT = 20;
74
75	/**
76	* The user-agent every preloader request sends.
77	*
78	* Filterable because the blocking rule lives on the server, not here: a
79	* host with its own bad-bot list can clear a warm without patching the
80	* plugin or waiting for a release. An empty filter return is ignored —
81	* sending no UA gets a request blocked at least as often. (#481)
82	*/
83	public static function user_agent(): string {
84	/**
85	* Filter the preloader's user-agent string.
86	*
87	* @param string $user_agent Default self::USER_AGENT.
88	*/
89	$ua = apply_filters( 'xspeed_preloader_user_agent', self::USER_AGENT );
90
91	return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : self::USER_AGENT;
92	}
93
94	/**
95	* Is this status code the signature of a firewall refusing our warmer?
96	*
97	* 403 and 406 are what bad-bot rules (7G/8G, mod_security, Wordfence)
98	* answer with. We only ever warm our OWN origin, and a page a visitor can
99	* load must be loadable by us too — so these codes mean the request was
100	* judged by its user-agent, not that the page is missing or broken. (#481)
101	*/
102	private static function is_firewall_block( int $code ): bool {
103	return in_array( $code, array( 403, 406 ), true );
104	}
105
106	/**
107	* Explain a warm failure in terms the admin can act on.
108	*
109	* A bare "HTTP 403" sent people hunting a broken page; the page is fine,
110	* and the fix is a server rule, so the message has to name the cause and
111	* the exact UA to allow. (#481)
112	*/
113	private static function failure_detail( int $code ): string {
114	if ( ! self::is_firewall_block( $code ) ) {
115	return sprintf( 'HTTP %d', $code );
116	}
117
118	return sprintf(
119	'HTTP %d — your server\'s firewall is blocking the xSpeed cache warmer by user-agent, so this page was not warmed. Allow the user-agent "%s" (on xCloud this is the 8G firewall\'s bad-bot rule), or change it with the xspeed_preloader_user_agent filter.',
120	$code,
121	self::user_agent()
122	);
123	}
124
125	/** Option holding the last firewall-shaped warm refusal. */
126	public const FIREWALL_BLOCK_OPTION = 'xspeed_preloader_firewall_block';
127
128	/**
129	* Record that the origin refused a warm by user-agent, for ui_notices().
130	*
131	* An option rather than a transient: the condition is a server rule that
132	* persists until someone changes it, and a notice that expired on its own
133	* would let a site go back to never warming, silently. Cleared by
134	* clear_firewall_block() on the first warm that succeeds. (#481)
135	*/
136	private static function remember_firewall_block( string $url, int $code ): void {
137	if ( ! function_exists( 'update_option' ) ) {
138	return;
139	}
140	update_option(
141	self::FIREWALL_BLOCK_OPTION,
142	array(
143	'url' => $url,
144	'code' => $code,
145	'user_agent' => self::user_agent(),
146	'ts' => time(),
147	),
148	false
149	);
150	}
151
152	/** Forget the firewall block once a warm gets through. */
153	public static function clear_firewall_block(): void {
154	if ( function_exists( 'delete_option' ) && self::firewall_block() ) {
155	delete_option( self::FIREWALL_BLOCK_OPTION );
156	}
157	}
158
159	/** The last firewall-shaped refusal, or null when there isn't one. */
160	public static function firewall_block(): ?array {
161	if ( ! function_exists( 'get_option' ) ) {
162	return null;
163	}
164	$block = get_option( self::FIREWALL_BLOCK_OPTION, null );
165
166	return ( is_array( $block ) && ! empty( $block['code'] ) ) ? $block : null;
167	}
168
169	/**
170	* How many new remote images one warmed page may resolve.
171	*/
172	private static function remote_dimension_limit(): int {
173	/**
174	* Filter the per-page cap on remote dimension lookups.
175	*
176	* @param int $limit Default 20. Values below 1 disable the lookup.
177	*/
178	return (int) apply_filters( 'xspeed_preloader_remote_dimension_limit', self::REMOTE_DIMENSION_LIMIT );
179	}
180
181	/**
182	* Why the top-level sitemap fetch failed on this request, or '' when it
183	* succeeded. Set by fetch_sitemap_urls(), read by resolve_queue() — the
184	* reason has to survive the return of an empty array, which is exactly
185	* what it could not do before. Request-scoped; never persisted. (#142)
186	*
187	* @var string
188	*/
189	private static $last_sitemap_error = '';
190
191	/**
192	* The sitemap URL the last error refers to. Kept beside the message so
193	* an error entry can carry a real `url` field like every other one,
194	* rather than repeating the URL already inside the message text.
195	*/
196	private static $last_sitemap_url = '';
197
198	/**
199	* How the queue for the current crawl was built — 'sitemap', 'fallback'
200	* (enumerated from the database because the sitemap was unreachable), or
201	* 'none'. Surfaced in the state so the panel, REST and CLI can each say
202	* what actually happened instead of reporting a bare zero. (#142)
203	*
204	* @var string
205	*/
206	private static $queue_source = 'none';
207
208	/** Largest number of URLs the database fallback will enumerate. */
209	private const FALLBACK_LIMIT = 500;
210
211	/**
212	* Kick off a fresh crawl. Returns the initial state.
213	*/
214	public static function start(): array {
215	$opts = Settings_Manager::get( 'preloader' );
216	$urls = self::resolve_queue( $opts );
217
218	// A crawl that queued nothing because the sitemap was unreachable is a
219	// FAILURE, and every layer above needs to be able to say so. It used
220	// to be indistinguishable from success: errors stayed empty, the REST
221	// route returned 200, and the CLI printed a green Success. (#142)
222	$sitemap_error = self::$last_sitemap_error;
223	$errors = array();
224	if ( '' !== $sitemap_error && empty( $urls ) ) {
225	// Same {url, error, ts} shape every other entry uses. A bare
226	// string here fataled `wp xspeed preloader status`, which
227	// destructures `$e['url']` over the list — and took the MCP
228	// `get_preloader_status` tool down with it, so an agent asking
229	// why the preload failed got "Cannot access offset of type
230	// string on string" instead of the reason this code records.
231	// `url` is the sitemap because that is what failed. (QA F1)
232	$errors[] = array(
233	'url' => self::$last_sitemap_url,
234	'error' => $sitemap_error,
235	'ts' => time(),
236	);
237	}
238
239	$state = array(
240	'running' => ! empty( $urls ),
241	'started_at' => time(),
242	'finished_at' => empty( $urls ) ? time() : 0,
243	'queue' => array_values( $urls ),
244	'processed' => 0,
245	'total' => count( $urls ),
246	'last_url' => '',
247	'errors' => $errors,
248	// Consumers render on these: the panel needs to distinguish
249	// "not started" from "ran and found nothing", and to tell the
250	// user when the queue came from the fallback rather than the
251	// sitemap they configured.
252	'source' => self::$queue_source,
253	'sitemap_error' => $sitemap_error,
254	);
255	set_transient( self::STATE_KEY, $state, self::STATE_TTL );
256
257	if ( '' !== $sitemap_error && 'fallback' === self::$queue_source ) {
258	$message = sprintf(
259	/* translators: 1: number of URLs, 2: the sitemap failure detail. */
260	__( 'Preloader queued %1$d URLs from the site content — %2$s', 'xspeed' ),
261	$state['total'],
262	$sitemap_error
263	);
264	$severity = Activity_Log::WARN;
265	} elseif ( '' !== $sitemap_error ) {
266	$message = sprintf(
267	/* translators: %s: the sitemap failure detail. */
268	__( 'Preloader could not start — %s', 'xspeed' ),
269	$sitemap_error
270	);
271	$severity = Activity_Log::WARN;
272	} else {
273	$message = sprintf(
274	/* translators: 1: number of URLs, 2: plural suffix. */
275	__( 'Preloader queued %1$d URL%2$s for warming.', 'xspeed' ),
276	$state['total'],
277	1 === $state['total'] ? '' : 's'
278	);
279	$severity = $state['total'] > 0 ? Activity_Log::INFO : Activity_Log::WARN;
280	}
281	Activity_Log::record( 'preloader_started', $message, $severity );
282
283	// Schedule the first tick ~5 seconds out so the kick-off REST call
284	// returns instantly; wp_schedule_single_event covers the
285	// "process the queue ASAP" path without a heavy synchronous loop.
286	if ( $state['running'] ) {
287	wp_schedule_single_event( time() + 5, self::CRON_HOOK );
288	}
289
290	return $state;
291	}
292
293	/**
294	* Cancel an in-flight crawl. Idempotent.
295	*/
296	public static function stop(): array {
297	$state = self::status();
298	if ( $state['running'] ) {
299	Activity_Log::record(
300	'preloader_stopped',
301	sprintf( 'Preloader stopped (%d/%d URLs warmed).', $state['processed'], $state['total'] ),
302	Activity_Log::INFO
303	);
304	}
305
306	// Clear scheduled ticks.
307	wp_clear_scheduled_hook( self::CRON_HOOK );
308
309	$state['running'] = false;
310	$state['finished_at'] = time();
311	$state['queue'] = array();
312	set_transient( self::STATE_KEY, $state, self::STATE_TTL );
313	return $state;
314	}
315
316	public static function status(): array {
317	$raw = get_transient( self::STATE_KEY );
318	if ( ! is_array( $raw ) ) {
319	return self::empty_state();
320	}
321	return wp_parse_args( $raw, self::empty_state() );
322	}
323
324	private static function empty_state(): array {
325	return array(
326	'running' => false,
327	'started_at' => 0,
328	'finished_at' => 0,
329	'queue' => array(),
330	'processed' => 0,
331	'total' => 0,
332	'last_url' => '',
333	'errors' => array(),
334	);
335	}
336
337	/**
338	* Tick handler — pulls up to batch_size URLs off the queue, warms
339	* each, persists state, and reschedules itself until the queue is
340	* empty. Called via the xspeed_preloader_tick action.
341	*/
342	public static function tick(): void {
343	$state = self::status();
344	if ( ! $state['running'] \|\| empty( $state['queue'] ) ) {
345	if ( $state['running'] ) {
346	self::mark_complete( $state );
347	}
348	return;
349	}
350
351	$opts = Settings_Manager::get( 'preloader' );
352	$batch = max( 1, min( 50, (int) ( $opts['batch_size'] ?? 5 ) ) );
353
354	$processed_this_tick = 0;
355	while ( $processed_this_tick < $batch && ! empty( $state['queue'] ) ) {
356	$url = array_shift( $state['queue'] );
357	self::warm_url( $url, $state );
358	$state['processed']++;
359	$state['last_url'] = $url;
360	$processed_this_tick++;
361	}
362
363	if ( empty( $state['queue'] ) ) {
364	self::mark_complete( $state );
365	return;
366	}
367
368	// More to do — persist + reschedule. Slight delay to avoid
369	// hammering the origin with parallel batches.
370	set_transient( self::STATE_KEY, $state, self::STATE_TTL );
371	wp_schedule_single_event( time() + 10, self::CRON_HOOK );
372	}
373
374	/**
375	* Fire a single warm request for one URL with no queue / no cron
376	* (the "content warmer" path: new post published → warm its URL
377	* immediately). Records an activity event so the user can see in
378	* the Health log that warming happened.
379	*
380	* Best-effort and non-blocking-feeling — uses a short timeout so a
381	* dead origin can't hang the calling request. Returns true if the
382	* fetch completed with a non-error status, false otherwise.
383	*/
384	public static function warm_one( string $url, string $cause = 'manual' ): bool {
385	if ( '' === $url ) {
386	return false;
387	}
388	$response = wp_remote_get(
389	$url,
390	array(
391	'timeout' => self::REQUEST_TIMEOUT,
392	'sslverify' => false,
393	'user-agent' => self::user_agent(),
394	'headers' => Self_Traffic::headers(),
395	'blocking' => true,
396	)
397	);
398	if ( is_wp_error( $response ) ) {
399	Activity_Log::record(
400	'preloader_warm_failed',
401	sprintf( 'Warm %s failed (%s): %s', $cause, $url, $response->get_error_message() ),
402	Activity_Log::WARN
403	);
404	return false;
405	}
406	$code = (int) wp_remote_retrieve_response_code( $response );
407	if ( $code >= 400 ) {
408	Activity_Log::record(
409	'preloader_warm_failed',
410	sprintf( 'Warm %s failed (%s): %s', $cause, $url, self::failure_detail( $code ) ),
411	Activity_Log::WARN
412	);
413	if ( self::is_firewall_block( $code ) ) {
414	self::remember_firewall_block( $url, $code );
415	}
416	return false;
417	}
418	// A warm that got through proves the firewall is no longer refusing us,
419	// so the notice must go — otherwise it outlives the problem. (#481)
420	self::clear_firewall_block();
421	Activity_Log::record(
422	'preloader_warmed_one',
423	sprintf( 'Warmed %s (%s)', $url, $cause ),
424	Activity_Log::INFO
425	);
426	return true;
427	}
428
429	private static function warm_url( string $url, array &$state ): void {
430	$response = wp_remote_get(
431	$url,
432	array(
433	'timeout' => self::REQUEST_TIMEOUT,
434	'sslverify' => false,
435	'user-agent' => self::user_agent(),
436	'headers' => Self_Traffic::headers(
437	array(
438	'Accept' => 'text/html,application/xhtml+xml',
439	)
440	),
441	'blocking' => true,
442	)
443	);
444	if ( is_wp_error( $response ) ) {
445	$state['errors'][] = array(
446	'url' => $url,
447	'error' => $response->get_error_message(),
448	'ts' => time(),
449	);
450	// Cap retained errors so a broken sitemap doesn't blow the
451	// transient size.
452	$state['errors'] = array_slice( $state['errors'], -20 );
453	return;
454	}
455	$code = (int) wp_remote_retrieve_response_code( $response );
456	if ( $code >= 400 ) {
457	$state['errors'][] = array(
458	'url' => $url,
459	'error' => self::failure_detail( $code ),
460	'ts' => time(),
461	);
462	$state['errors'] = array_slice( $state['errors'], -20 );
463	if ( self::is_firewall_block( $code ) ) {
464	self::remember_firewall_block( $url, $code );
465	}
466	return;
467	}
468
469	self::clear_firewall_block();
470	self::warm_remote_dimensions( (string) wp_remote_retrieve_body( $response ) );
471	}
472
473	/**
474	* Resolve dimensions for externally hosted images found on a warmed page.
475	*
476	* The crawl already has the HTML in hand, so harvesting image URLs from it
477	* costs nothing extra — and this is the one place where paying for a
478	* remote lookup is free of consequence, because no visitor is waiting.
479	*
480	* An image on another domain has no local file to measure, so the front
481	* end skips it and the page ships without width/height — which is layout
482	* shift, on precisely the sites least able to fix it by hand (a CDN, a
483	* sister site, a shared asset host). Warming here means the NEXT render
484	* finds the dimensions in cache and stamps them, with the visitor paying
485	* nothing.
486	*
487	* Deliberately bounded per page: a crawl should not turn into a scraper
488	* for a page embedding hundreds of third-party images.
489	*
490	* @param string $html The warmed page's HTML.
491	*/
492	private static function warm_remote_dimensions( string $html ): void {
493	if ( '' === $html \|\| ! class_exists( '\XSpeed\Lazy_Loader' ) ) {
494	return;
495	}
496
497	$opts = Settings_Manager::get( 'lazy' );
498	if ( empty( $opts['add_missing_dimensions'] ) ) {
499	return;
500	}
501
502	// Match any <img>, not only one carrying `src`. The URL worth warming
503	// may live in a lazy attribute instead — which is the whole point of
504	// #328 — and resolvable_image_url() below is what knows where to look.
505	if ( ! preg_match_all( '#<img\b[^>]*>#i', $html, $m, PREG_SET_ORDER ) ) {
506	return;
507	}
508
509	$home = wp_parse_url( home_url(), PHP_URL_HOST );
510	$targets = array();
511	foreach ( $m as $tag ) {
512	// Only tags MISSING a dimension are worth resolving — one that
513	// already declares both needs nothing.
514	// Same lookbehind as Lazy_Loader::ensure_dimensions(): a bare
515	// `\bwidth=` also matches `data-width=`, so a slider carrying its
516	// own metadata looked already-sized and was skipped from warming.
517	// The two must agree, or the collector skips exactly the tags the
518	// renderer still needs measured. (#333 review round 3, issue 2)
519	if ( preg_match( '#(?<![-\w])width\s=#i', $tag[0] ) && preg_match( '#(?<![-\w])height\s=#i', $tag[0] ) ) {
520	continue;
521	}
522	// Ask the same resolver the render path uses, rather than reading
523	// `src` directly. A slider parks a spacer in `src` and the real
524	// URL in `data-lazy`/`data-src`/`data-original`, so a collector
525	// looking only at `src` warmed the SPACER and never the image —
526	// leaving remotely-hosted slider images unresolvable at render
527	// time, the exact markup #328 is about. (#333 review round 2,
528	// issue 3)
529	// `false`: do not let the resolver settle a name-refused URL by
530	// MEASURING it. That is circular here — remote measurement is
531	// gated until warm_dimensions() sets $warming, and this collector
532	// is what feeds warm_dimensions(). Take the URL the tag offers and
533	// let the warm pass decide. (#333 review round 3, issue 3)
534	$src = Lazy_Loader::resolvable_image_url( $tag[0], false );
535	if ( '' === $src \|\| ! preg_match( '#^https?://#i', $src ) ) {
536	continue;
537	}
538	$host = wp_parse_url( $src, PHP_URL_HOST );
539	if ( ! $host \|\| $host === $home ) {
540	continue; // Local images already resolve from disk.
541	}
542	// Already resolved (or already known unresolvable) — looking it up
543	// again costs a request and teaches us nothing. Skipping it is
544	// also what makes the cap below advance: images appear in the same
545	// DOM order every crawl, so a collector that did not skip would
546	// re-pick the same first N for ever and never reach the rest.
547	if ( Lazy_Loader::dimensions_known( $src ) ) {
548	continue;
549	}
550	$targets[ $src ] = true;
551	if ( count( $targets ) >= self::remote_dimension_limit() ) {
552	break;
553	}
554	}
555
556	if ( $targets ) {
557	Lazy_Loader::warm_dimensions( array_keys( $targets ) );
558	}
559	}
560
561	private static function mark_complete( array $state ): void {
562	$state['running'] = false;
563	$state['finished_at'] = time();
564	$state['queue'] = array();
565	set_transient( self::STATE_KEY, $state, self::STATE_TTL );
566
567	Activity_Log::record(
568	'preloader_completed',
569	sprintf(
570	'Preloader finished — %d/%d URLs warmed, %d error%s.',
571	$state['processed'],
572	$state['total'],
573	count( $state['errors'] ),
574	1 === count( $state['errors'] ) ? '' : 's'
575	),
576	empty( $state['errors'] ) ? Activity_Log::SUCCESS : Activity_Log::WARN
577	);
578	}
579
580	/**
581	* Build the URL queue for a fresh crawl: parse the sitemap, follow
582	* nested indexes, drop excluded paths.
583	*
584	* @return string[]
585	*/
586	private static function resolve_queue( array $opts ): array {
587	// Request-scoped statics: reset so a previous crawl in the same
588	// process can't leak its verdict into this one.
589	self::$last_sitemap_error = '';
590	self::$last_sitemap_url = '';
591	self::$queue_source = 'none';
592
593	$sitemap = trim( (string) ( $opts['sitemap_url'] ?? '' ) );
594	if ( '' === $sitemap ) {
595	$sitemap = home_url( '/wp-sitemap.xml' );
596	}
597
598	$urls = self::fetch_sitemap_urls( $sitemap, 0 );
599	$from = empty( $urls ) ? 'none' : 'sitemap';
600
601	/*
602	* A missing sitemap must not disable the feature. Two very common
603	* setups produce one with no misconfiguration by the user:
604	* `blog_public = 0` (WordPress disables /wp-sitemap.xml outright,
605	* standard on staging and pre-launch sites), and an SEO plugin
606	* filtering `wp_sitemaps_enabled` to false while serving its own
607	* sitemap at a path we were never told about.
608	*
609	* Enumerate warmable URLs straight from the database instead. Only
610	* on a genuine fetch FAILURE — a sitemap that is reachable and
611	* legitimately empty is a real answer, and silently crawling
612	* something else would be worse than doing nothing. (#142)
613	*/
614	if ( empty( $urls ) && '' !== self::$last_sitemap_error ) {
615	$urls = self::fallback_urls();
616	$from = empty( $urls ) ? 'none' : 'fallback';
617	}
618
619	$cache_opts = Settings_Manager::get( 'cache' );
620	$excluded = is_array( $cache_opts['excluded_urls'] ?? null ) ? $cache_opts['excluded_urls'] : array();
621	if ( ! empty( $excluded ) ) {
622	$urls = array_filter(
623	$urls,
624	static function ( $u ) use ( $excluded ) {
625	$path = (string) wp_parse_url( $u, PHP_URL_PATH );
626	foreach ( $excluded as $needle ) {
627	if ( '' !== $needle && false !== strpos( $path, (string) $needle ) ) {
628	return false;
629	}
630	}
631	return true;
632	}
633	);
634	}
635
636	// Dedup + cap at 5000 to bound the transient size on huge sites.
637	$urls = array_values( array_unique( $urls ) );
638	$urls = array_slice( $urls, 0, 5000 );
639
640	/*
641	* Commit the verdict only now, AFTER the exclusion filter — the
642	* source describes what we ACTUALLY queued, not what we hoped to.
643	* Setting it earlier let a queue that the exclusions stripped to
644	* nothing still claim `fallback`, so the panel announced "Warmed
645	* from site content" over 0 URLs, and `crawlFailed` (which needs
646	* source !== 'fallback' at total 0) could never become true.
647	* One assignment fixes both. (QA F2/F3 on #155)
648	*/
649	self::$queue_source = empty( $urls ) ? 'none' : $from;
650
651	return $urls;
652	}
653
654	/**
655	* Enumerate warmable URLs from the database, for sites whose sitemap
656	* can't be fetched. Deliberately modest in scope: the home page, then
657	* the most recently modified public posts across every public post type.
658	* Newest-first is the right bias — those are the URLs most likely to be
659	* requested and least likely to be warm already.
660	*
661	* Uses WP_Query rather than SQL so post-type registration, status
662	* handling and multisite switching all behave the way the rest of
663	* WordPress does.
664	*
665	* @return string[]
666	*/
667	private static function fallback_urls(): array {
668	$urls = array();
669	$home = (string) home_url( '/' );
670	if ( '' !== trim( $home, '/' ) ) {
671	$urls[] = $home;
672	}
673
674	$types = get_post_types(
675	array(
676	'public' => true,
677	'publicly_queryable' => true,
678	)
679	);
680	// `page` is public but not publicly_queryable, so the query above
681	// misses it — and pages are exactly what a warm cache wants most.
682	// Only add it when the site has post types at all: an empty list
683	// means there is nothing to enumerate, and constructing a WP_Query
684	// for it would be wasted work.
685	if ( ! empty( $types ) ) {
686	$types['page'] = 'page';
687	unset( $types['attachment'] );
688	}
689
690	if ( empty( $types ) \|\| ! class_exists( '\WP_Query' ) ) {
691	return $urls;
692	}
693
694	$query = new \WP_Query(
695	array(
696	'post_type' => array_values( $types ),
697	'post_status' => 'publish',
698	'posts_per_page' => self::FALLBACK_LIMIT,
699	'orderby' => 'modified',
700	'order' => 'DESC',
701	'ignore_sticky_posts' => true,
702	'no_found_rows' => true,
703	'update_post_meta_cache' => false,
704	'update_post_term_cache' => false,
705	'fields' => 'ids',
706	)
707	);
708
709	foreach ( $query->posts as $post_id ) {
710	$permalink = get_permalink( (int) $post_id );
711	if ( is_string( $permalink ) && '' !== $permalink ) {
712	$urls[] = $permalink;
713	}
714	}
715
716	return array_values( array_unique( $urls ) );
717	}
718
719	/**
720	* Recursive sitemap parser. Depth-limited to 3 so a maliciously
721	* deep index can't stack-overflow.
722	*/
723	private static function fetch_sitemap_urls( string $sitemap_url, int $depth ): array {
724	if ( $depth > 3 ) {
725	return array();
726	}
727	$res = wp_remote_get(
728	$sitemap_url,
729	array(
730	'timeout' => self::REQUEST_TIMEOUT,
731	'sslverify' => false,
732	'user-agent' => self::user_agent(),
733	'headers' => Self_Traffic::headers(),
734	)
735	);
736	if ( is_wp_error( $res ) ) {
737	// Record WHY, don't just vanish. "Unreachable" and "valid but
738	// empty" both used to collapse into an empty array here, which is
739	// what made a sitemap-less site look like a successful crawl of
740	// zero URLs. Only the top-level fetch is recorded: a nested index
741	// failing is a partial result, not a dead crawl. (#142)
742	if ( 0 === $depth ) {
743	self::$last_sitemap_error = sprintf(
744	/* translators: 1: sitemap URL, 2: error detail. */
745	__( 'Could not fetch the sitemap at %1$s — %2$s', 'xspeed' ),
746	$sitemap_url,
747	$res->get_error_message()
748	);
749	self::$last_sitemap_url = (string) $sitemap_url;
750	}
751	return array();
752	}
753	$code = (int) wp_remote_retrieve_response_code( $res );
754	if ( $code >= 400 ) {
755	if ( 0 === $depth ) {
756	self::$last_sitemap_error = sprintf(
757	/* translators: 1: sitemap URL, 2: HTTP status code. */
758	__( 'Could not fetch the sitemap at %1$s — the server returned HTTP %2$d.', 'xspeed' ),
759	$sitemap_url,
760	$code
761	);
762	self::$last_sitemap_url = (string) $sitemap_url;
763	}
764	return array();
765	}
766	$body = (string) wp_remote_retrieve_body( $res );
767	if ( '' === $body ) {
768	return array();
769	}
770
771	$urls = array();
772	// Sitemap index → recurse.
773	if ( false !== strpos( $body, '<sitemapindex' ) ) {
774	if ( preg_match_all( '#<loc>([^<]+)</loc>#i', $body, $matches ) ) {
775	foreach ( $matches[1] as $child ) {
776	$urls = array_merge( $urls, self::fetch_sitemap_urls( trim( $child ), $depth + 1 ) );
777	}
778	}
779	return $urls;
780	}
781	// URL set → collect.
782	if ( preg_match_all( '#<loc>([^<]+)</loc>#i', $body, $matches ) ) {
783	foreach ( $matches[1] as $u ) {
784	$u = trim( $u );
785	if ( '' !== $u && false !== filter_var( $u, FILTER_VALIDATE_URL ) ) {
786	$urls[] = $u;
787	}
788	}
789	}
790	return $urls;
791	}
792
793	/**
794	* Apply the user's schedule choice. Called on settings change.
795	* Manual = no cron schedule (user must hit "Start now" to crawl).
796	*/
797	public static function apply_schedule( string $schedule ): void {
798	wp_clear_scheduled_hook( 'xspeed_preloader_recurring' );
799	if ( in_array( $schedule, array( 'hourly', 'daily', 'weekly' ), true ) ) {
800	if ( ! wp_next_scheduled( 'xspeed_preloader_recurring' ) ) {
801	wp_schedule_event( time() + 60, $schedule, 'xspeed_preloader_recurring' );
802	}
803	}
804	}
805
806	/**
807	* Recurring schedule hook handler — fires per the user's chosen
808	* cadence and kicks off a fresh crawl unless one is already running.
809	*/
810	public static function recurring_kickoff(): void {
811	$state = self::status();
812	if ( $state['running'] ) {
813	return;
814	}
815	self::start();
816	}
817	}
818