xspeed / includes / class-preloader.php

class-preloader.php in xSpeed Cache: AI-Powered Performance Hub with MCP, Caching & CDN 1.4.1, at includes/class-preloader.php

918 lines 32.7 KB

Raw Download Zip

1	<?php
2	/**
3	* Preloader — sitemap-driven cache warmer.
4	*
5	* On `start()`:
6	* 1. Fetches the configured sitemap (or auto-detects /wp-sitemap.xml).
7	* 2. Recursively follows nested sitemap indexes.
8	* 3. Filters URLs against the Cache module's excluded_urls list.
9	* 4. Queues the result in a transient.
10	* 5. Schedules the next WP-Cron tick.
11	*
12	* Each `tick()` processes up to `batch_size` URLs from the queue via
13	* `wp_remote_get()` (short timeout, sslverify off for local dev tolerance,
14	* a UA that flags itself so site owners can spot crawler traffic in
15	* access logs). Cache::should_cache() picks up the GET → writes the cache
16	* file on miss. The next visitor sees a HIT.
17	*
18	* State (transient `xspeed_preloader_state`):
19	* { running, started_at, finished_at, queue, processed, total,
20	* last_url, errors[] }
21	*
22	* Configuration comes from PreloaderModule's per-module option
23	* (xspeed_module_preloader) via Settings_Manager.
24	*
25	* @package XSpeed
26	*/
27
28	declare(strict_types=1);
29
30	namespace XSpeed;
31
32	defined( 'ABSPATH' ) \|\| exit;
33
34	final class Preloader {
35
36	public const STATE_KEY = 'xspeed_preloader_state';
37	public const STATE_TTL = 86400; // 24h — long enough for slow crawls.
38	public const CRON_HOOK = 'xspeed_preloader_tick';
39	/**
40	* User-agent for every request the preloader makes.
41	*
42	* Deliberately contains no substring from the 7G/8G bad-bot lists. The
43	* previous value, "xSpeed-Preloader/1.0", matched the `loader` token in
44	* the alphabetical slice `(linkscan\|linkwalker\|loader\|lwp-download\|...)`
45	* — a match on "Preloader" — so nginx ports of 8G answered every warm
46	* with 403 and newly published posts were never warmed. Upstream 8G
47	* v1.5 has since dropped `loader`, but forks and vendored copies (xCloud
48	* among them) still ship the older slice, so the name has to stay clear
49	* of it. "Warmer" matches nothing in either list. (#481)
50	*
51	* Read through user_agent() rather than using this constant directly, so
52	* the `xspeed_preloader_user_agent` filter applies.
53	*/
54	public const USER_AGENT = 'xSpeed-Warmer/1.0 (+cache warmer; admin-initiated)';
55	public const REQUEST_TIMEOUT = 8;
56
57	/**
58	* Default cap on NEW remote images resolved per warmed page.
59	*
60	* A crawl warms the cache; it is not a licence to hit third-party hosts
61	* hundreds of times for one page.
62	*
63	* The cap counts only images whose dimensions are not already known, and
64	* results persist between runs — so each crawl advances through a heavily
65	* embedded page rather than re-picking the same first N. That is what
66	* makes a cap safe here: without the skip it would strand everything past
67	* the limit permanently, because images appear in the same DOM order
68	* every time.
69	*
70	* 20 is a starting point, not a measurement. Sites that embed more can
71	* raise it via `xspeed_preloader_remote_dimension_limit`.
72	*/
73	private const REMOTE_DIMENSION_LIMIT = 20;
74
75	/**
76	* The user-agent every preloader request sends.
77	*
78	* Filterable because the blocking rule lives on the server, not here: a
79	* host with its own bad-bot list can clear a warm without patching the
80	* plugin or waiting for a release. An empty filter return is ignored —
81	* sending no UA gets a request blocked at least as often. (#481)
82	*/
83	public static function user_agent(): string {
84	/**
85	* Filter the preloader's user-agent string.
86	*
87	* @param string $user_agent Default self::USER_AGENT.
88	*/
89	$ua = apply_filters( 'xspeed_preloader_user_agent', self::USER_AGENT );
90
91	return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : self::USER_AGENT;
92	}
93
94	/**
95	* A real phone browser's user-agent, which the warmer's own is appended
96	* to for the phone copy.
97	*
98	* A real phone string rather than the warmer's UA plus a client hint:
99	* a theme or plugin that reads the user-agent itself (Mobile_Detect and
100	* the like) instead of calling wp_is_mobile() would otherwise render its
101	* desktop HTML into the phone copy. The warmer's name stays at the end so
102	* the request is still recognisable in access logs and still matches no
103	* bad-bot rule. (#596)
104	*/
105	public const MOBILE_UA_PREFIX = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1';
106
107	/** The user-agent the phone warm sends. */
108	public static function mobile_user_agent(): string {
109	$default = self::MOBILE_UA_PREFIX . ' ' . self::user_agent();
110	/**
111	* Filter the user-agent the preloader sends when it warms the phone
112	* copy of a page (Separate Mobile Cache on).
113	*
114	* It must still read as a phone: the cache files the response by the
115	* same test as wp_is_mobile(). An empty return is ignored.
116	*
117	* @param string $user_agent Default: an iPhone Safari string followed by the warmer's user-agent.
118	*/
119	$ua = apply_filters( 'xspeed_preloader_mobile_user_agent', $default );
120
121	return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : $default;
122	}
123
124	/**
125	* The copies of a page a warm fills: the desktop one, and the phone one
126	* too when Separate Mobile Cache keeps one per device. Before #596 every
127	* warm sent a desktop user-agent, so the first phone visitor to each
128	* page got an uncached render.
129	*
130	* @return string[] 'desktop', then 'mobile' when it applies.
131	*/
132	public static function devices(): array {
133	$cache = Settings_Manager::get( 'cache' );
134	$devices = empty( $cache['mobile_separate'] ) ? array( 'desktop' ) : array( 'desktop', 'mobile' );
135	/**
136	* Filter which copies of a page the preloader warms.
137	*
138	* Return array( 'desktop' ) to skip the phone copy, for example on a
139	* host where the extra requests cost too much. Unknown values are
140	* dropped; an empty list means desktop.
141	*
142	* @param string[] $devices 'desktop', plus 'mobile' when Separate Mobile Cache is on.
143	*/
144	$filtered = apply_filters( 'xspeed_preloader_devices', $devices );
145	$filtered = is_array( $filtered ) ? array_values( array_intersect( array( 'desktop', 'mobile' ), $filtered ) ) : $devices;
146
147	return empty( $filtered ) ? array( 'desktop' ) : $filtered;
148	}
149
150	/**
151	* Request arguments for one warm of one device copy.
152	*
153	* `Sec-CH-UA-Mobile` settles the device whatever the user-agent filters
154	* return: the cache checks that hint before the user-agent.
155	*
156	* @param array<string,string> $headers Extra request headers.
157	*/
158	private static function warm_args( string $device, array $headers = array() ): array {
159	$headers['Sec-CH-UA-Mobile'] = 'mobile' === $device ? '?1' : '?0';
160	return array(
161	'timeout' => self::REQUEST_TIMEOUT,
162	'sslverify' => false,
163	'user-agent' => 'mobile' === $device ? self::mobile_user_agent() : self::user_agent(),
164	'headers' => Self_Traffic::headers( $headers ),
165	'blocking' => true,
166	);
167	}
168
169	/**
170	* Warm every copy of $url that devices() names.
171	*
172	* @param array<string,string> $headers Extra request headers.
173	* @return array{failures: array<int,array{device:string,error:string}>, body: string}
174	* The failures, and the desktop response body ('' when it failed).
175	*/
176	private static function warm_devices( string $url, array $headers = array() ): array {
177	$failures = array();
178	$body = '';
179	foreach ( self::devices() as $device ) {
180	$args = self::warm_args( $device, $headers );
181	$response = wp_remote_get( $url, $args );
182	if ( is_wp_error( $response ) ) {
183	$failures[] = array( 'device' => $device, 'error' => $response->get_error_message() );
184	continue;
185	}
186	$code = (int) wp_remote_retrieve_response_code( $response );
187	if ( $code >= 400 ) {
188	$failures[] = array( 'device' => $device, 'error' => self::failure_detail( $code, $args['user-agent'] ) );
189	if ( self::is_firewall_block( $code ) ) {
190	self::remember_firewall_block( $url, $code, $args['user-agent'] );
191	}
192	continue;
193	}
194	if ( 'desktop' === $device ) {
195	$body = (string) wp_remote_retrieve_body( $response );
196	}
197	}
198	// The notice goes only once every copy got through. A host that
199	// blocks just the phone user-agent would otherwise have the notice
200	// cleared by each desktop warm right after the phone warm set it.
201	if ( empty( $failures ) ) {
202	self::clear_firewall_block();
203	}
204	return array( 'failures' => $failures, 'body' => $body );
205	}
206
207	/** "phone: " for a failure of the phone copy, so the log says which copy failed. */
208	private static function device_prefix( string $device ): string {
209	return 'mobile' === $device ? 'phone: ' : '';
210	}
211
212	/**
213	* Is this status code the signature of a firewall refusing our warmer?
214	*
215	* 403 and 406 are what bad-bot rules (7G/8G, mod_security, Wordfence)
216	* answer with. We only ever warm our OWN origin, and a page a visitor can
217	* load must be loadable by us too — so these codes mean the request was
218	* judged by its user-agent, not that the page is missing or broken. (#481)
219	*/
220	private static function is_firewall_block( int $code ): bool {
221	return in_array( $code, array( 403, 406 ), true );
222	}
223
224	/**
225	* Explain a warm failure in terms the admin can act on.
226	*
227	* A bare "HTTP 403" sent people hunting a broken page; the page is fine,
228	* and the fix is a server rule, so the message has to name the cause and
229	* the exact UA to allow. (#481)
230	*/
231	private static function failure_detail( int $code, string $user_agent = '' ): string {
232	if ( ! self::is_firewall_block( $code ) ) {
233	return sprintf( 'HTTP %d', $code );
234	}
235
236	return sprintf(
237	'HTTP %d — your server\'s firewall is blocking the xSpeed cache warmer by user-agent, so this page was not warmed. Allow the user-agent "%s" (on xCloud this is the 8G firewall\'s bad-bot rule), or change it with the xspeed_preloader_user_agent filter.',
238	$code,
239	'' !== $user_agent ? $user_agent : self::user_agent()
240	);
241	}
242
243	/** Option holding the last firewall-shaped warm refusal. */
244	public const FIREWALL_BLOCK_OPTION = 'xspeed_preloader_firewall_block';
245
246	/**
247	* Record that the origin refused a warm by user-agent, for ui_notices().
248	*
249	* An option rather than a transient: the condition is a server rule that
250	* persists until someone changes it, and a notice that expired on its own
251	* would let a site go back to never warming, silently. Cleared by
252	* clear_firewall_block() on the first warm that succeeds. (#481)
253	*/
254	private static function remember_firewall_block( string $url, int $code, string $user_agent = '' ): void {
255	if ( ! function_exists( 'update_option' ) ) {
256	return;
257	}
258	update_option(
259	self::FIREWALL_BLOCK_OPTION,
260	array(
261	'url' => $url,
262	'code' => $code,
263	'user_agent' => '' !== $user_agent ? $user_agent : self::user_agent(),
264	'ts' => time(),
265	),
266	false
267	);
268	}
269
270	/** Forget the firewall block once a warm gets through. */
271	public static function clear_firewall_block(): void {
272	if ( function_exists( 'delete_option' ) && self::firewall_block() ) {
273	delete_option( self::FIREWALL_BLOCK_OPTION );
274	}
275	}
276
277	/** The last firewall-shaped refusal, or null when there isn't one. */
278	public static function firewall_block(): ?array {
279	if ( ! function_exists( 'get_option' ) ) {
280	return null;
281	}
282	$block = get_option( self::FIREWALL_BLOCK_OPTION, null );
283
284	return ( is_array( $block ) && ! empty( $block['code'] ) ) ? $block : null;
285	}
286
287	/**
288	* How many new remote images one warmed page may resolve.
289	*/
290	private static function remote_dimension_limit(): int {
291	/**
292	* Filter the per-page cap on remote dimension lookups.
293	*
294	* @param int $limit Default 20. Values below 1 disable the lookup.
295	*/
296	return (int) apply_filters( 'xspeed_preloader_remote_dimension_limit', self::REMOTE_DIMENSION_LIMIT );
297	}
298
299	/**
300	* Why the top-level sitemap fetch failed on this request, or '' when it
301	* succeeded. Set by fetch_sitemap_urls(), read by resolve_queue() — the
302	* reason has to survive the return of an empty array, which is exactly
303	* what it could not do before. Request-scoped; never persisted. (#142)
304	*
305	* @var string
306	*/
307	private static $last_sitemap_error = '';
308
309	/**
310	* The sitemap URL the last error refers to. Kept beside the message so
311	* an error entry can carry a real `url` field like every other one,
312	* rather than repeating the URL already inside the message text.
313	*/
314	private static $last_sitemap_url = '';
315
316	/**
317	* How the queue for the current crawl was built — 'sitemap', 'fallback'
318	* (enumerated from the database because the sitemap was unreachable), or
319	* 'none'. Surfaced in the state so the panel, REST and CLI can each say
320	* what actually happened instead of reporting a bare zero. (#142)
321	*
322	* @var string
323	*/
324	private static $queue_source = 'none';
325
326	/** Largest number of URLs the database fallback will enumerate. */
327	private const FALLBACK_LIMIT = 500;
328
329	/**
330	* Kick off a fresh crawl. Returns the initial state.
331	*/
332	public static function start(): array {
333	$opts = Settings_Manager::get( 'preloader' );
334	$urls = self::resolve_queue( $opts );
335
336	// A crawl that queued nothing because the sitemap was unreachable is a
337	// FAILURE, and every layer above needs to be able to say so. It used
338	// to be indistinguishable from success: errors stayed empty, the REST
339	// route returned 200, and the CLI printed a green Success. (#142)
340	$sitemap_error = self::$last_sitemap_error;
341	$errors = array();
342	if ( '' !== $sitemap_error && empty( $urls ) ) {
343	// Same {url, error, ts} shape every other entry uses. A bare
344	// string here fataled `wp xspeed preloader status`, which
345	// destructures `$e['url']` over the list — and took the MCP
346	// `get_preloader_status` tool down with it, so an agent asking
347	// why the preload failed got "Cannot access offset of type
348	// string on string" instead of the reason this code records.
349	// `url` is the sitemap because that is what failed. (QA F1)
350	$errors[] = array(
351	'url' => self::$last_sitemap_url,
352	'error' => $sitemap_error,
353	'ts' => time(),
354	);
355	}
356
357	$state = array(
358	'running' => ! empty( $urls ),
359	'started_at' => time(),
360	'finished_at' => empty( $urls ) ? time() : 0,
361	'queue' => array_values( $urls ),
362	'processed' => 0,
363	'total' => count( $urls ),
364	'last_url' => '',
365	'errors' => $errors,
366	// Consumers render on these: the panel needs to distinguish
367	// "not started" from "ran and found nothing", and to tell the
368	// user when the queue came from the fallback rather than the
369	// sitemap they configured.
370	'source' => self::$queue_source,
371	'sitemap_error' => $sitemap_error,
372	// Which copies each URL gets, for the panel and the status
373	// command. Each tick refreshes it: turning Separate Mobile
374	// Cache on or off mid-crawl purges the cache, and the rest of
375	// the crawl should fill the copies that now exist.
376	'devices' => self::devices(),
377	);
378	set_transient( self::STATE_KEY, $state, self::STATE_TTL );
379
380	if ( '' !== $sitemap_error && 'fallback' === self::$queue_source ) {
381	$message = sprintf(
382	/* translators: 1: number of URLs, 2: the sitemap failure detail. */
383	__( 'Preloader queued %1$d URLs from the site content — %2$s', 'xspeed' ),
384	$state['total'],
385	$sitemap_error
386	);
387	$severity = Activity_Log::WARN;
388	} elseif ( '' !== $sitemap_error ) {
389	$message = sprintf(
390	/* translators: %s: the sitemap failure detail. */
391	__( 'Preloader could not start — %s', 'xspeed' ),
392	$sitemap_error
393	);
394	$severity = Activity_Log::WARN;
395	} else {
396	$message = sprintf(
397	/* translators: 1: number of URLs, 2: plural suffix. */
398	__( 'Preloader queued %1$d URL%2$s for warming.', 'xspeed' ),
399	$state['total'],
400	1 === $state['total'] ? '' : 's'
401	);
402	$severity = $state['total'] > 0 ? Activity_Log::INFO : Activity_Log::WARN;
403	}
404	Activity_Log::record( 'preloader_started', $message, $severity );
405
406	// Schedule the first tick ~5 seconds out so the kick-off REST call
407	// returns instantly; wp_schedule_single_event covers the
408	// "process the queue ASAP" path without a heavy synchronous loop.
409	if ( $state['running'] ) {
410	wp_schedule_single_event( time() + 5, self::CRON_HOOK );
411	}
412
413	return $state;
414	}
415
416	/**
417	* Cancel an in-flight crawl. Idempotent.
418	*/
419	public static function stop(): array {
420	$state = self::status();
421	if ( $state['running'] ) {
422	Activity_Log::record(
423	'preloader_stopped',
424	sprintf( 'Preloader stopped (%d/%d URLs warmed).', $state['processed'], $state['total'] ),
425	Activity_Log::INFO
426	);
427	}
428
429	// Clear scheduled ticks.
430	wp_clear_scheduled_hook( self::CRON_HOOK );
431
432	$state['running'] = false;
433	$state['finished_at'] = time();
434	$state['queue'] = array();
435	set_transient( self::STATE_KEY, $state, self::STATE_TTL );
436	return $state;
437	}
438
439	public static function status(): array {
440	$raw = get_transient( self::STATE_KEY );
441	if ( ! is_array( $raw ) ) {
442	return self::empty_state();
443	}
444	return wp_parse_args( $raw, self::empty_state() );
445	}
446
447	private static function empty_state(): array {
448	return array(
449	'running' => false,
450	'started_at' => 0,
451	'finished_at' => 0,
452	'queue' => array(),
453	'processed' => 0,
454	'total' => 0,
455	'last_url' => '',
456	'errors' => array(),
457	);
458	}
459
460	/**
461	* Tick handler — pulls up to batch_size URLs off the queue, warms
462	* each, persists state, and reschedules itself until the queue is
463	* empty. Called via the xspeed_preloader_tick action.
464	*/
465	public static function tick(): void {
466	$state = self::status();
467	if ( ! $state['running'] \|\| empty( $state['queue'] ) ) {
468	if ( $state['running'] ) {
469	self::mark_complete( $state );
470	}
471	return;
472	}
473
474	$opts = Settings_Manager::get( 'preloader' );
475	$batch = max( 1, min( 50, (int) ( $opts['batch_size'] ?? 5 ) ) );
476
477	/**
478	* Filter: xspeed_preload_batch_size
479	*
480	* How much one tick may warm, in the units the batch size setting
481	* counts. The setting is the site owner's
482	* intent; this is for anything that knows a lower ceiling applies —
483	* a CDN with a rate limit in front, say, which will answer a burst
484	* with a challenge and leave the queue looking warmed when it is not.
485	*
486	* Only ever lowers. A filter that raised it would let an add-on
487	* overrule a number the site owner chose, and the reason to reach for
488	* this is always that something cannot take the current rate.
489	*
490	* @param int $batch The batch this tick would otherwise use.
491	*/
492	$ceiling = (int) apply_filters( 'xspeed_preload_batch_size', $batch );
493	if ( $ceiling > 0 && $ceiling < $batch ) {
494	$batch = $ceiling;
495	}
496
497	// batch_size caps requests, not URLs, so the load on the origin per
498	// tick is the same with the phone copy on: each URL costs one request
499	// per device. A tick always warms at least one URL.
500	$devices = self::devices();
501	$state['devices'] = $devices;
502	$per_url = count( $devices );
503	$requests = 0;
504	while ( ! empty( $state['queue'] ) && ( 0 === $requests \|\| $requests + $per_url <= $batch ) ) {
505	$url = (string) array_shift( $state['queue'] );
506	self::warm_url( $url, $state );
507	$state['processed']++;
508	$state['last_url'] = $url;
509	$requests += $per_url;
510	}
511
512	if ( empty( $state['queue'] ) ) {
513	self::mark_complete( $state );
514	return;
515	}
516
517	// More to do — persist + reschedule. Slight delay to avoid
518	// hammering the origin with parallel batches.
519	set_transient( self::STATE_KEY, $state, self::STATE_TTL );
520	wp_schedule_single_event( time() + 10, self::CRON_HOOK );
521	}
522
523	/**
524	* Fire a single warm request for one URL with no queue / no cron
525	* (the "content warmer" path: new post published → warm its URL
526	* immediately). Records an activity event so the user can see in
527	* the Health log that warming happened.
528	*
529	* Best-effort and non-blocking-feeling — uses a short timeout so a
530	* dead origin can't hang the calling request. Returns true if the
531	* fetch completed with a non-error status, false otherwise.
532	*/
533	public static function warm_one( string $url, string $cause = 'manual' ): bool {
534	if ( '' === $url ) {
535	return false;
536	}
537	$result = self::warm_devices( $url );
538	foreach ( $result['failures'] as $failure ) {
539	Activity_Log::record(
540	'preloader_warm_failed',
541	sprintf( 'Warm %s failed (%s): %s%s', $cause, $url, self::device_prefix( $failure['device'] ), $failure['error'] ),
542	Activity_Log::WARN
543	);
544	}
545	if ( ! empty( $result['failures'] ) ) {
546	return false;
547	}
548	Activity_Log::record(
549	'preloader_warmed_one',
550	sprintf( 'Warmed %s (%s)', $url, $cause ),
551	Activity_Log::INFO
552	);
553	return true;
554	}
555
556	private static function warm_url( string $url, array &$state ): void {
557	$result = self::warm_devices( $url, array( 'Accept' => 'text/html,application/xhtml+xml' ) );
558	foreach ( $result['failures'] as $failure ) {
559	$state['errors'][] = array(
560	'url' => $url,
561	'error' => self::device_prefix( $failure['device'] ) . $failure['error'],
562	'ts' => time(),
563	);
564	}
565	// Cap retained errors so a broken sitemap doesn't blow the
566	// transient size.
567	$state['errors'] = array_slice( $state['errors'], -20 );
568	if ( '' !== $result['body'] ) {
569	self::warm_remote_dimensions( $result['body'] );
570	}
571	}
572
573	/**
574	* Resolve dimensions for externally hosted images found on a warmed page.
575	*
576	* The crawl already has the HTML in hand, so harvesting image URLs from it
577	* costs nothing extra — and this is the one place where paying for a
578	* remote lookup is free of consequence, because no visitor is waiting.
579	*
580	* An image on another domain has no local file to measure, so the front
581	* end skips it and the page ships without width/height — which is layout
582	* shift, on precisely the sites least able to fix it by hand (a CDN, a
583	* sister site, a shared asset host). Warming here means the NEXT render
584	* finds the dimensions in cache and stamps them, with the visitor paying
585	* nothing.
586	*
587	* Deliberately bounded per page: a crawl should not turn into a scraper
588	* for a page embedding hundreds of third-party images.
589	*
590	* @param string $html The warmed page's HTML.
591	*/
592	private static function warm_remote_dimensions( string $html ): void {
593	if ( '' === $html \|\| ! class_exists( '\XSpeed\Lazy_Loader' ) ) {
594	return;
595	}
596
597	$opts = Settings_Manager::get( 'lazy' );
598	if ( empty( $opts['add_missing_dimensions'] ) ) {
599	return;
600	}
601
602	// Match any <img>, not only one carrying `src`. The URL worth warming
603	// may live in a lazy attribute instead — which is the whole point of
604	// #328 — and resolvable_image_url() below is what knows where to look.
605	if ( ! preg_match_all( '#<img\b[^>]*>#i', $html, $m, PREG_SET_ORDER ) ) {
606	return;
607	}
608
609	$home = wp_parse_url( home_url(), PHP_URL_HOST );
610	$targets = array();
611	foreach ( $m as $tag ) {
612	// Only tags MISSING a dimension are worth resolving — one that
613	// already declares both needs nothing.
614	// Same lookbehind as Lazy_Loader::ensure_dimensions(): a bare
615	// `\bwidth=` also matches `data-width=`, so a slider carrying its
616	// own metadata looked already-sized and was skipped from warming.
617	// The two must agree, or the collector skips exactly the tags the
618	// renderer still needs measured. (#333 review round 3, issue 2)
619	if ( preg_match( '#(?<![-\w])width\s=#i', $tag[0] ) && preg_match( '#(?<![-\w])height\s=#i', $tag[0] ) ) {
620	continue;
621	}
622	// Ask the same resolver the render path uses, rather than reading
623	// `src` directly. A slider parks a spacer in `src` and the real
624	// URL in `data-lazy`/`data-src`/`data-original`, so a collector
625	// looking only at `src` warmed the SPACER and never the image —
626	// leaving remotely-hosted slider images unresolvable at render
627	// time, the exact markup #328 is about. (#333 review round 2,
628	// issue 3)
629	// `false`: do not let the resolver settle a name-refused URL by
630	// MEASURING it. That is circular here — remote measurement is
631	// gated until warm_dimensions() sets $warming, and this collector
632	// is what feeds warm_dimensions(). Take the URL the tag offers and
633	// let the warm pass decide. (#333 review round 3, issue 3)
634	$src = Lazy_Loader::resolvable_image_url( $tag[0], false );
635	if ( '' === $src \|\| ! preg_match( '#^https?://#i', $src ) ) {
636	continue;
637	}
638	$host = wp_parse_url( $src, PHP_URL_HOST );
639	if ( ! $host \|\| $host === $home ) {
640	continue; // Local images already resolve from disk.
641	}
642	// Already resolved (or already known unresolvable) — looking it up
643	// again costs a request and teaches us nothing. Skipping it is
644	// also what makes the cap below advance: images appear in the same
645	// DOM order every crawl, so a collector that did not skip would
646	// re-pick the same first N for ever and never reach the rest.
647	if ( Lazy_Loader::dimensions_known( $src ) ) {
648	continue;
649	}
650	$targets[ $src ] = true;
651	if ( count( $targets ) >= self::remote_dimension_limit() ) {
652	break;
653	}
654	}
655
656	if ( $targets ) {
657	Lazy_Loader::warm_dimensions( array_keys( $targets ) );
658	}
659	}
660
661	private static function mark_complete( array $state ): void {
662	$state['running'] = false;
663	$state['finished_at'] = time();
664	$state['queue'] = array();
665	set_transient( self::STATE_KEY, $state, self::STATE_TTL );
666
667	Activity_Log::record(
668	'preloader_completed',
669	sprintf(
670	'Preloader finished — %d/%d URLs warmed, %d error%s.',
671	$state['processed'],
672	$state['total'],
673	count( $state['errors'] ),
674	1 === count( $state['errors'] ) ? '' : 's'
675	),
676	empty( $state['errors'] ) ? Activity_Log::SUCCESS : Activity_Log::WARN
677	);
678	}
679
680	/**
681	* Build the URL queue for a fresh crawl: parse the sitemap, follow
682	* nested indexes, drop excluded paths.
683	*
684	* @return string[]
685	*/
686	private static function resolve_queue( array $opts ): array {
687	// Request-scoped statics: reset so a previous crawl in the same
688	// process can't leak its verdict into this one.
689	self::$last_sitemap_error = '';
690	self::$last_sitemap_url = '';
691	self::$queue_source = 'none';
692
693	$sitemap = trim( (string) ( $opts['sitemap_url'] ?? '' ) );
694	if ( '' === $sitemap ) {
695	$sitemap = home_url( '/wp-sitemap.xml' );
696	}
697
698	$urls = self::fetch_sitemap_urls( $sitemap, 0 );
699	$from = empty( $urls ) ? 'none' : 'sitemap';
700
701	/*
702	* A missing sitemap must not disable the feature. Two very common
703	* setups produce one with no misconfiguration by the user:
704	* `blog_public = 0` (WordPress disables /wp-sitemap.xml outright,
705	* standard on staging and pre-launch sites), and an SEO plugin
706	* filtering `wp_sitemaps_enabled` to false while serving its own
707	* sitemap at a path we were never told about.
708	*
709	* Enumerate warmable URLs straight from the database instead. Only
710	* on a genuine fetch FAILURE — a sitemap that is reachable and
711	* legitimately empty is a real answer, and silently crawling
712	* something else would be worse than doing nothing. (#142)
713	*/
714	if ( empty( $urls ) && '' !== self::$last_sitemap_error ) {
715	$urls = self::fallback_urls();
716	$from = empty( $urls ) ? 'none' : 'fallback';
717	}
718
719	$cache_opts = Settings_Manager::get( 'cache' );
720	$excluded = is_array( $cache_opts['excluded_urls'] ?? null ) ? $cache_opts['excluded_urls'] : array();
721	if ( ! empty( $excluded ) ) {
722	$urls = array_filter(
723	$urls,
724	static function ( $u ) use ( $excluded ) {
725	$path = (string) wp_parse_url( $u, PHP_URL_PATH );
726	foreach ( $excluded as $needle ) {
727	if ( '' !== $needle && false !== strpos( $path, (string) $needle ) ) {
728	return false;
729	}
730	}
731	return true;
732	}
733	);
734	}
735
736	// Dedup + cap at 5000 to bound the transient size on huge sites.
737	$urls = array_values( array_unique( $urls ) );
738	$urls = array_slice( $urls, 0, 5000 );
739
740	/*
741	* Commit the verdict only now, AFTER the exclusion filter — the
742	* source describes what we ACTUALLY queued, not what we hoped to.
743	* Setting it earlier let a queue that the exclusions stripped to
744	* nothing still claim `fallback`, so the panel announced "Warmed
745	* from site content" over 0 URLs, and `crawlFailed` (which needs
746	* source !== 'fallback' at total 0) could never become true.
747	* One assignment fixes both. (QA F2/F3 on #155)
748	*/
749	self::$queue_source = empty( $urls ) ? 'none' : $from;
750
751	return $urls;
752	}
753
754	/**
755	* Enumerate warmable URLs from the database, for sites whose sitemap
756	* can't be fetched. Deliberately modest in scope: the home page, then
757	* the most recently modified public posts across every public post type.
758	* Newest-first is the right bias — those are the URLs most likely to be
759	* requested and least likely to be warm already.
760	*
761	* Uses WP_Query rather than SQL so post-type registration, status
762	* handling and multisite switching all behave the way the rest of
763	* WordPress does.
764	*
765	* @return string[]
766	*/
767	private static function fallback_urls(): array {
768	$urls = array();
769	$home = (string) home_url( '/' );
770	if ( '' !== trim( $home, '/' ) ) {
771	$urls[] = $home;
772	}
773
774	$types = get_post_types(
775	array(
776	'public' => true,
777	'publicly_queryable' => true,
778	)
779	);
780	// `page` is public but not publicly_queryable, so the query above
781	// misses it — and pages are exactly what a warm cache wants most.
782	// Only add it when the site has post types at all: an empty list
783	// means there is nothing to enumerate, and constructing a WP_Query
784	// for it would be wasted work.
785	if ( ! empty( $types ) ) {
786	$types['page'] = 'page';
787	unset( $types['attachment'] );
788	}
789
790	if ( empty( $types ) \|\| ! class_exists( '\WP_Query' ) ) {
791	return $urls;
792	}
793
794	$query = new \WP_Query(
795	array(
796	'post_type' => array_values( $types ),
797	'post_status' => 'publish',
798	'posts_per_page' => self::FALLBACK_LIMIT,
799	'orderby' => 'modified',
800	'order' => 'DESC',
801	'ignore_sticky_posts' => true,
802	'no_found_rows' => true,
803	'update_post_meta_cache' => false,
804	'update_post_term_cache' => false,
805	'fields' => 'ids',
806	)
807	);
808
809	foreach ( $query->posts as $post_id ) {
810	$permalink = get_permalink( (int) $post_id );
811	if ( is_string( $permalink ) && '' !== $permalink ) {
812	$urls[] = $permalink;
813	}
814	}
815
816	return array_values( array_unique( $urls ) );
817	}
818
819	/**
820	* Recursive sitemap parser. Depth-limited to 3 so a maliciously
821	* deep index can't stack-overflow.
822	*/
823	private static function fetch_sitemap_urls( string $sitemap_url, int $depth ): array {
824	if ( $depth > 3 ) {
825	return array();
826	}
827	$res = wp_remote_get(
828	$sitemap_url,
829	array(
830	'timeout' => self::REQUEST_TIMEOUT,
831	'sslverify' => false,
832	'user-agent' => self::user_agent(),
833	'headers' => Self_Traffic::headers(),
834	)
835	);
836	if ( is_wp_error( $res ) ) {
837	// Record WHY, don't just vanish. "Unreachable" and "valid but
838	// empty" both used to collapse into an empty array here, which is
839	// what made a sitemap-less site look like a successful crawl of
840	// zero URLs. Only the top-level fetch is recorded: a nested index
841	// failing is a partial result, not a dead crawl. (#142)
842	if ( 0 === $depth ) {
843	self::$last_sitemap_error = sprintf(
844	/* translators: 1: sitemap URL, 2: error detail. */
845	__( 'Could not fetch the sitemap at %1$s — %2$s', 'xspeed' ),
846	$sitemap_url,
847	$res->get_error_message()
848	);
849	self::$last_sitemap_url = (string) $sitemap_url;
850	}
851	return array();
852	}
853	$code = (int) wp_remote_retrieve_response_code( $res );
854	if ( $code >= 400 ) {
855	if ( 0 === $depth ) {
856	self::$last_sitemap_error = sprintf(
857	/* translators: 1: sitemap URL, 2: HTTP status code. */
858	__( 'Could not fetch the sitemap at %1$s — the server returned HTTP %2$d.', 'xspeed' ),
859	$sitemap_url,
860	$code
861	);
862	self::$last_sitemap_url = (string) $sitemap_url;
863	}
864	return array();
865	}
866	$body = (string) wp_remote_retrieve_body( $res );
867	if ( '' === $body ) {
868	return array();
869	}
870
871	$urls = array();
872	// Sitemap index → recurse.
873	if ( false !== strpos( $body, '<sitemapindex' ) ) {
874	if ( preg_match_all( '#<loc>([^<]+)</loc>#i', $body, $matches ) ) {
875	foreach ( $matches[1] as $child ) {
876	$urls = array_merge( $urls, self::fetch_sitemap_urls( trim( $child ), $depth + 1 ) );
877	}
878	}
879	return $urls;
880	}
881	// URL set → collect.
882	if ( preg_match_all( '#<loc>([^<]+)</loc>#i', $body, $matches ) ) {
883	foreach ( $matches[1] as $u ) {
884	$u = trim( $u );
885	if ( '' !== $u && false !== filter_var( $u, FILTER_VALIDATE_URL ) ) {
886	$urls[] = $u;
887	}
888	}
889	}
890	return $urls;
891	}
892
893	/**
894	* Apply the user's schedule choice. Called on settings change.
895	* Manual = no cron schedule (user must hit "Start now" to crawl).
896	*/
897	public static function apply_schedule( string $schedule ): void {
898	wp_clear_scheduled_hook( 'xspeed_preloader_recurring' );
899	if ( in_array( $schedule, array( 'hourly', 'daily', 'weekly' ), true ) ) {
900	if ( ! wp_next_scheduled( 'xspeed_preloader_recurring' ) ) {
901	wp_schedule_event( time() + 60, $schedule, 'xspeed_preloader_recurring' );
902	}
903	}
904	}
905
906	/**
907	* Recurring schedule hook handler — fires per the user's chosen
908	* cadence and kicks off a fresh crawl unless one is already running.
909	*/
910	public static function recurring_kickoff(): void {
911	$state = self::status();
912	if ( $state['running'] ) {
913	return;
914	}
915	self::start();
916	}
917	}
918