← All changes | includes/class-preloader.php +616 -65 1.0.8 → 1.4.0 View file →

		@@ -35,12 +35,299 @@
35	35
36	36	public const STATE_KEY = 'xspeed_preloader_state';
37	37	public const STATE_TTL = 86400; // 24h — long enough for slow crawls.
38	38	public const CRON_HOOK = 'xspeed_preloader_tick';
39		- public const USER_AGENT = 'xSpeed-Preloader/1.0 (+cache warmer; admin-initiated)';
	39	+ /**
	40	+ * User-agent for every request the preloader makes.
	41	+ *
	42	+ * Deliberately contains no substring from the 7G/8G bad-bot lists. The
	43	+ * previous value, "xSpeed-Preloader/1.0", matched the `loader` token in
	44	+ * the alphabetical slice `(linkscan|linkwalker|loader|lwp-download|...)`
	45	+ * — a match on "Pre*loader*" — so nginx ports of 8G answered every warm
	46	+ * with 403 and newly published posts were never warmed. Upstream 8G
	47	+ * v1.5 has since dropped `loader`, but forks and vendored copies (xCloud
	48	+ * among them) still ship the older slice, so the name has to stay clear
	49	+ * of it. "Warmer" matches nothing in either list. (#481)
	50	+ *
	51	+ * Read through user_agent() rather than using this constant directly, so
	52	+ * the `xspeed_preloader_user_agent` filter applies.
	53	+ */
	54	+ public const USER_AGENT = 'xSpeed-Warmer/1.0 (+cache warmer; admin-initiated)';
40	55	public const REQUEST_TIMEOUT = 8;
41	56
42	57	/**
	58	+ * Default cap on NEW remote images resolved per warmed page.
	59	+ *
	60	+ * A crawl warms the cache; it is not a licence to hit third-party hosts
	61	+ * hundreds of times for one page.
	62	+ *
	63	+ * The cap counts only images whose dimensions are not already known, and
	64	+ * results persist between runs — so each crawl advances through a heavily
	65	+ * embedded page rather than re-picking the same first N. That is what
	66	+ * makes a cap safe here: without the skip it would strand everything past
	67	+ * the limit permanently, because images appear in the same DOM order
	68	+ * every time.
	69	+ *
	70	+ * 20 is a starting point, not a measurement. Sites that embed more can
	71	+ * raise it via `xspeed_preloader_remote_dimension_limit`.
	72	+ */
	73	+ private const REMOTE_DIMENSION_LIMIT = 20;
	74	+
	75	+ /**
	76	+ * The user-agent every preloader request sends.
	77	+ *
	78	+ * Filterable because the blocking rule lives on the server, not here: a
	79	+ * host with its own bad-bot list can clear a warm without patching the
	80	+ * plugin or waiting for a release. An empty filter return is ignored —
	81	+ * sending no UA gets a request blocked at least as often. (#481)
	82	+ */
	83	+ public static function user_agent(): string {
	84	+ /**
	85	+ * Filter the preloader's user-agent string.
	86	+ *
	87	+ * @param string $user_agent Default self::USER_AGENT.
	88	+ */
	89	+ $ua = apply_filters( 'xspeed_preloader_user_agent', self::USER_AGENT );
	90	+
	91	+ return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : self::USER_AGENT;
	92	+ }
	93	+
	94	+ /**
	95	+ * A real phone browser's user-agent, which the warmer's own is appended
	96	+ * to for the phone copy.
	97	+ *
	98	+ * A real phone string rather than the warmer's UA plus a client hint:
	99	+ * a theme or plugin that reads the user-agent itself (Mobile_Detect and
	100	+ * the like) instead of calling wp_is_mobile() would otherwise render its
	101	+ * desktop HTML into the phone copy. The warmer's name stays at the end so
	102	+ * the request is still recognisable in access logs and still matches no
	103	+ * bad-bot rule. (#596)
	104	+ */
	105	+ public const MOBILE_UA_PREFIX = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1';
	106	+
	107	+ /** The user-agent the phone warm sends. */
	108	+ public static function mobile_user_agent(): string {
	109	+ $default = self::MOBILE_UA_PREFIX . ' ' . self::user_agent();
	110	+ /**
	111	+ * Filter the user-agent the preloader sends when it warms the phone
	112	+ * copy of a page (Separate Mobile Cache on).
	113	+ *
	114	+ * It must still read as a phone: the cache files the response by the
	115	+ * same test as wp_is_mobile(). An empty return is ignored.
	116	+ *
	117	+ * @param string $user_agent Default: an iPhone Safari string followed by the warmer's user-agent.
	118	+ */
	119	+ $ua = apply_filters( 'xspeed_preloader_mobile_user_agent', $default );
	120	+
	121	+ return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : $default;
	122	+ }
	123	+
	124	+ /**
	125	+ * The copies of a page a warm fills: the desktop one, and the phone one
	126	+ * too when Separate Mobile Cache keeps one per device. Before #596 every
	127	+ * warm sent a desktop user-agent, so the first phone visitor to each
	128	+ * page got an uncached render.
	129	+ *
	130	+ * @return string[] 'desktop', then 'mobile' when it applies.
	131	+ */
	132	+ public static function devices(): array {
	133	+ $cache = Settings_Manager::get( 'cache' );
	134	+ $devices = empty( $cache['mobile_separate'] ) ? array( 'desktop' ) : array( 'desktop', 'mobile' );
	135	+ /**
	136	+ * Filter which copies of a page the preloader warms.
	137	+ *
	138	+ * Return array( 'desktop' ) to skip the phone copy, for example on a
	139	+ * host where the extra requests cost too much. Unknown values are
	140	+ * dropped; an empty list means desktop.
	141	+ *
	142	+ * @param string[] $devices 'desktop', plus 'mobile' when Separate Mobile Cache is on.
	143	+ */
	144	+ $filtered = apply_filters( 'xspeed_preloader_devices', $devices );
	145	+ $filtered = is_array( $filtered ) ? array_values( array_intersect( array( 'desktop', 'mobile' ), $filtered ) ) : $devices;
	146	+
	147	+ return empty( $filtered ) ? array( 'desktop' ) : $filtered;
	148	+ }
	149	+
	150	+ /**
	151	+ * Request arguments for one warm of one device copy.
	152	+ *
	153	+ * `Sec-CH-UA-Mobile` settles the device whatever the user-agent filters
	154	+ * return: the cache checks that hint before the user-agent.
	155	+ *
	156	+ * @param array<string,string> $headers Extra request headers.
	157	+ */
	158	+ private static function warm_args( string $device, array $headers = array() ): array {
	159	+ $headers['Sec-CH-UA-Mobile'] = 'mobile' === $device ? '?1' : '?0';
	160	+ return array(
	161	+ 'timeout' => self::REQUEST_TIMEOUT,
	162	+ 'sslverify' => false,
	163	+ 'user-agent' => 'mobile' === $device ? self::mobile_user_agent() : self::user_agent(),
	164	+ 'headers' => Self_Traffic::headers( $headers ),
	165	+ 'blocking' => true,
	166	+ );
	167	+ }
	168	+
	169	+ /**
	170	+ * Warm every copy of $url that devices() names.
	171	+ *
	172	+ * @param array<string,string> $headers Extra request headers.
	173	+ * @return array{failures: array<int,array{device:string,error:string}>, body: string}
	174	+ * The failures, and the desktop response body ('' when it failed).
	175	+ */
	176	+ private static function warm_devices( string $url, array $headers = array() ): array {
	177	+ $failures = array();
	178	+ $body = '';
	179	+ foreach ( self::devices() as $device ) {
	180	+ $args = self::warm_args( $device, $headers );
	181	+ $response = wp_remote_get( $url, $args );
	182	+ if ( is_wp_error( $response ) ) {
	183	+ $failures[] = array( 'device' => $device, 'error' => $response->get_error_message() );
	184	+ continue;
	185	+ }
	186	+ $code = (int) wp_remote_retrieve_response_code( $response );
	187	+ if ( $code >= 400 ) {
	188	+ $failures[] = array( 'device' => $device, 'error' => self::failure_detail( $code, $args['user-agent'] ) );
	189	+ if ( self::is_firewall_block( $code ) ) {
	190	+ self::remember_firewall_block( $url, $code, $args['user-agent'] );
	191	+ }
	192	+ continue;
	193	+ }
	194	+ if ( 'desktop' === $device ) {
	195	+ $body = (string) wp_remote_retrieve_body( $response );
	196	+ }
	197	+ }
	198	+ // The notice goes only once every copy got through. A host that
	199	+ // blocks just the phone user-agent would otherwise have the notice
	200	+ // cleared by each desktop warm right after the phone warm set it.
	201	+ if ( empty( $failures ) ) {
	202	+ self::clear_firewall_block();
	203	+ }
	204	+ return array( 'failures' => $failures, 'body' => $body );
	205	+ }
	206	+
	207	+ /** "phone: " for a failure of the phone copy, so the log says which copy failed. */
	208	+ private static function device_prefix( string $device ): string {
	209	+ return 'mobile' === $device ? 'phone: ' : '';
	210	+ }
	211	+
	212	+ /**
	213	+ * Is this status code the signature of a firewall refusing our warmer?
	214	+ *
	215	+ * 403 and 406 are what bad-bot rules (7G/8G, mod_security, Wordfence)
	216	+ * answer with. We only ever warm our OWN origin, and a page a visitor can
	217	+ * load must be loadable by us too — so these codes mean the request was
	218	+ * judged by its user-agent, not that the page is missing or broken. (#481)
	219	+ */
	220	+ private static function is_firewall_block( int $code ): bool {
	221	+ return in_array( $code, array( 403, 406 ), true );
	222	+ }
	223	+
	224	+ /**
	225	+ * Explain a warm failure in terms the admin can act on.
	226	+ *
	227	+ * A bare "HTTP 403" sent people hunting a broken page; the page is fine,
	228	+ * and the fix is a server rule, so the message has to name the cause and
	229	+ * the exact UA to allow. (#481)
	230	+ */
	231	+ private static function failure_detail( int $code, string $user_agent = '' ): string {
	232	+ if ( ! self::is_firewall_block( $code ) ) {
	233	+ return sprintf( 'HTTP %d', $code );
	234	+ }
	235	+
	236	+ return sprintf(
	237	+ 'HTTP %d — your server\'s firewall is blocking the xSpeed cache warmer by user-agent, so this page was not warmed. Allow the user-agent "%s" (on xCloud this is the 8G firewall\'s bad-bot rule), or change it with the xspeed_preloader_user_agent filter.',
	238	+ $code,
	239	+ '' !== $user_agent ? $user_agent : self::user_agent()
	240	+ );
	241	+ }
	242	+
	243	+ /** Option holding the last firewall-shaped warm refusal. */
	244	+ public const FIREWALL_BLOCK_OPTION = 'xspeed_preloader_firewall_block';
	245	+
	246	+ /**
	247	+ * Record that the origin refused a warm by user-agent, for ui_notices().
	248	+ *
	249	+ * An option rather than a transient: the condition is a server rule that
	250	+ * persists until someone changes it, and a notice that expired on its own
	251	+ * would let a site go back to never warming, silently. Cleared by
	252	+ * clear_firewall_block() on the first warm that succeeds. (#481)
	253	+ */
	254	+ private static function remember_firewall_block( string $url, int $code, string $user_agent = '' ): void {
	255	+ if ( ! function_exists( 'update_option' ) ) {
	256	+ return;
	257	+ }
	258	+ update_option(
	259	+ self::FIREWALL_BLOCK_OPTION,
	260	+ array(
	261	+ 'url' => $url,
	262	+ 'code' => $code,
	263	+ 'user_agent' => '' !== $user_agent ? $user_agent : self::user_agent(),
	264	+ 'ts' => time(),
	265	+ ),
	266	+ false
	267	+ );
	268	+ }
	269	+
	270	+ /** Forget the firewall block once a warm gets through. */
	271	+ public static function clear_firewall_block(): void {
	272	+ if ( function_exists( 'delete_option' ) && self::firewall_block() ) {
	273	+ delete_option( self::FIREWALL_BLOCK_OPTION );
	274	+ }
	275	+ }
	276	+
	277	+ /** The last firewall-shaped refusal, or null when there isn't one. */
	278	+ public static function firewall_block(): ?array {
	279	+ if ( ! function_exists( 'get_option' ) ) {
	280	+ return null;
	281	+ }
	282	+ $block = get_option( self::FIREWALL_BLOCK_OPTION, null );
	283	+
	284	+ return ( is_array( $block ) && ! empty( $block['code'] ) ) ? $block : null;
	285	+ }
	286	+
	287	+ /**
	288	+ * How many new remote images one warmed page may resolve.
	289	+ */
	290	+ private static function remote_dimension_limit(): int {
	291	+ /**
	292	+ * Filter the per-page cap on remote dimension lookups.
	293	+ *
	294	+ * @param int $limit Default 20. Values below 1 disable the lookup.
	295	+ */
	296	+ return (int) apply_filters( 'xspeed_preloader_remote_dimension_limit', self::REMOTE_DIMENSION_LIMIT );
	297	+ }
	298	+
	299	+ /**
	300	+ * Why the top-level sitemap fetch failed on this request, or '' when it
	301	+ * succeeded. Set by fetch_sitemap_urls(), read by resolve_queue() — the
	302	+ * reason has to survive the return of an empty array, which is exactly
	303	+ * what it could not do before. Request-scoped; never persisted. (#142)
	304	+ *
	305	+ * @var string
	306	+ */
	307	+ private static $last_sitemap_error = '';
	308	+
	309	+ /**
	310	+ * The sitemap URL the last error refers to. Kept beside the message so
	311	+ * an error entry can carry a real `url` field like every other one,
	312	+ * rather than repeating the URL already inside the message text.
	313	+ */
	314	+ private static $last_sitemap_url = '';
	315	+
	316	+ /**
	317	+ * How the queue for the current crawl was built — 'sitemap', 'fallback'
	318	+ * (enumerated from the database because the sitemap was unreachable), or
	319	+ * 'none'. Surfaced in the state so the panel, REST and CLI can each say
	320	+ * what actually happened instead of reporting a bare zero. (#142)
	321	+ *
	322	+ * @var string
	323	+ */
	324	+ private static $queue_source = 'none';
	325	+
	326	+ /** Largest number of URLs the database fallback will enumerate. */
	327	+ private const FALLBACK_LIMIT = 500;
	328	+
	329	+ /**
43	330	* Kick off a fresh crawl. Returns the initial state.
44	331	*/
45	332	public static function start(): array {
46	333	$opts = Settings_Manager::get( 'preloader' );
		@@ -45,25 +332,77 @@
45	332	public static function start(): array {
46	333	$opts = Settings_Manager::get( 'preloader' );
47	334	$urls = self::resolve_queue( $opts );
48	335
	336	+ // A crawl that queued nothing because the sitemap was unreachable is a
	337	+ // FAILURE, and every layer above needs to be able to say so. It used
	338	+ // to be indistinguishable from success: errors stayed empty, the REST
	339	+ // route returned 200, and the CLI printed a green Success. (#142)
	340	+ $sitemap_error = self::$last_sitemap_error;
	341	+ $errors = array();
	342	+ if ( '' !== $sitemap_error && empty( $urls ) ) {
	343	+ // Same {url, error, ts} shape every other entry uses. A bare
	344	+ // string here fataled `wp xspeed preloader status`, which
	345	+ // destructures `$e['url']` over the list — and took the MCP
	346	+ // `get_preloader_status` tool down with it, so an agent asking
	347	+ // why the preload failed got "Cannot access offset of type
	348	+ // string on string" instead of the reason this code records.
	349	+ // `url` is the sitemap because that is what failed. (QA F1)
	350	+ $errors[] = array(
	351	+ 'url' => self::$last_sitemap_url,
	352	+ 'error' => $sitemap_error,
	353	+ 'ts' => time(),
	354	+ );
	355	+ }
	356	+
49	357	$state = array(
50		- 'running' => ! empty( $urls ),
51		- 'started_at' => time(),
52		- 'finished_at' => 0,
53		- 'queue' => array_values( $urls ),
54		- 'processed' => 0,
55		- 'total' => count( $urls ),
56		- 'last_url' => '',
57		- 'errors' => array(),
	358	+ 'running' => ! empty( $urls ),
	359	+ 'started_at' => time(),
	360	+ 'finished_at' => empty( $urls ) ? time() : 0,
	361	+ 'queue' => array_values( $urls ),
	362	+ 'processed' => 0,
	363	+ 'total' => count( $urls ),
	364	+ 'last_url' => '',
	365	+ 'errors' => $errors,
	366	+ // Consumers render on these: the panel needs to distinguish
	367	+ // "not started" from "ran and found nothing", and to tell the
	368	+ // user when the queue came from the fallback rather than the
	369	+ // sitemap they configured.
	370	+ 'source' => self::$queue_source,
	371	+ 'sitemap_error' => $sitemap_error,
	372	+ // Which copies each URL gets, for the panel and the status
	373	+ // command. Each tick refreshes it: turning Separate Mobile
	374	+ // Cache on or off mid-crawl purges the cache, and the rest of
	375	+ // the crawl should fill the copies that now exist.
	376	+ 'devices' => self::devices(),
58	377	);
59	378	set_transient( self::STATE_KEY, $state, self::STATE_TTL );
60	379
61		- Activity_Log::record(
62		- 'preloader_started',
63		- sprintf( 'Preloader queued %d URL%s for warming.', $state['total'], 1 === $state['total'] ? '' : 's' ),
64		- $state['total'] > 0 ? Activity_Log::INFO : Activity_Log::WARN
65		- );
	380	+ if ( '' !== $sitemap_error && 'fallback' === self::$queue_source ) {
	381	+ $message = sprintf(
	382	+ /* translators: 1: number of URLs, 2: the sitemap failure detail. */
	383	+ __( 'Preloader queued %1$d URLs from the site content — %2$s', 'xspeed' ),
	384	+ $state['total'],
	385	+ $sitemap_error
	386	+ );
	387	+ $severity = Activity_Log::WARN;
	388	+ } elseif ( '' !== $sitemap_error ) {
	389	+ $message = sprintf(
	390	+ /* translators: %s: the sitemap failure detail. */
	391	+ __( 'Preloader could not start — %s', 'xspeed' ),
	392	+ $sitemap_error
	393	+ );
	394	+ $severity = Activity_Log::WARN;
	395	+ } else {
	396	+ $message = sprintf(
	397	+ /* translators: 1: number of URLs, 2: plural suffix. */
	398	+ __( 'Preloader queued %1$d URL%2$s for warming.', 'xspeed' ),
	399	+ $state['total'],
	400	+ 1 === $state['total'] ? '' : 's'
	401	+ );
	402	+ $severity = $state['total'] > 0 ? Activity_Log::INFO : Activity_Log::WARN;
	403	+ }
	404	+ Activity_Log::record( 'preloader_started', $message, $severity );
66	405
67	406	// Schedule the first tick ~5 seconds out so the kick-off REST call
68	407	// returns instantly; wp_schedule_single_event covers the
69	408	// "process the queue ASAP" path without a heavy synchronous loop.
		@@ -134,15 +473,41 @@
134	473
135	474	$opts = Settings_Manager::get( 'preloader' );
136	475	$batch = max( 1, min( 50, (int) ( $opts['batch_size'] ?? 5 ) ) );
137	476
138		- $processed_this_tick = 0;
139		- while ( $processed_this_tick < $batch && ! empty( $state['queue'] ) ) {
140		- $url = array_shift( $state['queue'] );
	477	+ /**
	478	+ * Filter: xspeed_preload_batch_size
	479	+ *
	480	+ * How much one tick may warm, in the units the batch size setting
	481	+ * counts. The setting is the site owner's
	482	+ * intent; this is for anything that knows a lower ceiling applies —
	483	+ * a CDN with a rate limit in front, say, which will answer a burst
	484	+ * with a challenge and leave the queue looking warmed when it is not.
	485	+ *
	486	+ * Only ever lowers. A filter that raised it would let an add-on
	487	+ * overrule a number the site owner chose, and the reason to reach for
	488	+ * this is always that something cannot take the current rate.
	489	+ *
	490	+ * @param int $batch The batch this tick would otherwise use.
	491	+ */
	492	+ $ceiling = (int) apply_filters( 'xspeed_preload_batch_size', $batch );
	493	+ if ( $ceiling > 0 && $ceiling < $batch ) {
	494	+ $batch = $ceiling;
	495	+ }
	496	+
	497	+ // batch_size caps requests, not URLs, so the load on the origin per
	498	+ // tick is the same with the phone copy on: each URL costs one request
	499	+ // per device. A tick always warms at least one URL.
	500	+ $devices = self::devices();
	501	+ $state['devices'] = $devices;
	502	+ $per_url = count( $devices );
	503	+ $requests = 0;
	504	+ while ( ! empty( $state['queue'] ) && ( 0 === $requests || $requests + $per_url <= $batch ) ) {
	505	+ $url = (string) array_shift( $state['queue'] );
141	506	self::warm_url( $url, $state );
142	507	$state['processed']++;
143	508	$state['last_url'] = $url;
144		- $processed_this_tick++;
	509	+ $requests += $per_url;
145	510	}
146	511
147	512	if ( empty( $state['queue'] ) ) {
148	513	self::mark_complete( $state );
		@@ -168,32 +533,17 @@
168	533	public static function warm_one( string $url, string $cause = 'manual' ): bool {
169	534	if ( '' === $url ) {
170	535	return false;
171	536	}
172		- $response = wp_remote_get(
173		- $url,
174		- array(
175		- 'timeout' => self::REQUEST_TIMEOUT,
176		- 'sslverify' => false,
177		- 'user-agent' => self::USER_AGENT,
178		- 'blocking' => true,
179		- )
180		- );
181		- if ( is_wp_error( $response ) ) {
	537	+ $result = self::warm_devices( $url );
	538	+ foreach ( $result['failures'] as $failure ) {
182	539	Activity_Log::record(
183	540	'preloader_warm_failed',
184		- sprintf( 'Warm %s failed (%s): %s', $cause, $url, $response->get_error_message() ),
	541	+ sprintf( 'Warm %s failed (%s): %s%s', $cause, $url, self::device_prefix( $failure['device'] ), $failure['error'] ),
185	542	Activity_Log::WARN
186	543	);
187		- return false;
188	544	}
189		- $code = (int) wp_remote_retrieve_response_code( $response );
190		- if ( $code >= 400 ) {
191		- Activity_Log::record(
192		- 'preloader_warm_failed',
193		- sprintf( 'Warm %s failed (%s): HTTP %d', $cause, $url, $code ),
194		- Activity_Log::WARN
195		- );
	545	+ if ( ! empty( $result['failures'] ) ) {
196	546	return false;
197	547	}
198	548	Activity_Log::record(
199	549	'preloader_warmed_one',
		@@ -203,40 +553,110 @@
203	553	return true;
204	554	}
205	555
206	556	private static function warm_url( string $url, array &$state ): void {
207		- $response = wp_remote_get(
208		- $url,
209		- array(
210		- 'timeout' => self::REQUEST_TIMEOUT,
211		- 'sslverify' => false,
212		- 'user-agent' => self::USER_AGENT,
213		- 'headers' => array(
214		- 'Accept' => 'text/html,application/xhtml+xml',
215		- ),
216		- 'blocking' => true,
217		- )
218		- );
219		- if ( is_wp_error( $response ) ) {
	557	+ $result = self::warm_devices( $url, array( 'Accept' => 'text/html,application/xhtml+xml' ) );
	558	+ foreach ( $result['failures'] as $failure ) {
220	559	$state['errors'][] = array(
221	560	'url' => $url,
222		- 'error' => $response->get_error_message(),
	561	+ 'error' => self::device_prefix( $failure['device'] ) . $failure['error'],
223	562	'ts' => time(),
224	563	);
225		- // Cap retained errors so a broken sitemap doesn't blow the
226		- // transient size.
227		- $state['errors'] = array_slice( $state['errors'], -20 );
	564	+ }
	565	+ // Cap retained errors so a broken sitemap doesn't blow the
	566	+ // transient size.
	567	+ $state['errors'] = array_slice( $state['errors'], -20 );
	568	+ if ( '' !== $result['body'] ) {
	569	+ self::warm_remote_dimensions( $result['body'] );
	570	+ }
	571	+ }
	572	+
	573	+ /**
	574	+ * Resolve dimensions for externally hosted images found on a warmed page.
	575	+ *
	576	+ * The crawl already has the HTML in hand, so harvesting image URLs from it
	577	+ * costs nothing extra — and this is the one place where paying for a
	578	+ * remote lookup is free of consequence, because no visitor is waiting.
	579	+ *
	580	+ * An image on another domain has no local file to measure, so the front
	581	+ * end skips it and the page ships without width/height — which is layout
	582	+ * shift, on precisely the sites least able to fix it by hand (a CDN, a
	583	+ * sister site, a shared asset host). Warming here means the NEXT render
	584	+ * finds the dimensions in cache and stamps them, with the visitor paying
	585	+ * nothing.
	586	+ *
	587	+ * Deliberately bounded per page: a crawl should not turn into a scraper
	588	+ * for a page embedding hundreds of third-party images.
	589	+ *
	590	+ * @param string $html The warmed page's HTML.
	591	+ */
	592	+ private static function warm_remote_dimensions( string $html ): void {
	593	+ if ( '' === $html || ! class_exists( '\XSpeed\Lazy_Loader' ) ) {
228	594	return;
229	595	}
230		- $code = (int) wp_remote_retrieve_response_code( $response );
231		- if ( $code >= 400 ) {
232		- $state['errors'][] = array(
233		- 'url' => $url,
234		- 'error' => sprintf( 'HTTP %d', $code ),
235		- 'ts' => time(),
236		- );
237		- $state['errors'] = array_slice( $state['errors'], -20 );
	596	+
	597	+ $opts = Settings_Manager::get( 'lazy' );
	598	+ if ( empty( $opts['add_missing_dimensions'] ) ) {
	599	+ return;
238	600	}
	601	+
	602	+ // Match any <img>, not only one carrying `src`. The URL worth warming
	603	+ // may live in a lazy attribute instead — which is the whole point of
	604	+ // #328 — and resolvable_image_url() below is what knows where to look.
	605	+ if ( ! preg_match_all( '#<img\b[^>]*>#i', $html, $m, PREG_SET_ORDER ) ) {
	606	+ return;
	607	+ }
	608	+
	609	+ $home = wp_parse_url( home_url(), PHP_URL_HOST );
	610	+ $targets = array();
	611	+ foreach ( $m as $tag ) {
	612	+ // Only tags MISSING a dimension are worth resolving — one that
	613	+ // already declares both needs nothing.
	614	+ // Same lookbehind as Lazy_Loader::ensure_dimensions(): a bare
	615	+ // `\bwidth=` also matches `data-width=`, so a slider carrying its
	616	+ // own metadata looked already-sized and was skipped from warming.
	617	+ // The two must agree, or the collector skips exactly the tags the
	618	+ // renderer still needs measured. (#333 review round 3, issue 2)
	619	+ if ( preg_match( '#(?<![-\w])width\s*=#i', $tag[0] ) && preg_match( '#(?<![-\w])height\s*=#i', $tag[0] ) ) {
	620	+ continue;
	621	+ }
	622	+ // Ask the same resolver the render path uses, rather than reading
	623	+ // `src` directly. A slider parks a spacer in `src` and the real
	624	+ // URL in `data-lazy`/`data-src`/`data-original`, so a collector
	625	+ // looking only at `src` warmed the SPACER and never the image —
	626	+ // leaving remotely-hosted slider images unresolvable at render
	627	+ // time, the exact markup #328 is about. (#333 review round 2,
	628	+ // issue 3)
	629	+ // `false`: do not let the resolver settle a name-refused URL by
	630	+ // MEASURING it. That is circular here — remote measurement is
	631	+ // gated until warm_dimensions() sets $warming, and this collector
	632	+ // is what feeds warm_dimensions(). Take the URL the tag offers and
	633	+ // let the warm pass decide. (#333 review round 3, issue 3)
	634	+ $src = Lazy_Loader::resolvable_image_url( $tag[0], false );
	635	+ if ( '' === $src || ! preg_match( '#^https?://#i', $src ) ) {
	636	+ continue;
	637	+ }
	638	+ $host = wp_parse_url( $src, PHP_URL_HOST );
	639	+ if ( ! $host || $host === $home ) {
	640	+ continue; // Local images already resolve from disk.
	641	+ }
	642	+ // Already resolved (or already known unresolvable) — looking it up
	643	+ // again costs a request and teaches us nothing. Skipping it is
	644	+ // also what makes the cap below advance: images appear in the same
	645	+ // DOM order every crawl, so a collector that did not skip would
	646	+ // re-pick the same first N for ever and never reach the rest.
	647	+ if ( Lazy_Loader::dimensions_known( $src ) ) {
	648	+ continue;
	649	+ }
	650	+ $targets[ $src ] = true;
	651	+ if ( count( $targets ) >= self::remote_dimension_limit() ) {
	652	+ break;
	653	+ }
	654	+ }
	655	+
	656	+ if ( $targets ) {
	657	+ Lazy_Loader::warm_dimensions( array_keys( $targets ) );
	658	+ }
239	659	}
240	660
241	661	private static function mark_complete( array $state ): void {
242	662	$state['running'] = false;
		@@ -263,8 +683,14 @@
263	683	*
264	684	* @return string[]
265	685	*/
266	686	private static function resolve_queue( array $opts ): array {
	687	+ // Request-scoped statics: reset so a previous crawl in the same
	688	+ // process can't leak its verdict into this one.
	689	+ self::$last_sitemap_error = '';
	690	+ self::$last_sitemap_url = '';
	691	+ self::$queue_source = 'none';
	692	+
267	693	$sitemap = trim( (string) ( $opts['sitemap_url'] ?? '' ) );
268	694	if ( '' === $sitemap ) {
269	695	$sitemap = home_url( '/wp-sitemap.xml' );
270	696	}
		@@ -269,9 +695,28 @@
269	695	$sitemap = home_url( '/wp-sitemap.xml' );
270	696	}
271	697
272	698	$urls = self::fetch_sitemap_urls( $sitemap, 0 );
	699	+ $from = empty( $urls ) ? 'none' : 'sitemap';
273	700
	701	+ /*
	702	+ * A missing sitemap must not disable the feature. Two very common
	703	+ * setups produce one with no misconfiguration by the user:
	704	+ * `blog_public = 0` (WordPress disables /wp-sitemap.xml outright,
	705	+ * standard on staging and pre-launch sites), and an SEO plugin
	706	+ * filtering `wp_sitemaps_enabled` to false while serving its own
	707	+ * sitemap at a path we were never told about.
	708	+ *
	709	+ * Enumerate warmable URLs straight from the database instead. Only
	710	+ * on a genuine fetch FAILURE — a sitemap that is reachable and
	711	+ * legitimately empty is a real answer, and silently crawling
	712	+ * something else would be worse than doing nothing. (#142)
	713	+ */
	714	+ if ( empty( $urls ) && '' !== self::$last_sitemap_error ) {
	715	+ $urls = self::fallback_urls();
	716	+ $from = empty( $urls ) ? 'none' : 'fallback';
	717	+ }
	718	+
274	719	$cache_opts = Settings_Manager::get( 'cache' );
275	720	$excluded = is_array( $cache_opts['excluded_urls'] ?? null ) ? $cache_opts['excluded_urls'] : array();
276	721	if ( ! empty( $excluded ) ) {
277	722	$urls = array_filter(
		@@ -289,12 +734,90 @@
289	734	}
290	735
291	736	// Dedup + cap at 5000 to bound the transient size on huge sites.
292	737	$urls = array_values( array_unique( $urls ) );
293		- return array_slice( $urls, 0, 5000 );
	738	+ $urls = array_slice( $urls, 0, 5000 );
	739	+
	740	+ /*
	741	+ * Commit the verdict only now, AFTER the exclusion filter — the
	742	+ * source describes what we ACTUALLY queued, not what we hoped to.
	743	+ * Setting it earlier let a queue that the exclusions stripped to
	744	+ * nothing still claim `fallback`, so the panel announced "Warmed
	745	+ * from site content" over 0 URLs, and `crawlFailed` (which needs
	746	+ * source !== 'fallback' at total 0) could never become true.
	747	+ * One assignment fixes both. (QA F2/F3 on #155)
	748	+ */
	749	+ self::$queue_source = empty( $urls ) ? 'none' : $from;
	750	+
	751	+ return $urls;
294	752	}
295	753
296	754	/**
	755	+ * Enumerate warmable URLs from the database, for sites whose sitemap
	756	+ * can't be fetched. Deliberately modest in scope: the home page, then
	757	+ * the most recently modified public posts across every public post type.
	758	+ * Newest-first is the right bias — those are the URLs most likely to be
	759	+ * requested and least likely to be warm already.
	760	+ *
	761	+ * Uses WP_Query rather than SQL so post-type registration, status
	762	+ * handling and multisite switching all behave the way the rest of
	763	+ * WordPress does.
	764	+ *
	765	+ * @return string[]
	766	+ */
	767	+ private static function fallback_urls(): array {
	768	+ $urls = array();
	769	+ $home = (string) home_url( '/' );
	770	+ if ( '' !== trim( $home, '/' ) ) {
	771	+ $urls[] = $home;
	772	+ }
	773	+
	774	+ $types = get_post_types(
	775	+ array(
	776	+ 'public' => true,
	777	+ 'publicly_queryable' => true,
	778	+ )
	779	+ );
	780	+ // `page` is public but not publicly_queryable, so the query above
	781	+ // misses it — and pages are exactly what a warm cache wants most.
	782	+ // Only add it when the site has post types at all: an empty list
	783	+ // means there is nothing to enumerate, and constructing a WP_Query
	784	+ // for it would be wasted work.
	785	+ if ( ! empty( $types ) ) {
	786	+ $types['page'] = 'page';
	787	+ unset( $types['attachment'] );
	788	+ }
	789	+
	790	+ if ( empty( $types ) || ! class_exists( '\WP_Query' ) ) {
	791	+ return $urls;
	792	+ }
	793	+
	794	+ $query = new \WP_Query(
	795	+ array(
	796	+ 'post_type' => array_values( $types ),
	797	+ 'post_status' => 'publish',
	798	+ 'posts_per_page' => self::FALLBACK_LIMIT,
	799	+ 'orderby' => 'modified',
	800	+ 'order' => 'DESC',
	801	+ 'ignore_sticky_posts' => true,
	802	+ 'no_found_rows' => true,
	803	+ 'update_post_meta_cache' => false,
	804	+ 'update_post_term_cache' => false,
	805	+ 'fields' => 'ids',
	806	+ )
	807	+ );
	808	+
	809	+ foreach ( $query->posts as $post_id ) {
	810	+ $permalink = get_permalink( (int) $post_id );
	811	+ if ( is_string( $permalink ) && '' !== $permalink ) {
	812	+ $urls[] = $permalink;
	813	+ }
	814	+ }
	815	+
	816	+ return array_values( array_unique( $urls ) );
	817	+ }
	818	+
	819	+ /**
297	820	* Recursive sitemap parser. Depth-limited to 3 so a maliciously
298	821	* deep index can't stack-overflow.
299	822	*/
300	823	private static function fetch_sitemap_urls( string $sitemap_url, int $depth ): array {
		@@ -305,12 +828,40 @@
305	828	$sitemap_url,
306	829	array(
307	830	'timeout' => self::REQUEST_TIMEOUT,
308	831	'sslverify' => false,
309		- 'user-agent' => self::USER_AGENT,
	832	+ 'user-agent' => self::user_agent(),
	833	+ 'headers' => Self_Traffic::headers(),
310	834	)
311	835	);
312		- if ( is_wp_error( $res ) || (int) wp_remote_retrieve_response_code( $res ) >= 400 ) {
	836	+ if ( is_wp_error( $res ) ) {
	837	+ // Record WHY, don't just vanish. "Unreachable" and "valid but
	838	+ // empty" both used to collapse into an empty array here, which is
	839	+ // what made a sitemap-less site look like a successful crawl of
	840	+ // zero URLs. Only the top-level fetch is recorded: a nested index
	841	+ // failing is a partial result, not a dead crawl. (#142)
	842	+ if ( 0 === $depth ) {
	843	+ self::$last_sitemap_error = sprintf(
	844	+ /* translators: 1: sitemap URL, 2: error detail. */
	845	+ __( 'Could not fetch the sitemap at %1$s — %2$s', 'xspeed' ),
	846	+ $sitemap_url,
	847	+ $res->get_error_message()
	848	+ );
	849	+ self::$last_sitemap_url = (string) $sitemap_url;
	850	+ }
	851	+ return array();
	852	+ }
	853	+ $code = (int) wp_remote_retrieve_response_code( $res );
	854	+ if ( $code >= 400 ) {
	855	+ if ( 0 === $depth ) {
	856	+ self::$last_sitemap_error = sprintf(
	857	+ /* translators: 1: sitemap URL, 2: HTTP status code. */
	858	+ __( 'Could not fetch the sitemap at %1$s — the server returned HTTP %2$d.', 'xspeed' ),
	859	+ $sitemap_url,
	860	+ $code
	861	+ );
	862	+ self::$last_sitemap_url = (string) $sitemap_url;
	863	+ }
313	864	return array();
314	865	}
315	866	$body = (string) wp_remote_retrieve_body( $res );
316	867	if ( '' === $body ) {