← All changes | includes/class-preloader.php +246 -6 1.1.5 → 1.3.6 View file →

		@@ -35,12 +35,151 @@
35	35
36	36	public const STATE_KEY = 'xspeed_preloader_state';
37	37	public const STATE_TTL = 86400; // 24h — long enough for slow crawls.
38	38	public const CRON_HOOK = 'xspeed_preloader_tick';
39		- public const USER_AGENT = 'xSpeed-Preloader/1.0 (+cache warmer; admin-initiated)';
	39	+ /**
	40	+ * User-agent for every request the preloader makes.
	41	+ *
	42	+ * Deliberately contains no substring from the 7G/8G bad-bot lists. The
	43	+ * previous value, "xSpeed-Preloader/1.0", matched the `loader` token in
	44	+ * the alphabetical slice `(linkscan|linkwalker|loader|lwp-download|...)`
	45	+ * — a match on "Pre*loader*" — so nginx ports of 8G answered every warm
	46	+ * with 403 and newly published posts were never warmed. Upstream 8G
	47	+ * v1.5 has since dropped `loader`, but forks and vendored copies (xCloud
	48	+ * among them) still ship the older slice, so the name has to stay clear
	49	+ * of it. "Warmer" matches nothing in either list. (#481)
	50	+ *
	51	+ * Read through user_agent() rather than using this constant directly, so
	52	+ * the `xspeed_preloader_user_agent` filter applies.
	53	+ */
	54	+ public const USER_AGENT = 'xSpeed-Warmer/1.0 (+cache warmer; admin-initiated)';
40	55	public const REQUEST_TIMEOUT = 8;
41	56
42	57	/**
	58	+ * Default cap on NEW remote images resolved per warmed page.
	59	+ *
	60	+ * A crawl warms the cache; it is not a licence to hit third-party hosts
	61	+ * hundreds of times for one page.
	62	+ *
	63	+ * The cap counts only images whose dimensions are not already known, and
	64	+ * results persist between runs — so each crawl advances through a heavily
	65	+ * embedded page rather than re-picking the same first N. That is what
	66	+ * makes a cap safe here: without the skip it would strand everything past
	67	+ * the limit permanently, because images appear in the same DOM order
	68	+ * every time.
	69	+ *
	70	+ * 20 is a starting point, not a measurement. Sites that embed more can
	71	+ * raise it via `xspeed_preloader_remote_dimension_limit`.
	72	+ */
	73	+ private const REMOTE_DIMENSION_LIMIT = 20;
	74	+
	75	+ /**
	76	+ * The user-agent every preloader request sends.
	77	+ *
	78	+ * Filterable because the blocking rule lives on the server, not here: a
	79	+ * host with its own bad-bot list can clear a warm without patching the
	80	+ * plugin or waiting for a release. An empty filter return is ignored —
	81	+ * sending no UA gets a request blocked at least as often. (#481)
	82	+ */
	83	+ public static function user_agent(): string {
	84	+ /**
	85	+ * Filter the preloader's user-agent string.
	86	+ *
	87	+ * @param string $user_agent Default self::USER_AGENT.
	88	+ */
	89	+ $ua = apply_filters( 'xspeed_preloader_user_agent', self::USER_AGENT );
	90	+
	91	+ return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : self::USER_AGENT;
	92	+ }
	93	+
	94	+ /**
	95	+ * Is this status code the signature of a firewall refusing our warmer?
	96	+ *
	97	+ * 403 and 406 are what bad-bot rules (7G/8G, mod_security, Wordfence)
	98	+ * answer with. We only ever warm our OWN origin, and a page a visitor can
	99	+ * load must be loadable by us too — so these codes mean the request was
	100	+ * judged by its user-agent, not that the page is missing or broken. (#481)
	101	+ */
	102	+ private static function is_firewall_block( int $code ): bool {
	103	+ return in_array( $code, array( 403, 406 ), true );
	104	+ }
	105	+
	106	+ /**
	107	+ * Explain a warm failure in terms the admin can act on.
	108	+ *
	109	+ * A bare "HTTP 403" sent people hunting a broken page; the page is fine,
	110	+ * and the fix is a server rule, so the message has to name the cause and
	111	+ * the exact UA to allow. (#481)
	112	+ */
	113	+ private static function failure_detail( int $code ): string {
	114	+ if ( ! self::is_firewall_block( $code ) ) {
	115	+ return sprintf( 'HTTP %d', $code );
	116	+ }
	117	+
	118	+ return sprintf(
	119	+ 'HTTP %d — your server\'s firewall is blocking the xSpeed cache warmer by user-agent, so this page was not warmed. Allow the user-agent "%s" (on xCloud this is the 8G firewall\'s bad-bot rule), or change it with the xspeed_preloader_user_agent filter.',
	120	+ $code,
	121	+ self::user_agent()
	122	+ );
	123	+ }
	124	+
	125	+ /** Option holding the last firewall-shaped warm refusal. */
	126	+ public const FIREWALL_BLOCK_OPTION = 'xspeed_preloader_firewall_block';
	127	+
	128	+ /**
	129	+ * Record that the origin refused a warm by user-agent, for ui_notices().
	130	+ *
	131	+ * An option rather than a transient: the condition is a server rule that
	132	+ * persists until someone changes it, and a notice that expired on its own
	133	+ * would let a site go back to never warming, silently. Cleared by
	134	+ * clear_firewall_block() on the first warm that succeeds. (#481)
	135	+ */
	136	+ private static function remember_firewall_block( string $url, int $code ): void {
	137	+ if ( ! function_exists( 'update_option' ) ) {
	138	+ return;
	139	+ }
	140	+ update_option(
	141	+ self::FIREWALL_BLOCK_OPTION,
	142	+ array(
	143	+ 'url' => $url,
	144	+ 'code' => $code,
	145	+ 'user_agent' => self::user_agent(),
	146	+ 'ts' => time(),
	147	+ ),
	148	+ false
	149	+ );
	150	+ }
	151	+
	152	+ /** Forget the firewall block once a warm gets through. */
	153	+ public static function clear_firewall_block(): void {
	154	+ if ( function_exists( 'delete_option' ) && self::firewall_block() ) {
	155	+ delete_option( self::FIREWALL_BLOCK_OPTION );
	156	+ }
	157	+ }
	158	+
	159	+ /** The last firewall-shaped refusal, or null when there isn't one. */
	160	+ public static function firewall_block(): ?array {
	161	+ if ( ! function_exists( 'get_option' ) ) {
	162	+ return null;
	163	+ }
	164	+ $block = get_option( self::FIREWALL_BLOCK_OPTION, null );
	165	+
	166	+ return ( is_array( $block ) && ! empty( $block['code'] ) ) ? $block : null;
	167	+ }
	168	+
	169	+ /**
	170	+ * How many new remote images one warmed page may resolve.
	171	+ */
	172	+ private static function remote_dimension_limit(): int {
	173	+ /**
	174	+ * Filter the per-page cap on remote dimension lookups.
	175	+ *
	176	+ * @param int $limit Default 20. Values below 1 disable the lookup.
	177	+ */
	178	+ return (int) apply_filters( 'xspeed_preloader_remote_dimension_limit', self::REMOTE_DIMENSION_LIMIT );
	179	+ }
	180	+
	181	+ /**
43	182	* Why the top-level sitemap fetch failed on this request, or '' when it
44	183	* succeeded. Set by fetch_sitemap_urls(), read by resolve_queue() — the
45	184	* reason has to survive the return of an empty array, which is exactly
46	185	* what it could not do before. Request-scoped; never persisted. (#142)
		@@ -250,9 +389,9 @@
250	389	$url,
251	390	array(
252	391	'timeout' => self::REQUEST_TIMEOUT,
253	392	'sslverify' => false,
254		- 'user-agent' => self::USER_AGENT,
	393	+ 'user-agent' => self::user_agent(),
255	394	'blocking' => true,
256	395	)
257	396	);
258	397	if ( is_wp_error( $response ) ) {
		@@ -266,13 +405,19 @@
266	405	$code = (int) wp_remote_retrieve_response_code( $response );
267	406	if ( $code >= 400 ) {
268	407	Activity_Log::record(
269	408	'preloader_warm_failed',
270		- sprintf( 'Warm %s failed (%s): HTTP %d', $cause, $url, $code ),
	409	+ sprintf( 'Warm %s failed (%s): %s', $cause, $url, self::failure_detail( $code ) ),
271	410	Activity_Log::WARN
272	411	);
	412	+ if ( self::is_firewall_block( $code ) ) {
	413	+ self::remember_firewall_block( $url, $code );
	414	+ }
273	415	return false;
274	416	}
	417	+ // A warm that got through proves the firewall is no longer refusing us,
	418	+ // so the notice must go — otherwise it outlives the problem. (#481)
	419	+ self::clear_firewall_block();
275	420	Activity_Log::record(
276	421	'preloader_warmed_one',
277	422	sprintf( 'Warmed %s (%s)', $url, $cause ),
278	423	Activity_Log::INFO
		@@ -285,9 +430,9 @@
285	430	$url,
286	431	array(
287	432	'timeout' => self::REQUEST_TIMEOUT,
288	433	'sslverify' => false,
289		- 'user-agent' => self::USER_AGENT,
	434	+ 'user-agent' => self::user_agent(),
290	435	'headers' => array(
291	436	'Accept' => 'text/html,application/xhtml+xml',
292	437	),
293	438	'blocking' => true,
		@@ -307,15 +452,110 @@
307	452	$code = (int) wp_remote_retrieve_response_code( $response );
308	453	if ( $code >= 400 ) {
309	454	$state['errors'][] = array(
310	455	'url' => $url,
311		- 'error' => sprintf( 'HTTP %d', $code ),
	456	+ 'error' => self::failure_detail( $code ),
312	457	'ts' => time(),
313	458	);
314	459	$state['errors'] = array_slice( $state['errors'], -20 );
	460	+ if ( self::is_firewall_block( $code ) ) {
	461	+ self::remember_firewall_block( $url, $code );
	462	+ }
	463	+ return;
315	464	}
	465	+
	466	+ self::clear_firewall_block();
	467	+ self::warm_remote_dimensions( (string) wp_remote_retrieve_body( $response ) );
316	468	}
317	469
	470	+ /**
	471	+ * Resolve dimensions for externally hosted images found on a warmed page.
	472	+ *
	473	+ * The crawl already has the HTML in hand, so harvesting image URLs from it
	474	+ * costs nothing extra — and this is the one place where paying for a
	475	+ * remote lookup is free of consequence, because no visitor is waiting.
	476	+ *
	477	+ * An image on another domain has no local file to measure, so the front
	478	+ * end skips it and the page ships without width/height — which is layout
	479	+ * shift, on precisely the sites least able to fix it by hand (a CDN, a
	480	+ * sister site, a shared asset host). Warming here means the NEXT render
	481	+ * finds the dimensions in cache and stamps them, with the visitor paying
	482	+ * nothing.
	483	+ *
	484	+ * Deliberately bounded per page: a crawl should not turn into a scraper
	485	+ * for a page embedding hundreds of third-party images.
	486	+ *
	487	+ * @param string $html The warmed page's HTML.
	488	+ */
	489	+ private static function warm_remote_dimensions( string $html ): void {
	490	+ if ( '' === $html || ! class_exists( '\XSpeed\Lazy_Loader' ) ) {
	491	+ return;
	492	+ }
	493	+
	494	+ $opts = Settings_Manager::get( 'lazy' );
	495	+ if ( empty( $opts['add_missing_dimensions'] ) ) {
	496	+ return;
	497	+ }
	498	+
	499	+ // Match any <img>, not only one carrying `src`. The URL worth warming
	500	+ // may live in a lazy attribute instead — which is the whole point of
	501	+ // #328 — and resolvable_image_url() below is what knows where to look.
	502	+ if ( ! preg_match_all( '#<img\b[^>]*>#i', $html, $m, PREG_SET_ORDER ) ) {
	503	+ return;
	504	+ }
	505	+
	506	+ $home = wp_parse_url( home_url(), PHP_URL_HOST );
	507	+ $targets = array();
	508	+ foreach ( $m as $tag ) {
	509	+ // Only tags MISSING a dimension are worth resolving — one that
	510	+ // already declares both needs nothing.
	511	+ // Same lookbehind as Lazy_Loader::ensure_dimensions(): a bare
	512	+ // `\bwidth=` also matches `data-width=`, so a slider carrying its
	513	+ // own metadata looked already-sized and was skipped from warming.
	514	+ // The two must agree, or the collector skips exactly the tags the
	515	+ // renderer still needs measured. (#333 review round 3, issue 2)
	516	+ if ( preg_match( '#(?<![-\w])width\s*=#i', $tag[0] ) && preg_match( '#(?<![-\w])height\s*=#i', $tag[0] ) ) {
	517	+ continue;
	518	+ }
	519	+ // Ask the same resolver the render path uses, rather than reading
	520	+ // `src` directly. A slider parks a spacer in `src` and the real
	521	+ // URL in `data-lazy`/`data-src`/`data-original`, so a collector
	522	+ // looking only at `src` warmed the SPACER and never the image —
	523	+ // leaving remotely-hosted slider images unresolvable at render
	524	+ // time, the exact markup #328 is about. (#333 review round 2,
	525	+ // issue 3)
	526	+ // `false`: do not let the resolver settle a name-refused URL by
	527	+ // MEASURING it. That is circular here — remote measurement is
	528	+ // gated until warm_dimensions() sets $warming, and this collector
	529	+ // is what feeds warm_dimensions(). Take the URL the tag offers and
	530	+ // let the warm pass decide. (#333 review round 3, issue 3)
	531	+ $src = Lazy_Loader::resolvable_image_url( $tag[0], false );
	532	+ if ( '' === $src || ! preg_match( '#^https?://#i', $src ) ) {
	533	+ continue;
	534	+ }
	535	+ $host = wp_parse_url( $src, PHP_URL_HOST );
	536	+ if ( ! $host || $host === $home ) {
	537	+ continue; // Local images already resolve from disk.
	538	+ }
	539	+ // Already resolved (or already known unresolvable) — looking it up
	540	+ // again costs a request and teaches us nothing. Skipping it is
	541	+ // also what makes the cap below advance: images appear in the same
	542	+ // DOM order every crawl, so a collector that did not skip would
	543	+ // re-pick the same first N for ever and never reach the rest.
	544	+ if ( Lazy_Loader::dimensions_known( $src ) ) {
	545	+ continue;
	546	+ }
	547	+ $targets[ $src ] = true;
	548	+ if ( count( $targets ) >= self::remote_dimension_limit() ) {
	549	+ break;
	550	+ }
	551	+ }
	552	+
	553	+ if ( $targets ) {
	554	+ Lazy_Loader::warm_dimensions( array_keys( $targets ) );
	555	+ }
	556	+ }
	557	+
318	558	private static function mark_complete( array $state ): void {
319	559	$state['running'] = false;
320	560	$state['finished_at'] = time();
321	561	$state['queue'] = array();
		@@ -485,9 +725,9 @@
485	725	$sitemap_url,
486	726	array(
487	727	'timeout' => self::REQUEST_TIMEOUT,
488	728	'sslverify' => false,
489		- 'user-agent' => self::USER_AGENT,
	729	+ 'user-agent' => self::user_agent(),
490	730	)
491	731	);
492	732	if ( is_wp_error( $res ) ) {
493	733	// Record WHY, don't just vanish. "Unreachable" and "valid but