| @@ -35,12 +35,299 @@ | ||
| 35 | 35 | |
| 36 | 36 | public const STATE_KEY = 'xspeed_preloader_state'; |
| 37 | 37 | public const STATE_TTL = 86400; // 24h — long enough for slow crawls. |
| 38 | 38 | public const CRON_HOOK = 'xspeed_preloader_tick'; |
| 39 | - public const USER_AGENT = 'xSpeed-Preloader/1.0 (+cache warmer; admin-initiated)'; | |
| 39 | + /** | |
| 40 | + * User-agent for every request the preloader makes. | |
| 41 | + * | |
| 42 | + * Deliberately contains no substring from the 7G/8G bad-bot lists. The | |
| 43 | + * previous value, "xSpeed-Preloader/1.0", matched the `loader` token in | |
| 44 | + * the alphabetical slice `(linkscan|linkwalker|loader|lwp-download|...)` | |
| 45 | + * — a match on "Pre*loader*" — so nginx ports of 8G answered every warm | |
| 46 | + * with 403 and newly published posts were never warmed. Upstream 8G | |
| 47 | + * v1.5 has since dropped `loader`, but forks and vendored copies (xCloud | |
| 48 | + * among them) still ship the older slice, so the name has to stay clear | |
| 49 | + * of it. "Warmer" matches nothing in either list. (#481) | |
| 50 | + * | |
| 51 | + * Read through user_agent() rather than using this constant directly, so | |
| 52 | + * the `xspeed_preloader_user_agent` filter applies. | |
| 53 | + */ | |
| 54 | + public const USER_AGENT = 'xSpeed-Warmer/1.0 (+cache warmer; admin-initiated)'; | |
| 40 | 55 | public const REQUEST_TIMEOUT = 8; |
| 41 | 56 | |
| 42 | 57 | /** |
| 58 | + * Default cap on NEW remote images resolved per warmed page. | |
| 59 | + * | |
| 60 | + * A crawl warms the cache; it is not a licence to hit third-party hosts | |
| 61 | + * hundreds of times for one page. | |
| 62 | + * | |
| 63 | + * The cap counts only images whose dimensions are not already known, and | |
| 64 | + * results persist between runs — so each crawl advances through a heavily | |
| 65 | + * embedded page rather than re-picking the same first N. That is what | |
| 66 | + * makes a cap safe here: without the skip it would strand everything past | |
| 67 | + * the limit permanently, because images appear in the same DOM order | |
| 68 | + * every time. | |
| 69 | + * | |
| 70 | + * 20 is a starting point, not a measurement. Sites that embed more can | |
| 71 | + * raise it via `xspeed_preloader_remote_dimension_limit`. | |
| 72 | + */ | |
| 73 | + private const REMOTE_DIMENSION_LIMIT = 20; | |
| 74 | + | |
| 75 | + /** | |
| 76 | + * The user-agent every preloader request sends. | |
| 77 | + * | |
| 78 | + * Filterable because the blocking rule lives on the server, not here: a | |
| 79 | + * host with its own bad-bot list can clear a warm without patching the | |
| 80 | + * plugin or waiting for a release. An empty filter return is ignored — | |
| 81 | + * sending no UA gets a request blocked at least as often. (#481) | |
| 82 | + */ | |
| 83 | + public static function user_agent(): string { | |
| 84 | + /** | |
| 85 | + * Filter the preloader's user-agent string. | |
| 86 | + * | |
| 87 | + * @param string $user_agent Default self::USER_AGENT. | |
| 88 | + */ | |
| 89 | + $ua = apply_filters( 'xspeed_preloader_user_agent', self::USER_AGENT ); | |
| 90 | + | |
| 91 | + return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : self::USER_AGENT; | |
| 92 | + } | |
| 93 | + | |
| 94 | + /** | |
| 95 | + * A real phone browser's user-agent, which the warmer's own is appended | |
| 96 | + * to for the phone copy. | |
| 97 | + * | |
| 98 | + * A real phone string rather than the warmer's UA plus a client hint: | |
| 99 | + * a theme or plugin that reads the user-agent itself (Mobile_Detect and | |
| 100 | + * the like) instead of calling wp_is_mobile() would otherwise render its | |
| 101 | + * desktop HTML into the phone copy. The warmer's name stays at the end so | |
| 102 | + * the request is still recognisable in access logs and still matches no | |
| 103 | + * bad-bot rule. (#596) | |
| 104 | + */ | |
| 105 | + public const MOBILE_UA_PREFIX = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1'; | |
| 106 | + | |
| 107 | + /** The user-agent the phone warm sends. */ | |
| 108 | + public static function mobile_user_agent(): string { | |
| 109 | + $default = self::MOBILE_UA_PREFIX . ' ' . self::user_agent(); | |
| 110 | + /** | |
| 111 | + * Filter the user-agent the preloader sends when it warms the phone | |
| 112 | + * copy of a page (Separate Mobile Cache on). | |
| 113 | + * | |
| 114 | + * It must still read as a phone: the cache files the response by the | |
| 115 | + * same test as wp_is_mobile(). An empty return is ignored. | |
| 116 | + * | |
| 117 | + * @param string $user_agent Default: an iPhone Safari string followed by the warmer's user-agent. | |
| 118 | + */ | |
| 119 | + $ua = apply_filters( 'xspeed_preloader_mobile_user_agent', $default ); | |
| 120 | + | |
| 121 | + return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : $default; | |
| 122 | + } | |
| 123 | + | |
| 124 | + /** | |
| 125 | + * The copies of a page a warm fills: the desktop one, and the phone one | |
| 126 | + * too when Separate Mobile Cache keeps one per device. Before #596 every | |
| 127 | + * warm sent a desktop user-agent, so the first phone visitor to each | |
| 128 | + * page got an uncached render. | |
| 129 | + * | |
| 130 | + * @return string[] 'desktop', then 'mobile' when it applies. | |
| 131 | + */ | |
| 132 | + public static function devices(): array { | |
| 133 | + $cache = Settings_Manager::get( 'cache' ); | |
| 134 | + $devices = empty( $cache['mobile_separate'] ) ? array( 'desktop' ) : array( 'desktop', 'mobile' ); | |
| 135 | + /** | |
| 136 | + * Filter which copies of a page the preloader warms. | |
| 137 | + * | |
| 138 | + * Return array( 'desktop' ) to skip the phone copy, for example on a | |
| 139 | + * host where the extra requests cost too much. Unknown values are | |
| 140 | + * dropped; an empty list means desktop. | |
| 141 | + * | |
| 142 | + * @param string[] $devices 'desktop', plus 'mobile' when Separate Mobile Cache is on. | |
| 143 | + */ | |
| 144 | + $filtered = apply_filters( 'xspeed_preloader_devices', $devices ); | |
| 145 | + $filtered = is_array( $filtered ) ? array_values( array_intersect( array( 'desktop', 'mobile' ), $filtered ) ) : $devices; | |
| 146 | + | |
| 147 | + return empty( $filtered ) ? array( 'desktop' ) : $filtered; | |
| 148 | + } | |
| 149 | + | |
| 150 | + /** | |
| 151 | + * Request arguments for one warm of one device copy. | |
| 152 | + * | |
| 153 | + * `Sec-CH-UA-Mobile` settles the device whatever the user-agent filters | |
| 154 | + * return: the cache checks that hint before the user-agent. | |
| 155 | + * | |
| 156 | + * @param array<string,string> $headers Extra request headers. | |
| 157 | + */ | |
| 158 | + private static function warm_args( string $device, array $headers = array() ): array { | |
| 159 | + $headers['Sec-CH-UA-Mobile'] = 'mobile' === $device ? '?1' : '?0'; | |
| 160 | + return array( | |
| 161 | + 'timeout' => self::REQUEST_TIMEOUT, | |
| 162 | + 'sslverify' => false, | |
| 163 | + 'user-agent' => 'mobile' === $device ? self::mobile_user_agent() : self::user_agent(), | |
| 164 | + 'headers' => Self_Traffic::headers( $headers ), | |
| 165 | + 'blocking' => true, | |
| 166 | + ); | |
| 167 | + } | |
| 168 | + | |
| 169 | + /** | |
| 170 | + * Warm every copy of $url that devices() names. | |
| 171 | + * | |
| 172 | + * @param array<string,string> $headers Extra request headers. | |
| 173 | + * @return array{failures: array<int,array{device:string,error:string}>, body: string} | |
| 174 | + * The failures, and the desktop response body ('' when it failed). | |
| 175 | + */ | |
| 176 | + private static function warm_devices( string $url, array $headers = array() ): array { | |
| 177 | + $failures = array(); | |
| 178 | + $body = ''; | |
| 179 | + foreach ( self::devices() as $device ) { | |
| 180 | + $args = self::warm_args( $device, $headers ); | |
| 181 | + $response = wp_remote_get( $url, $args ); | |
| 182 | + if ( is_wp_error( $response ) ) { | |
| 183 | + $failures[] = array( 'device' => $device, 'error' => $response->get_error_message() ); | |
| 184 | + continue; | |
| 185 | + } | |
| 186 | + $code = (int) wp_remote_retrieve_response_code( $response ); | |
| 187 | + if ( $code >= 400 ) { | |
| 188 | + $failures[] = array( 'device' => $device, 'error' => self::failure_detail( $code, $args['user-agent'] ) ); | |
| 189 | + if ( self::is_firewall_block( $code ) ) { | |
| 190 | + self::remember_firewall_block( $url, $code, $args['user-agent'] ); | |
| 191 | + } | |
| 192 | + continue; | |
| 193 | + } | |
| 194 | + if ( 'desktop' === $device ) { | |
| 195 | + $body = (string) wp_remote_retrieve_body( $response ); | |
| 196 | + } | |
| 197 | + } | |
| 198 | + // The notice goes only once every copy got through. A host that | |
| 199 | + // blocks just the phone user-agent would otherwise have the notice | |
| 200 | + // cleared by each desktop warm right after the phone warm set it. | |
| 201 | + if ( empty( $failures ) ) { | |
| 202 | + self::clear_firewall_block(); | |
| 203 | + } | |
| 204 | + return array( 'failures' => $failures, 'body' => $body ); | |
| 205 | + } | |
| 206 | + | |
| 207 | + /** "phone: " for a failure of the phone copy, so the log says which copy failed. */ | |
| 208 | + private static function device_prefix( string $device ): string { | |
| 209 | + return 'mobile' === $device ? 'phone: ' : ''; | |
| 210 | + } | |
| 211 | + | |
| 212 | + /** | |
| 213 | + * Is this status code the signature of a firewall refusing our warmer? | |
| 214 | + * | |
| 215 | + * 403 and 406 are what bad-bot rules (7G/8G, mod_security, Wordfence) | |
| 216 | + * answer with. We only ever warm our OWN origin, and a page a visitor can | |
| 217 | + * load must be loadable by us too — so these codes mean the request was | |
| 218 | + * judged by its user-agent, not that the page is missing or broken. (#481) | |
| 219 | + */ | |
| 220 | + private static function is_firewall_block( int $code ): bool { | |
| 221 | + return in_array( $code, array( 403, 406 ), true ); | |
| 222 | + } | |
| 223 | + | |
| 224 | + /** | |
| 225 | + * Explain a warm failure in terms the admin can act on. | |
| 226 | + * | |
| 227 | + * A bare "HTTP 403" sent people hunting a broken page; the page is fine, | |
| 228 | + * and the fix is a server rule, so the message has to name the cause and | |
| 229 | + * the exact UA to allow. (#481) | |
| 230 | + */ | |
| 231 | + private static function failure_detail( int $code, string $user_agent = '' ): string { | |
| 232 | + if ( ! self::is_firewall_block( $code ) ) { | |
| 233 | + return sprintf( 'HTTP %d', $code ); | |
| 234 | + } | |
| 235 | + | |
| 236 | + return sprintf( | |
| 237 | + 'HTTP %d — your server\'s firewall is blocking the xSpeed cache warmer by user-agent, so this page was not warmed. Allow the user-agent "%s" (on xCloud this is the 8G firewall\'s bad-bot rule), or change it with the xspeed_preloader_user_agent filter.', | |
| 238 | + $code, | |
| 239 | + '' !== $user_agent ? $user_agent : self::user_agent() | |
| 240 | + ); | |
| 241 | + } | |
| 242 | + | |
| 243 | + /** Option holding the last firewall-shaped warm refusal. */ | |
| 244 | + public const FIREWALL_BLOCK_OPTION = 'xspeed_preloader_firewall_block'; | |
| 245 | + | |
| 246 | + /** | |
| 247 | + * Record that the origin refused a warm by user-agent, for ui_notices(). | |
| 248 | + * | |
| 249 | + * An option rather than a transient: the condition is a server rule that | |
| 250 | + * persists until someone changes it, and a notice that expired on its own | |
| 251 | + * would let a site go back to never warming, silently. Cleared by | |
| 252 | + * clear_firewall_block() on the first warm that succeeds. (#481) | |
| 253 | + */ | |
| 254 | + private static function remember_firewall_block( string $url, int $code, string $user_agent = '' ): void { | |
| 255 | + if ( ! function_exists( 'update_option' ) ) { | |
| 256 | + return; | |
| 257 | + } | |
| 258 | + update_option( | |
| 259 | + self::FIREWALL_BLOCK_OPTION, | |
| 260 | + array( | |
| 261 | + 'url' => $url, | |
| 262 | + 'code' => $code, | |
| 263 | + 'user_agent' => '' !== $user_agent ? $user_agent : self::user_agent(), | |
| 264 | + 'ts' => time(), | |
| 265 | + ), | |
| 266 | + false | |
| 267 | + ); | |
| 268 | + } | |
| 269 | + | |
| 270 | + /** Forget the firewall block once a warm gets through. */ | |
| 271 | + public static function clear_firewall_block(): void { | |
| 272 | + if ( function_exists( 'delete_option' ) && self::firewall_block() ) { | |
| 273 | + delete_option( self::FIREWALL_BLOCK_OPTION ); | |
| 274 | + } | |
| 275 | + } | |
| 276 | + | |
| 277 | + /** The last firewall-shaped refusal, or null when there isn't one. */ | |
| 278 | + public static function firewall_block(): ?array { | |
| 279 | + if ( ! function_exists( 'get_option' ) ) { | |
| 280 | + return null; | |
| 281 | + } | |
| 282 | + $block = get_option( self::FIREWALL_BLOCK_OPTION, null ); | |
| 283 | + | |
| 284 | + return ( is_array( $block ) && ! empty( $block['code'] ) ) ? $block : null; | |
| 285 | + } | |
| 286 | + | |
| 287 | + /** | |
| 288 | + * How many new remote images one warmed page may resolve. | |
| 289 | + */ | |
| 290 | + private static function remote_dimension_limit(): int { | |
| 291 | + /** | |
| 292 | + * Filter the per-page cap on remote dimension lookups. | |
| 293 | + * | |
| 294 | + * @param int $limit Default 20. Values below 1 disable the lookup. | |
| 295 | + */ | |
| 296 | + return (int) apply_filters( 'xspeed_preloader_remote_dimension_limit', self::REMOTE_DIMENSION_LIMIT ); | |
| 297 | + } | |
| 298 | + | |
| 299 | + /** | |
| 300 | + * Why the top-level sitemap fetch failed on this request, or '' when it | |
| 301 | + * succeeded. Set by fetch_sitemap_urls(), read by resolve_queue() — the | |
| 302 | + * reason has to survive the return of an empty array, which is exactly | |
| 303 | + * what it could not do before. Request-scoped; never persisted. (#142) | |
| 304 | + * | |
| 305 | + * @var string | |
| 306 | + */ | |
| 307 | + private static $last_sitemap_error = ''; | |
| 308 | + | |
| 309 | + /** | |
| 310 | + * The sitemap URL the last error refers to. Kept beside the message so | |
| 311 | + * an error entry can carry a real `url` field like every other one, | |
| 312 | + * rather than repeating the URL already inside the message text. | |
| 313 | + */ | |
| 314 | + private static $last_sitemap_url = ''; | |
| 315 | + | |
| 316 | + /** | |
| 317 | + * How the queue for the current crawl was built — 'sitemap', 'fallback' | |
| 318 | + * (enumerated from the database because the sitemap was unreachable), or | |
| 319 | + * 'none'. Surfaced in the state so the panel, REST and CLI can each say | |
| 320 | + * what actually happened instead of reporting a bare zero. (#142) | |
| 321 | + * | |
| 322 | + * @var string | |
| 323 | + */ | |
| 324 | + private static $queue_source = 'none'; | |
| 325 | + | |
| 326 | + /** Largest number of URLs the database fallback will enumerate. */ | |
| 327 | + private const FALLBACK_LIMIT = 500; | |
| 328 | + | |
| 329 | + /** | |
| 43 | 330 | * Kick off a fresh crawl. Returns the initial state. |
| 44 | 331 | */ |
| 45 | 332 | public static function start(): array { |
| 46 | 333 | $opts = Settings_Manager::get( 'preloader' ); |
| @@ -45,25 +332,77 @@ | ||
| 45 | 332 | public static function start(): array { |
| 46 | 333 | $opts = Settings_Manager::get( 'preloader' ); |
| 47 | 334 | $urls = self::resolve_queue( $opts ); |
| 48 | 335 | |
| 336 | + // A crawl that queued nothing because the sitemap was unreachable is a | |
| 337 | + // FAILURE, and every layer above needs to be able to say so. It used | |
| 338 | + // to be indistinguishable from success: errors stayed empty, the REST | |
| 339 | + // route returned 200, and the CLI printed a green Success. (#142) | |
| 340 | + $sitemap_error = self::$last_sitemap_error; | |
| 341 | + $errors = array(); | |
| 342 | + if ( '' !== $sitemap_error && empty( $urls ) ) { | |
| 343 | + // Same {url, error, ts} shape every other entry uses. A bare | |
| 344 | + // string here fataled `wp xspeed preloader status`, which | |
| 345 | + // destructures `$e['url']` over the list — and took the MCP | |
| 346 | + // `get_preloader_status` tool down with it, so an agent asking | |
| 347 | + // why the preload failed got "Cannot access offset of type | |
| 348 | + // string on string" instead of the reason this code records. | |
| 349 | + // `url` is the sitemap because that is what failed. (QA F1) | |
| 350 | + $errors[] = array( | |
| 351 | + 'url' => self::$last_sitemap_url, | |
| 352 | + 'error' => $sitemap_error, | |
| 353 | + 'ts' => time(), | |
| 354 | + ); | |
| 355 | + } | |
| 356 | + | |
| 49 | 357 | $state = array( |
| 50 | - 'running' => ! empty( $urls ), | |
| 51 | - 'started_at' => time(), | |
| 52 | - 'finished_at' => 0, | |
| 53 | - 'queue' => array_values( $urls ), | |
| 54 | - 'processed' => 0, | |
| 55 | - 'total' => count( $urls ), | |
| 56 | - 'last_url' => '', | |
| 57 | - 'errors' => array(), | |
| 358 | + 'running' => ! empty( $urls ), | |
| 359 | + 'started_at' => time(), | |
| 360 | + 'finished_at' => empty( $urls ) ? time() : 0, | |
| 361 | + 'queue' => array_values( $urls ), | |
| 362 | + 'processed' => 0, | |
| 363 | + 'total' => count( $urls ), | |
| 364 | + 'last_url' => '', | |
| 365 | + 'errors' => $errors, | |
| 366 | + // Consumers render on these: the panel needs to distinguish | |
| 367 | + // "not started" from "ran and found nothing", and to tell the | |
| 368 | + // user when the queue came from the fallback rather than the | |
| 369 | + // sitemap they configured. | |
| 370 | + 'source' => self::$queue_source, | |
| 371 | + 'sitemap_error' => $sitemap_error, | |
| 372 | + // Which copies each URL gets, for the panel and the status | |
| 373 | + // command. Each tick refreshes it: turning Separate Mobile | |
| 374 | + // Cache on or off mid-crawl purges the cache, and the rest of | |
| 375 | + // the crawl should fill the copies that now exist. | |
| 376 | + 'devices' => self::devices(), | |
| 58 | 377 | ); |
| 59 | 378 | set_transient( self::STATE_KEY, $state, self::STATE_TTL ); |
| 60 | 379 | |
| 61 | - Activity_Log::record( | |
| 62 | - 'preloader_started', | |
| 63 | - sprintf( 'Preloader queued %d URL%s for warming.', $state['total'], 1 === $state['total'] ? '' : 's' ), | |
| 64 | - $state['total'] > 0 ? Activity_Log::INFO : Activity_Log::WARN | |
| 65 | - ); | |
| 380 | + if ( '' !== $sitemap_error && 'fallback' === self::$queue_source ) { | |
| 381 | + $message = sprintf( | |
| 382 | + /* translators: 1: number of URLs, 2: the sitemap failure detail. */ | |
| 383 | + __( 'Preloader queued %1$d URLs from the site content — %2$s', 'xspeed' ), | |
| 384 | + $state['total'], | |
| 385 | + $sitemap_error | |
| 386 | + ); | |
| 387 | + $severity = Activity_Log::WARN; | |
| 388 | + } elseif ( '' !== $sitemap_error ) { | |
| 389 | + $message = sprintf( | |
| 390 | + /* translators: %s: the sitemap failure detail. */ | |
| 391 | + __( 'Preloader could not start — %s', 'xspeed' ), | |
| 392 | + $sitemap_error | |
| 393 | + ); | |
| 394 | + $severity = Activity_Log::WARN; | |
| 395 | + } else { | |
| 396 | + $message = sprintf( | |
| 397 | + /* translators: 1: number of URLs, 2: plural suffix. */ | |
| 398 | + __( 'Preloader queued %1$d URL%2$s for warming.', 'xspeed' ), | |
| 399 | + $state['total'], | |
| 400 | + 1 === $state['total'] ? '' : 's' | |
| 401 | + ); | |
| 402 | + $severity = $state['total'] > 0 ? Activity_Log::INFO : Activity_Log::WARN; | |
| 403 | + } | |
| 404 | + Activity_Log::record( 'preloader_started', $message, $severity ); | |
| 66 | 405 | |
| 67 | 406 | // Schedule the first tick ~5 seconds out so the kick-off REST call |
| 68 | 407 | // returns instantly; wp_schedule_single_event covers the |
| 69 | 408 | // "process the queue ASAP" path without a heavy synchronous loop. |
| @@ -134,15 +473,41 @@ | ||
| 134 | 473 | |
| 135 | 474 | $opts = Settings_Manager::get( 'preloader' ); |
| 136 | 475 | $batch = max( 1, min( 50, (int) ( $opts['batch_size'] ?? 5 ) ) ); |
| 137 | 476 | |
| 138 | - $processed_this_tick = 0; | |
| 139 | - while ( $processed_this_tick < $batch && ! empty( $state['queue'] ) ) { | |
| 140 | - $url = array_shift( $state['queue'] ); | |
| 477 | + /** | |
| 478 | + * Filter: xspeed_preload_batch_size | |
| 479 | + * | |
| 480 | + * How much one tick may warm, in the units the batch size setting | |
| 481 | + * counts. The setting is the site owner's | |
| 482 | + * intent; this is for anything that knows a lower ceiling applies — | |
| 483 | + * a CDN with a rate limit in front, say, which will answer a burst | |
| 484 | + * with a challenge and leave the queue looking warmed when it is not. | |
| 485 | + * | |
| 486 | + * Only ever lowers. A filter that raised it would let an add-on | |
| 487 | + * overrule a number the site owner chose, and the reason to reach for | |
| 488 | + * this is always that something cannot take the current rate. | |
| 489 | + * | |
| 490 | + * @param int $batch The batch this tick would otherwise use. | |
| 491 | + */ | |
| 492 | + $ceiling = (int) apply_filters( 'xspeed_preload_batch_size', $batch ); | |
| 493 | + if ( $ceiling > 0 && $ceiling < $batch ) { | |
| 494 | + $batch = $ceiling; | |
| 495 | + } | |
| 496 | + | |
| 497 | + // batch_size caps requests, not URLs, so the load on the origin per | |
| 498 | + // tick is the same with the phone copy on: each URL costs one request | |
| 499 | + // per device. A tick always warms at least one URL. | |
| 500 | + $devices = self::devices(); | |
| 501 | + $state['devices'] = $devices; | |
| 502 | + $per_url = count( $devices ); | |
| 503 | + $requests = 0; | |
| 504 | + while ( ! empty( $state['queue'] ) && ( 0 === $requests || $requests + $per_url <= $batch ) ) { | |
| 505 | + $url = (string) array_shift( $state['queue'] ); | |
| 141 | 506 | self::warm_url( $url, $state ); |
| 142 | 507 | $state['processed']++; |
| 143 | 508 | $state['last_url'] = $url; |
| 144 | - $processed_this_tick++; | |
| 509 | + $requests += $per_url; | |
| 145 | 510 | } |
| 146 | 511 | |
| 147 | 512 | if ( empty( $state['queue'] ) ) { |
| 148 | 513 | self::mark_complete( $state ); |
| @@ -168,32 +533,17 @@ | ||
| 168 | 533 | public static function warm_one( string $url, string $cause = 'manual' ): bool { |
| 169 | 534 | if ( '' === $url ) { |
| 170 | 535 | return false; |
| 171 | 536 | } |
| 172 | - $response = wp_remote_get( | |
| 173 | - $url, | |
| 174 | - array( | |
| 175 | - 'timeout' => self::REQUEST_TIMEOUT, | |
| 176 | - 'sslverify' => false, | |
| 177 | - 'user-agent' => self::USER_AGENT, | |
| 178 | - 'blocking' => true, | |
| 179 | - ) | |
| 180 | - ); | |
| 181 | - if ( is_wp_error( $response ) ) { | |
| 537 | + $result = self::warm_devices( $url ); | |
| 538 | + foreach ( $result['failures'] as $failure ) { | |
| 182 | 539 | Activity_Log::record( |
| 183 | 540 | 'preloader_warm_failed', |
| 184 | - sprintf( 'Warm %s failed (%s): %s', $cause, $url, $response->get_error_message() ), | |
| 541 | + sprintf( 'Warm %s failed (%s): %s%s', $cause, $url, self::device_prefix( $failure['device'] ), $failure['error'] ), | |
| 185 | 542 | Activity_Log::WARN |
| 186 | 543 | ); |
| 187 | - return false; | |
| 188 | 544 | } |
| 189 | - $code = (int) wp_remote_retrieve_response_code( $response ); | |
| 190 | - if ( $code >= 400 ) { | |
| 191 | - Activity_Log::record( | |
| 192 | - 'preloader_warm_failed', | |
| 193 | - sprintf( 'Warm %s failed (%s): HTTP %d', $cause, $url, $code ), | |
| 194 | - Activity_Log::WARN | |
| 195 | - ); | |
| 545 | + if ( ! empty( $result['failures'] ) ) { | |
| 196 | 546 | return false; |
| 197 | 547 | } |
| 198 | 548 | Activity_Log::record( |
| 199 | 549 | 'preloader_warmed_one', |
| @@ -203,40 +553,110 @@ | ||
| 203 | 553 | return true; |
| 204 | 554 | } |
| 205 | 555 | |
| 206 | 556 | private static function warm_url( string $url, array &$state ): void { |
| 207 | - $response = wp_remote_get( | |
| 208 | - $url, | |
| 209 | - array( | |
| 210 | - 'timeout' => self::REQUEST_TIMEOUT, | |
| 211 | - 'sslverify' => false, | |
| 212 | - 'user-agent' => self::USER_AGENT, | |
| 213 | - 'headers' => array( | |
| 214 | - 'Accept' => 'text/html,application/xhtml+xml', | |
| 215 | - ), | |
| 216 | - 'blocking' => true, | |
| 217 | - ) | |
| 218 | - ); | |
| 219 | - if ( is_wp_error( $response ) ) { | |
| 557 | + $result = self::warm_devices( $url, array( 'Accept' => 'text/html,application/xhtml+xml' ) ); | |
| 558 | + foreach ( $result['failures'] as $failure ) { | |
| 220 | 559 | $state['errors'][] = array( |
| 221 | 560 | 'url' => $url, |
| 222 | - 'error' => $response->get_error_message(), | |
| 561 | + 'error' => self::device_prefix( $failure['device'] ) . $failure['error'], | |
| 223 | 562 | 'ts' => time(), |
| 224 | 563 | ); |
| 225 | - // Cap retained errors so a broken sitemap doesn't blow the | |
| 226 | - // transient size. | |
| 227 | - $state['errors'] = array_slice( $state['errors'], -20 ); | |
| 564 | + } | |
| 565 | + // Cap retained errors so a broken sitemap doesn't blow the | |
| 566 | + // transient size. | |
| 567 | + $state['errors'] = array_slice( $state['errors'], -20 ); | |
| 568 | + if ( '' !== $result['body'] ) { | |
| 569 | + self::warm_remote_dimensions( $result['body'] ); | |
| 570 | + } | |
| 571 | + } | |
| 572 | + | |
| 573 | + /** | |
| 574 | + * Resolve dimensions for externally hosted images found on a warmed page. | |
| 575 | + * | |
| 576 | + * The crawl already has the HTML in hand, so harvesting image URLs from it | |
| 577 | + * costs nothing extra — and this is the one place where paying for a | |
| 578 | + * remote lookup is free of consequence, because no visitor is waiting. | |
| 579 | + * | |
| 580 | + * An image on another domain has no local file to measure, so the front | |
| 581 | + * end skips it and the page ships without width/height — which is layout | |
| 582 | + * shift, on precisely the sites least able to fix it by hand (a CDN, a | |
| 583 | + * sister site, a shared asset host). Warming here means the NEXT render | |
| 584 | + * finds the dimensions in cache and stamps them, with the visitor paying | |
| 585 | + * nothing. | |
| 586 | + * | |
| 587 | + * Deliberately bounded per page: a crawl should not turn into a scraper | |
| 588 | + * for a page embedding hundreds of third-party images. | |
| 589 | + * | |
| 590 | + * @param string $html The warmed page's HTML. | |
| 591 | + */ | |
| 592 | + private static function warm_remote_dimensions( string $html ): void { | |
| 593 | + if ( '' === $html || ! class_exists( '\XSpeed\Lazy_Loader' ) ) { | |
| 228 | 594 | return; |
| 229 | 595 | } |
| 230 | - $code = (int) wp_remote_retrieve_response_code( $response ); | |
| 231 | - if ( $code >= 400 ) { | |
| 232 | - $state['errors'][] = array( | |
| 233 | - 'url' => $url, | |
| 234 | - 'error' => sprintf( 'HTTP %d', $code ), | |
| 235 | - 'ts' => time(), | |
| 236 | - ); | |
| 237 | - $state['errors'] = array_slice( $state['errors'], -20 ); | |
| 596 | + | |
| 597 | + $opts = Settings_Manager::get( 'lazy' ); | |
| 598 | + if ( empty( $opts['add_missing_dimensions'] ) ) { | |
| 599 | + return; | |
| 238 | 600 | } |
| 601 | + | |
| 602 | + // Match any <img>, not only one carrying `src`. The URL worth warming | |
| 603 | + // may live in a lazy attribute instead — which is the whole point of | |
| 604 | + // #328 — and resolvable_image_url() below is what knows where to look. | |
| 605 | + if ( ! preg_match_all( '#<img\b[^>]*>#i', $html, $m, PREG_SET_ORDER ) ) { | |
| 606 | + return; | |
| 607 | + } | |
| 608 | + | |
| 609 | + $home = wp_parse_url( home_url(), PHP_URL_HOST ); | |
| 610 | + $targets = array(); | |
| 611 | + foreach ( $m as $tag ) { | |
| 612 | + // Only tags MISSING a dimension are worth resolving — one that | |
| 613 | + // already declares both needs nothing. | |
| 614 | + // Same lookbehind as Lazy_Loader::ensure_dimensions(): a bare | |
| 615 | + // `\bwidth=` also matches `data-width=`, so a slider carrying its | |
| 616 | + // own metadata looked already-sized and was skipped from warming. | |
| 617 | + // The two must agree, or the collector skips exactly the tags the | |
| 618 | + // renderer still needs measured. (#333 review round 3, issue 2) | |
| 619 | + if ( preg_match( '#(?<![-\w])width\s*=#i', $tag[0] ) && preg_match( '#(?<![-\w])height\s*=#i', $tag[0] ) ) { | |
| 620 | + continue; | |
| 621 | + } | |
| 622 | + // Ask the same resolver the render path uses, rather than reading | |
| 623 | + // `src` directly. A slider parks a spacer in `src` and the real | |
| 624 | + // URL in `data-lazy`/`data-src`/`data-original`, so a collector | |
| 625 | + // looking only at `src` warmed the SPACER and never the image — | |
| 626 | + // leaving remotely-hosted slider images unresolvable at render | |
| 627 | + // time, the exact markup #328 is about. (#333 review round 2, | |
| 628 | + // issue 3) | |
| 629 | + // `false`: do not let the resolver settle a name-refused URL by | |
| 630 | + // MEASURING it. That is circular here — remote measurement is | |
| 631 | + // gated until warm_dimensions() sets $warming, and this collector | |
| 632 | + // is what feeds warm_dimensions(). Take the URL the tag offers and | |
| 633 | + // let the warm pass decide. (#333 review round 3, issue 3) | |
| 634 | + $src = Lazy_Loader::resolvable_image_url( $tag[0], false ); | |
| 635 | + if ( '' === $src || ! preg_match( '#^https?://#i', $src ) ) { | |
| 636 | + continue; | |
| 637 | + } | |
| 638 | + $host = wp_parse_url( $src, PHP_URL_HOST ); | |
| 639 | + if ( ! $host || $host === $home ) { | |
| 640 | + continue; // Local images already resolve from disk. | |
| 641 | + } | |
| 642 | + // Already resolved (or already known unresolvable) — looking it up | |
| 643 | + // again costs a request and teaches us nothing. Skipping it is | |
| 644 | + // also what makes the cap below advance: images appear in the same | |
| 645 | + // DOM order every crawl, so a collector that did not skip would | |
| 646 | + // re-pick the same first N for ever and never reach the rest. | |
| 647 | + if ( Lazy_Loader::dimensions_known( $src ) ) { | |
| 648 | + continue; | |
| 649 | + } | |
| 650 | + $targets[ $src ] = true; | |
| 651 | + if ( count( $targets ) >= self::remote_dimension_limit() ) { | |
| 652 | + break; | |
| 653 | + } | |
| 654 | + } | |
| 655 | + | |
| 656 | + if ( $targets ) { | |
| 657 | + Lazy_Loader::warm_dimensions( array_keys( $targets ) ); | |
| 658 | + } | |
| 239 | 659 | } |
| 240 | 660 | |
| 241 | 661 | private static function mark_complete( array $state ): void { |
| 242 | 662 | $state['running'] = false; |
| @@ -263,8 +683,14 @@ | ||
| 263 | 683 | * |
| 264 | 684 | * @return string[] |
| 265 | 685 | */ |
| 266 | 686 | private static function resolve_queue( array $opts ): array { |
| 687 | + // Request-scoped statics: reset so a previous crawl in the same | |
| 688 | + // process can't leak its verdict into this one. | |
| 689 | + self::$last_sitemap_error = ''; | |
| 690 | + self::$last_sitemap_url = ''; | |
| 691 | + self::$queue_source = 'none'; | |
| 692 | + | |
| 267 | 693 | $sitemap = trim( (string) ( $opts['sitemap_url'] ?? '' ) ); |
| 268 | 694 | if ( '' === $sitemap ) { |
| 269 | 695 | $sitemap = home_url( '/wp-sitemap.xml' ); |
| 270 | 696 | } |
| @@ -269,9 +695,28 @@ | ||
| 269 | 695 | $sitemap = home_url( '/wp-sitemap.xml' ); |
| 270 | 696 | } |
| 271 | 697 | |
| 272 | 698 | $urls = self::fetch_sitemap_urls( $sitemap, 0 ); |
| 699 | + $from = empty( $urls ) ? 'none' : 'sitemap'; | |
| 273 | 700 | |
| 701 | + /* | |
| 702 | + * A missing sitemap must not disable the feature. Two very common | |
| 703 | + * setups produce one with no misconfiguration by the user: | |
| 704 | + * `blog_public = 0` (WordPress disables /wp-sitemap.xml outright, | |
| 705 | + * standard on staging and pre-launch sites), and an SEO plugin | |
| 706 | + * filtering `wp_sitemaps_enabled` to false while serving its own | |
| 707 | + * sitemap at a path we were never told about. | |
| 708 | + * | |
| 709 | + * Enumerate warmable URLs straight from the database instead. Only | |
| 710 | + * on a genuine fetch FAILURE — a sitemap that is reachable and | |
| 711 | + * legitimately empty is a real answer, and silently crawling | |
| 712 | + * something else would be worse than doing nothing. (#142) | |
| 713 | + */ | |
| 714 | + if ( empty( $urls ) && '' !== self::$last_sitemap_error ) { | |
| 715 | + $urls = self::fallback_urls(); | |
| 716 | + $from = empty( $urls ) ? 'none' : 'fallback'; | |
| 717 | + } | |
| 718 | + | |
| 274 | 719 | $cache_opts = Settings_Manager::get( 'cache' ); |
| 275 | 720 | $excluded = is_array( $cache_opts['excluded_urls'] ?? null ) ? $cache_opts['excluded_urls'] : array(); |
| 276 | 721 | if ( ! empty( $excluded ) ) { |
| 277 | 722 | $urls = array_filter( |
| @@ -289,12 +734,90 @@ | ||
| 289 | 734 | } |
| 290 | 735 | |
| 291 | 736 | // Dedup + cap at 5000 to bound the transient size on huge sites. |
| 292 | 737 | $urls = array_values( array_unique( $urls ) ); |
| 293 | - return array_slice( $urls, 0, 5000 ); | |
| 738 | + $urls = array_slice( $urls, 0, 5000 ); | |
| 739 | + | |
| 740 | + /* | |
| 741 | + * Commit the verdict only now, AFTER the exclusion filter — the | |
| 742 | + * source describes what we ACTUALLY queued, not what we hoped to. | |
| 743 | + * Setting it earlier let a queue that the exclusions stripped to | |
| 744 | + * nothing still claim `fallback`, so the panel announced "Warmed | |
| 745 | + * from site content" over 0 URLs, and `crawlFailed` (which needs | |
| 746 | + * source !== 'fallback' at total 0) could never become true. | |
| 747 | + * One assignment fixes both. (QA F2/F3 on #155) | |
| 748 | + */ | |
| 749 | + self::$queue_source = empty( $urls ) ? 'none' : $from; | |
| 750 | + | |
| 751 | + return $urls; | |
| 294 | 752 | } |
| 295 | 753 | |
| 296 | 754 | /** |
| 755 | + * Enumerate warmable URLs from the database, for sites whose sitemap | |
| 756 | + * can't be fetched. Deliberately modest in scope: the home page, then | |
| 757 | + * the most recently modified public posts across every public post type. | |
| 758 | + * Newest-first is the right bias — those are the URLs most likely to be | |
| 759 | + * requested and least likely to be warm already. | |
| 760 | + * | |
| 761 | + * Uses WP_Query rather than SQL so post-type registration, status | |
| 762 | + * handling and multisite switching all behave the way the rest of | |
| 763 | + * WordPress does. | |
| 764 | + * | |
| 765 | + * @return string[] | |
| 766 | + */ | |
| 767 | + private static function fallback_urls(): array { | |
| 768 | + $urls = array(); | |
| 769 | + $home = (string) home_url( '/' ); | |
| 770 | + if ( '' !== trim( $home, '/' ) ) { | |
| 771 | + $urls[] = $home; | |
| 772 | + } | |
| 773 | + | |
| 774 | + $types = get_post_types( | |
| 775 | + array( | |
| 776 | + 'public' => true, | |
| 777 | + 'publicly_queryable' => true, | |
| 778 | + ) | |
| 779 | + ); | |
| 780 | + // `page` is public but not publicly_queryable, so the query above | |
| 781 | + // misses it — and pages are exactly what a warm cache wants most. | |
| 782 | + // Only add it when the site has post types at all: an empty list | |
| 783 | + // means there is nothing to enumerate, and constructing a WP_Query | |
| 784 | + // for it would be wasted work. | |
| 785 | + if ( ! empty( $types ) ) { | |
| 786 | + $types['page'] = 'page'; | |
| 787 | + unset( $types['attachment'] ); | |
| 788 | + } | |
| 789 | + | |
| 790 | + if ( empty( $types ) || ! class_exists( '\WP_Query' ) ) { | |
| 791 | + return $urls; | |
| 792 | + } | |
| 793 | + | |
| 794 | + $query = new \WP_Query( | |
| 795 | + array( | |
| 796 | + 'post_type' => array_values( $types ), | |
| 797 | + 'post_status' => 'publish', | |
| 798 | + 'posts_per_page' => self::FALLBACK_LIMIT, | |
| 799 | + 'orderby' => 'modified', | |
| 800 | + 'order' => 'DESC', | |
| 801 | + 'ignore_sticky_posts' => true, | |
| 802 | + 'no_found_rows' => true, | |
| 803 | + 'update_post_meta_cache' => false, | |
| 804 | + 'update_post_term_cache' => false, | |
| 805 | + 'fields' => 'ids', | |
| 806 | + ) | |
| 807 | + ); | |
| 808 | + | |
| 809 | + foreach ( $query->posts as $post_id ) { | |
| 810 | + $permalink = get_permalink( (int) $post_id ); | |
| 811 | + if ( is_string( $permalink ) && '' !== $permalink ) { | |
| 812 | + $urls[] = $permalink; | |
| 813 | + } | |
| 814 | + } | |
| 815 | + | |
| 816 | + return array_values( array_unique( $urls ) ); | |
| 817 | + } | |
| 818 | + | |
| 819 | + /** | |
| 297 | 820 | * Recursive sitemap parser. Depth-limited to 3 so a maliciously |
| 298 | 821 | * deep index can't stack-overflow. |
| 299 | 822 | */ |
| 300 | 823 | private static function fetch_sitemap_urls( string $sitemap_url, int $depth ): array { |
| @@ -305,12 +828,40 @@ | ||
| 305 | 828 | $sitemap_url, |
| 306 | 829 | array( |
| 307 | 830 | 'timeout' => self::REQUEST_TIMEOUT, |
| 308 | 831 | 'sslverify' => false, |
| 309 | - 'user-agent' => self::USER_AGENT, | |
| 832 | + 'user-agent' => self::user_agent(), | |
| 833 | + 'headers' => Self_Traffic::headers(), | |
| 310 | 834 | ) |
| 311 | 835 | ); |
| 312 | - if ( is_wp_error( $res ) || (int) wp_remote_retrieve_response_code( $res ) >= 400 ) { | |
| 836 | + if ( is_wp_error( $res ) ) { | |
| 837 | + // Record WHY, don't just vanish. "Unreachable" and "valid but | |
| 838 | + // empty" both used to collapse into an empty array here, which is | |
| 839 | + // what made a sitemap-less site look like a successful crawl of | |
| 840 | + // zero URLs. Only the top-level fetch is recorded: a nested index | |
| 841 | + // failing is a partial result, not a dead crawl. (#142) | |
| 842 | + if ( 0 === $depth ) { | |
| 843 | + self::$last_sitemap_error = sprintf( | |
| 844 | + /* translators: 1: sitemap URL, 2: error detail. */ | |
| 845 | + __( 'Could not fetch the sitemap at %1$s — %2$s', 'xspeed' ), | |
| 846 | + $sitemap_url, | |
| 847 | + $res->get_error_message() | |
| 848 | + ); | |
| 849 | + self::$last_sitemap_url = (string) $sitemap_url; | |
| 850 | + } | |
| 851 | + return array(); | |
| 852 | + } | |
| 853 | + $code = (int) wp_remote_retrieve_response_code( $res ); | |
| 854 | + if ( $code >= 400 ) { | |
| 855 | + if ( 0 === $depth ) { | |
| 856 | + self::$last_sitemap_error = sprintf( | |
| 857 | + /* translators: 1: sitemap URL, 2: HTTP status code. */ | |
| 858 | + __( 'Could not fetch the sitemap at %1$s — the server returned HTTP %2$d.', 'xspeed' ), | |
| 859 | + $sitemap_url, | |
| 860 | + $code | |
| 861 | + ); | |
| 862 | + self::$last_sitemap_url = (string) $sitemap_url; | |
| 863 | + } | |
| 313 | 864 | return array(); |
| 314 | 865 | } |
| 315 | 866 | $body = (string) wp_remote_retrieve_body( $res ); |
| 316 | 867 | if ( '' === $body ) { |
