| @@ -91,8 +91,126 @@ | ||
| 91 | 91 | return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : self::USER_AGENT; |
| 92 | 92 | } |
| 93 | 93 | |
| 94 | 94 | /** |
| 95 | + * A real phone browser's user-agent, which the warmer's own is appended | |
| 96 | + * to for the phone copy. | |
| 97 | + * | |
| 98 | + * A real phone string rather than the warmer's UA plus a client hint: | |
| 99 | + * a theme or plugin that reads the user-agent itself (Mobile_Detect and | |
| 100 | + * the like) instead of calling wp_is_mobile() would otherwise render its | |
| 101 | + * desktop HTML into the phone copy. The warmer's name stays at the end so | |
| 102 | + * the request is still recognisable in access logs and still matches no | |
| 103 | + * bad-bot rule. (#596) | |
| 104 | + */ | |
| 105 | + public const MOBILE_UA_PREFIX = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1'; | |
| 106 | + | |
| 107 | + /** The user-agent the phone warm sends. */ | |
| 108 | + public static function mobile_user_agent(): string { | |
| 109 | + $default = self::MOBILE_UA_PREFIX . ' ' . self::user_agent(); | |
| 110 | + /** | |
| 111 | + * Filter the user-agent the preloader sends when it warms the phone | |
| 112 | + * copy of a page (Separate Mobile Cache on). | |
| 113 | + * | |
| 114 | + * It must still read as a phone: the cache files the response by the | |
| 115 | + * same test as wp_is_mobile(). An empty return is ignored. | |
| 116 | + * | |
| 117 | + * @param string $user_agent Default: an iPhone Safari string followed by the warmer's user-agent. | |
| 118 | + */ | |
| 119 | + $ua = apply_filters( 'xspeed_preloader_mobile_user_agent', $default ); | |
| 120 | + | |
| 121 | + return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : $default; | |
| 122 | + } | |
| 123 | + | |
| 124 | + /** | |
| 125 | + * The copies of a page a warm fills: the desktop one, and the phone one | |
| 126 | + * too when Separate Mobile Cache keeps one per device. Before #596 every | |
| 127 | + * warm sent a desktop user-agent, so the first phone visitor to each | |
| 128 | + * page got an uncached render. | |
| 129 | + * | |
| 130 | + * @return string[] 'desktop', then 'mobile' when it applies. | |
| 131 | + */ | |
| 132 | + public static function devices(): array { | |
| 133 | + $cache = Settings_Manager::get( 'cache' ); | |
| 134 | + $devices = empty( $cache['mobile_separate'] ) ? array( 'desktop' ) : array( 'desktop', 'mobile' ); | |
| 135 | + /** | |
| 136 | + * Filter which copies of a page the preloader warms. | |
| 137 | + * | |
| 138 | + * Return array( 'desktop' ) to skip the phone copy, for example on a | |
| 139 | + * host where the extra requests cost too much. Unknown values are | |
| 140 | + * dropped; an empty list means desktop. | |
| 141 | + * | |
| 142 | + * @param string[] $devices 'desktop', plus 'mobile' when Separate Mobile Cache is on. | |
| 143 | + */ | |
| 144 | + $filtered = apply_filters( 'xspeed_preloader_devices', $devices ); | |
| 145 | + $filtered = is_array( $filtered ) ? array_values( array_intersect( array( 'desktop', 'mobile' ), $filtered ) ) : $devices; | |
| 146 | + | |
| 147 | + return empty( $filtered ) ? array( 'desktop' ) : $filtered; | |
| 148 | + } | |
| 149 | + | |
| 150 | + /** | |
| 151 | + * Request arguments for one warm of one device copy. | |
| 152 | + * | |
| 153 | + * `Sec-CH-UA-Mobile` settles the device whatever the user-agent filters | |
| 154 | + * return: the cache checks that hint before the user-agent. | |
| 155 | + * | |
| 156 | + * @param array<string,string> $headers Extra request headers. | |
| 157 | + */ | |
| 158 | + private static function warm_args( string $device, array $headers = array() ): array { | |
| 159 | + $headers['Sec-CH-UA-Mobile'] = 'mobile' === $device ? '?1' : '?0'; | |
| 160 | + return array( | |
| 161 | + 'timeout' => self::REQUEST_TIMEOUT, | |
| 162 | + 'sslverify' => false, | |
| 163 | + 'user-agent' => 'mobile' === $device ? self::mobile_user_agent() : self::user_agent(), | |
| 164 | + 'headers' => Self_Traffic::headers( $headers ), | |
| 165 | + 'blocking' => true, | |
| 166 | + ); | |
| 167 | + } | |
| 168 | + | |
| 169 | + /** | |
| 170 | + * Warm every copy of $url that devices() names. | |
| 171 | + * | |
| 172 | + * @param array<string,string> $headers Extra request headers. | |
| 173 | + * @return array{failures: array<int,array{device:string,error:string}>, body: string} | |
| 174 | + * The failures, and the desktop response body ('' when it failed). | |
| 175 | + */ | |
| 176 | + private static function warm_devices( string $url, array $headers = array() ): array { | |
| 177 | + $failures = array(); | |
| 178 | + $body = ''; | |
| 179 | + foreach ( self::devices() as $device ) { | |
| 180 | + $args = self::warm_args( $device, $headers ); | |
| 181 | + $response = wp_remote_get( $url, $args ); | |
| 182 | + if ( is_wp_error( $response ) ) { | |
| 183 | + $failures[] = array( 'device' => $device, 'error' => $response->get_error_message() ); | |
| 184 | + continue; | |
| 185 | + } | |
| 186 | + $code = (int) wp_remote_retrieve_response_code( $response ); | |
| 187 | + if ( $code >= 400 ) { | |
| 188 | + $failures[] = array( 'device' => $device, 'error' => self::failure_detail( $code, $args['user-agent'] ) ); | |
| 189 | + if ( self::is_firewall_block( $code ) ) { | |
| 190 | + self::remember_firewall_block( $url, $code, $args['user-agent'] ); | |
| 191 | + } | |
| 192 | + continue; | |
| 193 | + } | |
| 194 | + if ( 'desktop' === $device ) { | |
| 195 | + $body = (string) wp_remote_retrieve_body( $response ); | |
| 196 | + } | |
| 197 | + } | |
| 198 | + // The notice goes only once every copy got through. A host that | |
| 199 | + // blocks just the phone user-agent would otherwise have the notice | |
| 200 | + // cleared by each desktop warm right after the phone warm set it. | |
| 201 | + if ( empty( $failures ) ) { | |
| 202 | + self::clear_firewall_block(); | |
| 203 | + } | |
| 204 | + return array( 'failures' => $failures, 'body' => $body ); | |
| 205 | + } | |
| 206 | + | |
| 207 | + /** "phone: " for a failure of the phone copy, so the log says which copy failed. */ | |
| 208 | + private static function device_prefix( string $device ): string { | |
| 209 | + return 'mobile' === $device ? 'phone: ' : ''; | |
| 210 | + } | |
| 211 | + | |
| 212 | + /** | |
| 95 | 213 | * Is this status code the signature of a firewall refusing our warmer? |
| 96 | 214 | * |
| 97 | 215 | * 403 and 406 are what bad-bot rules (7G/8G, mod_security, Wordfence) |
| 98 | 216 | * answer with. We only ever warm our OWN origin, and a page a visitor can |
| @@ -109,9 +227,9 @@ | ||
| 109 | 227 | * A bare "HTTP 403" sent people hunting a broken page; the page is fine, |
| 110 | 228 | * and the fix is a server rule, so the message has to name the cause and |
| 111 | 229 | * the exact UA to allow. (#481) |
| 112 | 230 | */ |
| 113 | - private static function failure_detail( int $code ): string { | |
| 231 | + private static function failure_detail( int $code, string $user_agent = '' ): string { | |
| 114 | 232 | if ( ! self::is_firewall_block( $code ) ) { |
| 115 | 233 | return sprintf( 'HTTP %d', $code ); |
| 116 | 234 | } |
| 117 | 235 | |
| @@ -117,9 +235,9 @@ | ||
| 117 | 235 | |
| 118 | 236 | return sprintf( |
| 119 | 237 | 'HTTP %d — your server\'s firewall is blocking the xSpeed cache warmer by user-agent, so this page was not warmed. Allow the user-agent "%s" (on xCloud this is the 8G firewall\'s bad-bot rule), or change it with the xspeed_preloader_user_agent filter.', |
| 120 | 238 | $code, |
| 121 | - self::user_agent() | |
| 239 | + '' !== $user_agent ? $user_agent : self::user_agent() | |
| 122 | 240 | ); |
| 123 | 241 | } |
| 124 | 242 | |
| 125 | 243 | /** Option holding the last firewall-shaped warm refusal. */ |
| @@ -132,9 +250,9 @@ | ||
| 132 | 250 | * persists until someone changes it, and a notice that expired on its own |
| 133 | 251 | * would let a site go back to never warming, silently. Cleared by |
| 134 | 252 | * clear_firewall_block() on the first warm that succeeds. (#481) |
| 135 | 253 | */ |
| 136 | - private static function remember_firewall_block( string $url, int $code ): void { | |
| 254 | + private static function remember_firewall_block( string $url, int $code, string $user_agent = '' ): void { | |
| 137 | 255 | if ( ! function_exists( 'update_option' ) ) { |
| 138 | 256 | return; |
| 139 | 257 | } |
| 140 | 258 | update_option( |
| @@ -141,9 +259,9 @@ | ||
| 141 | 259 | self::FIREWALL_BLOCK_OPTION, |
| 142 | 260 | array( |
| 143 | 261 | 'url' => $url, |
| 144 | 262 | 'code' => $code, |
| 145 | - 'user_agent' => self::user_agent(), | |
| 263 | + 'user_agent' => '' !== $user_agent ? $user_agent : self::user_agent(), | |
| 146 | 264 | 'ts' => time(), |
| 147 | 265 | ), |
| 148 | 266 | false |
| 149 | 267 | ); |
| @@ -250,8 +368,13 @@ | ||
| 250 | 368 | // user when the queue came from the fallback rather than the |
| 251 | 369 | // sitemap they configured. |
| 252 | 370 | 'source' => self::$queue_source, |
| 253 | 371 | 'sitemap_error' => $sitemap_error, |
| 372 | + // Which copies each URL gets, for the panel and the status | |
| 373 | + // command. Each tick refreshes it: turning Separate Mobile | |
| 374 | + // Cache on or off mid-crawl purges the cache, and the rest of | |
| 375 | + // the crawl should fill the copies that now exist. | |
| 376 | + 'devices' => self::devices(), | |
| 254 | 377 | ); |
| 255 | 378 | set_transient( self::STATE_KEY, $state, self::STATE_TTL ); |
| 256 | 379 | |
| 257 | 380 | if ( '' !== $sitemap_error && 'fallback' === self::$queue_source ) { |
| @@ -350,15 +473,41 @@ | ||
| 350 | 473 | |
| 351 | 474 | $opts = Settings_Manager::get( 'preloader' ); |
| 352 | 475 | $batch = max( 1, min( 50, (int) ( $opts['batch_size'] ?? 5 ) ) ); |
| 353 | 476 | |
| 354 | - $processed_this_tick = 0; | |
| 355 | - while ( $processed_this_tick < $batch && ! empty( $state['queue'] ) ) { | |
| 356 | - $url = array_shift( $state['queue'] ); | |
| 477 | + /** | |
| 478 | + * Filter: xspeed_preload_batch_size | |
| 479 | + * | |
| 480 | + * How much one tick may warm, in the units the batch size setting | |
| 481 | + * counts. The setting is the site owner's | |
| 482 | + * intent; this is for anything that knows a lower ceiling applies — | |
| 483 | + * a CDN with a rate limit in front, say, which will answer a burst | |
| 484 | + * with a challenge and leave the queue looking warmed when it is not. | |
| 485 | + * | |
| 486 | + * Only ever lowers. A filter that raised it would let an add-on | |
| 487 | + * overrule a number the site owner chose, and the reason to reach for | |
| 488 | + * this is always that something cannot take the current rate. | |
| 489 | + * | |
| 490 | + * @param int $batch The batch this tick would otherwise use. | |
| 491 | + */ | |
| 492 | + $ceiling = (int) apply_filters( 'xspeed_preload_batch_size', $batch ); | |
| 493 | + if ( $ceiling > 0 && $ceiling < $batch ) { | |
| 494 | + $batch = $ceiling; | |
| 495 | + } | |
| 496 | + | |
| 497 | + // batch_size caps requests, not URLs, so the load on the origin per | |
| 498 | + // tick is the same with the phone copy on: each URL costs one request | |
| 499 | + // per device. A tick always warms at least one URL. | |
| 500 | + $devices = self::devices(); | |
| 501 | + $state['devices'] = $devices; | |
| 502 | + $per_url = count( $devices ); | |
| 503 | + $requests = 0; | |
| 504 | + while ( ! empty( $state['queue'] ) && ( 0 === $requests || $requests + $per_url <= $batch ) ) { | |
| 505 | + $url = (string) array_shift( $state['queue'] ); | |
| 357 | 506 | self::warm_url( $url, $state ); |
| 358 | 507 | $state['processed']++; |
| 359 | 508 | $state['last_url'] = $url; |
| 360 | - $processed_this_tick++; | |
| 509 | + $requests += $per_url; | |
| 361 | 510 | } |
| 362 | 511 | |
| 363 | 512 | if ( empty( $state['queue'] ) ) { |
| 364 | 513 | self::mark_complete( $state ); |
| @@ -384,41 +533,19 @@ | ||
| 384 | 533 | public static function warm_one( string $url, string $cause = 'manual' ): bool { |
| 385 | 534 | if ( '' === $url ) { |
| 386 | 535 | return false; |
| 387 | 536 | } |
| 388 | - $response = wp_remote_get( | |
| 389 | - $url, | |
| 390 | - array( | |
| 391 | - 'timeout' => self::REQUEST_TIMEOUT, | |
| 392 | - 'sslverify' => false, | |
| 393 | - 'user-agent' => self::user_agent(), | |
| 394 | - 'headers' => Self_Traffic::headers(), | |
| 395 | - 'blocking' => true, | |
| 396 | - ) | |
| 397 | - ); | |
| 398 | - if ( is_wp_error( $response ) ) { | |
| 537 | + $result = self::warm_devices( $url ); | |
| 538 | + foreach ( $result['failures'] as $failure ) { | |
| 399 | 539 | Activity_Log::record( |
| 400 | 540 | 'preloader_warm_failed', |
| 401 | - sprintf( 'Warm %s failed (%s): %s', $cause, $url, $response->get_error_message() ), | |
| 541 | + sprintf( 'Warm %s failed (%s): %s%s', $cause, $url, self::device_prefix( $failure['device'] ), $failure['error'] ), | |
| 402 | 542 | Activity_Log::WARN |
| 403 | 543 | ); |
| 404 | - return false; | |
| 405 | 544 | } |
| 406 | - $code = (int) wp_remote_retrieve_response_code( $response ); | |
| 407 | - if ( $code >= 400 ) { | |
| 408 | - Activity_Log::record( | |
| 409 | - 'preloader_warm_failed', | |
| 410 | - sprintf( 'Warm %s failed (%s): %s', $cause, $url, self::failure_detail( $code ) ), | |
| 411 | - Activity_Log::WARN | |
| 412 | - ); | |
| 413 | - if ( self::is_firewall_block( $code ) ) { | |
| 414 | - self::remember_firewall_block( $url, $code ); | |
| 415 | - } | |
| 545 | + if ( ! empty( $result['failures'] ) ) { | |
| 416 | 546 | return false; |
| 417 | 547 | } |
| 418 | - // A warm that got through proves the firewall is no longer refusing us, | |
| 419 | - // so the notice must go — otherwise it outlives the problem. (#481) | |
| 420 | - self::clear_firewall_block(); | |
| 421 | 548 | Activity_Log::record( |
| 422 | 549 | 'preloader_warmed_one', |
| 423 | 550 | sprintf( 'Warmed %s (%s)', $url, $cause ), |
| 424 | 551 | Activity_Log::INFO |
| @@ -426,49 +553,22 @@ | ||
| 426 | 553 | return true; |
| 427 | 554 | } |
| 428 | 555 | |
| 429 | 556 | private static function warm_url( string $url, array &$state ): void { |
| 430 | - $response = wp_remote_get( | |
| 431 | - $url, | |
| 432 | - array( | |
| 433 | - 'timeout' => self::REQUEST_TIMEOUT, | |
| 434 | - 'sslverify' => false, | |
| 435 | - 'user-agent' => self::user_agent(), | |
| 436 | - 'headers' => Self_Traffic::headers( | |
| 437 | - array( | |
| 438 | - 'Accept' => 'text/html,application/xhtml+xml', | |
| 439 | - ) | |
| 440 | - ), | |
| 441 | - 'blocking' => true, | |
| 442 | - ) | |
| 443 | - ); | |
| 444 | - if ( is_wp_error( $response ) ) { | |
| 557 | + $result = self::warm_devices( $url, array( 'Accept' => 'text/html,application/xhtml+xml' ) ); | |
| 558 | + foreach ( $result['failures'] as $failure ) { | |
| 445 | 559 | $state['errors'][] = array( |
| 446 | 560 | 'url' => $url, |
| 447 | - 'error' => $response->get_error_message(), | |
| 561 | + 'error' => self::device_prefix( $failure['device'] ) . $failure['error'], | |
| 448 | 562 | 'ts' => time(), |
| 449 | 563 | ); |
| 450 | - // Cap retained errors so a broken sitemap doesn't blow the | |
| 451 | - // transient size. | |
| 452 | - $state['errors'] = array_slice( $state['errors'], -20 ); | |
| 453 | - return; | |
| 454 | 564 | } |
| 455 | - $code = (int) wp_remote_retrieve_response_code( $response ); | |
| 456 | - if ( $code >= 400 ) { | |
| 457 | - $state['errors'][] = array( | |
| 458 | - 'url' => $url, | |
| 459 | - 'error' => self::failure_detail( $code ), | |
| 460 | - 'ts' => time(), | |
| 461 | - ); | |
| 462 | - $state['errors'] = array_slice( $state['errors'], -20 ); | |
| 463 | - if ( self::is_firewall_block( $code ) ) { | |
| 464 | - self::remember_firewall_block( $url, $code ); | |
| 465 | - } | |
| 466 | - return; | |
| 565 | + // Cap retained errors so a broken sitemap doesn't blow the | |
| 566 | + // transient size. | |
| 567 | + $state['errors'] = array_slice( $state['errors'], -20 ); | |
| 568 | + if ( '' !== $result['body'] ) { | |
| 569 | + self::warm_remote_dimensions( $result['body'] ); | |
| 467 | 570 | } |
| 468 | - | |
| 469 | - self::clear_firewall_block(); | |
| 470 | - self::warm_remote_dimensions( (string) wp_remote_retrieve_body( $response ) ); | |
| 471 | 571 | } |
| 472 | 572 | |
| 473 | 573 | /** |
| 474 | 574 | * Resolve dimensions for externally hosted images found on a warmed page. |
