| @@ -35,9 +35,24 @@ | ||
| 35 | 35 | |
| 36 | 36 | public const STATE_KEY = 'xspeed_preloader_state'; |
| 37 | 37 | public const STATE_TTL = 86400; // 24h — long enough for slow crawls. |
| 38 | 38 | public const CRON_HOOK = 'xspeed_preloader_tick'; |
| 39 | - public const USER_AGENT = 'xSpeed-Preloader/1.0 (+cache warmer; admin-initiated)'; | |
| 39 | + /** | |
| 40 | + * User-agent for every request the preloader makes. | |
| 41 | + * | |
| 42 | + * Deliberately contains no substring from the 7G/8G bad-bot lists. The | |
| 43 | + * previous value, "xSpeed-Preloader/1.0", matched the `loader` token in | |
| 44 | + * the alphabetical slice `(linkscan|linkwalker|loader|lwp-download|...)` | |
| 45 | + * — a match on "Pre*loader*" — so nginx ports of 8G answered every warm | |
| 46 | + * with 403 and newly published posts were never warmed. Upstream 8G | |
| 47 | + * v1.5 has since dropped `loader`, but forks and vendored copies (xCloud | |
| 48 | + * among them) still ship the older slice, so the name has to stay clear | |
| 49 | + * of it. "Warmer" matches nothing in either list. (#481) | |
| 50 | + * | |
| 51 | + * Read through user_agent() rather than using this constant directly, so | |
| 52 | + * the `xspeed_preloader_user_agent` filter applies. | |
| 53 | + */ | |
| 54 | + public const USER_AGENT = 'xSpeed-Warmer/1.0 (+cache warmer; admin-initiated)'; | |
| 40 | 55 | public const REQUEST_TIMEOUT = 8; |
| 41 | 56 | |
| 42 | 57 | /** |
| 43 | 58 | * Default cap on NEW remote images resolved per warmed page. |
| @@ -57,8 +72,220 @@ | ||
| 57 | 72 | */ |
| 58 | 73 | private const REMOTE_DIMENSION_LIMIT = 20; |
| 59 | 74 | |
| 60 | 75 | /** |
| 76 | + * The user-agent every preloader request sends. | |
| 77 | + * | |
| 78 | + * Filterable because the blocking rule lives on the server, not here: a | |
| 79 | + * host with its own bad-bot list can clear a warm without patching the | |
| 80 | + * plugin or waiting for a release. An empty filter return is ignored — | |
| 81 | + * sending no UA gets a request blocked at least as often. (#481) | |
| 82 | + */ | |
| 83 | + public static function user_agent(): string { | |
| 84 | + /** | |
| 85 | + * Filter the preloader's user-agent string. | |
| 86 | + * | |
| 87 | + * @param string $user_agent Default self::USER_AGENT. | |
| 88 | + */ | |
| 89 | + $ua = apply_filters( 'xspeed_preloader_user_agent', self::USER_AGENT ); | |
| 90 | + | |
| 91 | + return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : self::USER_AGENT; | |
| 92 | + } | |
| 93 | + | |
| 94 | + /** | |
| 95 | + * A real phone browser's user-agent, which the warmer's own is appended | |
| 96 | + * to for the phone copy. | |
| 97 | + * | |
| 98 | + * A real phone string rather than the warmer's UA plus a client hint: | |
| 99 | + * a theme or plugin that reads the user-agent itself (Mobile_Detect and | |
| 100 | + * the like) instead of calling wp_is_mobile() would otherwise render its | |
| 101 | + * desktop HTML into the phone copy. The warmer's name stays at the end so | |
| 102 | + * the request is still recognisable in access logs and still matches no | |
| 103 | + * bad-bot rule. (#596) | |
| 104 | + */ | |
| 105 | + public const MOBILE_UA_PREFIX = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1'; | |
| 106 | + | |
| 107 | + /** The user-agent the phone warm sends. */ | |
| 108 | + public static function mobile_user_agent(): string { | |
| 109 | + $default = self::MOBILE_UA_PREFIX . ' ' . self::user_agent(); | |
| 110 | + /** | |
| 111 | + * Filter the user-agent the preloader sends when it warms the phone | |
| 112 | + * copy of a page (Separate Mobile Cache on). | |
| 113 | + * | |
| 114 | + * It must still read as a phone: the cache files the response by the | |
| 115 | + * same test as wp_is_mobile(). An empty return is ignored. | |
| 116 | + * | |
| 117 | + * @param string $user_agent Default: an iPhone Safari string followed by the warmer's user-agent. | |
| 118 | + */ | |
| 119 | + $ua = apply_filters( 'xspeed_preloader_mobile_user_agent', $default ); | |
| 120 | + | |
| 121 | + return ( is_string( $ua ) && '' !== trim( $ua ) ) ? trim( $ua ) : $default; | |
| 122 | + } | |
| 123 | + | |
| 124 | + /** | |
| 125 | + * The copies of a page a warm fills: the desktop one, and the phone one | |
| 126 | + * too when Separate Mobile Cache keeps one per device. Before #596 every | |
| 127 | + * warm sent a desktop user-agent, so the first phone visitor to each | |
| 128 | + * page got an uncached render. | |
| 129 | + * | |
| 130 | + * @return string[] 'desktop', then 'mobile' when it applies. | |
| 131 | + */ | |
| 132 | + public static function devices(): array { | |
| 133 | + $cache = Settings_Manager::get( 'cache' ); | |
| 134 | + $devices = empty( $cache['mobile_separate'] ) ? array( 'desktop' ) : array( 'desktop', 'mobile' ); | |
| 135 | + /** | |
| 136 | + * Filter which copies of a page the preloader warms. | |
| 137 | + * | |
| 138 | + * Return array( 'desktop' ) to skip the phone copy, for example on a | |
| 139 | + * host where the extra requests cost too much. Unknown values are | |
| 140 | + * dropped; an empty list means desktop. | |
| 141 | + * | |
| 142 | + * @param string[] $devices 'desktop', plus 'mobile' when Separate Mobile Cache is on. | |
| 143 | + */ | |
| 144 | + $filtered = apply_filters( 'xspeed_preloader_devices', $devices ); | |
| 145 | + $filtered = is_array( $filtered ) ? array_values( array_intersect( array( 'desktop', 'mobile' ), $filtered ) ) : $devices; | |
| 146 | + | |
| 147 | + return empty( $filtered ) ? array( 'desktop' ) : $filtered; | |
| 148 | + } | |
| 149 | + | |
| 150 | + /** | |
| 151 | + * Request arguments for one warm of one device copy. | |
| 152 | + * | |
| 153 | + * `Sec-CH-UA-Mobile` settles the device whatever the user-agent filters | |
| 154 | + * return: the cache checks that hint before the user-agent. | |
| 155 | + * | |
| 156 | + * @param array<string,string> $headers Extra request headers. | |
| 157 | + */ | |
| 158 | + private static function warm_args( string $device, array $headers = array() ): array { | |
| 159 | + $headers['Sec-CH-UA-Mobile'] = 'mobile' === $device ? '?1' : '?0'; | |
| 160 | + return array( | |
| 161 | + 'timeout' => self::REQUEST_TIMEOUT, | |
| 162 | + 'sslverify' => false, | |
| 163 | + 'user-agent' => 'mobile' === $device ? self::mobile_user_agent() : self::user_agent(), | |
| 164 | + 'headers' => Self_Traffic::headers( $headers ), | |
| 165 | + 'blocking' => true, | |
| 166 | + ); | |
| 167 | + } | |
| 168 | + | |
| 169 | + /** | |
| 170 | + * Warm every copy of $url that devices() names. | |
| 171 | + * | |
| 172 | + * @param array<string,string> $headers Extra request headers. | |
| 173 | + * @return array{failures: array<int,array{device:string,error:string}>, body: string} | |
| 174 | + * The failures, and the desktop response body ('' when it failed). | |
| 175 | + */ | |
| 176 | + private static function warm_devices( string $url, array $headers = array() ): array { | |
| 177 | + $failures = array(); | |
| 178 | + $body = ''; | |
| 179 | + foreach ( self::devices() as $device ) { | |
| 180 | + $args = self::warm_args( $device, $headers ); | |
| 181 | + $response = wp_remote_get( $url, $args ); | |
| 182 | + if ( is_wp_error( $response ) ) { | |
| 183 | + $failures[] = array( 'device' => $device, 'error' => $response->get_error_message() ); | |
| 184 | + continue; | |
| 185 | + } | |
| 186 | + $code = (int) wp_remote_retrieve_response_code( $response ); | |
| 187 | + if ( $code >= 400 ) { | |
| 188 | + $failures[] = array( 'device' => $device, 'error' => self::failure_detail( $code, $args['user-agent'] ) ); | |
| 189 | + if ( self::is_firewall_block( $code ) ) { | |
| 190 | + self::remember_firewall_block( $url, $code, $args['user-agent'] ); | |
| 191 | + } | |
| 192 | + continue; | |
| 193 | + } | |
| 194 | + if ( 'desktop' === $device ) { | |
| 195 | + $body = (string) wp_remote_retrieve_body( $response ); | |
| 196 | + } | |
| 197 | + } | |
| 198 | + // The notice goes only once every copy got through. A host that | |
| 199 | + // blocks just the phone user-agent would otherwise have the notice | |
| 200 | + // cleared by each desktop warm right after the phone warm set it. | |
| 201 | + if ( empty( $failures ) ) { | |
| 202 | + self::clear_firewall_block(); | |
| 203 | + } | |
| 204 | + return array( 'failures' => $failures, 'body' => $body ); | |
| 205 | + } | |
| 206 | + | |
| 207 | + /** "phone: " for a failure of the phone copy, so the log says which copy failed. */ | |
| 208 | + private static function device_prefix( string $device ): string { | |
| 209 | + return 'mobile' === $device ? 'phone: ' : ''; | |
| 210 | + } | |
| 211 | + | |
| 212 | + /** | |
| 213 | + * Is this status code the signature of a firewall refusing our warmer? | |
| 214 | + * | |
| 215 | + * 403 and 406 are what bad-bot rules (7G/8G, mod_security, Wordfence) | |
| 216 | + * answer with. We only ever warm our OWN origin, and a page a visitor can | |
| 217 | + * load must be loadable by us too — so these codes mean the request was | |
| 218 | + * judged by its user-agent, not that the page is missing or broken. (#481) | |
| 219 | + */ | |
| 220 | + private static function is_firewall_block( int $code ): bool { | |
| 221 | + return in_array( $code, array( 403, 406 ), true ); | |
| 222 | + } | |
| 223 | + | |
| 224 | + /** | |
| 225 | + * Explain a warm failure in terms the admin can act on. | |
| 226 | + * | |
| 227 | + * A bare "HTTP 403" sent people hunting a broken page; the page is fine, | |
| 228 | + * and the fix is a server rule, so the message has to name the cause and | |
| 229 | + * the exact UA to allow. (#481) | |
| 230 | + */ | |
| 231 | + private static function failure_detail( int $code, string $user_agent = '' ): string { | |
| 232 | + if ( ! self::is_firewall_block( $code ) ) { | |
| 233 | + return sprintf( 'HTTP %d', $code ); | |
| 234 | + } | |
| 235 | + | |
| 236 | + return sprintf( | |
| 237 | + 'HTTP %d — your server\'s firewall is blocking the xSpeed cache warmer by user-agent, so this page was not warmed. Allow the user-agent "%s" (on xCloud this is the 8G firewall\'s bad-bot rule), or change it with the xspeed_preloader_user_agent filter.', | |
| 238 | + $code, | |
| 239 | + '' !== $user_agent ? $user_agent : self::user_agent() | |
| 240 | + ); | |
| 241 | + } | |
| 242 | + | |
| 243 | + /** Option holding the last firewall-shaped warm refusal. */ | |
| 244 | + public const FIREWALL_BLOCK_OPTION = 'xspeed_preloader_firewall_block'; | |
| 245 | + | |
| 246 | + /** | |
| 247 | + * Record that the origin refused a warm by user-agent, for ui_notices(). | |
| 248 | + * | |
| 249 | + * An option rather than a transient: the condition is a server rule that | |
| 250 | + * persists until someone changes it, and a notice that expired on its own | |
| 251 | + * would let a site go back to never warming, silently. Cleared by | |
| 252 | + * clear_firewall_block() on the first warm that succeeds. (#481) | |
| 253 | + */ | |
| 254 | + private static function remember_firewall_block( string $url, int $code, string $user_agent = '' ): void { | |
| 255 | + if ( ! function_exists( 'update_option' ) ) { | |
| 256 | + return; | |
| 257 | + } | |
| 258 | + update_option( | |
| 259 | + self::FIREWALL_BLOCK_OPTION, | |
| 260 | + array( | |
| 261 | + 'url' => $url, | |
| 262 | + 'code' => $code, | |
| 263 | + 'user_agent' => '' !== $user_agent ? $user_agent : self::user_agent(), | |
| 264 | + 'ts' => time(), | |
| 265 | + ), | |
| 266 | + false | |
| 267 | + ); | |
| 268 | + } | |
| 269 | + | |
| 270 | + /** Forget the firewall block once a warm gets through. */ | |
| 271 | + public static function clear_firewall_block(): void { | |
| 272 | + if ( function_exists( 'delete_option' ) && self::firewall_block() ) { | |
| 273 | + delete_option( self::FIREWALL_BLOCK_OPTION ); | |
| 274 | + } | |
| 275 | + } | |
| 276 | + | |
| 277 | + /** The last firewall-shaped refusal, or null when there isn't one. */ | |
| 278 | + public static function firewall_block(): ?array { | |
| 279 | + if ( ! function_exists( 'get_option' ) ) { | |
| 280 | + return null; | |
| 281 | + } | |
| 282 | + $block = get_option( self::FIREWALL_BLOCK_OPTION, null ); | |
| 283 | + | |
| 284 | + return ( is_array( $block ) && ! empty( $block['code'] ) ) ? $block : null; | |
| 285 | + } | |
| 286 | + | |
| 287 | + /** | |
| 61 | 288 | * How many new remote images one warmed page may resolve. |
| 62 | 289 | */ |
| 63 | 290 | private static function remote_dimension_limit(): int { |
| 64 | 291 | /** |
| @@ -141,8 +368,13 @@ | ||
| 141 | 368 | // user when the queue came from the fallback rather than the |
| 142 | 369 | // sitemap they configured. |
| 143 | 370 | 'source' => self::$queue_source, |
| 144 | 371 | 'sitemap_error' => $sitemap_error, |
| 372 | + // Which copies each URL gets, for the panel and the status | |
| 373 | + // command. Each tick refreshes it: turning Separate Mobile | |
| 374 | + // Cache on or off mid-crawl purges the cache, and the rest of | |
| 375 | + // the crawl should fill the copies that now exist. | |
| 376 | + 'devices' => self::devices(), | |
| 145 | 377 | ); |
| 146 | 378 | set_transient( self::STATE_KEY, $state, self::STATE_TTL ); |
| 147 | 379 | |
| 148 | 380 | if ( '' !== $sitemap_error && 'fallback' === self::$queue_source ) { |
| @@ -241,15 +473,41 @@ | ||
| 241 | 473 | |
| 242 | 474 | $opts = Settings_Manager::get( 'preloader' ); |
| 243 | 475 | $batch = max( 1, min( 50, (int) ( $opts['batch_size'] ?? 5 ) ) ); |
| 244 | 476 | |
| 245 | - $processed_this_tick = 0; | |
| 246 | - while ( $processed_this_tick < $batch && ! empty( $state['queue'] ) ) { | |
| 247 | - $url = array_shift( $state['queue'] ); | |
| 477 | + /** | |
| 478 | + * Filter: xspeed_preload_batch_size | |
| 479 | + * | |
| 480 | + * How much one tick may warm, in the units the batch size setting | |
| 481 | + * counts. The setting is the site owner's | |
| 482 | + * intent; this is for anything that knows a lower ceiling applies — | |
| 483 | + * a CDN with a rate limit in front, say, which will answer a burst | |
| 484 | + * with a challenge and leave the queue looking warmed when it is not. | |
| 485 | + * | |
| 486 | + * Only ever lowers. A filter that raised it would let an add-on | |
| 487 | + * overrule a number the site owner chose, and the reason to reach for | |
| 488 | + * this is always that something cannot take the current rate. | |
| 489 | + * | |
| 490 | + * @param int $batch The batch this tick would otherwise use. | |
| 491 | + */ | |
| 492 | + $ceiling = (int) apply_filters( 'xspeed_preload_batch_size', $batch ); | |
| 493 | + if ( $ceiling > 0 && $ceiling < $batch ) { | |
| 494 | + $batch = $ceiling; | |
| 495 | + } | |
| 496 | + | |
| 497 | + // batch_size caps requests, not URLs, so the load on the origin per | |
| 498 | + // tick is the same with the phone copy on: each URL costs one request | |
| 499 | + // per device. A tick always warms at least one URL. | |
| 500 | + $devices = self::devices(); | |
| 501 | + $state['devices'] = $devices; | |
| 502 | + $per_url = count( $devices ); | |
| 503 | + $requests = 0; | |
| 504 | + while ( ! empty( $state['queue'] ) && ( 0 === $requests || $requests + $per_url <= $batch ) ) { | |
| 505 | + $url = (string) array_shift( $state['queue'] ); | |
| 248 | 506 | self::warm_url( $url, $state ); |
| 249 | 507 | $state['processed']++; |
| 250 | 508 | $state['last_url'] = $url; |
| 251 | - $processed_this_tick++; | |
| 509 | + $requests += $per_url; | |
| 252 | 510 | } |
| 253 | 511 | |
| 254 | 512 | if ( empty( $state['queue'] ) ) { |
| 255 | 513 | self::mark_complete( $state ); |
| @@ -275,32 +533,17 @@ | ||
| 275 | 533 | public static function warm_one( string $url, string $cause = 'manual' ): bool { |
| 276 | 534 | if ( '' === $url ) { |
| 277 | 535 | return false; |
| 278 | 536 | } |
| 279 | - $response = wp_remote_get( | |
| 280 | - $url, | |
| 281 | - array( | |
| 282 | - 'timeout' => self::REQUEST_TIMEOUT, | |
| 283 | - 'sslverify' => false, | |
| 284 | - 'user-agent' => self::USER_AGENT, | |
| 285 | - 'blocking' => true, | |
| 286 | - ) | |
| 287 | - ); | |
| 288 | - if ( is_wp_error( $response ) ) { | |
| 537 | + $result = self::warm_devices( $url ); | |
| 538 | + foreach ( $result['failures'] as $failure ) { | |
| 289 | 539 | Activity_Log::record( |
| 290 | 540 | 'preloader_warm_failed', |
| 291 | - sprintf( 'Warm %s failed (%s): %s', $cause, $url, $response->get_error_message() ), | |
| 541 | + sprintf( 'Warm %s failed (%s): %s%s', $cause, $url, self::device_prefix( $failure['device'] ), $failure['error'] ), | |
| 292 | 542 | Activity_Log::WARN |
| 293 | 543 | ); |
| 294 | - return false; | |
| 295 | 544 | } |
| 296 | - $code = (int) wp_remote_retrieve_response_code( $response ); | |
| 297 | - if ( $code >= 400 ) { | |
| 298 | - Activity_Log::record( | |
| 299 | - 'preloader_warm_failed', | |
| 300 | - sprintf( 'Warm %s failed (%s): HTTP %d', $cause, $url, $code ), | |
| 301 | - Activity_Log::WARN | |
| 302 | - ); | |
| 545 | + if ( ! empty( $result['failures'] ) ) { | |
| 303 | 546 | return false; |
| 304 | 547 | } |
| 305 | 548 | Activity_Log::record( |
| 306 | 549 | 'preloader_warmed_one', |
| @@ -310,43 +553,22 @@ | ||
| 310 | 553 | return true; |
| 311 | 554 | } |
| 312 | 555 | |
| 313 | 556 | private static function warm_url( string $url, array &$state ): void { |
| 314 | - $response = wp_remote_get( | |
| 315 | - $url, | |
| 316 | - array( | |
| 317 | - 'timeout' => self::REQUEST_TIMEOUT, | |
| 318 | - 'sslverify' => false, | |
| 319 | - 'user-agent' => self::USER_AGENT, | |
| 320 | - 'headers' => array( | |
| 321 | - 'Accept' => 'text/html,application/xhtml+xml', | |
| 322 | - ), | |
| 323 | - 'blocking' => true, | |
| 324 | - ) | |
| 325 | - ); | |
| 326 | - if ( is_wp_error( $response ) ) { | |
| 557 | + $result = self::warm_devices( $url, array( 'Accept' => 'text/html,application/xhtml+xml' ) ); | |
| 558 | + foreach ( $result['failures'] as $failure ) { | |
| 327 | 559 | $state['errors'][] = array( |
| 328 | 560 | 'url' => $url, |
| 329 | - 'error' => $response->get_error_message(), | |
| 561 | + 'error' => self::device_prefix( $failure['device'] ) . $failure['error'], | |
| 330 | 562 | 'ts' => time(), |
| 331 | 563 | ); |
| 332 | - // Cap retained errors so a broken sitemap doesn't blow the | |
| 333 | - // transient size. | |
| 334 | - $state['errors'] = array_slice( $state['errors'], -20 ); | |
| 335 | - return; | |
| 336 | 564 | } |
| 337 | - $code = (int) wp_remote_retrieve_response_code( $response ); | |
| 338 | - if ( $code >= 400 ) { | |
| 339 | - $state['errors'][] = array( | |
| 340 | - 'url' => $url, | |
| 341 | - 'error' => sprintf( 'HTTP %d', $code ), | |
| 342 | - 'ts' => time(), | |
| 343 | - ); | |
| 344 | - $state['errors'] = array_slice( $state['errors'], -20 ); | |
| 345 | - return; | |
| 565 | + // Cap retained errors so a broken sitemap doesn't blow the | |
| 566 | + // transient size. | |
| 567 | + $state['errors'] = array_slice( $state['errors'], -20 ); | |
| 568 | + if ( '' !== $result['body'] ) { | |
| 569 | + self::warm_remote_dimensions( $result['body'] ); | |
| 346 | 570 | } |
| 347 | - | |
| 348 | - self::warm_remote_dimensions( (string) wp_remote_retrieve_body( $response ) ); | |
| 349 | 571 | } |
| 350 | 572 | |
| 351 | 573 | /** |
| 352 | 574 | * Resolve dimensions for externally hosted images found on a warmed page. |
| @@ -606,9 +828,10 @@ | ||
| 606 | 828 | $sitemap_url, |
| 607 | 829 | array( |
| 608 | 830 | 'timeout' => self::REQUEST_TIMEOUT, |
| 609 | 831 | 'sslverify' => false, |
| 610 | - 'user-agent' => self::USER_AGENT, | |
| 832 | + 'user-agent' => self::user_agent(), | |
| 833 | + 'headers' => Self_Traffic::headers(), | |
| 611 | 834 | ) |
| 612 | 835 | ); |
| 613 | 836 | if ( is_wp_error( $res ) ) { |
| 614 | 837 | // Record WHY, don't just vanish. "Unreachable" and "valid but |
