| @@ -46,13 +46,26 @@ | ||
| 46 | 46 | * in-request copy from a non-persistent cache can't shadow the DB value. |
| 47 | 47 | */ |
| 48 | 48 | public const OPT_KEY = 'xspeed_hit_buffer'; |
| 49 | 49 | |
| 50 | + /** Daily hit/miss aggregates (option, autoload off): 'Y-m-d' => {hits,misses}. */ | |
| 51 | + public const DAILY_OPT = 'xspeed_hit_daily'; | |
| 52 | + | |
| 53 | + /** Days of daily history to retain (the trend UI reads 7/30). */ | |
| 54 | + public const DAILY_MAX_DAYS = 120; | |
| 55 | + | |
| 50 | 56 | /** |
| 51 | - * @var array<int,int> Pending increments keyed by metric ('hit'|'miss'). | |
| 52 | - * Flushed to the transient on shutdown. | |
| 57 | + * @var array<string,int> Pending increments keyed by metric | |
| 58 | + * ('hit'|'miss'|'excluded'). Flushed on shutdown. | |
| 59 | + * `excluded` = requests that reached the render path | |
| 60 | + * but must NOT count toward cache performance — | |
| 61 | + * 404s and known-bot/scanner traffic (#118). | |
| 53 | 62 | */ |
| 54 | - private static $pending = array( 'hit' => 0, 'miss' => 0 ); | |
| 63 | + private static $pending = array( | |
| 64 | + 'hit' => 0, | |
| 65 | + 'miss' => 0, | |
| 66 | + 'excluded' => 0, | |
| 67 | + ); | |
| 55 | 68 | |
| 56 | 69 | /** |
| 57 | 70 | * @var bool Whether the shutdown flush is already registered. |
| 58 | 71 | */ |
| @@ -62,8 +75,70 @@ | ||
| 62 | 75 | ++self::$pending['hit']; |
| 63 | 76 | self::ensure_shutdown_flush(); |
| 64 | 77 | } |
| 65 | 78 | |
| 79 | + /** | |
| 80 | + * Record a request that reached the render path but must NOT count toward | |
| 81 | + * the hit ratio — a 404 or known-bot/scanner request. Kept as a separate | |
| 82 | + * line item ("you absorbed N scanner hits today") rather than polluting the | |
| 83 | + * cache-performance denominator, which a wave of `/wp-x7.php` 404s otherwise | |
| 84 | + * craters. Flushed inline like a miss so it's never lost. (#118) | |
| 85 | + */ | |
| 86 | + public static function record_excluded(): void { | |
| 87 | + ++self::$pending['excluded']; | |
| 88 | + self::flush_pending(); | |
| 89 | + } | |
| 90 | + | |
| 91 | + /** | |
| 92 | + * The bot / crawler / scanner alternation, without delimiters so the | |
| 93 | + * drop-in can compose it — see excluded_ua_regex(). | |
| 94 | + */ | |
| 95 | + public const BOT_UA_PATTERN = 'bot|crawl|spider|slurp|scan|curl|wget|python-requests|python-urllib|libwww|httpclient|go-http|okhttp|axios|node-fetch|headless|phantomjs|masscan|nikto|sqlmap|zgrab|semrush|ahrefs|mj12|dotbot|petalbot|bytespider|facebookexternalhit|preview|monitor|uptime|pingdom|gtmetrix|lighthouse|pagespeed'; | |
| 96 | + | |
| 97 | + /** | |
| 98 | + * Whether a User-Agent is a known bot / crawler / vulnerability scanner — | |
| 99 | + * its cache misses are cache-warming or hostile noise, not a signal of how | |
| 100 | + * the cache serves real visitors. Deliberately broad: matches the common | |
| 101 | + * crawler tokens plus the generic markers scanners and libraries carry. | |
| 102 | + * Unit-tested; no longer pure — Self_Traffic::is_self() runs the | |
| 103 | + * xspeed_self_user_agents filter, so the answer can vary per site. (#118) | |
| 104 | + */ | |
| 105 | + public static function is_bot_ua( string $ua ): bool { | |
| 106 | + if ( '' === $ua ) { | |
| 107 | + // No UA at all is overwhelmingly automated traffic, not a browser. | |
| 108 | + return true; | |
| 109 | + } | |
| 110 | + // Our own warmer, benchmark and verifier are warming the cache, not | |
| 111 | + // visiting it: `xSpeed-Warmer`, `xSpeed Benchmark`, and the rest. | |
| 112 | + // Callers with a request also check Self_Traffic::request_is_marked(). | |
| 113 | + if ( Self_Traffic::is_self( $ua ) ) { | |
| 114 | + return true; | |
| 115 | + } | |
| 116 | + return 1 === preg_match( '~(' . self::BOT_UA_PATTERN . ')~i', $ua ); | |
| 117 | + } | |
| 118 | + | |
| 119 | + /** | |
| 120 | + * The "do not count this user agent" alternation: bots and scanners, | |
| 121 | + * plus the fragments xSpeed's own requests carry. A renamed warmer is | |
| 122 | + * not in it on purpose; that request is recognised by | |
| 123 | + * Self_Traffic::HEADER, because its UA may be a real browser's. | |
| 124 | + * | |
| 125 | + * Baked into the drop-in at install time (`@@XSPEED_HIT_EXCLUDE_RE@@`). | |
| 126 | + * The drop-in runs before WordPress, so it cannot ask this class and the | |
| 127 | + * hits.log line it writes carries no user agent — nothing downstream can | |
| 128 | + * reclassify the line later, which is why the decision has to travel | |
| 129 | + * with the file. A hardcoded copy of the fragments drifted instead: it | |
| 130 | + * excluded the warmer but still counted every crawler HIT, and it could | |
| 131 | + * not know about an overridden `xspeed_preloader_user_agent`. | |
| 132 | + */ | |
| 133 | + public static function excluded_ua_regex(): string { | |
| 134 | + $parts = array( self::BOT_UA_PATTERN ); | |
| 135 | + foreach ( Self_Traffic::agents() as $agent ) { | |
| 136 | + $parts[] = preg_quote( $agent, '#' ); | |
| 137 | + } | |
| 138 | + return implode( '|', $parts ); | |
| 139 | + } | |
| 140 | + | |
| 66 | 141 | public static function record_miss(): void { |
| 67 | 142 | ++self::$pending['miss']; |
| 68 | 143 | // Flush misses INLINE, not at shutdown. A MISS is recorded ONLY here |
| 69 | 144 | // (HITs additionally have the durable hits.log drain as a backstop), |
| @@ -189,8 +264,15 @@ | ||
| 189 | 264 | // .htaccess can't header/log a static serve — see |
| 190 | 265 | // Cache::static_rewrite_allowed(). So Apache is the lone server that |
| 191 | 266 | // serves static hits below PHP yet logs them to the SERVER's access |
| 192 | 267 | // log, which is what we scan here. |
| 268 | + // | |
| 269 | + // LiteSpeed stays out even with the Static Fast Path opt-in (#509): | |
| 270 | + // its access log records the ORIGINAL request line ("GET / …"), not | |
| 271 | + // the rewritten static-file path, so the needle below can never | |
| 272 | + // match and scanning would only pretend to count. Verified on | |
| 273 | + // OpenLiteSpeed 1.8. Those hits are genuinely uncounted, which the | |
| 274 | + // dashboard discloses via stats.static_hits_uncounted. | |
| 193 | 275 | if ( Server::APACHE !== Server::type() ) { |
| 194 | 276 | return 0; |
| 195 | 277 | } |
| 196 | 278 | |
| @@ -268,9 +350,9 @@ | ||
| 268 | 350 | private static function read_buffer() { |
| 269 | 351 | // Drop the cached 'options' entry for our key so get_option() falls |
| 270 | 352 | // through to the DB. Harmless on a persistent cache (it just reloads |
| 271 | 353 | // from the DB once); essential on a non-persistent one. |
| 272 | - wp_cache_delete( self::OPT_KEY, 'options' ); | |
| 354 | + \wp_cache_delete( self::OPT_KEY, 'options' ); | |
| 273 | 355 | return get_option( self::OPT_KEY, array() ); |
| 274 | 356 | } |
| 275 | 357 | |
| 276 | 358 | private static function write_buffer( array $buf ): void { |
| @@ -292,11 +374,13 @@ | ||
| 292 | 374 | $out = array(); |
| 293 | 375 | foreach ( $buf as $b ) { |
| 294 | 376 | if ( is_array( $b ) && isset( $b['ts'], $b['hits'], $b['misses'] ) ) { |
| 295 | 377 | $out[] = array( |
| 296 | - 'ts' => (int) $b['ts'], | |
| 297 | - 'hits' => (int) $b['hits'], | |
| 298 | - 'misses' => (int) $b['misses'], | |
| 378 | + 'ts' => (int) $b['ts'], | |
| 379 | + 'hits' => (int) $b['hits'], | |
| 380 | + 'misses' => (int) $b['misses'], | |
| 381 | + // Older buckets (pre-#118) have no 'excluded' key — default 0. | |
| 382 | + 'excluded' => (int) ( $b['excluded'] ?? 0 ), | |
| 299 | 383 | ); |
| 300 | 384 | } |
| 301 | 385 | } |
| 302 | 386 | return $out; |
| @@ -302,32 +386,48 @@ | ||
| 302 | 386 | return $out; |
| 303 | 387 | } |
| 304 | 388 | |
| 305 | 389 | /** |
| 306 | - * Totals over the last 24h (sum across all buckets). | |
| 390 | + * Totals over the last 24h (sum across all buckets). `ratio` is computed | |
| 391 | + * over hits + real misses only; `excluded` (404s + bots) is reported | |
| 392 | + * alongside but kept OUT of the denominator so a scanner flood can't crater | |
| 393 | + * the number. (#118) | |
| 307 | 394 | * |
| 308 | - * @return array{hits:int,misses:int,ratio:float} | |
| 395 | + * @return array{hits:int,misses:int,excluded:int,ratio:float} | |
| 309 | 396 | */ |
| 310 | 397 | public static function totals_24h(): array { |
| 311 | - $buckets = self::buckets(); | |
| 312 | - $hits = 0; | |
| 313 | - $misses = 0; | |
| 398 | + $buckets = self::buckets(); | |
| 399 | + $hits = 0; | |
| 400 | + $misses = 0; | |
| 401 | + $excluded = 0; | |
| 314 | 402 | foreach ( $buckets as $b ) { |
| 315 | - $hits += $b['hits']; | |
| 316 | - $misses += $b['misses']; | |
| 403 | + $hits += $b['hits']; | |
| 404 | + $misses += $b['misses']; | |
| 405 | + $excluded += $b['excluded']; | |
| 317 | 406 | } |
| 318 | 407 | $total = $hits + $misses; |
| 319 | 408 | return array( |
| 320 | - 'hits' => $hits, | |
| 321 | - 'misses' => $misses, | |
| 322 | - 'ratio' => $total > 0 ? round( $hits / $total, 4 ) : 0.0, | |
| 409 | + 'hits' => $hits, | |
| 410 | + 'misses' => $misses, | |
| 411 | + 'excluded' => $excluded, | |
| 412 | + 'ratio' => $total > 0 ? round( $hits / $total, 4 ) : 0.0, | |
| 323 | 413 | ); |
| 324 | 414 | } |
| 325 | 415 | |
| 326 | 416 | public static function reset(): void { |
| 327 | 417 | delete_transient( self::TRANSIENT_KEY ); |
| 418 | + // The bucket buffer lives in the OPT_KEY option (migrated off the | |
| 419 | + // transient); reset() must clear it too, or record→reset leaves the | |
| 420 | + // old hit/miss buckets behind and buckets() still reports them. | |
| 421 | + delete_option( self::OPT_KEY ); | |
| 422 | + \wp_cache_delete( self::OPT_KEY, 'options' ); | |
| 328 | 423 | delete_option( self::SERVER_LOG_OFFSET_OPT ); |
| 329 | - self::$pending = array( 'hit' => 0, 'miss' => 0 ); | |
| 424 | + delete_option( self::DAILY_OPT ); | |
| 425 | + self::$pending = array( | |
| 426 | + 'hit' => 0, | |
| 427 | + 'miss' => 0, | |
| 428 | + 'excluded' => 0, | |
| 429 | + ); | |
| 330 | 430 | } |
| 331 | 431 | |
| 332 | 432 | /** |
| 333 | 433 | * One-shot register on first record_* call this request. |
| @@ -346,12 +446,16 @@ | ||
| 346 | 446 | * MAX_BUCKETS. |
| 347 | 447 | */ |
| 348 | 448 | public static function flush_pending(): void { |
| 349 | 449 | $pending = self::$pending; |
| 350 | - if ( 0 === $pending['hit'] && 0 === $pending['miss'] ) { | |
| 450 | + if ( 0 === $pending['hit'] && 0 === $pending['miss'] && 0 === $pending['excluded'] ) { | |
| 351 | 451 | return; |
| 352 | 452 | } |
| 353 | - self::$pending = array( 'hit' => 0, 'miss' => 0 ); | |
| 453 | + self::$pending = array( | |
| 454 | + 'hit' => 0, | |
| 455 | + 'miss' => 0, | |
| 456 | + 'excluded' => 0, | |
| 457 | + ); | |
| 354 | 458 | |
| 355 | 459 | $hour = (int) ( time() - ( time() % 3600 ) ); |
| 356 | 460 | $buf = self::buckets(); |
| 357 | 461 | $last = end( $buf ); |
| @@ -357,18 +461,21 @@ | ||
| 357 | 461 | $last = end( $buf ); |
| 358 | 462 | $updated = false; |
| 359 | 463 | |
| 360 | 464 | if ( $last && $last['ts'] === $hour ) { |
| 361 | - $buf[ count( $buf ) - 1 ]['hits'] += $pending['hit']; | |
| 362 | - $buf[ count( $buf ) - 1 ]['misses'] += $pending['miss']; | |
| 363 | - $updated = true; | |
| 465 | + $i = count( $buf ) - 1; | |
| 466 | + $buf[ $i ]['hits'] += $pending['hit']; | |
| 467 | + $buf[ $i ]['misses'] += $pending['miss']; | |
| 468 | + $buf[ $i ]['excluded'] += $pending['excluded']; | |
| 469 | + $updated = true; | |
| 364 | 470 | } |
| 365 | 471 | |
| 366 | 472 | if ( ! $updated ) { |
| 367 | 473 | $buf[] = array( |
| 368 | - 'ts' => $hour, | |
| 369 | - 'hits' => $pending['hit'], | |
| 370 | - 'misses' => $pending['miss'], | |
| 474 | + 'ts' => $hour, | |
| 475 | + 'hits' => $pending['hit'], | |
| 476 | + 'misses' => $pending['miss'], | |
| 477 | + 'excluded' => $pending['excluded'], | |
| 371 | 478 | ); |
| 372 | 479 | while ( count( $buf ) > self::MAX_BUCKETS ) { |
| 373 | 480 | array_shift( $buf ); |
| 374 | 481 | } |
| @@ -374,6 +481,71 @@ | ||
| 374 | 481 | } |
| 375 | 482 | } |
| 376 | 483 | |
| 377 | 484 | self::write_buffer( $buf ); |
| 485 | + self::bump_daily( $pending['hit'], $pending['miss'], $pending['excluded'] ); | |
| 486 | + } | |
| 487 | + | |
| 488 | + /** | |
| 489 | + * Fold the just-flushed counts into the persistent daily series. The | |
| 490 | + * hourly buckets expire after ~25h; this option is what makes 7/30-day | |
| 491 | + * hit-ratio trends possible (issue #44). Autoload off — it's only read | |
| 492 | + * by the dashboard/REST, never on the frontend hot path. | |
| 493 | + */ | |
| 494 | + private static function bump_daily( int $hits, int $misses, int $excluded = 0 ): void { | |
| 495 | + if ( $hits <= 0 && $misses <= 0 && $excluded <= 0 ) { | |
| 496 | + return; | |
| 497 | + } | |
| 498 | + $day = gmdate( 'Y-m-d' ); | |
| 499 | + $series = get_option( self::DAILY_OPT, array() ); | |
| 500 | + if ( ! is_array( $series ) ) { | |
| 501 | + $series = array(); | |
| 502 | + } | |
| 503 | + if ( ! isset( $series[ $day ] ) || ! is_array( $series[ $day ] ) ) { | |
| 504 | + $series[ $day ] = array( | |
| 505 | + 'hits' => 0, | |
| 506 | + 'misses' => 0, | |
| 507 | + 'excluded' => 0, | |
| 508 | + ); | |
| 509 | + } | |
| 510 | + $series[ $day ]['hits'] += $hits; | |
| 511 | + $series[ $day ]['misses'] += $misses; | |
| 512 | + $series[ $day ]['excluded'] = (int) ( $series[ $day ]['excluded'] ?? 0 ) + $excluded; | |
| 513 | + if ( count( $series ) > self::DAILY_MAX_DAYS ) { | |
| 514 | + ksort( $series ); | |
| 515 | + $series = array_slice( $series, -self::DAILY_MAX_DAYS, null, true ); | |
| 516 | + } | |
| 517 | + update_option( self::DAILY_OPT, $series, false ); | |
| 518 | + } | |
| 519 | + | |
| 520 | + /** | |
| 521 | + * The stored daily hit/miss series, oldest→newest, at most $days rows. | |
| 522 | + * | |
| 523 | + * @return array<int,array{date:string,hits:int,misses:int,ratio:float}> | |
| 524 | + */ | |
| 525 | + public static function daily_series( int $days = 30 ): array { | |
| 526 | + $series = get_option( self::DAILY_OPT, array() ); | |
| 527 | + if ( ! is_array( $series ) || empty( $series ) ) { | |
| 528 | + return array(); | |
| 529 | + } | |
| 530 | + ksort( $series ); | |
| 531 | + $series = array_slice( $series, -max( 1, $days ), null, true ); | |
| 532 | + $out = array(); | |
| 533 | + foreach ( $series as $date => $row ) { | |
| 534 | + if ( ! is_array( $row ) ) { | |
| 535 | + continue; | |
| 536 | + } | |
| 537 | + $hits = (int) ( $row['hits'] ?? 0 ); | |
| 538 | + $misses = (int) ( $row['misses'] ?? 0 ); | |
| 539 | + $excluded = (int) ( $row['excluded'] ?? 0 ); | |
| 540 | + $total = $hits + $misses; | |
| 541 | + $out[] = array( | |
| 542 | + 'date' => (string) $date, | |
| 543 | + 'hits' => $hits, | |
| 544 | + 'misses' => $misses, | |
| 545 | + 'excluded' => $excluded, | |
| 546 | + 'ratio' => $total > 0 ? round( $hits / $total, 4 ) : 0.0, | |
| 547 | + ); | |
| 548 | + } | |
| 549 | + return $out; | |
| 378 | 550 | } |
| 379 | 551 | } |
