metasync / otto / class-metasync-otto-bot-detector.php

class-metasync-otto-bot-detector.php in Search Atlas SEO – OTTO AI SEO Automation for WordPress 2.7.2, at otto/class-metasync-otto-bot-detector.php

744 lines 28.3 KB

Raw Download Zip

1	<?php
2	/**
3	* Bot Detection Service for OTTO
4	*
5	* Detects and identifies search engine crawlers and bots to allow
6	* selective processing of OTTO requests based on traffic source.
7	*
8	* @package Metasync
9	* @subpackage Metasync/otto
10	* @since 1.0.0
11	*/
12
13	// If this file is called directly, abort.
14	if (!defined('ABSPATH')) {
15	exit;
16	}
17
18	/**
19	* Bot Detection Service Class
20	*
21	* Provides bot detection functionality with whitelisting and blacklisting support.
22	* Implements singleton pattern for efficient memory usage.
23	*
24	* @since 1.0.0
25	*/
26	class Metasync_Otto_Bot_Detector {
27
28	/**
29	* Singleton instance
30	*
31	* @var Metasync_Otto_Bot_Detector\|null
32	*/
33	private static $instance = null;
34
35	/**
36	* Known bot signatures, each mapped to an explicit category.
37	*
38	* The key is a user-agent substring matched case-insensitively; the value is
39	* the category. Every entry carries an intentional category — nothing relies
40	* on an implicit fallthrough — because the category decides whether OTTO's
41	* render throttle applies (see metasync_start_otto() in otto_pixel.php).
42	*
43	* Categories, and the policy each one encodes:
44	*
45	* search_engine Never throttled. A crawler that builds a search index we
46	* want to rank in must always receive the full OTTO output,
47	* or it indexes un-optimized HTML.
48	* cache_warmer Never throttled. Host page-cache preloaders exist purely
49	* to populate the cache; a throttled response also sets
50	* DONOTCACHEPAGE, so throttling one means the cache can
51	* never fill and every human visitor pays an uncached render.
52	* perf_auditor Never throttled. Synthetic auditors are what customers
53	* measure the site with, so they must see what a real
54	* visitor sees.
55	* seo_tool Throttled, deliberately. Third-party SEO crawlers do not
56	* feed any search index, so serving them a throttled page
57	* cannot affect rankings, and their crawl volume is high
58	* enough that exempting them would defeat the throttle.
59	* social_media Throttled. Share-preview crawlers read the OG/Twitter
60	* tags, which the throttled response still contains.
61	* archiver Throttled. Snapshot/AI-training crawlers do not influence
62	* ranking for this site.
63	* generic Throttled. Unrecognized or low-value crawlers.
64	*
65	* Order matters: the first substring that matches wins, so a more specific
66	* signature must precede a broader one that would also match it
67	* (e.g. 'Applebot-Extended' before 'Applebot').
68	*
69	* Note that no needle is a bare brand word. Matching on bare brands is what
70	* previously mis-sorted crawlers whose product token omits the brand name
71	* (Yahoo's 'Slurp', Sogou, Exalead's 'Exabot', Apple's 'Applebot').
72	*
73	* Reverse-DNS verification of the Googlebot/Bingbot claim is deliberately NOT
74	* performed here. detect() runs inline on every front-end request, and a
75	* reverse lookup costs a blocking DNS round trip — on a timeout, seconds —
76	* on the render-critical path, which is a worse regression than the spoofing
77	* it would prevent. Spoofing a search-engine UA only buys an attacker the
78	* unthrottled render path (more of our CPU), not content access or data, so
79	* the trade is not worth paying on every request. Doing it safely needs a
80	* cached, non-blocking resolver, which is left to dedicated work; the
81	* coarse is_bot_ip() range check remains the cheap first-pass filter.
82	*
83	* @var array<string, string> Map of user-agent substring => category.
84	*/
85	private $common_bots = array(
86	// Search engines — crawlers that build an index we want to rank in.
87	'Googlebot' => 'search_engine',
88	'Googlebot-Image' => 'search_engine',
89	'Googlebot-News' => 'search_engine',
90	'Googlebot-Video' => 'search_engine',
91	'Google-InspectionTool' => 'search_engine', // Search Console live test
92	'Storebot-Google' => 'search_engine',
93	'GoogleOther' => 'search_engine',
94	'AdsBot-Google' => 'search_engine',
95	'Mediapartners-Google' => 'search_engine',
96	'APIs-Google' => 'search_engine',
97	'Bingbot' => 'search_engine',
98	'BingPreview' => 'search_engine',
99	'Slurp' => 'search_engine', // Yahoo
100	'DuckDuckBot' => 'search_engine', // DuckDuckGo
101	'Baiduspider' => 'search_engine', // Baidu
102	'YandexBot' => 'search_engine', // Yandex
103	'Sogou' => 'search_engine', // Sogou
104	'Exabot' => 'search_engine', // Exalead
105	'Applebot-Extended' => 'archiver', // Apple AI training, not search
106	'Applebot' => 'search_engine', // Apple (Siri/Spotlight)
107	'PetalBot' => 'search_engine', // Huawei Petal Search
108	'Qwantify' => 'search_engine', // Qwant
109	'SeekportBot' => 'search_engine', // Seekport
110
111	// Cache warmers — host preloaders that populate the page cache.
112	'PageCacheBot' => 'cache_warmer', // WP Cloud (Desktop\|Mobile)
113	'WP Rocket' => 'cache_warmer', // WP Rocket preload
114	'wprocket' => 'cache_warmer',
115	'lscache_runner' => 'cache_warmer', // LiteSpeed Cache crawler
116	'SG Optimizer' => 'cache_warmer', // SiteGround
117	'sg-optimizer' => 'cache_warmer',
118	'NitroPack' => 'cache_warmer',
119
120	// Performance auditors — what customers measure the site with.
121	'Chrome-Lighthouse' => 'perf_auditor',
122	'lighthouse' => 'perf_auditor',
123	'pagespeed' => 'perf_auditor', // PageSpeed Insights
124	'GTmetrix' => 'perf_auditor',
125	'WebPageTest' => 'perf_auditor',
126	'Pingdom' => 'perf_auditor',
127
128	// SEO tools — throttled on purpose; they feed no search index.
129	'AhrefsBot' => 'seo_tool',
130	'SemrushBot' => 'seo_tool',
131	'MJ12bot' => 'seo_tool', // Majestic
132	'DotBot' => 'seo_tool', // Moz
133	'rogerbot' => 'seo_tool', // Moz
134	'Screaming Frog' => 'seo_tool',
135	'SiteAuditBot' => 'seo_tool',
136	'serpstatbot' => 'seo_tool',
137	'DataForSeoBot' => 'seo_tool',
138	'SEOkicks' => 'seo_tool',
139	'SiteImprove' => 'seo_tool',
140
141	// Social media — share-preview crawlers.
142	'facebot' => 'social_media', // Facebook
143	'facebookexternalhit' => 'social_media',
144	'LinkedInBot' => 'social_media',
145	'Twitterbot' => 'social_media',
146	'PinterestBot' => 'social_media',
147
148	// Archivers and snapshot/AI-training crawlers.
149	'ia_archiver' => 'archiver', // Internet Archive / Alexa
150	'archive.org_bot' => 'archiver',
151	'ArchiveBot' => 'archiver',
152	'CCBot' => 'archiver', // Common Crawl
153
154	// Recognized but low-value crawlers.
155	'BLEXBot' => 'generic',
156	'MauiBot' => 'generic',
157	'AlphaBot' => 'generic',
158	'ZoominfoBot' => 'generic',
159	'WhatWeb' => 'generic',
160	);
161
162	/**
163	* Generic bot patterns
164	*
165	* @var array
166	*/
167	private $bot_patterns = array(
168	'/bot/i',
169	'/crawl/i',
170	'/spider/i',
171	'/slurp/i',
172	'/mediapartners/i',
173	'/scanner/i',
174	'/scraper/i',
175	'/checker/i',
176	'/validator/i',
177	'/fetcher/i',
178	'/archiver/i',
179	'/lighthouse/i',
180	'/pagespeed/i'
181	);
182
183	/**
184	* IP-based bot detection patterns
185	* Common IP ranges for known bots
186	*
187	* @var array
188	*/
189	private $bot_ip_ranges = array(
190	// Google IP ranges (partial list - full list is extensive)
191	'66.249.',
192	'66.102.',
193	'64.233.',
194	'72.14.',
195	'74.125.',
196	'209.85.',
197	'216.239.',
198	// Bing IP ranges
199	'40.77.',
200	'157.55.',
201	'207.46.',
202	// Yandex
203	'5.255.',
204	'77.88.',
205	'95.108.',
206	'100.43.',
207	'130.193.',
208	'141.8.',
209	'178.154.'
210	);
211
212	/**
213	* Current detection result cache
214	*
215	* @var array\|null
216	*/
217	private $detection_cache = null;
218
219	/**
220	* Get singleton instance
221	*
222	* @return Metasync_Otto_Bot_Detector
223	*/
224	public static function get_instance() {
225	if (self::$instance === null) {
226	self::$instance = new self();
227	}
228	return self::$instance;
229	}
230
231	/**
232	* Private constructor to prevent direct instantiation
233	*/
234	private function __construct() {
235	// Private constructor for singleton
236	}
237
238	/**
239	* Prevent cloning of the instance
240	*/
241	private function __clone() {
242	// Prevent cloning
243	}
244
245	/**
246	* Prevent unserialization of the instance
247	*/
248	public function __wakeup() {
249	// Prevent unserialization
250	}
251
252	/**
253	* Detect if current request is from a bot
254	*
255	* @param string\|null $user_agent Optional user agent to check (defaults to current request)
256	* @param string\|null $ip Optional IP address to check (defaults to current request)
257	* @return array Detection result with bot status and details
258	*/
259	public function detect($user_agent = null, $ip = null) {
260	// Use cache if available and no custom parameters provided
261	if ($this->detection_cache !== null && $user_agent === null && $ip === null) {
262	return $this->detection_cache;
263	}
264
265	// Get user agent from request if not provided
266	if ($user_agent === null) {
267	$user_agent = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '';
268	}
269
270	// Get IP from request if not provided
271	if ($ip === null) {
272	$ip = $this->get_client_ip();
273	}
274
275	// Initialize result
276	$result = array(
277	'is_bot' => false,
278	'bot_name' => null,
279	'bot_type' => null,
280	'user_agent' => $user_agent,
281	'ip_address' => $ip,
282	'detection_method' => null
283	);
284
285	// Check if user agent is empty (likely a bot)
286	if (empty($user_agent)) {
287	$result['is_bot'] = true;
288	$result['bot_name'] = 'Unknown Bot';
289	$result['bot_type'] = 'unknown';
290	$result['detection_method'] = 'empty_user_agent';
291
292	// Cache and return
293	$this->detection_cache = $result;
294	return $result;
295	}
296
297	// Check against the known bot signatures.
298	$match = $this->match_bot_signature($user_agent);
299	if ($match !== null) {
300	$result['is_bot'] = true;
301	$result['bot_name'] = $match[0];
302	$result['bot_type'] = $match[1];
303	$result['detection_method'] = 'user_agent_match';
304
305	// Cache and return
306	$this->detection_cache = $result;
307	return $result;
308	}
309
310	// Check against bot patterns
311	foreach ($this->bot_patterns as $pattern) {
312	if (preg_match($pattern, $user_agent)) {
313	$result['is_bot'] = true;
314	$result['bot_name'] = $this->extract_bot_name($user_agent);
315	$result['bot_type'] = 'generic';
316	$result['detection_method'] = 'pattern_match';
317
318	// Cache and return
319	$this->detection_cache = $result;
320	return $result;
321	}
322	}
323
324	// Check IP-based detection
325	if ($this->is_bot_ip($ip)) {
326	$result['is_bot'] = true;
327	$result['bot_name'] = 'Bot (IP-based detection)';
328	$result['bot_type'] = 'ip_based';
329	$result['detection_method'] = 'ip_range_match';
330
331	// Cache and return
332	$this->detection_cache = $result;
333	return $result;
334	}
335
336	// Cache and return
337	$this->detection_cache = $result;
338	return $result;
339	}
340
341	/**
342	* Check if current request should skip OTTO processing
343	* Takes into account settings, whitelist, and blacklist
344	*
345	* @return bool True if OTTO should be skipped, false otherwise
346	*/
347	public function should_skip_otto() {
348	// Get plugin settings
349	$options = get_option('metasync_options', array());
350	$general = isset($options['general']) ? $options['general'] : array();
351
352	// Check if bot filtering is enabled
353	$disable_for_bots = isset($general['otto_disable_for_bots']) ? (bool)$general['otto_disable_for_bots'] : false;
354
355	if (!$disable_for_bots) {
356	return false; // Feature is disabled, don't skip
357	}
358
359	// Detect bot
360	$detection = $this->detect();
361
362	if (!$detection['is_bot']) {
363	return false; // Not a bot, don't skip
364	}
365
366	// Check whitelist
367	$whitelist = isset($general['otto_bot_whitelist']) ? $general['otto_bot_whitelist'] : '';
368	if (!empty($whitelist)) {
369	$whitelist_array = array_map('trim', explode("\n", $whitelist));
370	foreach ($whitelist_array as $whitelisted_bot) {
371	if (empty($whitelisted_bot)) continue;
372
373	if (stripos($detection['user_agent'], $whitelisted_bot) !== false \|\|
374	stripos($detection['bot_name'], $whitelisted_bot) !== false) {
375	return false; // Bot is whitelisted, don't skip
376	}
377	}
378	}
379
380	// Check blacklist
381	$blacklist = isset($general['otto_bot_blacklist']) ? $general['otto_bot_blacklist'] : '';
382	if (!empty($blacklist)) {
383	$blacklist_array = array_map('trim', explode("\n", $blacklist));
384	foreach ($blacklist_array as $blacklisted_pattern) {
385	if (empty($blacklisted_pattern)) continue;
386
387	if (stripos($detection['user_agent'], $blacklisted_pattern) !== false \|\|
388	stripos($detection['bot_name'], $blacklisted_pattern) !== false) {
389	return true; // Bot is explicitly blacklisted, skip OTTO
390	}
391	}
392	}
393
394	// Bot detected and no whitelist override, skip OTTO
395	return true;
396	}
397
398	/**
399	* Find the first known bot signature contained in a user agent.
400	*
401	* Single source of truth for turning a user agent into a signature and a
402	* category: both detect() and categorize_bot() resolve through this method,
403	* so the two can never disagree.
404	*
405	* @param string $user_agent Raw user agent string.
406	* @return array\|null array(0 => matched signature, 1 => category), or null when
407	* no known signature is present.
408	*/
409	private function match_bot_signature($user_agent) {
410	// categorize_bot() is public, so loose input is normalized here rather
411	// than letting a non-string reach stripos(), where passing null has been
412	// deprecated since PHP 8.1.
413	$user_agent = (string) $user_agent;
414
415	if ($user_agent === '') {
416	return null;
417	}
418
419	foreach ($this->common_bots as $signature => $category) {
420	if (stripos($user_agent, $signature) !== false) {
421	return array($signature, $category);
422	}
423	}
424
425	return null;
426	}
427
428	/**
429	* Categorize a bot from its user agent.
430	*
431	* Matching is performed against the user agent itself, using the explicit
432	* $common_bots map. It deliberately does NOT match brand names against the
433	* detected bot's name: a crawler's product token frequently omits its
434	* brand (Yahoo ships 'Slurp', Exalead 'Exabot', Apple 'Applebot'), so
435	* brand matching silently mis-sorted real search engines into the throttled
436	* bucket and served them un-optimized HTML.
437	*
438	* An unrecognized crawler is 'generic', which is throttled — the safe
439	* default, since every category that bypasses the throttle is enumerated.
440	*
441	* @param string $user_agent Raw user agent string.
442	* @return string One of: search_engine, cache_warmer, perf_auditor, seo_tool,
443	* social_media, archiver, generic.
444	*/
445	public function categorize_bot($user_agent) {
446	$match = $this->match_bot_signature($user_agent);
447
448	return $match === null ? 'generic' : $match[1];
449	}
450
451	/**
452	* Categories whose requests must never hit OTTO's render throttle.
453	*
454	* Exposed so the throttle gate in otto_pixel.php and this classifier cannot
455	* drift apart: the gate asks this method rather than hard-coding the list.
456	*
457	* @return string[] Category names exempt from the render throttle.
458	*/
459	public static function get_unthrottled_categories() {
460	return array('search_engine', 'cache_warmer', 'perf_auditor');
461	}
462
463	/**
464	* Whether a detected category is exempt from OTTO's render throttle.
465	*
466	* @param string\|null $bot_type Category from detect()'s 'bot_type'.
467	* @return bool True when the request must bypass the render throttle.
468	*/
469	public static function is_unthrottled_category($bot_type) {
470	return in_array($bot_type, self::get_unthrottled_categories(), true);
471	}
472
473	/**
474	* Extract bot name from user agent string
475	*
476	* @param string $user_agent User agent string
477	* @return string Extracted bot name
478	*/
479	private function extract_bot_name($user_agent) {
480	// Try to extract a meaningful name from the user agent
481	if (preg_match('/([a-zA-Z]+[bB]ot[a-zA-Z]*)/i', $user_agent, $matches)) {
482	return $matches[1];
483	}
484
485	if (preg_match('/([a-zA-Z]+[sS]pider)/i', $user_agent, $matches)) {
486	return $matches[1];
487	}
488
489	if (preg_match('/([a-zA-Z]+[cC]rawler)/i', $user_agent, $matches)) {
490	return $matches[1];
491	}
492
493	return 'Unknown Bot';
494	}
495
496	/**
497	* Check if IP belongs to a known bot
498	*
499	* @param string $ip IP address
500	* @return bool True if IP is from a known bot
501	*/
502	private function is_bot_ip($ip) {
503	if (empty($ip)) {
504	return false;
505	}
506
507	foreach ($this->bot_ip_ranges as $range) {
508	if (strpos($ip, $range) === 0) {
509	return true;
510	}
511	}
512
513	return false;
514	}
515
516	/**
517	* Get client IP address.
518	*
519	* Resolves the visitor's IP using a Cloudflare-first priority order so that
520	* attacker-controlled headers cannot be used to spoof crawler IPs and bypass
521	* the IP-range step of bot detection:
522	*
523	* 1. HTTP_CF_CONNECTING_IP — trusted, but only when HTTP_CF_RAY confirms
524	* the request actually traversed Cloudflare.
525	* 2. HTTP_X_FORWARDED_FOR — fallback for non-Cloudflare reverse proxies.
526	* The chain is walked right-to-left and the first public IP is used;
527	* the leftmost value is attacker-controlled and must not be trusted.
528	* 3. REMOTE_ADDR — TCP peer; final fallback for direct (non-proxied) hits.
529	*
530	* Non-standard, attacker-controllable headers are intentionally not
531	* consulted at any point in this resolution chain.
532	*
533	* @return string Client IP address, or '' if none could be validated.
534	*/
535	private function get_client_ip() {
536	// 1. Cloudflare-proxied request: HTTP_CF_RAY is the canonical marker
537	// that Cloudflare actually handled the connection, which means
538	// HTTP_CF_CONNECTING_IP was set by Cloudflare and is trustworthy.
539	if (!empty($_SERVER['HTTP_CF_RAY']) && !empty($_SERVER['HTTP_CF_CONNECTING_IP'])) {
540	$cf_ip = $_SERVER['HTTP_CF_CONNECTING_IP'];
541	if (filter_var($cf_ip, FILTER_VALIDATE_IP)) {
542	return $cf_ip;
543	}
544	}
545
546	// 2. Non-Cloudflare reverse proxy: walk X-Forwarded-For right-to-left
547	// and return the first public IP. The trusted proxy appends to the
548	// right; everything to the left of that may have been supplied by
549	// the client and cannot be relied on.
550	if (!empty($_SERVER['HTTP_X_FORWARDED_FOR'])) {
551	$ips = array_reverse(explode(',', $_SERVER['HTTP_X_FORWARDED_FOR']));
552	foreach ($ips as $candidate) {
553	$candidate = trim($candidate);
554	if ($candidate === '') {
555	continue;
556	}
557	if (filter_var($candidate, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE \| FILTER_FLAG_NO_RES_RANGE)) {
558	return $candidate;
559	}
560	}
561	}
562
563	// 3. Direct connection: TCP peer.
564	if (!empty($_SERVER['REMOTE_ADDR']) && filter_var($_SERVER['REMOTE_ADDR'], FILTER_VALIDATE_IP)) {
565	return $_SERVER['REMOTE_ADDR'];
566	}
567
568	return '';
569	}
570
571	/**
572	* Log bot detection event
573	*
574	* @param array $detection Detection result
575	* @return void
576	*/
577	public function log_detection($detection) {
578	// Only log if bot was detected
579	if (!$detection['is_bot']) {
580	return;
581	}
582
583	// Get database instance
584	require_once plugin_dir_path(__FILE__) . 'class-metasync-otto-bot-statistics-database.php';
585	$db = Metasync_Otto_Bot_Statistics_Database::get_instance();
586
587	// Log the detection
588	$db->add_detection(
589	$detection['bot_name'],
590	$detection['bot_type'],
591	$detection['user_agent'],
592	$detection['ip_address'],
593	$detection['detection_method']
594	);
595	}
596
597	/**
598	* Is $method actually available on $class in this process right now?
599	*
600	* A partially updated install can leave a newer copy of one plugin file
601	* beside an older copy of another — stale opcache bytecode for a single file
602	* is enough. class_exists() is satisfied by the older copy, and calling a
603	* method it does not declare is a fatal. This runs on the front end, so that
604	* fatal is a white screen rather than a degraded feature.
605	*
606	* $class and $method are parameters rather than literals at the call site so
607	* the check survives static analysis, which would otherwise narrow a literal
608	* method_exists() on a known class to a constant true — the skew this guards
609	* against exists only at runtime.
610	*
611	* @param string $class Class about to be called.
612	* @param string $method Method about to be called on it.
613	* @return bool
614	*/
615	private static function class_provides( $class, $method ) {
616	return class_exists( $class ) && method_exists( $class, $method );
617	}
618
619	/**
620	* Push bot crawl log to SearchAtlas backend API (non-blocking fire-and-forget).
621	*
622	* Real search engine bots do not execute JavaScript, so the JS tracker
623	* (otto-tracker.js) never captures them. This method sends crawl monitoring
624	* data server-side via a non-blocking wp_remote_post so that the SA Crawl
625	* Monitoring dashboard is populated regardless of whether the bot runs JS.
626	*
627	* @param array $detection Detection result from detect()
628	* @param string $url The full URL being crawled
629	* @return void
630	*/
631	public function push_crawl_log_to_sa( $detection, $url ) {
632	if ( ! $detection['is_bot'] ) {
633	return;
634	}
635
636	// Require a configured OTTO UUID
637	if ( ! class_exists( 'Metasync_Otto_Config' ) ) {
638	return;
639	}
640	$otto_uuid = Metasync_Otto_Config::get_otto_uuid();
641	if ( empty( $otto_uuid ) ) {
642	return;
643	}
644
645	// This POST is documented below as non-blocking, but `blocking =>
646	// false` only skips waiting for the response — cURL still performs DNS
647	// resolution, the TCP connect and the TLS handshake synchronously. Measured
648	// against an unresponsive endpoint it blocked for 1.01s, on every bot
649	// request, on top of the OTTO suggestions call.
650	//
651	// So: if the endpoint is already known to be degraded, do not attempt it.
652	// Checked before the throttle transient is claimed, so a skipped push does
653	// not burn the 30-minute window and the crawl is still reported once the
654	// endpoint recovers.
655	// Resolve endpoint — respects production/staging mode toggle. Hoisted above
656	// the throttle so the health gate below can consult it.
657	if ( class_exists( 'Metasync_Endpoint_Manager' ) ) {
658	$endpoint = Metasync_Endpoint_Manager::get_endpoint( 'OTTO_CRAWL_LOGS' );
659	} else {
660	$endpoint = 'https://sa.searchatlas.com/api/v2/otto-page-crawl-logs';
661	}
662	if ( empty( $endpoint ) ) {
663	return;
664	}
665
666	// Backoff covers explicit 429/503 responses...
667	if (self::class_provides('Metasync_API_Backoff_Manager', 'is_endpoint_in_backoff')
668	&& Metasync_API_Backoff_Manager::get_instance()->is_endpoint_in_backoff($endpoint)) {
669	return;
670	}
671
672	// ...and the circuit breaker covers what backoff is blind to: timeouts,
673	// refused connections and DNS failures. Same host, so the suggestions
674	// path's breaker is authoritative for this endpoint too.
675	//
676	// class_provides() rather than class_exists(): a partially updated install
677	// can leave an older copy of the cache class loaded that predates
678	// is_host_breaker_open(), and calling it would fatal on the front end.
679	if (self::class_provides('Metasync_Otto_Transient_Cache', 'is_host_breaker_open')
680	&& Metasync_Otto_Transient_Cache::is_host_breaker_open($endpoint)) {
681	return;
682	}
683
684	// Throttle outbound POSTs to at most one per URL+bot every 30 minutes.
685	// Bot bursts can otherwise produce thousands of redundant requests to
686	// the crawl-log endpoint. Key includes bot_name so different crawlers
687	// hitting the same URL are each recorded once per window.
688	//
689	// Query string is stripped before hashing so a spoofed-bot scanner
690	// hitting /page?x=1, /page?x=2, ... cannot generate unbounded unique
691	// throttle keys (which would flood wp_options or evict legitimate
692	// entries from a persistent object cache).
693	$normalized_url = strtok( $url, '?' );
694	if ( $normalized_url === false ) {
695	$normalized_url = $url;
696	}
697	// Lowercase + length cap so attacker-controlled bot_name variations
698	// collapse to a bounded key space and cannot flood wp_options.
699	$bot_name = substr( strtolower( $detection['bot_name'] ?? 'unknown' ), 0, 32 );
700	$throttle_key = 'metasync_crawl_log_' . md5( $normalized_url . '\|' . $bot_name );
701	if ( get_transient( $throttle_key ) ) {
702	return;
703	}
704	set_transient( $throttle_key, 1, 30 * MINUTE_IN_SECONDS );
705
706	$payload = array(
707	'otto_uuid' => $otto_uuid,
708	'url' => $url,
709	'user_agent' => $detection['user_agent'],
710	'context' => null,
711	);
712
713	// Fire-and-forget: blocking=false means we never wait for the response body.
714	//
715	// `blocking => false` does NOT make the call free — cURL still does
716	// DNS + TCP connect + TLS handshake inline, which measured 1.01s against an
717	// unresponsive endpoint. connect_timeout caps that handshake explicitly;
718	// the health gate above is what removes it entirely once we know the host
719	// is bad. Together these bound this call at ~1s worst case for the first
720	// few failures, then ~0.
721	wp_remote_post(
722	trailingslashit( $endpoint ),
723	array(
724	'method' => 'POST',
725	'timeout' => 0.01,
726	'connect_timeout' => 1,
727	'blocking' => false,
728	'headers' => array( 'Content-Type' => 'application/json' ),
729	'body' => wp_json_encode( $payload ),
730	'data_format' => 'body',
731	)
732	);
733	}
734
735	/**
736	* Clear detection cache
737	*
738	* @return void
739	*/
740	public function clear_cache() {
741	$this->detection_cache = null;
742	}
743	}
744