metasync / otto / class-metasync-otto-bot-detector.php

class-metasync-otto-bot-detector.php in Search Atlas SEO – OTTO AI SEO Automation for WordPress 2.6.11, at otto/class-metasync-otto-bot-detector.php

575 lines 18.3 KB

Raw Download Zip

1	<?php
2	/**
3	* Bot Detection Service for OTTO
4	*
5	* Detects and identifies search engine crawlers and bots to allow
6	* selective processing of OTTO requests based on traffic source.
7	*
8	* @package Metasync
9	* @subpackage Metasync/otto
10	* @since 1.0.0
11	*/
12
13	// If this file is called directly, abort.
14	if (!defined('ABSPATH')) {
15	exit;
16	}
17
18	/**
19	* Bot Detection Service Class
20	*
21	* Provides bot detection functionality with whitelisting and blacklisting support.
22	* Implements singleton pattern for efficient memory usage.
23	*
24	* @since 1.0.0
25	*/
26	class Metasync_Otto_Bot_Detector {
27
28	/**
29	* Singleton instance
30	*
31	* @var Metasync_Otto_Bot_Detector\|null
32	*/
33	private static $instance = null;
34
35	/**
36	* Common search engine bot signatures
37	*
38	* @var array
39	*/
40	private $common_bots = array(
41	'Googlebot',
42	'Googlebot-Image',
43	'Googlebot-News',
44	'Googlebot-Video',
45	'Google-InspectionTool',
46	'Storebot-Google',
47	'GoogleOther',
48	'Bingbot',
49	'BingPreview',
50	'Slurp', // Yahoo
51	'DuckDuckBot', // DuckDuckGo
52	'Baiduspider', // Baidu
53	'YandexBot', // Yandex
54	'Sogou', // Sogou
55	'Exabot', // Exalead
56	'facebot', // Facebook
57	'ia_archiver', // Alexa
58	'AdsBot-Google',
59	'Mediapartners-Google',
60	'APIs-Google',
61	'AhrefsBot',
62	'SemrushBot',
63	'MJ12bot', // Majestic
64	'DotBot', // Moz
65	'Applebot', // Apple
66	'LinkedInBot',
67	'Twitterbot',
68	'PinterestBot',
69	'rogerbot', // Moz
70	'Screaming Frog',
71	'SiteAuditBot',
72	'ArchiveBot',
73	'archive.org_bot',
74	'CCBot', // Common Crawl
75	'BLEXBot',
76	'SEOkicks',
77	'Qwantify',
78	'PetalBot', // Huawei
79	'MauiBot',
80	'AlphaBot',
81	'SiteImprove',
82	'serpstatbot',
83	'WhatWeb',
84	'ZoominfoBot',
85	'SeekportBot',
86	'DataForSeoBot'
87	);
88
89	/**
90	* Generic bot patterns
91	*
92	* @var array
93	*/
94	private $bot_patterns = array(
95	'/bot/i',
96	'/crawl/i',
97	'/spider/i',
98	'/slurp/i',
99	'/mediapartners/i',
100	'/scanner/i',
101	'/scraper/i',
102	'/checker/i',
103	'/validator/i',
104	'/fetcher/i',
105	'/archiver/i',
106	'/lighthouse/i',
107	'/pagespeed/i'
108	);
109
110	/**
111	* IP-based bot detection patterns
112	* Common IP ranges for known bots
113	*
114	* @var array
115	*/
116	private $bot_ip_ranges = array(
117	// Google IP ranges (partial list - full list is extensive)
118	'66.249.',
119	'66.102.',
120	'64.233.',
121	'72.14.',
122	'74.125.',
123	'209.85.',
124	'216.239.',
125	// Bing IP ranges
126	'40.77.',
127	'157.55.',
128	'207.46.',
129	// Yandex
130	'5.255.',
131	'77.88.',
132	'95.108.',
133	'100.43.',
134	'130.193.',
135	'141.8.',
136	'178.154.'
137	);
138
139	/**
140	* Current detection result cache
141	*
142	* @var array\|null
143	*/
144	private $detection_cache = null;
145
146	/**
147	* Get singleton instance
148	*
149	* @return Metasync_Otto_Bot_Detector
150	*/
151	public static function get_instance() {
152	if (self::$instance === null) {
153	self::$instance = new self();
154	}
155	return self::$instance;
156	}
157
158	/**
159	* Private constructor to prevent direct instantiation
160	*/
161	private function __construct() {
162	// Private constructor for singleton
163	}
164
165	/**
166	* Prevent cloning of the instance
167	*/
168	private function __clone() {
169	// Prevent cloning
170	}
171
172	/**
173	* Prevent unserialization of the instance
174	*/
175	public function __wakeup() {
176	// Prevent unserialization
177	}
178
179	/**
180	* Detect if current request is from a bot
181	*
182	* @param string\|null $user_agent Optional user agent to check (defaults to current request)
183	* @param string\|null $ip Optional IP address to check (defaults to current request)
184	* @return array Detection result with bot status and details
185	*/
186	public function detect($user_agent = null, $ip = null) {
187	// Use cache if available and no custom parameters provided
188	if ($this->detection_cache !== null && $user_agent === null && $ip === null) {
189	return $this->detection_cache;
190	}
191
192	// Get user agent from request if not provided
193	if ($user_agent === null) {
194	$user_agent = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '';
195	}
196
197	// Get IP from request if not provided
198	if ($ip === null) {
199	$ip = $this->get_client_ip();
200	}
201
202	// Initialize result
203	$result = array(
204	'is_bot' => false,
205	'bot_name' => null,
206	'bot_type' => null,
207	'user_agent' => $user_agent,
208	'ip_address' => $ip,
209	'detection_method' => null
210	);
211
212	// Check if user agent is empty (likely a bot)
213	if (empty($user_agent)) {
214	$result['is_bot'] = true;
215	$result['bot_name'] = 'Unknown Bot';
216	$result['bot_type'] = 'unknown';
217	$result['detection_method'] = 'empty_user_agent';
218
219	// Cache and return
220	$this->detection_cache = $result;
221	return $result;
222	}
223
224	// Check against common bots list
225	foreach ($this->common_bots as $bot) {
226	if (stripos($user_agent, $bot) !== false) {
227	$result['is_bot'] = true;
228	$result['bot_name'] = $bot;
229	$result['bot_type'] = $this->categorize_bot($bot);
230	$result['detection_method'] = 'user_agent_match';
231
232	// Cache and return
233	$this->detection_cache = $result;
234	return $result;
235	}
236	}
237
238	// Check against bot patterns
239	foreach ($this->bot_patterns as $pattern) {
240	if (preg_match($pattern, $user_agent)) {
241	$result['is_bot'] = true;
242	$result['bot_name'] = $this->extract_bot_name($user_agent);
243	$result['bot_type'] = 'generic';
244	$result['detection_method'] = 'pattern_match';
245
246	// Cache and return
247	$this->detection_cache = $result;
248	return $result;
249	}
250	}
251
252	// Check IP-based detection
253	if ($this->is_bot_ip($ip)) {
254	$result['is_bot'] = true;
255	$result['bot_name'] = 'Bot (IP-based detection)';
256	$result['bot_type'] = 'ip_based';
257	$result['detection_method'] = 'ip_range_match';
258
259	// Cache and return
260	$this->detection_cache = $result;
261	return $result;
262	}
263
264	// Cache and return
265	$this->detection_cache = $result;
266	return $result;
267	}
268
269	/**
270	* Check if current request should skip OTTO processing
271	* Takes into account settings, whitelist, and blacklist
272	*
273	* @return bool True if OTTO should be skipped, false otherwise
274	*/
275	public function should_skip_otto() {
276	// Get plugin settings
277	$options = get_option('metasync_options', array());
278	$general = isset($options['general']) ? $options['general'] : array();
279
280	// Check if bot filtering is enabled
281	$disable_for_bots = isset($general['otto_disable_for_bots']) ? (bool)$general['otto_disable_for_bots'] : false;
282
283	if (!$disable_for_bots) {
284	return false; // Feature is disabled, don't skip
285	}
286
287	// Detect bot
288	$detection = $this->detect();
289
290	if (!$detection['is_bot']) {
291	return false; // Not a bot, don't skip
292	}
293
294	// Check whitelist
295	$whitelist = isset($general['otto_bot_whitelist']) ? $general['otto_bot_whitelist'] : '';
296	if (!empty($whitelist)) {
297	$whitelist_array = array_map('trim', explode("\n", $whitelist));
298	foreach ($whitelist_array as $whitelisted_bot) {
299	if (empty($whitelisted_bot)) continue;
300
301	if (stripos($detection['user_agent'], $whitelisted_bot) !== false \|\|
302	stripos($detection['bot_name'], $whitelisted_bot) !== false) {
303	return false; // Bot is whitelisted, don't skip
304	}
305	}
306	}
307
308	// Check blacklist
309	$blacklist = isset($general['otto_bot_blacklist']) ? $general['otto_bot_blacklist'] : '';
310	if (!empty($blacklist)) {
311	$blacklist_array = array_map('trim', explode("\n", $blacklist));
312	foreach ($blacklist_array as $blacklisted_pattern) {
313	if (empty($blacklisted_pattern)) continue;
314
315	if (stripos($detection['user_agent'], $blacklisted_pattern) !== false \|\|
316	stripos($detection['bot_name'], $blacklisted_pattern) !== false) {
317	return true; // Bot is explicitly blacklisted, skip OTTO
318	}
319	}
320	}
321
322	// Bot detected and no whitelist override, skip OTTO
323	return true;
324	}
325
326	/**
327	* Categorize bot type
328	*
329	* @param string $bot_name Bot name
330	* @return string Bot category
331	*/
332	private function categorize_bot($bot_name) {
333	$bot_lower = strtolower($bot_name);
334
335	// Search engines
336	if (stripos($bot_lower, 'google') !== false) return 'search_engine';
337	if (stripos($bot_lower, 'bing') !== false) return 'search_engine';
338	if (stripos($bot_lower, 'yahoo') !== false) return 'search_engine';
339	if (stripos($bot_lower, 'yandex') !== false) return 'search_engine';
340	if (stripos($bot_lower, 'baidu') !== false) return 'search_engine';
341	if (stripos($bot_lower, 'duckduck') !== false) return 'search_engine';
342
343	// SEO tools
344	if (stripos($bot_lower, 'ahrefs') !== false) return 'seo_tool';
345	if (stripos($bot_lower, 'semrush') !== false) return 'seo_tool';
346	if (stripos($bot_lower, 'moz') !== false) return 'seo_tool';
347	if (stripos($bot_lower, 'majestic') !== false) return 'seo_tool';
348	if (stripos($bot_lower, 'screaming') !== false) return 'seo_tool';
349
350	// Social media
351	if (stripos($bot_lower, 'facebook') !== false) return 'social_media';
352	if (stripos($bot_lower, 'twitter') !== false) return 'social_media';
353	if (stripos($bot_lower, 'linkedin') !== false) return 'social_media';
354	if (stripos($bot_lower, 'pinterest') !== false) return 'social_media';
355
356	// Archive
357	if (stripos($bot_lower, 'archive') !== false) return 'archiver';
358
359	return 'other';
360	}
361
362	/**
363	* Extract bot name from user agent string
364	*
365	* @param string $user_agent User agent string
366	* @return string Extracted bot name
367	*/
368	private function extract_bot_name($user_agent) {
369	// Try to extract a meaningful name from the user agent
370	if (preg_match('/([a-zA-Z]+[bB]ot[a-zA-Z]*)/i', $user_agent, $matches)) {
371	return $matches[1];
372	}
373
374	if (preg_match('/([a-zA-Z]+[sS]pider)/i', $user_agent, $matches)) {
375	return $matches[1];
376	}
377
378	if (preg_match('/([a-zA-Z]+[cC]rawler)/i', $user_agent, $matches)) {
379	return $matches[1];
380	}
381
382	return 'Unknown Bot';
383	}
384
385	/**
386	* Check if IP belongs to a known bot
387	*
388	* @param string $ip IP address
389	* @return bool True if IP is from a known bot
390	*/
391	private function is_bot_ip($ip) {
392	if (empty($ip)) {
393	return false;
394	}
395
396	foreach ($this->bot_ip_ranges as $range) {
397	if (strpos($ip, $range) === 0) {
398	return true;
399	}
400	}
401
402	return false;
403	}
404
405	/**
406	* Get client IP address.
407	*
408	* Resolves the visitor's IP using a Cloudflare-first priority order so that
409	* attacker-controlled headers cannot be used to spoof crawler IPs and bypass
410	* the IP-range step of bot detection:
411	*
412	* 1. HTTP_CF_CONNECTING_IP — trusted, but only when HTTP_CF_RAY confirms
413	* the request actually traversed Cloudflare.
414	* 2. HTTP_X_FORWARDED_FOR — fallback for non-Cloudflare reverse proxies.
415	* The chain is walked right-to-left and the first public IP is used;
416	* the leftmost value is attacker-controlled and must not be trusted.
417	* 3. REMOTE_ADDR — TCP peer; final fallback for direct (non-proxied) hits.
418	*
419	* Non-standard, attacker-controllable headers are intentionally not
420	* consulted at any point in this resolution chain.
421	*
422	* @return string Client IP address, or '' if none could be validated.
423	*/
424	private function get_client_ip() {
425	// 1. Cloudflare-proxied request: HTTP_CF_RAY is the canonical marker
426	// that Cloudflare actually handled the connection, which means
427	// HTTP_CF_CONNECTING_IP was set by Cloudflare and is trustworthy.
428	if (!empty($_SERVER['HTTP_CF_RAY']) && !empty($_SERVER['HTTP_CF_CONNECTING_IP'])) {
429	$cf_ip = $_SERVER['HTTP_CF_CONNECTING_IP'];
430	if (filter_var($cf_ip, FILTER_VALIDATE_IP)) {
431	return $cf_ip;
432	}
433	}
434
435	// 2. Non-Cloudflare reverse proxy: walk X-Forwarded-For right-to-left
436	// and return the first public IP. The trusted proxy appends to the
437	// right; everything to the left of that may have been supplied by
438	// the client and cannot be relied on.
439	if (!empty($_SERVER['HTTP_X_FORWARDED_FOR'])) {
440	$ips = array_reverse(explode(',', $_SERVER['HTTP_X_FORWARDED_FOR']));
441	foreach ($ips as $candidate) {
442	$candidate = trim($candidate);
443	if ($candidate === '') {
444	continue;
445	}
446	if (filter_var($candidate, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE \| FILTER_FLAG_NO_RES_RANGE)) {
447	return $candidate;
448	}
449	}
450	}
451
452	// 3. Direct connection: TCP peer.
453	if (!empty($_SERVER['REMOTE_ADDR']) && filter_var($_SERVER['REMOTE_ADDR'], FILTER_VALIDATE_IP)) {
454	return $_SERVER['REMOTE_ADDR'];
455	}
456
457	return '';
458	}
459
460	/**
461	* Log bot detection event
462	*
463	* @param array $detection Detection result
464	* @return void
465	*/
466	public function log_detection($detection) {
467	// Only log if bot was detected
468	if (!$detection['is_bot']) {
469	return;
470	}
471
472	// Get database instance
473	require_once plugin_dir_path(__FILE__) . 'class-metasync-otto-bot-statistics-database.php';
474	$db = Metasync_Otto_Bot_Statistics_Database::get_instance();
475
476	// Log the detection
477	$db->add_detection(
478	$detection['bot_name'],
479	$detection['bot_type'],
480	$detection['user_agent'],
481	$detection['ip_address'],
482	$detection['detection_method']
483	);
484	}
485
486	/**
487	* Push bot crawl log to SearchAtlas backend API (non-blocking fire-and-forget).
488	*
489	* Real search engine bots do not execute JavaScript, so the JS tracker
490	* (otto-tracker.js) never captures them. This method sends crawl monitoring
491	* data server-side via a non-blocking wp_remote_post so that the SA Crawl
492	* Monitoring dashboard is populated regardless of whether the bot runs JS.
493	*
494	* @param array $detection Detection result from detect()
495	* @param string $url The full URL being crawled
496	* @return void
497	*/
498	public function push_crawl_log_to_sa( $detection, $url ) {
499	if ( ! $detection['is_bot'] ) {
500	return;
501	}
502
503	// Require a configured OTTO UUID
504	if ( ! class_exists( 'Metasync_Otto_Config' ) ) {
505	return;
506	}
507	$otto_uuid = Metasync_Otto_Config::get_otto_uuid();
508	if ( empty( $otto_uuid ) ) {
509	return;
510	}
511
512	// Throttle outbound POSTs to at most one per URL+bot every 30 minutes.
513	// Bot bursts can otherwise produce thousands of redundant requests to
514	// the crawl-log endpoint. Key includes bot_name so different crawlers
515	// hitting the same URL are each recorded once per window.
516	//
517	// Query string is stripped before hashing so a spoofed-bot scanner
518	// hitting /page?x=1, /page?x=2, ... cannot generate unbounded unique
519	// throttle keys (which would flood wp_options or evict legitimate
520	// entries from a persistent object cache).
521	$normalized_url = strtok( $url, '?' );
522	if ( $normalized_url === false ) {
523	$normalized_url = $url;
524	}
525	// Lowercase + length cap so attacker-controlled bot_name variations
526	// collapse to a bounded key space and cannot flood wp_options.
527	$bot_name = substr( strtolower( $detection['bot_name'] ?? 'unknown' ), 0, 32 );
528	$throttle_key = 'metasync_crawl_log_' . md5( $normalized_url . '\|' . $bot_name );
529	if ( get_transient( $throttle_key ) ) {
530	return;
531	}
532	set_transient( $throttle_key, 1, 30 * MINUTE_IN_SECONDS );
533
534	// Resolve endpoint — respects production/staging mode toggle
535	if ( class_exists( 'Metasync_Endpoint_Manager' ) ) {
536	$endpoint = Metasync_Endpoint_Manager::get_endpoint( 'OTTO_CRAWL_LOGS' );
537	} else {
538	$endpoint = 'https://sa.searchatlas.com/api/v2/otto-page-crawl-logs';
539	}
540	if ( empty( $endpoint ) ) {
541	return;
542	}
543
544	$payload = array(
545	'otto_uuid' => $otto_uuid,
546	'url' => $url,
547	'user_agent' => $detection['user_agent'],
548	'context' => null,
549	);
550
551	// Non-blocking: timeout=0.01 + blocking=false so we never wait for a response.
552	// This prevents any latency impact on the bot's page request.
553	wp_remote_post(
554	trailingslashit( $endpoint ),
555	array(
556	'method' => 'POST',
557	'timeout' => 0.01,
558	'blocking' => false,
559	'headers' => array( 'Content-Type' => 'application/json' ),
560	'body' => wp_json_encode( $payload ),
561	'data_format' => 'body',
562	)
563	);
564	}
565
566	/**
567	* Clear detection cache
568	*
569	* @return void
570	*/
571	public function clear_cache() {
572	$this->detection_cache = null;
573	}
574	}
575