metasync / otto / class-metasync-otto-bot-detector.php

class-metasync-otto-bot-detector.php in Search Atlas SEO – OTTO AI SEO Automation for WordPress 2.6.7, at otto/class-metasync-otto-bot-detector.php

534 lines 15.7 KB

Raw Download Zip

1	<?php
2	/**
3	* Bot Detection Service for OTTO
4	*
5	* Detects and identifies search engine crawlers and bots to allow
6	* selective processing of OTTO requests based on traffic source.
7	*
8	* @package Metasync
9	* @subpackage Metasync/otto
10	* @since 1.0.0
11	*/
12
13	// If this file is called directly, abort.
14	if (!defined('ABSPATH')) {
15	exit;
16	}
17
18	/**
19	* Bot Detection Service Class
20	*
21	* Provides bot detection functionality with whitelisting and blacklisting support.
22	* Implements singleton pattern for efficient memory usage.
23	*
24	* @since 1.0.0
25	*/
26	class Metasync_Otto_Bot_Detector {
27
28	/**
29	* Singleton instance
30	*
31	* @var Metasync_Otto_Bot_Detector\|null
32	*/
33	private static $instance = null;
34
35	/**
36	* Common search engine bot signatures
37	*
38	* @var array
39	*/
40	private $common_bots = array(
41	'Googlebot',
42	'Googlebot-Image',
43	'Googlebot-News',
44	'Googlebot-Video',
45	'Google-InspectionTool',
46	'Storebot-Google',
47	'GoogleOther',
48	'Bingbot',
49	'BingPreview',
50	'Slurp', // Yahoo
51	'DuckDuckBot', // DuckDuckGo
52	'Baiduspider', // Baidu
53	'YandexBot', // Yandex
54	'Sogou', // Sogou
55	'Exabot', // Exalead
56	'facebot', // Facebook
57	'ia_archiver', // Alexa
58	'AdsBot-Google',
59	'Mediapartners-Google',
60	'APIs-Google',
61	'AhrefsBot',
62	'SemrushBot',
63	'MJ12bot', // Majestic
64	'DotBot', // Moz
65	'Applebot', // Apple
66	'LinkedInBot',
67	'Twitterbot',
68	'PinterestBot',
69	'rogerbot', // Moz
70	'Screaming Frog',
71	'SiteAuditBot',
72	'ArchiveBot',
73	'archive.org_bot',
74	'CCBot', // Common Crawl
75	'BLEXBot',
76	'SEOkicks',
77	'Qwantify',
78	'PetalBot', // Huawei
79	'MauiBot',
80	'AlphaBot',
81	'SiteImprove',
82	'serpstatbot',
83	'WhatWeb',
84	'ZoominfoBot',
85	'SeekportBot',
86	'DataForSeoBot'
87	);
88
89	/**
90	* Generic bot patterns
91	*
92	* @var array
93	*/
94	private $bot_patterns = array(
95	'/bot/i',
96	'/crawl/i',
97	'/spider/i',
98	'/slurp/i',
99	'/mediapartners/i',
100	'/scanner/i',
101	'/scraper/i',
102	'/checker/i',
103	'/validator/i',
104	'/fetcher/i',
105	'/archiver/i',
106	'/lighthouse/i',
107	'/pagespeed/i'
108	);
109
110	/**
111	* IP-based bot detection patterns
112	* Common IP ranges for known bots
113	*
114	* @var array
115	*/
116	private $bot_ip_ranges = array(
117	// Google IP ranges (partial list - full list is extensive)
118	'66.249.',
119	'66.102.',
120	'64.233.',
121	'72.14.',
122	'74.125.',
123	'209.85.',
124	'216.239.',
125	// Bing IP ranges
126	'40.77.',
127	'157.55.',
128	'207.46.',
129	// Yandex
130	'5.255.',
131	'77.88.',
132	'95.108.',
133	'100.43.',
134	'130.193.',
135	'141.8.',
136	'178.154.'
137	);
138
139	/**
140	* Current detection result cache
141	*
142	* @var array\|null
143	*/
144	private $detection_cache = null;
145
146	/**
147	* Get singleton instance
148	*
149	* @return Metasync_Otto_Bot_Detector
150	*/
151	public static function get_instance() {
152	if (self::$instance === null) {
153	self::$instance = new self();
154	}
155	return self::$instance;
156	}
157
158	/**
159	* Private constructor to prevent direct instantiation
160	*/
161	private function __construct() {
162	// Private constructor for singleton
163	}
164
165	/**
166	* Prevent cloning of the instance
167	*/
168	private function __clone() {
169	// Prevent cloning
170	}
171
172	/**
173	* Prevent unserialization of the instance
174	*/
175	public function __wakeup() {
176	// Prevent unserialization
177	}
178
179	/**
180	* Detect if current request is from a bot
181	*
182	* @param string\|null $user_agent Optional user agent to check (defaults to current request)
183	* @param string\|null $ip Optional IP address to check (defaults to current request)
184	* @return array Detection result with bot status and details
185	*/
186	public function detect($user_agent = null, $ip = null) {
187	// Use cache if available and no custom parameters provided
188	if ($this->detection_cache !== null && $user_agent === null && $ip === null) {
189	return $this->detection_cache;
190	}
191
192	// Get user agent from request if not provided
193	if ($user_agent === null) {
194	$user_agent = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '';
195	}
196
197	// Get IP from request if not provided
198	if ($ip === null) {
199	$ip = $this->get_client_ip();
200	}
201
202	// Initialize result
203	$result = array(
204	'is_bot' => false,
205	'bot_name' => null,
206	'bot_type' => null,
207	'user_agent' => $user_agent,
208	'ip_address' => $ip,
209	'detection_method' => null
210	);
211
212	// Check if user agent is empty (likely a bot)
213	if (empty($user_agent)) {
214	$result['is_bot'] = true;
215	$result['bot_name'] = 'Unknown Bot';
216	$result['bot_type'] = 'unknown';
217	$result['detection_method'] = 'empty_user_agent';
218
219	// Cache and return
220	$this->detection_cache = $result;
221	return $result;
222	}
223
224	// Check against common bots list
225	foreach ($this->common_bots as $bot) {
226	if (stripos($user_agent, $bot) !== false) {
227	$result['is_bot'] = true;
228	$result['bot_name'] = $bot;
229	$result['bot_type'] = $this->categorize_bot($bot);
230	$result['detection_method'] = 'user_agent_match';
231
232	// Cache and return
233	$this->detection_cache = $result;
234	return $result;
235	}
236	}
237
238	// Check against bot patterns
239	foreach ($this->bot_patterns as $pattern) {
240	if (preg_match($pattern, $user_agent)) {
241	$result['is_bot'] = true;
242	$result['bot_name'] = $this->extract_bot_name($user_agent);
243	$result['bot_type'] = 'generic';
244	$result['detection_method'] = 'pattern_match';
245
246	// Cache and return
247	$this->detection_cache = $result;
248	return $result;
249	}
250	}
251
252	// Check IP-based detection
253	if ($this->is_bot_ip($ip)) {
254	$result['is_bot'] = true;
255	$result['bot_name'] = 'Bot (IP-based detection)';
256	$result['bot_type'] = 'ip_based';
257	$result['detection_method'] = 'ip_range_match';
258
259	// Cache and return
260	$this->detection_cache = $result;
261	return $result;
262	}
263
264	// Cache and return
265	$this->detection_cache = $result;
266	return $result;
267	}
268
269	/**
270	* Check if current request should skip OTTO processing
271	* Takes into account settings, whitelist, and blacklist
272	*
273	* @return bool True if OTTO should be skipped, false otherwise
274	*/
275	public function should_skip_otto() {
276	// Get plugin settings
277	$options = get_option('metasync_options', array());
278	$general = isset($options['general']) ? $options['general'] : array();
279
280	// Check if bot filtering is enabled
281	$disable_for_bots = isset($general['otto_disable_for_bots']) ? (bool)$general['otto_disable_for_bots'] : false;
282
283	if (!$disable_for_bots) {
284	return false; // Feature is disabled, don't skip
285	}
286
287	// Detect bot
288	$detection = $this->detect();
289
290	if (!$detection['is_bot']) {
291	return false; // Not a bot, don't skip
292	}
293
294	// Check whitelist
295	$whitelist = isset($general['otto_bot_whitelist']) ? $general['otto_bot_whitelist'] : '';
296	if (!empty($whitelist)) {
297	$whitelist_array = array_map('trim', explode("\n", $whitelist));
298	foreach ($whitelist_array as $whitelisted_bot) {
299	if (empty($whitelisted_bot)) continue;
300
301	if (stripos($detection['user_agent'], $whitelisted_bot) !== false \|\|
302	stripos($detection['bot_name'], $whitelisted_bot) !== false) {
303	return false; // Bot is whitelisted, don't skip
304	}
305	}
306	}
307
308	// Check blacklist
309	$blacklist = isset($general['otto_bot_blacklist']) ? $general['otto_bot_blacklist'] : '';
310	if (!empty($blacklist)) {
311	$blacklist_array = array_map('trim', explode("\n", $blacklist));
312	foreach ($blacklist_array as $blacklisted_pattern) {
313	if (empty($blacklisted_pattern)) continue;
314
315	if (stripos($detection['user_agent'], $blacklisted_pattern) !== false \|\|
316	stripos($detection['bot_name'], $blacklisted_pattern) !== false) {
317	return true; // Bot is explicitly blacklisted, skip OTTO
318	}
319	}
320	}
321
322	// Bot detected and no whitelist override, skip OTTO
323	return true;
324	}
325
326	/**
327	* Categorize bot type
328	*
329	* @param string $bot_name Bot name
330	* @return string Bot category
331	*/
332	private function categorize_bot($bot_name) {
333	$bot_lower = strtolower($bot_name);
334
335	// Search engines
336	if (stripos($bot_lower, 'google') !== false) return 'search_engine';
337	if (stripos($bot_lower, 'bing') !== false) return 'search_engine';
338	if (stripos($bot_lower, 'yahoo') !== false) return 'search_engine';
339	if (stripos($bot_lower, 'yandex') !== false) return 'search_engine';
340	if (stripos($bot_lower, 'baidu') !== false) return 'search_engine';
341	if (stripos($bot_lower, 'duckduck') !== false) return 'search_engine';
342
343	// SEO tools
344	if (stripos($bot_lower, 'ahrefs') !== false) return 'seo_tool';
345	if (stripos($bot_lower, 'semrush') !== false) return 'seo_tool';
346	if (stripos($bot_lower, 'moz') !== false) return 'seo_tool';
347	if (stripos($bot_lower, 'majestic') !== false) return 'seo_tool';
348	if (stripos($bot_lower, 'screaming') !== false) return 'seo_tool';
349
350	// Social media
351	if (stripos($bot_lower, 'facebook') !== false) return 'social_media';
352	if (stripos($bot_lower, 'twitter') !== false) return 'social_media';
353	if (stripos($bot_lower, 'linkedin') !== false) return 'social_media';
354	if (stripos($bot_lower, 'pinterest') !== false) return 'social_media';
355
356	// Archive
357	if (stripos($bot_lower, 'archive') !== false) return 'archiver';
358
359	return 'other';
360	}
361
362	/**
363	* Extract bot name from user agent string
364	*
365	* @param string $user_agent User agent string
366	* @return string Extracted bot name
367	*/
368	private function extract_bot_name($user_agent) {
369	// Try to extract a meaningful name from the user agent
370	if (preg_match('/([a-zA-Z]+[bB]ot[a-zA-Z]*)/i', $user_agent, $matches)) {
371	return $matches[1];
372	}
373
374	if (preg_match('/([a-zA-Z]+[sS]pider)/i', $user_agent, $matches)) {
375	return $matches[1];
376	}
377
378	if (preg_match('/([a-zA-Z]+[cC]rawler)/i', $user_agent, $matches)) {
379	return $matches[1];
380	}
381
382	return 'Unknown Bot';
383	}
384
385	/**
386	* Check if IP belongs to a known bot
387	*
388	* @param string $ip IP address
389	* @return bool True if IP is from a known bot
390	*/
391	private function is_bot_ip($ip) {
392	if (empty($ip)) {
393	return false;
394	}
395
396	foreach ($this->bot_ip_ranges as $range) {
397	if (strpos($ip, $range) === 0) {
398	return true;
399	}
400	}
401
402	return false;
403	}
404
405	/**
406	* Get client IP address
407	* Handles various proxy configurations
408	*
409	* @return string Client IP address
410	*/
411	private function get_client_ip() {
412	$ip = '';
413
414	// Check for shared internet/ISP IP
415	if (!empty($_SERVER['HTTP_CLIENT_IP'])) {
416	$ip = $_SERVER['HTTP_CLIENT_IP'];
417	}
418	// Check for IPs passing through proxies
419	elseif (!empty($_SERVER['HTTP_X_FORWARDED_FOR'])) {
420	// HTTP_X_FORWARDED_FOR can contain multiple IPs
421	$ips = explode(',', $_SERVER['HTTP_X_FORWARDED_FOR']);
422	$ip = trim($ips[0]);
423	}
424	// Check for Cloudflare
425	elseif (!empty($_SERVER['HTTP_CF_CONNECTING_IP'])) {
426	$ip = $_SERVER['HTTP_CF_CONNECTING_IP'];
427	}
428	// Regular remote address
429	elseif (!empty($_SERVER['REMOTE_ADDR'])) {
430	$ip = $_SERVER['REMOTE_ADDR'];
431	}
432
433	// Validate IP
434	if (filter_var($ip, FILTER_VALIDATE_IP)) {
435	return $ip;
436	}
437
438	return '';
439	}
440
441	/**
442	* Log bot detection event
443	*
444	* @param array $detection Detection result
445	* @return void
446	*/
447	public function log_detection($detection) {
448	// Only log if bot was detected
449	if (!$detection['is_bot']) {
450	return;
451	}
452
453	// Get database instance
454	require_once plugin_dir_path(__FILE__) . 'class-metasync-otto-bot-statistics-database.php';
455	$db = Metasync_Otto_Bot_Statistics_Database::get_instance();
456
457	// Log the detection
458	$db->add_detection(
459	$detection['bot_name'],
460	$detection['bot_type'],
461	$detection['user_agent'],
462	$detection['ip_address'],
463	$detection['detection_method']
464	);
465	}
466
467	/**
468	* Push bot crawl log to SearchAtlas backend API (non-blocking fire-and-forget).
469	*
470	* Real search engine bots do not execute JavaScript, so the JS tracker
471	* (otto-tracker.js) never captures them. This method sends crawl monitoring
472	* data server-side via a non-blocking wp_remote_post so that the SA Crawl
473	* Monitoring dashboard is populated regardless of whether the bot runs JS.
474	*
475	* @param array $detection Detection result from detect()
476	* @param string $url The full URL being crawled
477	* @return void
478	*/
479	public function push_crawl_log_to_sa( $detection, $url ) {
480	if ( ! $detection['is_bot'] ) {
481	return;
482	}
483
484	// Require a configured OTTO UUID
485	if ( ! class_exists( 'Metasync_Otto_Config' ) ) {
486	return;
487	}
488	$otto_uuid = Metasync_Otto_Config::get_otto_uuid();
489	if ( empty( $otto_uuid ) ) {
490	return;
491	}
492
493	// Resolve endpoint — respects production/staging mode toggle
494	if ( class_exists( 'Metasync_Endpoint_Manager' ) ) {
495	$endpoint = Metasync_Endpoint_Manager::get_endpoint( 'OTTO_CRAWL_LOGS' );
496	} else {
497	$endpoint = 'https://sa.searchatlas.com/api/v2/otto-page-crawl-logs';
498	}
499	if ( empty( $endpoint ) ) {
500	return;
501	}
502
503	$payload = array(
504	'otto_uuid' => $otto_uuid,
505	'url' => $url,
506	'user_agent' => $detection['user_agent'],
507	'context' => null,
508	);
509
510	// Non-blocking: timeout=0.01 + blocking=false so we never wait for a response.
511	// This prevents any latency impact on the bot's page request.
512	wp_remote_post(
513	trailingslashit( $endpoint ),
514	array(
515	'method' => 'POST',
516	'timeout' => 0.01,
517	'blocking' => false,
518	'headers' => array( 'Content-Type' => 'application/json' ),
519	'body' => wp_json_encode( $payload ),
520	'data_format' => 'body',
521	)
522	);
523	}
524
525	/**
526	* Clear detection cache
527	*
528	* @return void
529	*/
530	public function clear_cache() {
531	$this->detection_cache = null;
532	}
533	}
534