wfCrawl.php — Wordfence Security – Firewall, Malware Scan, and Login Security 6.3.2

wordfence / lib / wfCrawl.php

wordfence / lib Last commit date

wfCrawl.php

183 lines

1	<?php
2	require_once('wfUtils.php');
3	class wfCrawl {
4	public static function isCrawler($UA){
5	$browscap = new wfBrowscap();
6	$b = $browscap->getBrowser($UA);
7	if (!$b \|\| $b['Parent'] == 'DefaultProperties') {
8	$log = new wfLog(wfConfig::get('apiKey'), wfUtils::getWPVersion());
9	$IP = wfUtils::getIP();
10	return !(isset($_COOKIE['wordfence_verifiedHuman']) && $log->validateVerifiedHumanCookie($_COOKIE['wordfence_verifiedHuman'], $UA, $IP));
11	}
12	else if (isset($b['Crawler']) && $b['Crawler']) {
13	return true;
14	}
15
16	return false;
17	}
18	public static function verifyCrawlerPTR($hostPattern, $IP){
19	global $wpdb; $table = $wpdb->base_prefix . 'wfCrawlers';
20	$db = new wfDB();
21	$IPn = wfUtils::inet_pton($IP);
22	$status = $db->querySingle("select status from $table where IP=%s and patternSig=UNHEX(MD5('%s')) and lastUpdate > unix_timestamp() - %d", $IPn, $hostPattern, WORDFENCE_CRAWLER_VERIFY_CACHE_TIME);
23	if($status){
24	if($status == 'verified'){
25	return true;
26	} else {
27	return false;
28	}
29	}
30	$host = wfUtils::reverseLookup($IP);
31	if(! $host){
32	$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values (%s, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $IPn, $hostPattern, 'noPTR', '', 'noPTR', '');
33	return false;
34	}
35	if(preg_match($hostPattern, $host)){
36	$resultIPs = wfUtils::resolveDomainName($host);
37	$addrsMatch = false;
38	foreach($resultIPs as $resultIP){
39	if($resultIP == $IP){
40	$addrsMatch = true;
41	break;
42	}
43	}
44	if($addrsMatch){
45	$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values (%s, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $IPn, $hostPattern, 'verified', $host, 'verified', $host);
46	return true;
47	} else {
48	$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values (%s, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $IPn, $hostPattern, 'fwdFail', $host, 'fwdFail', $host);
49	return false;
50	}
51	} else {
52	$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values (%s, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $IPn, $hostPattern, 'badPTR', $host, 'badPTR', $host);
53	return false;
54	}
55	}
56	public static function isGooglebot($userAgent = null){
57	if ($userAgent === null) {
58	$userAgent = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '';
59	}
60	return (bool) preg_match('/Googlebot\/\d\.\d/', $userAgent);
61	}
62	public static function isGoogleCrawler($userAgent = null){
63	if ($userAgent === null) {
64	$userAgent = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '';
65	}
66	foreach (self::$googPat as $pat) {
67	if (preg_match($pat . 'i', $userAgent)) {
68	return true;
69	}
70	}
71	return false;
72	}
73	private static $googPat = array(
74	'@^Mozilla/5\\.0 \$.Google Keyword Tool.\$$@',
75	'@^Mozilla/5\\.0 \$.Feedfetcher\\-Google.\$$@',
76	'@^Feedfetcher\\-Google\\-iGoogleGadgets.*$@',
77	'@^searchbot admin\\@google\\.com$@',
78	'@^Google\\-Site\\-Verification.*$@',
79	'@^Google OpenSocial agent.*$@',
80	'@^.Googlebot\\-Mobile/2\\..$@',
81	'@^AdsBot\\-Google\\-Mobile.*$@',
82	'@^google \$.Enterprise.\$$@',
83	'@^Mediapartners\\-Google.*$@',
84	'@^GoogleFriendConnect.*$@',
85	'@^googlebot\\-urlconsole$@',
86	'@^.Google Web Preview.$@',
87	'@^Feedfetcher\\-Google.*$@',
88	'@^AppEngine\\-Google.*$@',
89	'@^Googlebot\\-Video.*$@',
90	'@^Googlebot\\-Image.*$@',
91	'@^Google\\-Sitemaps.*$@',
92	'@^Googlebot/Test.*$@',
93	'@^Googlebot\\-News.*$@',
94	'@^.Googlebot/2\\.1.$@',
95	'@^AdsBot\\-Google.*$@',
96	'@^Google$@'
97	);
98
99
100	/**
101	* Has correct user agent and PTR record points to .googlebot.com domain.
102	*
103	* @param string\|null $ip
104	* @param string\|null $ua
105	* @return bool
106	*/
107	public static function isVerifiedGoogleCrawler($ip = null, $ua = null) {
108	static $verified;
109	if (!isset($verified)) {
110	$verified = array();
111	}
112	if ($ip === null) {
113	$ip = wfUtils::getIP();
114	}
115	if (array_key_exists($ip, $verified)) {
116	return $verified[$ip];
117	}
118	if (self::isGoogleCrawler($ua)) {
119	if (self::verifyCrawlerPTR(wordfence::getLog()->getGooglePattern(), $ip)) {
120	$verified[$ip] = true;
121	return $verified[$ip];
122	}
123	if (self::verifyGooglebotViaNOC1($ip)) {
124	$verified[$ip] = true;
125	return $verified[$ip];
126	}
127	}
128	$verified[$ip] = false;
129	return $verified[$ip];
130	}
131
132	/**
133	* @param string\|null $ip
134	* @return bool
135	*/
136	public static function verifyGooglebotViaNOC1($ip = null) {
137	global $wpdb;
138	$table = $wpdb->base_prefix . 'wfCrawlers';
139	if ($ip === null) {
140	$ip = wfUtils::getIP();
141	}
142	$db = new wfDB();
143	$IPn = wfUtils::inet_pton($ip);
144	$patternSig = 'googlenoc1';
145	$status = $db->querySingle("select status from $table
146	where IP=%s
147	and patternSig=UNHEX(MD5('%s'))
148	and lastUpdate > unix_timestamp() - %d",
149	$IPn,
150	$patternSig,
151	WORDFENCE_CRAWLER_VERIFY_CACHE_TIME);
152	if ($status === 'verified') {
153	return true;
154	} else if ($status === 'fakeBot') {
155	return false;
156	}
157
158	$api = new wfAPI(wfConfig::get('apiKey'), wfUtils::getWPVersion());
159	try {
160	$data = $api->call('verify_googlebot', array(
161	'ip' => $ip,
162	));
163	if (is_array($data) && !empty($data['verified'])) {
164	// Cache results
165	$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate)
166	values (%s, UNHEX(MD5('%s')), '%s', unix_timestamp())
167	ON DUPLICATE KEY UPDATE status='%3\$s', lastUpdate=unix_timestamp()",
168	$IPn, $patternSig, 'verified');
169	return true;
170	} else {
171	$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate)
172	values (%s, UNHEX(MD5('%s')), '%s', unix_timestamp())
173	ON DUPLICATE KEY UPDATE status='%3\$s', lastUpdate=unix_timestamp()",
174	$IPn, $patternSig, 'fakeBot');
175	}
176	} catch (Exception $e) {
177	// Do nothing, bail
178	}
179	return false;
180	}
181	}
182	?>
183