wordfenceURLHoover.php — Wordfence Security – Firewall, Malware Scan, and Login Security 5.2.4

wordfence / lib / wordfenceURLHoover.php

wordfence / lib Last commit date

wordfenceURLHoover.php

276 lines

1	<?php
2	require_once('wfAPI.php');
3	require_once('wfArray.php');
4	class wordfenceURLHoover {
5	private $debug = false;
6	public $errorMsg = false;
7	private $hostsToAdd = false;
8	private $table = '';
9	private $apiKey = false;
10	private $wordpressVersion = false;
11	private $useDB = true;
12	private $hostKeys = array();
13	private $hostList = array();
14	public $currentHooverID = false;
15	private $dRegex = 'aero\|asia\|biz\|cat\|com\|coop\|edu\|gov\|info\|int\|jobs\|mil\|mobi\|museum\|name\|net\|org\|pro\|tel\|travel\|xxx\|ac\|ad\|ae\|af\|ag\|ai\|al\|am\|an\|ao\|aq\|ar\|as\|at\|au\|aw\|ax\|az\|ba\|bb\|bd\|be\|bf\|bg\|bh\|bi\|bj\|bm\|bn\|bo\|br\|bs\|bt\|bv\|bw\|by\|bz\|ca\|cc\|cd\|cf\|cg\|ch\|ci\|ck\|cl\|cm\|cn\|co\|cr\|cs\|cu\|cv\|cx\|cy\|cz\|dd\|de\|dj\|dk\|dm\|do\|dz\|ec\|ee\|eg\|eh\|er\|es\|et\|eu\|fi\|fj\|fk\|fm\|fo\|fr\|ga\|gb\|gd\|ge\|gf\|gg\|gh\|gi\|gl\|gm\|gn\|gp\|gq\|gr\|gs\|gt\|gu\|gw\|gy\|hk\|hm\|hn\|hr\|ht\|hu\|id\|ie\|il\|im\|in\|io\|iq\|ir\|is\|it\|je\|jm\|jo\|jp\|ke\|kg\|kh\|ki\|km\|kn\|kp\|kr\|kw\|ky\|kz\|la\|lb\|lc\|li\|lk\|lr\|ls\|lt\|lu\|lv\|ly\|ma\|mc\|md\|me\|mg\|mh\|mk\|ml\|mm\|mn\|mo\|mp\|mq\|mr\|ms\|mt\|mu\|mv\|mw\|mx\|my\|mz\|na\|nc\|ne\|nf\|ng\|ni\|nl\|no\|np\|nr\|nu\|nz\|om\|pa\|pe\|pf\|pg\|ph\|pk\|pl\|pm\|pn\|pr\|ps\|pt\|pw\|py\|qa\|re\|ro\|rs\|ru\|rw\|sa\|sb\|sc\|sd\|se\|sg\|sh\|si\|sj\|sk\|sl\|sm\|sn\|so\|sr\|ss\|st\|su\|sv\|sy\|sz\|tc\|td\|tf\|tg\|th\|tj\|tk\|tl\|tm\|tn\|to\|tp\|tr\|tt\|tv\|tw\|tz\|ua\|ug\|uk\|us\|uy\|uz\|va\|vc\|ve\|vg\|vi\|vn\|vu\|wf\|ws\|ye\|yt\|za\|zm\|zw\|xn--lgbbat1ad8j\|xn--fiqs8s\|xn--fiqz9s\|xn--wgbh1c\|xn--j6w193g\|xn--h2brj9c\|xn--mgbbh1a71e\|xn--fpcrj9c3d\|xn--gecrj9c\|xn--s9brj9c\|xn--xkc2dl3a5ee0h\|xn--45brj9c\|xn--mgba3a4f16a\|xn--mgbayh7gpa\|xn--mgbc0a9azcg\|xn--ygbi2ammx\|xn--wgbl6a\|xn--p1ai\|xn--mgberp4a5d4ar\|xn--90a3ac\|xn--yfro4i67o\|xn--clchc0ea0b2g2a9gcd\|xn--3e0b707e\|xn--fzc2c9e2c\|xn--xkc2al3hye2a\|xn--mgbtf8fl\|xn--kprw13d\|xn--kpry57d\|xn--o3cw4h\|xn--pgbs0dh\|xn--mgbaam7a8h\|xn--54b7fta0cc\|xn--90ae\|xn--node\|xn--4dbrk0ce\|xn--80ao21a\|xn--mgb9awbf\|xn--mgbai9azgqp6j\|xn--j1amh\|xn--mgb2ddes\|xn--kgbechtv\|xn--hgbk6aj7f53bba\|xn--0zwm56d\|xn--g6w251d\|xn--80akhbyknj4f\|xn--11b5bs3a9aj6g\|xn--jxalpdlp\|xn--9t4b11yi5a\|xn--deba0ad\|xn--zckzah\|xn--hlcj6aya9esc7a';
16	private $api = false;
17	private $db = false;
18	public function __sleep(){
19	$this->writeHosts();
20	return array('debug', 'errorMsg', 'table', 'apiKey', 'wordpressVersion', 'dRegex');
21	}
22	public function __wakeup(){
23	$this->hostsToAdd = new wfArray(array('owner', 'host', 'path', 'hostKey'));
24	$this->api = new wfAPI($this->apiKey, $this->wordpressVersion);
25	$this->db = new wfDB();
26	}
27	public function __construct($apiKey, $wordpressVersion, $db = false){
28	$this->hostsToAdd = new wfArray(array('owner', 'host', 'path', 'hostKey'));
29	$this->apiKey = $apiKey;
30	$this->wordpressVersion = $wordpressVersion;
31	$this->api = new wfAPI($apiKey, $wordpressVersion);
32	if($db){
33	$this->db = $db;
34	} else {
35	$this->db = new wfDB();
36	}
37	global $wpdb;
38	if(isset($wpdb)){
39	$this->table = $wpdb->base_prefix . 'wfHoover';
40	} else {
41	$this->table = 'wp_wfHoover';
42	}
43	$this->db->truncate($this->table);
44	}
45	public function cleanup(){
46	$this->db->truncate($this->table);
47	}
48	public function hoover($id, $data){
49	if(strpos($data, '.') === false){
50	return;
51	}
52	$this->currentHooverID = $id;
53	try {
54	@preg_replace_callback("/(?<=^\|[^a-zA-Z0-9\-])((?:[a-zA-Z0-9\-]+\.)+)(" . $this->dRegex . ")($\|[\r\n\s\t]\|\/[^\r\n\s\t\"\'\$\{\}<>]*)/i", array($this, 'addHost'), $data);
55	//((?:$\|[^a-zA-Z0-9\-\.\'\"])[^\r\n\s\t\"\'\$\{\}<>]*)
56	//"\$this->" . "addHost(\$id, '$1$2', '$3')", $data);
57	} catch(Exception $e){
58	//error_log("Regex error 1: $e");
59	}
60	@preg_replace_callback("/(?<=[^\d]\|^)(\d{8,10}\|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})($\|[\r\n\s\t]\|\/[^\r\n\s\t\"\'\$\{\}<>]*)/", array($this, 'addIP'), $data);
61	//([^\d\'\"][^\r\n\s\t\"\'\$\{\}<>]*)
62	//"\$this->" . "addIP(\$id, \"$1\",\"$2\")", $data);
63	$this->writeHosts();
64	}
65	private function dbg($msg){
66	if($this->debug){
67	//error_log("DEBUG: $msg\n");
68	}
69	}
70	public function addHost($matches){
71	$id = $this->currentHooverID;
72	$host = $matches[1] . $matches[2];
73	$path = $matches[3];
74	if(strpos($path, '/') !== 0){
75	$path = '/';
76	} else {
77	$path = preg_replace_callback('/([^A-Za-z0-9\-\.\_\~:\/\?\#\[\]\@\!\$\&\'\*\+\,;\=]+)/', 'wordfenceURLHoover::urlenc', $path);
78	}
79	$host = strtolower($host);
80	$hostParts = explode('.', $host);
81	if(sizeof($hostParts) == 2){
82	$hostKey = substr(hash('sha256', $hostParts[0] . '.' . $hostParts[1] . '/', true), 0, 4);
83	$this->hostsToAdd->push(array('owner' => $id, 'host' => $host, 'path' => $path, 'hostKey' => $hostKey));
84	} else if(sizeof($hostParts) > 2){
85	$hostKeyThreeParts = substr(hash('sha256',$hostParts[sizeof($hostParts) - 3] . '.' . $hostParts[sizeof($hostParts) - 2] . '.' . $hostParts[sizeof($hostParts) - 1] . '/', true), 0, 4);
86	$hostKeyTwoParts = substr(hash('sha256', $hostParts[sizeof($hostParts) - 2] . '.' . $hostParts[sizeof($hostParts) - 1] . '/', true), 0, 4);
87	$this->hostsToAdd->push(array('owner' => $id, 'host' => $host, 'path' => $path, 'hostKey' => $hostKeyThreeParts));
88	$this->hostsToAdd->push(array('owner' => $id, 'host' => $host, 'path' => $path, 'hostKey' => $hostKeyTwoParts));
89	}
90	if($this->hostsToAdd->size() > 1000){ $this->writeHosts(); }
91	}
92	public function addIP($matches){
93	$id = $this->currentHooverID;
94	$ipdata = $matches[1];
95	$path = $matches[2];
96	$this->dbg("Add IP called with $ipdata $path");
97	if(strstr($ipdata, '.') === false){
98	if($ipdata >= 16777216 && $ipdata <= 4026531840){
99	$ipdata = long2ip($ipdata);
100	} else {
101	return; //Is int but invalid address.
102	}
103	}
104	$parts = explode('.', $ipdata);
105	foreach($parts as $part){
106	if($part < 0 \|\| $part > 255){
107	return;
108	}
109	}
110	if(wfUtils::isPrivateAddress($ipdata) ){
111	return;
112	}
113	if(strlen($path) == 1){
114	$path = '/'; //Because it's either a whitespace char or a / anyway.
115	} else if(strlen($path) > 1){
116	$path = preg_replace_callback('/([^A-Za-z0-9\-\.\_\~:\/\?\#\[\]\@\!\$\&\'\*\+\,;\=]+)/', 'wordfenceURLHoover::urlenc', $path);
117	}
118	$hostKey = substr(hash('sha256', $ipdata . '/', true), 0, 4);
119	$this->hostsToAdd->push(array('owner' => $id, 'host' => $ipdata, 'path' => $path, 'hostKey' => $hostKey));
120	if($this->hostsToAdd->size() > 1000){ $this->writeHosts(); }
121	}
122	public static function urlenc($m){
123	return urlencode($m[1]);
124	}
125	private function writeHosts(){
126	if($this->hostsToAdd->size() < 1){ return; }
127	if($this->useDB){
128	$sql = "insert into " . $this->table . " (owner, host, path, hostKey) values ";
129	while($elem = $this->hostsToAdd->shift()){
130	//This may be an issue for hyperDB or other abstraction layers, but leaving it for now.
131	$sql .= sprintf("('%s', '%s', '%s', '%s'),",
132	$this->db->realEscape($elem['owner']),
133	$this->db->realEscape($elem['host']),
134	$this->db->realEscape($elem['path']),
135	$this->db->realEscape($elem['hostKey'])
136	);
137	}
138	$sql = rtrim($sql, ',');
139	$this->db->queryWrite($sql);
140	} else {
141	while($elem = $this->hostsToAdd->shift()){
142	$this->hostKeys[] = $elem['hostKey'];
143	$this->hostList[] = array(
144	'owner' => $elem['owner'],
145	'host' => $elem['host'],
146	'path' => $elem['path'],
147	'hostKey' => $elem['hostKey']
148	);
149	}
150	}
151	}
152	public function getBaddies(){
153	$allHostKeys = array();
154	$stime = microtime(true);
155	$allHostKeys = array();
156	if($this->useDB){
157	$q1 = $this->db->querySelect("select distinct hostKey as hostKey from $this->table");
158	foreach($q1 as $hRec){
159	$allHostKeys[] = $hRec['hostKey'];
160	}
161	} else {
162	$allHostKeys = $this->hostKeys;
163	}
164	//Now call API and check if any hostkeys are bad.
165	//This is a shortcut, because if no hostkeys are bad it saves us having to check URLs
166	if(sizeof($allHostKeys) > 0){ //If we don't have any hostkeys, then we won't have any URL's to check either.
167	//Hostkeys are 4 byte sha256 prefixes
168	//Returned value is 2 byte shorts which are array indexes for bad keys that were passed in the original list
169	$this->dbg("Checking " . sizeof($allHostKeys) . " hostkeys");
170	if($this->debug){
171	foreach($allHostKeys as $key){
172	$this->dbg("Checking hostkey: " . bin2hex($key));
173	}
174	}
175	wordfence::status(2, 'info', "Checking " . sizeof($allHostKeys) . " host keys against Wordfence scanning servers.");
176	$resp = $this->api->binCall('check_host_keys', implode('', $allHostKeys));
177	wordfence::status(2, 'info', "Done host key check.");
178	$this->dbg("Done hostkey check");
179
180	$badHostKeys = array();
181	if($resp['code'] == 200){
182	if(strlen($resp['data']) > 0){
183	$dataLen = strlen($resp['data']);
184	if($dataLen % 2 != 0){
185	$this->errorMsg = "Invalid data length received from Wordfence server: " . $dataLen;
186	return false;
187	}
188	for($i = 0; $i < $dataLen; $i += 2){
189	$idxArr = unpack('n', substr($resp['data'], $i, 2));
190	$idx = $idxArr[1];
191	if(isset($allHostKeys[$idx]) ){
192	$badHostKeys[] = $allHostKeys[$idx];
193	$this->dbg("Got bad hostkey for record: " . var_export($allHostKeys[$idx], true));
194	} else {
195	$this->dbg("Bad allHostKeys index: $idx");
196	$this->errorMsg = "Bad allHostKeys index: $idx";
197	return false;
198	}
199	}
200	}
201	} else {
202	$this->errorMsg = "Wordfence server responded with an error. HTTP code " . $resp['code'] . " and data: " . $resp['data'];
203	return false;
204	}
205	if(sizeof($badHostKeys) > 0){
206	$urlsToCheck = array();
207	$totalURLs = 0;
208	//need to figure out which id's have bad hostkeys
209	//need to feed in all URL's from those id's where the hostkey matches a URL
210	foreach($badHostKeys as $badHostKey){
211	if($this->useDB){
212	//Putting a 10000 limit in here for sites that have a huge number of items with the same URL that repeats.
213	// This is an edge case. But if the URLs are malicious then presumably the admin will fix the malicious URLs
214	// and on subsequent scans the items (owners) that are above the 10000 limit will appear.
215	$q1 = $this->db->querySelect("select owner, host, path from $this->table where hostKey='%s' limit 10000", $badHostKey);
216	foreach($q1 as $rec){
217	$url = 'http://' . $rec['host'] . $rec['path'];
218	if(! isset($urlsToCheck[$rec['owner']])){
219	$urlsToCheck[$rec['owner']] = array();
220	}
221	if(! in_array($url, $urlsToCheck[$rec['owner']])){
222	$urlsToCheck[$rec['owner']][] = $url;
223	$totalURLs++;
224	}
225	}
226	} else {
227	foreach($this->hostList as $rec){
228	if($rec['hostKey'] == $badHostKey){
229	$url = 'http://' . $rec['host'] . $rec['path'];
230	if(! isset($urlsToCheck[$rec['owner']])){
231	$urlsToCheck[$rec['owner']] = array();
232	}
233	if(! in_array($url, $urlsToCheck[$rec['owner']])){
234	$urlsToCheck[$rec['owner']][] = $url;
235	$totalURLs++;
236	}
237	}
238	}
239	}
240	}
241
242	if(sizeof($urlsToCheck) > 0){
243	wordfence::status(2, 'info', "Checking " . $totalURLs . " URLs from " . sizeof($urlsToCheck) . " sources.");
244	$badURLs = $this->api->call('check_bad_urls', array(), array( 'toCheck' => json_encode($urlsToCheck)) );
245	wordfence::status(2, 'info', "Done URL check.");
246	$this->dbg("Done URL check");
247	if(is_array($badURLs) && sizeof($badURLs) > 0){
248	$finalResults = array();
249	foreach($badURLs as $file => $badSiteList){
250	if(! isset($finalResults[$file])){
251	$finalResults[$file] = array();
252	}
253	foreach($badSiteList as $badSite){
254	$finalResults[$file][] = array(
255	'URL' => $badSite[0],
256	'badList' => $badSite[1]
257	);
258	}
259	}
260	return $finalResults;
261	} else {
262	return array();
263	}
264	} else {
265	return array();
266	}
267	} else {
268	return array();
269	}
270	} else {
271	return array();
272	}
273	}
274	}
275	?>
276