crawler-detect.php — Post Views Counter 1.3.8

post-views-counter / includes / crawler-detect.php

post-views-counter / includes Last commit date

crawler-detect.php

970 lines

1	<?php
2	// exit if accessed directly
3	if ( ! defined( 'ABSPATH' ) )
4	exit;
5
6	/**
7	* Post_Views_Counter_Crawler_Detect class.
8	*
9	* Based on CrawlerDetect php class adjusted to PHP 5.2
10	* https://github.com/JayBizzle/Crawler-Detect/blob/master/src/CrawlerDetect.php
11	*
12	* @since 1.2.4
13	* @class Post_Views_Counter_Crawler_Detect
14	*/
15	class Post_Views_Counter_Crawler_Detect {
16
17	/**
18	* The user agent.
19	*
20	* @var null
21	*/
22	protected $user_agent = null;
23
24	/**
25	* Headers that contain a user agent.
26	*
27	* @var array
28	*/
29	protected $http_headers = array();
30
31	/**
32	* Store regex matches.
33	*
34	* @var array
35	*/
36	protected $matches = array();
37
38	/**
39	* Crawlers object.
40	*
41	* @var object
42	*/
43	protected $crawlers = array();
44
45	/**
46	* Exclusions object.
47	*
48	* @var object
49	*/
50	protected $exclusions = array();
51
52	/**
53	* Headers object.
54	*
55	* @var object
56	*/
57	protected $ua_http_headers;
58
59	/**
60	* Class constructor.
61	*/
62	public function __construct() {
63	$this->crawlers = $this->get_crawlers_list();
64	$this->exclusions = $this->get_exclusions_list();
65
66	add_action( 'after_setup_theme', array( $this, 'init' ) );
67	}
68
69	/**
70	* Initialize class.
71	*/
72	public function init() {
73	// break on admin side
74	if ( is_admin() && ! ( defined( 'DOING_AJAX' ) && DOING_AJAX ) )
75	return;
76
77	$this->ua_http_headers = $this->get_headers_list();
78	$this->set_http_headers();
79	$this->set_user_agent();
80	}
81
82	/**
83	* Set HTTP headers.
84	*
85	* @param array $http_headers
86	*/
87	public function set_http_headers( $http_headers = null ) {
88	// use global _SERVER if $http_headers aren't defined
89	if ( ! is_array( $http_headers ) \|\| ! count( $http_headers ) ) {
90	$http_headers = $_SERVER;
91	}
92	// clear existing headers
93	$this->http_headers = array();
94	// only save HTTP headers - in PHP land, that means only _SERVER vars that start with HTTP_.
95	foreach ( $http_headers as $key => $value ) {
96	if ( substr( $key, 0, 5 ) === 'HTTP_' ) {
97	$this->http_headers[$key] = $value;
98	}
99	}
100	}
101
102	/**
103	* Return user agent headers.
104	*
105	* @return array
106	*/
107	public function get_ua_http_headers() {
108	return $this->ua_http_headers;
109	}
110
111	/**
112	* Return the user agent.
113	*
114	* @return string
115	*/
116	public function get_user_agent() {
117	return $this->user_agent;
118	}
119
120	/**
121	* Set the user agent.
122	*
123	* @param string $user_agent
124	*/
125	public function set_user_agent( $user_agent = null ) {
126	if ( false === empty( $user_agent ) ) {
127	return $this->user_agent = $user_agent;
128	} else {
129	$this->user_agent = null;
130	foreach ( $this->get_ua_http_headers() as $alt_header ) {
131	if ( false === empty( $this->http_headers[$alt_header] ) ) { // @todo: should use get_http_header(), but it would be slow.
132	$this->user_agent .= $this->http_headers[$alt_header] . ' ';
133	}
134	}
135	return $this->user_agent = ( ! empty( $this->user_agent ) ? trim( $this->user_agent ) : null);
136	}
137	}
138
139	/**
140	* Build the user agent regex.
141	*
142	* @return string
143	*/
144	public function get_regex() {
145	return '(' . implode( '\|', $this->crawlers ) . ')';
146	}
147
148	/**
149	* Build the replacement regex.
150	*
151	* @return string
152	*/
153	public function get_exclusions() {
154	return '(' . implode( '\|', $this->exclusions ) . ')';
155	}
156
157	/**
158	* Check user agent string against the regex.
159	*
160	* @param string $user_agent
161	*
162	* @return bool
163	*/
164	public function is_crawler( $user_agent = null ) {
165	$agent = is_null( $user_agent ) ? $this->user_agent : $user_agent;
166	$agent = preg_replace( '/' . $this->get_exclusions() . '/i', '', $agent );
167	if ( strlen( trim( $agent ) ) == 0 ) {
168	return false;
169	} else {
170	$result = preg_match( '/' . $this->get_regex() . '/i', trim( $agent ), $matches );
171	}
172	if ( $matches ) {
173	$this->matches = $matches;
174	}
175	return (bool) $result;
176	}
177
178	/**
179	* Return the matches.
180	*
181	* @return string
182	*/
183	public function get_matches() {
184	return isset( $this->matches[0] ) ? $this->matches[0] : null;
185	}
186
187	/**
188	* Return the regular expressions to match against the user agent.
189	*
190	* @return array
191	*/
192	protected function get_crawlers_list() {
193	$data = array(
194	'.Java.outbrain',
195	'008\/',
196	'192.comAgent',
197	'2ip\.ru',
198	'404checker',
199	'^bluefish ',
200	'^FDM ',
201	'^Goose\/',
202	'^Java\/',
203	'^Mget',
204	'^NG\/[0-9\.]',
205	'^NING\/',
206	'^PHP\/[0-9]',
207	'^RMA\/',
208	'^Ruby\|Ruby\/[0-9]',
209	'^scrutiny\/',
210	'^VSE\/[0-9]',
211	'^WordPress\.com',
212	'^XRL\/[0-9]',
213	'a3logics\.in',
214	'A6-Indexer',
215	'a\.pr-cy\.ru',
216	'Aboundex',
217	'aboutthedomain',
218	'Accoona-AI-Agent',
219	'acoon',
220	'acrylicapps\.com\/pulp',
221	'adbeat',
222	'AddThis',
223	'ADmantX',
224	'adressendeutschland',
225	'Advanced Email Extractor v',
226	'agentslug',
227	'AHC',
228	'aihit',
229	'aiohttp\/',
230	'Airmail',
231	'akula\/',
232	'alertra',
233	'alexa site audit',
234	'alyze\.info',
235	'amagit',
236	'AndroidDownloadManager',
237	'Anemone',
238	'Ant\.com',
239	'Anturis Agent',
240	'AnyEvent-HTTP\/',
241	'Apache-HttpClient\/',
242	'AportWorm\/[0-9]',
243	'AppEngine-Google',
244	'Arachmo',
245	'arachnode',
246	'Arachnophilia',
247	'archive-com',
248	'aria2',
249	'asafaweb.com',
250	'AskQuickly',
251	'Astute',
252	'autocite',
253	'Autonomy',
254	'B-l-i-t-z-B-O-T',
255	'Backlink-Ceck\.de',
256	'Bad-Neighborhood',
257	'baidu\.com',
258	'baypup\/[0-9]',
259	'baypup\/colbert',
260	'BazQux',
261	'BCKLINKS',
262	'BDFetch',
263	'BegunAdvertising\/',
264	'bibnum\.bnf',
265	'BigBozz',
266	'biglotron',
267	'BingLocalSearch',
268	'BingPreview',
269	'binlar',
270	'biz_Directory',
271	'Blackboard Safeassign',
272	'Bloglovin',
273	'BlogPulseLive',
274	'BlogSearch',
275	'Blogtrottr',
276	'boitho\.com-dc',
277	'BPImageWalker',
278	'Braintree-Webhooks',
279	'Branch Metrics API',
280	'Branch-Passthrough',
281	'Browsershots',
282	'BUbiNG',
283	'Butterfly\/',
284	'BuzzSumo',
285	'CakePHP',
286	'CapsuleChecker',
287	'CaretNail',
288	'cb crawl',
289	'CC Metadata Scaper',
290	'Cerberian Drtrs',
291	'CERT\.at-Statistics-Survey',
292	'cg-eye',
293	'changedetection',
294	'Charlotte',
295	'CheckHost',
296	'chkme\.com',
297	'CirrusExplorer\/',
298	'CISPA Vulnerability Notification',
299	'CJNetworkQuality',
300	'clips\.ua\.ac\.be',
301	'Cloud mapping experiment',
302	'CloudFlare-AlwaysOnline',
303	'Cloudinary\/[0-9]',
304	'cmcm\.com',
305	'coccoc',
306	'CommaFeed',
307	'Commons-HttpClient',
308	'Comodo SSL Checker',
309	'contactbigdatafr',
310	'convera',
311	'copyright sheriff',
312	'cosmos\/[0-9]',
313	'Covario-IDS',
314	'CrawlForMe\/[0-9]',
315	'cron-job\.org',
316	'Crowsnest',
317	'curb',
318	'Curious George',
319	'curl',
320	'cuwhois\/[0-9]',
321	'CyberPatrol',
322	'cybo\.com',
323	'DareBoost',
324	'DataparkSearch',
325	'dataprovider',
326	'Daum(oa)?[ \/][0-9]',
327	'DeuSu',
328	'developers\.google\.com\/\+\/web\/snippet\/',
329	'Digg',
330	'Dispatch\/',
331	'dlvr',
332	'DNS-Tools Header-Analyzer',
333	'DNSPod-reporting',
334	'docoloc',
335	'DomainAppender',
336	'dotSemantic',
337	'downforeveryoneorjustme',
338	'downnotifier\.com',
339	'DowntimeDetector',
340	'Dragonfly File Reader',
341	'drupact',
342	'Drupal (\+http:\/\/drupal\.org\/)',
343	'dubaiindex',
344	'EARTHCOM',
345	'Easy-Thumb',
346	'ec2linkfinder',
347	'eCairn-Grabber',
348	'ECCP',
349	'ElectricMonk',
350	'elefent',
351	'EMail Exractor',
352	'EmailWolf',
353	'Embed PHP Library',
354	'Embedly',
355	'europarchive\.org',
356	'evc-batch\/[0-9]',
357	'EventMachine HttpClient',
358	'Evidon',
359	'Evrinid',
360	'ExactSearch',
361	'ExaleadCloudview',
362	'Excel\/',
363	'Exploratodo',
364	'ezooms',
365	'facebookexternalhit',
366	'facebookplatform',
367	'fairshare',
368	'Faraday v',
369	'Faveeo',
370	'Favicon downloader',
371	'FavOrg',
372	'Feed Wrangler',
373	'Feedbin',
374	'FeedBooster',
375	'FeedBucket',
376	'FeedBurner',
377	'FeedChecker',
378	'Feedly',
379	'Feedspot',
380	'feeltiptop',
381	'Fetch API',
382	'Fetch\/[0-9]',
383	'Fever\/[0-9]',
384	'findlink',
385	'findthatfile',
386	'Flamingo_SearchEngine',
387	'FlipboardBrowserProxy',
388	'FlipboardProxy',
389	'FlipboardRSS',
390	'fluffy',
391	'flynxapp',
392	'forensiq',
393	'FoundSeoTool\/[0-9]',
394	'free thumbnails',
395	'FreeWebMonitoring SiteChecker',
396	'Funnelback',
397	'g00g1e\.net',
398	'GAChecker',
399	'geek-tools',
400	'Genderanalyzer',
401	'Genieo',
402	'GentleSource',
403	'GetLinkInfo',
404	'getprismatic\.com',
405	'GetURLInfo\/[0-9]',
406	'GigablastOpenSource',
407	'Go [\d\.]* package http',
408	'Go-http-client',
409	'GomezAgent',
410	'gooblog',
411	'Goodzer\/[0-9]',
412	'Google favicon',
413	'Google Keyword Suggestion',
414	'Google Keyword Tool',
415	'Google Page Speed Insights',
416	'Google PP Default',
417	'Google Search Console',
418	'Google Web Preview',
419	'Google-Adwords',
420	'Google-Apps-Script',
421	'Google-Calendar-Importer',
422	'Google-HTTP-Java-Client',
423	'Google-Publisher-Plugin',
424	'Google-SearchByImage',
425	'Google-Site-Verification',
426	'Google-Structured-Data-Testing-Tool',
427	'google_partner_monitoring',
428	'GoogleDocs',
429	'GoogleHC\/',
430	'GoogleProducer',
431	'GoScraper',
432	'GoSpotCheck',
433	'GoSquared-Status-Checker',
434	'gosquared-thumbnailer',
435	'GotSiteMonitor',
436	'Grammarly',
437	'grouphigh',
438	'grub-client',
439	'GTmetrix',
440	'Hatena',
441	'hawkReader',
442	'HEADMasterSEO',
443	'HeartRails_Capture',
444	'heritrix',
445	'hledejLevne\.cz\/[0-9]',
446	'Holmes',
447	'HootSuite Image proxy',
448	'Hootsuite-WebFeed\/[0-9]',
449	'HostTracker',
450	'ht:\/\/check',
451	'htdig',
452	'HTMLParser\/',
453	'HTTP-Header-Abfrage',
454	'http-kit',
455	'HTTP-Tiny',
456	'HTTP_Compression_Test',
457	'http_request2',
458	'http_requester',
459	'HttpComponents',
460	'httphr',
461	'HTTPMon',
462	'httpscheck',
463	'httpssites_power',
464	'httpunit',
465	'HttpUrlConnection',
466	'httrack',
467	'hosterstats',
468	'huaweisymantec',
469	'HubPages.*crawlingpolicy',
470	'HubSpot Connect',
471	'HubSpot Marketing Grader',
472	'HyperZbozi.cz Feeder',
473	'ichiro',
474	'IdeelaborPlagiaat',
475	'IDG Twitter Links Resolver',
476	'IDwhois\/[0-9]',
477	'Iframely',
478	'igdeSpyder',
479	'IlTrovatore',
480	'ImageEngine\/',
481	'Imagga',
482	'InAGist',
483	'inbound\.li parser',
484	'InDesign%20CC',
485	'infegy',
486	'infohelfer',
487	'InfoWizards Reciprocal Link System PRO',
488	'inpwrd\.com',
489	'Integrity',
490	'integromedb',
491	'internet_archive',
492	'InternetSeer',
493	'internetVista monitor',
494	'IODC',
495	'IOI',
496	'ips-agent',
497	'iqdb\/',
498	'Irokez',
499	'isitup\.org',
500	'iskanie',
501	'iZSearch',
502	'janforman',
503	'Jigsaw',
504	'Jobboerse',
505	'jobo',
506	'Jobrapido',
507	'KeepRight OpenStreetMap Checker',
508	'KimonoLabs\/',
509	'knows\.is',
510	'kouio',
511	'KrOWLer',
512	'kulturarw3',
513	'KumKie',
514	'L\.webis',
515	'Larbin',
516	'LayeredExtractor',
517	'LibVLC',
518	'libwww',
519	'link checker',
520	'Link Valet',
521	'link_thumbnailer',
522	'linkCheck',
523	'linkdex',
524	'LinkExaminer',
525	'linkfluence',
526	'linkpeek',
527	'LinkTiger',
528	'LinkWalker',
529	'Lipperhey',
530	'livedoor ScreenShot',
531	'LoadImpactPageAnalyzer',
532	'LoadImpactRload',
533	'LongURL API',
534	'looksystems\.net',
535	'ltx71',
536	'lwp-trivial',
537	'lycos',
538	'LYT\.SR',
539	'mabontland',
540	'MagpieRSS',
541	'Mail.Ru',
542	'MailChimp\.com',
543	'Mandrill',
544	'marketinggrader',
545	'Mediapartners-Google',
546	'MegaIndex\.ru',
547	'Melvil Rawi\/',
548	'MergeFlow-PageReader',
549	'MetaInspector',
550	'Metaspinner',
551	'MetaURI',
552	'Microsearch',
553	'Microsoft Office ',
554	'Microsoft Windows Network Diagnostics',
555	'Mindjet',
556	'Miniflux',
557	'Mnogosearch',
558	'mogimogi',
559	'Mojolicious (Perl)',
560	'monitis',
561	'Monitority\/[0-9]',
562	'montastic',
563	'MonTools',
564	'Moreover',
565	'Morning Paper',
566	'mowser',
567	'Mrcgiguy',
568	'mShots',
569	'MVAClient',
570	'nagios',
571	'Najdi\.si\/',
572	'NETCRAFT',
573	'NetLyzer FastProbe',
574	'netresearch',
575	'NetShelter ContentScan',
576	'NetTrack',
577	'Netvibes',
578	'Neustar WPM',
579	'NeutrinoAPI',
580	'NewsBlur .*Finder',
581	'NewsGator',
582	'newsme',
583	'newspaper\/',
584	'NG-Search',
585	'nineconnections\.com',
586	'NLNZ_IAHarvester',
587	'Nmap Scripting Engine',
588	'node-superagent',
589	'node\.io',
590	'nominet\.org\.uk',
591	'Norton-Safeweb',
592	'Notifixious',
593	'notifyninja',
594	'nuhk',
595	'nutch',
596	'Nuzzel',
597	'nWormFeedFinder',
598	'Nymesis',
599	'Ocelli\/[0-9]',
600	'oegp',
601	'okhttp',
602	'Omea Reader',
603	'omgili',
604	'Online Domain Tools',
605	'OpenCalaisSemanticProxy',
606	'Openstat\/',
607	'OpenVAS',
608	'Optimizer',
609	'Orbiter',
610	'OrgProbe\/[0-9]',
611	'ow\.ly',
612	'ownCloud News',
613	'Page Analyzer',
614	'Page Valet',
615	'page2rss',
616	'page_verifier',
617	'PagePeeker',
618	'Pagespeed\/[0-9]',
619	'Panopta',
620	'panscient',
621	'parsijoo',
622	'PayPal IPN',
623	'Pcore-HTTP',
624	'Pearltrees',
625	'peerindex',
626	'Peew',
627	'PhantomJS\/',
628	'Photon\/',
629	'phpcrawl',
630	'phpservermon',
631	'Pi-Monster',
632	'Pingdom\.com',
633	'Pingoscope',
634	'PingSpot',
635	'Pinterest',
636	'Pizilla',
637	'Ploetz \+ Zeller',
638	'Plukkie',
639	'PocketParser',
640	'Pompos',
641	'Porkbun',
642	'Port Monitor',
643	'postano',
644	'PostPost',
645	'postrank',
646	'PowerPoint\/',
647	'Priceonomics Analysis Engine',
648	'Prlog',
649	'probethenet',
650	'Project 25499',
651	'Promotion_Tools_www.searchenginepromotionhelp.com',
652	'prospectb2b',
653	'Protopage',
654	'proximic',
655	'PTST ',
656	'PTST\/[0-9]+',
657	'Pulsepoint XT3 web scraper',
658	'Python-httplib2',
659	'python-requests',
660	'Python-urllib',
661	'Qirina Hurdler',
662	'Qseero',
663	'Qualidator.com SiteAnalyzer',
664	'Quora Link Preview',
665	'Qwantify',
666	'Radian6',
667	'RankSonicSiteAuditor',
668	'Readability',
669	'RealPlayer%20Downloader',
670	'RebelMouse',
671	'redback\/',
672	'Redirect Checker Tool',
673	'ReederForMac',
674	'ResponseCodeTest\/[0-9]',
675	'RestSharp',
676	'RetrevoPageAnalyzer',
677	'Riddler',
678	'Rival IQ',
679	'Robosourcer',
680	'Robozilla\/[0-9]',
681	'ROI Hunter',
682	'SalesIntelligent',
683	'SauceNAO',
684	'SBIder',
685	'Scoop',
686	'scooter',
687	'ScoutJet',
688	'ScoutURLMonitor',
689	'Scrapy',
690	'Scrubby',
691	'SearchSight',
692	'semanticdiscovery',
693	'semanticjuice',
694	'SEO Browser',
695	'Seo Servis',
696	'seo-nastroj.cz',
697	'Seobility',
698	'SEOCentro',
699	'SeoCheck',
700	'SeopultContentAnalyzer',
701	'SEOstats',
702	'Server Density Service Monitoring',
703	'servernfo\.com',
704	'Seznam screenshot-generator',
705	'Shelob',
706	'Shoppimon Analyzer',
707	'ShoppimonAgent\/[0-9]',
708	'ShopWiki',
709	'ShortLinkTranslate',
710	'shrinktheweb',
711	'SilverReader',
712	'SimplePie',
713	'SimplyFast',
714	'Site-Shot\/',
715	'Site24x7',
716	'SiteBar',
717	'SiteCondor',
718	'siteexplorer\.info',
719	'SiteGuardian',
720	'Siteimprove\.com',
721	'Sitemap(s)? Generator',
722	'Siteshooter B0t',
723	'SiteTruth',
724	'sitexy\.com',
725	'SkypeUriPreview',
726	'slider\.com',
727	'slurp',
728	'SMRF URL Expander',
729	'Snappy',
730	'SniffRSS',
731	'sniptracker',
732	'Snoopy',
733	'sogou web',
734	'SortSite',
735	'spaziodati',
736	'Specificfeeds',
737	'speedy',
738	'SPEng',
739	'Spinn3r',
740	'spray-can',
741	'Sprinklr ',
742	'spyonweb',
743	'Sqworm',
744	'SSL Labs',
745	'StackRambler',
746	'Statastico\/',
747	'StatusCake',
748	'Stratagems Kumo',
749	'Stroke.cz',
750	'StudioFACA',
751	'suchen',
752	'summify',
753	'Super Monitoring',
754	'Surphace Scout',
755	'SwiteScraper',
756	'Symfony2 BrowserKit',
757	'Sysomos',
758	'T0PHackTeam',
759	'Tarantula\/',
760	'teoma',
761	'terrainformatica\.com',
762	'The Expert HTML Source Viewer',
763	'theinternetrules',
764	'theoldreader\.com',
765	'Thumbshots',
766	'ThumbSniper',
767	'TinEye',
768	'Tiny Tiny RSS',
769	'topster',
770	'touche.com',
771	'Traackr.com',
772	'truwoGPS',
773	'tweetedtimes\.com',
774	'Tweetminster',
775	'Twikle',
776	'Twingly',
777	'Typhoeus',
778	'ubermetrics-technologies',
779	'uclassify',
780	'UdmSearch',
781	'UnwindFetchor',
782	'updated',
783	'Upflow',
784	'URLChecker',
785	'URLitor.com',
786	'urlresolver',
787	'Urlstat',
788	'UrlTrends Ranking Updater',
789	'Vagabondo',
790	'via ggpht\.com GoogleImageProxy',
791	'visionutils',
792	'vkShare',
793	'voltron',
794	'Vortex\/[0-9]',
795	'voyager\/',
796	'VSAgent\/[0-9]',
797	'VSB-TUO\/[0-9]',
798	'VYU2',
799	'w3af\.org',
800	'W3C-checklink',
801	'W3C-mobileOK',
802	'W3C_I18n-Checker',
803	'W3C_Unicorn',
804	'wangling',
805	'Wappalyzer',
806	'WatchMouse',
807	'WbSrch\/',
808	'web-capture\.net',
809	'Web-Monitoring',
810	'Web-sniffer',
811	'Webauskunft',
812	'WebCapture',
813	'webcollage',
814	'WebCookies',
815	'WebCorp',
816	'WebDoc',
817	'WebFetch',
818	'WebImages',
819	'WebIndex',
820	'webkit2png',
821	'webmastercoffee',
822	'webmon ',
823	'webscreenie',
824	'Webshot',
825	'Website Analyzer\/',
826	'websitepulse[+ ]checker',
827	'Websnapr\/',
828	'Websquash\.com',
829	'Webthumb\/[0-9]',
830	'WebThumbnail',
831	'WeCrawlForThePeace',
832	'WeLikeLinks',
833	'WEPA',
834	'WeSEE',
835	'wf84',
836	'wget',
837	'WhatsApp',
838	'WhatsMyIP',
839	'WhatWeb',
840	'Whibse',
841	'Whynder Magnet',
842	'Windows-RSS-Platform',
843	'WinHttpRequest',
844	'wkhtmlto',
845	'wmtips',
846	'Woko',
847	'WomlpeFactory',
848	'Word\/',
849	'WordPress\/',
850	'wotbox',
851	'WP Engine Install Performance API',
852	'WPScan',
853	'wscheck',
854	'WWW-Mechanize',
855	'www\.monitor\.us',
856	'XaxisSemanticsClassifier',
857	'Xenu Link Sleuth',
858	'XING-contenttabreceiver\/[0-9]',
859	'XmlSitemapGenerator',
860	'xpymep([0-9]?)\.exe',
861	'Y!J-(ASR\|BSC)',
862	'Yaanb',
863	'yacy',
864	'Yahoo Ad monitoring',
865	'Yahoo Link Preview',
866	'YahooCacheSystem',
867	'YahooSeeker',
868	'YahooYSMcm',
869	'YandeG',
870	'yandex',
871	'yanga',
872	'yeti',
873	'Yo-yo',
874	'Yoleo Consumer',
875	'yoogliFetchAgent',
876	'YottaaMonitor',
877	'yourls\.org',
878	'Zao',
879	'Zemanta Aggregator',
880	'Zend\\\\Http\\\\Client',
881	'Zend_Http_Client',
882	'zgrab',
883	'ZnajdzFoto',
884	'ZyBorg',
885	'[a-z0-9\-_]*((?<!cu)bot\|crawler\|archiver\|transcoder\|spider\|uptime\|validator\|fetcher)',
886	);
887
888	return $data;
889	}
890
891	/**
892	* Return the list of strings to remove from the user agent before running the crawler regex.
893	*
894	* @return array
895	*/
896	public function get_exclusions_list() {
897	$data = array(
898	'Safari.[\d\.]*',
899	'Firefox.[\d\.]*',
900	'Chrome.[\d\.]*',
901	'Chromium.[\d\.]*',
902	'MSIE.[\d\.]',
903	'Opera\/[\d\.]*',
904	'Mozilla.[\d\.]*',
905	'AppleWebKit.[\d\.]*',
906	'Trident.[\d\.]*',
907	'Windows NT.[\d\.]*',
908	'Android [\d\.]*',
909	'Macintosh.',
910	'Ubuntu',
911	'Linux',
912	'[ ]Intel',
913	'Mac OS X [\d_]*',
914	'(like )?Gecko(.[\d\.]*)?',
915	'KHTML,',
916	'CriOS.[\d\.]*',
917	'CPU iPhone OS ([0-9_])* like Mac OS X',
918	'CPU OS ([0-9_])* like Mac OS X',
919	'iPod',
920	'compatible',
921	'x86_..',
922	'i686',
923	'x64',
924	'X11',
925	'rv:[\d\.]*',
926	'Version.[\d\.]*',
927	'WOW64',
928	'Win64',
929	'Dalvik.[\d\.]*',
930	' \.NET CLR [\d\.]*',
931	'Presto.[\d\.]*',
932	'Media Center PC',
933	'BlackBerry',
934	'Build',
935	'Opera Mini\/\d{1,2}\.\d{1,2}\.[\d\.]*\/\d{1,2}\.',
936	'Opera',
937	' \.NET[\d\.]*',
938	'$\|$\|;\|,', // remove the following characters ( ) : ,
939	);
940
941	return $data;
942	}
943
944	/**
945	* Return all possible HTTP headers that represent the User-Agent string.
946	*
947	* @return array
948	*/
949	public function get_headers_list() {
950	$data = array(
951	// the default User-Agent string.
952	'HTTP_USER_AGENT',
953	// header can occur on devices using Opera Mini.
954	'HTTP_X_OPERAMINI_PHONE_UA',
955	// vodafone specific header: http://www.seoprinciple.com/mobile-web-community-still-angry-at-vodafone/24/
956	'HTTP_X_DEVICE_USER_AGENT',
957	'HTTP_X_ORIGINAL_USER_AGENT',
958	'HTTP_X_SKYFIRE_PHONE',
959	'HTTP_X_BOLT_PHONE_UA',
960	'HTTP_DEVICE_STOCK_UA',
961	'HTTP_X_UCBROWSER_DEVICE_UA',
962	// sometimes, bots (especially Google) use a genuine user agent, but fill this header in with their email address
963	'HTTP_FROM',
964	);
965
966	return $data;
967	}
968
969	}
970