__init__.py — 10Web Booster – Website speed optimization, Cache & Page Speed optimizer trunk

tenweb-speed-optimizer / vendor / monperrus / crawler-user-agents / __init__.py

tenweb-speed-optimizer / vendor / monperrus / crawler-user-agents Last commit date

__init__.py

52 lines

1	import re
2	import json
3	from functools import cached_property
4	from pathlib import Path
5
6
7	class CrawlerPatterns:
8	def __init__(self):
9	pass
10
11	@cached_property
12	def case_insensitive(self):
13	return re.compile(
14	"\|".join(i["pattern"] for i in CRAWLER_USER_AGENTS_DATA),
15	re.IGNORECASE
16	)
17
18	@cached_property
19	def case_sensitive(self):
20	return re.compile("\|".join(i["pattern"] for i in CRAWLER_USER_AGENTS_DATA))
21
22
23	def load_json():
24	cwd = Path(__file__).parent
25	user_agents_file_path = cwd / "crawler-user-agents.json"
26	with user_agents_file_path.open() as patterns_file:
27	return json.load(patterns_file)
28
29
30	CRAWLER_USER_AGENTS_DATA = load_json()
31	CRAWLER_PATTERNS = CrawlerPatterns()
32
33
34	def is_crawler(user_agent: str, case_sensitive: bool = True) -> bool:
35	"""Return True if the given User-Agent matches a known crawler."""
36	if case_sensitive:
37	return bool(re.search(CRAWLER_PATTERNS.case_sensitive, user_agent))
38	return bool(re.search(CRAWLER_PATTERNS.case_insensitive, user_agent))
39
40
41	def matching_crawlers(user_agent: str, case_sensitive: bool = True) -> list[int]:
42	"""
43	Return a list of the indices in CRAWLER_USER_AGENTS_DATA of any crawlers
44	matching the given User-Agent.
45	"""
46	result = []
47	if is_crawler(user_agent, case_sensitive):
48	for num, crawler_user_agent in enumerate(CRAWLER_USER_AGENTS_DATA):
49	if re.search(crawler_user_agent["pattern"], user_agent, 0 if case_sensitive else re.IGNORECASE):
50	result.append(num)
51	return result
52