thinkrank / includes / seo / class-ai-crawlers.php

class-ai-crawlers.php in ThinkRank AI SEO – AI SEO Plugin for WordPress: Schema, XML Sitemaps, Meta Tags, Search Console & Local SEO 2.5.0, at includes/seo/class-ai-crawlers.php

360 lines 12.2 KB

Raw Download Zip

1	<?php
2	/**
3	* The AI crawler registry — one list, two consumers.
4	*
5	* `AI_Traffic_Tracker` classifies an inbound user agent against these tokens
6	* to record crawler hits; the Robots.txt panel turns the same list into
7	* per-agent allow/block directives. The list lived as a private constant on
8	* the tracker with no accessor and no filter, so the robots side had no way to
9	* read it — and a second hardcoded copy is how the two drift into disagreeing
10	* about which bots exist (#657).
11	*
12	* @package ThinkRank\SEO
13	* @since 2.5.0
14	*/
15
16	declare(strict_types=1);
17
18	namespace ThinkRank\SEO;
19
20	if (!defined('ABSPATH')) {
21	exit;
22	}
23
24	/**
25	* Known AI crawlers, their vendors, and what each one is for.
26	*/
27	class AI_Crawlers {
28
29	/**
30	* Vendor slug → display name, in the order the UI groups them.
31	*
32	* @var array<string, string>
33	*/
34	private const VENDORS = [
35	'openai' => 'OpenAI',
36	'anthropic' => 'Anthropic',
37	'google' => 'Google',
38	'perplexity' => 'Perplexity',
39	'meta' => 'Meta',
40	'apple' => 'Apple',
41	'other' => 'Other',
42	];
43
44	/**
45	* Crawler slug → definition.
46	*
47	* ORDER IS SIGNIFICANT. classify_crawler() takes the first token that
48	* appears anywhere in the user agent, so a token containing another must
49	* come first: `Claude-SearchBot` and `Claude-User` both contain
50	* `ClaudeBot`'s prefix, and `ChatGPT-User` would otherwise never be
51	* distinguished. The tracker relied on this ordering while the list was
52	* its own constant, and moving the list must not quietly lose it.
53	*
54	* `token` is matched case-insensitively as a substring of the user agent,
55	* and is also the exact string written after `User-agent:` in robots.txt —
56	* the two have to be the same value or a site would block a bot it is not
57	* detecting, and vice versa.
58	*
59	* @var array<string, array{token: string, label: string, vendor: string, purpose: string}>
60	*/
61	private const AGENTS = [
62	'oai-searchbot' => [
63	'token' => 'OAI-SearchBot',
64	'label' => 'OAI-SearchBot',
65	'vendor' => 'openai',
66	'purpose' => 'Indexes pages so they can be surfaced and linked in ChatGPT search results.',
67	],
68	'chatgpt-user' => [
69	'token' => 'ChatGPT-User',
70	'label' => 'ChatGPT-User',
71	'vendor' => 'openai',
72	'purpose' => 'Fetches a page live when a ChatGPT user follows or asks about that specific link.',
73	],
74	'gptbot' => [
75	'token' => 'GPTBot',
76	'label' => 'GPTBot',
77	'vendor' => 'openai',
78	'purpose' => 'Collects page content to train OpenAI models.',
79	],
80	'perplexity-user' => [
81	'token' => 'Perplexity-User',
82	'label' => 'Perplexity-User',
83	'vendor' => 'perplexity',
84	'purpose' => 'Fetches a page live in response to a Perplexity user asking about it.',
85	],
86	'perplexitybot' => [
87	'token' => 'PerplexityBot',
88	'label' => 'PerplexityBot',
89	'vendor' => 'perplexity',
90	'purpose' => 'Indexes pages so Perplexity can cite them in answers.',
91	],
92	'claude-searchbot' => [
93	'token' => 'Claude-SearchBot',
94	'label' => 'Claude-SearchBot',
95	'vendor' => 'anthropic',
96	'purpose' => 'Indexes pages so Claude can cite them when answering with search.',
97	],
98	'claude-user' => [
99	'token' => 'Claude-User',
100	'label' => 'Claude-User',
101	'vendor' => 'anthropic',
102	'purpose' => 'Fetches a page live when a Claude user asks about that specific link.',
103	],
104	'claudebot' => [
105	'token' => 'ClaudeBot',
106	'label' => 'ClaudeBot',
107	'vendor' => 'anthropic',
108	'purpose' => 'Collects page content to train Anthropic models.',
109	],
110	'anthropic-ai' => [
111	'token' => 'anthropic-ai',
112	'label' => 'anthropic-ai',
113	'vendor' => 'anthropic',
114	'purpose' => 'Anthropic’s earlier crawler token, still seen in the wild.',
115	],
116	'google-extended' => [
117	'token' => 'Google-Extended',
118	'label' => 'Google-Extended',
119	'vendor' => 'google',
120	'purpose' => 'Controls whether your content trains Gemini and grounds its answers. Blocking it does NOT affect normal Google Search indexing or your rankings.',
121	],
122	'applebot-extended' => [
123	'token' => 'Applebot-Extended',
124	'label' => 'Applebot-Extended',
125	'vendor' => 'apple',
126	'purpose' => 'Controls whether your content trains Apple Intelligence. Blocking it does not affect Siri or Spotlight search results.',
127	],
128	'meta-externalagent' => [
129	'token' => 'meta-externalagent',
130	'label' => 'meta-externalagent',
131	'vendor' => 'meta',
132	'purpose' => 'Collects page content to train Meta AI.',
133	],
134	'meta-externalfetcher' => [
135	'token' => 'meta-externalfetcher',
136	'label' => 'meta-externalfetcher',
137	'vendor' => 'meta',
138	'purpose' => 'Fetches a page live in response to a Meta AI user asking about it.',
139	],
140	'bytespider' => [
141	'token' => 'Bytespider',
142	'label' => 'Bytespider',
143	'vendor' => 'other',
144	'purpose' => 'ByteDance’s crawler, collecting content to train its models.',
145	],
146	'amazonbot' => [
147	'token' => 'Amazonbot',
148	'label' => 'Amazonbot',
149	'vendor' => 'other',
150	'purpose' => 'Amazon’s crawler, feeding Alexa answers and Amazon’s AI products.',
151	],
152	'ccbot' => [
153	'token' => 'CCBot',
154	'label' => 'CCBot',
155	'vendor' => 'other',
156	'purpose' => 'Common Crawl’s crawler. Its public archive is a training source for many AI models, so blocking it reaches more than one company.',
157	],
158	'cohere-ai' => [
159	'token' => 'cohere-ai',
160	'label' => 'cohere-ai',
161	'vendor' => 'other',
162	'purpose' => 'Cohere’s crawler, collecting content for its models.',
163	],
164	'mistral-user' => [
165	'token' => 'MistralAI-User',
166	'label' => 'MistralAI-User',
167	'vendor' => 'other',
168	'purpose' => 'Fetches a page live in response to a Le Chat user asking about it.',
169	],
170	];
171
172	/**
173	* Every known crawler, keyed by slug.
174	*
175	* Filterable so a site can add a crawler that shipped after this release
176	* without editing the plugin. A filtered entry is normalised and anything
177	* without a usable token is dropped: a malformed entry reaching robots.txt
178	* would write a `User-agent:` line with no agent on it, which changes the
179	* meaning of the group that follows.
180	*
181	* @since 2.5.0
182	*
183	* @return array<string, array{token: string, label: string, vendor: string, purpose: string}>
184	*/
185	public static function all(): array {
186	/**
187	* Filter the known AI crawler registry.
188	*
189	* Order is significant: classification takes the first token that
190	* matches, so a token that contains another must come first.
191	*
192	* @since 2.5.0
193	*
194	* @param array<string, array{token: string, label: string, vendor: string, purpose: string}> $agents Registry keyed by slug.
195	*/
196	$agents = apply_filters('thinkrank_ai_crawlers', self::AGENTS);
197
198	if (!is_array($agents)) {
199	return self::AGENTS;
200	}
201
202	$normalised = [];
203
204	foreach ($agents as $slug => $agent) {
205	$slug = sanitize_key((string) $slug);
206
207	if ('' === $slug \|\| !is_array($agent)) {
208	continue;
209	}
210
211	$token = trim((string) ($agent['token'] ?? ''));
212
213	// A token with whitespace or a colon in it cannot be written after
214	// `User-agent:` without breaking the record it opens.
215	if ('' === $token \|\| preg_match('/[\s:]/', $token)) {
216	continue;
217	}
218
219	$vendor = (string) ($agent['vendor'] ?? 'other');
220
221	$normalised[$slug] = [
222	'token' => $token,
223	'label' => trim((string) ($agent['label'] ?? '')) ?: $token,
224	'vendor' => isset(self::VENDORS[$vendor]) ? $vendor : 'other',
225	'purpose' => (string) ($agent['purpose'] ?? ''),
226	];
227	}
228
229	return $normalised ?: self::AGENTS;
230	}
231
232	/**
233	* User-agent token → crawler slug, in match order.
234	*
235	* The shape `AI_Traffic_Tracker::classify_crawler()` walks.
236	*
237	* @since 2.5.0
238	*
239	* @return array<string, string>
240	*/
241	public static function token_map(): array {
242	$map = [];
243
244	foreach (self::all() as $slug => $agent) {
245	$map[$agent['token']] = $slug;
246	}
247
248	return $map;
249	}
250
251	/**
252	* Vendor slug → display name, including any vendor only a filter knows.
253	*
254	* @since 2.5.0
255	*
256	* @return array<string, string>
257	*/
258	public static function vendors(): array {
259	return self::VENDORS;
260	}
261
262	/**
263	* Is this a crawler slug the registry knows?
264	*
265	* @since 2.5.0
266	*
267	* @param string $slug Crawler slug.
268	* @return bool
269	*/
270	public static function exists(string $slug): bool {
271	return isset(self::all()[$slug]);
272	}
273
274	/**
275	* The registry as the admin screen and the REST API consume it: a flat,
276	* ordered list carrying the slug, so JSON preserves the grouping order.
277	*
278	* @since 2.5.0
279	*
280	* @return array<int, array{slug: string, token: string, label: string, vendor: string, vendor_label: string, purpose: string}>
281	*/
282	public static function for_display(): array {
283	$vendors = self::vendors();
284	$list = [];
285
286	foreach (self::all() as $slug => $agent) {
287	$list[] = [
288	'slug' => $slug,
289	'token' => $agent['token'],
290	'label' => $agent['label'],
291	'vendor' => $agent['vendor'],
292	'vendor_label' => $vendors[$agent['vendor']] ?? $vendors['other'],
293	'purpose' => $agent['purpose'],
294	];
295	}
296
297	return $list;
298	}
299
300	/**
301	* Normalise a stored/posted rule map to `slug => 'allow'\|'block'`.
302	*
303	* Unknown slugs are dropped rather than stored: a slug with no agent
304	* behind it can never emit a directive, but it would round-trip through
305	* every settings response forever. Anything that is not the string
306	* `block` reads as allow, so a half-written payload defaults to the
307	* permissive answer rather than silently blocking a crawler.
308	*
309	* @since 2.5.0
310	*
311	* @param mixed $rules Raw rule map.
312	* @return array<string, string> Normalised rules.
313	*/
314	public static function normalize_rules($rules): array {
315	if (!is_array($rules)) {
316	return [];
317	}
318
319	$known = self::all();
320	$normalised = [];
321
322	foreach ($rules as $slug => $rule) {
323	$slug = sanitize_key((string) $slug);
324
325	if (!isset($known[$slug])) {
326	continue;
327	}
328
329	$normalised[$slug] = ('block' === strtolower(trim((string) $rule))) ? 'block' : 'allow';
330	}
331
332	return $normalised;
333	}
334
335	/**
336	* The slugs currently set to block, in registry order.
337	*
338	* Registry order rather than the stored map's order, so the robots.txt
339	* block is stable between saves and a diff of the served file shows only
340	* real changes.
341	*
342	* @since 2.5.0
343	*
344	* @param mixed $rules Stored rule map.
345	* @return string[] Blocked crawler slugs.
346	*/
347	public static function blocked_slugs($rules): array {
348	$rules = self::normalize_rules($rules);
349	$blocked = [];
350
351	foreach (array_keys(self::all()) as $slug) {
352	if (($rules[$slug] ?? 'allow') === 'block') {
353	$blocked[] = $slug;
354	}
355	}
356
357	return $blocked;
358	}
359	}
360