jetpack / jetpack_vendor / automattic / jetpack-seo / src / class-ai-crawlers.php

class-ai-crawlers.php in Jetpack – WP Security, Backup, Speed, & Growth 16.3, at jetpack_vendor/automattic/jetpack-seo/src/class-ai-crawlers.php

400 lines 13.1 KB

Raw Download Zip

1	<?php
2	/**
3	* AI crawler access controls.
4	*
5	* Lets a site owner allow or block individual AI crawlers (training and
6	* answer-engine bots) by appending per-user-agent `Disallow` rules to the
7	* WordPress-generated robots.txt.
8	*
9	* Persistence uses a STORE-INTENT model rather than a literal blocked list. The
10	* durable option `jetpack_seo_ai_crawler_overrides` holds only deviations from
11	* each bot's default policy (training crawlers blocked, answer-engine crawlers
12	* allowed); a bot with no stored override falls back to its default.
13	* This keeps the stored map sparse and means newly added training crawlers are
14	* covered automatically without a migration.
15	*
16	* Caveat: the `robots_txt` filter only feeds WordPress's virtual robots.txt.
17	* A detected static file in the WordPress installation is not modified — a
18	* limitation surfaced in the AI tab (see {@see self::has_static_robots_txt()}).
19	*
20	* @package automattic/jetpack-seo-package
21	*/
22
23	namespace Automattic\Jetpack\SEO;
24
25	/**
26	* Emits robots.txt directives that block opted-out AI crawlers.
27	*/
28	class Ai_Crawlers {
29
30	/**
31	* Option holding the sparse map of per-crawler overrides
32	* (`slug => bool`, true = blocked). Only entries that deviate from the bot's
33	* default policy are stored. Mirrored in the plugin's settings endpoint
34	* whitelist (`jp_group => 'seo-tools'`).
35	*
36	* @var string
37	*/
38	const OPTION = 'jetpack_seo_ai_crawler_overrides';
39
40	/**
41	* Wire the robots.txt filter.
42	*
43	* @return void
44	*/
45	public static function init() {
46	add_filter( 'robots_txt', array( __CLASS__, 'append_directives' ), 10, 2 );
47	}
48
49	/**
50	* The known AI crawler catalog: slug => [ label, user_agent, type ].
51	*
52	* `slug` is the stable key persisted in the override map and sent by the AI
53	* tab; `user_agent` is the token written to the `User-agent:` robots line;
54	* `label` is the human name shown in the UI; `type` is `answer` (fetches to
55	* cite in live AI answers) or `training` (collects to train models; some, like
56	* Google Gemini, also ground AI answers but are filed here). Training crawlers
57	* are blocked by default; answer-engine crawlers are allowed.
58	*
59	* @return array<string, array<string, string>>
60	*/
61	public static function get_catalog() {
62	// Labels are the name a user is most likely to recognize, with the exact
63	// robots.txt user-agent token in parentheses so a technical user can verify
64	// the emitted directive (and to disambiguate the search/training pairs, e.g.
65	// ChatGPT's GPTBot vs OAI-SearchBot).
66	$catalog = array(
67	// Answer-engine crawlers fetch pages so AI assistants can cite them in
68	// live answers. Allowed by default — blocking them costs AI visibility.
69	'oai-searchbot' => array(
70	'label' => __( 'ChatGPT Search (OAI-SearchBot)', 'jetpack-seo' ),
71	'user_agent' => 'OAI-SearchBot',
72	'type' => 'answer',
73	),
74	'claude-searchbot' => array(
75	'label' => __( 'Claude Search (Claude-SearchBot)', 'jetpack-seo' ),
76	'user_agent' => 'Claude-SearchBot',
77	'type' => 'answer',
78	),
79	'perplexitybot' => array(
80	'label' => __( 'Perplexity (PerplexityBot)', 'jetpack-seo' ),
81	'user_agent' => 'PerplexityBot',
82	'type' => 'answer',
83	),
84	'amzn-searchbot' => array(
85	'label' => __( 'Amazon Alexa (Amzn-SearchBot)', 'jetpack-seo' ),
86	'user_agent' => 'Amzn-SearchBot',
87	'type' => 'answer',
88	),
89	// Training crawlers collect content to train AI models. Blocked by
90	// default. Some (e.g. Google Gemini) are also used to ground live AI
91	// answers, so blocking them protects privacy but can cost that
92	// visibility — filed here as training and blocked by default so the
93	// owner actively opts in.
94	'gptbot' => array(
95	'label' => __( 'ChatGPT (GPTBot)', 'jetpack-seo' ),
96	'user_agent' => 'GPTBot',
97	'type' => 'training',
98	),
99	'claudebot' => array(
100	'label' => __( 'Claude (ClaudeBot)', 'jetpack-seo' ),
101	'user_agent' => 'ClaudeBot',
102	'type' => 'training',
103	),
104	'google-extended' => array(
105	'label' => __( 'Google Gemini (Google-Extended)', 'jetpack-seo' ),
106	'user_agent' => 'Google-Extended',
107	'type' => 'training',
108	),
109	'applebot-extended' => array(
110	'label' => __( 'Apple Intelligence (Applebot-Extended)', 'jetpack-seo' ),
111	'user_agent' => 'Applebot-Extended',
112	'type' => 'training',
113	),
114	'meta-externalagent' => array(
115	'label' => __( 'Meta AI (meta-externalagent)', 'jetpack-seo' ),
116	'user_agent' => 'meta-externalagent',
117	'type' => 'training',
118	),
119	'bytespider' => array(
120	'label' => __( 'ByteDance (Bytespider)', 'jetpack-seo' ),
121	'user_agent' => 'Bytespider',
122	'type' => 'training',
123	),
124	'ccbot' => array(
125	'label' => __( 'Common Crawl (CCBot)', 'jetpack-seo' ),
126	'user_agent' => 'CCBot',
127	'type' => 'training',
128	),
129	'amazonbot' => array(
130	// Amazon's own docs: Amazonbot "may be used to train Amazon AI
131	// models" (Amzn-SearchBot is the answer-engine bot above).
132	'label' => __( 'Amazon (Amazonbot)', 'jetpack-seo' ),
133	'user_agent' => 'Amazonbot',
134	'type' => 'training',
135	),
136	);
137
138	return $catalog;
139	}
140
141	/**
142	* Whether a catalog bot is blocked when the owner hasn't set an override for
143	* it: training crawlers are blocked by default; answer-engine crawlers are
144	* allowed.
145	*
146	* @param string $slug Catalog slug.
147	* @return bool
148	*/
149	private static function is_blocked_by_default( $slug ) {
150	$catalog = self::get_catalog();
151	return isset( $catalog[ $slug ] ) && 'training' === $catalog[ $slug ]['type'];
152	}
153
154	/**
155	* Whether WordPress.com's site-wide data-sharing opt-out is enabled.
156	*
157	* Its later `robots_txt` filter remains authoritative over per-bot settings.
158	*
159	* @return bool
160	*/
161	public static function has_data_sharing_opt_out() {
162	return (bool) get_option( 'wpcom_data_sharing_opt_out' );
163	}
164
165	/**
166	* Whether the existing data-sharing policy forces a catalog bot to be blocked.
167	*
168	* @param string $slug Catalog slug.
169	* @return bool
170	*/
171	private static function is_blocked_by_data_sharing_policy( $slug ) {
172	return self::has_data_sharing_opt_out()
173	&& in_array(
174	$slug,
175	array(
176	'amazonbot',
177	'applebot-extended',
178	'bytespider',
179	'ccbot',
180	'claudebot',
181	'google-extended',
182	'gptbot',
183	'meta-externalagent',
184	'perplexitybot',
185	),
186	true
187	);
188	}
189
190	/**
191	* The resolved, sparse override map: `slug => bool` (true = blocked).
192	*
193	* Reads the stored option, drops unknown slugs, casts values to bool, and
194	* drops any entry that equals the bot's default policy — so the returned map
195	* contains only real deviations. A bot absent from this map falls back to its
196	* default in {@see self::is_blocked()}.
197	*
198	* @return array<string, bool>
199	*/
200	public static function get_overrides() {
201	$stored = get_option( self::OPTION, array() );
202	if ( ! is_array( $stored ) ) {
203	return array();
204	}
205
206	$catalog = self::get_catalog();
207	$overrides = array();
208	foreach ( $stored as $slug => $blocked ) {
209	if ( ! isset( $catalog[ $slug ] ) ) {
210	continue;
211	}
212	$blocked = (bool) $blocked;
213	// Only keep entries that actually deviate from the default policy
214	// (training blocked, answer-engine allowed) — resolved from the
215	// catalog we already hold rather than re-fetching it per slug.
216	if ( $blocked === ( 'training' === $catalog[ $slug ]['type'] ) ) {
217	continue;
218	}
219	$overrides[ $slug ] = $blocked;
220	}
221
222	return $overrides;
223	}
224
225	/**
226	* Whether a given crawler is currently blocked, applying an override when one
227	* exists and otherwise the bot's default policy. Unknown slugs return false.
228	*
229	* @param string $slug Catalog slug.
230	* @return bool
231	*/
232	public static function is_blocked( $slug ) {
233	$catalog = self::get_catalog();
234	if ( ! isset( $catalog[ $slug ] ) ) {
235	return false;
236	}
237	if ( self::is_blocked_by_data_sharing_policy( $slug ) ) {
238	return true;
239	}
240
241	$overrides = self::get_overrides();
242	return array_key_exists( $slug, $overrides )
243	? $overrides[ $slug ]
244	: self::is_blocked_by_default( $slug );
245	}
246
247	/**
248	* The catalog slugs currently blocked (override or default).
249	*
250	* @return string[]
251	*/
252	public static function get_blocked_slugs() {
253	// Resolve the override map and catalog once, then fold in the per-type
254	// default — rather than calling is_blocked() per slug (which would re-read
255	// the option and rebuild the catalog on every robots.txt render).
256	$overrides = self::get_overrides();
257	$blocked = array();
258	foreach ( self::get_catalog() as $slug => $info ) {
259	$is_blocked = self::is_blocked_by_data_sharing_policy( $slug )
260	\|\| ( array_key_exists( $slug, $overrides )
261	? $overrides[ $slug ]
262	: ( 'training' === $info['type'] ) );
263	if ( $is_blocked ) {
264	$blocked[] = $slug;
265	}
266	}
267	return $blocked;
268	}
269
270	/**
271	* Whether the site allows search-engine (and therefore AI-crawler) indexing.
272	*
273	* When `blog_public` is off, WordPress disallows everything in robots.txt, so
274	* per-crawler controls are moot — the AI tab surfaces this instead of showing
275	* toggles that can't take effect.
276	*
277	* @return bool
278	*/
279	public static function search_engines_allowed() {
280	return (int) get_option( 'blog_public', 1 ) === 1;
281	}
282
283	/**
284	* Whether the site is served from a WordPress.com staging subdomain
285	* (`*.wpcomstaging.com`) where the platform blocks all crawling regardless of
286	* the site's own settings — so AI-crawler controls can't take effect.
287	*
288	* @return bool
289	*/
290	public static function is_crawl_restricted_subdomain() {
291	$host = (string) wp_parse_url( home_url(), PHP_URL_HOST );
292	$suffix = '.wpcomstaging.com';
293	return strlen( $host ) > strlen( $suffix )
294	&& substr( $host, -strlen( $suffix ) ) === $suffix;
295	}
296
297	/**
298	* Whether a static `robots.txt` file exists in the WordPress installation
299	* directory. These controls only change WordPress's virtual robots.txt and
300	* cannot modify that file.
301	*
302	* @return bool
303	*/
304	public static function has_static_robots_txt() {
305	return file_exists( ABSPATH . 'robots.txt' );
306	}
307
308	/**
309	* Whether this is a path-based multisite network.
310	*
311	* Such networks share one origin-level robots.txt, so per-site controls
312	* cannot be represented safely.
313	*
314	* @return bool
315	*/
316	public static function is_path_based_multisite() {
317	return is_multisite() && ! is_subdomain_install();
318	}
319
320	/**
321	* Append `Disallow: /` blocks for each blocked AI crawler to robots.txt.
322	*
323	* @param string $output The robots.txt content assembled so far.
324	* @param bool $public Whether the site is public (`blog_public`). When
325	* false WordPress already disallows everything, but the
326	* explicit per-bot blocks are still valid and harmless.
327	* @return string The robots.txt content with AI-crawler directives appended.
328	*/
329	public static function append_directives( $output, $public ) {
330	unset( $public );
331
332	// WordPress.com's existing privacy filter owns the site-wide opt-out and
333	// runs later at priority 12. Defer to it instead of emitting duplicate
334	// directives from this per-bot filter.
335	if ( self::is_path_based_multisite() \|\| self::has_data_sharing_opt_out() ) {
336	return $output;
337	}
338
339	$catalog = self::get_catalog();
340	$blocked = self::get_blocked_slugs();
341	if ( empty( $blocked ) ) {
342	return $output;
343	}
344
345	$lines = array( '', '# AI crawlers blocked via Jetpack SEO.' );
346	foreach ( $blocked as $slug ) {
347	$lines[] = 'User-agent: ' . $catalog[ $slug ]['user_agent'];
348	$lines[] = 'Disallow: /';
349	$lines[] = '';
350	}
351
352	return $output . implode( "\n", $lines ) . "\n";
353	}
354
355	/**
356	* The AI tab's crawler bootstrap payload: the catalog (camelCased for JS), the
357	* current sparse override map, and the environment flags that decide whether
358	* the controls render or are replaced by an explanation.
359	*
360	* @return array
361	*/
362	public static function get_bootstrap_data() {
363	$catalog = array();
364	foreach ( self::get_catalog() as $slug => $info ) {
365	$catalog[] = array(
366	'slug' => $slug,
367	'label' => $info['label'],
368	'userAgent' => $info['user_agent'],
369	'type' => $info['type'],
370	);
371	}
372
373	return array(
374	'catalog' => $catalog,
375	'overrides' => (object) self::get_overrides(),
376	'searchEnginesVisible' => self::search_engines_allowed(),
377	'restrictedSubdomain' => self::is_crawl_restricted_subdomain(),
378	'staticRobotsTxt' => self::has_static_robots_txt(),
379	'dataSharingOptOut' => self::has_data_sharing_opt_out(),
380	'pathBasedMultisite' => self::is_path_based_multisite(),
381	'privacySettingsUrl' => self::privacy_settings_url(),
382	'robotsTxtUrl' => home_url( '/robots.txt' ),
383	);
384	}
385
386	/**
387	* URL of the settings screen holding "Prevent third-party sharing", linked from
388	* the AI tab when that setting is governing crawler access.
389	*
390	* Points at wp-admin → Settings → Reading, which is the same-origin home of the
391	* option on the sites where it exists (WordPress.com). Left as a package method
392	* so a Calypso deep link can replace it without touching the client.
393	*
394	* @return string
395	*/
396	public static function privacy_settings_url() {
397	return admin_url( 'options-reading.php' );
398	}
399	}
400