class-admin-page.php
4 weeks ago
class-ai-crawlers.php
4 weeks ago
class-author-schema-node.php
1 month ago
class-breadcrumb-schema-node.php
1 month ago
class-content-coverage.php
4 weeks ago
class-dashboard-data.php
3 weeks ago
class-initializer.php
3 weeks ago
class-llms-txt.php
4 weeks ago
class-local-business-schema-node.php
1 month ago
class-organization-schema-node.php
1 month ago
class-post-schema-node.php
1 month ago
class-post-types.php
4 weeks ago
class-schema-builder.php
1 month ago
class-schema-graph.php
1 month ago
class-schema-node-ids.php
1 month ago
class-schema-settings-controller.php
1 month ago
class-schema-settings.php
1 month ago
class-surface-visibility.php
3 weeks ago
class-website-schema-node.php
1 month ago
class-ai-crawlers.php
400 lines
| 1 | <?php |
| 2 | /** |
| 3 | * AI crawler access controls. |
| 4 | * |
| 5 | * Lets a site owner allow or block individual AI crawlers (training and |
| 6 | * answer-engine bots) by appending per-user-agent `Disallow` rules to the |
| 7 | * WordPress-generated robots.txt. |
| 8 | * |
| 9 | * Persistence uses a STORE-INTENT model rather than a literal blocked list. The |
| 10 | * durable option `jetpack_seo_ai_crawler_overrides` holds only *deviations* from |
| 11 | * each bot's default policy (training crawlers blocked, answer-engine crawlers |
| 12 | * allowed); a bot with no stored override falls back to its default. |
| 13 | * This keeps the stored map sparse and means newly added training crawlers are |
| 14 | * covered automatically without a migration. |
| 15 | * |
| 16 | * Caveat: the `robots_txt` filter only feeds WordPress's *virtual* robots.txt. |
| 17 | * A detected static file in the WordPress installation is not modified — a |
| 18 | * limitation surfaced in the AI tab (see {@see self::has_static_robots_txt()}). |
| 19 | * |
| 20 | * @package automattic/jetpack-seo-package |
| 21 | */ |
| 22 | |
| 23 | namespace Automattic\Jetpack\SEO; |
| 24 | |
| 25 | /** |
| 26 | * Emits robots.txt directives that block opted-out AI crawlers. |
| 27 | */ |
| 28 | class Ai_Crawlers { |
| 29 | |
| 30 | /** |
| 31 | * Option holding the sparse map of per-crawler overrides |
| 32 | * (`slug => bool`, true = blocked). Only entries that deviate from the bot's |
| 33 | * default policy are stored. Mirrored in the plugin's settings endpoint |
| 34 | * whitelist (`jp_group => 'seo-tools'`). |
| 35 | * |
| 36 | * @var string |
| 37 | */ |
| 38 | const OPTION = 'jetpack_seo_ai_crawler_overrides'; |
| 39 | |
| 40 | /** |
| 41 | * Wire the robots.txt filter. |
| 42 | * |
| 43 | * @return void |
| 44 | */ |
| 45 | public static function init() { |
| 46 | add_filter( 'robots_txt', array( __CLASS__, 'append_directives' ), 10, 2 ); |
| 47 | } |
| 48 | |
| 49 | /** |
| 50 | * The known AI crawler catalog: slug => [ label, user_agent, type ]. |
| 51 | * |
| 52 | * `slug` is the stable key persisted in the override map and sent by the AI |
| 53 | * tab; `user_agent` is the token written to the `User-agent:` robots line; |
| 54 | * `label` is the human name shown in the UI; `type` is `answer` (fetches to |
| 55 | * cite in live AI answers) or `training` (collects to train models; some, like |
| 56 | * Google Gemini, also ground AI answers but are filed here). Training crawlers |
| 57 | * are blocked by default; answer-engine crawlers are allowed. |
| 58 | * |
| 59 | * @return array<string, array<string, string>> |
| 60 | */ |
| 61 | public static function get_catalog() { |
| 62 | // Labels are the name a user is most likely to recognize, with the exact |
| 63 | // robots.txt user-agent token in parentheses so a technical user can verify |
| 64 | // the emitted directive (and to disambiguate the search/training pairs, e.g. |
| 65 | // ChatGPT's GPTBot vs OAI-SearchBot). |
| 66 | $catalog = array( |
| 67 | // Answer-engine crawlers fetch pages so AI assistants can cite them in |
| 68 | // live answers. Allowed by default — blocking them costs AI visibility. |
| 69 | 'oai-searchbot' => array( |
| 70 | 'label' => __( 'ChatGPT Search (OAI-SearchBot)', 'jetpack-seo' ), |
| 71 | 'user_agent' => 'OAI-SearchBot', |
| 72 | 'type' => 'answer', |
| 73 | ), |
| 74 | 'claude-searchbot' => array( |
| 75 | 'label' => __( 'Claude Search (Claude-SearchBot)', 'jetpack-seo' ), |
| 76 | 'user_agent' => 'Claude-SearchBot', |
| 77 | 'type' => 'answer', |
| 78 | ), |
| 79 | 'perplexitybot' => array( |
| 80 | 'label' => __( 'Perplexity (PerplexityBot)', 'jetpack-seo' ), |
| 81 | 'user_agent' => 'PerplexityBot', |
| 82 | 'type' => 'answer', |
| 83 | ), |
| 84 | 'amzn-searchbot' => array( |
| 85 | 'label' => __( 'Amazon Alexa (Amzn-SearchBot)', 'jetpack-seo' ), |
| 86 | 'user_agent' => 'Amzn-SearchBot', |
| 87 | 'type' => 'answer', |
| 88 | ), |
| 89 | // Training crawlers collect content to train AI models. Blocked by |
| 90 | // default. Some (e.g. Google Gemini) are also used to ground live AI |
| 91 | // answers, so blocking them protects privacy but can cost that |
| 92 | // visibility — filed here as training and blocked by default so the |
| 93 | // owner actively opts in. |
| 94 | 'gptbot' => array( |
| 95 | 'label' => __( 'ChatGPT (GPTBot)', 'jetpack-seo' ), |
| 96 | 'user_agent' => 'GPTBot', |
| 97 | 'type' => 'training', |
| 98 | ), |
| 99 | 'claudebot' => array( |
| 100 | 'label' => __( 'Claude (ClaudeBot)', 'jetpack-seo' ), |
| 101 | 'user_agent' => 'ClaudeBot', |
| 102 | 'type' => 'training', |
| 103 | ), |
| 104 | 'google-extended' => array( |
| 105 | 'label' => __( 'Google Gemini (Google-Extended)', 'jetpack-seo' ), |
| 106 | 'user_agent' => 'Google-Extended', |
| 107 | 'type' => 'training', |
| 108 | ), |
| 109 | 'applebot-extended' => array( |
| 110 | 'label' => __( 'Apple Intelligence (Applebot-Extended)', 'jetpack-seo' ), |
| 111 | 'user_agent' => 'Applebot-Extended', |
| 112 | 'type' => 'training', |
| 113 | ), |
| 114 | 'meta-externalagent' => array( |
| 115 | 'label' => __( 'Meta AI (meta-externalagent)', 'jetpack-seo' ), |
| 116 | 'user_agent' => 'meta-externalagent', |
| 117 | 'type' => 'training', |
| 118 | ), |
| 119 | 'bytespider' => array( |
| 120 | 'label' => __( 'ByteDance (Bytespider)', 'jetpack-seo' ), |
| 121 | 'user_agent' => 'Bytespider', |
| 122 | 'type' => 'training', |
| 123 | ), |
| 124 | 'ccbot' => array( |
| 125 | 'label' => __( 'Common Crawl (CCBot)', 'jetpack-seo' ), |
| 126 | 'user_agent' => 'CCBot', |
| 127 | 'type' => 'training', |
| 128 | ), |
| 129 | 'amazonbot' => array( |
| 130 | // Amazon's own docs: Amazonbot "may be used to train Amazon AI |
| 131 | // models" (Amzn-SearchBot is the answer-engine bot above). |
| 132 | 'label' => __( 'Amazon (Amazonbot)', 'jetpack-seo' ), |
| 133 | 'user_agent' => 'Amazonbot', |
| 134 | 'type' => 'training', |
| 135 | ), |
| 136 | ); |
| 137 | |
| 138 | return $catalog; |
| 139 | } |
| 140 | |
| 141 | /** |
| 142 | * Whether a catalog bot is blocked when the owner hasn't set an override for |
| 143 | * it: training crawlers are blocked by default; answer-engine crawlers are |
| 144 | * allowed. |
| 145 | * |
| 146 | * @param string $slug Catalog slug. |
| 147 | * @return bool |
| 148 | */ |
| 149 | private static function is_blocked_by_default( $slug ) { |
| 150 | $catalog = self::get_catalog(); |
| 151 | return isset( $catalog[ $slug ] ) && 'training' === $catalog[ $slug ]['type']; |
| 152 | } |
| 153 | |
| 154 | /** |
| 155 | * Whether WordPress.com's site-wide data-sharing opt-out is enabled. |
| 156 | * |
| 157 | * Its later `robots_txt` filter remains authoritative over per-bot settings. |
| 158 | * |
| 159 | * @return bool |
| 160 | */ |
| 161 | public static function has_data_sharing_opt_out() { |
| 162 | return (bool) get_option( 'wpcom_data_sharing_opt_out' ); |
| 163 | } |
| 164 | |
| 165 | /** |
| 166 | * Whether the existing data-sharing policy forces a catalog bot to be blocked. |
| 167 | * |
| 168 | * @param string $slug Catalog slug. |
| 169 | * @return bool |
| 170 | */ |
| 171 | private static function is_blocked_by_data_sharing_policy( $slug ) { |
| 172 | return self::has_data_sharing_opt_out() |
| 173 | && in_array( |
| 174 | $slug, |
| 175 | array( |
| 176 | 'amazonbot', |
| 177 | 'applebot-extended', |
| 178 | 'bytespider', |
| 179 | 'ccbot', |
| 180 | 'claudebot', |
| 181 | 'google-extended', |
| 182 | 'gptbot', |
| 183 | 'meta-externalagent', |
| 184 | 'perplexitybot', |
| 185 | ), |
| 186 | true |
| 187 | ); |
| 188 | } |
| 189 | |
| 190 | /** |
| 191 | * The resolved, sparse override map: `slug => bool` (true = blocked). |
| 192 | * |
| 193 | * Reads the stored option, drops unknown slugs, casts values to bool, and |
| 194 | * drops any entry that equals the bot's default policy — so the returned map |
| 195 | * contains only real deviations. A bot absent from this map falls back to its |
| 196 | * default in {@see self::is_blocked()}. |
| 197 | * |
| 198 | * @return array<string, bool> |
| 199 | */ |
| 200 | public static function get_overrides() { |
| 201 | $stored = get_option( self::OPTION, array() ); |
| 202 | if ( ! is_array( $stored ) ) { |
| 203 | return array(); |
| 204 | } |
| 205 | |
| 206 | $catalog = self::get_catalog(); |
| 207 | $overrides = array(); |
| 208 | foreach ( $stored as $slug => $blocked ) { |
| 209 | if ( ! isset( $catalog[ $slug ] ) ) { |
| 210 | continue; |
| 211 | } |
| 212 | $blocked = (bool) $blocked; |
| 213 | // Only keep entries that actually deviate from the default policy |
| 214 | // (training blocked, answer-engine allowed) — resolved from the |
| 215 | // catalog we already hold rather than re-fetching it per slug. |
| 216 | if ( $blocked === ( 'training' === $catalog[ $slug ]['type'] ) ) { |
| 217 | continue; |
| 218 | } |
| 219 | $overrides[ $slug ] = $blocked; |
| 220 | } |
| 221 | |
| 222 | return $overrides; |
| 223 | } |
| 224 | |
| 225 | /** |
| 226 | * Whether a given crawler is currently blocked, applying an override when one |
| 227 | * exists and otherwise the bot's default policy. Unknown slugs return false. |
| 228 | * |
| 229 | * @param string $slug Catalog slug. |
| 230 | * @return bool |
| 231 | */ |
| 232 | public static function is_blocked( $slug ) { |
| 233 | $catalog = self::get_catalog(); |
| 234 | if ( ! isset( $catalog[ $slug ] ) ) { |
| 235 | return false; |
| 236 | } |
| 237 | if ( self::is_blocked_by_data_sharing_policy( $slug ) ) { |
| 238 | return true; |
| 239 | } |
| 240 | |
| 241 | $overrides = self::get_overrides(); |
| 242 | return array_key_exists( $slug, $overrides ) |
| 243 | ? $overrides[ $slug ] |
| 244 | : self::is_blocked_by_default( $slug ); |
| 245 | } |
| 246 | |
| 247 | /** |
| 248 | * The catalog slugs currently blocked (override or default). |
| 249 | * |
| 250 | * @return string[] |
| 251 | */ |
| 252 | public static function get_blocked_slugs() { |
| 253 | // Resolve the override map and catalog once, then fold in the per-type |
| 254 | // default — rather than calling is_blocked() per slug (which would re-read |
| 255 | // the option and rebuild the catalog on every robots.txt render). |
| 256 | $overrides = self::get_overrides(); |
| 257 | $blocked = array(); |
| 258 | foreach ( self::get_catalog() as $slug => $info ) { |
| 259 | $is_blocked = self::is_blocked_by_data_sharing_policy( $slug ) |
| 260 | || ( array_key_exists( $slug, $overrides ) |
| 261 | ? $overrides[ $slug ] |
| 262 | : ( 'training' === $info['type'] ) ); |
| 263 | if ( $is_blocked ) { |
| 264 | $blocked[] = $slug; |
| 265 | } |
| 266 | } |
| 267 | return $blocked; |
| 268 | } |
| 269 | |
| 270 | /** |
| 271 | * Whether the site allows search-engine (and therefore AI-crawler) indexing. |
| 272 | * |
| 273 | * When `blog_public` is off, WordPress disallows everything in robots.txt, so |
| 274 | * per-crawler controls are moot — the AI tab surfaces this instead of showing |
| 275 | * toggles that can't take effect. |
| 276 | * |
| 277 | * @return bool |
| 278 | */ |
| 279 | public static function search_engines_allowed() { |
| 280 | return (int) get_option( 'blog_public', 1 ) === 1; |
| 281 | } |
| 282 | |
| 283 | /** |
| 284 | * Whether the site is served from a WordPress.com staging subdomain |
| 285 | * (`*.wpcomstaging.com`) where the platform blocks all crawling regardless of |
| 286 | * the site's own settings — so AI-crawler controls can't take effect. |
| 287 | * |
| 288 | * @return bool |
| 289 | */ |
| 290 | public static function is_crawl_restricted_subdomain() { |
| 291 | $host = (string) wp_parse_url( home_url(), PHP_URL_HOST ); |
| 292 | $suffix = '.wpcomstaging.com'; |
| 293 | return strlen( $host ) > strlen( $suffix ) |
| 294 | && substr( $host, -strlen( $suffix ) ) === $suffix; |
| 295 | } |
| 296 | |
| 297 | /** |
| 298 | * Whether a static `robots.txt` file exists in the WordPress installation |
| 299 | * directory. These controls only change WordPress's virtual robots.txt and |
| 300 | * cannot modify that file. |
| 301 | * |
| 302 | * @return bool |
| 303 | */ |
| 304 | public static function has_static_robots_txt() { |
| 305 | return file_exists( ABSPATH . 'robots.txt' ); |
| 306 | } |
| 307 | |
| 308 | /** |
| 309 | * Whether this is a path-based multisite network. |
| 310 | * |
| 311 | * Such networks share one origin-level robots.txt, so per-site controls |
| 312 | * cannot be represented safely. |
| 313 | * |
| 314 | * @return bool |
| 315 | */ |
| 316 | public static function is_path_based_multisite() { |
| 317 | return is_multisite() && ! is_subdomain_install(); |
| 318 | } |
| 319 | |
| 320 | /** |
| 321 | * Append `Disallow: /` blocks for each blocked AI crawler to robots.txt. |
| 322 | * |
| 323 | * @param string $output The robots.txt content assembled so far. |
| 324 | * @param bool $public Whether the site is public (`blog_public`). When |
| 325 | * false WordPress already disallows everything, but the |
| 326 | * explicit per-bot blocks are still valid and harmless. |
| 327 | * @return string The robots.txt content with AI-crawler directives appended. |
| 328 | */ |
| 329 | public static function append_directives( $output, $public ) { |
| 330 | unset( $public ); |
| 331 | |
| 332 | // WordPress.com's existing privacy filter owns the site-wide opt-out and |
| 333 | // runs later at priority 12. Defer to it instead of emitting duplicate |
| 334 | // directives from this per-bot filter. |
| 335 | if ( self::is_path_based_multisite() || self::has_data_sharing_opt_out() ) { |
| 336 | return $output; |
| 337 | } |
| 338 | |
| 339 | $catalog = self::get_catalog(); |
| 340 | $blocked = self::get_blocked_slugs(); |
| 341 | if ( empty( $blocked ) ) { |
| 342 | return $output; |
| 343 | } |
| 344 | |
| 345 | $lines = array( '', '# AI crawlers blocked via Jetpack SEO.' ); |
| 346 | foreach ( $blocked as $slug ) { |
| 347 | $lines[] = 'User-agent: ' . $catalog[ $slug ]['user_agent']; |
| 348 | $lines[] = 'Disallow: /'; |
| 349 | $lines[] = ''; |
| 350 | } |
| 351 | |
| 352 | return $output . implode( "\n", $lines ) . "\n"; |
| 353 | } |
| 354 | |
| 355 | /** |
| 356 | * The AI tab's crawler bootstrap payload: the catalog (camelCased for JS), the |
| 357 | * current sparse override map, and the environment flags that decide whether |
| 358 | * the controls render or are replaced by an explanation. |
| 359 | * |
| 360 | * @return array |
| 361 | */ |
| 362 | public static function get_bootstrap_data() { |
| 363 | $catalog = array(); |
| 364 | foreach ( self::get_catalog() as $slug => $info ) { |
| 365 | $catalog[] = array( |
| 366 | 'slug' => $slug, |
| 367 | 'label' => $info['label'], |
| 368 | 'userAgent' => $info['user_agent'], |
| 369 | 'type' => $info['type'], |
| 370 | ); |
| 371 | } |
| 372 | |
| 373 | return array( |
| 374 | 'catalog' => $catalog, |
| 375 | 'overrides' => (object) self::get_overrides(), |
| 376 | 'searchEnginesVisible' => self::search_engines_allowed(), |
| 377 | 'restrictedSubdomain' => self::is_crawl_restricted_subdomain(), |
| 378 | 'staticRobotsTxt' => self::has_static_robots_txt(), |
| 379 | 'dataSharingOptOut' => self::has_data_sharing_opt_out(), |
| 380 | 'pathBasedMultisite' => self::is_path_based_multisite(), |
| 381 | 'privacySettingsUrl' => self::privacy_settings_url(), |
| 382 | 'robotsTxtUrl' => home_url( '/robots.txt' ), |
| 383 | ); |
| 384 | } |
| 385 | |
| 386 | /** |
| 387 | * URL of the settings screen holding "Prevent third-party sharing", linked from |
| 388 | * the AI tab when that setting is governing crawler access. |
| 389 | * |
| 390 | * Points at wp-admin → Settings → Reading, which is the same-origin home of the |
| 391 | * option on the sites where it exists (WordPress.com). Left as a package method |
| 392 | * so a Calypso deep link can replace it without touching the client. |
| 393 | * |
| 394 | * @return string |
| 395 | */ |
| 396 | public static function privacy_settings_url() { |
| 397 | return admin_url( 'options-reading.php' ); |
| 398 | } |
| 399 | } |
| 400 |