| @@ -383,11 +383,17 @@ | ||
| 383 | 383 | // static file directly (bypassing the robots_txt filter), so without |
| 384 | 384 | // this the file goes stale and /robots.txt shows the old content |
| 385 | 385 | // while the textarea shows the new — regardless of what the frontend |
| 386 | 386 | // believed about the file's existence. |
| 387 | + // | |
| 388 | + // ai_crawler_rules counts as a robots.txt change even though it is | |
| 389 | + // not the body: the directives it produces are composed into the | |
| 390 | + // served output at render time, so a physical file left alone here | |
| 391 | + // would keep serving the previous allow/block set (#657). | |
| 387 | 392 | if ($context_type === 'site' |
| 388 | 393 | && (array_key_exists('robots_txt_content', $settings) |
| 389 | - || array_key_exists('robots_txt_enabled', $settings)) | |
| 394 | + || array_key_exists('robots_txt_enabled', $settings) | |
| 395 | + || array_key_exists('ai_crawler_rules', $settings)) | |
| 390 | 396 | ) { |
| 391 | 397 | $this->identity_manager->sync_robots_txt_file(); |
| 392 | 398 | } |
| 393 | 399 | |
| @@ -565,8 +571,24 @@ | ||
| 565 | 571 | // show the served output next to the editable body and flag a |
| 566 | 572 | // physical file in the web root that has drifted from the settings. |
| 567 | 573 | $robots_data['effective'] = $this->identity_manager->get_robots_txt_delivery(); |
| 568 | 574 | |
| 575 | + // The per-agent AI crawler surface (#657). The registry ships with | |
| 576 | + // the response rather than being duplicated in the bundle, so a | |
| 577 | + // crawler added by the `thinkrank_ai_crawlers` filter appears in | |
| 578 | + // the UI without a rebuild. Rules are returned normalised, so a | |
| 579 | + // crawler with nothing stored comes back explicitly allowed rather | |
| 580 | + // than as an absence the client has to interpret. | |
| 581 | + $settings = $this->identity_manager->get_settings('site'); | |
| 582 | + $rules = \ThinkRank\SEO\AI_Crawlers::normalize_rules($settings['ai_crawler_rules'] ?? []); | |
| 583 | + | |
| 584 | + $robots_data['ai_crawlers'] = \ThinkRank\SEO\AI_Crawlers::for_display(); | |
| 585 | + $robots_data['ai_crawler_rules'] = []; | |
| 586 | + | |
| 587 | + foreach ($robots_data['ai_crawlers'] as $agent) { | |
| 588 | + $robots_data['ai_crawler_rules'][$agent['slug']] = $rules[$agent['slug']] ?? 'allow'; | |
| 589 | + } | |
| 590 | + | |
| 569 | 591 | return new WP_REST_Response([ |
| 570 | 592 | 'success' => true, |
| 571 | 593 | 'data' => $robots_data, |
| 572 | 594 | 'message' => 'Robots.txt data retrieved successfully' |
| @@ -622,8 +644,17 @@ | ||
| 622 | 644 | } |
| 623 | 645 | |
| 624 | 646 | $settings = ['robots_txt_enabled' => $enable_management]; |
| 625 | 647 | |
| 648 | + // Per-agent AI crawler rules (#657). Only written when the caller | |
| 649 | + // sends them: this route is also the plain "re-sync the file" save, | |
| 650 | + // and defaulting a missing parameter to an empty map there would | |
| 651 | + // unblock every crawler the site had blocked. | |
| 652 | + $ai_rules = $request->get_param('ai_crawler_rules'); | |
| 653 | + if (null !== $ai_rules) { | |
| 654 | + $settings['ai_crawler_rules'] = \ThinkRank\SEO\AI_Crawlers::normalize_rules($ai_rules); | |
| 655 | + } | |
| 656 | + | |
| 626 | 657 | if ($regenerate) { |
| 627 | 658 | // Generate and validate robots.txt from the rules. |
| 628 | 659 | $robots_data = $this->identity_manager->generate_robots_txt($custom_rules); |
| 629 | 660 | |
| @@ -1105,8 +1136,17 @@ | ||
| 1105 | 1136 | // sends it, and "false" is a non-empty string — truthy — so the |
| 1106 | 1137 | // file was written when the caller had asked it not to be. ("0" is |
| 1107 | 1138 | // falsy, which is why the failure was asymmetric.) Registering it |
| 1108 | 1139 | // gets core's boolean coercion (#394). |
| 1140 | + 'ai_crawler_rules' => [ | |
| 1141 | + 'required' => false, | |
| 1142 | + 'type' => 'object', | |
| 1143 | + 'description' => 'Per-agent AI crawler rules, keyed by crawler slug, each "allow" or "block".', | |
| 1144 | + 'additionalProperties' => [ | |
| 1145 | + 'type' => 'string', | |
| 1146 | + 'enum' => ['allow', 'block'], | |
| 1147 | + ], | |
| 1148 | + ], | |
| 1109 | 1149 | 'write_to_file' => [ |
| 1110 | 1150 | 'required' => false, |
| 1111 | 1151 | 'type' => 'boolean', |
| 1112 | 1152 | 'default' => true, |
