| @@ -97,36 +97,8 @@ | ||
| 97 | 97 | 'kimi.com' => 'kimi', |
| 98 | 98 | ]; |
| 99 | 99 | |
| 100 | 100 | /** |
| 101 | - * User-agent fragments → AI crawler slug. Case-insensitive substring | |
| 102 | - * match. Order matters where one token contains another — more specific | |
| 103 | - * entries first. | |
| 104 | - * | |
| 105 | - * @var array<string, string> | |
| 106 | - */ | |
| 107 | - private const CRAWLER_AGENTS = [ | |
| 108 | - 'OAI-SearchBot' => 'oai-searchbot', | |
| 109 | - 'ChatGPT-User' => 'chatgpt-user', | |
| 110 | - 'GPTBot' => 'gptbot', | |
| 111 | - 'Perplexity-User' => 'perplexity-user', | |
| 112 | - 'PerplexityBot' => 'perplexitybot', | |
| 113 | - 'Claude-SearchBot' => 'claude-searchbot', | |
| 114 | - 'Claude-User' => 'claude-user', | |
| 115 | - 'ClaudeBot' => 'claudebot', | |
| 116 | - 'anthropic-ai' => 'anthropic-ai', | |
| 117 | - 'Google-Extended' => 'google-extended', | |
| 118 | - 'Applebot-Extended' => 'applebot-extended', | |
| 119 | - 'meta-externalagent' => 'meta-externalagent', | |
| 120 | - 'meta-externalfetcher' => 'meta-externalfetcher', | |
| 121 | - 'Bytespider' => 'bytespider', | |
| 122 | - 'Amazonbot' => 'amazonbot', | |
| 123 | - 'CCBot' => 'ccbot', | |
| 124 | - 'cohere-ai' => 'cohere-ai', | |
| 125 | - 'MistralAI-User' => 'mistral-user', | |
| 126 | - ]; | |
| 127 | - | |
| 128 | - /** | |
| 129 | 101 | * Wire the front-end recorder and the retention cron. |
| 130 | 102 | * |
| 131 | 103 | * @return void |
| 132 | 104 | */ |
| @@ -169,8 +141,12 @@ | ||
| 169 | 141 | |
| 170 | 142 | /** |
| 171 | 143 | * Classify a user agent as an AI crawler. |
| 172 | 144 | * |
| 145 | + * The agent list is `AI_Crawlers`, shared with the robots.txt panel — the | |
| 146 | + * two must agree about which bots exist, or a site blocks a crawler it is | |
| 147 | + * not counting (#657). | |
| 148 | + * | |
| 173 | 149 | * @param string $user_agent Raw user agent (may be empty). |
| 174 | 150 | * @return string|null Crawler slug, or null when not a known AI crawler. |
| 175 | 151 | */ |
| 176 | 152 | public static function classify_crawler(string $user_agent): ?string { |
| @@ -177,9 +153,12 @@ | ||
| 177 | 153 | if ('' === $user_agent) { |
| 178 | 154 | return null; |
| 179 | 155 | } |
| 180 | 156 | |
| 181 | - foreach (self::CRAWLER_AGENTS as $fragment => $slug) { | |
| 157 | + // Token order is significant and owned by the registry: the first | |
| 158 | + // token found wins, so `Claude-SearchBot` has to be tested before | |
| 159 | + // `ClaudeBot`. See AI_Crawlers::AGENTS. | |
| 160 | + foreach (AI_Crawlers::token_map() as $fragment => $slug) { | |
| 182 | 161 | if (false !== stripos($user_agent, $fragment)) { |
| 183 | 162 | return $slug; |
| 184 | 163 | } |
| 185 | 164 | } |
