thinkrank / includes / seo / class-ai-traffic-tracker.php

class-ai-traffic-tracker.php in ThinkRank AI SEO – AI SEO Plugin for WordPress: Schema, XML Sitemaps, Meta Tags, Search Console & Local SEO 2.0.0, at includes/seo/class-ai-traffic-tracker.php

387 lines 14.2 KB

Raw Download Zip

1	<?php
2	/**
3	* AI referral traffic + AI crawler tracking.
4	*
5	* AI platforms (ChatGPT, Perplexity, Gemini, Claude, Copilot…) send real human
6	* visitors, but analytics tools misattribute much of it: the platforms strip
7	* or rewrite referrers, so GA4 files a large share under "Direct". WordPress
8	* serves its own pages, so — unlike a hosted storefront — the plugin IS in the
9	* request path and can read the referrer first-party, with no pixel and no
10	* JavaScript.
11	*
12	* What is stored (and deliberately nothing more): daily aggregate counters,
13	* one row per (day, kind, source, path). Three kinds:
14	*
15	* referral — a human pageview whose referrer host matched an AI platform
16	* crawler — a request whose user agent matched a known AI crawler
17	* baseline — every human pageview (source 'all', no path), so the dashboard
18	* can say "AI referrals are N% of traffic" without Google
19	*
20	* No IPs, no raw user agents, no cookies, no per-visit rows — nothing that
21	* identifies a visitor. That keeps the table small and the feature clean
22	* under wordpress.org privacy expectations.
23	*
24	* @package ThinkRank\SEO
25	* @since 1.27.0
26	*/
27
28	declare(strict_types=1);
29
30	namespace ThinkRank\SEO;
31
32	if (!defined('ABSPATH')) {
33	exit;
34	}
35
36	/**
37	* Records AI referrals/crawlers and serves the dashboard summary.
38	*/
39	class Ai_Traffic_Tracker {
40
41	/**
42	* Cron hook for pruning old aggregate rows.
43	*/
44	private const PRUNE_HOOK = 'thinkrank_ai_traffic_prune';
45
46	/**
47	* Days of history to keep. The dashboard reads 30; keep 6 months so a
48	* longer range is possible later without changing collection.
49	*/
50	private const RETENTION_DAYS = 180;
51
52	/**
53	* Referrer host fragments → platform slug. Checked with substring match
54	* against the referrer host, so subdomains are covered.
55	*
56	* @var array<string, string>
57	*/
58	private const REFERRER_PLATFORMS = [
59	'chatgpt.com' => 'chatgpt',
60	'chat.openai.com' => 'chatgpt',
61	'perplexity.ai' => 'perplexity',
62	'pplx.ai' => 'perplexity',
63	'gemini.google.com' => 'gemini',
64	'bard.google.com' => 'gemini',
65	'claude.ai' => 'claude',
66	'copilot.microsoft.com' => 'copilot',
67	'meta.ai' => 'meta-ai',
68	'you.com' => 'you',
69	'poe.com' => 'poe',
70	'grok.com' => 'grok',
71	'x.ai' => 'grok',
72	'chat.mistral.ai' => 'mistral',
73	'chat.deepseek.com' => 'deepseek',
74	'kimi.com' => 'kimi',
75	];
76
77	/**
78	* User-agent fragments → AI crawler slug. Case-insensitive substring
79	* match. Order matters where one token contains another — more specific
80	* entries first.
81	*
82	* @var array<string, string>
83	*/
84	private const CRAWLER_AGENTS = [
85	'OAI-SearchBot' => 'oai-searchbot',
86	'ChatGPT-User' => 'chatgpt-user',
87	'GPTBot' => 'gptbot',
88	'Perplexity-User' => 'perplexity-user',
89	'PerplexityBot' => 'perplexitybot',
90	'Claude-SearchBot' => 'claude-searchbot',
91	'Claude-User' => 'claude-user',
92	'ClaudeBot' => 'claudebot',
93	'anthropic-ai' => 'anthropic-ai',
94	'Google-Extended' => 'google-extended',
95	'Applebot-Extended' => 'applebot-extended',
96	'meta-externalagent' => 'meta-externalagent',
97	'meta-externalfetcher' => 'meta-externalfetcher',
98	'Bytespider' => 'bytespider',
99	'Amazonbot' => 'amazonbot',
100	'CCBot' => 'ccbot',
101	'cohere-ai' => 'cohere-ai',
102	'MistralAI-User' => 'mistral-user',
103	];
104
105	/**
106	* Wire the front-end recorder and the retention cron.
107	*
108	* @return void
109	*/
110	public function init(): void {
111	// Priority 1: record before any template logic can redirect/exit.
112	add_action('template_redirect', [$this, 'record'], 1);
113
114	add_action(self::PRUNE_HOOK, [$this, 'prune']);
115	if (!wp_next_scheduled(self::PRUNE_HOOK)) {
116	wp_schedule_event(time() + DAY_IN_SECONDS, 'daily', self::PRUNE_HOOK);
117	}
118	}
119
120	/**
121	* Classify a referrer URL as an AI platform.
122	*
123	* @param string $referrer Full referrer URL (may be empty).
124	* @return string\|null Platform slug, or null when not an AI platform.
125	*/
126	public static function classify_referrer(string $referrer): ?string {
127	if ('' === $referrer) {
128	return null;
129	}
130
131	$host = strtolower((string) wp_parse_url($referrer, PHP_URL_HOST));
132	if ('' === $host) {
133	return null;
134	}
135
136	foreach (self::REFERRER_PLATFORMS as $fragment => $slug) {
137	// Suffix match on the host so evil.com/?q=claude.ai can't spoof
138	// via path, and subdomains (www.perplexity.ai) still match.
139	if ($host === $fragment \|\| str_ends_with($host, '.' . $fragment)) {
140	return $slug;
141	}
142	}
143
144	return null;
145	}
146
147	/**
148	* Classify a user agent as an AI crawler.
149	*
150	* @param string $user_agent Raw user agent (may be empty).
151	* @return string\|null Crawler slug, or null when not a known AI crawler.
152	*/
153	public static function classify_crawler(string $user_agent): ?string {
154	if ('' === $user_agent) {
155	return null;
156	}
157
158	foreach (self::CRAWLER_AGENTS as $fragment => $slug) {
159	if (false !== stripos($user_agent, $fragment)) {
160	return $slug;
161	}
162	}
163
164	return null;
165	}
166
167	/**
168	* Record the current front-end request into the daily aggregates.
169	*
170	* @return void
171	*/
172	public function record(): void {
173	if (is_admin() \|\| wp_doing_ajax() \|\| wp_doing_cron()) {
174	return;
175	}
176	if (is_feed() \|\| is_preview() \|\| is_robots() \|\| is_404()) {
177	return;
178	}
179	$method = isset($_SERVER['REQUEST_METHOD']) ? strtoupper((string) wp_unslash($_SERVER['REQUEST_METHOD'])) : 'GET'; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput
180	if ('GET' !== $method) {
181	return;
182	}
183
184	$user_agent = isset($_SERVER['HTTP_USER_AGENT']) ? (string) wp_unslash($_SERVER['HTTP_USER_AGENT']) : ''; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- classified, never stored raw.
185
186	// AI crawler: count it and stop — a bot is not part of the human
187	// baseline and has no meaningful referrer.
188	$bot = self::classify_crawler($user_agent);
189	if (null !== $bot) {
190	$this->bump('crawler', $bot);
191	return;
192	}
193
194	// Editors/admins browsing their own site would skew small sites.
195	if (is_user_logged_in() && current_user_can('edit_posts')) {
196	return;
197	}
198
199	$this->bump('baseline', 'all');
200
201	$referrer = isset($_SERVER['HTTP_REFERER']) ? (string) wp_unslash($_SERVER['HTTP_REFERER']) : ''; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- classified, never stored raw.
202	$platform = self::classify_referrer($referrer);
203	if (null !== $platform) {
204	$this->bump('referral', $platform, $this->current_path());
205	}
206	}
207
208	/**
209	* Record a page served as Markdown to an AI agent.
210	*
211	* Called by Pro's Markdown for AI feature at serve time. Lives here rather
212	* than in Pro because this class owns the aggregate table; Pro owning a
213	* second writer to it would couple the schema to two repos.
214	*
215	* @param string $source Crawler slug when the agent is a known AI crawler,
216	* 'header' for Accept-negotiated requests, 'link' for
217	* ?format=markdown / .md URLs.
218	* @param string $path Path of the post served.
219	* @return void
220	*/
221	public function record_served_markdown(string $source, string $path = ''): void {
222	$source = sanitize_key($source);
223	if ('' === $source) {
224	$source = 'other';
225	}
226	$this->bump('markdown', $source, substr($path, 0, 191));
227	}
228
229	/**
230	* Total Markdown-for-AI responses served in the last N days.
231	*
232	* @param int $days Range in days (bounded 1–180).
233	* @return int
234	*/
235	public function served_markdown_count(int $days = 30): int {
236	global $wpdb;
237
238	$days = max(1, min(self::RETENTION_DAYS, $days));
239	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
240	$since = gmdate('Y-m-d', time() - $days * DAY_IN_SECONDS);
241
242	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- read-only aggregate over our own table.
243	return (int) $wpdb->get_var(
244	$wpdb->prepare(
245	"SELECT COALESCE(SUM(hits), 0) FROM {$table} WHERE kind = 'markdown' AND day >= %s",
246	$since
247	)
248	);
249	// phpcs:enable
250	}
251
252	/**
253	* The current request path, normalized for the aggregate key.
254	*
255	* @return string
256	*/
257	private function current_path(): string {
258	$uri = isset($_SERVER['REQUEST_URI']) ? (string) wp_unslash($_SERVER['REQUEST_URI']) : '/'; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- reduced to its path component below.
259	$path = (string) wp_parse_url($uri, PHP_URL_PATH);
260	if ('' === $path) {
261	$path = '/';
262	}
263	return substr($path, 0, 191);
264	}
265
266	/**
267	* Increment one daily aggregate bucket.
268	*
269	* @param string $kind 'referral' \| 'crawler' \| 'baseline'.
270	* @param string $source Platform/bot slug, or 'all' for baseline.
271	* @param string $path Landing path (referrals only).
272	* @return void
273	*/
274	private function bump(string $kind, string $source, string $path = ''): void {
275	global $wpdb;
276
277	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
278
279	// Single cheap upsert per pageview; the unique key is the bucket.
280	// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- aggregate counter upsert; table name is prefix-derived.
281	$wpdb->query(
282	// phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- table name is $wpdb->prefix plus a literal, and every value is passed as a placeholder replacement.
283	$wpdb->prepare(
284	"INSERT INTO {$table} (day, kind, source, path, hits) VALUES (%s, %s, %s, %s, 1)
285	ON DUPLICATE KEY UPDATE hits = hits + 1",
286	current_time('Y-m-d'),
287	$kind,
288	$source,
289	$path
290	)
291	);
292	// phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared
293	}
294
295	/**
296	* Dashboard summary for the last N days.
297	*
298	* @param int $days Range in days (bounded 1–180).
299	* @return array<string, mixed>
300	*/
301	public function summary(int $days = 30): array {
302	global $wpdb;
303
304	$days = max(1, min(self::RETENTION_DAYS, $days));
305	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
306	$since = gmdate('Y-m-d', time() - $days * DAY_IN_SECONDS);
307
308	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- read-only aggregates over our own table.
309	$rows = $wpdb->get_results(
310	$wpdb->prepare(
311	"SELECT day, kind, source, path, hits FROM {$table} WHERE day >= %s",
312	$since
313	),
314	ARRAY_A
315	);
316	// phpcs:enable
317
318	$baseline = 0;
319	$referrals = 0;
320	$platforms = [];
321	$trend = [];
322	$pages = [];
323	$crawlers = [];
324	$markdown = 0;
325
326	foreach ((array) $rows as $row) {
327	$hits = (int) $row['hits'];
328	switch ($row['kind']) {
329	case 'baseline':
330	$baseline += $hits;
331	break;
332	case 'referral':
333	$referrals += $hits;
334	$platforms[$row['source']] = ($platforms[$row['source']] ?? 0) + $hits;
335	$trend[$row['day']] = ($trend[$row['day']] ?? 0) + $hits;
336	if ('' !== $row['path']) {
337	$pages[$row['path']] = ($pages[$row['path']] ?? 0) + $hits;
338	}
339	break;
340	case 'crawler':
341	$crawlers[$row['source']] = ($crawlers[$row['source']] ?? 0) + $hits;
342	break;
343	case 'markdown':
344	$markdown += $hits;
345	break;
346	}
347	}
348
349	arsort($platforms);
350	arsort($pages);
351	arsort($crawlers);
352	ksort($trend);
353
354	return [
355	'days' => $days,
356	'baseline' => $baseline,
357	'ai_sessions' => $referrals,
358	'ai_share' => $baseline > 0 ? round($referrals / $baseline * 100, 1) : 0.0,
359	'platforms' => $platforms,
360	'trend' => $trend,
361	'top_pages' => array_slice($pages, 0, 10, true),
362	'crawlers' => $crawlers,
363	// Whether llms.txt is being served, so the crawler panel can pair
364	// "bots are coming" with "and here's what we feed them".
365	'llms_txt' => file_exists(ABSPATH . 'llms.txt'),
366	// Pages served as Markdown by Pro's Markdown for AI feature
367	// (kind 'markdown', written via record_served_markdown()).
368	'markdown_served' => $markdown,
369	];
370	}
371
372	/**
373	* Drop aggregate rows past the retention window.
374	*
375	* @return void
376	*/
377	public function prune(): void {
378	global $wpdb;
379
380	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
381	$cutoff = gmdate('Y-m-d', time() - self::RETENTION_DAYS * DAY_IN_SECONDS);
382
383	// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- retention delete on our own table.
384	$wpdb->query($wpdb->prepare("DELETE FROM {$table} WHERE day < %s", $cutoff));
385	}
386	}
387