thinkrank / includes / seo / class-ai-traffic-tracker.php

class-ai-traffic-tracker.php in ThinkRank AI SEO – AI SEO Plugin for WordPress: Schema, XML Sitemaps, Meta Tags, Search Console & Local SEO 2.1.1, at includes/seo/class-ai-traffic-tracker.php

465 lines 16.9 KB

Raw Download Zip

1	<?php
2	/**
3	* AI referral traffic + AI crawler tracking.
4	*
5	* AI platforms (ChatGPT, Perplexity, Gemini, Claude, Copilot…) send real human
6	* visitors, but analytics tools misattribute much of it: the platforms strip
7	* or rewrite referrers, so GA4 files a large share under "Direct". WordPress
8	* serves its own pages, so — unlike a hosted storefront — the plugin IS in the
9	* request path and can read the referrer first-party, with no pixel and no
10	* JavaScript.
11	*
12	* What is stored (and deliberately nothing more): daily aggregate counters,
13	* one row per (day, kind, source, path). Three kinds:
14	*
15	* referral — a human pageview whose referrer host matched an AI platform
16	* crawler — a request whose user agent matched a known AI crawler
17	* baseline — every human pageview (source 'all', no path), so the dashboard
18	* can say "AI referrals are N% of traffic" without Google
19	*
20	* No IPs, no raw user agents, no cookies, no per-visit rows — nothing that
21	* identifies a visitor. That keeps the table small and the feature clean
22	* under wordpress.org privacy expectations.
23	*
24	* @package ThinkRank\SEO
25	* @since 1.27.0
26	*/
27
28	declare(strict_types=1);
29
30	namespace ThinkRank\SEO;
31
32	if (!defined('ABSPATH')) {
33	exit;
34	}
35
36	/**
37	* Records AI referrals/crawlers and serves the dashboard summary.
38	*/
39	class Ai_Traffic_Tracker {
40
41	/**
42	* Cron hook for pruning old aggregate rows.
43	*/
44	private const PRUNE_HOOK = 'thinkrank_ai_traffic_prune';
45
46	/**
47	* Object-cache group for the buffered hit counters.
48	*/
49	private const COUNTER_GROUP = 'thinkrank_traffic';
50
51	/**
52	* Key prefix for those counters.
53	*/
54	private const COUNTER_PREFIX = 'tr_traffic_';
55
56	/**
57	* Flush a bucket once it has this many buffered hits.
58	*/
59	private const FLUSH_AT = 50;
60
61	/**
62	* ...or once its oldest buffered hit is this many seconds old, so a quiet
63	* site still records its traffic.
64	*/
65	private const FLUSH_AFTER = 300;
66
67	/**
68	* Days of history to keep. The dashboard reads 30; keep 6 months so a
69	* longer range is possible later without changing collection.
70	*/
71	private const RETENTION_DAYS = 180;
72
73	/**
74	* Referrer host fragments → platform slug. Checked with substring match
75	* against the referrer host, so subdomains are covered.
76	*
77	* @var array<string, string>
78	*/
79	private const REFERRER_PLATFORMS = [
80	'chatgpt.com' => 'chatgpt',
81	'chat.openai.com' => 'chatgpt',
82	'perplexity.ai' => 'perplexity',
83	'pplx.ai' => 'perplexity',
84	'gemini.google.com' => 'gemini',
85	'bard.google.com' => 'gemini',
86	'claude.ai' => 'claude',
87	'copilot.microsoft.com' => 'copilot',
88	'meta.ai' => 'meta-ai',
89	'you.com' => 'you',
90	'poe.com' => 'poe',
91	'grok.com' => 'grok',
92	'x.ai' => 'grok',
93	'chat.mistral.ai' => 'mistral',
94	'chat.deepseek.com' => 'deepseek',
95	'kimi.com' => 'kimi',
96	];
97
98	/**
99	* User-agent fragments → AI crawler slug. Case-insensitive substring
100	* match. Order matters where one token contains another — more specific
101	* entries first.
102	*
103	* @var array<string, string>
104	*/
105	private const CRAWLER_AGENTS = [
106	'OAI-SearchBot' => 'oai-searchbot',
107	'ChatGPT-User' => 'chatgpt-user',
108	'GPTBot' => 'gptbot',
109	'Perplexity-User' => 'perplexity-user',
110	'PerplexityBot' => 'perplexitybot',
111	'Claude-SearchBot' => 'claude-searchbot',
112	'Claude-User' => 'claude-user',
113	'ClaudeBot' => 'claudebot',
114	'anthropic-ai' => 'anthropic-ai',
115	'Google-Extended' => 'google-extended',
116	'Applebot-Extended' => 'applebot-extended',
117	'meta-externalagent' => 'meta-externalagent',
118	'meta-externalfetcher' => 'meta-externalfetcher',
119	'Bytespider' => 'bytespider',
120	'Amazonbot' => 'amazonbot',
121	'CCBot' => 'ccbot',
122	'cohere-ai' => 'cohere-ai',
123	'MistralAI-User' => 'mistral-user',
124	];
125
126	/**
127	* Wire the front-end recorder and the retention cron.
128	*
129	* @return void
130	*/
131	public function init(): void {
132	// Priority 1: record before any template logic can redirect/exit.
133	add_action('template_redirect', [$this, 'record'], 1);
134
135	add_action(self::PRUNE_HOOK, [$this, 'prune']);
136	if (!wp_next_scheduled(self::PRUNE_HOOK)) {
137	wp_schedule_event(time() + DAY_IN_SECONDS, 'daily', self::PRUNE_HOOK);
138	}
139	}
140
141	/**
142	* Classify a referrer URL as an AI platform.
143	*
144	* @param string $referrer Full referrer URL (may be empty).
145	* @return string\|null Platform slug, or null when not an AI platform.
146	*/
147	public static function classify_referrer(string $referrer): ?string {
148	if ('' === $referrer) {
149	return null;
150	}
151
152	$host = strtolower((string) wp_parse_url($referrer, PHP_URL_HOST));
153	if ('' === $host) {
154	return null;
155	}
156
157	foreach (self::REFERRER_PLATFORMS as $fragment => $slug) {
158	// Suffix match on the host so evil.com/?q=claude.ai can't spoof
159	// via path, and subdomains (www.perplexity.ai) still match.
160	if ($host === $fragment \|\| str_ends_with($host, '.' . $fragment)) {
161	return $slug;
162	}
163	}
164
165	return null;
166	}
167
168	/**
169	* Classify a user agent as an AI crawler.
170	*
171	* @param string $user_agent Raw user agent (may be empty).
172	* @return string\|null Crawler slug, or null when not a known AI crawler.
173	*/
174	public static function classify_crawler(string $user_agent): ?string {
175	if ('' === $user_agent) {
176	return null;
177	}
178
179	foreach (self::CRAWLER_AGENTS as $fragment => $slug) {
180	if (false !== stripos($user_agent, $fragment)) {
181	return $slug;
182	}
183	}
184
185	return null;
186	}
187
188	/**
189	* Record the current front-end request into the daily aggregates.
190	*
191	* @return void
192	*/
193	public function record(): void {
194	if (is_admin() \|\| wp_doing_ajax() \|\| wp_doing_cron()) {
195	return;
196	}
197	if (is_feed() \|\| is_preview() \|\| is_robots() \|\| is_404()) {
198	return;
199	}
200	$method = isset($_SERVER['REQUEST_METHOD']) ? strtoupper((string) wp_unslash($_SERVER['REQUEST_METHOD'])) : 'GET'; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput
201	if ('GET' !== $method) {
202	return;
203	}
204
205	$user_agent = isset($_SERVER['HTTP_USER_AGENT']) ? (string) wp_unslash($_SERVER['HTTP_USER_AGENT']) : ''; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- classified, never stored raw.
206
207	// AI crawler: count it and stop — a bot is not part of the human
208	// baseline and has no meaningful referrer.
209	$bot = self::classify_crawler($user_agent);
210	if (null !== $bot) {
211	$this->bump('crawler', $bot);
212	return;
213	}
214
215	// Editors/admins browsing their own site would skew small sites.
216	if (is_user_logged_in() && current_user_can('edit_posts')) {
217	return;
218	}
219
220	$this->bump('baseline', 'all');
221
222	$referrer = isset($_SERVER['HTTP_REFERER']) ? (string) wp_unslash($_SERVER['HTTP_REFERER']) : ''; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- classified, never stored raw.
223	$platform = self::classify_referrer($referrer);
224	if (null !== $platform) {
225	$this->bump('referral', $platform, $this->current_path());
226	}
227	}
228
229	/**
230	* Record a page served as Markdown to an AI agent.
231	*
232	* Called by Pro's Markdown for AI feature at serve time. Lives here rather
233	* than in Pro because this class owns the aggregate table; Pro owning a
234	* second writer to it would couple the schema to two repos.
235	*
236	* @param string $source Crawler slug when the agent is a known AI crawler,
237	* 'header' for Accept-negotiated requests, 'link' for
238	* ?format=markdown / .md URLs.
239	* @param string $path Path of the post served.
240	* @return void
241	*/
242	public function record_served_markdown(string $source, string $path = ''): void {
243	$source = sanitize_key($source);
244	if ('' === $source) {
245	$source = 'other';
246	}
247	$this->bump('markdown', $source, substr($path, 0, 191));
248	}
249
250	/**
251	* Total Markdown-for-AI responses served in the last N days.
252	*
253	* @param int $days Range in days (bounded 1–180).
254	* @return int
255	*/
256	public function served_markdown_count(int $days = 30): int {
257	global $wpdb;
258
259	$days = max(1, min(self::RETENTION_DAYS, $days));
260	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
261	$since = gmdate('Y-m-d', time() - $days * DAY_IN_SECONDS);
262
263	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- read-only aggregate over our own table.
264	return (int) $wpdb->get_var(
265	$wpdb->prepare(
266	"SELECT COALESCE(SUM(hits), 0) FROM {$table} WHERE kind = 'markdown' AND day >= %s",
267	$since
268	)
269	);
270	// phpcs:enable
271	}
272
273	/**
274	* The current request path, normalized for the aggregate key.
275	*
276	* @return string
277	*/
278	private function current_path(): string {
279	$uri = isset($_SERVER['REQUEST_URI']) ? (string) wp_unslash($_SERVER['REQUEST_URI']) : '/'; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- reduced to its path component below.
280	$path = (string) wp_parse_url($uri, PHP_URL_PATH);
281	if ('' === $path) {
282	$path = '/';
283	}
284	return substr($path, 0, 191);
285	}
286
287	/**
288	* Increment one daily aggregate bucket.
289	*
290	* @param string $kind 'referral' \| 'crawler' \| 'baseline'.
291	* @param string $source Platform/bot slug, or 'all' for baseline.
292	* @param string $path Landing path (referrals only).
293	* @return void
294	*/
295	private function bump(string $kind, string $source, string $path = ''): void {
296	// Without a persistent object cache there is nowhere to buffer, so keep
297	// the direct write rather than counting into per-request memory that is
298	// thrown away — that would lose hits outright.
299	if (!wp_using_ext_object_cache()) {
300	$this->write_bucket($kind, $source, $path, 1);
301
302	return;
303	}
304
305	// With one, buffer and flush in batches. The unique key is
306	// (day, kind, source, path), so all baseline traffic funnels into a
307	// single row per day: InnoDB took an exclusive row lock on it for every
308	// visitor, serialising concurrent anonymous traffic, and made every
309	// pageview a write even when the response was fully cacheable (#402).
310	$bucket = self::COUNTER_PREFIX . md5($kind . '\|' . $source . '\|' . $path);
311	$since = $bucket . '_since';
312
313	$hits = wp_cache_incr($bucket, 1, self::COUNTER_GROUP);
314
315	if (false === $hits) {
316	wp_cache_add($bucket, 1, self::COUNTER_GROUP, 0);
317	wp_cache_add($since, time(), self::COUNTER_GROUP, 0);
318	$hits = 1;
319	}
320
321	$started = (int) wp_cache_get($since, self::COUNTER_GROUP);
322
323	// Flush on either bound, so a busy site writes rarely and a quiet one
324	// still lands its hits — an eviction can cost at most one window.
325	if ($hits < self::FLUSH_AT && $started > 0 && (time() - $started) < self::FLUSH_AFTER) {
326	return;
327	}
328
329	wp_cache_set($bucket, 0, self::COUNTER_GROUP, 0);
330	wp_cache_set($since, time(), self::COUNTER_GROUP, 0);
331
332	$this->write_bucket($kind, $source, $path, (int) $hits);
333	}
334
335	/**
336	* Add hits to a bucket's row.
337	*
338	* @since 2.0.1
339	*
340	* @param string $kind 'referral' \| 'crawler' \| 'baseline'.
341	* @param string $source Platform/bot slug, or 'all' for baseline.
342	* @param string $path Landing path (referrals only).
343	* @param int $hits How many hits to add.
344	* @return void
345	*/
346	private function write_bucket(string $kind, string $source, string $path, int $hits): void {
347	if ($hits < 1) {
348	return;
349	}
350
351	global $wpdb;
352
353	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
354
355	// Aggregate counter upsert; the unique key is the bucket.
356	// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- aggregate counter upsert; table name is prefix-derived.
357	$wpdb->query(
358	// phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- table name is $wpdb->prefix plus a literal, and every value is passed as a placeholder replacement.
359	$wpdb->prepare(
360	"INSERT INTO {$table} (day, kind, source, path, hits) VALUES (%s, %s, %s, %s, %d)
361	ON DUPLICATE KEY UPDATE hits = hits + %d",
362	current_time('Y-m-d'),
363	$kind,
364	$source,
365	$path,
366	$hits,
367	$hits
368	)
369	);
370	// phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared
371	}
372
373	/**
374	* Dashboard summary for the last N days.
375	*
376	* @param int $days Range in days (bounded 1–180).
377	* @return array<string, mixed>
378	*/
379	public function summary(int $days = 30): array {
380	global $wpdb;
381
382	$days = max(1, min(self::RETENTION_DAYS, $days));
383	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
384	$since = gmdate('Y-m-d', time() - $days * DAY_IN_SECONDS);
385
386	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- read-only aggregates over our own table.
387	$rows = $wpdb->get_results(
388	$wpdb->prepare(
389	"SELECT day, kind, source, path, hits FROM {$table} WHERE day >= %s",
390	$since
391	),
392	ARRAY_A
393	);
394	// phpcs:enable
395
396	$baseline = 0;
397	$referrals = 0;
398	$platforms = [];
399	$trend = [];
400	$pages = [];
401	$crawlers = [];
402	$markdown = 0;
403
404	foreach ((array) $rows as $row) {
405	$hits = (int) $row['hits'];
406	switch ($row['kind']) {
407	case 'baseline':
408	$baseline += $hits;
409	break;
410	case 'referral':
411	$referrals += $hits;
412	$platforms[$row['source']] = ($platforms[$row['source']] ?? 0) + $hits;
413	$trend[$row['day']] = ($trend[$row['day']] ?? 0) + $hits;
414	if ('' !== $row['path']) {
415	$pages[$row['path']] = ($pages[$row['path']] ?? 0) + $hits;
416	}
417	break;
418	case 'crawler':
419	$crawlers[$row['source']] = ($crawlers[$row['source']] ?? 0) + $hits;
420	break;
421	case 'markdown':
422	$markdown += $hits;
423	break;
424	}
425	}
426
427	arsort($platforms);
428	arsort($pages);
429	arsort($crawlers);
430	ksort($trend);
431
432	return [
433	'days' => $days,
434	'baseline' => $baseline,
435	'ai_sessions' => $referrals,
436	'ai_share' => $baseline > 0 ? round($referrals / $baseline * 100, 1) : 0.0,
437	'platforms' => $platforms,
438	'trend' => $trend,
439	'top_pages' => array_slice($pages, 0, 10, true),
440	'crawlers' => $crawlers,
441	// Whether llms.txt is being served, so the crawler panel can pair
442	// "bots are coming" with "and here's what we feed them".
443	'llms_txt' => file_exists(ABSPATH . 'llms.txt'),
444	// Pages served as Markdown by Pro's Markdown for AI feature
445	// (kind 'markdown', written via record_served_markdown()).
446	'markdown_served' => $markdown,
447	];
448	}
449
450	/**
451	* Drop aggregate rows past the retention window.
452	*
453	* @return void
454	*/
455	public function prune(): void {
456	global $wpdb;
457
458	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
459	$cutoff = gmdate('Y-m-d', time() - self::RETENTION_DAYS * DAY_IN_SECONDS);
460
461	// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- retention delete on our own table.
462	$wpdb->query($wpdb->prepare("DELETE FROM {$table} WHERE day < %s", $cutoff));
463	}
464	}
465