thinkrank / includes / seo / class-ai-traffic-tracker.php

class-ai-traffic-tracker.php in ThinkRank AI SEO – AI SEO Plugin for WordPress: Schema, XML Sitemaps, Meta Tags, Search Console & Local SEO 2.4.0, at includes/seo/class-ai-traffic-tracker.php

612 lines 23.1 KB

Raw Download Zip

1	<?php
2	/**
3	* AI referral traffic + AI crawler tracking.
4	*
5	* AI platforms (ChatGPT, Perplexity, Gemini, Claude, Copilot…) send real human
6	* visitors, but analytics tools misattribute much of it: the platforms strip
7	* or rewrite referrers, so GA4 files a large share under "Direct". WordPress
8	* serves its own pages, so — unlike a hosted storefront — the plugin IS in the
9	* request path and can read the referrer first-party, with no pixel and no
10	* JavaScript.
11	*
12	* What is stored (and deliberately nothing more): daily aggregate counters,
13	* one row per (day, kind, source, path). Three kinds:
14	*
15	* referral — a human pageview whose referrer host matched an AI platform
16	* crawler — a request whose user agent matched a known AI crawler
17	* baseline — every human pageview (source 'all', no path), so the dashboard
18	* can say "AI referrals are N% of traffic" without Google
19	*
20	* No IPs, no raw user agents, no cookies, no per-visit rows — nothing that
21	* identifies a visitor. That keeps the table small and the feature clean
22	* under wordpress.org privacy expectations.
23	*
24	* @package ThinkRank\SEO
25	* @since 1.27.0
26	*/
27
28	declare(strict_types=1);
29
30	namespace ThinkRank\SEO;
31
32	use DateTimeImmutable;
33
34	if (!defined('ABSPATH')) {
35	exit;
36	}
37
38	/**
39	* Records AI referrals/crawlers and serves the dashboard summary.
40	*/
41	class Ai_Traffic_Tracker {
42
43	/**
44	* Cron hook for pruning old aggregate rows.
45	*/
46	private const PRUNE_HOOK = 'thinkrank_ai_traffic_prune';
47
48	/**
49	* Object-cache group for the buffered hit counters.
50	*/
51	private const COUNTER_GROUP = 'thinkrank_traffic';
52
53	/**
54	* Key prefix for those counters.
55	*/
56	private const COUNTER_PREFIX = 'tr_traffic_';
57
58	/**
59	* Flush a bucket once it has this many buffered hits.
60	*/
61	private const FLUSH_AT = 50;
62
63	/**
64	* ...or once its oldest buffered hit is this many seconds old, so a quiet
65	* site still records its traffic.
66	*/
67	private const FLUSH_AFTER = 300;
68
69	/**
70	* Days of history to keep. The dashboard reads 30; keep 6 months so a
71	* longer range is possible later without changing collection.
72	*/
73	private const RETENTION_DAYS = 180;
74
75	/**
76	* Referrer host fragments → platform slug. Checked with substring match
77	* against the referrer host, so subdomains are covered.
78	*
79	* @var array<string, string>
80	*/
81	private const REFERRER_PLATFORMS = [
82	'chatgpt.com' => 'chatgpt',
83	'chat.openai.com' => 'chatgpt',
84	'perplexity.ai' => 'perplexity',
85	'pplx.ai' => 'perplexity',
86	'gemini.google.com' => 'gemini',
87	'bard.google.com' => 'gemini',
88	'claude.ai' => 'claude',
89	'copilot.microsoft.com' => 'copilot',
90	'meta.ai' => 'meta-ai',
91	'you.com' => 'you',
92	'poe.com' => 'poe',
93	'grok.com' => 'grok',
94	'x.ai' => 'grok',
95	'chat.mistral.ai' => 'mistral',
96	'chat.deepseek.com' => 'deepseek',
97	'kimi.com' => 'kimi',
98	];
99
100	/**
101	* User-agent fragments → AI crawler slug. Case-insensitive substring
102	* match. Order matters where one token contains another — more specific
103	* entries first.
104	*
105	* @var array<string, string>
106	*/
107	private const CRAWLER_AGENTS = [
108	'OAI-SearchBot' => 'oai-searchbot',
109	'ChatGPT-User' => 'chatgpt-user',
110	'GPTBot' => 'gptbot',
111	'Perplexity-User' => 'perplexity-user',
112	'PerplexityBot' => 'perplexitybot',
113	'Claude-SearchBot' => 'claude-searchbot',
114	'Claude-User' => 'claude-user',
115	'ClaudeBot' => 'claudebot',
116	'anthropic-ai' => 'anthropic-ai',
117	'Google-Extended' => 'google-extended',
118	'Applebot-Extended' => 'applebot-extended',
119	'meta-externalagent' => 'meta-externalagent',
120	'meta-externalfetcher' => 'meta-externalfetcher',
121	'Bytespider' => 'bytespider',
122	'Amazonbot' => 'amazonbot',
123	'CCBot' => 'ccbot',
124	'cohere-ai' => 'cohere-ai',
125	'MistralAI-User' => 'mistral-user',
126	];
127
128	/**
129	* Wire the front-end recorder and the retention cron.
130	*
131	* @return void
132	*/
133	public function init(): void {
134	// Priority 1: record before any template logic can redirect/exit.
135	add_action('template_redirect', [$this, 'record'], 1);
136
137	add_action(self::PRUNE_HOOK, [$this, 'prune']);
138	if (!wp_next_scheduled(self::PRUNE_HOOK)) {
139	wp_schedule_event(time() + DAY_IN_SECONDS, 'daily', self::PRUNE_HOOK);
140	}
141	}
142
143	/**
144	* Classify a referrer URL as an AI platform.
145	*
146	* @param string $referrer Full referrer URL (may be empty).
147	* @return string\|null Platform slug, or null when not an AI platform.
148	*/
149	public static function classify_referrer(string $referrer): ?string {
150	if ('' === $referrer) {
151	return null;
152	}
153
154	$host = strtolower((string) wp_parse_url($referrer, PHP_URL_HOST));
155	if ('' === $host) {
156	return null;
157	}
158
159	foreach (self::REFERRER_PLATFORMS as $fragment => $slug) {
160	// Suffix match on the host so evil.com/?q=claude.ai can't spoof
161	// via path, and subdomains (www.perplexity.ai) still match.
162	if ($host === $fragment \|\| str_ends_with($host, '.' . $fragment)) {
163	return $slug;
164	}
165	}
166
167	return null;
168	}
169
170	/**
171	* Classify a user agent as an AI crawler.
172	*
173	* @param string $user_agent Raw user agent (may be empty).
174	* @return string\|null Crawler slug, or null when not a known AI crawler.
175	*/
176	public static function classify_crawler(string $user_agent): ?string {
177	if ('' === $user_agent) {
178	return null;
179	}
180
181	foreach (self::CRAWLER_AGENTS as $fragment => $slug) {
182	if (false !== stripos($user_agent, $fragment)) {
183	return $slug;
184	}
185	}
186
187	return null;
188	}
189
190	/**
191	* Record the current front-end request into the daily aggregates.
192	*
193	* @return void
194	*/
195	public function record(): void {
196	if (is_admin() \|\| wp_doing_ajax() \|\| wp_doing_cron()) {
197	return;
198	}
199	if (is_feed() \|\| is_preview() \|\| is_robots() \|\| is_404()) {
200	return;
201	}
202	$method = isset($_SERVER['REQUEST_METHOD']) ? strtoupper((string) wp_unslash($_SERVER['REQUEST_METHOD'])) : 'GET'; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput
203	if ('GET' !== $method) {
204	return;
205	}
206
207	$user_agent = isset($_SERVER['HTTP_USER_AGENT']) ? (string) wp_unslash($_SERVER['HTTP_USER_AGENT']) : ''; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- classified, never stored raw.
208
209	// AI crawler: count it and stop — a bot is not part of the human
210	// baseline and has no meaningful referrer.
211	$bot = self::classify_crawler($user_agent);
212	if (null !== $bot) {
213	$this->bump('crawler', $bot);
214	return;
215	}
216
217	// Editors/admins browsing their own site would skew small sites.
218	if (is_user_logged_in() && current_user_can('edit_posts')) {
219	return;
220	}
221
222	$this->bump('baseline', 'all');
223
224	$referrer = isset($_SERVER['HTTP_REFERER']) ? (string) wp_unslash($_SERVER['HTTP_REFERER']) : ''; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- classified, never stored raw.
225	$platform = self::classify_referrer($referrer);
226	if (null !== $platform) {
227	$this->bump('referral', $platform, $this->current_path());
228	}
229	}
230
231	/**
232	* Record a page served as Markdown to an AI agent.
233	*
234	* Called by Pro's Markdown for AI feature at serve time. Lives here rather
235	* than in Pro because this class owns the aggregate table; Pro owning a
236	* second writer to it would couple the schema to two repos.
237	*
238	* @param string $source Crawler slug when the agent is a known AI crawler,
239	* 'header' for Accept-negotiated requests, 'link' for
240	* ?format=markdown / .md URLs.
241	* @param string $path Path of the post served.
242	* @return void
243	*/
244	public function record_served_markdown(string $source, string $path = ''): void {
245	$source = sanitize_key($source);
246	if ('' === $source) {
247	$source = 'other';
248	}
249	$this->bump('markdown', $source, substr($path, 0, 191));
250	}
251
252	/**
253	* Total Markdown-for-AI responses served in the last N days.
254	*
255	* @param int $days Range in days (bounded 1–180).
256	* @return int
257	*/
258	public function served_markdown_count(int $days = 30): int {
259	global $wpdb;
260
261	$days = max(1, min(self::RETENTION_DAYS, $days));
262	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
263	// Same clock as write_bucket(), and counted in calendar days so a
264	// DST transition inside the window does not move the boundary.
265	$since = $this->day_key_offset($days);
266
267	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- read-only aggregate over our own table.
268	return (int) $wpdb->get_var(
269	$wpdb->prepare(
270	"SELECT COALESCE(SUM(hits), 0) FROM {$table} WHERE kind = 'markdown' AND day >= %s",
271	$since
272	)
273	);
274	// phpcs:enable
275	}
276
277	/**
278	* The current request path, normalized for the aggregate key.
279	*
280	* @return string
281	*/
282	private function current_path(): string {
283	$uri = isset($_SERVER['REQUEST_URI']) ? (string) wp_unslash($_SERVER['REQUEST_URI']) : '/'; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- reduced to its path component below.
284	$path = (string) wp_parse_url($uri, PHP_URL_PATH);
285	if ('' === $path) {
286	$path = '/';
287	}
288	return substr($path, 0, 191);
289	}
290
291	/**
292	* Increment one daily aggregate bucket.
293	*
294	* @param string $kind 'referral' \| 'crawler' \| 'baseline'.
295	* @param string $source Platform/bot slug, or 'all' for baseline.
296	* @param string $path Landing path (referrals only).
297	* @return void
298	*/
299	private function bump(string $kind, string $source, string $path = ''): void {
300	// Without a persistent object cache there is nowhere to buffer, so keep
301	// the direct write rather than counting into per-request memory that is
302	// thrown away — that would lose hits outright.
303	if (!wp_using_ext_object_cache()) {
304	$this->write_bucket($kind, $source, $path, 1);
305
306	return;
307	}
308
309	// With one, buffer and flush in batches. The unique key is
310	// (day, kind, source, path), so all baseline traffic funnels into a
311	// single row per day: InnoDB took an exclusive row lock on it for every
312	// visitor, serialising concurrent anonymous traffic, and made every
313	// pageview a write even when the response was fully cacheable (#402).
314	$bucket = self::COUNTER_PREFIX . md5($kind . '\|' . $source . '\|' . $path);
315	$since = $bucket . '_since';
316
317	$hits = wp_cache_incr($bucket, 1, self::COUNTER_GROUP);
318
319	if (false === $hits) {
320	wp_cache_add($bucket, 1, self::COUNTER_GROUP, 0);
321	wp_cache_add($since, time(), self::COUNTER_GROUP, 0);
322	$hits = 1;
323	}
324
325	$started = (int) wp_cache_get($since, self::COUNTER_GROUP);
326
327	// Flush on either bound, so a busy site writes rarely and a quiet one
328	// still lands its hits — an eviction can cost at most one window.
329	if ($hits < self::FLUSH_AT && $started > 0 && (time() - $started) < self::FLUSH_AFTER) {
330	return;
331	}
332
333	wp_cache_set($bucket, 0, self::COUNTER_GROUP, 0);
334	wp_cache_set($since, time(), self::COUNTER_GROUP, 0);
335
336	$this->write_bucket($kind, $source, $path, (int) $hits);
337	}
338
339	/**
340	* Add hits to a bucket's row.
341	*
342	* @since 2.0.1
343	*
344	* @param string $kind 'referral' \| 'crawler' \| 'baseline'.
345	* @param string $source Platform/bot slug, or 'all' for baseline.
346	* @param string $path Landing path (referrals only).
347	* @param int $hits How many hits to add.
348	* @return void
349	*/
350	private function write_bucket(string $kind, string $source, string $path, int $hits): void {
351	if ($hits < 1) {
352	return;
353	}
354
355	// `day` is the SITE-LOCAL date (see day_key()), not UTC. The column is
356	// a bare `date` with no zone attached, so the clock that writes it is
357	// the only thing that gives it meaning — and these keys reach the user
358	// as the trend chart's dates, where the site's own calendar is what
359	// they expect to read.
360	//
361	// Every range boundary and retention cutoff must be derived with
362	// day_key() too. A gmdate() boundary against these rows drifts by a
363	// day for part of every day on a non-UTC site.
364
365	global $wpdb;
366
367	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
368
369	// Aggregate counter upsert; the unique key is the bucket.
370	// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- aggregate counter upsert; table name is prefix-derived.
371	$wpdb->query(
372	// phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- table name is $wpdb->prefix plus a literal, and every value is passed as a placeholder replacement.
373	$wpdb->prepare(
374	"INSERT INTO {$table} (day, kind, source, path, hits) VALUES (%s, %s, %s, %s, %d)
375	ON DUPLICATE KEY UPDATE hits = hits + %d",
376	current_time('Y-m-d'),
377	$kind,
378	$source,
379	$path,
380	$hits,
381	$hits
382	)
383	);
384	// phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared
385	}
386
387	/**
388	* The site-local date key for an instant, matching write_bucket().
389	*
390	* Every consumer of the `day` column goes through this, so the read side
391	* cannot drift onto a different calendar from the write side.
392	*
393	* @param int\|null $timestamp Unix timestamp, or null for now.
394	* @return string `Y-m-d` on the site's clock.
395	*/
396	private function day_key(?int $timestamp = null): string {
397	return wp_date('Y-m-d', $timestamp ?? time());
398	}
399
400	/**
401	* Midday on a given site-local date.
402	*
403	* Midday, not midnight: a handful of zones start DST at 00:00, so
404	* midnight on a transition date can be a time that does not exist and
405	* PHP quietly rolls it forward. Noon is never inside a DST gap, so
406	* every date in the year is representable.
407	*
408	* @param string $day `Y-m-d` on the site's clock.
409	* @return DateTimeImmutable
410	*/
411	private function local_noon(string $day): DateTimeImmutable {
412	return new DateTimeImmutable($day . ' 12:00:00', wp_timezone());
413	}
414
415	/**
416	* The site-local date key N calendar days before today.
417	*
418	* Not `time() - N * DAY_IN_SECONDS`: a fixed 86400-second step is not a
419	* day on a clock that shifts. Around a DST transition that arithmetic
420	* lands an hour early or late, which moves the date for the hour either
421	* side of midnight.
422	*
423	* @param int $days_ago Whole days back.
424	* @return string `Y-m-d`.
425	*/
426	private function day_key_offset(int $days_ago): string {
427	return $this->local_noon($this->day_key())
428	->modify('-' . max(0, $days_ago) . ' day')
429	->format('Y-m-d');
430	}
431
432	/**
433	* Every site-local date from $from to $to inclusive.
434	*
435	* Walks the calendar rather than stepping by 86400 seconds, so a DST
436	* transition inside the range neither duplicates a date nor skips one.
437	* Skipping one used to drop that day's referrals out of the trend while
438	* they stayed in the totals.
439	*
440	* @param string $from `Y-m-d`, inclusive.
441	* @param string $to `Y-m-d`, inclusive.
442	* @return string[] Ordered, contiguous date keys.
443	*/
444	private function day_range(string $from, string $to): array {
445	$cursor = $this->local_noon($from);
446	$end = $this->local_noon($to);
447
448	$days = [];
449	// Bounded by the caller's window (<= RETENTION_DAYS), with headroom
450	// so a malformed pair can never spin here.
451	$guard = self::RETENTION_DAYS + 2;
452	$steps = 0;
453	while ($cursor <= $end && $steps < $guard) {
454	$days[] = $cursor->format('Y-m-d');
455	$cursor = $cursor->modify('+1 day');
456	$steps++;
457	}
458
459	return $days;
460	}
461
462	/**
463	* Dashboard summary for the last N days.
464	*
465	* @param int $days Range in days (bounded 1–180).
466	* @return array<string, mixed>
467	*/
468	public function summary(int $days = 30): array {
469	global $wpdb;
470
471	$days = max(1, min(self::RETENTION_DAYS, $days));
472	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
473	// Same clock as write_bucket(), and counted in calendar days so a
474	// DST transition inside the window does not move the boundary.
475	$since = $this->day_key_offset($days);
476
477	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- read-only aggregates over our own table.
478	$rows = $wpdb->get_results(
479	$wpdb->prepare(
480	"SELECT day, kind, source, path, hits FROM {$table} WHERE day >= %s",
481	$since
482	),
483	ARRAY_A
484	);
485	// phpcs:enable
486
487	$baseline = 0;
488	$referrals = 0;
489	$platforms = [];
490	$trend = [];
491	$pages = [];
492	$crawlers = [];
493	$markdown = 0;
494
495	foreach ((array) $rows as $row) {
496	$hits = (int) $row['hits'];
497	switch ($row['kind']) {
498	case 'baseline':
499	$baseline += $hits;
500	break;
501	case 'referral':
502	$referrals += $hits;
503	$platforms[$row['source']] = ($platforms[$row['source']] ?? 0) + $hits;
504	$trend[$row['day']] = ($trend[$row['day']] ?? 0) + $hits;
505	if ('' !== $row['path']) {
506	$pages[$row['path']] = ($pages[$row['path']] ?? 0) + $hits;
507	}
508	break;
509	case 'crawler':
510	$crawlers[$row['source']] = ($crawlers[$row['source']] ?? 0) + $hits;
511	break;
512	case 'markdown':
513	$markdown += $hits;
514	break;
515	}
516	}
517
518	arsort($platforms);
519	arsort($pages);
520	arsort($crawlers);
521	ksort($trend);
522
523	// Fill every day the query covered, zeroes included. Only days that
524	// had a referral produce a $trend key above, and the chart positions
525	// points by index — so a sparse map drew a three-week gap exactly
526	// like a one-day gap. A contiguous series makes even spacing correct,
527	// and distinguishes "no referrals that day" from "no data".
528	//
529	// The range mirrors the WHERE clause (day >= $since, through today)
530	// so the series covers exactly what was counted, and it is built on
531	// day_key() so the keys match how the rows were written.
532	$filled = [];
533	foreach ($this->day_range($since, $this->day_key()) as $day) {
534	$filled[$day] = $trend[$day] ?? 0;
535	}
536
537	// Safety net for anything the window did not cover — a row dated
538	// ahead of today, which a site that moved timezone can hold. Union
539	// keeps the filled zeroes and adds only keys not already present, so
540	// the series can never total less than ai_sessions.
541	$filled += $trend;
542	ksort($filled);
543
544	$trend = $filled;
545
546	return [
547	'days' => $days,
548	'baseline' => $baseline,
549	'ai_sessions' => $referrals,
550	'ai_share' => $baseline > 0 ? round($referrals / $baseline * 100, 1) : 0.0,
551	'platforms' => $platforms,
552	'trend' => $trend,
553	'top_pages' => array_slice($pages, 0, 10, true),
554	'crawlers' => $crawlers,
555	// Whether llms.txt is being served, so the crawler panel can pair
556	// "bots are coming" with "and here's what we feed them".
557	//
558	// Ask the manager, not the filesystem: `dynamic` delivery — the
559	// resolved default on every non-Apache stack — publishes no
560	// physical file and answers from serve_llms_txt(), so a
561	// file_exists() probe reports "not published" for a live document.
562	'llms_txt' => $this->llms_txt_published(),
563	// Pages served as Markdown by Pro's Markdown for AI feature
564	// (kind 'markdown', written via record_served_markdown()).
565	'markdown_served' => $markdown,
566	];
567	}
568
569	/**
570	* Whether llms.txt is currently being served, in either delivery mode.
571	*
572	* `static` publishes a file at ABSPATH; `dynamic` keeps the document in
573	* an option and serves it from a PHP route. Only the manager knows which
574	* is in force, so it is the single source of truth here.
575	*
576	* @return bool
577	*/
578	private function llms_txt_published(): bool {
579	// Spelt exactly as the class is declared. The autoloader routes this
580	// one through a case-SENSITIVE special-case map, and while a
581	// mis-cased name happens to fall through to the generic rule and
582	// resolve anyway, that is a coincidence — a change to that rule would
583	// silently make class_exists() false here, and the badge would go
584	// back to reporting "No llms.txt" for a live document.
585	if (!class_exists(LLMs_Txt_Manager::class)) {
586	// Defensive: a partial load must not claim llms.txt is live.
587	return false;
588	}
589
590	// is_published(), not get_llms_txt_status(): the latter resolves the
591	// delivery mode, may fire a loopback probe and touches the filesystem
592	// API, which is far too much work for a dashboard boolean.
593	return (new LLMs_Txt_Manager())->is_published();
594	}
595
596	/**
597	* Drop aggregate rows past the retention window.
598	*
599	* @return void
600	*/
601	public function prune(): void {
602	global $wpdb;
603
604	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
605	// Same clock as write_bucket(), counted in calendar days.
606	$cutoff = $this->day_key_offset(self::RETENTION_DAYS);
607
608	// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- retention delete on our own table.
609	$wpdb->query($wpdb->prepare("DELETE FROM {$table} WHERE day < %s", $cutoff));
610	}
611	}
612