thinkrank / includes / seo / class-ai-traffic-tracker.php

class-ai-traffic-tracker.php in ThinkRank AI SEO – AI SEO Plugin for WordPress: Schema, XML Sitemaps, Meta Tags, Search Console & Local SEO 2.7.0, at includes/seo/class-ai-traffic-tracker.php

591 lines 22.4 KB

Raw Download Zip

1	<?php
2	/**
3	* AI referral traffic + AI crawler tracking.
4	*
5	* AI platforms (ChatGPT, Perplexity, Gemini, Claude, Copilot…) send real human
6	* visitors, but analytics tools misattribute much of it: the platforms strip
7	* or rewrite referrers, so GA4 files a large share under "Direct". WordPress
8	* serves its own pages, so — unlike a hosted storefront — the plugin IS in the
9	* request path and can read the referrer first-party, with no pixel and no
10	* JavaScript.
11	*
12	* What is stored (and deliberately nothing more): daily aggregate counters,
13	* one row per (day, kind, source, path). Three kinds:
14	*
15	* referral — a human pageview whose referrer host matched an AI platform
16	* crawler — a request whose user agent matched a known AI crawler
17	* baseline — every human pageview (source 'all', no path), so the dashboard
18	* can say "AI referrals are N% of traffic" without Google
19	*
20	* No IPs, no raw user agents, no cookies, no per-visit rows — nothing that
21	* identifies a visitor. That keeps the table small and the feature clean
22	* under wordpress.org privacy expectations.
23	*
24	* @package ThinkRank\SEO
25	* @since 1.27.0
26	*/
27
28	declare(strict_types=1);
29
30	namespace ThinkRank\SEO;
31
32	use DateTimeImmutable;
33
34	if (!defined('ABSPATH')) {
35	exit;
36	}
37
38	/**
39	* Records AI referrals/crawlers and serves the dashboard summary.
40	*/
41	class Ai_Traffic_Tracker {
42
43	/**
44	* Cron hook for pruning old aggregate rows.
45	*/
46	private const PRUNE_HOOK = 'thinkrank_ai_traffic_prune';
47
48	/**
49	* Object-cache group for the buffered hit counters.
50	*/
51	private const COUNTER_GROUP = 'thinkrank_traffic';
52
53	/**
54	* Key prefix for those counters.
55	*/
56	private const COUNTER_PREFIX = 'tr_traffic_';
57
58	/**
59	* Flush a bucket once it has this many buffered hits.
60	*/
61	private const FLUSH_AT = 50;
62
63	/**
64	* ...or once its oldest buffered hit is this many seconds old, so a quiet
65	* site still records its traffic.
66	*/
67	private const FLUSH_AFTER = 300;
68
69	/**
70	* Days of history to keep. The dashboard reads 30; keep 6 months so a
71	* longer range is possible later without changing collection.
72	*/
73	private const RETENTION_DAYS = 180;
74
75	/**
76	* Referrer host fragments → platform slug. Checked with substring match
77	* against the referrer host, so subdomains are covered.
78	*
79	* @var array<string, string>
80	*/
81	private const REFERRER_PLATFORMS = [
82	'chatgpt.com' => 'chatgpt',
83	'chat.openai.com' => 'chatgpt',
84	'perplexity.ai' => 'perplexity',
85	'pplx.ai' => 'perplexity',
86	'gemini.google.com' => 'gemini',
87	'bard.google.com' => 'gemini',
88	'claude.ai' => 'claude',
89	'copilot.microsoft.com' => 'copilot',
90	'meta.ai' => 'meta-ai',
91	'you.com' => 'you',
92	'poe.com' => 'poe',
93	'grok.com' => 'grok',
94	'x.ai' => 'grok',
95	'chat.mistral.ai' => 'mistral',
96	'chat.deepseek.com' => 'deepseek',
97	'kimi.com' => 'kimi',
98	];
99
100	/**
101	* Wire the front-end recorder and the retention cron.
102	*
103	* @return void
104	*/
105	public function init(): void {
106	// Priority 1: record before any template logic can redirect/exit.
107	add_action('template_redirect', [$this, 'record'], 1);
108
109	add_action(self::PRUNE_HOOK, [$this, 'prune']);
110	if (!wp_next_scheduled(self::PRUNE_HOOK)) {
111	wp_schedule_event(time() + DAY_IN_SECONDS, 'daily', self::PRUNE_HOOK);
112	}
113	}
114
115	/**
116	* Classify a referrer URL as an AI platform.
117	*
118	* @param string $referrer Full referrer URL (may be empty).
119	* @return string\|null Platform slug, or null when not an AI platform.
120	*/
121	public static function classify_referrer(string $referrer): ?string {
122	if ('' === $referrer) {
123	return null;
124	}
125
126	$host = strtolower((string) wp_parse_url($referrer, PHP_URL_HOST));
127	if ('' === $host) {
128	return null;
129	}
130
131	foreach (self::REFERRER_PLATFORMS as $fragment => $slug) {
132	// Suffix match on the host so evil.com/?q=claude.ai can't spoof
133	// via path, and subdomains (www.perplexity.ai) still match.
134	if ($host === $fragment \|\| str_ends_with($host, '.' . $fragment)) {
135	return $slug;
136	}
137	}
138
139	return null;
140	}
141
142	/**
143	* Classify a user agent as an AI crawler.
144	*
145	* The agent list is `AI_Crawlers`, shared with the robots.txt panel — the
146	* two must agree about which bots exist, or a site blocks a crawler it is
147	* not counting (#657).
148	*
149	* @param string $user_agent Raw user agent (may be empty).
150	* @return string\|null Crawler slug, or null when not a known AI crawler.
151	*/
152	public static function classify_crawler(string $user_agent): ?string {
153	if ('' === $user_agent) {
154	return null;
155	}
156
157	// Token order is significant and owned by the registry: the first
158	// token found wins, so `Claude-SearchBot` has to be tested before
159	// `ClaudeBot`. See AI_Crawlers::AGENTS.
160	foreach (AI_Crawlers::token_map() as $fragment => $slug) {
161	if (false !== stripos($user_agent, $fragment)) {
162	return $slug;
163	}
164	}
165
166	return null;
167	}
168
169	/**
170	* Record the current front-end request into the daily aggregates.
171	*
172	* @return void
173	*/
174	public function record(): void {
175	if (is_admin() \|\| wp_doing_ajax() \|\| wp_doing_cron()) {
176	return;
177	}
178	if (is_feed() \|\| is_preview() \|\| is_robots() \|\| is_404()) {
179	return;
180	}
181	$method = isset($_SERVER['REQUEST_METHOD']) ? strtoupper((string) wp_unslash($_SERVER['REQUEST_METHOD'])) : 'GET'; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput
182	if ('GET' !== $method) {
183	return;
184	}
185
186	$user_agent = isset($_SERVER['HTTP_USER_AGENT']) ? (string) wp_unslash($_SERVER['HTTP_USER_AGENT']) : ''; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- classified, never stored raw.
187
188	// AI crawler: count it and stop — a bot is not part of the human
189	// baseline and has no meaningful referrer.
190	$bot = self::classify_crawler($user_agent);
191	if (null !== $bot) {
192	$this->bump('crawler', $bot);
193	return;
194	}
195
196	// Editors/admins browsing their own site would skew small sites.
197	if (is_user_logged_in() && current_user_can('edit_posts')) {
198	return;
199	}
200
201	$this->bump('baseline', 'all');
202
203	$referrer = isset($_SERVER['HTTP_REFERER']) ? (string) wp_unslash($_SERVER['HTTP_REFERER']) : ''; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- classified, never stored raw.
204	$platform = self::classify_referrer($referrer);
205	if (null !== $platform) {
206	$this->bump('referral', $platform, $this->current_path());
207	}
208	}
209
210	/**
211	* Record a page served as Markdown to an AI agent.
212	*
213	* Called by Pro's Markdown for AI feature at serve time. Lives here rather
214	* than in Pro because this class owns the aggregate table; Pro owning a
215	* second writer to it would couple the schema to two repos.
216	*
217	* @param string $source Crawler slug when the agent is a known AI crawler,
218	* 'header' for Accept-negotiated requests, 'link' for
219	* ?format=markdown / .md URLs.
220	* @param string $path Path of the post served.
221	* @return void
222	*/
223	public function record_served_markdown(string $source, string $path = ''): void {
224	$source = sanitize_key($source);
225	if ('' === $source) {
226	$source = 'other';
227	}
228	$this->bump('markdown', $source, substr($path, 0, 191));
229	}
230
231	/**
232	* Total Markdown-for-AI responses served in the last N days.
233	*
234	* @param int $days Range in days (bounded 1–180).
235	* @return int
236	*/
237	public function served_markdown_count(int $days = 30): int {
238	global $wpdb;
239
240	$days = max(1, min(self::RETENTION_DAYS, $days));
241	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
242	// Same clock as write_bucket(), and counted in calendar days so a
243	// DST transition inside the window does not move the boundary.
244	$since = $this->day_key_offset($days);
245
246	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- read-only aggregate over our own table.
247	return (int) $wpdb->get_var(
248	$wpdb->prepare(
249	"SELECT COALESCE(SUM(hits), 0) FROM {$table} WHERE kind = 'markdown' AND day >= %s",
250	$since
251	)
252	);
253	// phpcs:enable
254	}
255
256	/**
257	* The current request path, normalized for the aggregate key.
258	*
259	* @return string
260	*/
261	private function current_path(): string {
262	$uri = isset($_SERVER['REQUEST_URI']) ? (string) wp_unslash($_SERVER['REQUEST_URI']) : '/'; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- reduced to its path component below.
263	$path = (string) wp_parse_url($uri, PHP_URL_PATH);
264	if ('' === $path) {
265	$path = '/';
266	}
267	return substr($path, 0, 191);
268	}
269
270	/**
271	* Increment one daily aggregate bucket.
272	*
273	* @param string $kind 'referral' \| 'crawler' \| 'baseline'.
274	* @param string $source Platform/bot slug, or 'all' for baseline.
275	* @param string $path Landing path (referrals only).
276	* @return void
277	*/
278	private function bump(string $kind, string $source, string $path = ''): void {
279	// Without a persistent object cache there is nowhere to buffer, so keep
280	// the direct write rather than counting into per-request memory that is
281	// thrown away — that would lose hits outright.
282	if (!wp_using_ext_object_cache()) {
283	$this->write_bucket($kind, $source, $path, 1);
284
285	return;
286	}
287
288	// With one, buffer and flush in batches. The unique key is
289	// (day, kind, source, path), so all baseline traffic funnels into a
290	// single row per day: InnoDB took an exclusive row lock on it for every
291	// visitor, serialising concurrent anonymous traffic, and made every
292	// pageview a write even when the response was fully cacheable (#402).
293	$bucket = self::COUNTER_PREFIX . md5($kind . '\|' . $source . '\|' . $path);
294	$since = $bucket . '_since';
295
296	$hits = wp_cache_incr($bucket, 1, self::COUNTER_GROUP);
297
298	if (false === $hits) {
299	wp_cache_add($bucket, 1, self::COUNTER_GROUP, 0);
300	wp_cache_add($since, time(), self::COUNTER_GROUP, 0);
301	$hits = 1;
302	}
303
304	$started = (int) wp_cache_get($since, self::COUNTER_GROUP);
305
306	// Flush on either bound, so a busy site writes rarely and a quiet one
307	// still lands its hits — an eviction can cost at most one window.
308	if ($hits < self::FLUSH_AT && $started > 0 && (time() - $started) < self::FLUSH_AFTER) {
309	return;
310	}
311
312	wp_cache_set($bucket, 0, self::COUNTER_GROUP, 0);
313	wp_cache_set($since, time(), self::COUNTER_GROUP, 0);
314
315	$this->write_bucket($kind, $source, $path, (int) $hits);
316	}
317
318	/**
319	* Add hits to a bucket's row.
320	*
321	* @since 2.0.1
322	*
323	* @param string $kind 'referral' \| 'crawler' \| 'baseline'.
324	* @param string $source Platform/bot slug, or 'all' for baseline.
325	* @param string $path Landing path (referrals only).
326	* @param int $hits How many hits to add.
327	* @return void
328	*/
329	private function write_bucket(string $kind, string $source, string $path, int $hits): void {
330	if ($hits < 1) {
331	return;
332	}
333
334	// `day` is the SITE-LOCAL date (see day_key()), not UTC. The column is
335	// a bare `date` with no zone attached, so the clock that writes it is
336	// the only thing that gives it meaning — and these keys reach the user
337	// as the trend chart's dates, where the site's own calendar is what
338	// they expect to read.
339	//
340	// Every range boundary and retention cutoff must be derived with
341	// day_key() too. A gmdate() boundary against these rows drifts by a
342	// day for part of every day on a non-UTC site.
343
344	global $wpdb;
345
346	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
347
348	// Aggregate counter upsert; the unique key is the bucket.
349	// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- aggregate counter upsert; table name is prefix-derived.
350	$wpdb->query(
351	// phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- table name is $wpdb->prefix plus a literal, and every value is passed as a placeholder replacement.
352	$wpdb->prepare(
353	"INSERT INTO {$table} (day, kind, source, path, hits) VALUES (%s, %s, %s, %s, %d)
354	ON DUPLICATE KEY UPDATE hits = hits + %d",
355	current_time('Y-m-d'),
356	$kind,
357	$source,
358	$path,
359	$hits,
360	$hits
361	)
362	);
363	// phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared
364	}
365
366	/**
367	* The site-local date key for an instant, matching write_bucket().
368	*
369	* Every consumer of the `day` column goes through this, so the read side
370	* cannot drift onto a different calendar from the write side.
371	*
372	* @param int\|null $timestamp Unix timestamp, or null for now.
373	* @return string `Y-m-d` on the site's clock.
374	*/
375	private function day_key(?int $timestamp = null): string {
376	return wp_date('Y-m-d', $timestamp ?? time());
377	}
378
379	/**
380	* Midday on a given site-local date.
381	*
382	* Midday, not midnight: a handful of zones start DST at 00:00, so
383	* midnight on a transition date can be a time that does not exist and
384	* PHP quietly rolls it forward. Noon is never inside a DST gap, so
385	* every date in the year is representable.
386	*
387	* @param string $day `Y-m-d` on the site's clock.
388	* @return DateTimeImmutable
389	*/
390	private function local_noon(string $day): DateTimeImmutable {
391	return new DateTimeImmutable($day . ' 12:00:00', wp_timezone());
392	}
393
394	/**
395	* The site-local date key N calendar days before today.
396	*
397	* Not `time() - N * DAY_IN_SECONDS`: a fixed 86400-second step is not a
398	* day on a clock that shifts. Around a DST transition that arithmetic
399	* lands an hour early or late, which moves the date for the hour either
400	* side of midnight.
401	*
402	* @param int $days_ago Whole days back.
403	* @return string `Y-m-d`.
404	*/
405	private function day_key_offset(int $days_ago): string {
406	return $this->local_noon($this->day_key())
407	->modify('-' . max(0, $days_ago) . ' day')
408	->format('Y-m-d');
409	}
410
411	/**
412	* Every site-local date from $from to $to inclusive.
413	*
414	* Walks the calendar rather than stepping by 86400 seconds, so a DST
415	* transition inside the range neither duplicates a date nor skips one.
416	* Skipping one used to drop that day's referrals out of the trend while
417	* they stayed in the totals.
418	*
419	* @param string $from `Y-m-d`, inclusive.
420	* @param string $to `Y-m-d`, inclusive.
421	* @return string[] Ordered, contiguous date keys.
422	*/
423	private function day_range(string $from, string $to): array {
424	$cursor = $this->local_noon($from);
425	$end = $this->local_noon($to);
426
427	$days = [];
428	// Bounded by the caller's window (<= RETENTION_DAYS), with headroom
429	// so a malformed pair can never spin here.
430	$guard = self::RETENTION_DAYS + 2;
431	$steps = 0;
432	while ($cursor <= $end && $steps < $guard) {
433	$days[] = $cursor->format('Y-m-d');
434	$cursor = $cursor->modify('+1 day');
435	$steps++;
436	}
437
438	return $days;
439	}
440
441	/**
442	* Dashboard summary for the last N days.
443	*
444	* @param int $days Range in days (bounded 1–180).
445	* @return array<string, mixed>
446	*/
447	public function summary(int $days = 30): array {
448	global $wpdb;
449
450	$days = max(1, min(self::RETENTION_DAYS, $days));
451	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
452	// Same clock as write_bucket(), and counted in calendar days so a
453	// DST transition inside the window does not move the boundary.
454	$since = $this->day_key_offset($days);
455
456	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- read-only aggregates over our own table.
457	$rows = $wpdb->get_results(
458	$wpdb->prepare(
459	"SELECT day, kind, source, path, hits FROM {$table} WHERE day >= %s",
460	$since
461	),
462	ARRAY_A
463	);
464	// phpcs:enable
465
466	$baseline = 0;
467	$referrals = 0;
468	$platforms = [];
469	$trend = [];
470	$pages = [];
471	$crawlers = [];
472	$markdown = 0;
473
474	foreach ((array) $rows as $row) {
475	$hits = (int) $row['hits'];
476	switch ($row['kind']) {
477	case 'baseline':
478	$baseline += $hits;
479	break;
480	case 'referral':
481	$referrals += $hits;
482	$platforms[$row['source']] = ($platforms[$row['source']] ?? 0) + $hits;
483	$trend[$row['day']] = ($trend[$row['day']] ?? 0) + $hits;
484	if ('' !== $row['path']) {
485	$pages[$row['path']] = ($pages[$row['path']] ?? 0) + $hits;
486	}
487	break;
488	case 'crawler':
489	$crawlers[$row['source']] = ($crawlers[$row['source']] ?? 0) + $hits;
490	break;
491	case 'markdown':
492	$markdown += $hits;
493	break;
494	}
495	}
496
497	arsort($platforms);
498	arsort($pages);
499	arsort($crawlers);
500	ksort($trend);
501
502	// Fill every day the query covered, zeroes included. Only days that
503	// had a referral produce a $trend key above, and the chart positions
504	// points by index — so a sparse map drew a three-week gap exactly
505	// like a one-day gap. A contiguous series makes even spacing correct,
506	// and distinguishes "no referrals that day" from "no data".
507	//
508	// The range mirrors the WHERE clause (day >= $since, through today)
509	// so the series covers exactly what was counted, and it is built on
510	// day_key() so the keys match how the rows were written.
511	$filled = [];
512	foreach ($this->day_range($since, $this->day_key()) as $day) {
513	$filled[$day] = $trend[$day] ?? 0;
514	}
515
516	// Safety net for anything the window did not cover — a row dated
517	// ahead of today, which a site that moved timezone can hold. Union
518	// keeps the filled zeroes and adds only keys not already present, so
519	// the series can never total less than ai_sessions.
520	$filled += $trend;
521	ksort($filled);
522
523	$trend = $filled;
524
525	return [
526	'days' => $days,
527	'baseline' => $baseline,
528	'ai_sessions' => $referrals,
529	'ai_share' => $baseline > 0 ? round($referrals / $baseline * 100, 1) : 0.0,
530	'platforms' => $platforms,
531	'trend' => $trend,
532	'top_pages' => array_slice($pages, 0, 10, true),
533	'crawlers' => $crawlers,
534	// Whether llms.txt is being served, so the crawler panel can pair
535	// "bots are coming" with "and here's what we feed them".
536	//
537	// Ask the manager, not the filesystem: `dynamic` delivery — the
538	// resolved default on every non-Apache stack — publishes no
539	// physical file and answers from serve_llms_txt(), so a
540	// file_exists() probe reports "not published" for a live document.
541	'llms_txt' => $this->llms_txt_published(),
542	// Pages served as Markdown by Pro's Markdown for AI feature
543	// (kind 'markdown', written via record_served_markdown()).
544	'markdown_served' => $markdown,
545	];
546	}
547
548	/**
549	* Whether llms.txt is currently being served, in either delivery mode.
550	*
551	* `static` publishes a file at ABSPATH; `dynamic` keeps the document in
552	* an option and serves it from a PHP route. Only the manager knows which
553	* is in force, so it is the single source of truth here.
554	*
555	* @return bool
556	*/
557	private function llms_txt_published(): bool {
558	// Spelt exactly as the class is declared. The autoloader routes this
559	// one through a case-SENSITIVE special-case map, and while a
560	// mis-cased name happens to fall through to the generic rule and
561	// resolve anyway, that is a coincidence — a change to that rule would
562	// silently make class_exists() false here, and the badge would go
563	// back to reporting "No llms.txt" for a live document.
564	if (!class_exists(LLMs_Txt_Manager::class)) {
565	// Defensive: a partial load must not claim llms.txt is live.
566	return false;
567	}
568
569	// is_published(), not get_llms_txt_status(): the latter resolves the
570	// delivery mode, may fire a loopback probe and touches the filesystem
571	// API, which is far too much work for a dashboard boolean.
572	return (new LLMs_Txt_Manager())->is_published();
573	}
574
575	/**
576	* Drop aggregate rows past the retention window.
577	*
578	* @return void
579	*/
580	public function prune(): void {
581	global $wpdb;
582
583	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
584	// Same clock as write_bucket(), counted in calendar days.
585	$cutoff = $this->day_key_offset(self::RETENTION_DAYS);
586
587	// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- retention delete on our own table.
588	$wpdb->query($wpdb->prepare("DELETE FROM {$table} WHERE day < %s", $cutoff));
589	}
590	}
591