← All changes | includes/seo/class-ai-traffic-tracker.php +291 -36 1.29.0 → 2.10.0 View file →

		@@ -28,8 +28,10 @@
28	28	declare(strict_types=1);
29	29
30	30	namespace ThinkRank\SEO;
31	31
	32	+use DateTimeImmutable;
	33	+
32	34	if (!defined('ABSPATH')) {
33	35	exit;
34	36	}
35	37
		@@ -43,8 +45,29 @@
43	45	*/
44	46	private const PRUNE_HOOK = 'thinkrank_ai_traffic_prune';
45	47
46	48	/**
	49	+ * Object-cache group for the buffered hit counters.
	50	+ */
	51	+ private const COUNTER_GROUP = 'thinkrank_traffic';
	52	+
	53	+ /**
	54	+ * Key prefix for those counters.
	55	+ */
	56	+ private const COUNTER_PREFIX = 'tr_traffic_';
	57	+
	58	+ /**
	59	+ * Flush a bucket once it has this many buffered hits.
	60	+ */
	61	+ private const FLUSH_AT = 50;
	62	+
	63	+ /**
	64	+ * ...or once its oldest buffered hit is this many seconds old, so a quiet
	65	+ * site still records its traffic.
	66	+ */
	67	+ private const FLUSH_AFTER = 300;
	68	+
	69	+ /**
47	70	* Days of history to keep. The dashboard reads 30; keep 6 months so a
48	71	* longer range is possible later without changing collection.
49	72	*/
50	73	private const RETENTION_DAYS = 180;
		@@ -74,36 +97,8 @@
74	97	'kimi.com' => 'kimi',
75	98	];
76	99
77	100	/**
78		- * User-agent fragments → AI crawler slug. Case-insensitive substring
79		- * match. Order matters where one token contains another — more specific
80		- * entries first.
81		- *
82		- * @var array<string, string>
83		- */
84		- private const CRAWLER_AGENTS = [
85		- 'OAI-SearchBot' => 'oai-searchbot',
86		- 'ChatGPT-User' => 'chatgpt-user',
87		- 'GPTBot' => 'gptbot',
88		- 'Perplexity-User' => 'perplexity-user',
89		- 'PerplexityBot' => 'perplexitybot',
90		- 'Claude-SearchBot' => 'claude-searchbot',
91		- 'Claude-User' => 'claude-user',
92		- 'ClaudeBot' => 'claudebot',
93		- 'anthropic-ai' => 'anthropic-ai',
94		- 'Google-Extended' => 'google-extended',
95		- 'Applebot-Extended' => 'applebot-extended',
96		- 'meta-externalagent' => 'meta-externalagent',
97		- 'meta-externalfetcher' => 'meta-externalfetcher',
98		- 'Bytespider' => 'bytespider',
99		- 'Amazonbot' => 'amazonbot',
100		- 'CCBot' => 'ccbot',
101		- 'cohere-ai' => 'cohere-ai',
102		- 'MistralAI-User' => 'mistral-user',
103		- ];
104		-
105		- /**
106	101	* Wire the front-end recorder and the retention cron.
107	102	*
108	103	* @return void
109	104	*/
		@@ -146,8 +141,12 @@
146	141
147	142	/**
148	143	* Classify a user agent as an AI crawler.
149	144	*
	145	+ * The agent list is `AI_Crawlers`, shared with the robots.txt panel — the
	146	+ * two must agree about which bots exist, or a site blocks a crawler it is
	147	+ * not counting (#657).
	148	+ *
150	149	* @param string $user_agent Raw user agent (may be empty).
151	150	* @return string|null Crawler slug, or null when not a known AI crawler.
152	151	*/
153	152	public static function classify_crawler(string $user_agent): ?string {
		@@ -154,9 +153,12 @@
154	153	if ('' === $user_agent) {
155	154	return null;
156	155	}
157	156
158		- foreach (self::CRAWLER_AGENTS as $fragment => $slug) {
	157	+ // Token order is significant and owned by the registry: the first
	158	+ // token found wins, so `Claude-SearchBot` has to be tested before
	159	+ // `ClaudeBot`. See AI_Crawlers::AGENTS.
	160	+ foreach (AI_Crawlers::token_map() as $fragment => $slug) {
159	161	if (false !== stripos($user_agent, $fragment)) {
160	162	return $slug;
161	163	}
162	164	}
		@@ -205,8 +207,54 @@
205	207	}
206	208	}
207	209
208	210	/**
	211	+ * Record a page served as Markdown to an AI agent.
	212	+ *
	213	+ * Called by Pro's Markdown for AI feature at serve time. Lives here rather
	214	+ * than in Pro because this class owns the aggregate table; Pro owning a
	215	+ * second writer to it would couple the schema to two repos.
	216	+ *
	217	+ * @param string $source Crawler slug when the agent is a known AI crawler,
	218	+ * 'header' for Accept-negotiated requests, 'link' for
	219	+ * ?format=markdown / .md URLs.
	220	+ * @param string $path Path of the post served.
	221	+ * @return void
	222	+ */
	223	+ public function record_served_markdown(string $source, string $path = ''): void {
	224	+ $source = sanitize_key($source);
	225	+ if ('' === $source) {
	226	+ $source = 'other';
	227	+ }
	228	+ $this->bump('markdown', $source, substr($path, 0, 191));
	229	+ }
	230	+
	231	+ /**
	232	+ * Total Markdown-for-AI responses served in the last N days.
	233	+ *
	234	+ * @param int $days Range in days (bounded 1–180).
	235	+ * @return int
	236	+ */
	237	+ public function served_markdown_count(int $days = 30): int {
	238	+ global $wpdb;
	239	+
	240	+ $days = max(1, min(self::RETENTION_DAYS, $days));
	241	+ $table = $wpdb->prefix . 'thinkrank_ai_traffic';
	242	+ // Same clock as write_bucket(), and counted in calendar days so a
	243	+ // DST transition inside the window does not move the boundary.
	244	+ $since = $this->day_key_offset($days);
	245	+
	246	+ // phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- read-only aggregate over our own table.
	247	+ return (int) $wpdb->get_var(
	248	+ $wpdb->prepare(
	249	+ "SELECT COALESCE(SUM(hits), 0) FROM {$table} WHERE kind = 'markdown' AND day >= %s",
	250	+ $since
	251	+ )
	252	+ );
	253	+ // phpcs:enable
	254	+ }
	255	+
	256	+ /**
209	257	* The current request path, normalized for the aggregate key.
210	258	*
211	259	* @return string
212	260	*/
		@@ -227,23 +275,90 @@
227	275	* @param string $path Landing path (referrals only).
228	276	* @return void
229	277	*/
230	278	private function bump(string $kind, string $source, string $path = ''): void {
	279	+ // Without a persistent object cache there is nowhere to buffer, so keep
	280	+ // the direct write rather than counting into per-request memory that is
	281	+ // thrown away — that would lose hits outright.
	282	+ if (!wp_using_ext_object_cache()) {
	283	+ $this->write_bucket($kind, $source, $path, 1);
	284	+
	285	+ return;
	286	+ }
	287	+
	288	+ // With one, buffer and flush in batches. The unique key is
	289	+ // (day, kind, source, path), so all baseline traffic funnels into a
	290	+ // single row per day: InnoDB took an exclusive row lock on it for every
	291	+ // visitor, serialising concurrent anonymous traffic, and made every
	292	+ // pageview a write even when the response was fully cacheable (#402).
	293	+ $bucket = self::COUNTER_PREFIX . md5($kind . '|' . $source . '|' . $path);
	294	+ $since = $bucket . '_since';
	295	+
	296	+ $hits = wp_cache_incr($bucket, 1, self::COUNTER_GROUP);
	297	+
	298	+ if (false === $hits) {
	299	+ wp_cache_add($bucket, 1, self::COUNTER_GROUP, 0);
	300	+ wp_cache_add($since, time(), self::COUNTER_GROUP, 0);
	301	+ $hits = 1;
	302	+ }
	303	+
	304	+ $started = (int) wp_cache_get($since, self::COUNTER_GROUP);
	305	+
	306	+ // Flush on either bound, so a busy site writes rarely and a quiet one
	307	+ // still lands its hits — an eviction can cost at most one window.
	308	+ if ($hits < self::FLUSH_AT && $started > 0 && (time() - $started) < self::FLUSH_AFTER) {
	309	+ return;
	310	+ }
	311	+
	312	+ wp_cache_set($bucket, 0, self::COUNTER_GROUP, 0);
	313	+ wp_cache_set($since, time(), self::COUNTER_GROUP, 0);
	314	+
	315	+ $this->write_bucket($kind, $source, $path, (int) $hits);
	316	+ }
	317	+
	318	+ /**
	319	+ * Add hits to a bucket's row.
	320	+ *
	321	+ * @since 2.0.1
	322	+ *
	323	+ * @param string $kind 'referral' | 'crawler' | 'baseline'.
	324	+ * @param string $source Platform/bot slug, or 'all' for baseline.
	325	+ * @param string $path Landing path (referrals only).
	326	+ * @param int $hits How many hits to add.
	327	+ * @return void
	328	+ */
	329	+ private function write_bucket(string $kind, string $source, string $path, int $hits): void {
	330	+ if ($hits < 1) {
	331	+ return;
	332	+ }
	333	+
	334	+ // `day` is the SITE-LOCAL date (see day_key()), not UTC. The column is
	335	+ // a bare `date` with no zone attached, so the clock that writes it is
	336	+ // the only thing that gives it meaning — and these keys reach the user
	337	+ // as the trend chart's dates, where the site's own calendar is what
	338	+ // they expect to read.
	339	+ //
	340	+ // Every range boundary and retention cutoff must be derived with
	341	+ // day_key() too. A gmdate() boundary against these rows drifts by a
	342	+ // day for part of every day on a non-UTC site.
	343	+
231	344	global $wpdb;
232	345
233	346	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
234	347
235		- // Single cheap upsert per pageview; the unique key is the bucket.
	348	+ // Aggregate counter upsert; the unique key is the bucket.
236	349	// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- aggregate counter upsert; table name is prefix-derived.
237	350	$wpdb->query(
238	351	// phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- table name is $wpdb->prefix plus a literal, and every value is passed as a placeholder replacement.
239	352	$wpdb->prepare(
240		- "INSERT INTO {$table} (day, kind, source, path, hits) VALUES (%s, %s, %s, %s, 1)
241		- ON DUPLICATE KEY UPDATE hits = hits + 1",
	353	+ "INSERT INTO {$table} (day, kind, source, path, hits) VALUES (%s, %s, %s, %s, %d)
	354	+ ON DUPLICATE KEY UPDATE hits = hits + %d",
242	355	current_time('Y-m-d'),
243	356	$kind,
244	357	$source,
245		- $path
	358	+ $path,
	359	+ $hits,
	360	+ $hits
246	361	)
247	362	);
248	363	// phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared
249	364	}
		@@ -248,8 +363,83 @@
248	363	// phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared
249	364	}
250	365
251	366	/**
	367	+ * The site-local date key for an instant, matching write_bucket().
	368	+ *
	369	+ * Every consumer of the `day` column goes through this, so the read side
	370	+ * cannot drift onto a different calendar from the write side.
	371	+ *
	372	+ * @param int|null $timestamp Unix timestamp, or null for now.
	373	+ * @return string `Y-m-d` on the site's clock.
	374	+ */
	375	+ private function day_key(?int $timestamp = null): string {
	376	+ return wp_date('Y-m-d', $timestamp ?? time());
	377	+ }
	378	+
	379	+ /**
	380	+ * Midday on a given site-local date.
	381	+ *
	382	+ * Midday, not midnight: a handful of zones start DST at 00:00, so
	383	+ * midnight on a transition date can be a time that does not exist and
	384	+ * PHP quietly rolls it forward. Noon is never inside a DST gap, so
	385	+ * every date in the year is representable.
	386	+ *
	387	+ * @param string $day `Y-m-d` on the site's clock.
	388	+ * @return DateTimeImmutable
	389	+ */
	390	+ private function local_noon(string $day): DateTimeImmutable {
	391	+ return new DateTimeImmutable($day . ' 12:00:00', wp_timezone());
	392	+ }
	393	+
	394	+ /**
	395	+ * The site-local date key N *calendar* days before today.
	396	+ *
	397	+ * Not `time() - N * DAY_IN_SECONDS`: a fixed 86400-second step is not a
	398	+ * day on a clock that shifts. Around a DST transition that arithmetic
	399	+ * lands an hour early or late, which moves the date for the hour either
	400	+ * side of midnight.
	401	+ *
	402	+ * @param int $days_ago Whole days back.
	403	+ * @return string `Y-m-d`.
	404	+ */
	405	+ private function day_key_offset(int $days_ago): string {
	406	+ return $this->local_noon($this->day_key())
	407	+ ->modify('-' . max(0, $days_ago) . ' day')
	408	+ ->format('Y-m-d');
	409	+ }
	410	+
	411	+ /**
	412	+ * Every site-local date from $from to $to inclusive.
	413	+ *
	414	+ * Walks the calendar rather than stepping by 86400 seconds, so a DST
	415	+ * transition inside the range neither duplicates a date nor skips one.
	416	+ * Skipping one used to drop that day's referrals out of the trend while
	417	+ * they stayed in the totals.
	418	+ *
	419	+ * @param string $from `Y-m-d`, inclusive.
	420	+ * @param string $to `Y-m-d`, inclusive.
	421	+ * @return string[] Ordered, contiguous date keys.
	422	+ */
	423	+ private function day_range(string $from, string $to): array {
	424	+ $cursor = $this->local_noon($from);
	425	+ $end = $this->local_noon($to);
	426	+
	427	+ $days = [];
	428	+ // Bounded by the caller's window (<= RETENTION_DAYS), with headroom
	429	+ // so a malformed pair can never spin here.
	430	+ $guard = self::RETENTION_DAYS + 2;
	431	+ $steps = 0;
	432	+ while ($cursor <= $end && $steps < $guard) {
	433	+ $days[] = $cursor->format('Y-m-d');
	434	+ $cursor = $cursor->modify('+1 day');
	435	+ $steps++;
	436	+ }
	437	+
	438	+ return $days;
	439	+ }
	440	+
	441	+ /**
252	442	* Dashboard summary for the last N days.
253	443	*
254	444	* @param int $days Range in days (bounded 1–180).
255	445	* @return array<string, mixed>
		@@ -258,9 +448,11 @@
258	448	global $wpdb;
259	449
260	450	$days = max(1, min(self::RETENTION_DAYS, $days));
261	451	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
262		- $since = gmdate('Y-m-d', time() - $days * DAY_IN_SECONDS);
	452	+ // Same clock as write_bucket(), and counted in calendar days so a
	453	+ // DST transition inside the window does not move the boundary.
	454	+ $since = $this->day_key_offset($days);
263	455
264	456	// phpcs:disable WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- read-only aggregates over our own table.
265	457	$rows = $wpdb->get_results(
266	458	$wpdb->prepare(
		@@ -276,8 +468,9 @@
276	468	$platforms = [];
277	469	$trend = [];
278	470	$pages = [];
279	471	$crawlers = [];
	472	+ $markdown = 0;
280	473
281	474	foreach ((array) $rows as $row) {
282	475	$hits = (int) $row['hits'];
283	476	switch ($row['kind']) {
		@@ -294,8 +487,11 @@
294	487	break;
295	488	case 'crawler':
296	489	$crawlers[$row['source']] = ($crawlers[$row['source']] ?? 0) + $hits;
297	490	break;
	491	+ case 'markdown':
	492	+ $markdown += $hits;
	493	+ break;
298	494	}
299	495	}
300	496
301	497	arsort($platforms);
		@@ -302,8 +498,31 @@
302	498	arsort($pages);
303	499	arsort($crawlers);
304	500	ksort($trend);
305	501
	502	+ // Fill every day the query covered, zeroes included. Only days that
	503	+ // had a referral produce a $trend key above, and the chart positions
	504	+ // points by index — so a sparse map drew a three-week gap exactly
	505	+ // like a one-day gap. A contiguous series makes even spacing correct,
	506	+ // and distinguishes "no referrals that day" from "no data".
	507	+ //
	508	+ // The range mirrors the WHERE clause (day >= $since, through today)
	509	+ // so the series covers exactly what was counted, and it is built on
	510	+ // day_key() so the keys match how the rows were written.
	511	+ $filled = [];
	512	+ foreach ($this->day_range($since, $this->day_key()) as $day) {
	513	+ $filled[$day] = $trend[$day] ?? 0;
	514	+ }
	515	+
	516	+ // Safety net for anything the window did not cover — a row dated
	517	+ // ahead of today, which a site that moved timezone can hold. Union
	518	+ // keeps the filled zeroes and adds only keys not already present, so
	519	+ // the series can never total less than ai_sessions.
	520	+ $filled += $trend;
	521	+ ksort($filled);
	522	+
	523	+ $trend = $filled;
	524	+
306	525	return [
307	526	'days' => $days,
308	527	'baseline' => $baseline,
309	528	'ai_sessions' => $referrals,
		@@ -313,13 +532,48 @@
313	532	'top_pages' => array_slice($pages, 0, 10, true),
314	533	'crawlers' => $crawlers,
315	534	// Whether llms.txt is being served, so the crawler panel can pair
316	535	// "bots are coming" with "and here's what we feed them".
317		- 'llms_txt' => file_exists(ABSPATH . 'llms.txt'),
	536	+ //
	537	+ // Ask the manager, not the filesystem: `dynamic` delivery — the
	538	+ // resolved default on every non-Apache stack — publishes no
	539	+ // physical file and answers from serve_llms_txt(), so a
	540	+ // file_exists() probe reports "not published" for a live document.
	541	+ 'llms_txt' => $this->llms_txt_published(),
	542	+ // Pages served as Markdown by Pro's Markdown for AI feature
	543	+ // (kind 'markdown', written via record_served_markdown()).
	544	+ 'markdown_served' => $markdown,
318	545	];
319	546	}
320	547
321	548	/**
	549	+ * Whether llms.txt is currently being served, in either delivery mode.
	550	+ *
	551	+ * `static` publishes a file at ABSPATH; `dynamic` keeps the document in
	552	+ * an option and serves it from a PHP route. Only the manager knows which
	553	+ * is in force, so it is the single source of truth here.
	554	+ *
	555	+ * @return bool
	556	+ */
	557	+ private function llms_txt_published(): bool {
	558	+ // Spelt exactly as the class is declared. The autoloader routes this
	559	+ // one through a case-SENSITIVE special-case map, and while a
	560	+ // mis-cased name happens to fall through to the generic rule and
	561	+ // resolve anyway, that is a coincidence — a change to that rule would
	562	+ // silently make class_exists() false here, and the badge would go
	563	+ // back to reporting "No llms.txt" for a live document.
	564	+ if (!class_exists(LLMs_Txt_Manager::class)) {
	565	+ // Defensive: a partial load must not claim llms.txt is live.
	566	+ return false;
	567	+ }
	568	+
	569	+ // is_published(), not get_llms_txt_status(): the latter resolves the
	570	+ // delivery mode, may fire a loopback probe and touches the filesystem
	571	+ // API, which is far too much work for a dashboard boolean.
	572	+ return (new LLMs_Txt_Manager())->is_published();
	573	+ }
	574	+
	575	+ /**
322	576	* Drop aggregate rows past the retention window.
323	577	*
324	578	* @return void
325	579	*/
		@@ -326,9 +580,10 @@
326	580	public function prune(): void {
327	581	global $wpdb;
328	582
329	583	$table = $wpdb->prefix . 'thinkrank_ai_traffic';
330		- $cutoff = gmdate('Y-m-d', time() - self::RETENTION_DAYS * DAY_IN_SECONDS);
	584	+ // Same clock as write_bucket(), counted in calendar days.
	585	+ $cutoff = $this->day_key_offset(self::RETENTION_DAYS);
331	586
332	587	// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- retention delete on our own table.
333	588	$wpdb->query($wpdb->prepare("DELETE FROM {$table} WHERE day < %s", $cutoff));
334	589	}