metasync / otto / class-metasync-otto-transient-cache.php

class-metasync-otto-transient-cache.php in Search Atlas SEO – OTTO AI SEO Automation for WordPress 2.6.22, at otto/class-metasync-otto-transient-cache.php

1,276 lines 51.0 KB

Raw Download Zip

1	<?php
2	/**
3	* OTTO Transient Cache Manager
4	* Implements Option 1: On-Demand Transient Caching with mitigations
5	*
6	* Features:
7	* - Transient-based caching (30 min TTL)
8	* - Rate limiting (max 10 API calls per minute)
9	* - Request locking (prevents thundering herd)
10	* - API timeout with fallback
11	* - Stale cache fallback on API failure
12	*/
13
14	if (!defined('ABSPATH')) {
15	exit;
16	}
17
18	class Metasync_Otto_Transient_Cache {
19
20	/**
21	* Transient prefix
22	*/
23	private const TRANSIENT_PREFIX = 'otto_suggestions_';
24
25	/**
26	* Lock transient prefix
27	*/
28	private const LOCK_PREFIX = 'otto_lock_';
29
30	/**
31	* Rate limit transient prefix
32	*/
33	private const RATE_LIMIT_PREFIX = 'otto_api_rate_';
34
35	/**
36	* Stale cache prefix (for fallback)
37	*/
38	private const STALE_PREFIX = 'otto_stale_';
39
40	/**
41	* TTL for "no suggestions" cache (5 minutes)
42	*/
43	private const NO_SUGGESTIONS_TTL = 5 * MINUTE_IN_SECONDS;
44
45	/**
46	* Lock timeout (5 seconds)
47	*/
48	private const LOCK_TIMEOUT = 5;
49
50	/**
51	* Recovery probe lock timeout (10 seconds).
52	* Must exceed warm_cache()'s longest API timeout (8 seconds), so a slow
53	* recovery probe cannot overlap with a second probe.
54	*/
55	private const BREAKER_PROBE_LOCK_TIMEOUT = 10;
56
57	/**
58	* Circuit breaker prefix (per API host, not per URL — the failure being
59	* tracked is "the OTTO API is unreachable", which is host-wide)
60	*/
61	private const BREAKER_PREFIX = 'otto_breaker_';
62
63	/**
64	* Circuit breaker consecutive-failure tally prefix
65	*/
66	private const BREAKER_FAIL_PREFIX = 'otto_breaker_fail_';
67
68	/**
69	* Circuit breaker probe-lock prefix (half-open state)
70	*/
71	private const BREAKER_PROBE_PREFIX = 'otto_breaker_probe_';
72
73	/**
74	* Object-cache group for atomic counters (rate limiter, breaker tally).
75	* These are NOT stored in the transient namespace when an external object
76	* cache is present, so they must always be read and deleted via
77	* counter_get() / counter_delete().
78	*/
79	private const COUNTER_GROUP = 'otto_counters';
80
81	/**
82	* Consecutive hard failures before the breaker opens.
83	*/
84	private const BREAKER_FAILURE_THRESHOLD = 3;
85
86	/**
87	* How long the breaker stays open before allowing a single probe (seconds).
88	* Deliberately short: long enough to shed load off the PHP workers, short
89	* enough that a brief upstream blip does not visibly de-OTTO the site.
90	*/
91	private const BREAKER_COOLOFF = 60;
92
93	/**
94	* Max API calls per minute
95	*/
96	public const MAX_API_CALLS_PER_MINUTE = 10;
97
98	/**
99	* API timeout (2 seconds)
100	*/
101	private const API_TIMEOUT = 2;
102
103	/**
104	* OTTO UUID
105	*/
106	private $otto_uuid;
107
108	/**
109	* Whether the last fetch_from_api() call was answered with HTTP 404.
110	*
111	* Tracked explicitly rather than inferred from the returned value: a 404 is
112	* represented as an empty array, but json_decode() also yields an empty array
113	* for a 200 body of `{}` or `[]`. Distinguishing the two by emptiness would
114	* silently misclassify such a response as "no data held" and stop clearing the
115	* stale copy — resurfacing a withdrawn title, which is the bug the stale wipe
116	* exists to prevent.
117	*
118	* @var bool
119	*/
120	private $last_fetch_was_404 = false;
121
122	/**
123	* OTTO API endpoint
124	*/
125	private $api_endpoint;
126
127	/**
128	* API fetch timeout for the current operation.
129	* Page-load requests use the short API_TIMEOUT (2s).
130	* warm_cache() uses a longer timeout since it runs in a webhook handler.
131	*/
132	private $fetch_timeout = self::API_TIMEOUT;
133
134	/**
135	* Constructor
136	*/
137	public function __construct($otto_uuid) {
138	$this->otto_uuid = $otto_uuid;
139
140	# Use endpoint manager if available, otherwise fallback to production
141	if (class_exists('Metasync_Endpoint_Manager')) {
142	$this->api_endpoint = Metasync_Endpoint_Manager::get_endpoint('OTTO_URL_DETAILS');
143	} else {
144	$this->api_endpoint = 'https://sa.searchatlas.com/api/v2/otto-url-details';
145	}
146	}
147
148	/**
149	* Cache status tracker (for header)
150	*/
151	private static $cache_status = [];
152
153	/**
154	* Get OTTO suggestions for a URL (with transient caching)
155	*
156	* @param string $url The page URL
157	* @param string $track_key Optional key to track cache status for this request
158	* @return array\|false OTTO suggestions data or false if none
159	*/
160	public function get_suggestions($url, $track_key = null) {
161	if (empty($url) \|\| empty($this->otto_uuid)) {
162	return false;
163	}
164
165	# PERFORMANCE OPTIMIZATION: Generate all cache keys once
166	$keys = $this->get_cache_keys($url);
167	$cache_status_key = $track_key ?: $keys['track'];
168
169	# Step 1: Check transient cache first
170	$cached = get_transient($keys['transient']);
171	if ($cached !== false) {
172	# Cache hit. Distinguish a real payload from the "checked, nothing
173	# here" marker so the X-MetaSync-OTTO-Cache header stays truthful —
174	# reporting HIT for a URL OTTO holds no data for would be misleading
175	# when debugging, and the two cases are handled differently upstream.
176	self::$cache_status[$cache_status_key] = $this->has_payload($cached)
177	? 'HIT'
178	: 'NO_SUGGESTIONS';
179	return $cached;
180	}
181
182	# Step 1.5: circuit breaker.
183	#
184	# Checked BEFORE the rate limiter on purpose. can_make_api_call() increments
185	# a per-minute counter, so running it first would spend the site's API budget
186	# on requests the breaker is about to block anyway — leaving OTTO throttled
187	# for the rest of the minute even after the upstream recovers.
188	if ($this->is_breaker_open()) {
189	$stale = get_transient($keys['stale']);
190	if ($stale !== false) {
191	self::$cache_status[$cache_status_key] = 'STALE';
192	return $stale;
193	}
194	self::$cache_status[$cache_status_key] = 'BREAKER_OPEN';
195	return false;
196	}
197
198	# Step 2: Acquire lock atomically (test-and-set).
199	# If another worker already holds the lock, wait briefly for them to populate
200	# the cache, then either return their result or bail out — DO NOT fall through
201	# to fetch_from_api() (that was the original bug: duplicate concurrent API calls).
202	if (!$this->acquire_lock_atomic($keys['lock'], self::LOCK_TIMEOUT)) {
203	usleep(500000); // 0.5 seconds
204	$cached = get_transient($keys['transient']);
205	if ($cached !== false) {
206	self::$cache_status[$cache_status_key] = 'HIT';
207	return $cached;
208	}
209	# The lock holder did not finish in time. Before giving up and letting
210	# the caller serve un-optimized HTML, reuse the stale copy — it is the
211	# same data one TTL older, which is far better for a crawler than the
212	# original title. Previously only the rate-limit path consulted it.
213	$stale = get_transient($keys['stale']);
214	if ($stale !== false) {
215	self::$cache_status[$cache_status_key] = 'STALE';
216	return $stale;
217	}
218	self::$cache_status[$cache_status_key] = 'LOCKED';
219	return false;
220	}
221
222	try {
223	# Another request may have populated the cache between our initial
224	# miss and this lock acquisition. Recheck before spending API budget.
225	$cached = get_transient($keys['transient']);
226	if ($cached !== false) {
227	self::$cache_status[$cache_status_key] = $this->has_payload($cached)
228	? 'HIT'
229	: 'NO_SUGGESTIONS';
230	return $cached;
231	}
232
233	# Step 3: Count only the lock winner that can make a real API call.
234	if (!$this->can_make_api_call()) {
235	# Rate limited - try to use stale cache
236	$stale = get_transient($keys['stale']);
237	if ($stale !== false) {
238	# NEW: Structured error logging with category and code
239	if (class_exists('Metasync_Error_Logger')) {
240	Metasync_Error_Logger::log(
241	Metasync_Error_Logger::CATEGORY_API_RATE_LIMIT,
242	Metasync_Error_Logger::SEVERITY_INFO,
243	'OTTO suggestions throttled - served from cache',
244	[
245	'url' => $url,
246	'fallback' => 'stale_cache',
247	'api_endpoint' => 'OTTO Suggestions API',
248	'operation' => 'get_suggestions'
249	]
250	);
251	}
252
253	error_log('MetaSync OTTO: Throttled, serving suggestions from cache for ' . $url);
254	self::$cache_status[$cache_status_key] = 'STALE';
255	return $stale;
256	}
257	# No stale cache available - return false
258	self::$cache_status[$cache_status_key] = 'RATE_LIMITED';
259	return false;
260	}
261
262	# Step 4: Fetch from API (with timeout and error handling)
263	$suggestions = $this->fetch_from_api($url);
264
265	# NitroPack purge removed from page-load path to prevent feedback loop:
266	# get_suggestions() runs on every visitor request when the transient expires,
267	# and purging NitroPack here causes it to flush its cache (and potentially the
268	# object cache holding our transients), triggering another API fetch → purge cycle.
269	# The webhook handler (otto_pixel.php → purge_single_url) already covers the
270	# legitimate case where suggestions change after an OTTO crawl.
271
272	# Step 5: Store result in transient
273	if ($suggestions && $this->has_payload($suggestions)) {
274	# Has suggestions - cache for the admin-configured TTL (default 30 min)
275	$ttl = Metasync_Otto_Config::get_otto_cache_ttl_seconds();
276	set_transient($keys['transient'], $suggestions, $ttl);
277	# Also store as stale cache for fallback (2x the primary TTL)
278	set_transient($keys['stale'], $suggestions, $ttl * 2);
279	self::$cache_status[$cache_status_key] = 'MISS'; // Cache miss, fetched from API
280	} elseif ($suggestions !== false) {
281	# API responded 200 OK but genuinely no OTTO suggestions for this URL,
282	# or answered 404 (no data held for this URL at all).
283	#
284	# Cache the negative result so we stop asking. This MUST be stored as
285	# an empty array rather than `false`: WordPress cannot distinguish a
286	# stored `false` from an absent key, so a `false` marker never reads
287	# back as a hit and every request re-calls the API — which then
288	# exhausts the per-minute budget and pushes these URLs into
289	# RATE_LIMITED, a status treated as a transient failure. An empty
290	# array reads back cleanly, and has_payload([]) is false, so every
291	# consumer still takes the "no suggestions" path.
292	set_transient($keys['transient'], [], self::NO_SUGGESTIONS_TTL);
293
294	# Wipe the stale fallback for any empty answer that was NOT a 404 —
295	# a 200 whose body is empty or hollow, and a 204. Both mean the URL is
296	# known to OTTO but currently holds no suggestions, and without the
297	# wipe a rate-limit event within the next hour would serve the
298	# previously-deployed (now undeployed) suggestion, keeping the wrong
299	# title alive on the page.
300	#
301	# A 404 is different, and the difference is who answered. A 204 or an
302	# empty 200 came from the application itself: it was asked about this
303	# URL and deliberately said "nothing here". A 404 may never have
304	# reached the application at all — a changed endpoint path, a gateway,
305	# or a CDN in front of the API can produce it for every URL on every
306	# site at once, for reasons that say nothing about whether suggestions
307	# exist. Wiping stale copies on that would destroy the fallback
308	# everywhere simultaneously, precisely when it is the only thing
309	# keeping correct SEO on the page. So a 404 leaves it alone.
310	#
311	# Keyed off the recorded status code rather than the emptiness of the
312	# decoded body: json_decode() also returns an empty array for a 200
313	# body of `{}` or `[]`, which would otherwise be mistaken for a 404.
314	if (!$this->last_fetch_was_404) {
315	delete_transient($keys['stale']);
316	}
317
318	self::$cache_status[$cache_status_key] = 'NO_SUGGESTIONS';
319	} else {
320	# fetch_from_api() returned false — network error, timeout, or a
321	# non-200 response other than 404 and 204 (both of those return []
322	# and are handled by the "no suggestions" branch above).
323	# Do NOT cache the failure: a stale false would poison subsequent MISS requests
324	# (Kinsta sees MISS → PHP runs → transient HIT = false → OTTO skips → old title cached).
325	# Leave the transient empty so the very next page load retries the API call.
326	#
327	# Before reporting the failure upward, reuse the stale copy if we
328	# have one — same data, one TTL older, which still renders correct
329	# SEO instead of falling through to the original markup.
330	$stale = get_transient($keys['stale']);
331	if ($stale !== false) {
332	self::$cache_status[$cache_status_key] = 'STALE';
333	$suggestions = $stale;
334	} else {
335	self::$cache_status[$cache_status_key] = 'API_ERROR';
336	}
337	}
338	} finally {
339	# Step 6: Release lock — guaranteed to run even if fetch_from_api() throws
340	# or the storage block errors, so we never leak the lock for the full TTL.
341	# Note: acquire_lock_atomic() uses raw SQL (INSERT IGNORE / CAS UPDATE) on
342	# the MySQL path, but delete_transient() via the WP API is safe here because
343	# the rows use autoload='no' and are not in the alloptions cache.
344	delete_transient($keys['lock']);
345	}
346
347	return $suggestions;
348	}
349
350	/**
351	* Get cache status for a request
352	*
353	* @param string $key The tracking key
354	* @return string Cache status (HIT, MISS, STALE, RATE_LIMITED, NO_SUGGESTIONS, LOCKED, UNKNOWN)
355	*/
356	public static function get_cache_status($key) {
357	return self::$cache_status[$key] ?? 'UNKNOWN';
358	}
359
360	/**
361	* Check if URL has OTTO suggestions (quick check)
362	*
363	* @param string $url The page URL
364	* @return bool True if URL has suggestions
365	*/
366	public function has_suggestions($url) {
367	$suggestions = $this->get_suggestions($url);
368	return $suggestions !== false && $this->has_payload($suggestions);
369	}
370
371	/**
372	* Invalidate cache for a URL
373	*
374	* @param string $url The page URL
375	* @return bool Success
376	*/
377	public function invalidate($url) {
378	# PERFORMANCE OPTIMIZATION: Generate all cache keys once
379	$keys = $this->get_cache_keys($url);
380
381	delete_transient($keys['transient']);
382	delete_transient($keys['stale']);
383	delete_transient($keys['lock']);
384
385	return true;
386	}
387
388	/**
389	* Warm cache for a URL (pre-fetch and cache)
390	* Called from the OTTO webhook handler — NOT from a page load.
391	* Uses a longer API timeout so transients are reliably populated before
392	* the Kinsta/WP Engine cache purge fires, preventing the "false transient
393	* poisoning" race condition (MISS → false transient HIT → OTTO skips →
394	* old Yoast title cached).
395	*
396	* @param string $url The page URL
397	* @return array\|false Suggestions or false
398	*/
399	public function warm_cache($url) {
400	# Invalidate existing cache first
401	$this->invalidate($url);
402
403	# Use a longer timeout for this pre-warm request (webhook context, not page load).
404	$saved_timeout = $this->fetch_timeout;
405	$this->fetch_timeout = 8; // 8 seconds — enough for cross-region API calls
406
407	# Fetch and cache (with the longer timeout)
408	$result = $this->get_suggestions($url);
409
410	# Restore original timeout
411	$this->fetch_timeout = $saved_timeout;
412
413	return $result;
414	}
415
416	/**
417	* Fetch suggestions from OTTO API
418	*
419	* @param string $url The page URL
420	* @return array\|false API response or false on failure
421	*/
422	private function fetch_from_api($url) {
423	# Reset per call so a previous 404 cannot influence this one's handling.
424	$this->last_fetch_was_404 = false;
425
426	# Check if endpoint is in backoff mode (explicit check for better error handling)
427	if (class_exists('Metasync_API_Backoff_Manager')) {
428	$backoff_manager = Metasync_API_Backoff_Manager::get_instance();
429	if ($backoff_manager->is_endpoint_in_backoff($this->api_endpoint)) {
430	error_log('MetaSync OTTO: Request deferred - retry scheduled for this endpoint');
431	# Try to use stale cache if available
432	$stale = get_transient($this->get_stale_key($url));
433	if ($stale !== false) {
434	error_log('MetaSync OTTO: Serving suggestions from cache while retry is pending');
435	return $stale;
436	}
437	return false;
438	}
439	}
440
441	# NOTE: the circuit breaker gate lives in get_suggestions(), which is
442	# this method's only caller — placed there so an open breaker also skips the
443	# rate-limit increment. Failure recording stays here, at the point where the
444	# transport result is known.
445
446	# Construct API URL
447	$api_url = add_query_arg(
448	[
449	'url' => $url,
450	'uuid' => $this->otto_uuid,
451	],
452	$this->api_endpoint
453	);
454
455	# Make API request — timeout comes from $this->fetch_timeout so warm_cache()
456	# can override it without affecting page-load requests.
457	$response = wp_remote_get($api_url, [
458	'timeout' => $this->fetch_timeout,
459	'redirection' => 5,
460	'user-agent' => 'MetaSync-OTTO-Transient-Cache/1.0',
461	'sslverify' => true,
462	]);
463
464	# Check for errors
465	if (is_wp_error($response)) {
466	$error_message = $response->get_error_message();
467
468	# Check if error is due to backoff
469	if ($response->get_error_code() === 'api_backoff_active') {
470	error_log('MetaSync OTTO: Request deferred by retry schedule - ' . $error_message);
471	# Try to use stale cache
472	$stale = get_transient($this->get_stale_key($url));
473	if ($stale !== false) {
474	error_log('MetaSync OTTO: Serving suggestions from cache while retry is pending');
475	return $stale;
476	}
477	} else {
478	# A WP_Error that is not a backoff block is a hard transport
479	# failure (cURL 28 timeout, connection refused, DNS failure). These are
480	# the failures that cost a full API_TIMEOUT of PHP-FPM worker time.
481	$this->record_breaker_failure();
482	error_log('MetaSync OTTO: API call failed for ' . $url . ' - ' . $error_message);
483	}
484	return false;
485	}
486
487	# Check response code
488	$response_code = wp_remote_retrieve_response_code($response);
489	if ($response_code !== 200) {
490	# 404 and 204 are ANSWERS, not failures: the API is telling us it
491	# holds no OTTO data for this URL (never crawled, outside the project,
492	# or successfully answered with no content).
493	# Returning false here would classify it as API_ERROR, which is never
494	# cached and therefore re-requested on every uncached page load —
495	# burning the per-minute API budget on URLs that will keep answering
496	# 404 or 204, and (once fail-open emits no-cache headers) making every
497	# not-yet-crawled page uncacheable. Newly published posts are in this
498	# state until OTTO crawls them, so this is the common case, not an edge
499	# case. Return an empty array instead: get_suggestions() then takes the
500	# "no suggestions" branch and negative-caches it like any other
501	# legitimately empty result.
502	#
503	# These also close the circuit breaker, for the same reason a 200 does:
504	# the host answered, so it is reachable. Without that, a project whose
505	# URLs all answer 404 (not yet crawled) would leave a previously-opened
506	# breaker stuck half-open, re-probing every LOCK_TIMEOUT seconds against
507	# an API that is in fact perfectly healthy.
508	if ($response_code === 404) {
509	$this->last_fetch_was_404 = true;
510	$this->reset_breaker();
511	return [];
512	}
513	if ($response_code === 204) {
514	$this->reset_breaker();
515	return [];
516	}
517
518	# 5xx means the upstream is unhealthy — count it toward the breaker.
519	# Checked after the 404/204 answers above so they are never miscounted
520	# as failures. 429/503 are left to the backoff manager as well (503 is
521	# counted by both, which is correct: it is simultaneously "slow down"
522	# and "upstream unhealthy").
523	if ($response_code >= 500) {
524	$this->record_breaker_failure();
525	}
526
527	# For 429/503, the backoff manager will handle it automatically
528	# Try to use stale cache for these errors
529	if (in_array($response_code, [429, 503], true)) {
530	$stale = get_transient($this->get_stale_key($url));
531	if ($stale !== false) {
532	error_log('MetaSync OTTO: Serving suggestions from cache - source temporarily unavailable');
533	return $stale;
534	}
535	}
536	return false;
537	}
538
539	# Parse response body
540	$body = wp_remote_retrieve_body($response);
541	if (empty($body)) {
542	return false;
543	}
544
545	$data = json_decode($body, true);
546	if (json_last_error() !== JSON_ERROR_NONE) {
547	return false;
548	}
549
550	# A clean 200 means the upstream is healthy again — close the breaker.
551	$this->reset_breaker();
552
553	return $data;
554	}
555
556	// ------------------------------------------------------------------
557	// Circuit breaker
558	//
559	// Scoped to the API host, not the URL: when sa.searchatlas.com is
560	// unreachable it is unreachable for every URL, so a per-URL breaker would
561	// need BREAKER_FAILURE_THRESHOLD failures per URL before it helped — which
562	// on a site with thousands of URLs is never.
563	// ------------------------------------------------------------------
564
565	/**
566	* Transient key for the breaker state (per API host, per site).
567	*
568	* @return string
569	*/
570	private function get_breaker_key() {
571	return self::breaker_key_for_host(wp_parse_url($this->api_endpoint, PHP_URL_HOST) ?: 'unknown');
572	}
573
574	/**
575	* Single derivation of the breaker key, shared by the instance and static
576	* paths so the two cannot drift apart.
577	*
578	* @param string $host API host.
579	* @return string
580	*/
581	private static function breaker_key_for_host($host) {
582	$site_id = is_multisite() ? get_current_blog_id() : 0;
583	return self::BREAKER_PREFIX . $site_id . '_' . md5($host);
584	}
585
586	/**
587	* Is the circuit breaker currently open (i.e. should we skip the API call)?
588	*
589	* Three states:
590	* - closed : fewer than BREAKER_FAILURE_THRESHOLD consecutive failures. Allow.
591	* - open : threshold reached, still inside the cool-off. Block.
592	* - half-open : cool-off elapsed. Allow exactly ONE request through to probe
593	* the upstream; block the rest. Without the probe lock every
594	* concurrent request would retry at once and we would recreate
595	* the stampede the breaker exists to prevent.
596	*
597	* @return bool True if the API call should be skipped.
598	*/
599	private function is_breaker_open() {
600	# Escape hatch: lets support disable the breaker on a single site without
601	# shipping a release, mirroring how other OTTO behaviour is gated.
602	if (!apply_filters('metasync_otto_circuit_breaker_enabled', true)) {
603	return false;
604	}
605
606	$open_until = (int) get_transient($this->get_breaker_key());
607
608	if ($open_until <= 0) {
609	# Closed (or counting failures but not yet tripped).
610	return false;
611	}
612
613	if (time() < $open_until) {
614	return true;
615	}
616
617	# Half-open: the first caller to win the probe lock gets to test the
618	# upstream. Without this every concurrent request would retry at once and
619	# recreate the stampede the breaker exists to prevent.
620	# RETRY-LOOP-SAFETY: bounded recovery; do not add a naive retry handler here.
621	# Concurrency: one host-level lock lasting beyond the 8-second transport timeout.
622	# Attempts/backoff: one transport attempt; failure re-arms the 60-second cool-off.
623	# Blocked callers return immediately; inline retries would hold PHP workers.
624	$probe_key = self::BREAKER_PROBE_PREFIX . md5($this->get_breaker_key());
625	if ($this->acquire_lock_atomic($probe_key, self::BREAKER_PROBE_LOCK_TIMEOUT)) {
626	return false;
627	}
628
629	return true;
630	}
631
632	/**
633	* Record a hard upstream failure and open the breaker once the threshold is hit.
634	*
635	* The failure tally is incremented atomically: a read-modify-write here loses
636	* increments under exactly the concurrency the breaker is meant to protect
637	* against, which delays the trip and lets the damage continue. Verified in
638	* load testing — the naive version opened several seconds late.
639	*
640	* The open-until timestamp is a plain write because every racing worker
641	* computes the same value, so a lost update is harmless.
642	*/
643	private function record_breaker_failure() {
644	$open_key = $this->get_breaker_key();
645
646	$failures = $this->atomic_increment(
647	$this->get_breaker_fail_key(),
648	self::BREAKER_COOLOFF * 10
649	);
650
651	# Re-arm when the tally crosses the threshold, OR when the breaker is
652	# already tripped.
653	#
654	# The second condition is not redundant. The failure tally's expiry is
655	# written once, on first insert, and is never refreshed — so during an
656	# outage longer than its lifetime the tally silently resets to 1. Without
657	# this check a failed half-open probe would then fail to extend the
658	# cool-off, and the breaker would decay into one probe every LOCK_TIMEOUT
659	# seconds instead of one every BREAKER_COOLOFF.
660	#
661	# Short-circuits, so the extra read only happens on the sub-threshold
662	# failure path, which is rare.
663	if ($failures >= self::BREAKER_FAILURE_THRESHOLD \|\| get_transient($open_key) !== false) {
664	# Re-writing this also refreshes its TTL, so the open marker cannot
665	# expire underneath a sustained outage.
666	set_transient($open_key, time() + self::BREAKER_COOLOFF, self::BREAKER_COOLOFF * 10);
667	}
668	}
669
670	/**
671	* Close the breaker after a successful call.
672	*/
673	private function reset_breaker() {
674	$open_key = $this->get_breaker_key();
675	$fail_key = $this->get_breaker_fail_key();
676
677	# Guarded so a healthy site is not issuing deletes on every cache MISS.
678	# counter_get()/counter_delete() for the tally because with an object
679	# cache it does not live in the transient namespace.
680	if (get_transient($open_key) !== false \|\| $this->counter_get($fail_key) > 0) {
681	delete_transient($open_key);
682	$this->counter_delete($fail_key);
683	}
684	}
685
686	/**
687	* Transient key for the breaker's consecutive-failure tally.
688	*
689	* @return string
690	*/
691	private function get_breaker_fail_key() {
692	return self::BREAKER_FAIL_PREFIX . substr($this->get_breaker_key(), strlen(self::BREAKER_PREFIX));
693	}
694
695	/**
696	* Is the breaker open for the host serving the given endpoint?
697	*
698	* Static so other SearchAtlas callers on the same host (the bot crawl-log
699	* push in particular) can cheaply avoid an outbound call we already know
700	* will hang. Read-only: never opens, closes or probes the breaker, so a
701	* caller using this cannot disturb the suggestions path's state machine.
702	*
703	* @param string $endpoint_url Any URL on the host in question.
704	* @return bool
705	*/
706	public static function is_host_breaker_open($endpoint_url) {
707	if (!apply_filters('metasync_otto_circuit_breaker_enabled', true)) {
708	return false;
709	}
710
711	$host = wp_parse_url($endpoint_url, PHP_URL_HOST);
712	if (empty($host)) {
713	return false;
714	}
715
716	$open_until = (int) get_transient(self::breaker_key_for_host($host));
717
718	return $open_until > 0 && time() < $open_until;
719	}
720
721	/**
722	* Circuit breaker state, for Site Health / debug surfaces.
723	*
724	* @return array{open:bool, failures:int, seconds_remaining:int}
725	*/
726	public function get_breaker_status() {
727	$remaining = max(0, (int) get_transient($this->get_breaker_key()) - time());
728
729	return [
730	'open' => $remaining > 0,
731	'failures' => $this->counter_get($this->get_breaker_fail_key()),
732	'seconds_remaining' => $remaining,
733	];
734	}
735
736	/**
737	* Purge NitroPack cache for a specific URL
738	* Called when OTTO API returns new suggestions to ensure NitroPack regenerates cache with fresh content
739	* @param string $url The page URL to purge from NitroPack cache
740	* @return bool True if purge was attempted, false if NitroPack not available
741	*/
742	private function purge_nitropack_cache($url) {
743	# Check if NitroPack functions are available
744	if (!function_exists('nitropack_sdk_purge')) {
745	return false;
746	}
747
748	try {
749	# Purge NitroPack's cache (both local and remote) for this URL
750	# Using nitropack_sdk_purge() ensures remote API cache is also cleared
751	return nitropack_sdk_purge($url, NULL, 'MetaSync OTTO API call - fresh suggestions fetched');
752	} catch (\Exception $e) {
753	return false;
754	}
755	}
756
757	/**
758	* Check if API response has payload (suggestions)
759	* Public method for external use
760	*
761	* @param array $data API response data
762	* @return bool True if has suggestions
763	*/
764	public function has_payload($data) {
765	if (empty($data) \|\| !is_array($data)) {
766	return false;
767	}
768
769	# Undeployed OTTO responses ship harmless whitespace ("\n") and
770	# empty wrappers ({"images": {}}) that PHP's empty() reads as non-empty.
771	# Without these guards, has_payload() flagged undeployed pages as "live"
772	# and downstream code (otto_has_live_suggestions(), title filter) kept
773	# applying the stale _metasync_otto_title meta from the previous deploy.
774
775	$has_string = static function ($value) {
776	return is_string($value) && trim($value) !== '';
777	};
778
779	$has_container = static function ($value) {
780	if (!is_array($value) \|\| empty($value)) {
781	return false;
782	}
783	foreach ($value as $child) {
784	if (!empty($child)) {
785	return true;
786	}
787	}
788	return false;
789	};
790
791	if (!empty($data['header_replacements']) && is_array($data['header_replacements'])) {
792	foreach ($data['header_replacements'] as $replacement) {
793	if (is_array($replacement) && !empty($replacement['recommended_value'])) {
794	return true;
795	}
796	}
797	}
798
799	return $has_string($data['header_html_insertion'] ?? '')
800	\|\| $has_container($data['body_substitutions'] ?? [])
801	\|\| $has_string($data['body_top_html_insertion'] ?? '')
802	\|\| $has_string($data['body_bottom_html_insertion'] ?? '')
803	\|\| $has_string($data['footer_html_insertion'] ?? '')
804	\|\| $has_container($data['image_missing_alt'] ?? []);
805	}
806
807	/**
808	* Get OTTO API rate limit from execution settings
809	* Falls back to default constant if setting not configured
810	*
811	* @return int Rate limit (calls per minute)
812	*/
813	private function get_rate_limit() {
814	$execution_settings = get_option('metasync_execution_settings', array());
815	if (isset($execution_settings['otto_rate_limit'])) {
816	return (int) $execution_settings['otto_rate_limit'];
817	}
818	// Fallback to default constant
819	return self::MAX_API_CALLS_PER_MINUTE;
820	}
821
822	/**
823	* Check if we can make an API call (rate limiting)
824	*
825	* @return bool True if under rate limit
826	* @note With a persistent object cache (Redis/Memcached) the increment is atomic and race-free.
827	* Without one, falls back to transients (minor TOCTOU race under concurrency, but the counter persists across requests).
828	*/
829	private function can_make_api_call(): bool {
830	# Scope rate-limit key per site so each multisite subsite has its own
831	# 10 req/min budget. Without this, one subsite's traffic spike
832	# exhausts the network-wide bucket and silently rate-limits every
833	# other subsite for the rest of the minute.
834	$site_id = is_multisite() ? get_current_blog_id() : 0;
835	# Create rate limit key (per site, per minute)
836	$rate_key = self::RATE_LIMIT_PREFIX . $site_id . '_' . date('Y-m-d-H-i');
837
838	# Get rate limit from execution settings
839	$rate_limit = $this->get_rate_limit();
840
841	if (wp_using_ext_object_cache()) {
842	# Atomic path — race-free on Redis/Memcached
843	wp_cache_add($rate_key, 0, 'otto_rate', MINUTE_IN_SECONDS);
844	$new_count = wp_cache_incr($rate_key, 1, 'otto_rate');
845	return $new_count <= $rate_limit;
846	}
847
848	# MySQL transient fallback, atomic. The previous
849	# read-then-write lost increments whenever two PHP workers interleaved,
850	# which is precisely the traffic pattern the limiter exists for.
851	# Measured effect: ~22 calls/min against a configured cap of 10.
852	return $this->atomic_increment($rate_key, MINUTE_IN_SECONDS) <= $rate_limit;
853	}
854
855	/**
856	* Atomically increment a transient-backed counter and return its new value.
857	*
858	* Shared by the rate limiter and the circuit breaker. Both are counters that
859	* are only ever consulted while multiple PHP workers are racing, so a
860	* read-modify-write (get_transient + set_transient) silently undercounts
861	* exactly when accuracy matters.
862	*
863	* On MySQL, INSERT ... ON DUPLICATE KEY UPDATE is atomic under the unique
864	* key on option_name, and the LAST_INSERT_ID(expr) idiom hands back the
865	* post-increment value on the same connection without a second, racy SELECT.
866	*
867	* Caveat: the raw INSERT bypasses WordPress's in-process option cache, so a
868	* get_transient() call for the same key later in the SAME request may return
869	* a stale value. Nothing reads these counters that way — both callers use the
870	* return value here — but diagnostic surfaces should re-read from the DB.
871	*
872	* @param string $key Transient key (without the _transient_ prefix).
873	* @param int $ttl Lifetime in seconds.
874	* @return int New counter value.
875	*/
876	private function atomic_increment($key, $ttl) {
877	if (wp_using_ext_object_cache()) {
878	# Redis/Memcached INCR is already atomic.
879	wp_cache_add($key, 0, self::COUNTER_GROUP, $ttl);
880	return (int) wp_cache_incr($key, 1, self::COUNTER_GROUP);
881	}
882
883	global $wpdb;
884	$value_option = '_transient_' . $key;
885	$timeout_option = '_transient_timeout_' . $key;
886
887	# Let WordPress reap an expired window first. get_transient() deletes both
888	# rows when the timeout has passed; without this the raw INSERT below would
889	# happily increment a counter left over from a previous, long-finished
890	# window — so a single fresh failure could inherit an old tally and trip
891	# the breaker immediately.
892	get_transient($key);
893
894	# Timeout row first so WP's transient GC can reap the counter.
895	$wpdb->query(
896	$wpdb->prepare(
897	"INSERT IGNORE INTO {$wpdb->options} (option_name, option_value, autoload) VALUES (%s, %s, 'no')",
898	$timeout_option,
899	(string) (time() + (int) $ttl)
900	)
901	);
902
903	# MySQL reports 1 affected row for an insert and 2 for an update, which is
904	# how we distinguish "first increment" from "incremented again".
905	$affected = $wpdb->query(
906	$wpdb->prepare(
907	"INSERT INTO {$wpdb->options} (option_name, option_value, autoload) VALUES (%s, '1', 'no')
908	ON DUPLICATE KEY UPDATE option_value = LAST_INSERT_ID(CAST(option_value AS UNSIGNED) + 1)",
909	$value_option
910	)
911	);
912
913	# Fresh insert — the counter started at 1.
914	if ((int) $affected === 1) {
915	return 1;
916	}
917
918	# Increment — LAST_INSERT_ID() carries the new value on this connection.
919	$new_value = (int) $wpdb->insert_id;
920	if ((int) $affected > 1 && $new_value > 0) {
921	return $new_value;
922	}
923
924	# Anything else means the storage layer did not behave as expected;
925	# wpdb::query() returns false on any SQL error. Do NOT trust insert_id
926	# here — it may hold a stale value from an unrelated statement, and a
927	# large one reads as "budget exhausted", silently switching OTTO off for
928	# the whole site until the minute rolls over. Fall back to a plain
929	# read-modify-write: less precise under concurrency, but it fails open
930	# rather than closed.
931	$count = (int) get_transient($key) + 1;
932	set_transient($key, $count, $ttl);
933
934	return $count;
935	}
936
937	/**
938	* Read a counter written by atomic_increment().
939	*
940	* Must branch on the same backend as the writer. With an external object
941	* cache the counter lives in COUNTER_GROUP, NOT in the transient namespace,
942	* so reading it with get_transient() would always report zero.
943	*
944	* Reads zero if atomic_increment() already ran for this key in the SAME
945	* request, because that writes via raw SQL and WordPress's in-process option
946	* cache still holds the pre-write miss. Harmless for both callers: a request
947	* that recorded a failure does not also reset the breaker, and the diagnostic
948	* surface reads on a separate request.
949	*
950	* @param string $key Counter key.
951	* @return int
952	*/
953	private function counter_get($key) {
954	if (wp_using_ext_object_cache()) {
955	return (int) wp_cache_get($key, self::COUNTER_GROUP);
956	}
957
958	return (int) get_transient($key);
959	}
960
961	/**
962	* Delete a counter written by atomic_increment().
963	*
964	* Backend-matched for the same reason as counter_get(). Getting this wrong
965	* leaves the tally growing forever on object-cached sites, so every later
966	* incident would trip the breaker on its first failure.
967	*
968	* @param string $key Counter key.
969	*/
970	private function counter_delete($key) {
971	if (wp_using_ext_object_cache()) {
972	wp_cache_delete($key, self::COUNTER_GROUP);
973	return;
974	}
975
976	delete_transient($key);
977	}
978
979	/**
980	* Atomically acquire a lock (test-and-set).
981	*
982	* On Redis/Memcached object cache backends, wp_cache_add() maps to SET NX —
983	* the kernel guarantees only one caller succeeds. On the MySQL transient
984	* backend we INSERT IGNORE the timeout and value rows; the unique key on
985	* option_name makes that atomic. For an expired-but-not-yet-deleted lock
986	* we use a compare-and-swap UPDATE so only one racing worker can claim it.
987	*
988	* @param string $lock_key Transient key for the lock (without _transient_ prefix)
989	* @param int $ttl Lock timeout in seconds
990	* @return bool True if this caller acquired the lock, false if it was already held
991	*/
992	private function acquire_lock_atomic($lock_key, $ttl) {
993	# Fast path: external object cache (Redis/Memcached) — wp_cache_add is atomic.
994	if (wp_using_ext_object_cache()) {
995	return (bool) wp_cache_add($lock_key, '1', 'transient', $ttl);
996	}
997
998	# MySQL transient backend — emulate compare-and-set with INSERT IGNORE on the
999	# _transient_timeout_ row (which carries the unique option_name constraint).
1000	global $wpdb;
1001	$now = time();
1002	$new_expires = $now + (int) $ttl;
1003	$timeout_option = '_transient_timeout_' . $lock_key;
1004	$value_option = '_transient_' . $lock_key;
1005
1006	$inserted = $wpdb->query(
1007	$wpdb->prepare(
1008	"INSERT IGNORE INTO {$wpdb->options} (option_name, option_value, autoload) VALUES (%s, %s, 'no')",
1009	$timeout_option,
1010	(string) $new_expires
1011	)
1012	);
1013
1014	if ($inserted === 1) {
1015	# We won the insert race — now write the value row. INSERT IGNORE keeps it
1016	# safe if a stale value row from a prior expired lock is still around.
1017	$wpdb->query(
1018	$wpdb->prepare(
1019	"INSERT IGNORE INTO {$wpdb->options} (option_name, option_value, autoload) VALUES (%s, %s, 'no')",
1020	$value_option,
1021	'1'
1022	)
1023	);
1024	return true;
1025	}
1026
1027	# Insert was ignored — a timeout row already exists. Check whether the
1028	# existing lock has expired so we can attempt to take it over.
1029	$existing_expires = (int) $wpdb->get_var(
1030	$wpdb->prepare(
1031	"SELECT option_value FROM {$wpdb->options} WHERE option_name = %s",
1032	$timeout_option
1033	)
1034	);
1035
1036	if ($existing_expires > $now) {
1037	# Lock is still active — we did not acquire.
1038	return false;
1039	}
1040
1041	# Stale lock — try to claim it via compare-and-swap on the timeout column.
1042	# Only the worker whose UPDATE actually changes a row wins the race.
1043	$updated = $wpdb->query(
1044	$wpdb->prepare(
1045	"UPDATE {$wpdb->options} SET option_value = %s WHERE option_name = %s AND CAST(option_value AS UNSIGNED) = %d",
1046	(string) $new_expires,
1047	$timeout_option,
1048	$existing_expires
1049	)
1050	);
1051
1052	if ($updated === 1) {
1053	# We won the takeover. Make sure the value row exists.
1054	$wpdb->query(
1055	$wpdb->prepare(
1056	"INSERT IGNORE INTO {$wpdb->options} (option_name, option_value, autoload) VALUES (%s, %s, 'no')",
1057	$value_option,
1058	'1'
1059	)
1060	);
1061	return true;
1062	}
1063
1064	return false;
1065	}
1066
1067	/**
1068	* PERFORMANCE OPTIMIZATION: Generate all cache keys at once
1069	* Reduces redundant URL normalization and MD5 hashing
1070	*
1071	* @param string $url The page URL
1072	* @return array All cache keys ['transient', 'lock', 'stale', 'track']
1073	*/
1074	private function get_cache_keys($url) {
1075	# Normalize URL once (remove trailing slash, lowercase)
1076	$normalized = rtrim(strtolower($url), '/');
1077	# Compute MD5 hash once
1078	$hash = md5($normalized);
1079	# Get site ID once
1080	$site_id = is_multisite() ? get_current_blog_id() : 0;
1081
1082	# All keys scoped by $site_id so multisite subsites don't collide on a
1083	# shared Redis/Memcached object cache.
1084	return [
1085	'transient' => self::TRANSIENT_PREFIX . $site_id . '_' . $hash,
1086	'lock' => self::LOCK_PREFIX . $site_id . '_' . $hash,
1087	'stale' => self::STALE_PREFIX . $site_id . '_' . $hash,
1088	'track' => 'otto_' . $site_id . '_' . $hash,
1089	];
1090	}
1091
1092	/**
1093	* Get transient key for URL
1094	*
1095	* @param string $url The page URL
1096	* @return string Transient key
1097	*/
1098	private function get_transient_key($url) {
1099	$keys = $this->get_cache_keys($url);
1100	return $keys['transient'];
1101	}
1102
1103	/**
1104	* Get lock key for URL
1105	*
1106	* @param string $url The page URL
1107	* @return string Lock key
1108	*/
1109	private function get_lock_key($url) {
1110	$keys = $this->get_cache_keys($url);
1111	return $keys['lock'];
1112	}
1113
1114	/**
1115	* Get stale cache key for URL
1116	*
1117	* @param string $url The page URL
1118	* @return string Stale cache key
1119	*/
1120	private function get_stale_key($url) {
1121	$keys = $this->get_cache_keys($url);
1122	return $keys['stale'];
1123	}
1124
1125	/**
1126	* Get cache statistics for debugging
1127	*
1128	* @param string $url The page URL
1129	* @return array Statistics
1130	*/
1131	public function get_stats($url) {
1132	$transient_key = $this->get_transient_key($url);
1133	$cached = get_transient($transient_key);
1134
1135	# Match the scoped rate-limit key produced by can_make_api_call().
1136	$site_id = is_multisite() ? get_current_blog_id() : 0;
1137	$rate_limit_key = self::RATE_LIMIT_PREFIX . $site_id . '_' . date('Y-m-d-H-i');
1138
1139	return [
1140	'url' => $url,
1141	'has_cache' => $cached !== false,
1142	'has_suggestions' => $cached !== false && $this->has_payload($cached),
1143	'cache_key' => $transient_key,
1144	'rate_limit_key' => $rate_limit_key,
1145	'current_rate_count' => wp_using_ext_object_cache()
1146	? (int) wp_cache_get($rate_limit_key, 'otto_rate')
1147	: (int) get_transient($rate_limit_key),
1148	];
1149	}
1150
1151	/**
1152	* Clear all OTTO transient caches
1153	*
1154	* @return array Results with count of cleared items
1155	*/
1156	public static function clear_all_transients() {
1157	global $wpdb;
1158
1159	# PERFORMANCE OPTIMIZATION: Batch delete in single query instead of N+1 loop
1160	# Build the LIKE conditions for all prefixes
1161	$prefixes = [
1162	self::TRANSIENT_PREFIX,
1163	self::LOCK_PREFIX,
1164	self::STALE_PREFIX,
1165	self::RATE_LIMIT_PREFIX,
1166	# Render-failure report throttles. Without this they survive a full
1167	# OTTO cache clear, so a support-led "clear the cache" would leave
1168	# reporting suppressed for up to 5 more minutes per URL.
1169	'metasync_otto_fail_',
1170	# Circuit breaker state, for the same reason: clearing the cache is
1171	# what support does to make a site retry immediately, so it must also
1172	# un-stick the breaker rather than leaving OTTO off for another
1173	# cool-off. This one prefix covers the open marker, the failure tally
1174	# and the half-open probe lock.
1175	self::BREAKER_PREFIX,
1176	];
1177
1178	# Build WHERE clause for all transient and timeout variants
1179	$where_parts = [];
1180	foreach ($prefixes as $prefix) {
1181	$where_parts[] = $wpdb->prepare("option_name LIKE %s", '_transient_' . $prefix . '%');
1182	$where_parts[] = $wpdb->prepare("option_name LIKE %s", '_transient_timeout_' . $prefix . '%');
1183	}
1184	$where_clause = implode(' OR ', $where_parts);
1185
1186	# Count entries before deletion
1187	$count_query = "SELECT COUNT(*) FROM {$wpdb->options} WHERE " . $where_clause;
1188	$cleared_count = (int) $wpdb->get_var($count_query);
1189
1190	# Batch delete all matching transients in single query
1191	$delete_query = "DELETE FROM {$wpdb->options} WHERE " . $where_clause;
1192	$wpdb->query($delete_query);
1193
1194	# Also clear object cache if available
1195	if (function_exists('wp_cache_flush_group')) {
1196	wp_cache_flush_group('transient');
1197	# Counters live in their own group when an external object cache is
1198	# present, so the raw SQL above cannot reach them. Covers both the
1199	# rate limiter and the breaker's failure tally.
1200	wp_cache_flush_group(self::COUNTER_GROUP);
1201	}
1202
1203	delete_transient('metasync_otto_js_detected');
1204
1205	return [
1206	'success' => true,
1207	'cleared_count' => $cleared_count,
1208	'message' => sprintf('Cleared %d OTTO transient cache entries', $cleared_count)
1209	];
1210	}
1211
1212	/**
1213	* Clear transient cache for a specific URL
1214	*
1215	* @param string $url The page URL
1216	* @return array Results
1217	*/
1218	public static function clear_url_transient($url) {
1219	if (empty($url)) {
1220	return [
1221	'success' => false,
1222	'message' => 'URL is required'
1223	];
1224	}
1225
1226	# Normalize URL for transient key (matches get_transient_key)
1227	$normalized = rtrim(strtolower($url), '/');
1228	$site_id = is_multisite() ? get_current_blog_id() : 0;
1229
1230	# Use the same normalization as get_cache_keys() for all three keys.
1231	# Previously lock/stale used raw $url (no normalization), which generated
1232	# different hashes than the actual keys — they were never actually deleted.
1233	$hash = md5($normalized);
1234	# All three keys must include $site_id to match get_cache_keys().
1235	$transient_key = self::TRANSIENT_PREFIX . $site_id . '_' . $hash;
1236	$lock_key = self::LOCK_PREFIX . $site_id . '_' . $hash;
1237	$stale_key = self::STALE_PREFIX . $site_id . '_' . $hash;
1238
1239	$keys_to_clear = [$transient_key, $lock_key, $stale_key];
1240
1241	$cleared_count = 0;
1242	foreach ($keys_to_clear as $key) {
1243	if (delete_transient($key)) {
1244	$cleared_count++;
1245	}
1246	}
1247
1248	return [
1249	'success' => true,
1250	'cleared_count' => $cleared_count,
1251	'url' => $url,
1252	'message' => sprintf('Cleared cache for URL: %s (%d entries)', $url, $cleared_count)
1253	];
1254	}
1255
1256	/**
1257	* Get count of cached transients
1258	*
1259	* @return int Number of cached transients
1260	*/
1261	public static function get_cache_count() {
1262	global $wpdb;
1263
1264	$count = $wpdb->get_var(
1265	$wpdb->prepare(
1266	"SELECT COUNT(*) FROM {$wpdb->options}
1267	WHERE option_name LIKE %s",
1268	'_transient_' . self::TRANSIENT_PREFIX . '%'
1269	)
1270	);
1271
1272	return (int) $count;
1273	}
1274	}
1275
1276