' . "\n"; echo '' . "\n"; # adding the otto tag to pages $plugin_version = defined('METASYNC_VERSION') ? METASYNC_VERSION : 'unknown'; # OPTIMIZED: use cached uuid $otto_tag = ''; # out the otto tag echo $otto_tag; }, 1); # Priority 1 to output early in head /** * Scheduling seam for the OTTO pipeline. * * wp_schedule_single_event() returns false on failure and the old pipeline * ignored it, so a broken cron table acknowledged URLs that were never * queued. Every schedule in this pipeline goes through this seam so the * return value is honored (and tests can observe scheduling). */ function metasync_otto_schedule_single_event($timestamp, $hook, $args = array()) { return wp_schedule_single_event($timestamp, $hook, $args) === true; } /** * Dedupe seam: does this URL already have live per-URL work in flight? * * Combines the durable status store with the raw cron table so a re-delivered * webhook cannot stack a second job behind a live one. */ function metasync_otto_job_already_scheduled($route) { if (class_exists('Metasync_Otto_Job_Status', false) && Metasync_Otto_Job_Status::has_active($route)) { return true; } return wp_next_scheduled('metasync_process_otto_crawl_url_job', array($route)) !== false; } /** * Cron-table dedupe seam: does a per-URL cron event already exist for this URL? * * The pending-queue drainer must dedupe against the scheduler table only. * Every URL it takes off the durable queue carries a fresh webhook * accepted/batch_overflow ledger row; that row exists so a re-delivered * webhook cannot stack a second job behind live work, and consulting it * from the drainer — after take_pending() has already removed the URL * from the queue — silently dropped every overflow URL instead of * draining it. Only a real, still-pending cron event may veto here. */ function metasync_otto_job_event_pending($route) { return wp_next_scheduled('metasync_process_otto_crawl_url_job', array($route)) !== false; } /** * Start end point to handle requests on page updates * The Otto Crawler will call this end point once a page is updated **/ # function to register the route function metasync_otto_crawl_notify($request){ # get request data params $data = $request->get_json_params(); # fields $fields = ['domain', 'urls']; # validate the json request foreach ($fields as $key => $value) { # if the value is empty stop tehre if(empty($data[$value])){ # Handle the POST request return new WP_REST_Response(array( 'success' => false, 'message' => 'Invalid Field : '. $value, ), 400); } } # load otto pixel $otto_pixel = new Metasync_otto_pixel(false); # save the otto data first (cheap local DB write — kept synchronous) $otto_pixel->save_crawl_data($data); # Defer the expensive per-URL work (OTTO API fetch, post meta sync, # per-URL cache purge) to background jobs. The webhook caller enforces a # strict response-time budget, so the response acknowledges SCHEDULING, # never execution. The per-URL status store records what actually # happened to each URL once the jobs run. # # Cap the number of per-URL cron jobs scheduled per webhook batch so a # large crawl cannot overwhelm WP-Cron on shared hosts. URLs past the cap # are queued durably and worked off by a self-rescheduling drainer job — # never silently dropped. $max_jobs_per_batch = defined('METASYNC_MAX_JOBS_PER_BATCH') ? METASYNC_MAX_JOBS_PER_BATCH : 25; $now = time(); $routes_to_process = array(); $accepted = 0; $deferred = 0; $rejected = 0; $scheduled_this_batch = 0; foreach($data['urls'] AS $key => $url){ # prepare the route $route = $data['domain'] . $url; # validate the route $route = rtrim($route, '/'); # Resolve redirect table: use final destination URL before excluded/404 checks and OTTO processing $route = metasync_otto_resolve_redirect_to_final_url($route); # OTTO has confirmed this URL is crawlable. Remove any auto-exclusion that was # previously set (e.g. because url_to_postid() returned 0 for a custom post type # and metasync_otto_is_url_available() incorrectly treated it as a 404). # Manual exclusions (auto_excluded = 0) are left untouched. metasync_otto_remove_auto_exclusion($route); # Skip manually excluded URLs - don't queue OTTO processing for them if (metasync_is_otto_url_excluded($route)) { continue; } # Every valid URL belongs to the batched cache-purge set even when its # per-URL meta-sync job is deduped or deferred: the crawler hit all of # them, so the caches for all of them must be refreshed. $routes_to_process[] = $route; # Duplicate delivery: an active job already owns this URL. Idempotent # no-op — not re-scheduled, not queued, not counted as new work. if (metasync_otto_job_already_scheduled($route)) { continue; } if ($scheduled_this_batch >= $max_jobs_per_batch) { # Batch overflow: queue durably for the drainer instead of dropping. if (Metasync_Otto_Job_Status::enqueue_pending($route)) { Metasync_Otto_Job_Status::record($route, Metasync_Otto_Job_Status::STATE_ACCEPTED, 'queue', 0, 'batch_overflow'); $deferred++; } else { $rejected++; } continue; } # Queue the per-URL processing for background execution. The handler # (metasync_handle_otto_crawl_url_job) performs transient warming, # SEO meta sync, and per-URL host cache purge. # Offset each event by $key seconds to avoid wp_schedule_single_event() # silently dropping duplicates when timestamp + hook + args collide. # A false return means cron refused the event — fall back to the # durable queue rather than acknowledging a URL nobody will process. if (metasync_otto_schedule_single_event($now + $key, 'metasync_process_otto_crawl_url_job', array($route))) { Metasync_Otto_Job_Status::record($route, Metasync_Otto_Job_Status::STATE_ACCEPTED, 'queue', 0, ''); $accepted++; $scheduled_this_batch++; } elseif (Metasync_Otto_Job_Status::enqueue_pending($route)) { Metasync_Otto_Job_Status::record($route, Metasync_Otto_Job_Status::STATE_ACCEPTED, 'queue', 0, 'cron_refused'); $deferred++; } else { $rejected++; } } # Schedule a single batch job for the edge CDN purge. # The batch waits for per-URL stragglers (retries) before purging, so a # retried URL's fresh content is what reaches the edge. Multi-URL purge # in a single API call keeps edge-provider request counts low. if (!empty($routes_to_process)) { $batch_time = $now + count($data['urls']) + 5; // run after all per-URL jobs metasync_otto_schedule_single_event($batch_time, 'metasync_process_otto_batch_cache_job', array($routes_to_process)); } # Work off any overflow (or cron-refused URLs) from the durable queue. if ($deferred > 0) { metasync_otto_schedule_single_event($now + 120, 'metasync_otto_pending_drainer', array()); } # Return 200 immediately so the webhook caller does not time out, with an # honest accounting of what happened to the batch. return new WP_REST_Response(array( 'success' => true, 'message' => 'OTTO crawl notification received', 'accepted' => $accepted, 'deferred' => $deferred, 'rejected' => $rejected, ), 200); } /** * Background handler for a single OTTO crawl-notify URL. * * Pipeline per URL: * 1. Warm the OTTO transient cache — this is the ONE live API call the job * makes; the warmed suggestions are handed to the meta sync so the * endpoint is never fetched twice per URL. * 2. Sync SEO post meta (metasync_process_otto_seo_data) and branch on its * outcome: only SUCCESS / NO_CHANGE reach the purge step. * 3. Clear the per-URL host cache and re-warm it. * * Retryable outcomes (API timeout / 5xx / 429 / lock contention / CPU deferral) * retry with exponential backoff up to METASYNC_OTTO_JOB_MAX_RETRIES; the * exhaustion event is recorded via metasync_record_failed_action() for * Site Health. Permanent outcomes (not connected, excluded/404 URL, API auth * or input error) never retry and never purge. * * @param string $route Fully-qualified URL to process. * @param int $retry_count Current retry attempt (0 = first run). */ function metasync_handle_otto_crawl_url_job($route = '', $retry_count = 0) { // WP-Cron events persist independently of plugin code. A stale or malformed // event may therefore invoke this callback without its required route. // Docblock type isn't enforced at runtime; a stale cron record can pass a non-string. // @phpstan-ignore-next-line function.alreadyNarrowedType if (!is_string($route) || $route === '') { error_log('MetaSync OTTO: skipping crawl-url job without a valid route.'); return; } $max_retries = defined('METASYNC_OTTO_JOB_MAX_RETRIES') ? METASYNC_OTTO_JOB_MAX_RETRIES : 3; # Shared retry/terminal bookkeeping so every failure path stays consistent. $schedule_retry = function ($reason) use ($route, $retry_count, $max_retries) { if ($retry_count < $max_retries) { # Exponential backoff (60s, 120s, 240s) plus jitter so a batch of # failures cannot stampede the API on the same tick. $delay = 60 * pow(2, $retry_count) + wp_rand(0, 15); $scheduled = metasync_otto_schedule_single_event(time() + $delay, 'metasync_process_otto_crawl_url_job', array($route, $retry_count + 1)); if (!$scheduled && Metasync_Otto_Job_Status::enqueue_pending($route)) { Metasync_Otto_Job_Status::record($route, Metasync_Otto_Job_Status::STATE_ACCEPTED, 'queue', $retry_count + 1, $reason . '/cron_refused'); } else { Metasync_Otto_Job_Status::record($route, Metasync_Otto_Job_Status::STATE_RETRYING, 'job', $retry_count + 1, $reason); } error_log('MetaSync OTTO: retrying crawl-url job for ' . $route . ' (attempt ' . ($retry_count + 1) . '/' . $max_retries . ') reason=' . $reason . ' in ' . $delay . 's'); } else { metasync_record_failed_action('metasync_process_otto_crawl_url_job'); Metasync_Otto_Job_Status::record($route, Metasync_Otto_Job_Status::STATE_FAILED, 'job', $retry_count, 'exhausted:' . $reason); error_log('MetaSync OTTO: background crawl-url job permanently failed for ' . $route . ' after ' . $max_retries . ' retries, last reason=' . $reason); } }; try { # Not connected: retrying cannot help, and there is nothing to purge. $otto_uuid = Metasync_Otto_Config::get_otto_uuid(); # A corrupted option row can hold a non-string UUID; the docblock # contract alone does not enforce that at runtime. # @phpstan-ignore-next-line function.alreadyNarrowedType if (!is_string($otto_uuid) || $otto_uuid === '') { Metasync_Otto_Job_Status::record($route, Metasync_Otto_Job_Status::STATE_FAILED, 'job', $retry_count, 'not_connected'); return; } # Step 1: Warm OTTO transient cache (fetch fresh suggestions from OTTO # API into WP transient — the rate limiter, circuit breaker, and # request lock all live inside this call). $transient_cache = new Metasync_Otto_Transient_Cache($otto_uuid); $warmed = $transient_cache->warm_cache($route); if ($warmed === false) { # A 4xx refusal (bad key, revoked project) is permanent: retrying # cannot change the answer, so fail fast instead of burning the # attempt budget on warm calls that will keep being rejected. if ($transient_cache->last_failure_is_permanent()) { metasync_record_failed_action('metasync_process_otto_crawl_url_job'); Metasync_Otto_Job_Status::record($route, Metasync_Otto_Job_Status::STATE_FAILED, 'job', $retry_count, 'permanent:warm_rejected'); error_log('MetaSync OTTO: crawl-url job failed permanently for ' . $route . ' (warm request rejected by the API)'); return; } # Transient API failure (timeout / 5xx / breaker open). No meta # write, no purge — straight to the retry path. $schedule_retry('warm_failed'); return; } # Step 2: Write SEO post meta synchronously BEFORE the cache purge. # This ensures the DB is fully up-to-date when the host re-populates # the cache on the very next request. The warmed suggestions are # passed through so the sync makes NO second API call. # allow_defer=false: the crawl_url_job retry mechanism owns failure # handling here; letting the sync self-reschedule caused an # unbounded cron pile-up. # warm_cache() may hand back a non-array truthy on contract drift; # the sync treats that as "no prefetched suggestions". # @phpstan-ignore-next-line function.alreadyNarrowedType $prefetched = is_array($warmed) ? $warmed : null; $outcome = metasync_process_otto_seo_data($route, false, 0, $prefetched); if ($outcome === Metasync_Otto_Job_Status::OUTCOME_RETRYABLE || $outcome === Metasync_Otto_Job_Status::OUTCOME_DEFERRED) { $schedule_retry('outcome_' . $outcome); return; } if ($outcome !== Metasync_Otto_Job_Status::OUTCOME_SUCCESS && $outcome !== Metasync_Otto_Job_Status::OUTCOME_NO_CHANGE) { # Permanent failure: no retry, no purge — caches must keep serving # the last good content. Metasync_Otto_Job_Status::record($route, Metasync_Otto_Job_Status::STATE_FAILED, 'meta', $retry_count, 'permanent:' . $outcome); return; } # Step 3: meta is current (changed or confirmed unchanged) — clear the # per-URL host cache entry and re-warm it so OTTO-modified output is # what gets stored. $otto_pixel = new Metasync_otto_pixel(false); $otto_pixel->refresh_cache($route); Metasync_Cache_Purge::purge_single_url($route); Metasync_Cache_Purge::warm_urls(array($route)); # A URL that completed on a retry missed its batch's edge purge (the # batch only waits so long). Late completers purge the edge themselves. if ($retry_count > 0) { Metasync_Edge_Cache_Purge::purge(array($route)); } Metasync_Otto_Job_Status::record($route, Metasync_Otto_Job_Status::STATE_COMPLETED, 'job', $retry_count, $outcome); } catch (Exception $e) { $schedule_retry('exception:' . $e->getMessage()); } } /** * Batch handler for the edge CDN purge. * * Runs after all per-URL jobs have had time to complete. Per-URL host caches * are cleared and re-warmed by the individual jobs; this batched event only * handles the edge CDN purge, which benefits from multi-URL API calls * (Cloudflare, Fastly, Akamai, Sucuri, Sevella, etc.). * * URLs still retrying are stragglers: the batch waits (up to * METASYNC_OTTO_BATCH_MAX_WAITS reschedules 60s apart) so their freshly * synced content — not stale content — reaches the edge. URLs that complete * after the wait budget purge the edge themselves from the retry path. * * @param array $routes List of fully-qualified URLs in this batch. * @param int $wait_count How many times this batch has already waited. */ function metasync_handle_otto_batch_cache_job($routes = array(), $wait_count = 0) { // WP-Cron events persist independently of plugin code. A stale or malformed // event may therefore invoke this callback without its required route list. // Docblock type isn't enforced at runtime; a stale cron record can pass a non-array. // @phpstan-ignore-next-line function.alreadyNarrowedType if (!is_array($routes) || empty($routes)) { error_log('MetaSync OTTO: skipping batch cache job without valid routes.'); return; } $max_waits = defined('METASYNC_OTTO_BATCH_MAX_WAITS') ? METASYNC_OTTO_BATCH_MAX_WAITS : 3; // Docblock type isn't enforced at runtime; a stale cron record can pass a non-numeric. // @phpstan-ignore-next-line function.alreadyNarrowedType $wait_count = is_numeric($wait_count) ? (int) $wait_count : 0; # Split the batch: terminally-done URLs vs. URLs with work still in # flight. Unknown-state URLs (legacy events scheduled before the status # store existed, or deduped duplicates whose entry expired) count as done # — their per-URL job has either finished or never will, and both mean # the edge purge should not stall behind them. $ready = array(); $stragglers = 0; foreach ($routes as $route) { $entry = Metasync_Otto_Job_Status::get($route); $state = is_array($entry) && isset($entry['state']) ? $entry['state'] : null; if ($state === Metasync_Otto_Job_Status::STATE_RETRYING || $state === Metasync_Otto_Job_Status::STATE_ACCEPTED) { $stragglers++; } else { # completed / failed / unknown: nothing left to wait for. $ready[] = $route; } } # Still live work in the batch and wait budget left: hold the edge purge. if ($stragglers > 0 && $wait_count < $max_waits) { metasync_otto_schedule_single_event(time() + 60, 'metasync_process_otto_batch_cache_job', array($routes, $wait_count + 1)); return; } if (empty($ready)) { return; } try { # Purge edge CDN caches (Cloudflare, Fastly, Akamai, Sucuri, Sevalla, etc.) # Tag-based providers purge only the affected posts; full-flush providers fire once per batch. Metasync_Edge_Cache_Purge::purge($ready); } catch (Exception $e) { metasync_record_failed_action('metasync_process_otto_batch_cache_job'); error_log('MetaSync OTTO: batch cache job failed for ' . count($ready) . ' URLs: ' . $e->getMessage() . ' in ' . $e->getFile() . ':' . $e->getLine()); } } /** * Drainer: work off URLs that could not be scheduled inline (batch overflow * or a cron that refused the event) from the durable pending queue. * * Processes at most 25 URLs per run to stay friendly to WP-Cron, schedules * the same batched edge purge as the webhook path for the chunk it handled, * and reschedules itself while the queue is non-empty. URLs it takes off * the queue are scheduled unconditionally unless a real cron event already * exists for them — the webhook's overflow bookkeeping must never veto, * because by then the URL has already left the durable queue. */ function metasync_handle_otto_pending_drainer() { $chunk = Metasync_Otto_Job_Status::take_pending(25); if (empty($chunk)) { return; } $now = time(); $scheduled = 0; foreach ($chunk as $key => $route) { # This run owns the chunk: take_pending() already removed these URLs # from the durable queue, so skipping here would drop them. Only a # real cron event vetoes — the webhook's overflow ledger row must # not (see metasync_otto_job_event_pending). if (metasync_otto_job_event_pending($route)) { continue; } if (metasync_otto_schedule_single_event($now + $key, 'metasync_process_otto_crawl_url_job', array($route))) { Metasync_Otto_Job_Status::record($route, Metasync_Otto_Job_Status::STATE_ACCEPTED, 'drain', 0, ''); $scheduled++; } else { # Cron still refusing: put it back at the end of the queue. Metasync_Otto_Job_Status::enqueue_pending($route); } } if ($scheduled > 0) { metasync_otto_schedule_single_event($now + count($chunk) + 5, 'metasync_process_otto_batch_cache_job', array($chunk)); } if (Metasync_Otto_Job_Status::pending_count() > 0) { metasync_otto_schedule_single_event($now + 60, 'metasync_otto_pending_drainer', array()); } } # NOTE: Cache system removed - these functions are no longer needed # Kept for backward compatibility in case old cache directories need cleanup function metasync_deleteDir($dir) { if (!is_dir($dir)) { return false; } $files = array_diff(scandir($dir), array('.', '..')); foreach ($files as $file) { $filePath = $dir . DIRECTORY_SEPARATOR . $file; if (is_dir($filePath)) { metasync_deleteDir($filePath); } else { unlink($filePath); } } return rmdir($dir); } # Cleanup function for removing old cache directories (if they exist) function metasync_invalidate_all_caches($folder = ''){ # Cache system removed - this function only exists to clean up old cache directories if(!defined('WP_CONTENT_DIR')){ return false; } $wp_content_dir = WP_CONTENT_DIR; $cache_dir = $wp_content_dir . '/metasync_caches'; if(in_array($folder, ['posts', 'pages'])){ $cache_dir = $cache_dir . '/' . $folder; } if(is_dir($cache_dir)){ metasync_deleteDir($cache_dir); } } // metasync_is_custom_or_lps_page() now lives in includes/metasync-helpers.php // (loaded unconditionally before this file) so all SEO surfaces share one rule. /** * Detect an Elementor editor / preview request that OTTO must never process. * * Elementor renders its editing canvas in a front-end