' . "\n"; echo '' . "\n"; # adding the otto tag to pages $plugin_version = defined('METASYNC_VERSION') ? METASYNC_VERSION : 'unknown'; # OPTIMIZED: use cached uuid $otto_tag = ''; # out the otto tag echo $otto_tag; }, 1); # Priority 1 to output early in head /** * Start end point to handle requests on page updates * The Otto Crawler will call this end point once a page is updated **/ # function to register the route function metasync_otto_crawl_notify($request){ # get request data params $data = $request->get_json_params(); # fields $fields = ['domain', 'urls']; # validate the json request foreach ($fields as $key => $value) { # if the value is empty stop tehre if(empty($data[$value])){ # Handle the POST request return new WP_REST_Response(array( 'success' => false, 'message' => 'Invalid Field : '. $value, ), 400); } } # load otto pixel $otto_pixel = new Metasync_otto_pixel(false); # save the otto data first (cheap local DB write — kept synchronous) $otto_pixel->save_crawl_data($data); # Defer the expensive per-URL work (OTTO API fetch, post meta sync, # per-URL cache purge) to background jobs. The webhook caller enforces a # strict response-time budget; doing this work inline previously blocked # the response for up to 30s × N URLs and timed the caller out. # Each URL gets its own scheduled event so failures are isolated. # # Cap the number of per-URL cron jobs scheduled per webhook batch. # Large crawl batches (100+ URLs) previously created hundreds of cron events # that overwhelmed WP-Cron on shared/managed hosts like WP Engine. Excess # URLs beyond the cap are silently skipped — OTTO will re-crawl them on the # next webhook cycle. $max_jobs_per_batch = defined('METASYNC_MAX_JOBS_PER_BATCH') ? METASYNC_MAX_JOBS_PER_BATCH : 25; $now = time(); $routes_to_process = array(); $scheduled_count = 0; foreach($data['urls'] AS $key => $url){ # Enforce per-batch cap to prevent cron overload if ($scheduled_count >= $max_jobs_per_batch) { break; } # prepare the route $route = $data['domain'] . $url; # validate the route $route = rtrim($route, '/'); # Resolve redirect table: use final destination URL before excluded/404 checks and OTTO processing $route = metasync_otto_resolve_redirect_to_final_url($route); # OTTO has confirmed this URL is crawlable. Remove any auto-exclusion that was # previously set (e.g. because url_to_postid() returned 0 for a custom post type # and metasync_otto_is_url_available() incorrectly treated it as a 404). # Manual exclusions (auto_excluded = 0) are left untouched. metasync_otto_remove_auto_exclusion($route); # Skip manually excluded URLs - don't queue OTTO processing for them if (metasync_is_otto_url_excluded($route)) { continue; } # Queue the per-URL processing for background execution. The handler # (metasync_handle_otto_crawl_url_job) performs transient warming, # SEO meta sync, and per-URL host cache purge. # Offset each event by $key seconds to avoid wp_schedule_single_event() # silently dropping duplicates when timestamp + hook + args collide. wp_schedule_single_event($now + $key, 'metasync_process_otto_crawl_url_job', array($route)); $routes_to_process[] = $route; $scheduled_count++; } # Schedule a single batch job for cache warming and edge CDN purge. # These operations benefit from batching: warm_urls() uses concurrent # fire-and-forget requests, and edge providers (Cloudflare, Fastly) # support multi-URL purge in a single API call. if (!empty($routes_to_process)) { $batch_time = $now + count($data['urls']) + 5; // run after all per-URL jobs wp_schedule_single_event($batch_time, 'metasync_process_otto_batch_cache_job', array($routes_to_process)); } # Track OTTO optimization event in GA4 (local, non-blocking) try { Metasync_GA4::get_instance()->track_otto_optimization($data); } catch (Exception $e) { // Analytics tracking failed, continue } # Return 200 immediately so the webhook caller does not time out. return new WP_REST_Response(array( 'success' => true, 'message' => 'OTTO crawl notification received', ), 200); } /** * Background handler for a single OTTO crawl-notify URL. * * Runs the per-URL sequence that was previously inline in metasync_otto_crawl_notify(): * warm the OTTO transient cache, sync SEO post meta, and clear the per-URL host cache. * * Cache warming and edge CDN purge are handled separately by * metasync_handle_otto_batch_cache_job() in a single batched event, because * those operations benefit from multi-URL batching (fewer API calls). * * On failure, retries up to METASYNC_OTTO_JOB_MAX_RETRIES times with 60s backoff. * After all retries are exhausted the failure is recorded via * metasync_record_failed_action() for surfacing in Site Health. * * @param string $route Fully-qualified URL to process. * @param int $retry_count Current retry attempt (0 = first run). */ function metasync_handle_otto_crawl_url_job($route, $retry_count = 0) { try { # Step 1: Warm OTTO transient cache (fetch fresh suggestions from OTTO API into WP transient) $otto_uuid = Metasync_Otto_Config::get_otto_uuid(); if (!empty($otto_uuid)) { $transient_cache = new Metasync_Otto_Transient_Cache($otto_uuid); $transient_cache->warm_cache($route); } # Step 2: Write SEO post meta synchronously BEFORE the cache purge. # This ensures the DB is fully up-to-date when Kinsta (or any host) re-populates # the cache on the very next request — eliminating the race condition where the # 1-second scheduled job hadn't run yet and stale meta got cached. # allow_defer=false: do NOT reschedule a new metasync_process_seo_job cron # event from this synchronous path — the crawl_url_job retry mechanism already # handles failures, and rescheduling here caused an unbounded cron pile-up. metasync_process_otto_seo_data($route, false); # Step 3: Clear the per-URL cache entry $otto_pixel = new Metasync_otto_pixel(false); $otto_pixel->refresh_cache($route); Metasync_Cache_Purge::purge_single_url($route); } catch (Exception $e) { $max_retries = defined('METASYNC_OTTO_JOB_MAX_RETRIES') ? METASYNC_OTTO_JOB_MAX_RETRIES : 3; if ($retry_count < $max_retries) { # Schedule a retry with exponential backoff: 60s, 120s, 240s $delay = 60 * pow(2, $retry_count); wp_schedule_single_event(time() + $delay, 'metasync_process_otto_crawl_url_job', array($route, $retry_count + 1)); error_log('MetaSync OTTO: retrying crawl-url job for ' . $route . ' (attempt ' . ($retry_count + 1) . '/' . $max_retries . ') in ' . $delay . 's: ' . $e->getMessage()); } else { metasync_record_failed_action('metasync_process_otto_crawl_url_job'); error_log('MetaSync OTTO: background crawl-url job permanently failed for ' . $route . ' after ' . $max_retries . ' retries: ' . $e->getMessage() . ' in ' . $e->getFile() . ':' . $e->getLine()); } } } /** * Batch handler for cache warming and edge CDN purge. * * Runs after all per-URL jobs have had time to complete. Batching these * operations avoids N separate API calls to edge providers (Cloudflare * supports up to 30 tags per request) and lets warm_urls() issue concurrent * fire-and-forget requests. * * @param array $routes List of fully-qualified URLs to warm and purge. */ function metasync_handle_otto_batch_cache_job($routes) { try { # Re-populate the cache so OTTO-modified output is what gets stored. # Per-URL host cache was already cleared by individual jobs. # warm_urls() hits each URL with a non-blocking request so our code — # with fresh transients and post meta — is first to populate the host cache. Metasync_Cache_Purge::warm_urls($routes); # Purge edge CDN caches (Cloudflare, Fastly, Akamai, Sucuri, Sevalla, etc.) # Tag-based providers purge only the affected posts; full-flush providers fire once per batch. Metasync_Edge_Cache_Purge::purge($routes); } catch (Exception $e) { metasync_record_failed_action('metasync_process_otto_batch_cache_job'); error_log('MetaSync OTTO: batch cache job failed for ' . count($routes) . ' URLs: ' . $e->getMessage() . ' in ' . $e->getFile() . ':' . $e->getLine()); } } # NOTE: Cache system removed - these functions are no longer needed # Kept for backward compatibility in case old cache directories need cleanup function metasync_deleteDir($dir) { if (!is_dir($dir)) { return false; } $files = array_diff(scandir($dir), array('.', '..')); foreach ($files as $file) { $filePath = $dir . DIRECTORY_SEPARATOR . $file; if (is_dir($filePath)) { metasync_deleteDir($filePath); } else { unlink($filePath); } } return rmdir($dir); } # Cleanup function for removing old cache directories (if they exist) function metasync_invalidate_all_caches($folder = ''){ # Cache system removed - this function only exists to clean up old cache directories if(!defined('WP_CONTENT_DIR')){ return false; } $wp_content_dir = WP_CONTENT_DIR; $cache_dir = $wp_content_dir . '/metasync_caches'; if(in_array($folder, ['posts', 'pages'])){ $cache_dir = $cache_dir . '/' . $folder; } if(is_dir($cache_dir)){ metasync_deleteDir($cache_dir); } } // metasync_is_custom_or_lps_page() now lives in includes/metasync-helpers.php // (loaded unconditionally before this file) so all SEO surfaces share one rule. /** * Detect an Elementor editor / preview request that OTTO must never process. * * Elementor renders its editing canvas in a front-end