Product
3 days ago
Async_Request.php
2 years ago
Background_Job.php
3 days ago
Category.php
3 months ago
Connection.php
5 months ago
Email.php
1 year ago
Order.php
3 days ago
Order_Sync.php
11 months ago
Product.php
3 days ago
Products.php
1 month ago
Sync.php
3 years ago
Background_Job.php
755 lines
| 1 | <?php |
| 2 | /** |
| 3 | * WooCommerce Square |
| 4 | * |
| 5 | * This source file is subject to the GNU General Public License v3.0 |
| 6 | * that is bundled with this package in the file license.txt. |
| 7 | * It is also available through the world-wide-web at this URL: |
| 8 | * http://www.gnu.org/licenses/gpl-3.0.html GNU General Public License v3.0 or later |
| 9 | * If you did not receive a copy of the license and are unable to |
| 10 | * obtain it through the world-wide-web, please send an email |
| 11 | * to license@woocommerce.com so we can send you a copy immediately. |
| 12 | * |
| 13 | * DISCLAIMER |
| 14 | * |
| 15 | * Do not edit or add to this file if you wish to upgrade WooCommerce Square to newer |
| 16 | * versions in the future. If you wish to customize WooCommerce Square for your |
| 17 | * needs please refer to https://docs.woocommerce.com/document/woocommerce-square/ |
| 18 | * |
| 19 | */ |
| 20 | |
| 21 | namespace WooCommerce\Square\Handlers; |
| 22 | |
| 23 | use WooCommerce\Square\Framework\Utilities\Background_Job_Handler; |
| 24 | use WooCommerce\Square\Sync\Job; |
| 25 | use WooCommerce\Square\Sync\Records; |
| 26 | use WooCommerce\Square\Sync\Interval_Polling; |
| 27 | use WooCommerce\Square\Sync\Manual_Synchronization; |
| 28 | use WooCommerce\Square\Sync\Product_Import; |
| 29 | |
| 30 | defined( 'ABSPATH' ) || exit; |
| 31 | |
| 32 | /** |
| 33 | * Product and Inventory Synchronization handler class. |
| 34 | * |
| 35 | * This class handles manual and interval synchronization jobs. |
| 36 | * It is a wrapper for the framework background handler and as such it only handles loopback business to keep the queue processing. |
| 37 | * See the individual job implementations: |
| 38 | * |
| 39 | * @see Manual_Synchronization manual jobs re-process ALL synced products |
| 40 | * @see Interval_Polling interval (polling) jobs perform API requests for ONLY the latest changes and update the associated products |
| 41 | * |
| 42 | * @since 2.0.0 |
| 43 | */ |
| 44 | class Background_Job extends Background_Job_Handler { |
| 45 | |
| 46 | |
| 47 | /** |
| 48 | * Initializes the background sync handler. |
| 49 | * |
| 50 | * @since 2.0.0 |
| 51 | */ |
| 52 | public function __construct() { |
| 53 | |
| 54 | $this->prefix = 'wc_square'; |
| 55 | $this->action = 'background_sync'; |
| 56 | $this->data_key = 'product_ids'; |
| 57 | |
| 58 | parent::__construct(); |
| 59 | |
| 60 | add_action( "{$this->identifier}_job_complete", array( $this, 'job_complete' ) ); |
| 61 | add_action( "{$this->identifier}_job_failed", array( $this, 'job_failed' ) ); |
| 62 | add_filter( 'woocommerce_debug_tools', array( $this, 'add_debug_tool' ) ); |
| 63 | add_action( 'wc_square_job_runner', array( $this, 'handle' ) ); |
| 64 | |
| 65 | // Sync healthcheck |
| 66 | add_action( $this->cron_hook_identifier, array( $this, 'handle_sync_healthcheck' ) ); |
| 67 | |
| 68 | // Safety net for sites where cron/Action Scheduler execution is broken (the reported |
| 69 | // incident had the healthcheck actions themselves failing for a month): any admin page |
| 70 | // load can also detect and recover a stalled sync. Throttled internally. |
| 71 | add_action( 'admin_init', array( $this, 'maybe_recover_stuck_sync_from_admin' ) ); |
| 72 | } |
| 73 | |
| 74 | |
| 75 | /** |
| 76 | * Creates a new job. |
| 77 | * |
| 78 | * @since 2.0.0 |
| 79 | * |
| 80 | * @param array $attrs array of job attributes |
| 81 | * @return \stdClass|null |
| 82 | */ |
| 83 | public function create_job( $attrs ) { |
| 84 | |
| 85 | $sor = wc_square()->get_settings_handler()->get_system_of_record(); |
| 86 | |
| 87 | return parent::create_job( |
| 88 | wp_parse_args( |
| 89 | $attrs, |
| 90 | array( |
| 91 | 'action' => '', // job action |
| 92 | 'catalog_processed' => false, // whether the Square catalog has been processed |
| 93 | 'cursor' => '', // job advancement position |
| 94 | 'manual' => false, // whether it's a sync job triggered manually |
| 95 | 'percentage' => 0, // percentage completed |
| 96 | 'product_ids' => array(), // products to process |
| 97 | 'processed_product_ids' => array(), // newly imported products processed |
| 98 | 'updated_product_ids' => array(), // updated products processed |
| 99 | 'skipped_products' => array(), // remote product IDs that were skipped |
| 100 | 'system_of_record' => $sor, // Sync setting used |
| 101 | ) |
| 102 | ) |
| 103 | ); |
| 104 | } |
| 105 | |
| 106 | |
| 107 | /** |
| 108 | * Handles job execution. |
| 109 | * |
| 110 | * Overridden to support our multi-step job structure. There are steps that can take a long time to process, so this |
| 111 | * ensures only one step is performed for each background request. |
| 112 | * |
| 113 | * @since 2.0.0 |
| 114 | */ |
| 115 | public function handle() { |
| 116 | |
| 117 | // Schedule sync healthcheck event if not already scheduled. |
| 118 | $this->schedule_event(); |
| 119 | |
| 120 | $this->lock_process(); |
| 121 | |
| 122 | // Get next job in the queue |
| 123 | $job = $this->get_job(); |
| 124 | |
| 125 | // handle PHP errors from here on out |
| 126 | register_shutdown_function( array( $this, 'handle_shutdown' ), $job ); |
| 127 | |
| 128 | // Start processing |
| 129 | $this->process_job( $job ); |
| 130 | |
| 131 | $this->unlock_process(); |
| 132 | |
| 133 | // Start next job or complete process |
| 134 | if ( ! $this->is_queue_empty() ) { |
| 135 | // If the job has a retry count set, we'll retry the job after a delay. |
| 136 | if ( isset( $job->retry ) && is_numeric( $job->retry ) && $job->retry > 0 ) { |
| 137 | $base_delay = 30; // Base delay in seconds for rate limit errors. 30 seconds. |
| 138 | $delay = $base_delay * ( pow( 2, $job->retry ) ); |
| 139 | wc_square()->log( "Retrying in {$delay} seconds." ); |
| 140 | as_schedule_single_action( time() + $delay, 'wc_square_job_runner' ); |
| 141 | } else { |
| 142 | as_enqueue_async_action( 'wc_square_job_runner' ); |
| 143 | } |
| 144 | } else { |
| 145 | $this->complete(); |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | |
| 150 | /** |
| 151 | * Processes a background job. |
| 152 | * |
| 153 | * @since 2.0.0 |
| 154 | * |
| 155 | * @param object|\stdClass $job |
| 156 | * @param null $items_per_batch |
| 157 | * @return false|object|\stdClass |
| 158 | */ |
| 159 | public function process_job( $job, $items_per_batch = null ) { |
| 160 | |
| 161 | if ( ! $job ) { |
| 162 | return; |
| 163 | } |
| 164 | |
| 165 | // indicate that the job has started processing |
| 166 | if ( 'processing' !== $job->status ) { |
| 167 | |
| 168 | $job->status = 'processing'; |
| 169 | $job->started_processing_at = current_time( 'mysql' ); |
| 170 | |
| 171 | // A sync the merchant started has taken over, so the recovery notice has served its |
| 172 | // purpose. Clearing it here rather than on completion means a long sync does not keep |
| 173 | // showing a warning about the previous one for hours. Interval poll jobs are excluded: |
| 174 | // they start on their own every few minutes and would dismiss the notice before anyone |
| 175 | // had a chance to read it. |
| 176 | if ( 'poll' !== ( $job->action ?? '' ) ) { |
| 177 | delete_option( 'wc_square_sync_auto_recovered_at' ); |
| 178 | } |
| 179 | |
| 180 | $this->update_job( $job ); |
| 181 | |
| 182 | // Confirm the row still exists rather than trusting update_job(), which returns the |
| 183 | // supplied object even when the option has been removed concurrently (the Clear Square |
| 184 | // Sync tool). The object itself is deliberately NOT replaced with a fresh read: the |
| 185 | // shutdown handler registered in handle() holds this instance, and swapping it would |
| 186 | // leave that handler writing a stale snapshot after a fatal. |
| 187 | if ( ! $this->job_exists( $job->id ) ) { |
| 188 | return; |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | if ( 'poll' === $job->action ) { |
| 193 | |
| 194 | $job = new Interval_Polling( $job ); |
| 195 | |
| 196 | } elseif ( 'product_import' === $job->action ) { |
| 197 | |
| 198 | $job = new Product_Import( $job ); |
| 199 | |
| 200 | } elseif ( ! empty( $job->manual ) ) { |
| 201 | |
| 202 | $job = new Manual_Synchronization( $job ); |
| 203 | } |
| 204 | |
| 205 | if ( $job instanceof Job ) { |
| 206 | $current_user_id = get_current_user_id(); |
| 207 | $job = $job->run(); |
| 208 | wp_set_current_user( $current_user_id ); // phpcs:ignore Generic.PHP.ForbiddenFunctions.Discouraged -- required for background job processing |
| 209 | } |
| 210 | |
| 211 | // Heartbeat: recorded only after the step has finished, never at the start of an attempt. |
| 212 | // A job that keeps dying mid step (for example an action scheduler timeout loop) must not |
| 213 | // refresh its own heartbeat on every retry, or it would never look stalled and never be |
| 214 | // recovered. started_processing_at is stamped once, so it cannot tell a slow large catalog |
| 215 | // sync apart from a stuck one; a per completed step heartbeat can. |
| 216 | if ( $job && 'processing' === ( $job->status ?? '' ) ) { |
| 217 | $job->last_activity_at = time(); |
| 218 | $job = $this->update_job( $job ); |
| 219 | } |
| 220 | |
| 221 | return $job; |
| 222 | } |
| 223 | |
| 224 | |
| 225 | /** |
| 226 | * Checks whether a job row still exists. |
| 227 | * |
| 228 | * Used instead of re-reading the job, because callers hold an object that other code (including |
| 229 | * the shutdown handler registered in handle()) keeps mutating, and replacing it would strand |
| 230 | * those references. |
| 231 | * |
| 232 | * @since 5.5.0 |
| 233 | * |
| 234 | * @param string $job_id job ID |
| 235 | * @return bool |
| 236 | */ |
| 237 | protected function job_exists( $job_id ) { |
| 238 | global $wpdb; |
| 239 | |
| 240 | if ( ! $job_id ) { |
| 241 | return false; |
| 242 | } |
| 243 | |
| 244 | // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching |
| 245 | return (bool) $wpdb->get_var( $wpdb->prepare( "SELECT option_id FROM {$wpdb->options} WHERE option_name = %s LIMIT 1", $this->identifier . '_job_' . $job_id ) ); |
| 246 | } |
| 247 | |
| 248 | |
| 249 | /** |
| 250 | * Handles actions after a sync job is complete. |
| 251 | * |
| 252 | * @since 2.0.0 |
| 253 | * |
| 254 | * @param $job |
| 255 | */ |
| 256 | public function job_complete( $job ) { |
| 257 | |
| 258 | // Normally cleared when the sync started; repeated here for a job that was already running |
| 259 | // when this release was installed. Interval polls are excluded for the same reason as at the |
| 260 | // start: they run on their own every few minutes and would dismiss the notice before anyone |
| 261 | // had a chance to read it. |
| 262 | if ( 'poll' !== ( $job->action ?? '' ) ) { |
| 263 | delete_option( 'wc_square_sync_auto_recovered_at' ); |
| 264 | } |
| 265 | |
| 266 | wc_square()->get_sync_handler()->set_last_synced_at(); |
| 267 | |
| 268 | wc_square()->get_sync_handler()->record_sync( $job->processed_product_ids, $job ); |
| 269 | |
| 270 | wc_square()->get_email_handler()->get_sync_completed_email()->trigger( $job ); |
| 271 | } |
| 272 | |
| 273 | |
| 274 | /** |
| 275 | * Handles actions after a sync job has failed. |
| 276 | * |
| 277 | * @since 2.0.0 |
| 278 | * |
| 279 | * @param $job |
| 280 | */ |
| 281 | public function job_failed( $job ) { |
| 282 | |
| 283 | $message = empty( $job->auto_failed ) |
| 284 | ? __( 'Sync failed. Please try again', 'woocommerce-square' ) |
| 285 | : __( 'A sync stopped responding and was stopped automatically. Product data may be out of date, please start a new sync.', 'woocommerce-square' ); |
| 286 | |
| 287 | Records::set_record( |
| 288 | array( |
| 289 | 'type' => 'failed', |
| 290 | 'message' => $message, |
| 291 | ) |
| 292 | ); |
| 293 | |
| 294 | wc_square()->get_email_handler()->get_sync_completed_email()->trigger( $job ); |
| 295 | } |
| 296 | |
| 297 | |
| 298 | /** |
| 299 | * No-op: implements framework parent abstract method. |
| 300 | * |
| 301 | * @since 2.0.0 |
| 302 | * |
| 303 | * @param null $item |
| 304 | * @param \stdClass $job |
| 305 | */ |
| 306 | protected function process_item( $item, $job ) {} |
| 307 | |
| 308 | /** |
| 309 | * Adds some helpful debug tools. |
| 310 | * |
| 311 | * @since 2.0.0 |
| 312 | * |
| 313 | * @param array $tools existing debug tools |
| 314 | * @return array |
| 315 | */ |
| 316 | public function add_debug_tool( $tools ) { |
| 317 | |
| 318 | // this key is not unique to the plugin to avoid duplicate tools |
| 319 | $tools['wc_square_clear_background_jobs'] = array( |
| 320 | 'name' => __( 'Clear Square Sync', 'woocommerce-square' ), |
| 321 | 'button' => __( 'Clear', 'woocommerce-square' ), |
| 322 | 'desc' => __( 'This tool will clear any ongoing Square product syncs.', 'woocommerce-square' ), |
| 323 | 'callback' => array( $this, 'run_clear_background_jobs' ), |
| 324 | ); |
| 325 | |
| 326 | return $tools; |
| 327 | } |
| 328 | |
| 329 | |
| 330 | /** |
| 331 | * Clear all background jobs of any status. |
| 332 | * |
| 333 | * @since 2.0.0 |
| 334 | */ |
| 335 | public function clear_all_jobs() { |
| 336 | |
| 337 | $jobs = $this->get_jobs(); |
| 338 | |
| 339 | if ( is_array( $jobs ) ) { |
| 340 | $this->delete_jobs( $jobs ); |
| 341 | } |
| 342 | |
| 343 | delete_transient( 'wc_square_background_sync_process_lock' ); |
| 344 | } |
| 345 | |
| 346 | |
| 347 | /** |
| 348 | * Deletes a set of background jobs. |
| 349 | * |
| 350 | * @since 2.0.0 |
| 351 | * |
| 352 | * @param object[] $jobs jobs to delete |
| 353 | */ |
| 354 | public function delete_jobs( $jobs ) { |
| 355 | |
| 356 | foreach ( $jobs as $job ) { |
| 357 | $this->delete_job( $job ); |
| 358 | } |
| 359 | } |
| 360 | |
| 361 | /** |
| 362 | * Runs the "Clear Square Sync" tool. |
| 363 | * |
| 364 | * Provides a way for merchants to clear any ongoing or stuck product syncs. |
| 365 | * |
| 366 | * @since 2.0.0 |
| 367 | */ |
| 368 | public function run_clear_background_jobs() { |
| 369 | |
| 370 | $this->clear_all_jobs(); |
| 371 | |
| 372 | $this->debug_message = esc_html__( 'Success! You can now sync your products.', 'woocommerce-square' ); |
| 373 | |
| 374 | return true; |
| 375 | } |
| 376 | |
| 377 | /** |
| 378 | * Runs the stalled-sync recovery check from admin page loads, throttled. |
| 379 | * |
| 380 | * The scheduled healthcheck is the primary trigger, but on sites where cron or Action |
| 381 | * Scheduler execution is broken (as in the reported incident, where the healthcheck actions |
| 382 | * themselves failed for a month) it never runs. Admin page loads are the one context such a |
| 383 | * site still exercises, so use them as a fallback trigger. Throttled to once per five minutes |
| 384 | * and restricted to users who can manage WooCommerce. |
| 385 | * |
| 386 | * @since 5.5.0 |
| 387 | */ |
| 388 | public function maybe_recover_stuck_sync_from_admin() { |
| 389 | |
| 390 | if ( ! current_user_can( 'manage_woocommerce' ) ) { // phpcs:ignore WordPress.WP.Capabilities.Unknown |
| 391 | return; |
| 392 | } |
| 393 | |
| 394 | if ( get_transient( 'wc_square_admin_recovery_check' ) ) { |
| 395 | return; |
| 396 | } |
| 397 | set_transient( 'wc_square_admin_recovery_check', 1, 5 * MINUTE_IN_SECONDS ); |
| 398 | |
| 399 | // Deliberately NOT the full healthcheck. Its tail enqueues a job runner whenever the queue is |
| 400 | // non empty and Action Scheduler has nothing scheduled, and handle() takes the process lock |
| 401 | // without checking it first, so making every admin page load a third enqueue trigger would |
| 402 | // widen the window for two runners to process the same step and push the same objects twice. |
| 403 | // Recovery and housekeeping are safe here; the queue is only restarted when this call |
| 404 | // actually failed a stalled job, which is the case where nothing else will restart it. |
| 405 | $recovered = $this->maybe_recover_stuck_sync(); |
| 406 | |
| 407 | $this->cleanup_stale_failed_actions(); |
| 408 | |
| 409 | if ( $recovered && ! $this->is_queue_empty() && ! $this->has_pending_job_runner() ) { |
| 410 | as_enqueue_async_action( 'wc_square_job_runner' ); |
| 411 | } |
| 412 | } |
| 413 | |
| 414 | /** |
| 415 | * Detects a sync job stalled in "processing" and recovers it so the queue can resume. |
| 416 | * |
| 417 | * A job is considered stalled when it has been in "processing" without any step activity for |
| 418 | * longer than a filterable threshold (measured against the per-step heartbeat, so a legitimately |
| 419 | * long sync is not affected). Recovery marks the job failed and releases the process lock; |
| 420 | * scheduled wc_square_job_runner actions are left alone so other queued sync jobs keep |
| 421 | * processing. A flag is stored so the admin notice can prompt a re-run. |
| 422 | * |
| 423 | * @since 5.5.0 |
| 424 | * |
| 425 | * @return bool whether a stalled job was failed by this call |
| 426 | */ |
| 427 | protected function maybe_recover_stuck_sync() { |
| 428 | |
| 429 | // Ask for processing jobs specifically. get_job() returns the oldest queued OR processing row, |
| 430 | // so an older queued job would otherwise hide a newer stalled one from this check entirely. |
| 431 | // ASC because get_jobs() defaults to DESC and the job blocking the queue is the oldest one. |
| 432 | $processing = $this->get_jobs( |
| 433 | array( |
| 434 | 'status' => 'processing', |
| 435 | 'order' => 'ASC', |
| 436 | 'orderby' => 'option_id', |
| 437 | ) |
| 438 | ); |
| 439 | $job = is_array( $processing ) ? reset( $processing ) : null; |
| 440 | |
| 441 | if ( ! $job || ! isset( $job->status ) || 'processing' !== $job->status ) { |
| 442 | $this->clear_recovery_grace(); |
| 443 | return false; |
| 444 | } |
| 445 | |
| 446 | /** |
| 447 | * Filters how long (in seconds) a sync job may sit in "processing" without any step activity |
| 448 | * before it is treated as stalled and automatically recovered. |
| 449 | * |
| 450 | * @since 5.5.0 |
| 451 | * |
| 452 | * Note when lowering this: Action Scheduler stamps a runner action's last attempt time once, |
| 453 | * when the worker picks it up, and does not refresh it while the step runs. A threshold below |
| 454 | * the longest single step therefore reads a live worker as stale, and with no pending action |
| 455 | * to earn a grace window that job would be failed while it is still working. The 15 minute |
| 456 | * default sits well above any step this plugin runs. |
| 457 | * |
| 458 | * @param int $threshold threshold in seconds (default 15 minutes) |
| 459 | */ |
| 460 | $threshold = (int) apply_filters( 'wc_square_stuck_job_threshold', 15 * MINUTE_IN_SECONDS ); |
| 461 | |
| 462 | $is_stalled = function ( $job ) use ( $threshold ) { |
| 463 | if ( ! $job || 'processing' !== ( $job->status ?? '' ) ) { |
| 464 | return false; |
| 465 | } |
| 466 | // Prefer the per-step heartbeat; fall back to the one-time start stamp for jobs created |
| 467 | // before this change shipped. started_processing_at is a site-local mysql string, so |
| 468 | // convert to GMT before comparing against the UTC epoch from time(). |
| 469 | if ( ! empty( $job->last_activity_at ) ) { |
| 470 | $reference = (int) $job->last_activity_at; |
| 471 | } elseif ( ! empty( $job->started_processing_at ) ) { |
| 472 | $reference = (int) strtotime( get_gmt_from_date( $job->started_processing_at ) ); |
| 473 | } else { |
| 474 | return false; |
| 475 | } |
| 476 | |
| 477 | return $reference > 0 && ( time() - $reference ) > $threshold; |
| 478 | }; |
| 479 | |
| 480 | if ( ! $is_stalled( $job ) ) { |
| 481 | $this->clear_recovery_grace(); |
| 482 | return false; |
| 483 | } |
| 484 | |
| 485 | // A live worker still holds the process lock: the job is progressing, not stuck. Leave it. |
| 486 | if ( $this->is_process_running() ) { |
| 487 | return false; |
| 488 | } |
| 489 | |
| 490 | // A runner action was touched recently, so a worker is still on it. The process lock only |
| 491 | // lasts 60 seconds, so a single long step outlives it and would otherwise look abandoned. |
| 492 | // |
| 493 | // The recency bound is essential rather than cosmetic: an in progress row is only cleared by |
| 494 | // Action Scheduler's own cleaner, which runs from its queue runner, so on a site where Action |
| 495 | // Scheduler is not executing (the incident this recovery exists for) a worker killed mid |
| 496 | // action leaves that row in progress forever. Without the bound this guard would then block |
| 497 | // recovery permanently and rebuild the same deadlock in a new shape. |
| 498 | if ( function_exists( 'as_get_scheduled_actions' ) && function_exists( 'as_get_datetime_object' ) && class_exists( 'ActionScheduler_Store' ) ) { |
| 499 | $running = as_get_scheduled_actions( |
| 500 | array( |
| 501 | 'hook' => 'wc_square_job_runner', |
| 502 | 'status' => \ActionScheduler_Store::STATUS_RUNNING, |
| 503 | 'modified' => as_get_datetime_object( $threshold . ' seconds ago' ), |
| 504 | 'modified_compare' => '>', |
| 505 | 'per_page' => 1, |
| 506 | 'orderby' => 'none', |
| 507 | ), |
| 508 | 'ids' |
| 509 | ); |
| 510 | |
| 511 | if ( ! empty( $running ) ) { |
| 512 | return false; |
| 513 | } |
| 514 | } |
| 515 | |
| 516 | // A runner action is still queued: the queue may be paused, not dead (low traffic sites can |
| 517 | // go quiet long enough for the threshold to pass, then resume on the visit that triggered |
| 518 | // this very check). Give the queue one grace window to make progress; recover only if the |
| 519 | // job is still stalled with the same queued action after that window. |
| 520 | // A pending action means the queue is waiting its turn rather than dead, so it earns a grace |
| 521 | // window. An in progress row does not count: see has_pending_job_runner(). |
| 522 | if ( $this->has_pending_job_runner() ) { |
| 523 | |
| 524 | /** |
| 525 | * Filters how long a stalled sync job is given to resume when a job runner action is |
| 526 | * still queued, before it is treated as dead. |
| 527 | * |
| 528 | * A queued action means the queue may simply be paused rather than broken, and the |
| 529 | * request that runs this check usually gives Action Scheduler its chance to run, so this |
| 530 | * only needs to be long enough for that to happen. It is deliberately shorter than the |
| 531 | * stall threshold: with both at their defaults a paused queue is left alone for 15 |
| 532 | * minutes and a genuinely dead one is failed after 20, not 30. |
| 533 | * |
| 534 | * @since 5.5.0 |
| 535 | * |
| 536 | * @param int $grace_period grace period in seconds (default a third of the stall threshold) |
| 537 | */ |
| 538 | $grace_period = max( MINUTE_IN_SECONDS, (int) apply_filters( 'wc_square_stuck_job_grace_period', (int) round( $threshold / 3 ) ) ); |
| 539 | |
| 540 | // The grace marker is scoped to the job it was started for. A global timestamp could |
| 541 | // outlive its job (the Clear Square Sync tool, or a job that simply finished) and the next |
| 542 | // stall would then read an ancient timestamp, skip the grace window entirely, and fail a |
| 543 | // merely paused queue on first detection. |
| 544 | $grace = (array) get_option( 'wc_square_recovery_grace', array() ); |
| 545 | $grace_started = (int) ( $grace['at'] ?? 0 ); |
| 546 | |
| 547 | if ( ! $grace_started || ( $grace['job'] ?? '' ) !== $job->id ) { |
| 548 | update_option( |
| 549 | 'wc_square_recovery_grace', |
| 550 | array( |
| 551 | 'job' => $job->id, |
| 552 | 'at' => time(), |
| 553 | ), |
| 554 | false |
| 555 | ); |
| 556 | wc_square()->log( sprintf( 'Stalled sync has a queued runner action; allowing %d seconds for the queue to resume before auto-failing.', $grace_period ) ); |
| 557 | return false; |
| 558 | } |
| 559 | |
| 560 | if ( ( time() - $grace_started ) < $grace_period ) { |
| 561 | return false; |
| 562 | } |
| 563 | } |
| 564 | |
| 565 | // Re-read immediately before acting: a concurrent step may have advanced the job in the |
| 566 | // window since get_job() above, in which case we must not clobber it with a stale snapshot. |
| 567 | $job = $this->get_job( $job->id ); |
| 568 | if ( ! $is_stalled( $job ) ) { |
| 569 | $this->clear_recovery_grace(); |
| 570 | return false; |
| 571 | } |
| 572 | |
| 573 | // Recover: fail the stalled job and release the lock. Scheduled job_runner actions are |
| 574 | // deliberately left in place so any other sync job waiting in the queue keeps processing; |
| 575 | // the failed job no longer comes back from get_job(), so the next run picks up the rest. |
| 576 | // Flagged so job_failed() can record why this job failed instead of the generic message. |
| 577 | $job->auto_failed = true; |
| 578 | |
| 579 | $this->fail_job( $job, __( 'Sync job stalled and was automatically marked as failed.', 'woocommerce-square' ) ); |
| 580 | $this->unlock_process(); |
| 581 | $this->clear_recovery_grace(); |
| 582 | |
| 583 | // Recorded so the admin notice can tell the merchant a stalled sync was stopped. |
| 584 | update_option( 'wc_square_sync_auto_recovered_at', time() ); |
| 585 | |
| 586 | wc_square()->log( 'Auto-failed a stalled sync job (' . ( isset( $job->id ) ? $job->id : 'unknown' ) . '). The queue lock was released so the next sync can run.' ); |
| 587 | |
| 588 | return true; |
| 589 | } |
| 590 | |
| 591 | |
| 592 | /** |
| 593 | * Checks whether a job runner action is waiting to run. |
| 594 | * |
| 595 | * Deliberately not as_has_scheduled_action() or as_next_scheduled_action(): both also report true |
| 596 | * for an action that is in progress, and an in progress row can outlive its worker indefinitely |
| 597 | * because only Action Scheduler's own cleaner clears it. Treating that as "the queue will run" |
| 598 | * would both grant a dead queue a grace window and stop the queue ever being restarted. |
| 599 | * |
| 600 | * @since 5.5.0 |
| 601 | * |
| 602 | * @return bool |
| 603 | */ |
| 604 | protected function has_pending_job_runner() { |
| 605 | |
| 606 | if ( ! function_exists( 'as_get_scheduled_actions' ) || ! class_exists( 'ActionScheduler_Store' ) ) { |
| 607 | return false; |
| 608 | } |
| 609 | |
| 610 | $pending = as_get_scheduled_actions( |
| 611 | array( |
| 612 | 'hook' => 'wc_square_job_runner', |
| 613 | 'status' => \ActionScheduler_Store::STATUS_PENDING, |
| 614 | 'per_page' => 1, |
| 615 | 'orderby' => 'none', |
| 616 | ), |
| 617 | 'ids' |
| 618 | ); |
| 619 | |
| 620 | return ! empty( $pending ); |
| 621 | } |
| 622 | |
| 623 | |
| 624 | /** |
| 625 | * Clears the stalled sync grace marker. |
| 626 | * |
| 627 | * @since 5.5.0 |
| 628 | */ |
| 629 | protected function clear_recovery_grace() { |
| 630 | |
| 631 | if ( get_option( 'wc_square_recovery_grace', false ) ) { |
| 632 | delete_option( 'wc_square_recovery_grace' ); |
| 633 | } |
| 634 | } |
| 635 | |
| 636 | /** |
| 637 | * Deletes stale failed Square Action Scheduler actions to prevent table bloat. |
| 638 | * |
| 639 | * Runs at most once per day. Removes actions for the plugin's sync hooks that are in the failed |
| 640 | * state and older than a filterable retention window. Never touches pending or in-progress |
| 641 | * actions, and does not change Action Scheduler's own timeout handling. |
| 642 | * |
| 643 | * @since 5.5.0 |
| 644 | */ |
| 645 | protected function cleanup_stale_failed_actions() { |
| 646 | |
| 647 | $last_run = (int) get_option( 'wc_square_failed_action_cleanup_at', 0 ); |
| 648 | if ( $last_run && ( time() - $last_run ) < DAY_IN_SECONDS ) { |
| 649 | return; |
| 650 | } |
| 651 | update_option( 'wc_square_failed_action_cleanup_at', time(), false ); |
| 652 | |
| 653 | if ( ! function_exists( 'as_get_scheduled_actions' ) || ! class_exists( 'ActionScheduler_Store' ) ) { |
| 654 | return; |
| 655 | } |
| 656 | |
| 657 | /** |
| 658 | * Filters how many days a failed Square action is retained before automatic cleanup. |
| 659 | * |
| 660 | * @since 5.5.0 |
| 661 | * |
| 662 | * @param int $days retention in days (default 30) |
| 663 | */ |
| 664 | $retention_days = max( 1, (int) apply_filters( 'wc_square_failed_action_retention_days', 30 ) ); |
| 665 | $cutoff = gmdate( 'Y-m-d H:i:s', time() - ( $retention_days * DAY_IN_SECONDS ) ); |
| 666 | |
| 667 | $hooks = array( 'wc_square_job_runner', 'wc_square_background_sync_cron', 'wc_square_sync', 'wc_square_sync_orders' ); |
| 668 | |
| 669 | try { |
| 670 | $store = \ActionScheduler_Store::instance(); |
| 671 | |
| 672 | /** |
| 673 | * Filters how many failed actions are deleted per cleanup batch. |
| 674 | * |
| 675 | * @since 5.5.0 |
| 676 | * |
| 677 | * @param int $batch_size actions per batch (default 200) |
| 678 | */ |
| 679 | $batch_size = max( 10, (int) apply_filters( 'wc_square_failed_action_cleanup_batch', 200 ) ); |
| 680 | $max_batches = 25; // hard bound per run: up to 5,000 deletions, far above the incident growth rate. |
| 681 | |
| 682 | foreach ( $hooks as $hook ) { |
| 683 | for ( $batch = 0; $batch < $max_batches; $batch++ ) { |
| 684 | $action_ids = as_get_scheduled_actions( |
| 685 | array( |
| 686 | 'hook' => $hook, |
| 687 | 'status' => \ActionScheduler_Store::STATUS_FAILED, |
| 688 | 'date' => $cutoff, |
| 689 | 'date_compare' => '<=', |
| 690 | 'per_page' => $batch_size, |
| 691 | 'orderby' => 'none', |
| 692 | ), |
| 693 | 'ids' |
| 694 | ); |
| 695 | |
| 696 | foreach ( (array) $action_ids as $action_id ) { |
| 697 | try { |
| 698 | $store->delete_action( $action_id ); |
| 699 | } catch ( \Exception $e ) { |
| 700 | // One undeletable action must not abandon the rest of the backlog for a day. |
| 701 | wc_square()->log( 'Could not delete failed action ' . $action_id . ': ' . $e->getMessage() ); |
| 702 | } |
| 703 | } |
| 704 | |
| 705 | // A short page means the backlog for this hook is exhausted. |
| 706 | if ( count( (array) $action_ids ) < $batch_size ) { |
| 707 | break; |
| 708 | } |
| 709 | } |
| 710 | } |
| 711 | } catch ( \Exception $e ) { |
| 712 | wc_square()->log( 'Failed-action cleanup skipped: ' . $e->getMessage() ); |
| 713 | } |
| 714 | } |
| 715 | |
| 716 | /** |
| 717 | * Handle Sync healthcheck |
| 718 | * |
| 719 | * Restart the background sync process if not already running |
| 720 | * and data exists in the queue. |
| 721 | * |
| 722 | * @since 3.8.2 |
| 723 | */ |
| 724 | public function handle_sync_healthcheck() { |
| 725 | |
| 726 | // Auto-recover a stalled sync first, on purpose. A job stuck in "processing" (timeout, fatal, |
| 727 | // worker kill) or a cascade of failing wc_square_job_runner actions keeps the queue |
| 728 | // non-empty, so the as_has_scheduled_action() guard below would otherwise never let the sync |
| 729 | // restart. Running recovery before the early returns is what breaks that deadlock. |
| 730 | $this->maybe_recover_stuck_sync(); |
| 731 | |
| 732 | // Housekeeping: prune old failed Square actions so the Action Scheduler store does not bloat |
| 733 | // (the reported incident left 14,000+ failed actions behind). Throttled internally. |
| 734 | $this->cleanup_stale_failed_actions(); |
| 735 | |
| 736 | if ( $this->is_process_running() ) { |
| 737 | // background process already running |
| 738 | return; |
| 739 | } |
| 740 | |
| 741 | if ( $this->is_queue_empty() ) { |
| 742 | // no data to process |
| 743 | return; |
| 744 | } |
| 745 | |
| 746 | if ( as_has_scheduled_action( 'wc_square_job_runner' ) ) { |
| 747 | // scheduled action for trigger sync is already exists |
| 748 | return; |
| 749 | } |
| 750 | |
| 751 | // Start the sync process |
| 752 | as_enqueue_async_action( 'wc_square_job_runner' ); |
| 753 | } |
| 754 | } |
| 755 |