PluginProbe
Jetpack – WP Security, Backup, Speed, & Growth / 16.3-a.7
Jetpack – WP Security, Backup, Speed, & Growth v16.3-a.7
16.3-a.5 16.3-a.7 16.3-a.3 16.3-a.1 16.2 16.2-beta 12.0.3 12.1.3 12.2.3 12.3.2 12.4.2 12.5.2 12.6.4 12.7.3 12.8.3 12.9.5 13.0.2 13.1.5 13.2.4 13.3.3 13.4.5 13.5.2 13.6.2 13.7.2 13.8.3 All 506 releases
jetpack / jetpack_vendor / automattic / jetpack-sync / src / modules / class-full-sync.php

class-full-sync.php in Jetpack – WP Security, Backup, Speed, & Growth 16.3-a.7, at jetpack_vendor/automattic/jetpack-sync/src/modules/class-full-sync.php

740 lines 20.6 KB
No matching file
Up and down to move Enter to open Esc to close
Raw Download Zip
1 <?php
2 /**
3 * Full sync module.
4 *
5 * @package automattic/jetpack-sync
6 */
7
8 namespace Automattic\Jetpack\Sync\Modules;
9
10 use Automattic\Jetpack\Sync\Listener;
11 use Automattic\Jetpack\Sync\Lock;
12 use Automattic\Jetpack\Sync\Modules;
13 use Automattic\Jetpack\Sync\Queue;
14 use Automattic\Jetpack\Sync\Settings;
15
16 if ( ! defined( 'ABSPATH' ) ) {
17 exit( 0 );
18 }
19
20 /**
21 * This class does a full resync of the database by
22 * enqueuing an outbound action for every single object
23 * that we care about.
24 *
25 * This class, and its related class Jetpack_Sync_Module, contain a few non-obvious optimisations that should be explained:
26 * - we fire an action called jetpack_full_sync_start so that WPCOM can erase the contents of the cached database
27 * - for each object type, we page through the object IDs and enqueue them by firing some monitored actions
28 * - we load the full objects for those IDs in chunks of Jetpack_Sync_Module::ARRAY_CHUNK_SIZE (to reduce the number of MySQL calls)
29 * - we fire a trigger for the entire array which the Automattic\Jetpack\Sync\Listener then serializes and queues.
30 */
31 class Full_Sync extends Module {
32 /**
33 * Prefix of the full sync status option name.
34 *
35 * @var string
36 */
37 const STATUS_OPTION_PREFIX = 'jetpack_sync_full_';
38
39 /**
40 * Enqueue Lock name.
41 *
42 * @var string
43 */
44 const ENQUEUE_LOCK_NAME = 'full_sync_enqueue';
45
46 /**
47 * Sync module name.
48 *
49 * @access public
50 *
51 * @return string
52 */
53 public function name() {
54 return 'full-sync';
55 }
56
57 /**
58 * Initialize action listeners for full sync.
59 *
60 * @access public
61 *
62 * @param callable $callable Action handler callable.
63 */
64 public function init_full_sync_listeners( $callable ) {
65 // Synthetic actions for full sync.
66 add_action( 'jetpack_full_sync_start', $callable, 10, 3 );
67 add_action( 'jetpack_full_sync_end', $callable, 10, 2 );
68 add_action( 'jetpack_full_sync_cancelled', $callable );
69 }
70
71 /**
72 * Initialize the module in the sender.
73 *
74 * @access public
75 */
76 public function init_before_send() {
77 // This is triggered after actions have been processed on the server.
78 add_action( 'jetpack_sync_processed_actions', array( $this, 'update_sent_progress_action' ) );
79 }
80
81 /**
82 * Start a full sync.
83 *
84 * @access public
85 *
86 * @param array $module_configs Full sync configuration for all sync modules.
87 * @param mixed $context Context for the full sync.
88 * @return bool Always returns true at success.
89 */
90 public function start( $module_configs = null, $context = null ) { // phpcs:ignore VariableAnalysis.CodeAnalysis.VariableAnalysis.UnusedVariable
91 $was_already_running = $this->is_started() && ! $this->is_finished();
92
93 // Remove all evidence of previous full sync items and status.
94 $this->reset_data();
95
96 if ( $was_already_running ) {
97 /**
98 * Fires when a full sync is cancelled.
99 *
100 * @since 1.6.3
101 * @since-jetpack 4.2.0
102 */
103 do_action( 'jetpack_full_sync_cancelled' );
104 }
105
106 $this->update_status_option( 'started', time() );
107 $this->update_status_option( 'params', $module_configs );
108
109 $enqueue_status = array();
110 $full_sync_config = array();
111 $include_empty = false;
112 $empty = array();
113
114 // Default value is full sync.
115 if ( ! is_array( $module_configs ) ) {
116 $module_configs = array();
117 $include_empty = true;
118 foreach ( Modules::get_modules() as $module ) {
119 $module_configs[ $module->name() ] = true;
120 }
121 }
122
123 // Set default configuration, calculate totals, and save configuration if totals > 0.
124 foreach ( Modules::get_modules() as $module ) {
125 $module_name = $module->name();
126 $module_config = $module_configs[ $module_name ] ?? false;
127
128 if ( ! $module_config ) {
129 continue;
130 }
131
132 if ( 'users' === $module_name && 'initial' === $module_config ) {
133 '@phan-var Users $module';
134 $module_config = $module->get_initial_sync_user_config();
135 }
136
137 $enqueue_status[ $module_name ] = false;
138
139 $total_items = $module->estimate_full_sync_actions( $module_config );
140
141 // If there's information to process, configure this module.
142 if ( $total_items !== null && $total_items > 0 ) {
143 $full_sync_config[ $module_name ] = $module_config;
144 $enqueue_status[ $module_name ] = array(
145 $total_items, // Total.
146 0, // Queued.
147 false, // Current state.
148 );
149 } elseif ( $include_empty && 0 === $total_items ) {
150 $empty[ $module_name ] = true;
151 }
152 }
153
154 $this->set_config( $full_sync_config );
155 $this->set_enqueue_status( $enqueue_status );
156
157 $range = $this->get_content_range( $full_sync_config );
158 /**
159 * Fires when a full sync begins. This action is serialized
160 * and sent to the server so that it knows a full sync is coming.
161 *
162 * @since 1.6.3
163 * @since-jetpack 4.2.0
164 * @since-jetpack 7.3.0 Added $range arg.
165 * @since-jetpack 7.4.0 Added $empty arg.
166 *
167 * @param array $full_sync_config Sync configuration for all sync modules.
168 * @param array $range Range of the sync items, containing min and max IDs for some item types.
169 * @param array $empty The modules with no items to sync during a full sync.
170 */
171 do_action( 'jetpack_full_sync_start', $full_sync_config, $range, $empty );
172
173 $this->continue_enqueuing( $full_sync_config );
174
175 return true;
176 }
177
178 /**
179 * Enqueue the next items to sync.
180 *
181 * @access public
182 *
183 * @param array $configs Full sync configuration for all sync modules.
184 */
185 public function continue_enqueuing( $configs = null ) {
186 // Return early if not in progress.
187 if ( ! $this->get_status_option( 'started' ) || $this->get_status_option( 'queue_finished' ) ) {
188 return;
189 }
190
191 // Attempt to obtain lock.
192 $lock = new Lock();
193 $lock_expiration = $lock->attempt( self::ENQUEUE_LOCK_NAME );
194
195 // Return if unable to obtain lock.
196 if ( false === $lock_expiration ) {
197 return;
198 }
199
200 // enqueue full sync actions.
201 $this->enqueue( $configs );
202
203 // Remove lock.
204 $lock->remove( self::ENQUEUE_LOCK_NAME, $lock_expiration );
205 }
206
207 /**
208 * Get Modules that are configured to Full Sync and haven't finished enqueuing
209 *
210 * @param array $configs Full sync configuration for all sync modules.
211 *
212 * @return array
213 */
214 public function get_remaining_modules_to_enqueue( $configs ) {
215 $enqueue_status = $this->get_enqueue_status();
216 return array_filter(
217 Modules::get_modules(),
218 /**
219 * Select configured and not finished modules.
220 *
221 * @param Module $module
222 * @return bool
223 */
224 function ( $module ) use ( $configs, $enqueue_status ) {
225 // Skip module if not configured for this sync or module is done.
226 if ( ! isset( $configs[ $module->name() ] ) ) {
227 return false;
228 }
229 if ( ! $configs[ $module->name() ] ) {
230 return false;
231 }
232 if ( isset( $enqueue_status[ $module->name() ][2] ) ) {
233 if ( true === $enqueue_status[ $module->name() ][2] ) {
234 return false;
235 }
236 }
237
238 return true;
239 }
240 );
241 }
242
243 /**
244 * Enqueue the next items to sync.
245 *
246 * @access public
247 *
248 * @param array $configs Full sync configuration for all sync modules.
249 */
250 public function enqueue( $configs = null ) {
251 if ( ! $configs ) {
252 $configs = $this->get_config();
253 }
254
255 $enqueue_status = $this->get_enqueue_status();
256 $full_sync_queue = new Queue( 'full_sync' );
257 $available_queue_slots = Settings::get_setting( 'max_queue_size_full_sync' ) - $full_sync_queue->size();
258
259 if ( $available_queue_slots <= 0 ) {
260 return;
261 }
262
263 $remaining_items_to_enqueue = min( Settings::get_setting( 'max_enqueue_full_sync' ), $available_queue_slots );
264
265 /**
266 * If a module exits early (e.g. because it ran out of full sync queue slots, or we ran out of request time)
267 * then it should exit early
268 */
269 foreach ( $this->get_remaining_modules_to_enqueue( $configs ) as $module ) {
270 list( $items_enqueued, $next_enqueue_state ) = $module->enqueue_full_sync_actions( $configs[ $module->name() ], $remaining_items_to_enqueue, $enqueue_status[ $module->name() ][2] );
271
272 $enqueue_status[ $module->name() ][2] = $next_enqueue_state;
273
274 // If items were processed, subtract them from the limit.
275 if ( $items_enqueued !== null && $items_enqueued > 0 ) {
276 $enqueue_status[ $module->name() ][1] += $items_enqueued;
277 $remaining_items_to_enqueue -= $items_enqueued;
278 }
279
280 if ( 0 >= $remaining_items_to_enqueue || true !== $next_enqueue_state ) {
281 $this->set_enqueue_status( $enqueue_status );
282 return;
283 }
284 }
285
286 $this->queue_full_sync_end( $configs );
287 $this->set_enqueue_status( $enqueue_status );
288 }
289
290 /**
291 * Enqueue 'jetpack_full_sync_end' and update 'queue_finished' status.
292 *
293 * @access public
294 *
295 * @param array $configs Full sync configuration for all sync modules.
296 */
297 public function queue_full_sync_end( $configs ) {
298 $range = $this->get_content_range( $configs );
299
300 /**
301 * Fires when a full sync ends. This action is serialized
302 * and sent to the server.
303 *
304 * @since 1.6.3
305 * @since-jetpack 4.2.0
306 * @since-jetpack 7.3.0 Added $range arg.
307 *
308 * @param string $checksum Deprecated since 7.3.0 - @see https://github.com/Automattic/jetpack/pull/11945/
309 * @param array $range Range of the sync items, containing min and max IDs for some item types.
310 */
311 do_action( 'jetpack_full_sync_end', '', $range );
312
313 // Setting autoload to true means that it's faster to check whether we should continue enqueuing.
314 $this->update_status_option( 'queue_finished', time(), true );
315 }
316
317 /**
318 * Get the range (min ID, max ID and total items) of items to sync.
319 *
320 * @access public
321 *
322 * @param string $type Type of sync item to get the range for.
323 * @return array Array of min ID, max ID and total items in the range.
324 */
325 public function get_range( $type ) {
326 global $wpdb;
327 if ( ! in_array( $type, array( 'comments', 'posts' ), true ) ) {
328 return array();
329 }
330
331 switch ( $type ) {
332 case 'posts':
333 $table = $wpdb->posts;
334 $id = 'ID';
335 $where_sql = Settings::get_blacklisted_post_types_sql();
336
337 break;
338 case 'comments':
339 $table = $wpdb->comments;
340 $id = 'comment_ID';
341 $where_sql = Settings::get_comments_filter_sql();
342 break;
343 default:
344 // This should never be reached due to the guard condition above,
345 // but Phan complains so let's make it happy.
346 return array();
347 }
348
349 // TODO: Call $wpdb->prepare on the following query.
350 // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared
351 $results = $wpdb->get_results( "SELECT MAX({$id}) as max, MIN({$id}) as min, COUNT({$id}) as count FROM {$table} WHERE {$where_sql}" );
352 if ( isset( $results[0] ) ) {
353 return $results[0];
354 }
355
356 return array();
357 }
358
359 /**
360 * Get the range for content (posts and comments) to sync.
361 *
362 * @access private
363 *
364 * @param array $config Full sync configuration for this all sync modules.
365 * @return array Array of range (min ID, max ID, total items) for all content types.
366 */
367 private function get_content_range( $config ) {
368 $range = array();
369 // Only when we are sending the whole range do we want to send also the range.
370 if ( ! empty( $config['posts'] ) ) {
371 $range['posts'] = $this->get_range( 'posts' );
372 }
373
374 if ( ! empty( $config['comments'] ) ) {
375 $range['comments'] = $this->get_range( 'comments' );
376 }
377 return $range;
378 }
379
380 /**
381 * Update the progress after sync modules actions have been processed on the server.
382 *
383 * @access public
384 *
385 * @param array $actions Actions that have been processed on the server.
386 */
387 public function update_sent_progress_action( $actions ) {
388 // Quick way to map to first items with an array of arrays.
389 $actions_with_counts = array_count_values( array_filter( array_map( array( $this, 'get_action_name' ), $actions ) ) );
390
391 // Total item counts for each action.
392 $actions_with_total_counts = $this->get_actions_totals( $actions );
393
394 if ( ! $this->is_started() || $this->is_finished() ) {
395 return;
396 }
397
398 if ( isset( $actions_with_counts['jetpack_full_sync_start'] ) ) {
399 $this->update_status_option( 'send_started', time() );
400 }
401
402 foreach ( Modules::get_modules() as $module ) {
403 $module_actions = $module->get_full_sync_actions();
404 $status_option_name = "{$module->name()}_sent";
405 $total_option_name = "{$status_option_name}_total";
406 $items_sent = $this->get_status_option( $status_option_name, 0 );
407 $items_sent_total = $this->get_status_option( $total_option_name, 0 );
408
409 foreach ( $module_actions as $module_action ) {
410 if ( isset( $actions_with_counts[ $module_action ] ) ) {
411 $items_sent += $actions_with_counts[ $module_action ];
412 }
413
414 if ( ! empty( $actions_with_total_counts[ $module_action ] ) ) {
415 $items_sent_total += $actions_with_total_counts[ $module_action ];
416 }
417 }
418
419 if ( $items_sent > 0 ) {
420 $this->update_status_option( $status_option_name, $items_sent );
421 }
422
423 if ( 0 !== $items_sent_total ) {
424 $this->update_status_option( $total_option_name, $items_sent_total );
425 }
426 }
427
428 if ( isset( $actions_with_counts['jetpack_full_sync_end'] ) ) {
429 $this->update_status_option( 'finished', time() );
430 }
431 }
432
433 /**
434 * Returns the progress percentage of a full sync.
435 *
436 * @access public
437 *
438 * @return int|null
439 */
440 public function get_sync_progress_percentage() {
441 if ( ! $this->is_started() || $this->is_finished() ) {
442 return null;
443 }
444 $status = $this->get_status();
445 if ( ! $status['queue'] || ! $status['sent'] || ! $status['total'] ) {
446 return null;
447 }
448 $queued_multiplier = 0.1;
449 $sent_multiplier = 0.9;
450 $count_queued = array_reduce(
451 $status['queue'],
452 function ( $sum, $value ) {
453 return $sum + $value;
454 },
455 0
456 );
457 $count_sent = array_reduce(
458 $status['sent'],
459 function ( $sum, $value ) {
460 return $sum + $value;
461 },
462 0
463 );
464 $count_total = array_reduce(
465 $status['total'],
466 function ( $sum, $value ) {
467 return $sum + $value;
468 },
469 0
470 );
471 $percent_queued = ( $count_queued / $count_total ) * $queued_multiplier * 100;
472 $percent_sent = ( $count_sent / $count_total ) * $sent_multiplier * 100;
473 return ceil( $percent_queued + $percent_sent );
474 }
475
476 /**
477 * Get the name of the action for an item in the sync queue.
478 *
479 * @access public
480 *
481 * @param array $queue_item Item of the sync queue.
482 * @return string|boolean Name of the action, false if queue item is invalid.
483 */
484 public function get_action_name( $queue_item ) {
485 if ( is_array( $queue_item ) && isset( $queue_item[0] ) ) {
486 return $queue_item[0];
487 }
488 return false;
489 }
490
491 /**
492 * Retrieve the total number of items we're syncing in a particular queue item (action).
493 * `$queue_item[1]` is expected to contain chunks of items, and `$queue_item[1][0]`
494 * represents the first (and only) chunk of items to sync in that action.
495 *
496 * @access public
497 *
498 * @param array $queue_item Item of the sync queue that corresponds to a particular action.
499 * @return int Total number of items in the action.
500 */
501 public function get_action_totals( $queue_item ) {
502 if ( is_array( $queue_item ) && isset( $queue_item[1][0] ) ) {
503 if ( is_array( $queue_item[1][0] ) ) {
504 // Let's count the items we sync in this action.
505 return count( $queue_item[1][0] );
506 }
507 // -1 indicates that this action syncs all items by design.
508 return -1;
509 }
510 return 0;
511 }
512
513 /**
514 * Retrieve the total number of items for a set of actions, grouped by action name.
515 *
516 * @access public
517 *
518 * @param array $actions An array of actions.
519 * @return array An array, representing the total number of items, grouped per action.
520 */
521 public function get_actions_totals( $actions ) {
522 $totals = array();
523
524 foreach ( $actions as $action ) {
525 $name = $this->get_action_name( $action );
526 $action_totals = $this->get_action_totals( $action );
527 if ( ! isset( $totals[ $name ] ) ) {
528 $totals[ $name ] = 0;
529 }
530 $totals[ $name ] += $action_totals;
531 }
532
533 return $totals;
534 }
535
536 /**
537 * Whether full sync has started.
538 *
539 * @access public
540 *
541 * @return boolean
542 */
543 public function is_started() {
544 return (bool) $this->get_status_option( 'started' );
545 }
546
547 /**
548 * Whether full sync has finished.
549 *
550 * @access public
551 *
552 * @return boolean
553 */
554 public function is_finished() {
555 return (bool) $this->get_status_option( 'finished' );
556 }
557
558 /**
559 * Retrieve the status of the current full sync.
560 *
561 * @access public
562 *
563 * @return array Full sync status.
564 */
565 public function get_status() {
566 $status = array(
567 'started' => $this->get_status_option( 'started' ),
568 'queue_finished' => $this->get_status_option( 'queue_finished' ),
569 'send_started' => $this->get_status_option( 'send_started' ),
570 'finished' => $this->get_status_option( 'finished' ),
571 'sent' => array(),
572 'sent_total' => array(),
573 'queue' => array(),
574 'config' => $this->get_status_option( 'params' ),
575 'total' => array(),
576 );
577
578 $enqueue_status = $this->get_enqueue_status();
579
580 foreach ( Modules::get_modules() as $module ) {
581 $name = $module->name();
582
583 if ( ! isset( $enqueue_status[ $name ] ) ) {
584 continue;
585 }
586
587 list( $total, $queued ) = $enqueue_status[ $name ];
588
589 if ( $total ) {
590 $status['total'][ $name ] = $total;
591 }
592
593 if ( $queued ) {
594 $status['queue'][ $name ] = $queued;
595 }
596
597 $sent = $this->get_status_option( "{$name}_sent" );
598 if ( $sent ) {
599 $status['sent'][ $name ] = $sent;
600 }
601
602 $sent_total = $this->get_status_option( "{$name}_sent_total" );
603 if ( $sent_total ) {
604 $status['sent_total'][ $name ] = $sent_total;
605 }
606 }
607
608 return $status;
609 }
610
611 /**
612 * Clear all the full sync status options.
613 *
614 * @access public
615 */
616 public function clear_status() {
617 $prefix = self::STATUS_OPTION_PREFIX;
618 \Jetpack_Options::delete_raw_option( "{$prefix}_started" );
619 \Jetpack_Options::delete_raw_option( "{$prefix}_params" );
620 \Jetpack_Options::delete_raw_option( "{$prefix}_queue_finished" );
621 \Jetpack_Options::delete_raw_option( "{$prefix}_send_started" );
622 \Jetpack_Options::delete_raw_option( "{$prefix}_finished" );
623
624 $this->delete_enqueue_status();
625
626 foreach ( Modules::get_modules() as $module ) {
627 \Jetpack_Options::delete_raw_option( "{$prefix}_{$module->name()}_sent" );
628 \Jetpack_Options::delete_raw_option( "{$prefix}_{$module->name()}_sent_total" );
629 }
630 }
631
632 /**
633 * Clear all the full sync data.
634 *
635 * @access public
636 */
637 public function reset_data() {
638 $this->clear_status();
639 $this->delete_config();
640 ( new Lock() )->remove( self::ENQUEUE_LOCK_NAME, true );
641
642 $listener = Listener::get_instance();
643 $listener->get_full_sync_queue()->reset();
644 }
645
646 /**
647 * Get the value of a full sync status option.
648 *
649 * @access private
650 *
651 * @param string $name Name of the option.
652 * @param mixed $default Default value of the option.
653 * @return mixed Option value.
654 */
655 private function get_status_option( $name, $default = null ) {
656 $value = \Jetpack_Options::get_raw_option( self::STATUS_OPTION_PREFIX . "_$name", $default );
657
658 return is_numeric( $value ) ? (int) $value : $value;
659 }
660
661 /**
662 * Update the value of a full sync status option.
663 *
664 * @access private
665 *
666 * @param string $name Name of the option.
667 * @param mixed $value Value of the option.
668 * @param boolean $autoload Whether the option should be autoloaded at the beginning of the request.
669 */
670 private function update_status_option( $name, $value, $autoload = false ) {
671 \Jetpack_Options::update_raw_option( self::STATUS_OPTION_PREFIX . "_$name", $value, $autoload );
672 }
673
674 /**
675 * Set the full sync enqueue status.
676 *
677 * @access private
678 *
679 * @param array $new_status The new full sync enqueue status.
680 */
681 private function set_enqueue_status( $new_status ) {
682 \Jetpack_Options::update_raw_option( 'jetpack_sync_full_enqueue_status', $new_status );
683 }
684
685 /**
686 * Delete full sync enqueue status.
687 *
688 * @access private
689 *
690 * @return boolean Whether the status was deleted.
691 */
692 private function delete_enqueue_status() {
693 return \Jetpack_Options::delete_raw_option( 'jetpack_sync_full_enqueue_status' );
694 }
695
696 /**
697 * Retrieve the current full sync enqueue status.
698 *
699 * @access private
700 *
701 * @return array Full sync enqueue status.
702 */
703 public function get_enqueue_status() {
704 return \Jetpack_Options::get_raw_option( 'jetpack_sync_full_enqueue_status' );
705 }
706
707 /**
708 * Set the full sync enqueue configuration.
709 *
710 * @access private
711 *
712 * @param array $config The new full sync enqueue configuration.
713 */
714 private function set_config( $config ) {
715 \Jetpack_Options::update_raw_option( 'jetpack_sync_full_config', $config );
716 }
717
718 /**
719 * Delete full sync configuration.
720 *
721 * @access private
722 *
723 * @return boolean Whether the configuration was deleted.
724 */
725 private function delete_config() {
726 return \Jetpack_Options::delete_raw_option( 'jetpack_sync_full_config' );
727 }
728
729 /**
730 * Retrieve the current full sync enqueue config.
731 *
732 * @access private
733 *
734 * @return array Full sync enqueue config.
735 */
736 private function get_config() {
737 return \Jetpack_Options::get_raw_option( 'jetpack_sync_full_config' );
738 }
739 }
740