PluginProbe
Jetpack – WP Security, Backup, Speed, & Growth / 12.3.2
Jetpack – WP Security, Backup, Speed, & Growth v12.3.2
12.0.3 12.1.3 12.2.3 12.3.2 12.4.2 12.5.2 12.6.4 12.7.3 12.8.3 12.9.5 13.0.2 13.1.5 13.2.4 13.3.3 13.4.5 13.5.2 13.6.2 13.7.2 13.8.3 13.9.2 14.0.1 14.1.1 14.2.2 14.3.1 14.4.2 All 500 releases
jetpack / jetpack_vendor / automattic / jetpack-sync / src / class-queue.php
class-queue.php
770 lines 17.9 KB
No matching file
Up and down to move Enter to open Esc to close
Raw Download Zip
1 <?php
2 /**
3 * The class that describes the Queue for the sync package.
4 *
5 * @package automattic/jetpack-sync
6 */
7
8 namespace Automattic\Jetpack\Sync;
9
10 use WP_Error;
11
12 /**
13 * A persistent queue that can be flushed in increments of N items,
14 * and which blocks reads until checked-out buffers are checked in or
15 * closed. This uses raw SQL for two reasons: speed, and not triggering
16 * tons of added_option callbacks.
17 */
18 class Queue {
19 /**
20 * The queue id.
21 *
22 * @var string
23 */
24 public $id;
25
26 /**
27 * Keeps track of the rows.
28 *
29 * @var int
30 */
31 private $row_iterator;
32
33 /**
34 * Random number.
35 *
36 * @var int
37 */
38 public $random_int;
39
40 /**
41 * Queue constructor.
42 *
43 * @param string $id Name of the queue.
44 */
45 public function __construct( $id ) {
46 $this->id = str_replace( '-', '_', $id ); // Necessary to ensure we don't have ID collisions in the SQL.
47 $this->row_iterator = 0;
48 $this->random_int = wp_rand( 1, 1000000 );
49 }
50
51 /**
52 * Add a single item to the queue.
53 *
54 * @param object $item Event object to add to queue.
55 */
56 public function add( $item ) {
57 global $wpdb;
58 $added = false;
59
60 // If empty, don't add.
61 if ( empty( $item ) ) {
62 return;
63 }
64
65 // Attempt to serialize data, if an exception (closures) return early.
66 try {
67 $item = serialize( $item ); // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.serialize_serialize
68 } catch ( \Exception $ex ) {
69 return;
70 }
71
72 // This basically tries to add the option until enough time has elapsed that
73 // it has a unique (microtime-based) option key.
74 while ( ! $added ) {
75 $rows_added = $wpdb->query(
76 $wpdb->prepare(
77 "INSERT INTO $wpdb->options (option_name, option_value, autoload) VALUES (%s, %s,%s)",
78 $this->get_next_data_row_option_name(),
79 $item,
80 'no'
81 )
82 );
83 $added = ( 0 !== $rows_added );
84 }
85 }
86
87 /**
88 * Insert all the items in a single SQL query. May be subject to query size limits!
89 *
90 * @param array $items Array of events to add to the queue.
91 *
92 * @return bool|\WP_Error
93 */
94 public function add_all( $items ) {
95 global $wpdb;
96 $base_option_name = $this->get_next_data_row_option_name();
97
98 $query = "INSERT INTO $wpdb->options (option_name, option_value, autoload) VALUES ";
99
100 $rows = array();
101 $count_items = count( $items );
102 for ( $i = 0; $i < $count_items; ++$i ) {
103 // skip empty items.
104 if ( empty( $items[ $i ] ) ) {
105 continue;
106 }
107 try {
108 $option_name = esc_sql( $base_option_name . '-' . $i );
109 $option_value = esc_sql( serialize( $items[ $i ] ) ); // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.serialize_serialize
110 $rows[] = "('$option_name', '$option_value', 'no')";
111 } catch ( \Exception $e ) {
112 // Item cannot be serialized so skip.
113 continue;
114 }
115 }
116
117 $rows_added = $wpdb->query( $query . implode( ',', $rows ) ); // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared, WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching
118
119 if ( count( $items ) !== $rows_added ) {
120 return new WP_Error( 'row_count_mismatch', "The number of rows inserted didn't match the size of the input array" );
121 }
122 return true;
123 }
124
125 /**
126 * Get the front-most item on the queue without checking it out.
127 *
128 * @param int $count Number of items to return when looking at the items.
129 *
130 * @return array
131 */
132 public function peek( $count = 1 ) {
133 $items = $this->fetch_items( $count );
134 if ( $items ) {
135 return Utils::get_item_values( $items );
136 }
137
138 return array();
139 }
140
141 /**
142 * Gets items with particular IDs.
143 *
144 * @param array $item_ids Array of item IDs to retrieve.
145 *
146 * @return array
147 */
148 public function peek_by_id( $item_ids ) {
149 $items = $this->fetch_items_by_id( $item_ids );
150 if ( $items ) {
151 return Utils::get_item_values( $items );
152 }
153
154 return array();
155 }
156
157 /**
158 * Gets the queue lag.
159 * Lag is the difference in time between the age of the oldest item
160 * (aka first or frontmost item) and the current time.
161 *
162 * @param microtime $now The current time in microtime.
163 *
164 * @return float|int|mixed|null
165 */
166 public function lag( $now = null ) {
167 global $wpdb;
168
169 $first_item_name = $wpdb->get_var(
170 $wpdb->prepare(
171 "SELECT option_name FROM $wpdb->options WHERE option_name LIKE %s ORDER BY option_name ASC LIMIT 1",
172 "jpsq_{$this->id}-%"
173 )
174 );
175
176 if ( ! $first_item_name ) {
177 return 0;
178 }
179
180 if ( null === $now ) {
181 $now = microtime( true );
182 }
183
184 // Break apart the item name to get the timestamp.
185 $matches = null;
186 if ( preg_match( '/^jpsq_' . $this->id . '-(\d+\.\d+)-/', $first_item_name, $matches ) ) {
187 return $now - (float) $matches[1];
188 } else {
189 return 0;
190 }
191 }
192
193 /**
194 * Resets the queue.
195 */
196 public function reset() {
197 global $wpdb;
198 $this->delete_checkout_id();
199 $wpdb->query(
200 $wpdb->prepare(
201 "DELETE FROM $wpdb->options WHERE option_name LIKE %s",
202 "jpsq_{$this->id}-%"
203 )
204 );
205 }
206
207 /**
208 * Return the size of the queue.
209 *
210 * @return int
211 */
212 public function size() {
213 global $wpdb;
214
215 return (int) $wpdb->get_var(
216 $wpdb->prepare(
217 "SELECT count(*) FROM $wpdb->options WHERE option_name LIKE %s",
218 "jpsq_{$this->id}-%"
219 )
220 );
221 }
222
223 /**
224 * Lets you know if there is any items in the queue.
225 *
226 * We use this peculiar implementation because it's much faster than count(*).
227 *
228 * @return bool
229 */
230 public function has_any_items() {
231 global $wpdb;
232 $value = $wpdb->get_var(
233 $wpdb->prepare(
234 "SELECT exists( SELECT option_name FROM $wpdb->options WHERE option_name LIKE %s )",
235 "jpsq_{$this->id}-%"
236 )
237 );
238
239 return ( '1' === $value );
240 }
241
242 /**
243 * Used to checkout the queue.
244 *
245 * @param int $buffer_size Size of the buffer to checkout.
246 *
247 * @return Automattic\Jetpack\Sync\Queue_Buffer|bool|int|\WP_Error
248 */
249 public function checkout( $buffer_size ) {
250 if ( $this->get_checkout_id() ) {
251 return new WP_Error( 'unclosed_buffer', 'There is an unclosed buffer' );
252 }
253
254 $buffer_id = uniqid();
255
256 $result = $this->set_checkout_id( $buffer_id );
257
258 if ( ! $result || is_wp_error( $result ) ) {
259 return $result;
260 }
261
262 $items = $this->fetch_items( $buffer_size );
263
264 if ( ! is_countable( $items ) ) {
265 return false;
266 }
267
268 if ( count( $items ) === 0 ) {
269 return false;
270 }
271
272 $buffer = new Queue_Buffer( $buffer_id, array_slice( $items, 0, $buffer_size ) );
273
274 return $buffer;
275 }
276
277 /**
278 * Given a list of items return the items ids.
279 *
280 * @param array $items List of item objects.
281 *
282 * @return array Ids of the items.
283 */
284 public function get_ids( $items ) {
285 return array_map(
286 function ( $item ) {
287 return $item->id;
288 },
289 $items
290 );
291 }
292
293 /**
294 * Pop elements from the queue.
295 *
296 * @param int $limit Number of items to pop from the queue.
297 *
298 * @return array|object|null
299 */
300 public function pop( $limit ) {
301 $items = $this->fetch_items( $limit );
302
303 $ids = $this->get_ids( $items );
304
305 $this->delete( $ids );
306
307 return $items;
308 }
309
310 /**
311 * Get the items from the queue with a memory limit.
312 *
313 * This checks out rows until it either empties the queue or hits a certain memory limit
314 * it loads the sizes from the DB first so that it doesn't accidentally
315 * load more data into memory than it needs to.
316 * The only way it will load more items than $max_size is if a single queue item
317 * exceeds the memory limit, but in that case it will send that item by itself.
318 *
319 * @param int $max_memory (bytes) Maximum memory threshold.
320 * @param int $max_buffer_size Maximum buffer size (number of items).
321 *
322 * @return Automattic\Jetpack\Sync\Queue_Buffer|bool|int|\WP_Error
323 */
324 public function checkout_with_memory_limit( $max_memory, $max_buffer_size = 500 ) {
325 if ( $this->get_checkout_id() ) {
326 return new WP_Error( 'unclosed_buffer', 'There is an unclosed buffer' );
327 }
328
329 $buffer_id = uniqid();
330
331 $result = $this->set_checkout_id( $buffer_id );
332
333 if ( ! $result || is_wp_error( $result ) ) {
334 return $result;
335 }
336
337 // Get the map of buffer_id -> memory_size.
338 global $wpdb;
339
340 $items_with_size = $wpdb->get_results(
341 $wpdb->prepare(
342 "SELECT option_name AS id, LENGTH(option_value) AS value_size FROM $wpdb->options WHERE option_name LIKE %s ORDER BY option_name ASC LIMIT %d",
343 "jpsq_{$this->id}-%",
344 $max_buffer_size
345 ),
346 OBJECT
347 );
348
349 if ( ! is_countable( $items_with_size ) ) {
350 return false;
351 }
352
353 if ( count( $items_with_size ) === 0 ) {
354 return false;
355 }
356
357 $total_memory = 0;
358 $max_item_id = $items_with_size[0]->id;
359 $min_item_id = $max_item_id;
360
361 foreach ( $items_with_size as $id => $item_with_size ) {
362 $total_memory += $item_with_size->value_size;
363
364 // If this is the first item and it exceeds memory, allow loop to continue
365 // we will exit on the next iteration instead.
366 if ( $total_memory > $max_memory && $id > 0 ) {
367 break;
368 }
369
370 $max_item_id = $item_with_size->id;
371 }
372
373 $query = $wpdb->prepare(
374 "SELECT option_name AS id, option_value AS value FROM $wpdb->options WHERE option_name >= %s and option_name <= %s ORDER BY option_name ASC",
375 $min_item_id,
376 $max_item_id
377 );
378
379 $items = $wpdb->get_results( $query, OBJECT ); // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared
380 $items_count = is_countable( $items ) ? count( $items ) : 0;
381
382 if ( $items_count > 0 ) {
383 foreach ( $items as $item ) {
384 // @codingStandardsIgnoreStart
385 $item->value = @unserialize( $item->value );
386 // @codingStandardsIgnoreEnd
387 }
388 }
389
390 if ( $items_count === 0 ) {
391 $this->delete_checkout_id();
392 return false;
393 }
394
395 $buffer = new Queue_Buffer( $buffer_id, $items );
396
397 return $buffer;
398 }
399
400 /**
401 * Check in the queue.
402 *
403 * @param Automattic\Jetpack\Sync\Queue_Buffer $buffer Queue_Buffer object.
404 *
405 * @return bool|\WP_Error
406 */
407 public function checkin( $buffer ) {
408 $is_valid = $this->validate_checkout( $buffer );
409
410 if ( is_wp_error( $is_valid ) ) {
411 return $is_valid;
412 }
413
414 $this->delete_checkout_id();
415
416 return true;
417 }
418
419 /**
420 * Close the buffer.
421 *
422 * @param Automattic\Jetpack\Sync\Queue_Buffer $buffer Queue_Buffer object.
423 * @param null|array $ids_to_remove Ids to remove from the queue.
424 *
425 * @return bool|\WP_Error
426 */
427 public function close( $buffer, $ids_to_remove = null ) {
428 $is_valid = $this->validate_checkout( $buffer );
429
430 if ( is_wp_error( $is_valid ) ) {
431 // Always delete ids_to_remove even when buffer is no longer checked-out.
432 // They were processed by WP.com so safe to remove from queue.
433 if ( $ids_to_remove !== null ) {
434 $this->delete( $ids_to_remove );
435 }
436 return $is_valid;
437 }
438
439 $this->delete_checkout_id();
440
441 // By default clear all items in the buffer.
442 if ( $ids_to_remove === null ) {
443 $ids_to_remove = $buffer->get_item_ids();
444 }
445
446 $this->delete( $ids_to_remove );
447
448 return true;
449 }
450
451 /**
452 * Delete elements from the queue.
453 *
454 * @param array $ids Ids to delete.
455 *
456 * @return bool|int
457 */
458 private function delete( $ids ) {
459 if ( array() === $ids ) {
460 return 0;
461 }
462 global $wpdb;
463 $sql = "DELETE FROM $wpdb->options WHERE option_name IN (" . implode( ', ', array_fill( 0, count( $ids ), '%s' ) ) . ')';
464 $query = call_user_func_array( array( $wpdb, 'prepare' ), array_merge( array( $sql ), $ids ) );
465
466 return $wpdb->query( $query ); // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared
467 }
468
469 /**
470 * Flushes all items from the queue.
471 *
472 * @return array
473 */
474 public function flush_all() {
475 $items = Utils::get_item_values( $this->fetch_items() );
476 $this->reset();
477
478 return $items;
479 }
480
481 /**
482 * Get all the items from the queue.
483 *
484 * @return array|object|null
485 */
486 public function get_all() {
487 return $this->fetch_items();
488 }
489
490 /**
491 * Forces Checkin of the queue.
492 * Use with caution, this could allow multiple processes to delete
493 * and send from the queue at the same time
494 */
495 public function force_checkin() {
496 $this->delete_checkout_id();
497 }
498
499 /**
500 * Checks if the queue is locked.
501 *
502 * @return bool
503 */
504 public function is_locked() {
505 return (bool) $this->get_checkout_id();
506 }
507
508 /**
509 * Locks checkouts from the queue
510 * tries to wait up to $timeout seconds for the queue to be empty.
511 *
512 * @param int $timeout The wait time in seconds for the queue to be empty.
513 *
514 * @return bool|int|\WP_Error
515 */
516 public function lock( $timeout = 30 ) {
517 $tries = 0;
518
519 while ( $this->has_any_items() && $tries < $timeout ) {
520 sleep( 1 );
521 ++$tries;
522 }
523
524 if ( 30 === $tries ) {
525 return new WP_Error( 'lock_timeout', 'Timeout waiting for sync queue to empty' );
526 }
527
528 if ( $this->get_checkout_id() ) {
529 return new WP_Error( 'unclosed_buffer', 'There is an unclosed buffer' );
530 }
531
532 // Hopefully this means we can acquire a checkout?
533 $result = $this->set_checkout_id( 'lock' );
534
535 if ( ! $result || is_wp_error( $result ) ) {
536 return $result;
537 }
538
539 return true;
540 }
541
542 /**
543 * Unlocks the queue.
544 *
545 * @return bool|int
546 */
547 public function unlock() {
548 return $this->delete_checkout_id();
549 }
550
551 /**
552 * This option is specifically chosen to, as much as possible, preserve time order
553 * and minimise the possibility of collisions between multiple processes working
554 * at the same time.
555 *
556 * @return string
557 */
558 protected function generate_option_name_timestamp() {
559 return sprintf( '%.6f', microtime( true ) );
560 }
561
562 /**
563 * Gets the checkout ID.
564 *
565 * @return bool|string
566 */
567 private function get_checkout_id() {
568 global $wpdb;
569 $checkout_value = $wpdb->get_var(
570 $wpdb->prepare(
571 "SELECT option_value FROM $wpdb->options WHERE option_name = %s",
572 $this->get_lock_option_name()
573 )
574 );
575
576 if ( $checkout_value ) {
577 list( $checkout_id, $timestamp ) = explode( ':', $checkout_value );
578 if ( (int) $timestamp > time() ) {
579 return $checkout_id;
580 }
581 }
582
583 return false;
584 }
585
586 /**
587 * Sets the checkout id.
588 *
589 * @param string $checkout_id The ID of the checkout.
590 *
591 * @return bool|int
592 */
593 private function set_checkout_id( $checkout_id ) {
594 global $wpdb;
595
596 $expires = time() + Defaults::$default_sync_queue_lock_timeout;
597 $updated_num = $wpdb->query(
598 $wpdb->prepare(
599 "UPDATE $wpdb->options SET option_value = %s WHERE option_name = %s",
600 "$checkout_id:$expires",
601 $this->get_lock_option_name()
602 )
603 );
604
605 if ( ! $updated_num ) {
606 $updated_num = $wpdb->query(
607 $wpdb->prepare(
608 "INSERT INTO $wpdb->options ( option_name, option_value, autoload ) VALUES ( %s, %s, 'no' )",
609 $this->get_lock_option_name(),
610 "$checkout_id:$expires"
611 )
612 );
613 }
614
615 return $updated_num;
616 }
617
618 /**
619 * Deletes the checkout ID.
620 *
621 * @return bool|int
622 */
623 private function delete_checkout_id() {
624 global $wpdb;
625 // Rather than delete, which causes fragmentation, we update in place.
626 return $wpdb->query(
627 $wpdb->prepare(
628 "UPDATE $wpdb->options SET option_value = %s WHERE option_name = %s",
629 '0:0',
630 $this->get_lock_option_name()
631 )
632 );
633 }
634
635 /**
636 * Return the lock option name.
637 *
638 * @return string
639 */
640 private function get_lock_option_name() {
641 return "jpsq_{$this->id}_checkout";
642 }
643
644 /**
645 * Return the next data row option name.
646 *
647 * @return string
648 */
649 private function get_next_data_row_option_name() {
650 $timestamp = $this->generate_option_name_timestamp();
651
652 // Row iterator is used to avoid collisions where we're writing data waaay fast in a single process.
653 if ( PHP_INT_MAX === $this->row_iterator ) {
654 $this->row_iterator = 0;
655 } else {
656 $this->row_iterator += 1;
657 }
658
659 return 'jpsq_' . $this->id . '-' . $timestamp . '-' . $this->random_int . '-' . $this->row_iterator;
660 }
661
662 /**
663 * Return the items in the queue.
664 *
665 * @param null|int $limit Limit to the number of items we fetch at once.
666 *
667 * @return array|object|null
668 */
669 private function fetch_items( $limit = null ) {
670 global $wpdb;
671
672 if ( $limit ) {
673 $items = $wpdb->get_results(
674 $wpdb->prepare(
675 "SELECT option_name AS id, option_value AS value FROM $wpdb->options WHERE option_name LIKE %s ORDER BY option_name ASC LIMIT %d",
676 "jpsq_{$this->id}-%",
677 $limit
678 ),
679 OBJECT
680 );
681 } else {
682 $items = $wpdb->get_results(
683 $wpdb->prepare(
684 "SELECT option_name AS id, option_value AS value FROM $wpdb->options WHERE option_name LIKE %s ORDER BY option_name ASC",
685 "jpsq_{$this->id}-%"
686 ),
687 OBJECT
688 );
689 }
690
691 return $this->unserialize_values( $items );
692 }
693
694 /**
695 * Return items with specific ids.
696 *
697 * @param array $items_ids Array of event ids.
698 *
699 * @return array|object|null
700 */
701 private function fetch_items_by_id( $items_ids ) {
702 global $wpdb;
703
704 // return early if $items_ids is empty or not an array.
705 if ( empty( $items_ids ) || ! is_array( $items_ids ) ) {
706 return null;
707 }
708
709 $ids_placeholders = implode( ', ', array_fill( 0, count( $items_ids ), '%s' ) );
710 $query_with_placeholders = "SELECT option_name AS id, option_value AS value
711 FROM $wpdb->options
712 WHERE option_name IN ( $ids_placeholders )";
713 $items = $wpdb->get_results(
714 $wpdb->prepare(
715 $query_with_placeholders, // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared
716 $items_ids
717 ),
718 OBJECT
719 );
720
721 return $this->unserialize_values( $items );
722 }
723
724 /**
725 * Unserialize item values.
726 *
727 * @param array $items Events from the Queue to be unserialized.
728 *
729 * @return mixed
730 */
731 private function unserialize_values( $items ) {
732 array_walk(
733 $items,
734 function ( $item ) {
735 // @codingStandardsIgnoreStart
736 $item->value = @unserialize( $item->value );
737 // @codingStandardsIgnoreEnd
738 }
739 );
740
741 return $items;
742 }
743
744 /**
745 * Return true if the buffer is still valid or an Error other wise.
746 *
747 * @param Automattic\Jetpack\Sync\Queue_Buffer $buffer The Queue_Buffer.
748 *
749 * @return bool|WP_Error
750 */
751 private function validate_checkout( $buffer ) {
752 if ( ! $buffer instanceof Queue_Buffer ) {
753 return new WP_Error( 'not_a_buffer', 'You must checkin an instance of Automattic\\Jetpack\\Sync\\Queue_Buffer' );
754 }
755
756 $checkout_id = $this->get_checkout_id();
757
758 if ( ! $checkout_id ) {
759 return new WP_Error( 'buffer_not_checked_out', 'There are no checked out buffers' );
760 }
761
762 // TODO: change to strict comparison.
763 if ( $checkout_id != $buffer->id ) { // phpcs:ignore Universal.Operators.StrictComparisons.LooseNotEqual
764 return new WP_Error( 'buffer_mismatch', 'The buffer you checked in was not checked out' );
765 }
766
767 return true;
768 }
769 }
770