PluginProbe
Automatic YouTube Gallery – Embed Auto-Updating YouTube Video Galleries, Feeds, Playlists & Channels / 2.8.1
Automatic YouTube Gallery – Embed Auto-Updating YouTube Video Galleries, Feeds, Playlists & Channels v2.8.1
2.9.1 2.9.0 trunk 1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 1.6.1 1.6.2 1.6.3 1.6.4 1.6.5 2.0.0 2.1.0 2.2.0 2.3.2 2.3.3 2.3.5 2.3.6 2.3.8 2.3.9 2.4.3 All 37 releases
automatic-youtube-gallery / includes / import.php

import.php in Automatic YouTube Gallery – Embed Auto-Updating YouTube Video Galleries, Feeds, Playlists & Channels 2.8.1, at includes/import.php

537 lines 19.3 KB
No matching file
Up and down to move Enter to open Esc to close
Raw Download Zip
1 <?php
2
3 /**
4 * Gallery builder import engine.
5 *
6 * @link https://plugins360.com
7 * @since 2.8.0
8 *
9 * @package Automatic_YouTube_Gallery
10 */
11
12 // Exit if accessed directly
13 if ( ! defined( 'WPINC' ) ) {
14 die;
15 }
16
17 /**
18 * AYG_Import class.
19 *
20 * @since 2.8.0
21 */
22 class AYG_Import {
23
24 /**
25 * Import a single batch of videos for the given gallery.
26 *
27 * @since 2.8.0
28 * @param int $gallery_id Gallery ID.
29 * @param string $page_token Page token to resume from. Empty string starts from the
30 * saved token (if any) or the first page.
31 * @param bool $manual True for a manual "Create / Update Gallery" run: full re-scan,
32 * no per-run cap, and prune videos no longer in the source. Cron
33 * passes false (incremental, capped, never prunes).
34 * @return array { imported, total_so_far, next_page_token, done }
35 * on success, or { error, quota_exceeded } on failure.
36 */
37 public function import_batch( $gallery_id, $page_token = '', $manual = false ) {
38 global $wpdb;
39
40 $galleries_table = $wpdb->prefix . 'ayg_galleries';
41 $rel_table = $wpdb->prefix . 'ayg_gallery_relationships';
42
43 $gallery = ayg_get_gallery( $gallery_id );
44
45 if ( ! $gallery ) {
46 return array( 'error' => __( 'Gallery not found.', 'automatic-youtube-gallery' ) );
47 }
48
49 if ( in_array( $gallery->source_type, array( 'search', 'livestream', 'video' ), true ) ) {
50 return array( 'error' => __( 'Live galleries (search / livestream / single video) query the live API at display time and cannot be imported.', 'automatic-youtube-gallery' ) );
51 }
52
53 $params = json_decode( (string) $gallery->params, true );
54 if ( ! is_array( $params ) ) {
55 $params = array();
56 }
57
58 // Empty incoming token = first batch of a run (the loop passes the returned token back).
59 $is_first_batch = ( '' === $page_token );
60
61 // A genuinely fresh start has neither an incoming nor a saved token (a resume has a saved one).
62 $is_fresh_start = ( $is_first_batch && empty( $params['page_token'] ) );
63
64 // Resume from the saved token when the client does not supply one (e.g. after a quota pause).
65 if ( '' === $page_token && ! empty( $params['page_token'] ) ) {
66 $page_token = (string) $params['page_token'];
67 }
68
69 // Manual "Update Gallery" runs a full refresh: full re-scan (no early-stop), no per-run cap
70 // (the browser completes it in one go), metadata refreshed via upsert, and a prune of videos
71 // no longer in the source on the final batch. Cron stays incremental + capped + never prunes.
72 //
73 // On a fresh start, reset the run's counters and (for manual) stamp the run's start time. Each
74 // batch then marks the links it re-stores with this timestamp; on the final batch any link not
75 // re-stamped (i.e. removed at the source) is pruned. A resume keeps both so totals + the prune
76 // marker stay correct across the whole run.
77 if ( $is_fresh_start ) {
78 $params['count_imported'] = 0;
79 $params['count_updated'] = 0;
80
81 if ( $manual ) {
82 $params['sync_run_start'] = current_time( 'mysql' );
83 }
84 }
85
86 // Completed gallery syncs incrementally; a new / never-finished one does a full scan.
87 // Stable for the whole run: last_imported_at only flips on this run's final batch.
88 $incremental = ! empty( $gallery->last_imported_at );
89
90 $now = current_time( 'mysql' );
91
92 if ( 'running' !== $gallery->import_status ) {
93 $wpdb->update(
94 $galleries_table,
95 array(
96 'import_status' => 'running',
97 'updated_at' => $now
98 ),
99 array( 'id' => $gallery->id ),
100 array( '%s', '%s' ),
101 array( '%d' )
102 );
103 }
104
105 // Relationship count before this page is stored. Used by the channel/username early-stop and to
106 // split this batch into new (imported) vs already-linked (updated) videos.
107 $rel_count_before = (int) $wpdb->get_var(
108 $wpdb->prepare( "SELECT COUNT(*) FROM $rel_table WHERE gallery_id = %s", strval( $gallery->id ) )
109 );
110
111 // Fetch one page. The query carries uid + exclude + advanced mode, so request_videos() also
112 // stores the video rows + relationships and enriches duration / video_type as a side effect.
113 $response = $this->request_videos( $gallery, $params, $page_token );
114
115 if ( isset( $response->error ) ) {
116 // A real API failure stops the run. But an incremental sync that simply found an empty
117 // result set (nothing to import) is a clean, complete run — fall through with an empty
118 // video set so the normal completion path finalizes it.
119 if ( ! $incremental || empty( $response->no_results ) ) {
120 return $this->handle_api_error( $gallery, $params, $response, $page_token );
121 }
122
123 $videos = array();
124 } else {
125 $videos = $response->videos;
126 }
127
128 // Manual full refresh: stamp this batch's links with the run's start time so the final batch can
129 // prune links not seen this run. One UPDATE per batch — no growing id list, scales to any size.
130 if ( $manual && ! empty( $videos ) ) {
131 $batch_ids = array();
132
133 foreach ( $videos as $video ) {
134 if ( ! empty( $video->id ) ) {
135 $batch_ids[] = $video->id;
136 }
137 }
138
139 if ( $batch_ids ) {
140 $run_start = isset( $params['sync_run_start'] ) ? (string) $params['sync_run_start'] : $now;
141 $placeholders = implode( ',', array_fill( 0, count( $batch_ids ), '%s' ) );
142
143 $wpdb->query(
144 $wpdb->prepare(
145 "UPDATE $rel_table SET synced_at = %s WHERE gallery_id = %s AND video_id IN ( $placeholders )",
146 array_merge( array( $run_start, strval( $gallery->id ) ), $batch_ids )
147 )
148 );
149 }
150 }
151
152 $video_count = (int) $wpdb->get_var(
153 $wpdb->prepare( "SELECT COUNT(*) FROM $rel_table WHERE gallery_id = %s", strval( $gallery->id ) )
154 );
155
156 // New links added this batch (imported). Accumulate across batches for the run total.
157 $new_this_batch = max( 0, $video_count - $rel_count_before );
158 $params['count_imported'] = ( isset( $params['count_imported'] ) ? (int) $params['count_imported'] : 0 ) + $new_this_batch;
159
160 // "Updated" (existing videos re-synced) is only counted for a manual full refresh. Cron's
161 // incremental re-fetch of the newest page would otherwise log a misleading "updated" every run
162 // even when nothing changed; cron keeps updated = 0 and reports only genuinely new imports.
163 if ( $manual ) {
164 $updated_this_batch = max( 0, count( $videos ) - $new_this_batch );
165 $params['count_updated'] = ( isset( $params['count_updated'] ) ? (int) $params['count_updated'] : 0 ) + $updated_this_batch;
166 }
167
168 $next_page_token = '';
169 if ( isset( $response->page_info ) && ! empty( $response->page_info['next_page_token'] ) ) {
170 $next_page_token = (string) $response->page_info['next_page_token'];
171 }
172
173 // Incremental channel / username sync stops at the first already-stored video instead of
174 // re-paging the whole channel: the uploads playlist is reverse-chronological and never
175 // reordered, so a known video means all older ones are known. A fully-new page (cron stalled,
176 // > 50 new) keeps paging so the backlog isn't lost. Overlap = fewer new rows than page videos.
177 if ( $incremental && ! $manual && in_array( $gallery->source_type, array( 'channel', 'username' ), true ) ) {
178 if ( ( $video_count - $rel_count_before ) < count( $videos ) ) {
179 $next_page_token = '';
180 }
181 }
182
183 // No next page token means the run is done — either the source has no more pages, or
184 // the early-stop above cleared the token after reaching an already-stored video.
185 $done = ( '' === $next_page_token );
186
187 // Per-run safety cap: a run (one browser loop or one cron fire) processes at most
188 // max_videos_per_run() videos, then pauses with the token saved so cron finishes the rest.
189 $run_processed = ( $is_first_batch ? 0 : (int) ( isset( $params['run_processed'] ) ? $params['run_processed'] : 0 ) ) + count( $videos );
190 $params['run_processed'] = $run_processed;
191
192 // Manual runs are uncapped — the browser completes the whole scan in one session.
193 $capped = ( ! $manual && ! $done && $run_processed >= $this->max_videos_per_run() );
194
195 $params['page_token'] = $done ? '' : $next_page_token;
196
197 // Run totals (accumulated across batches): imported = new, updated = existing re-synced,
198 // deleted = pruned on the final manual batch.
199 $imported = isset( $params['count_imported'] ) ? (int) $params['count_imported'] : 0;
200 $updated = isset( $params['count_updated'] ) ? (int) $params['count_updated'] : 0;
201 $deleted = 0;
202
203 $data = array( 'updated_at' => $now );
204
205 if ( $done ) {
206 $schedule = isset( $params['schedule'] ) ? absint( $params['schedule'] ) : 0;
207 $is_recurring = $schedule > 0 && ! in_array( $gallery->source_type, array( 'video', 'videos' ), true );
208
209 $data['import_status'] = $is_recurring ? 'idle' : 'completed';
210 $data['import_error'] = '';
211 $data['last_imported_at'] = $now;
212 $data['next_import_at'] = $is_recurring ? date( 'Y-m-d H:i:s', current_time( 'timestamp' ) + $schedule ) : null;
213
214 unset( $params['run_processed'] );
215
216 // Manual full refresh: this run scanned the whole source and stamped every link it saw with
217 // $run_start, so prune the gallery's links NOT stamped this run (removed at the source), then
218 // recount. Cron never does this. Guard: only prune when the run actually stamped something
219 // (imported + updated > 0) so an empty/failed scan can't wipe the gallery.
220 if ( $manual ) {
221 if ( ( $imported + $updated ) > 0 && ! empty( $params['sync_run_start'] ) ) {
222 $deleted = $this->prune_deleted_videos( $gallery, (string) $params['sync_run_start'] );
223
224 $video_count = (int) $wpdb->get_var(
225 $wpdb->prepare( "SELECT COUNT(*) FROM $rel_table WHERE gallery_id = %s", strval( $gallery->id ) )
226 );
227 }
228
229 unset( $params['sync_run_start'] );
230 }
231
232 // One log entry per completed run, recording this run's imported / updated / deleted totals.
233 $data['import_log'] = $this->append_log_entry( $gallery, $imported, $updated, $deleted, $data['import_status'], '' );
234
235 unset( $params['count_imported'], $params['count_updated'] );
236 } elseif ( $capped ) {
237 // Hand off to cron: idle + due now so the next fire resumes from the saved token.
238 // last_imported_at stays unset, so the resume keeps the same import mode.
239 $data['import_status'] = 'idle';
240 $data['next_import_at'] = $now;
241 }
242
243 // params / video_count are appended last so their values reflect the cleanup above.
244 $data['params'] = wp_json_encode( $params );
245 $data['video_count'] = $video_count;
246
247 $wpdb->update(
248 $galleries_table,
249 $data,
250 array( 'id' => $gallery->id ),
251 array(),
252 array( '%d' )
253 );
254
255 return array(
256 'imported' => $imported, // Running run totals; final on the done batch.
257 'updated' => $updated,
258 'deleted' => $deleted,
259 'total_so_far' => $video_count,
260 'next_page_token' => $next_page_token,
261 'capped' => $capped,
262 'done' => $done || $capped // A capped run reports done so the browser / cron loop stops; cron continues it.
263 );
264 }
265
266 /**
267 * Scheduled sync entry point. Invoked by the ayg_cron_schedule cron dispatcher.
268 *
269 * @since 2.8.0
270 */
271 public function sync() {
272 $this->recover_stuck_syncs();
273
274 $gallery_id = $this->get_due_gallery_id();
275 if ( ! $gallery_id ) {
276 return;
277 }
278
279 // Cron has no client to drive the batches, so loop here. import_batch() self-caps each
280 // fire and reports done; the guard is just a backstop against a non-terminating loop.
281 $page_token = '';
282 $guard = 0;
283
284 do {
285 $result = $this->import_batch( $gallery_id, $page_token );
286
287 if ( isset( $result['error'] ) ) {
288 break;
289 }
290
291 $page_token = $result['next_page_token'];
292 } while ( empty( $result['done'] ) && ++$guard < 1000 );
293 }
294
295 /**
296 * Fetch and store one page of videos from the YouTube API for the gallery source.
297 *
298 * @since 2.8.0
299 * @access private
300 * @param object $gallery Gallery row from wp_ayg_galleries.
301 * @param array $params Decoded gallery params.
302 * @param string $page_token Page token to fetch.
303 * @return mixed Response object, or error object from the API class.
304 */
305 private function request_videos( $gallery, $params, $page_token ) {
306 $query = array(
307 'type' => $gallery->source_type,
308 'src' => $gallery->source_value,
309 'uid' => strval( $gallery->id ),
310 'exclude' => ( isset( $params['exclude'] ) && is_array( $params['exclude'] ) ) ? $params['exclude'] : array(),
311 'mode' => 'advanced',
312 'maxResults' => 50, // YouTube's max per call; one quota unit regardless, so always page at 50
313 'pageToken' => $page_token,
314 'cache' => 0
315 );
316
317 $api = new AYG_YouTube_API();
318
319 // query() also persists the videos + relationships (see note above), not just fetch.
320 return $api->query( $query );
321 }
322
323 /**
324 * Maximum number of videos to process in a single run before pausing for cron.
325 *
326 * @since 2.8.0
327 * @access private
328 * @return int Videos per run (always >= 1).
329 */
330 private function max_videos_per_run() {
331 $limit = (int) apply_filters( 'ayg_import_max_videos_per_run', 1000 );
332 return $limit > 0 ? $limit : 1000;
333 }
334
335 /**
336 * Prune a gallery's videos that were not seen during a manual full scan.
337 *
338 * A complete manual run stamps every link it re-stores with $run_start (see import_batch). This
339 * removes the gallery's links NOT stamped this run (i.e. removed at the source), then deletes any
340 * wp_ayg_videos rows left orphaned (not linked by ANY gallery — shared rows are kept). The caller
341 * only invokes this after a non-empty scan, so a transient API failure can't empty the gallery.
342 *
343 * @since 2.8.0
344 * @access private
345 * @param object $gallery Gallery row.
346 * @param string $run_start Timestamp this run stamped its links with (MySQL datetime).
347 * @return int Number of gallery links removed.
348 */
349 private function prune_deleted_videos( $gallery, $run_start ) {
350 global $wpdb;
351
352 if ( '' === (string) $run_start ) {
353 return 0;
354 }
355
356 $rel_table = $wpdb->prefix . 'ayg_gallery_relationships';
357 $videos_table = $wpdb->prefix . 'ayg_videos';
358
359 // 1) Drop this gallery's links not re-stamped by this run (NULL = never synced, or an older run).
360 $deleted = (int) $wpdb->query(
361 $wpdb->prepare(
362 "DELETE FROM $rel_table WHERE gallery_id = %s AND ( synced_at IS NULL OR synced_at < %s )",
363 strval( $gallery->id ),
364 $run_start
365 )
366 );
367
368 // 2) Delete video rows now orphaned (linked by no gallery). Rows shared by others are kept.
369 $wpdb->query(
370 "DELETE v FROM $videos_table v
371 LEFT JOIN $rel_table r ON v.video_id = r.video_id
372 WHERE r.video_id IS NULL"
373 );
374
375 return $deleted;
376 }
377
378 /**
379 * Maximum number of run entries kept in a gallery's import_log JSON.
380 *
381 * @since 2.8.0
382 * @access private
383 * @return int Log entries to keep (always >= 1).
384 */
385 private function max_log_entries() {
386 $max = (int) apply_filters( 'ayg_import_max_log_entries', 25 );
387 return $max > 0 ? $max : 25;
388 }
389
390 /**
391 * Recover galleries left stuck in the 'running' state.
392 *
393 * @since 2.8.0
394 * @access private
395 */
396 private function recover_stuck_syncs() {
397 global $wpdb;
398
399 $galleries_table = $wpdb->prefix . 'ayg_galleries';
400
401 $now = current_time( 'mysql' );
402 $threshold = date( 'Y-m-d H:i:s', current_time( 'timestamp' ) - 10 * MINUTE_IN_SECONDS );
403
404 $wpdb->query(
405 $wpdb->prepare(
406 "UPDATE $galleries_table
407 SET import_status = 'idle', next_import_at = %s, updated_at = %s
408 WHERE import_status = 'running' AND updated_at < %s",
409 $now,
410 $now,
411 $threshold
412 )
413 );
414 }
415
416 /**
417 * Find the earliest gallery that is due for a scheduled sync.
418 *
419 * @since 2.8.0
420 * @access private
421 * @return int|null Gallery ID, or null when none are due.
422 */
423 private function get_due_gallery_id() {
424 global $wpdb;
425
426 $galleries_table = $wpdb->prefix . 'ayg_galleries';
427
428 $gallery_id = $wpdb->get_var(
429 $wpdb->prepare(
430 "SELECT id FROM $galleries_table
431 WHERE next_import_at IS NOT NULL
432 AND next_import_at < %s
433 AND import_status NOT IN ( 'running', 'paused', 'error' )
434 AND source_type NOT IN ( 'search', 'livestream', 'video' )
435 ORDER BY next_import_at ASC
436 LIMIT 1",
437 current_time( 'mysql' )
438 )
439 );
440
441 return $gallery_id ? (int) $gallery_id : null;
442 }
443
444 /**
445 * Handle an API error response for the current batch.
446 *
447 * @since 2.8.0
448 * @access private
449 * @param object $gallery Gallery row from wp_ayg_galleries.
450 * @param array $params Decoded gallery params.
451 * @param object $response Error object from the API class.
452 * @param string $page_token Page token the failed batch was fetching.
453 * @return array { error, quota_exceeded }
454 */
455 private function handle_api_error( $gallery, $params, $response, $page_token ) {
456 global $wpdb;
457
458 $message = wp_strip_all_tags( $response->error_message );
459 $quota_exceeded = ( false !== stripos( $message, 'quotaExceeded' ) );
460 $status = $quota_exceeded ? 'paused' : 'error';
461
462 // Log what earlier batches imported/updated before the error (deleted = 0: the prune runs only on
463 // a clean completion). Then reset the counters so a resume logs only its own delta rather than
464 // re-counting these. sync_run_start + page_token are kept so a resume continues from the same
465 // point with the same prune marker; only the per-run cap counter restarts.
466 $imported = isset( $params['count_imported'] ) ? (int) $params['count_imported'] : 0;
467 $updated = isset( $params['count_updated'] ) ? (int) $params['count_updated'] : 0;
468
469 $params['page_token'] = $page_token;
470 $params['count_imported'] = 0;
471 $params['count_updated'] = 0;
472 unset( $params['run_processed'] );
473
474 $wpdb->update(
475 $wpdb->prefix . 'ayg_galleries',
476 array(
477 'params' => wp_json_encode( $params ),
478 'import_status' => $status,
479 'import_error' => $message,
480 'updated_at' => current_time( 'mysql' ),
481 'import_log' => $this->append_log_entry( $gallery, $imported, $updated, 0, $status, $message ) // Record the stopped run (counts so far + message) so it shows in the log.
482 ),
483 array( 'id' => $gallery->id ),
484 array(),
485 array( '%d' )
486 );
487
488 return array(
489 'error' => $message,
490 'quota_exceeded' => $quota_exceeded
491 );
492 }
493
494 /**
495 * Append a run entry to the gallery's import_log JSON and return the encoded result.
496 *
497 * @since 2.8.0
498 * @access private
499 * @param object $gallery Gallery row from wp_ayg_galleries.
500 * @param int $imported Number of new videos added by this run.
501 * @param int $updated Number of already-linked videos re-synced this run.
502 * @param int $deleted Number of videos pruned this run (removed at the source).
503 * @param string $status Final run status ('idle' / 'completed' / 'paused' / 'error').
504 * @param string $error Error message for failed runs, empty otherwise.
505 * @return string JSON-encoded import_log.
506 */
507 private function append_log_entry( $gallery, $imported, $updated, $deleted, $status, $error = '' ) {
508 $logs = json_decode( (string) $gallery->import_log, true );
509 if ( ! is_array( $logs ) ) {
510 $logs = array();
511 }
512
513 $entry = array(
514 'date' => current_time( 'mysql' ),
515 'imported' => (int) $imported,
516 'updated' => (int) $updated,
517 'deleted' => (int) $deleted,
518 'status' => $status
519 );
520
521 // Record the error message on failed runs so it can be shown in the log.
522 if ( '' !== $error ) {
523 $entry['error'] = $error;
524 }
525
526 $logs[] = $entry;
527
528 $max = $this->max_log_entries();
529 if ( count( $logs ) > $max ) {
530 $logs = array_slice( $logs, -$max );
531 }
532
533 return wp_json_encode( $logs );
534 }
535
536 }
537