PluginProbe
Automatic YouTube Gallery – Embed Auto-Updating YouTube Video Galleries, Feeds, Playlists & Channels / trunk
Automatic YouTube Gallery – Embed Auto-Updating YouTube Video Galleries, Feeds, Playlists & Channels vtrunk
2.9.1 2.9.0 trunk 1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 1.6.1 1.6.2 1.6.3 1.6.4 1.6.5 2.0.0 2.1.0 2.2.0 2.3.2 2.3.3 2.3.5 2.3.6 2.3.8 2.3.9 2.4.3 All 37 releases
automatic-youtube-gallery / includes / import.php

import.php in Automatic YouTube Gallery – Embed Auto-Updating YouTube Video Galleries, Feeds, Playlists & Channels trunk, at includes/import.php

538 lines 19.4 KB
No matching file
Up and down to move Enter to open Esc to close
Raw Download Zip
1 <?php
2
3 /**
4 * Gallery builder import engine.
5 *
6 * @link https://plugins360.com
7 * @since 2.8.0
8 *
9 * @package Automatic_YouTube_Gallery
10 */
11
12 // Exit if accessed directly
13 if ( ! defined( 'WPINC' ) ) {
14 die;
15 }
16
17 /**
18 * AYG_Import class.
19 *
20 * @since 2.8.0
21 */
22 class AYG_Import {
23
24 /**
25 * Import a single batch of videos for the given gallery.
26 *
27 * @since 2.8.0
28 * @param int $gallery_id Gallery ID.
29 * @param string $page_token Page token to resume from. Empty string starts from the
30 * saved token (if any) or the first page.
31 * @param bool $manual True for a manual "Create / Update Gallery" run: full re-scan,
32 * no per-run cap, and prune videos no longer in the source. Cron
33 * passes false (incremental, capped, never prunes).
34 * @return array { imported, total_so_far, next_page_token, done }
35 * on success, or { error, quota_exceeded } on failure.
36 */
37 public function import_batch( $gallery_id, $page_token = '', $manual = false ) {
38 global $wpdb;
39
40 $galleries_table = $wpdb->prefix . 'ayg_galleries';
41 $rel_table = $wpdb->prefix . 'ayg_gallery_relationships';
42
43 $gallery = ayg_get_gallery( $gallery_id );
44
45 if ( ! $gallery ) {
46 return array( 'error' => __( 'Gallery not found.', 'automatic-youtube-gallery' ) );
47 }
48
49 if ( in_array( $gallery->source_type, array( 'search', 'livestream', 'video' ), true ) ) {
50 return array( 'error' => __( 'Live galleries (search / livestream / single video) query the live API at display time and cannot be imported.', 'automatic-youtube-gallery' ) );
51 }
52
53 $params = json_decode( (string) $gallery->params, true );
54 if ( ! is_array( $params ) ) {
55 $params = array();
56 }
57
58 // Empty incoming token = first batch of a run (the loop passes the returned token back).
59 $is_first_batch = ( '' === $page_token );
60
61 // A genuinely fresh start has neither an incoming nor a saved token (a resume has a saved one).
62 $is_fresh_start = ( $is_first_batch && empty( $params['page_token'] ) );
63
64 // Resume from the saved token when the client does not supply one (e.g. after a quota pause).
65 if ( '' === $page_token && ! empty( $params['page_token'] ) ) {
66 $page_token = (string) $params['page_token'];
67 }
68
69 // Manual "Update Gallery" runs a full refresh: full re-scan (no early-stop), no per-run cap
70 // (the browser completes it in one go), metadata refreshed via upsert, and a prune of videos
71 // no longer in the source on the final batch. Cron stays incremental + capped + never prunes.
72 //
73 // On a fresh start, reset the run's counters and (for manual) stamp the run's start time. Each
74 // batch then marks the links it re-stores with this timestamp; on the final batch any link not
75 // re-stamped (i.e. removed at the source) is pruned. A resume keeps both so totals + the prune
76 // marker stay correct across the whole run.
77 if ( $is_fresh_start ) {
78 $params['count_imported'] = 0;
79 $params['count_updated'] = 0;
80
81 if ( $manual ) {
82 $params['sync_run_start'] = current_time( 'mysql' );
83 }
84 }
85
86 // Completed gallery syncs incrementally; a new / never-finished one does a full scan.
87 // Stable for the whole run: last_imported_at only flips on this run's final batch.
88 $incremental = ! empty( $gallery->last_imported_at );
89
90 $now = current_time( 'mysql' );
91
92 if ( 'running' !== $gallery->import_status ) {
93 $wpdb->update(
94 $galleries_table,
95 array(
96 'import_status' => 'running',
97 'updated_at' => $now
98 ),
99 array( 'id' => $gallery->id ),
100 array( '%s', '%s' ),
101 array( '%d' )
102 );
103 }
104
105 // Relationship count before this page is stored. Used by the channel/username early-stop and to
106 // split this batch into new (imported) vs already-linked (updated) videos.
107 $rel_count_before = (int) $wpdb->get_var(
108 $wpdb->prepare( "SELECT COUNT(*) FROM $rel_table WHERE gallery_id = %s", strval( $gallery->id ) )
109 );
110
111 // Fetch one page. The query carries uid + exclude + advanced mode, so request_videos() also
112 // stores the video rows + relationships and enriches duration / video_type as a side effect.
113 $response = $this->request_videos( $gallery, $params, $page_token );
114
115 if ( isset( $response->error ) ) {
116 // A real API failure stops the run. But an incremental sync that simply found an empty
117 // result set (nothing to import) is a clean, complete run — fall through with an empty
118 // video set so the normal completion path finalizes it.
119 if ( ! $incremental || empty( $response->no_results ) ) {
120 return $this->handle_api_error( $gallery, $params, $response, $page_token );
121 }
122
123 $videos = array();
124 } else {
125 $videos = $response->videos;
126 }
127
128 // Manual full refresh: stamp this batch's links with the run's start time so the final batch can
129 // prune links not seen this run. One UPDATE per batch — no growing id list, scales to any size.
130 if ( $manual && ! empty( $videos ) ) {
131 $batch_ids = array();
132
133 foreach ( $videos as $video ) {
134 if ( ! empty( $video->id ) ) {
135 $batch_ids[] = $video->id;
136 }
137 }
138
139 if ( $batch_ids ) {
140 $run_start = isset( $params['sync_run_start'] ) ? (string) $params['sync_run_start'] : $now;
141 $placeholders = implode( ',', array_fill( 0, count( $batch_ids ), '%s' ) );
142
143 $wpdb->query(
144 $wpdb->prepare(
145 "UPDATE $rel_table SET synced_at = %s WHERE gallery_id = %s AND video_id IN ( $placeholders )",
146 array_merge( array( $run_start, strval( $gallery->id ) ), $batch_ids )
147 )
148 );
149 }
150 }
151
152 $video_count = (int) $wpdb->get_var(
153 $wpdb->prepare( "SELECT COUNT(*) FROM $rel_table WHERE gallery_id = %s", strval( $gallery->id ) )
154 );
155
156 // New links added this batch (imported). Accumulate across batches for the run total.
157 $new_this_batch = max( 0, $video_count - $rel_count_before );
158 $params['count_imported'] = ( isset( $params['count_imported'] ) ? (int) $params['count_imported'] : 0 ) + $new_this_batch;
159
160 // "Updated" (existing videos re-synced) is only counted for a manual full refresh. Cron's
161 // incremental re-fetch of the newest page would otherwise log a misleading "updated" every run
162 // even when nothing changed; cron keeps updated = 0 and reports only genuinely new imports.
163 if ( $manual ) {
164 $updated_this_batch = max( 0, count( $videos ) - $new_this_batch );
165 $params['count_updated'] = ( isset( $params['count_updated'] ) ? (int) $params['count_updated'] : 0 ) + $updated_this_batch;
166 }
167
168 $next_page_token = '';
169 if ( isset( $response->page_info ) && ! empty( $response->page_info['next_page_token'] ) ) {
170 $next_page_token = (string) $response->page_info['next_page_token'];
171 }
172
173 // Incremental channel / username sync stops at the first already-stored video instead of
174 // re-paging the whole channel: the uploads playlist is reverse-chronological and never
175 // reordered, so a known video means all older ones are known. A fully-new page (cron stalled,
176 // > 50 new) keeps paging so the backlog isn't lost. Overlap = fewer new rows than page videos.
177 if ( $incremental && ! $manual && in_array( $gallery->source_type, array( 'channel', 'username' ), true ) ) {
178 if ( ( $video_count - $rel_count_before ) < count( $videos ) ) {
179 $next_page_token = '';
180 }
181 }
182
183 // No next page token means the run is done — either the source has no more pages, or
184 // the early-stop above cleared the token after reaching an already-stored video.
185 $done = ( '' === $next_page_token );
186
187 // Per-run safety cap: a run (one browser loop or one cron fire) processes at most
188 // max_videos_per_run() videos, then pauses with the token saved so cron finishes the rest.
189 $run_processed = ( $is_first_batch ? 0 : (int) ( isset( $params['run_processed'] ) ? $params['run_processed'] : 0 ) ) + count( $videos );
190 $params['run_processed'] = $run_processed;
191
192 // Manual runs are uncapped — the browser completes the whole scan in one session.
193 $capped = ( ! $manual && ! $done && $run_processed >= $this->max_videos_per_run() );
194
195 $params['page_token'] = $done ? '' : $next_page_token;
196
197 // Run totals (accumulated across batches): imported = new, updated = existing re-synced,
198 // deleted = pruned on the final manual batch.
199 $imported = isset( $params['count_imported'] ) ? (int) $params['count_imported'] : 0;
200 $updated = isset( $params['count_updated'] ) ? (int) $params['count_updated'] : 0;
201 $deleted = 0;
202
203 $data = array( 'updated_at' => $now );
204
205 if ( $done ) {
206 $schedule = isset( $params['schedule'] ) ? absint( $params['schedule'] ) : 0;
207 $is_recurring = $schedule > 0 && ! in_array( $gallery->source_type, array( 'video', 'videos' ), true );
208
209 $data['import_status'] = $is_recurring ? 'idle' : 'completed';
210 $data['import_error'] = '';
211 $data['last_imported_at'] = $now;
212 $data['next_import_at'] = $is_recurring ? date( 'Y-m-d H:i:s', current_time( 'timestamp' ) + $schedule ) : null;
213
214 unset( $params['run_processed'] );
215
216 // Manual full refresh: this run scanned the whole source and stamped every link it saw with
217 // $run_start, so prune the gallery's links NOT stamped this run (removed at the source), then
218 // recount. Cron never does this. Guard: only prune when the run actually stamped something
219 // (imported + updated > 0) so an empty/failed scan can't wipe the gallery.
220 if ( $manual ) {
221 if ( ( $imported + $updated ) > 0 && ! empty( $params['sync_run_start'] ) ) {
222 $deleted = $this->prune_deleted_videos( $gallery, (string) $params['sync_run_start'] );
223
224 $video_count = (int) $wpdb->get_var(
225 $wpdb->prepare( "SELECT COUNT(*) FROM $rel_table WHERE gallery_id = %s", strval( $gallery->id ) )
226 );
227 }
228
229 unset( $params['sync_run_start'] );
230 }
231
232 // One log entry per completed run, recording this run's imported / updated / deleted totals.
233 $data['import_log'] = $this->append_log_entry( $gallery, $imported, $updated, $deleted, $data['import_status'], '' );
234
235 unset( $params['count_imported'], $params['count_updated'] );
236 } elseif ( $capped ) {
237 // Hand off to cron: idle + due now so the next fire resumes from the saved token.
238 // last_imported_at stays unset, so the resume keeps the same import mode.
239 $data['import_status'] = 'idle';
240 $data['next_import_at'] = $now;
241 }
242
243 // params / video_count are appended last so their values reflect the cleanup above.
244 $data['params'] = wp_json_encode( $params );
245 $data['video_count'] = $video_count;
246
247 $wpdb->update(
248 $galleries_table,
249 $data,
250 array( 'id' => $gallery->id ),
251 array(),
252 array( '%d' )
253 );
254
255 return array(
256 'imported' => $imported, // Running run totals; final on the done batch.
257 'updated' => $updated,
258 'deleted' => $deleted,
259 'total_so_far' => $video_count,
260 'next_page_token' => $next_page_token,
261 'capped' => $capped,
262 'done' => $done || $capped // A capped run reports done so the browser / cron loop stops; cron continues it.
263 );
264 }
265
266 /**
267 * Scheduled sync entry point. Invoked by the ayg_cron_schedule cron dispatcher.
268 *
269 * @since 2.8.0
270 */
271 public function sync() {
272 $this->recover_stuck_syncs();
273
274 $gallery_id = $this->get_due_gallery_id();
275 if ( ! $gallery_id ) {
276 return;
277 }
278
279 // Cron has no client to drive the batches, so loop here. import_batch() self-caps each
280 // fire and reports done; the guard is just a backstop against a non-terminating loop.
281 $page_token = '';
282 $guard = 0;
283
284 do {
285 $result = $this->import_batch( $gallery_id, $page_token );
286
287 if ( isset( $result['error'] ) ) {
288 break;
289 }
290
291 $page_token = $result['next_page_token'];
292 } while ( empty( $result['done'] ) && ++$guard < 1000 );
293 }
294
295 /**
296 * Fetch and store one page of videos from the YouTube API for the gallery source.
297 *
298 * @since 2.8.0
299 * @access private
300 * @param object $gallery Gallery row from wp_ayg_galleries.
301 * @param array $params Decoded gallery params.
302 * @param string $page_token Page token to fetch.
303 * @return mixed Response object, or error object from the API class.
304 */
305 private function request_videos( $gallery, $params, $page_token ) {
306 $query = array(
307 'type' => $gallery->source_type,
308 'src' => $gallery->source_value,
309 'uid' => strval( $gallery->id ),
310 'store' => true, // Trusted context: admin "Update Gallery" and cron, with the source read from the gallery row.
311 'exclude' => ( isset( $params['exclude'] ) && is_array( $params['exclude'] ) ) ? $params['exclude'] : array(),
312 'mode' => 'advanced',
313 'maxResults' => 50, // YouTube's max per call; one quota unit regardless, so always page at 50
314 'pageToken' => $page_token,
315 'cache' => 0
316 );
317
318 $api = new AYG_YouTube_API();
319
320 // query() also persists the videos + relationships (see note above), not just fetch.
321 return $api->query( $query );
322 }
323
324 /**
325 * Maximum number of videos to process in a single run before pausing for cron.
326 *
327 * @since 2.8.0
328 * @access private
329 * @return int Videos per run (always >= 1).
330 */
331 private function max_videos_per_run() {
332 $limit = (int) apply_filters( 'ayg_import_max_videos_per_run', 1000 );
333 return $limit > 0 ? $limit : 1000;
334 }
335
336 /**
337 * Prune a gallery's videos that were not seen during a manual full scan.
338 *
339 * A complete manual run stamps every link it re-stores with $run_start (see import_batch). This
340 * removes the gallery's links NOT stamped this run (i.e. removed at the source), then deletes any
341 * wp_ayg_videos rows left orphaned (not linked by ANY gallery — shared rows are kept). The caller
342 * only invokes this after a non-empty scan, so a transient API failure can't empty the gallery.
343 *
344 * @since 2.8.0
345 * @access private
346 * @param object $gallery Gallery row.
347 * @param string $run_start Timestamp this run stamped its links with (MySQL datetime).
348 * @return int Number of gallery links removed.
349 */
350 private function prune_deleted_videos( $gallery, $run_start ) {
351 global $wpdb;
352
353 if ( '' === (string) $run_start ) {
354 return 0;
355 }
356
357 $rel_table = $wpdb->prefix . 'ayg_gallery_relationships';
358 $videos_table = $wpdb->prefix . 'ayg_videos';
359
360 // 1) Drop this gallery's links not re-stamped by this run (NULL = never synced, or an older run).
361 $deleted = (int) $wpdb->query(
362 $wpdb->prepare(
363 "DELETE FROM $rel_table WHERE gallery_id = %s AND ( synced_at IS NULL OR synced_at < %s )",
364 strval( $gallery->id ),
365 $run_start
366 )
367 );
368
369 // 2) Delete video rows now orphaned (linked by no gallery). Rows shared by others are kept.
370 $wpdb->query(
371 "DELETE v FROM $videos_table v
372 LEFT JOIN $rel_table r ON v.video_id = r.video_id
373 WHERE r.video_id IS NULL"
374 );
375
376 return $deleted;
377 }
378
379 /**
380 * Maximum number of run entries kept in a gallery's import_log JSON.
381 *
382 * @since 2.8.0
383 * @access private
384 * @return int Log entries to keep (always >= 1).
385 */
386 private function max_log_entries() {
387 $max = (int) apply_filters( 'ayg_import_max_log_entries', 25 );
388 return $max > 0 ? $max : 25;
389 }
390
391 /**
392 * Recover galleries left stuck in the 'running' state.
393 *
394 * @since 2.8.0
395 * @access private
396 */
397 private function recover_stuck_syncs() {
398 global $wpdb;
399
400 $galleries_table = $wpdb->prefix . 'ayg_galleries';
401
402 $now = current_time( 'mysql' );
403 $threshold = date( 'Y-m-d H:i:s', current_time( 'timestamp' ) - 10 * MINUTE_IN_SECONDS );
404
405 $wpdb->query(
406 $wpdb->prepare(
407 "UPDATE $galleries_table
408 SET import_status = 'idle', next_import_at = %s, updated_at = %s
409 WHERE import_status = 'running' AND updated_at < %s",
410 $now,
411 $now,
412 $threshold
413 )
414 );
415 }
416
417 /**
418 * Find the earliest gallery that is due for a scheduled sync.
419 *
420 * @since 2.8.0
421 * @access private
422 * @return int|null Gallery ID, or null when none are due.
423 */
424 private function get_due_gallery_id() {
425 global $wpdb;
426
427 $galleries_table = $wpdb->prefix . 'ayg_galleries';
428
429 $gallery_id = $wpdb->get_var(
430 $wpdb->prepare(
431 "SELECT id FROM $galleries_table
432 WHERE next_import_at IS NOT NULL
433 AND next_import_at < %s
434 AND import_status NOT IN ( 'running', 'paused', 'error' )
435 AND source_type NOT IN ( 'search', 'livestream', 'video' )
436 ORDER BY next_import_at ASC
437 LIMIT 1",
438 current_time( 'mysql' )
439 )
440 );
441
442 return $gallery_id ? (int) $gallery_id : null;
443 }
444
445 /**
446 * Handle an API error response for the current batch.
447 *
448 * @since 2.8.0
449 * @access private
450 * @param object $gallery Gallery row from wp_ayg_galleries.
451 * @param array $params Decoded gallery params.
452 * @param object $response Error object from the API class.
453 * @param string $page_token Page token the failed batch was fetching.
454 * @return array { error, quota_exceeded }
455 */
456 private function handle_api_error( $gallery, $params, $response, $page_token ) {
457 global $wpdb;
458
459 $message = wp_strip_all_tags( $response->error_message );
460 $quota_exceeded = ( false !== stripos( $message, 'quotaExceeded' ) );
461 $status = $quota_exceeded ? 'paused' : 'error';
462
463 // Log what earlier batches imported/updated before the error (deleted = 0: the prune runs only on
464 // a clean completion). Then reset the counters so a resume logs only its own delta rather than
465 // re-counting these. sync_run_start + page_token are kept so a resume continues from the same
466 // point with the same prune marker; only the per-run cap counter restarts.
467 $imported = isset( $params['count_imported'] ) ? (int) $params['count_imported'] : 0;
468 $updated = isset( $params['count_updated'] ) ? (int) $params['count_updated'] : 0;
469
470 $params['page_token'] = $page_token;
471 $params['count_imported'] = 0;
472 $params['count_updated'] = 0;
473 unset( $params['run_processed'] );
474
475 $wpdb->update(
476 $wpdb->prefix . 'ayg_galleries',
477 array(
478 'params' => wp_json_encode( $params ),
479 'import_status' => $status,
480 'import_error' => $message,
481 'updated_at' => current_time( 'mysql' ),
482 'import_log' => $this->append_log_entry( $gallery, $imported, $updated, 0, $status, $message ) // Record the stopped run (counts so far + message) so it shows in the log.
483 ),
484 array( 'id' => $gallery->id ),
485 array(),
486 array( '%d' )
487 );
488
489 return array(
490 'error' => $message,
491 'quota_exceeded' => $quota_exceeded
492 );
493 }
494
495 /**
496 * Append a run entry to the gallery's import_log JSON and return the encoded result.
497 *
498 * @since 2.8.0
499 * @access private
500 * @param object $gallery Gallery row from wp_ayg_galleries.
501 * @param int $imported Number of new videos added by this run.
502 * @param int $updated Number of already-linked videos re-synced this run.
503 * @param int $deleted Number of videos pruned this run (removed at the source).
504 * @param string $status Final run status ('idle' / 'completed' / 'paused' / 'error').
505 * @param string $error Error message for failed runs, empty otherwise.
506 * @return string JSON-encoded import_log.
507 */
508 private function append_log_entry( $gallery, $imported, $updated, $deleted, $status, $error = '' ) {
509 $logs = json_decode( (string) $gallery->import_log, true );
510 if ( ! is_array( $logs ) ) {
511 $logs = array();
512 }
513
514 $entry = array(
515 'date' => current_time( 'mysql' ),
516 'imported' => (int) $imported,
517 'updated' => (int) $updated,
518 'deleted' => (int) $deleted,
519 'status' => $status
520 );
521
522 // Record the error message on failed runs so it can be shown in the log.
523 if ( '' !== $error ) {
524 $entry['error'] = $error;
525 }
526
527 $logs[] = $entry;
528
529 $max = $this->max_log_entries();
530 if ( count( $logs ) > $max ) {
531 $logs = array_slice( $logs, -$max );
532 }
533
534 return wp_json_encode( $logs );
535 }
536
537 }
538