| 1 |
<?php |
| 2 |
/** |
| 3 |
* Fetch upcoming attachments a few at a time, in parallel. |
| 4 |
* |
| 5 |
* WPImport used to download every image with its own blocking request on its own fresh |
| 6 |
* connection. On a high-latency link that is almost pure waiting: a 37 KB image cost ~4s |
| 7 |
* (connect, TLS, first byte and body at roughly one round trip each), and an import with 65 |
| 8 |
* remote images spent 207 of its 230 seconds that way. Six of the same image requested at |
| 9 |
* once finished in the same 4s as one. Bandwidth was never the limit; round trips were. |
| 10 |
* |
| 11 |
* HOW IT IS USED |
| 12 |
* |
| 13 |
* 1. A caller that knows what it is about to import `enqueue()`s those URLs, in order. |
| 14 |
* 2. `WPImport::fetch_remote_file()` asks `take( $url )` before doing its own request. |
| 15 |
* A hit hands back a finished temp file. A miss on a QUEUED url fetches a window — |
| 16 |
* that url plus the next few queued ones — in parallel, then answers. |
| 17 |
* A url nobody queued is never fetched here: `take()` returns null and the importer's |
| 18 |
* existing serial request runs exactly as it always has. |
| 19 |
* |
| 20 |
* Windowed and lazy, rather than "download everything first", for two reasons. The import |
| 21 |
* request works to a ~25s budget and streams progress; an up-front pass over every image |
| 22 |
* would be a long silent stall with nothing imported at the end of it. And files land in |
| 23 |
* the SESSION's temp directory, so a window fetched just before a chunk boundary is still |
| 24 |
* there for the next request — and is removed with the session. |
| 25 |
* |
| 26 |
* THREE RULES THIS CLASS MUST NOT BREAK |
| 27 |
* |
| 28 |
* 1. **It may only ever make an import faster, never different.** Every failure — an |
| 29 |
* unusable URL, a non-200, a redirect, a short file, no parallel transport, a proxy — |
| 30 |
* ends in `null`, and the importer's own request then decides the outcome. Nothing here |
| 31 |
* is allowed to turn a download that would have worked into one that does not. |
| 32 |
* 2. **The same URL checks as `wp_safe_remote_get()`.** A pack can point an image anywhere, |
| 33 |
* so this path must not become the way around `wp_http_validate_url()` or |
| 34 |
* WP_HTTP_BLOCK_EXTERNAL. Redirects are NOT followed: core validates each hop, a raw |
| 35 |
* transport call does not, so a redirecting URL is left to the serial path that does. |
| 36 |
* 3. **Never throw, never echo.** It runs inside a live SSE stream. |
| 37 |
* |
| 38 |
* @package Templately\Modules\FullSiteImport |
| 39 |
*/ |
| 40 |
|
| 41 |
namespace Templately\Modules\FullSiteImport\Utils; |
| 42 |
|
| 43 |
use Templately\Core\Capabilities; |
| 44 |
|
| 45 |
class AttachmentPrefetcher { |
| 46 |
|
| 47 |
/** How many requests run at once. Filterable: `templately_fsi_prefetch_concurrency`. */ |
| 48 |
const DEFAULT_CONCURRENCY = 6; |
| 49 |
|
| 50 |
/** A parallel request is a shortcut, not the last word — keep it short; the serial path retries. */ |
| 51 |
const DEFAULT_TIMEOUT = 30; |
| 52 |
|
| 53 |
/** @var string[] Queued URLs, in import order. */ |
| 54 |
private static $queue = []; |
| 55 |
|
| 56 |
/** @var array<string,true> URLs already attempted in this request — never retried here. */ |
| 57 |
private static $attempted = []; |
| 58 |
|
| 59 |
/** @var string|null */ |
| 60 |
private static $dir = null; |
| 61 |
|
| 62 |
/** @var callable|null Receives one line per window, for the import's own event log. */ |
| 63 |
private static $log = null; |
| 64 |
|
| 65 |
/** @var array{hits:int,windows:int,fetched:int,failed:int} */ |
| 66 |
private static $stats = [ 'hits' => 0, 'windows' => 0, 'fetched' => 0, 'failed' => 0 ]; |
| 67 |
|
| 68 |
/** |
| 69 |
* Point the cache at the session's temp directory. Without one, nothing is prefetched. |
| 70 |
* |
| 71 |
* @param string $session_dir Absolute path of the session's working directory. |
| 72 |
* @param callable|null $log Optional sink for one line per window. This class never |
| 73 |
* writes to the output stream itself. |
| 74 |
* @return void |
| 75 |
*/ |
| 76 |
public static function boot( $session_dir, $log = null ) { |
| 77 |
self::$log = is_callable( $log ) ? $log : null; |
| 78 |
self::$queue = []; |
| 79 |
self::$attempted = []; |
| 80 |
self::$stats = [ 'hits' => 0, 'windows' => 0, 'fetched' => 0, 'failed' => 0 ]; |
| 81 |
self::$dir = null; |
| 82 |
|
| 83 |
if ( ! is_string( $session_dir ) || '' === $session_dir || ! is_dir( $session_dir ) ) { |
| 84 |
return; |
| 85 |
} |
| 86 |
|
| 87 |
$dir = trailingslashit( $session_dir ) . 'prefetch'; |
| 88 |
|
| 89 |
if ( ! is_dir( $dir ) && ! wp_mkdir_p( $dir ) ) { |
| 90 |
return; |
| 91 |
} |
| 92 |
|
| 93 |
self::$dir = trailingslashit( $dir ); |
| 94 |
} |
| 95 |
|
| 96 |
/** |
| 97 |
* Whether parallel fetching can run on this host, right now. |
| 98 |
* |
| 99 |
* @return bool |
| 100 |
*/ |
| 101 |
public static function is_available() { |
| 102 |
if ( null === self::$dir ) { |
| 103 |
return false; |
| 104 |
} |
| 105 |
|
| 106 |
if ( ! Capabilities::get_instance()->has( 'http-parallel-requests' ) ) { |
| 107 |
return false; |
| 108 |
} |
| 109 |
|
| 110 |
// A configured proxy is honoured by WP_Http and unknown to a raw transport call. |
| 111 |
// Going around it could fail on a locked-down network or, worse, succeed where the |
| 112 |
// site owner meant requests to be routed. Leave those sites on the serial path. |
| 113 |
if ( defined( 'WP_PROXY_HOST' ) && WP_PROXY_HOST ) { |
| 114 |
return false; |
| 115 |
} |
| 116 |
|
| 117 |
return (bool) apply_filters( 'templately_fsi_prefetch_enabled', true ); |
| 118 |
} |
| 119 |
|
| 120 |
/** |
| 121 |
* Register URLs that are about to be imported, in the order they will be asked for. |
| 122 |
* |
| 123 |
* @param string[] $urls |
| 124 |
* @return void |
| 125 |
*/ |
| 126 |
public static function enqueue( array $urls ) { |
| 127 |
if ( ! self::is_available() ) { |
| 128 |
return; |
| 129 |
} |
| 130 |
|
| 131 |
foreach ( $urls as $url ) { |
| 132 |
if ( ! is_string( $url ) || '' === $url ) { |
| 133 |
continue; |
| 134 |
} |
| 135 |
if ( isset( self::$attempted[ $url ] ) || in_array( $url, self::$queue, true ) ) { |
| 136 |
continue; |
| 137 |
} |
| 138 |
self::$queue[] = $url; |
| 139 |
} |
| 140 |
} |
| 141 |
|
| 142 |
/** |
| 143 |
* A finished download for this URL, or null. |
| 144 |
* |
| 145 |
* The file is handed over: the caller moves it, and the cache entry is gone. |
| 146 |
* |
| 147 |
* @param string $url |
| 148 |
* @return array{file:string,code:int,headers:array<string,string>}|null |
| 149 |
*/ |
| 150 |
public static function take( $url ) { |
| 151 |
if ( ! is_string( $url ) || '' === $url || ! self::is_available() ) { |
| 152 |
return null; |
| 153 |
} |
| 154 |
|
| 155 |
$cached = self::read( $url ); |
| 156 |
|
| 157 |
if ( null === $cached && in_array( $url, self::$queue, true ) ) { |
| 158 |
self::fetch_window( $url ); |
| 159 |
$cached = self::read( $url ); |
| 160 |
} |
| 161 |
|
| 162 |
// Asked for, so no longer upcoming — whatever the outcome. |
| 163 |
self::$queue = array_values( array_diff( self::$queue, [ $url ] ) ); |
| 164 |
|
| 165 |
if ( null === $cached ) { |
| 166 |
return null; |
| 167 |
} |
| 168 |
|
| 169 |
self::$stats['hits']++; |
| 170 |
@unlink( self::meta_path( $url ) ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged |
| 171 |
|
| 172 |
return $cached; |
| 173 |
} |
| 174 |
|
| 175 |
/** |
| 176 |
* Counters for the log — how much of the import this actually carried. |
| 177 |
* |
| 178 |
* @return array{hits:int,windows:int,fetched:int,failed:int} |
| 179 |
*/ |
| 180 |
public static function stats() { |
| 181 |
return self::$stats; |
| 182 |
} |
| 183 |
|
| 184 |
/** |
| 185 |
* Remove whatever was fetched and never asked for. Called once the import is over. |
| 186 |
* |
| 187 |
* @return void |
| 188 |
*/ |
| 189 |
public static function purge() { |
| 190 |
if ( null === self::$dir || ! is_dir( self::$dir ) ) { |
| 191 |
return; |
| 192 |
} |
| 193 |
|
| 194 |
foreach ( (array) glob( self::$dir . '*' ) as $file ) { |
| 195 |
if ( is_string( $file ) && is_file( $file ) ) { |
| 196 |
@unlink( $file ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged |
| 197 |
} |
| 198 |
} |
| 199 |
@rmdir( self::$dir ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged |
| 200 |
|
| 201 |
self::$queue = []; |
| 202 |
} |
| 203 |
|
| 204 |
/** |
| 205 |
* Fetch `$url` and the next queued URLs together. |
| 206 |
* |
| 207 |
* @param string $url |
| 208 |
* @return void |
| 209 |
*/ |
| 210 |
private static function fetch_window( $url ) { |
| 211 |
$size = (int) apply_filters( 'templately_fsi_prefetch_concurrency', self::DEFAULT_CONCURRENCY ); |
| 212 |
$size = max( 1, min( 12, $size ) ); |
| 213 |
|
| 214 |
$start = array_search( $url, self::$queue, true ); |
| 215 |
$window = array_slice( self::$queue, false === $start ? 0 : $start, $size ); |
| 216 |
|
| 217 |
$requests = []; |
| 218 |
foreach ( $window as $candidate ) { |
| 219 |
self::$attempted[ $candidate ] = true; |
| 220 |
|
| 221 |
if ( null !== self::read( $candidate ) || ! self::is_fetchable( $candidate ) ) { |
| 222 |
continue; |
| 223 |
} |
| 224 |
|
| 225 |
$requests[ $candidate ] = [ |
| 226 |
'url' => $candidate, |
| 227 |
'type' => 'GET', |
| 228 |
'headers' => [ 'Accept-Encoding' => 'identity' ], |
| 229 |
'options' => [ 'filename' => self::body_path( $candidate ) ], |
| 230 |
]; |
| 231 |
} |
| 232 |
|
| 233 |
// Whatever was in the window has had its one chance here. |
| 234 |
self::$queue = array_values( array_diff( self::$queue, $window, [ $url ] ) ); |
| 235 |
|
| 236 |
if ( empty( $requests ) ) { |
| 237 |
return; |
| 238 |
} |
| 239 |
|
| 240 |
self::$stats['windows']++; |
| 241 |
$started = microtime( true ); |
| 242 |
|
| 243 |
$options = [ |
| 244 |
'timeout' => (int) apply_filters( 'templately_fsi_prefetch_timeout', self::DEFAULT_TIMEOUT ), |
| 245 |
'connect_timeout' => 10, |
| 246 |
'follow_redirects' => false, |
| 247 |
'useragent' => 'WordPress/' . get_bloginfo( 'version' ) . '; ' . get_bloginfo( 'url' ), |
| 248 |
'verify' => ABSPATH . WPINC . '/certificates/ca-bundle.crt', |
| 249 |
'verifyname' => true, |
| 250 |
]; |
| 251 |
|
| 252 |
/** This filter is documented in wp-includes/class-wp-http.php */ |
| 253 |
if ( ! apply_filters( 'https_ssl_verify', true, '' ) ) { |
| 254 |
$options['verify'] = false; |
| 255 |
$options['verifyname'] = false; |
| 256 |
} |
| 257 |
|
| 258 |
try { |
| 259 |
$responses = \WpOrg\Requests\Requests::request_multiple( $requests, $options ); |
| 260 |
} catch ( \Throwable $e ) { |
| 261 |
$responses = []; |
| 262 |
} |
| 263 |
|
| 264 |
$kept = 0; |
| 265 |
foreach ( $requests as $candidate => $request ) { |
| 266 |
$response = isset( $responses[ $candidate ] ) ? $responses[ $candidate ] : null; |
| 267 |
|
| 268 |
if ( self::keep( $candidate, $response ) ) { |
| 269 |
self::$stats['fetched']++; |
| 270 |
$kept++; |
| 271 |
continue; |
| 272 |
} |
| 273 |
|
| 274 |
self::$stats['failed']++; |
| 275 |
self::discard( $candidate ); |
| 276 |
} |
| 277 |
|
| 278 |
if ( null !== self::$log ) { |
| 279 |
try { |
| 280 |
call_user_func( |
| 281 |
self::$log, |
| 282 |
sprintf( 'Fetched %d of %d attachments in parallel in %.1fs', $kept, count( $requests ), microtime( true ) - $started ) |
| 283 |
); |
| 284 |
} catch ( \Throwable $e ) { |
| 285 |
// A logger that fails is not a reason to fail a download. |
| 286 |
unset( $e ); |
| 287 |
} |
| 288 |
} |
| 289 |
} |
| 290 |
|
| 291 |
/** |
| 292 |
* Decide whether a parallel response is good enough to stand in for the serial one. |
| 293 |
* |
| 294 |
* Deliberately strict. A rejected file costs one ordinary request; an accepted bad one |
| 295 |
* becomes a broken image on the customer's site. |
| 296 |
* |
| 297 |
* @param string $url |
| 298 |
* @param mixed $response |
| 299 |
* @return bool |
| 300 |
*/ |
| 301 |
private static function keep( $url, $response ) { |
| 302 |
if ( ! is_object( $response ) || ! isset( $response->status_code ) || 200 !== (int) $response->status_code ) { |
| 303 |
return false; |
| 304 |
} |
| 305 |
|
| 306 |
$body = self::body_path( $url ); |
| 307 |
$size = file_exists( $body ) ? (int) filesize( $body ) : 0; |
| 308 |
|
| 309 |
if ( $size <= 0 ) { |
| 310 |
return false; |
| 311 |
} |
| 312 |
|
| 313 |
$headers = []; |
| 314 |
foreach ( [ 'content-type', 'content-length', 'content-encoding', 'content-disposition' ] as $name ) { |
| 315 |
$value = isset( $response->headers[ $name ] ) ? $response->headers[ $name ] : null; |
| 316 |
if ( is_string( $value ) && '' !== $value ) { |
| 317 |
$headers[ $name ] = $value; |
| 318 |
} |
| 319 |
} |
| 320 |
|
| 321 |
if ( ! isset( $headers['content-encoding'] ) && isset( $headers['content-length'] ) && (int) $headers['content-length'] !== $size ) { |
| 322 |
return false; |
| 323 |
} |
| 324 |
|
| 325 |
$meta = wp_json_encode( [ 'url' => $url, 'code' => 200, 'headers' => $headers ] ); |
| 326 |
|
| 327 |
return false !== $meta && false !== file_put_contents( self::meta_path( $url ), $meta ); // phpcs:ignore WordPress.WP.AlternativeFunctions |
| 328 |
} |
| 329 |
|
| 330 |
/** |
| 331 |
* The checks `wp_safe_remote_get()` would have made before sending anything. |
| 332 |
* |
| 333 |
* @param string $url |
| 334 |
* @return bool |
| 335 |
*/ |
| 336 |
private static function is_fetchable( $url ) { |
| 337 |
if ( ! wp_http_validate_url( $url ) ) { |
| 338 |
return false; |
| 339 |
} |
| 340 |
|
| 341 |
$scheme = strtolower( (string) wp_parse_url( $url, PHP_URL_SCHEME ) ); |
| 342 |
if ( 'https' !== $scheme && 'http' !== $scheme ) { |
| 343 |
return false; |
| 344 |
} |
| 345 |
|
| 346 |
$http = _wp_http_get_object(); |
| 347 |
|
| 348 |
return ! $http->block_request( $url ); |
| 349 |
} |
| 350 |
|
| 351 |
/** |
| 352 |
* @param string $url |
| 353 |
* @return array{file:string,code:int,headers:array<string,string>}|null |
| 354 |
*/ |
| 355 |
private static function read( $url ) { |
| 356 |
if ( null === self::$dir ) { |
| 357 |
return null; |
| 358 |
} |
| 359 |
|
| 360 |
$meta_path = self::meta_path( $url ); |
| 361 |
$body_path = self::body_path( $url ); |
| 362 |
|
| 363 |
if ( ! file_exists( $meta_path ) || ! file_exists( $body_path ) ) { |
| 364 |
return null; |
| 365 |
} |
| 366 |
|
| 367 |
$meta = json_decode( (string) file_get_contents( $meta_path ), true ); // phpcs:ignore WordPress.WP.AlternativeFunctions |
| 368 |
|
| 369 |
// The key is a hash, so confirm the entry really is for this URL. |
| 370 |
if ( ! is_array( $meta ) || ! isset( $meta['url'] ) || $meta['url'] !== $url ) { |
| 371 |
return null; |
| 372 |
} |
| 373 |
|
| 374 |
return [ |
| 375 |
'file' => $body_path, |
| 376 |
'code' => isset( $meta['code'] ) ? (int) $meta['code'] : 0, |
| 377 |
'headers' => isset( $meta['headers'] ) && is_array( $meta['headers'] ) ? $meta['headers'] : [], |
| 378 |
]; |
| 379 |
} |
| 380 |
|
| 381 |
private static function discard( $url ) { |
| 382 |
@unlink( self::body_path( $url ) ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged |
| 383 |
@unlink( self::meta_path( $url ) ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged |
| 384 |
} |
| 385 |
|
| 386 |
private static function body_path( $url ) { |
| 387 |
return self::$dir . sha1( $url ) . '.bin'; |
| 388 |
} |
| 389 |
|
| 390 |
private static function meta_path( $url ) { |
| 391 |
return self::$dir . sha1( $url ) . '.json'; |
| 392 |
} |
| 393 |
} |
| 394 |
|
