| @@ -69,8 +69,416 @@ | ||
| 69 | 69 | } |
| 70 | 70 | } |
| 71 | 71 | |
| 72 | 72 | /** |
| 73 | + * Source URLs deferred this request - raster images left pointing at the source CDN | |
| 74 | + * because $defer_media was true. Not populated from the cloud's per-template | |
| 75 | + * `site_images` field, which many templates never have populated - the importer records | |
| 76 | + * these itself, at the exact point it decides to defer, so the background sideload pass | |
| 77 | + * always has an accurate list regardless of what (if anything) the cloud sent. | |
| 78 | + * | |
| 79 | + * @since 2.7.1 | |
| 80 | + * | |
| 81 | + * @var array | |
| 82 | + */ | |
| 83 | + private static $deferred_urls = array(); | |
| 84 | + | |
| 85 | + /** | |
| 86 | + * Every image this request could not localise, one entry per failure. | |
| 87 | + * | |
| 88 | + * `log_failed_download()` writes to `Wdkit_Import_Log` for developers reading the | |
| 89 | + * debug log after the fact, but nothing summed those entries into a count a caller | |
| 90 | + * could act on — an import that failed to download ten images still reported full | |
| 91 | + * success everywhere a person or the website actually looks. This is that count, | |
| 92 | + * read the same read-and-clear way `$deferred_urls` above is: once per page/section, | |
| 93 | + * by `Wdkit_Page_Importer::import()`, so it ends up on that page's own result and | |
| 94 | + * from there in the run's aggregate the same way `failed_pages` already is. See | |
| 95 | + * ClickUp 14ynqxywnae. | |
| 96 | + * | |
| 97 | + * @since 2.7.4 | |
| 98 | + * | |
| 99 | + * @var array | |
| 100 | + */ | |
| 101 | + private static $failed_downloads = array(); | |
| 102 | + | |
| 103 | + /** | |
| 104 | + * Hand back every image URL that failed to localise so far this request, and empty | |
| 105 | + * the list. Read-and-clear, matching get_and_clear_deferred_urls() above and for the | |
| 106 | + * same reason: each page must only see its own failures, not a previous page's. | |
| 107 | + * | |
| 108 | + * @since 2.7.4 | |
| 109 | + * | |
| 110 | + * @return array | |
| 111 | + */ | |
| 112 | + public static function get_and_clear_failed_downloads() { | |
| 113 | + $failed = self::$failed_downloads; | |
| 114 | + self::$failed_downloads = array(); | |
| 115 | + | |
| 116 | + return $failed; | |
| 117 | + } | |
| 118 | + | |
| 119 | + /** | |
| 120 | + * Hand back every URL deferred so far this request, and empty the list. | |
| 121 | + * | |
| 122 | + * Read-and-clear in one call so a caller can never forget to reset it for the next item - | |
| 123 | + * each page/section in a kit must only see the URLs its own content deferred, not a | |
| 124 | + * previous item's. | |
| 125 | + * | |
| 126 | + * @since 2.7.1 | |
| 127 | + * | |
| 128 | + * @return array | |
| 129 | + */ | |
| 130 | + public static function get_and_clear_deferred_urls() { | |
| 131 | + $urls = array_values( array_unique( self::$deferred_urls ) ); | |
| 132 | + self::$deferred_urls = array(); | |
| 133 | + | |
| 134 | + return $urls; | |
| 135 | + } | |
| 136 | + | |
| 137 | + /** | |
| 138 | + * Same list, without emptying it. | |
| 139 | + * | |
| 140 | + * The deferral decision is made per-image deep inside the media walk, but the content | |
| 141 | + * rewrite that has to react to it happens back at the top of that walk - which needs to | |
| 142 | + * read the list while the owning caller still has its own read-and-clear to make. | |
| 143 | + * | |
| 144 | + * @since 2.7.1 | |
| 145 | + * | |
| 146 | + * @return array | |
| 147 | + */ | |
| 148 | + public static function peek_deferred_urls() { | |
| 149 | + return array_values( array_unique( self::$deferred_urls ) ); | |
| 150 | + } | |
| 151 | + | |
| 152 | + /** | |
| 153 | + * Record URLs that a caller deferred itself, outside the per-image media walk. | |
| 154 | + * | |
| 155 | + * The Gutenberg walk reaches wdkit_Import_media() once per image and can flag each one | |
| 156 | + * as it goes. The Elementor walk cannot: it hands every control to Elementor's own | |
| 157 | + * on_import(), which downloads and rewrites in a single step, so deferring means skipping | |
| 158 | + * that walk entirely and there is no per-image hook left to observe. The Elementor branch | |
| 159 | + * therefore scans the element tree for foreign media itself and reports the result here, | |
| 160 | + * so both builders end up feeding the same list that get_and_clear_deferred_urls() | |
| 161 | + * returns to the import response. | |
| 162 | + * | |
| 163 | + * @since 2.7.1 | |
| 164 | + * | |
| 165 | + * @param array $urls Source media URLs left pointing at the template's own site. | |
| 166 | + * @return void | |
| 167 | + */ | |
| 168 | + public static function record_deferred_urls( $urls ) { | |
| 169 | + if ( ! is_array( $urls ) ) { | |
| 170 | + return; | |
| 171 | + } | |
| 172 | + | |
| 173 | + foreach ( $urls as $url ) { | |
| 174 | + if ( is_string( $url ) && '' !== $url ) { | |
| 175 | + self::$deferred_urls[] = $url; | |
| 176 | + } | |
| 177 | + } | |
| 178 | + } | |
| 179 | + | |
| 180 | + /** | |
| 181 | + * How many files the prefetch pass fetches at once, and its safety bounds. | |
| 182 | + * | |
| 183 | + * Concurrency is deliberately modest: this points at one origin (the template CDN), | |
| 184 | + * and the win here is eliminating per-file connection setup, not saturating the host. | |
| 185 | + */ | |
| 186 | + const PREFETCH_CONCURRENCY = 8; | |
| 187 | + const PREFETCH_MAX_FILES = 250; | |
| 188 | + const PREFETCH_MAX_BYTES = 2097152; | |
| 189 | + | |
| 190 | + /** | |
| 191 | + * Ceiling on the bodies held in self::$prefetched at once. | |
| 192 | + * | |
| 193 | + * The per-file cap alone does not bound this: 250 files x 2 MB is 500 MB of retained | |
| 194 | + * strings, on top of whatever the import itself needs, and the cap was only applied | |
| 195 | + * after the whole body had already been buffered. Warming stops once this is reached; | |
| 196 | + * everything past it falls through to the sequential path, which is the slow case, not | |
| 197 | + * a broken one. | |
| 198 | + */ | |
| 199 | + const PREFETCH_MAX_TOTAL_BYTES = 33554432; | |
| 200 | + | |
| 201 | + /** | |
| 202 | + * Response bodies already fetched by prefetch_remote_files(), keyed by source URL. | |
| 203 | + * | |
| 204 | + * @since 2.7.1 | |
| 205 | + * | |
| 206 | + * @var array | |
| 207 | + */ | |
| 208 | + private static $prefetched = array(); | |
| 209 | + | |
| 210 | + /** | |
| 211 | + * Fetch a batch of remote files concurrently, so wdkit_Import_media() below can take | |
| 212 | + * them from memory instead of paying a blocking round trip each. | |
| 213 | + * | |
| 214 | + * Why this exists: importing a kit walks the block tree one node at a time, and every | |
| 215 | + * node that carries a foreign image used to make its own sequential HTTP request. On a | |
| 216 | + * real 13-template kit that is 65 SVGs at roughly 0.7s apiece - about 44 seconds, of | |
| 217 | + * which almost none is work: it is connection setup and latency, repeated 65 times | |
| 218 | + * against a single origin. | |
| 219 | + * | |
| 220 | + * Strictly an optimisation, never a dependency: | |
| 221 | + * | |
| 222 | + * - Every URL is put through wdesignkit_validate_external_url() first, the same DNS | |
| 223 | + * resolution + reserved-range check wdesignkit_safe_remote_get() performs, so this | |
| 224 | + * cannot become an SSRF hole that the sequential path would have refused. | |
| 225 | + * - Redirects are NOT followed. wp_safe_remote_get() re-validates each hop; curl would | |
| 226 | + * not, so a redirect to an internal address would slip past the check above. A | |
| 227 | + * redirecting URL is simply left uncached and falls through to the safe path. | |
| 228 | + * - Anything that is not a clean, size-capped 200 is left uncached and falls through | |
| 229 | + * the same way. A failed prefetch costs nothing but the original request. | |
| 230 | + * - The site's own HTTP policy is honoured before dialling: WP_HTTP_BLOCK_EXTERNAL | |
| 231 | + * with WP_ACCESSIBLE_HOSTS, and a configured proxy. curl here would otherwise reach | |
| 232 | + * hosts wp_safe_remote_get() refuses, or leave the proxy - which on a locked-down | |
| 233 | + * host is the only route out - unused. | |
| 234 | + * - Transfers abort mid-flight once they pass PREFETCH_MAX_BYTES, and warming stops | |
| 235 | + * at PREFETCH_MAX_TOTAL_BYTES retained, so neither a single large file nor a large | |
| 236 | + * kit can buffer without bound. | |
| 237 | + * | |
| 238 | + * @since 2.7.1 | |
| 239 | + * | |
| 240 | + * @param array $urls Absolute URLs to warm. | |
| 241 | + * @return void | |
| 242 | + */ | |
| 243 | + public static function prefetch_remote_files( $urls ) { | |
| 244 | + if ( ! is_array( $urls ) || empty( $urls ) || ! function_exists( 'curl_multi_init' ) ) { | |
| 245 | + return; | |
| 246 | + } | |
| 247 | + | |
| 248 | + $queue = array(); | |
| 249 | + | |
| 250 | + foreach ( $urls as $url ) { | |
| 251 | + if ( ! is_string( $url ) || '' === $url || isset( self::$prefetched[ $url ] ) || isset( $queue[ $url ] ) ) { | |
| 252 | + continue; | |
| 253 | + } | |
| 254 | + | |
| 255 | + // Same gate the sequential path applies, applied before anything is dialled. | |
| 256 | + if ( ! function_exists( 'wdesignkit_validate_external_url' ) || ! wdesignkit_validate_external_url( $url ) ) { | |
| 257 | + continue; | |
| 258 | + } | |
| 259 | + | |
| 260 | + // And the same policy WP_Http would apply to this host. | |
| 261 | + if ( ! self::prefetch_allowed_by_http_policy( $url ) ) { | |
| 262 | + continue; | |
| 263 | + } | |
| 264 | + | |
| 265 | + $queue[ $url ] = true; | |
| 266 | + | |
| 267 | + if ( count( $queue ) >= self::PREFETCH_MAX_FILES ) { | |
| 268 | + break; | |
| 269 | + } | |
| 270 | + } | |
| 271 | + | |
| 272 | + $queue = array_keys( $queue ); | |
| 273 | + | |
| 274 | + if ( empty( $queue ) ) { | |
| 275 | + return; | |
| 276 | + } | |
| 277 | + | |
| 278 | + $user_agent = 'WordPress/' . get_bloginfo( 'version' ) . '; ' . home_url( '/' ); | |
| 279 | + $retained = self::prefetched_bytes(); | |
| 280 | + $has_progress = defined( 'CURLOPT_XFERINFOFUNCTION' ); | |
| 281 | + | |
| 282 | + foreach ( array_chunk( $queue, self::PREFETCH_CONCURRENCY ) as $chunk ) { | |
| 283 | + if ( $retained >= self::PREFETCH_MAX_TOTAL_BYTES ) { | |
| 284 | + break; | |
| 285 | + } | |
| 286 | + | |
| 287 | + $multi = curl_multi_init(); | |
| 288 | + | |
| 289 | + if ( false === $multi ) { | |
| 290 | + return; | |
| 291 | + } | |
| 292 | + | |
| 293 | + $handles = array(); | |
| 294 | + | |
| 295 | + foreach ( $chunk as $url ) { | |
| 296 | + $handle = curl_init(); | |
| 297 | + | |
| 298 | + // curl_init() can fail, and every call below would then be handed false. | |
| 299 | + // On PHP 8 that is a TypeError, i.e. a fatal in the middle of an import. | |
| 300 | + if ( false === $handle ) { | |
| 301 | + continue; | |
| 302 | + } | |
| 303 | + | |
| 304 | + $options = array( | |
| 305 | + CURLOPT_URL => $url, | |
| 306 | + CURLOPT_RETURNTRANSFER => true, | |
| 307 | + // See the docblock: following a redirect here would skip the | |
| 308 | + // per-hop validation wp_safe_remote_get() does for us. | |
| 309 | + CURLOPT_FOLLOWLOCATION => false, | |
| 310 | + CURLOPT_CONNECTTIMEOUT => 10, | |
| 311 | + CURLOPT_TIMEOUT => 30, | |
| 312 | + CURLOPT_SSL_VERIFYPEER => true, | |
| 313 | + CURLOPT_SSL_VERIFYHOST => 2, | |
| 314 | + CURLOPT_USERAGENT => $user_agent, | |
| 315 | + // Refuses up front when the server declares an oversized body. | |
| 316 | + CURLOPT_MAXFILESIZE => self::PREFETCH_MAX_BYTES, | |
| 317 | + ); | |
| 318 | + | |
| 319 | + // And catches the servers that declare nothing: without this a chunked | |
| 320 | + // response of any size is buffered in full before being measured and | |
| 321 | + // thrown away. | |
| 322 | + if ( $has_progress ) { | |
| 323 | + $options[ CURLOPT_NOPROGRESS ] = false; | |
| 324 | + $options[ CURLOPT_XFERINFOFUNCTION ] = static function ( $handle, $expected, $received ) { | |
| 325 | + unset( $handle, $expected ); | |
| 326 | + | |
| 327 | + return ( $received > self::PREFETCH_MAX_BYTES ) ? 1 : 0; | |
| 328 | + }; | |
| 329 | + } | |
| 330 | + | |
| 331 | + curl_setopt_array( $handle, $options ); | |
| 332 | + | |
| 333 | + curl_multi_add_handle( $multi, $handle ); | |
| 334 | + $handles[ $url ] = $handle; | |
| 335 | + } | |
| 336 | + | |
| 337 | + if ( empty( $handles ) ) { | |
| 338 | + curl_multi_close( $multi ); | |
| 339 | + continue; | |
| 340 | + } | |
| 341 | + | |
| 342 | + $running = null; | |
| 343 | + $iterations = 0; | |
| 344 | + | |
| 345 | + // Belt and braces on top of the status check: 30s CURLOPT_TIMEOUT against a | |
| 346 | + // 0.5s select is ~60 turns per handle, so this can only fire if the loop is | |
| 347 | + // not waiting at all - the exact shape of the busy-loop being guarded against. | |
| 348 | + $max_iterations = 2000; | |
| 349 | + | |
| 350 | + do { | |
| 351 | + $status = curl_multi_exec( $multi, $running ); | |
| 352 | + | |
| 353 | + // A handle-level failure leaves $running untouched, so a loop that only | |
| 354 | + // looks at $running spins forever on it - and curl_multi_select() returns | |
| 355 | + // -1 immediately on error, so the 0.5s wait does not apply either. | |
| 356 | + if ( CURLM_OK !== $status && CURLM_CALL_MULTI_PERFORM !== $status ) { | |
| 357 | + break; | |
| 358 | + } | |
| 359 | + | |
| 360 | + if ( ++$iterations > $max_iterations ) { | |
| 361 | + break; | |
| 362 | + } | |
| 363 | + | |
| 364 | + if ( $running > 0 && CURLM_CALL_MULTI_PERFORM !== $status ) { | |
| 365 | + curl_multi_select( $multi, 0.5 ); | |
| 366 | + } | |
| 367 | + } while ( $running > 0 ); | |
| 368 | + | |
| 369 | + foreach ( $handles as $url => $handle ) { | |
| 370 | + $body = curl_multi_getcontent( $handle ); | |
| 371 | + $code = (int) curl_getinfo( $handle, CURLINFO_RESPONSE_CODE ); | |
| 372 | + | |
| 373 | + if ( 200 === $code && is_string( $body ) && '' !== $body && strlen( $body ) <= self::PREFETCH_MAX_BYTES ) { | |
| 374 | + if ( ( $retained + strlen( $body ) ) <= self::PREFETCH_MAX_TOTAL_BYTES ) { | |
| 375 | + self::$prefetched[ $url ] = $body; | |
| 376 | + $retained += strlen( $body ); | |
| 377 | + } | |
| 378 | + } | |
| 379 | + | |
| 380 | + unset( $body ); | |
| 381 | + | |
| 382 | + curl_multi_remove_handle( $multi, $handle ); | |
| 383 | + curl_close( $handle ); | |
| 384 | + } | |
| 385 | + | |
| 386 | + curl_multi_close( $multi ); | |
| 387 | + } | |
| 388 | + } | |
| 389 | + | |
| 390 | + /** | |
| 391 | + * Bytes currently held in the warm cache. | |
| 392 | + * | |
| 393 | + * @since 2.7.1 | |
| 394 | + * | |
| 395 | + * @return int | |
| 396 | + */ | |
| 397 | + private static function prefetched_bytes() { | |
| 398 | + $bytes = 0; | |
| 399 | + | |
| 400 | + foreach ( self::$prefetched as $body ) { | |
| 401 | + $bytes += strlen( (string) $body ); | |
| 402 | + } | |
| 403 | + | |
| 404 | + return $bytes; | |
| 405 | + } | |
| 406 | + | |
| 407 | + /** | |
| 408 | + * Would WP_Http let this request out? | |
| 409 | + * | |
| 410 | + * The prefetch pass talks to curl directly, which means none of the site's own HTTP | |
| 411 | + * configuration applies to it. Two cases matter, and in both the answer is to skip | |
| 412 | + * warming and let the sequential wp_safe_remote_get() path handle the URL properly: | |
| 413 | + * | |
| 414 | + * - WP_HTTP_BLOCK_EXTERNAL (with WP_ACCESSIBLE_HOSTS as the allowlist). A site that | |
| 415 | + * blocks outbound HTTP means it; reaching the host anyway because we happen to hold | |
| 416 | + * our own curl handle is not a decision this optimisation gets to make. | |
| 417 | + * - A configured proxy. On a host where the proxy is the only route out, curl would | |
| 418 | + * simply fail - so this is also the difference between a slow import and no images. | |
| 419 | + * | |
| 420 | + * @since 2.7.1 | |
| 421 | + * | |
| 422 | + * @param string $url URL about to be warmed. | |
| 423 | + * @return bool True when the request may be made directly. | |
| 424 | + */ | |
| 425 | + private static function prefetch_allowed_by_http_policy( $url ) { | |
| 426 | + $host = (string) wp_parse_url( $url, PHP_URL_HOST ); | |
| 427 | + | |
| 428 | + if ( '' === $host ) { | |
| 429 | + return false; | |
| 430 | + } | |
| 431 | + | |
| 432 | + if ( defined( 'WP_HTTP_BLOCK_EXTERNAL' ) && WP_HTTP_BLOCK_EXTERNAL ) { | |
| 433 | + $accessible = defined( 'WP_ACCESSIBLE_HOSTS' ) ? preg_split( '|,\s*|', WP_ACCESSIBLE_HOSTS ) : array(); | |
| 434 | + $allowed = false; | |
| 435 | + | |
| 436 | + foreach ( (array) $accessible as $pattern ) { | |
| 437 | + $pattern = trim( (string) $pattern ); | |
| 438 | + | |
| 439 | + if ( '' === $pattern ) { | |
| 440 | + continue; | |
| 441 | + } | |
| 442 | + | |
| 443 | + // WP_ACCESSIBLE_HOSTS supports a leading wildcard, e.g. *.example.com. | |
| 444 | + $regex = '|^' . str_replace( '\*\.', '([a-zA-Z0-9-]+\.)*', preg_quote( $pattern, '|' ) ) . '$|i'; | |
| 445 | + | |
| 446 | + if ( preg_match( $regex, $host ) ) { | |
| 447 | + $allowed = true; | |
| 448 | + break; | |
| 449 | + } | |
| 450 | + } | |
| 451 | + | |
| 452 | + if ( ! $allowed ) { | |
| 453 | + return false; | |
| 454 | + } | |
| 455 | + } | |
| 456 | + | |
| 457 | + if ( class_exists( 'WP_HTTP_Proxy' ) ) { | |
| 458 | + $proxy = new WP_HTTP_Proxy(); | |
| 459 | + | |
| 460 | + if ( $proxy->is_enabled() && $proxy->send_through_proxy( $url ) ) { | |
| 461 | + return false; | |
| 462 | + } | |
| 463 | + } | |
| 464 | + | |
| 465 | + return true; | |
| 466 | + } | |
| 467 | + | |
| 468 | + /** | |
| 469 | + * Drop every warmed body. Called once a kit's walk is done so the bodies do not sit in | |
| 470 | + * memory for the rest of the request. | |
| 471 | + * | |
| 472 | + * @since 2.7.1 | |
| 473 | + * | |
| 474 | + * @return void | |
| 475 | + */ | |
| 476 | + public static function clear_prefetched() { | |
| 477 | + self::$prefetched = array(); | |
| 478 | + } | |
| 479 | + | |
| 480 | + /** | |
| 73 | 481 | * Get attachment url image hash sha1. |
| 74 | 482 | * |
| 75 | 483 | * Retrieve the sha1 hash of the image URL. |
| 76 | 484 | * |
| @@ -175,11 +583,14 @@ | ||
| 175 | 583 | * attachment metadata. |
| 176 | 584 | * |
| 177 | 585 | * @since 1.0.0 |
| 178 | 586 | * |
| 179 | - * @param array $attachment The attachment. | |
| 587 | + * @param array $attachment The attachment. | |
| 588 | + * @param bool $defer_media Record the URL for the cron sideload instead of downloading | |
| 589 | + * now. Video always defers regardless; SVG never does, because | |
| 590 | + * the cron has no user and so no `svg` mime type. | |
| 180 | 591 | */ |
| 181 | - public static function wdkit_Import_media( $attachment ) { | |
| 592 | + public static function wdkit_Import_media( $attachment, $defer_media = false ) { | |
| 182 | 593 | $stored_image = self::get_store_image_saved( $attachment ); |
| 183 | 594 | |
| 184 | 595 | if ( $stored_image ) { |
| 185 | 596 | self::remember_url( $attachment['url'] ?? '', $stored_image['url'] ?? '' ); |
| @@ -189,15 +600,40 @@ | ||
| 189 | 600 | if ( empty( $attachment['url'] ) || ! is_string( $attachment['url'] ) ) { |
| 190 | 601 | return $attachment; |
| 191 | 602 | } |
| 192 | 603 | |
| 193 | - // Take the file name from the URL *path*. basename() on the whole URL keeps any query | |
| 194 | - // string, and "photo.jpeg?w=1920&auto=compress" is not a filename WordPress will | |
| 195 | - // accept - wp_upload_bits() rejects it and every step after that worked on nothing. | |
| 604 | + // Deferred imports skip the network fetch for anything that safely tolerates staying | |
| 605 | + // remote until the background pass localises it - except SVGs. Nexter's animated icon | |
| 606 | + // blocks embed an SVG via <object data="..."> and read object.contentDocument in JS to | |
| 607 | + // drive the draw-on-scroll effect; a cross-origin <object> cannot be read that way, so | |
| 608 | + // the icon never reveals itself and the block renders as an empty box until the | |
| 609 | + // background pass catches up. SVGs are also the cheapest node to import synchronously | |
| 610 | + // (no raster decode - see the processing_resize timing on any .svg in this importer's | |
| 611 | + // own investigation log), so keeping them eager here does not reintroduce the cost this | |
| 612 | + // was written to remove. | |
| 196 | 613 | if ( ! class_exists( 'Wdkit_Image_Guard' ) ) { |
| 197 | 614 | require_once WDKIT_INCLUDES . 'admin/class-wdkit-image-guard.php'; |
| 198 | 615 | } |
| 199 | 616 | |
| 617 | + $media_path = (string) wp_parse_url( $attachment['url'], PHP_URL_PATH ); | |
| 618 | + $is_svg = (bool) preg_match( '/\.svg$/i', $media_path ); | |
| 619 | + | |
| 620 | + // Video defers whether or not the caller asked for deferral. It is the largest thing | |
| 621 | + // a kit references and this function runs inside the request the user is waiting on, | |
| 622 | + // so fetching one here is exactly the cost the deferred path exists to avoid. The | |
| 623 | + // background pass handles it (see wdkit_sideload_deferred_media()); a caller that | |
| 624 | + // never schedules that pass leaves the video remote, which is what it did before | |
| 625 | + // video was importable at all. | |
| 626 | + $is_video = (bool) preg_match( '/\.(?:' . Wdkit_Image_Guard::VIDEO_EXTENSIONS . ')$/i', $media_path ); | |
| 627 | + | |
| 628 | + if ( ( $defer_media || $is_video ) && ! $is_svg ) { | |
| 629 | + self::$deferred_urls[] = $attachment['url']; | |
| 630 | + return $attachment; | |
| 631 | + } | |
| 632 | + | |
| 633 | + // Take the file name from the URL *path*. basename() on the whole URL keeps any query | |
| 634 | + // string, and "photo.jpeg?w=1920&auto=compress" is not a filename WordPress will | |
| 635 | + // accept - wp_upload_bits() rejects it and every step after that worked on nothing. | |
| 200 | 636 | $file_name = Wdkit_Image_Guard::filename_from_url( $attachment['url'] ); |
| 201 | 637 | |
| 202 | 638 | if ( '' === $file_name ) { |
| 203 | 639 | return $attachment; |
| @@ -226,23 +662,45 @@ | ||
| 226 | 662 | // nothing to import and the URL itself still works. |
| 227 | 663 | return $attachment; |
| 228 | 664 | } |
| 229 | 665 | |
| 230 | - $response = wp_safe_remote_get( $attachment['url'] ); | |
| 666 | + // wdesignkit_safe_remote_get(), not wp_safe_remote_get(): this importer ingests the | |
| 667 | + // largest volume of externally supplied URLs in the plugin, and wp_safe_remote_get() | |
| 668 | + // never resolves the hostname — it only rejects a literal-IP host, so a name that | |
| 669 | + // resolves to 127.0.0.1 or 169.254.169.254 passes straight through (SSRF, CWE-918, | |
| 670 | + // ClickUp 86d41ce9w). The wrapper resolves the host and blocks the reserved ranges. | |
| 671 | + // Already pulled down by the concurrent warm-up (see prefetch_remote_files())? Use | |
| 672 | + // that copy. Only a validated, non-redirected, size-capped 200 ever lands there, so | |
| 673 | + // this is the same payload this request would have fetched - just without paying for | |
| 674 | + // the round trip a second time. A miss simply falls through to the request below, | |
| 675 | + // which is what happens for every URL when curl_multi is unavailable. | |
| 676 | + if ( isset( self::$prefetched[ $attachment['url'] ] ) ) { | |
| 677 | + $file_content = self::$prefetched[ $attachment['url'] ]; | |
| 678 | + } else { | |
| 679 | + $response = wdesignkit_safe_remote_get( $attachment['url'] ); | |
| 231 | 680 | |
| 232 | - // A 404 body is still a body: without this check the error page was written to disk | |
| 233 | - // as a .png and inserted as a real attachment, giving a broken image rather than an | |
| 234 | - // honest failure. | |
| 235 | - if ( is_wp_error( $response ) || 200 !== (int) wp_remote_retrieve_response_code( $response ) ) { | |
| 236 | - return $attachment; | |
| 681 | + // A 404 body is still a body: without this check the error page was written to disk | |
| 682 | + // as a .png and inserted as a real attachment, giving a broken image rather than an | |
| 683 | + // honest failure. | |
| 684 | + if ( is_wp_error( $response ) || 200 !== (int) wp_remote_retrieve_response_code( $response ) ) { | |
| 685 | + self::log_failed_download( | |
| 686 | + $attachment, | |
| 687 | + 'fetch_failed', | |
| 688 | + is_wp_error( $response ) ? $response->get_error_message() : 'HTTP ' . wp_remote_retrieve_response_code( $response ) | |
| 689 | + ); | |
| 690 | + | |
| 691 | + return $attachment; | |
| 692 | + } | |
| 693 | + | |
| 694 | + $file_content = wp_remote_retrieve_body( $response ); | |
| 237 | 695 | } |
| 238 | 696 | |
| 239 | - $file_content = wp_remote_retrieve_body( $response ); | |
| 240 | - | |
| 241 | 697 | if ( empty( $file_content ) || ! self::is_image_payload( $file_content, $file_name ) ) { |
| 242 | 698 | // Returning the attachment unchanged keeps the remote URL working. Returning |
| 243 | 699 | // false - as this used to - made the caller overwrite the whole media attribute, |
| 244 | 700 | // so a single failed download removed the image from the page entirely. |
| 701 | + self::log_failed_download( $attachment, 'not_an_image', 'empty body or unrecognised image payload' ); | |
| 702 | + | |
| 245 | 703 | return $attachment; |
| 246 | 704 | } |
| 247 | 705 | |
| 248 | 706 | $upload_data = wp_upload_bits( $file_name, null, $file_content ); |
| @@ -247,8 +705,10 @@ | ||
| 247 | 705 | |
| 248 | 706 | $upload_data = wp_upload_bits( $file_name, null, $file_content ); |
| 249 | 707 | |
| 250 | 708 | if ( ! empty( $upload_data['error'] ) || empty( $upload_data['file'] ) || empty( $upload_data['url'] ) ) { |
| 709 | + self::log_failed_download( $attachment, 'upload_failed', ! empty( $upload_data['error'] ) ? (string) $upload_data['error'] : 'wp_upload_bits() returned no file' ); | |
| 710 | + | |
| 251 | 711 | return $attachment; |
| 252 | 712 | } |
| 253 | 713 | |
| 254 | 714 | $info = wp_check_filetype( $upload_data['file'] ); |
| @@ -258,8 +718,10 @@ | ||
| 258 | 718 | // unknown type was inserted with an empty post_mime_type. |
| 259 | 719 | if ( empty( $info['type'] ) ) { |
| 260 | 720 | wp_delete_file( $upload_data['file'] ); |
| 261 | 721 | |
| 722 | + self::log_failed_download( $attachment, 'unrecognised_filetype', $file_name ); | |
| 723 | + | |
| 262 | 724 | return $attachment; |
| 263 | 725 | } |
| 264 | 726 | |
| 265 | 727 | $post_image = array( |
| @@ -270,8 +732,10 @@ | ||
| 270 | 732 | |
| 271 | 733 | $post_id = wp_insert_attachment( $post_image, $upload_data['file'] ); |
| 272 | 734 | |
| 273 | 735 | if ( is_wp_error( $post_id ) || ! $post_id ) { |
| 736 | + self::log_failed_download( $attachment, 'insert_failed', is_wp_error( $post_id ) ? $post_id->get_error_message() : 'wp_insert_attachment() returned no id' ); | |
| 737 | + | |
| 274 | 738 | return $attachment; |
| 275 | 739 | } |
| 276 | 740 | |
| 277 | 741 | // On REST requests. |
| @@ -297,8 +761,44 @@ | ||
| 297 | 761 | |
| 298 | 762 | self::remember_url( $attachment['url'], $upload_data['url'] ); |
| 299 | 763 | |
| 300 | 764 | return $new_attachment_img; |
| 765 | + } | |
| 766 | + | |
| 767 | + /** | |
| 768 | + * Record that this image could not be localised, so the failure is diagnosable instead | |
| 769 | + * of invisible. The caller still keeps the original remote URL on the page - that part is | |
| 770 | + * deliberate (see the callers' own comments) - this only makes sure someone can find out | |
| 771 | + * it happened, via `wp eval 'print_r( Wdkit_Import_Log::read() );'` or | |
| 772 | + * `Wdkit_Import_Log::report()`. See ClickUp 14ynqxywnae. | |
| 773 | + * | |
| 774 | + * @since 2.7.3 | |
| 775 | + * | |
| 776 | + * @param array $attachment The attachment that failed to import. | |
| 777 | + * @param string $code Short machine-readable reason. | |
| 778 | + * @param string $message Detail for a human reading the log. | |
| 779 | + * @return void | |
| 780 | + */ | |
| 781 | + private static function log_failed_download( $attachment, $code, $message = '' ) { | |
| 782 | + if ( ! class_exists( 'Wdkit_Import_Log' ) ) { | |
| 783 | + require_once WDKIT_INCLUDES . 'admin/import/class-wdkit-import-log.php'; | |
| 784 | + } | |
| 785 | + | |
| 786 | + Wdkit_Import_Log::add( | |
| 787 | + 'failure', | |
| 788 | + array( | |
| 789 | + 'step' => 'media_import', | |
| 790 | + 'code' => $code, | |
| 791 | + 'message' => $message, | |
| 792 | + 'url' => isset( $attachment['url'] ) ? (string) $attachment['url'] : '', | |
| 793 | + ) | |
| 794 | + ); | |
| 795 | + | |
| 796 | + self::$failed_downloads[] = array( | |
| 797 | + 'url' => isset( $attachment['url'] ) ? (string) $attachment['url'] : '', | |
| 798 | + 'code' => $code, | |
| 799 | + 'message' => $message, | |
| 800 | + ); | |
| 301 | 801 | } |
| 302 | 802 | |
| 303 | 803 | /** |
| 304 | 804 | * Get store saved image. |
