| @@ -34,8 +34,451 @@ | ||
| 34 | 34 | */ |
| 35 | 35 | private static $new_image_ids = array(); |
| 36 | 36 | |
| 37 | 37 | /** |
| 38 | + * Source URL => local URL for everything localised this request. | |
| 39 | + * | |
| 40 | + * Block markup keeps a rendered copy of the image in innerHTML, and rewriting only the | |
| 41 | + * attributes leaves that copy pointing at the site the template came from. | |
| 42 | + * | |
| 43 | + * @since 2.6.2 | |
| 44 | + * | |
| 45 | + * @var array | |
| 46 | + */ | |
| 47 | + private static $url_map = array(); | |
| 48 | + | |
| 49 | + /** | |
| 50 | + * Source URL => local URL for media localised during this request. | |
| 51 | + * | |
| 52 | + * @since 2.6.2 | |
| 53 | + * | |
| 54 | + * @return array | |
| 55 | + */ | |
| 56 | + public static function get_url_map() { | |
| 57 | + return self::$url_map; | |
| 58 | + } | |
| 59 | + | |
| 60 | + /** | |
| 61 | + * Record that a source URL now lives locally. | |
| 62 | + * | |
| 63 | + * @param string $from Source URL. | |
| 64 | + * @param string $to Local URL. | |
| 65 | + */ | |
| 66 | + private static function remember_url( $from, $to ) { | |
| 67 | + if ( is_string( $from ) && is_string( $to ) && '' !== $from && '' !== $to && $from !== $to ) { | |
| 68 | + self::$url_map[ $from ] = $to; | |
| 69 | + } | |
| 70 | + } | |
| 71 | + | |
| 72 | + /** | |
| 73 | + * Source URLs deferred this request - raster images left pointing at the source CDN | |
| 74 | + * because $defer_media was true. Not populated from the cloud's per-template | |
| 75 | + * `site_images` field, which many templates never have populated - the importer records | |
| 76 | + * these itself, at the exact point it decides to defer, so the background sideload pass | |
| 77 | + * always has an accurate list regardless of what (if anything) the cloud sent. | |
| 78 | + * | |
| 79 | + * @since 2.7.1 | |
| 80 | + * | |
| 81 | + * @var array | |
| 82 | + */ | |
| 83 | + private static $deferred_urls = array(); | |
| 84 | + | |
| 85 | + /** | |
| 86 | + * Every image this request could not localise, one entry per failure. | |
| 87 | + * | |
| 88 | + * `log_failed_download()` writes to `Wdkit_Import_Log` for developers reading the | |
| 89 | + * debug log after the fact, but nothing summed those entries into a count a caller | |
| 90 | + * could act on — an import that failed to download ten images still reported full | |
| 91 | + * success everywhere a person or the website actually looks. This is that count, | |
| 92 | + * read the same read-and-clear way `$deferred_urls` above is: once per page/section, | |
| 93 | + * by `Wdkit_Page_Importer::import()`, so it ends up on that page's own result and | |
| 94 | + * from there in the run's aggregate the same way `failed_pages` already is. See | |
| 95 | + * ClickUp 14ynqxywnae. | |
| 96 | + * | |
| 97 | + * @since 2.7.4 | |
| 98 | + * | |
| 99 | + * @var array | |
| 100 | + */ | |
| 101 | + private static $failed_downloads = array(); | |
| 102 | + | |
| 103 | + /** | |
| 104 | + * Hand back every image URL that failed to localise so far this request, and empty | |
| 105 | + * the list. Read-and-clear, matching get_and_clear_deferred_urls() above and for the | |
| 106 | + * same reason: each page must only see its own failures, not a previous page's. | |
| 107 | + * | |
| 108 | + * @since 2.7.4 | |
| 109 | + * | |
| 110 | + * @return array | |
| 111 | + */ | |
| 112 | + public static function get_and_clear_failed_downloads() { | |
| 113 | + $failed = self::$failed_downloads; | |
| 114 | + self::$failed_downloads = array(); | |
| 115 | + | |
| 116 | + return $failed; | |
| 117 | + } | |
| 118 | + | |
| 119 | + /** | |
| 120 | + * Hand back every URL deferred so far this request, and empty the list. | |
| 121 | + * | |
| 122 | + * Read-and-clear in one call so a caller can never forget to reset it for the next item - | |
| 123 | + * each page/section in a kit must only see the URLs its own content deferred, not a | |
| 124 | + * previous item's. | |
| 125 | + * | |
| 126 | + * @since 2.7.1 | |
| 127 | + * | |
| 128 | + * @return array | |
| 129 | + */ | |
| 130 | + public static function get_and_clear_deferred_urls() { | |
| 131 | + $urls = array_values( array_unique( self::$deferred_urls ) ); | |
| 132 | + self::$deferred_urls = array(); | |
| 133 | + | |
| 134 | + return $urls; | |
| 135 | + } | |
| 136 | + | |
| 137 | + /** | |
| 138 | + * Same list, without emptying it. | |
| 139 | + * | |
| 140 | + * The deferral decision is made per-image deep inside the media walk, but the content | |
| 141 | + * rewrite that has to react to it happens back at the top of that walk - which needs to | |
| 142 | + * read the list while the owning caller still has its own read-and-clear to make. | |
| 143 | + * | |
| 144 | + * @since 2.7.1 | |
| 145 | + * | |
| 146 | + * @return array | |
| 147 | + */ | |
| 148 | + public static function peek_deferred_urls() { | |
| 149 | + return array_values( array_unique( self::$deferred_urls ) ); | |
| 150 | + } | |
| 151 | + | |
| 152 | + /** | |
| 153 | + * Record URLs that a caller deferred itself, outside the per-image media walk. | |
| 154 | + * | |
| 155 | + * The Gutenberg walk reaches wdkit_Import_media() once per image and can flag each one | |
| 156 | + * as it goes. The Elementor walk cannot: it hands every control to Elementor's own | |
| 157 | + * on_import(), which downloads and rewrites in a single step, so deferring means skipping | |
| 158 | + * that walk entirely and there is no per-image hook left to observe. The Elementor branch | |
| 159 | + * therefore scans the element tree for foreign media itself and reports the result here, | |
| 160 | + * so both builders end up feeding the same list that get_and_clear_deferred_urls() | |
| 161 | + * returns to the import response. | |
| 162 | + * | |
| 163 | + * @since 2.7.1 | |
| 164 | + * | |
| 165 | + * @param array $urls Source media URLs left pointing at the template's own site. | |
| 166 | + * @return void | |
| 167 | + */ | |
| 168 | + public static function record_deferred_urls( $urls ) { | |
| 169 | + if ( ! is_array( $urls ) ) { | |
| 170 | + return; | |
| 171 | + } | |
| 172 | + | |
| 173 | + foreach ( $urls as $url ) { | |
| 174 | + if ( is_string( $url ) && '' !== $url ) { | |
| 175 | + self::$deferred_urls[] = $url; | |
| 176 | + } | |
| 177 | + } | |
| 178 | + } | |
| 179 | + | |
| 180 | + /** | |
| 181 | + * How many files the prefetch pass fetches at once, and its safety bounds. | |
| 182 | + * | |
| 183 | + * Concurrency is deliberately modest: this points at one origin (the template CDN), | |
| 184 | + * and the win here is eliminating per-file connection setup, not saturating the host. | |
| 185 | + */ | |
| 186 | + const PREFETCH_CONCURRENCY = 8; | |
| 187 | + const PREFETCH_MAX_FILES = 250; | |
| 188 | + const PREFETCH_MAX_BYTES = 2097152; | |
| 189 | + | |
| 190 | + /** | |
| 191 | + * Ceiling on the bodies held in self::$prefetched at once. | |
| 192 | + * | |
| 193 | + * The per-file cap alone does not bound this: 250 files x 2 MB is 500 MB of retained | |
| 194 | + * strings, on top of whatever the import itself needs, and the cap was only applied | |
| 195 | + * after the whole body had already been buffered. Warming stops once this is reached; | |
| 196 | + * everything past it falls through to the sequential path, which is the slow case, not | |
| 197 | + * a broken one. | |
| 198 | + */ | |
| 199 | + const PREFETCH_MAX_TOTAL_BYTES = 33554432; | |
| 200 | + | |
| 201 | + /** | |
| 202 | + * Response bodies already fetched by prefetch_remote_files(), keyed by source URL. | |
| 203 | + * | |
| 204 | + * @since 2.7.1 | |
| 205 | + * | |
| 206 | + * @var array | |
| 207 | + */ | |
| 208 | + private static $prefetched = array(); | |
| 209 | + | |
| 210 | + /** | |
| 211 | + * Fetch a batch of remote files concurrently, so wdkit_Import_media() below can take | |
| 212 | + * them from memory instead of paying a blocking round trip each. | |
| 213 | + * | |
| 214 | + * Why this exists: importing a kit walks the block tree one node at a time, and every | |
| 215 | + * node that carries a foreign image used to make its own sequential HTTP request. On a | |
| 216 | + * real 13-template kit that is 65 SVGs at roughly 0.7s apiece - about 44 seconds, of | |
| 217 | + * which almost none is work: it is connection setup and latency, repeated 65 times | |
| 218 | + * against a single origin. | |
| 219 | + * | |
| 220 | + * Strictly an optimisation, never a dependency: | |
| 221 | + * | |
| 222 | + * - Every URL is put through wdesignkit_validate_external_url() first, the same DNS | |
| 223 | + * resolution + reserved-range check wdesignkit_safe_remote_get() performs, so this | |
| 224 | + * cannot become an SSRF hole that the sequential path would have refused. | |
| 225 | + * - Redirects are NOT followed. wp_safe_remote_get() re-validates each hop; curl would | |
| 226 | + * not, so a redirect to an internal address would slip past the check above. A | |
| 227 | + * redirecting URL is simply left uncached and falls through to the safe path. | |
| 228 | + * - Anything that is not a clean, size-capped 200 is left uncached and falls through | |
| 229 | + * the same way. A failed prefetch costs nothing but the original request. | |
| 230 | + * - The site's own HTTP policy is honoured before dialling: WP_HTTP_BLOCK_EXTERNAL | |
| 231 | + * with WP_ACCESSIBLE_HOSTS, and a configured proxy. curl here would otherwise reach | |
| 232 | + * hosts wp_safe_remote_get() refuses, or leave the proxy - which on a locked-down | |
| 233 | + * host is the only route out - unused. | |
| 234 | + * - Transfers abort mid-flight once they pass PREFETCH_MAX_BYTES, and warming stops | |
| 235 | + * at PREFETCH_MAX_TOTAL_BYTES retained, so neither a single large file nor a large | |
| 236 | + * kit can buffer without bound. | |
| 237 | + * | |
| 238 | + * @since 2.7.1 | |
| 239 | + * | |
| 240 | + * @param array $urls Absolute URLs to warm. | |
| 241 | + * @return void | |
| 242 | + */ | |
| 243 | + public static function prefetch_remote_files( $urls ) { | |
| 244 | + if ( ! is_array( $urls ) || empty( $urls ) || ! function_exists( 'curl_multi_init' ) ) { | |
| 245 | + return; | |
| 246 | + } | |
| 247 | + | |
| 248 | + $queue = array(); | |
| 249 | + | |
| 250 | + foreach ( $urls as $url ) { | |
| 251 | + if ( ! is_string( $url ) || '' === $url || isset( self::$prefetched[ $url ] ) || isset( $queue[ $url ] ) ) { | |
| 252 | + continue; | |
| 253 | + } | |
| 254 | + | |
| 255 | + // Same gate the sequential path applies, applied before anything is dialled. | |
| 256 | + if ( ! function_exists( 'wdesignkit_validate_external_url' ) || ! wdesignkit_validate_external_url( $url ) ) { | |
| 257 | + continue; | |
| 258 | + } | |
| 259 | + | |
| 260 | + // And the same policy WP_Http would apply to this host. | |
| 261 | + if ( ! self::prefetch_allowed_by_http_policy( $url ) ) { | |
| 262 | + continue; | |
| 263 | + } | |
| 264 | + | |
| 265 | + $queue[ $url ] = true; | |
| 266 | + | |
| 267 | + if ( count( $queue ) >= self::PREFETCH_MAX_FILES ) { | |
| 268 | + break; | |
| 269 | + } | |
| 270 | + } | |
| 271 | + | |
| 272 | + $queue = array_keys( $queue ); | |
| 273 | + | |
| 274 | + if ( empty( $queue ) ) { | |
| 275 | + return; | |
| 276 | + } | |
| 277 | + | |
| 278 | + $user_agent = 'WordPress/' . get_bloginfo( 'version' ) . '; ' . home_url( '/' ); | |
| 279 | + $retained = self::prefetched_bytes(); | |
| 280 | + $has_progress = defined( 'CURLOPT_XFERINFOFUNCTION' ); | |
| 281 | + | |
| 282 | + foreach ( array_chunk( $queue, self::PREFETCH_CONCURRENCY ) as $chunk ) { | |
| 283 | + if ( $retained >= self::PREFETCH_MAX_TOTAL_BYTES ) { | |
| 284 | + break; | |
| 285 | + } | |
| 286 | + | |
| 287 | + $multi = curl_multi_init(); | |
| 288 | + | |
| 289 | + if ( false === $multi ) { | |
| 290 | + return; | |
| 291 | + } | |
| 292 | + | |
| 293 | + $handles = array(); | |
| 294 | + | |
| 295 | + foreach ( $chunk as $url ) { | |
| 296 | + $handle = curl_init(); | |
| 297 | + | |
| 298 | + // curl_init() can fail, and every call below would then be handed false. | |
| 299 | + // On PHP 8 that is a TypeError, i.e. a fatal in the middle of an import. | |
| 300 | + if ( false === $handle ) { | |
| 301 | + continue; | |
| 302 | + } | |
| 303 | + | |
| 304 | + $options = array( | |
| 305 | + CURLOPT_URL => $url, | |
| 306 | + CURLOPT_RETURNTRANSFER => true, | |
| 307 | + // See the docblock: following a redirect here would skip the | |
| 308 | + // per-hop validation wp_safe_remote_get() does for us. | |
| 309 | + CURLOPT_FOLLOWLOCATION => false, | |
| 310 | + CURLOPT_CONNECTTIMEOUT => 10, | |
| 311 | + CURLOPT_TIMEOUT => 30, | |
| 312 | + CURLOPT_SSL_VERIFYPEER => true, | |
| 313 | + CURLOPT_SSL_VERIFYHOST => 2, | |
| 314 | + CURLOPT_USERAGENT => $user_agent, | |
| 315 | + // Refuses up front when the server declares an oversized body. | |
| 316 | + CURLOPT_MAXFILESIZE => self::PREFETCH_MAX_BYTES, | |
| 317 | + ); | |
| 318 | + | |
| 319 | + // And catches the servers that declare nothing: without this a chunked | |
| 320 | + // response of any size is buffered in full before being measured and | |
| 321 | + // thrown away. | |
| 322 | + if ( $has_progress ) { | |
| 323 | + $options[ CURLOPT_NOPROGRESS ] = false; | |
| 324 | + $options[ CURLOPT_XFERINFOFUNCTION ] = static function ( $handle, $expected, $received ) { | |
| 325 | + unset( $handle, $expected ); | |
| 326 | + | |
| 327 | + return ( $received > self::PREFETCH_MAX_BYTES ) ? 1 : 0; | |
| 328 | + }; | |
| 329 | + } | |
| 330 | + | |
| 331 | + curl_setopt_array( $handle, $options ); | |
| 332 | + | |
| 333 | + curl_multi_add_handle( $multi, $handle ); | |
| 334 | + $handles[ $url ] = $handle; | |
| 335 | + } | |
| 336 | + | |
| 337 | + if ( empty( $handles ) ) { | |
| 338 | + curl_multi_close( $multi ); | |
| 339 | + continue; | |
| 340 | + } | |
| 341 | + | |
| 342 | + $running = null; | |
| 343 | + $iterations = 0; | |
| 344 | + | |
| 345 | + // Belt and braces on top of the status check: 30s CURLOPT_TIMEOUT against a | |
| 346 | + // 0.5s select is ~60 turns per handle, so this can only fire if the loop is | |
| 347 | + // not waiting at all - the exact shape of the busy-loop being guarded against. | |
| 348 | + $max_iterations = 2000; | |
| 349 | + | |
| 350 | + do { | |
| 351 | + $status = curl_multi_exec( $multi, $running ); | |
| 352 | + | |
| 353 | + // A handle-level failure leaves $running untouched, so a loop that only | |
| 354 | + // looks at $running spins forever on it - and curl_multi_select() returns | |
| 355 | + // -1 immediately on error, so the 0.5s wait does not apply either. | |
| 356 | + if ( CURLM_OK !== $status && CURLM_CALL_MULTI_PERFORM !== $status ) { | |
| 357 | + break; | |
| 358 | + } | |
| 359 | + | |
| 360 | + if ( ++$iterations > $max_iterations ) { | |
| 361 | + break; | |
| 362 | + } | |
| 363 | + | |
| 364 | + if ( $running > 0 && CURLM_CALL_MULTI_PERFORM !== $status ) { | |
| 365 | + curl_multi_select( $multi, 0.5 ); | |
| 366 | + } | |
| 367 | + } while ( $running > 0 ); | |
| 368 | + | |
| 369 | + foreach ( $handles as $url => $handle ) { | |
| 370 | + $body = curl_multi_getcontent( $handle ); | |
| 371 | + $code = (int) curl_getinfo( $handle, CURLINFO_RESPONSE_CODE ); | |
| 372 | + | |
| 373 | + if ( 200 === $code && is_string( $body ) && '' !== $body && strlen( $body ) <= self::PREFETCH_MAX_BYTES ) { | |
| 374 | + if ( ( $retained + strlen( $body ) ) <= self::PREFETCH_MAX_TOTAL_BYTES ) { | |
| 375 | + self::$prefetched[ $url ] = $body; | |
| 376 | + $retained += strlen( $body ); | |
| 377 | + } | |
| 378 | + } | |
| 379 | + | |
| 380 | + unset( $body ); | |
| 381 | + | |
| 382 | + curl_multi_remove_handle( $multi, $handle ); | |
| 383 | + curl_close( $handle ); | |
| 384 | + } | |
| 385 | + | |
| 386 | + curl_multi_close( $multi ); | |
| 387 | + } | |
| 388 | + } | |
| 389 | + | |
| 390 | + /** | |
| 391 | + * Bytes currently held in the warm cache. | |
| 392 | + * | |
| 393 | + * @since 2.7.1 | |
| 394 | + * | |
| 395 | + * @return int | |
| 396 | + */ | |
| 397 | + private static function prefetched_bytes() { | |
| 398 | + $bytes = 0; | |
| 399 | + | |
| 400 | + foreach ( self::$prefetched as $body ) { | |
| 401 | + $bytes += strlen( (string) $body ); | |
| 402 | + } | |
| 403 | + | |
| 404 | + return $bytes; | |
| 405 | + } | |
| 406 | + | |
| 407 | + /** | |
| 408 | + * Would WP_Http let this request out? | |
| 409 | + * | |
| 410 | + * The prefetch pass talks to curl directly, which means none of the site's own HTTP | |
| 411 | + * configuration applies to it. Two cases matter, and in both the answer is to skip | |
| 412 | + * warming and let the sequential wp_safe_remote_get() path handle the URL properly: | |
| 413 | + * | |
| 414 | + * - WP_HTTP_BLOCK_EXTERNAL (with WP_ACCESSIBLE_HOSTS as the allowlist). A site that | |
| 415 | + * blocks outbound HTTP means it; reaching the host anyway because we happen to hold | |
| 416 | + * our own curl handle is not a decision this optimisation gets to make. | |
| 417 | + * - A configured proxy. On a host where the proxy is the only route out, curl would | |
| 418 | + * simply fail - so this is also the difference between a slow import and no images. | |
| 419 | + * | |
| 420 | + * @since 2.7.1 | |
| 421 | + * | |
| 422 | + * @param string $url URL about to be warmed. | |
| 423 | + * @return bool True when the request may be made directly. | |
| 424 | + */ | |
| 425 | + private static function prefetch_allowed_by_http_policy( $url ) { | |
| 426 | + $host = (string) wp_parse_url( $url, PHP_URL_HOST ); | |
| 427 | + | |
| 428 | + if ( '' === $host ) { | |
| 429 | + return false; | |
| 430 | + } | |
| 431 | + | |
| 432 | + if ( defined( 'WP_HTTP_BLOCK_EXTERNAL' ) && WP_HTTP_BLOCK_EXTERNAL ) { | |
| 433 | + $accessible = defined( 'WP_ACCESSIBLE_HOSTS' ) ? preg_split( '|,\s*|', WP_ACCESSIBLE_HOSTS ) : array(); | |
| 434 | + $allowed = false; | |
| 435 | + | |
| 436 | + foreach ( (array) $accessible as $pattern ) { | |
| 437 | + $pattern = trim( (string) $pattern ); | |
| 438 | + | |
| 439 | + if ( '' === $pattern ) { | |
| 440 | + continue; | |
| 441 | + } | |
| 442 | + | |
| 443 | + // WP_ACCESSIBLE_HOSTS supports a leading wildcard, e.g. *.example.com. | |
| 444 | + $regex = '|^' . str_replace( '\*\.', '([a-zA-Z0-9-]+\.)*', preg_quote( $pattern, '|' ) ) . '$|i'; | |
| 445 | + | |
| 446 | + if ( preg_match( $regex, $host ) ) { | |
| 447 | + $allowed = true; | |
| 448 | + break; | |
| 449 | + } | |
| 450 | + } | |
| 451 | + | |
| 452 | + if ( ! $allowed ) { | |
| 453 | + return false; | |
| 454 | + } | |
| 455 | + } | |
| 456 | + | |
| 457 | + if ( class_exists( 'WP_HTTP_Proxy' ) ) { | |
| 458 | + $proxy = new WP_HTTP_Proxy(); | |
| 459 | + | |
| 460 | + if ( $proxy->is_enabled() && $proxy->send_through_proxy( $url ) ) { | |
| 461 | + return false; | |
| 462 | + } | |
| 463 | + } | |
| 464 | + | |
| 465 | + return true; | |
| 466 | + } | |
| 467 | + | |
| 468 | + /** | |
| 469 | + * Drop every warmed body. Called once a kit's walk is done so the bodies do not sit in | |
| 470 | + * memory for the rest of the request. | |
| 471 | + * | |
| 472 | + * @since 2.7.1 | |
| 473 | + * | |
| 474 | + * @return void | |
| 475 | + */ | |
| 476 | + public static function clear_prefetched() { | |
| 477 | + self::$prefetched = array(); | |
| 478 | + } | |
| 479 | + | |
| 480 | + /** | |
| 38 | 481 | * Get attachment url image hash sha1. |
| 39 | 482 | * |
| 40 | 483 | * Retrieve the sha1 hash of the image URL. |
| 41 | 484 | * |
| @@ -48,8 +491,92 @@ | ||
| 48 | 491 | return sha1( $attachment_url ); |
| 49 | 492 | } |
| 50 | 493 | |
| 51 | 494 | /** |
| 495 | + * Is the file behind this attachment something we can actually show? | |
| 496 | + * | |
| 497 | + * Guards against reusing a previously stored file that turned out empty. An SVG needs at | |
| 498 | + * least one drawing element - a sanitiser that fails mid-way leaves a valid but blank | |
| 499 | + * `<svg></svg>`, which renders as nothing and is easy to mistake for a missing import. | |
| 500 | + * Raster files are checked by asking whether they can be read as an image at all. | |
| 501 | + * | |
| 502 | + * @since 2.6.2 | |
| 503 | + * | |
| 504 | + * @param int $attachment_id Attachment to check. | |
| 505 | + * @return bool True when the file is present and has real content. | |
| 506 | + */ | |
| 507 | + private static function attachment_file_is_usable( $attachment_id ) { | |
| 508 | + $file = get_attached_file( $attachment_id ); | |
| 509 | + | |
| 510 | + if ( empty( $file ) || ! file_exists( $file ) || ! filesize( $file ) ) { | |
| 511 | + return false; | |
| 512 | + } | |
| 513 | + | |
| 514 | + if ( preg_match( '/\.svg$/i', $file ) ) { | |
| 515 | + $svg = (string) file_get_contents( $file ); // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents -- local file, no HTTP involved. | |
| 516 | + | |
| 517 | + return (bool) preg_match( '/<(?:path|circle|rect|polygon|polyline|ellipse|line|g|use|image|text)[\s>\/]/i', $svg ); | |
| 518 | + } | |
| 519 | + | |
| 520 | + return false !== @getimagesize( $file ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- an unreadable file is the case being detected. | |
| 521 | + } | |
| 522 | + | |
| 523 | + /** | |
| 524 | + * Resolve one of our own upload URLs back to its attachment ID. | |
| 525 | + * | |
| 526 | + * Mirrors the size suffixes WordPress adds but does not record in _wp_attached_file: the | |
| 527 | + * "-scaled" copy kept for oversized originals, and any "-1920x1280" intermediate size. | |
| 528 | + * attachment_url_to_postid() misses both. | |
| 529 | + * | |
| 530 | + * @since 2.6.2 | |
| 531 | + * | |
| 532 | + * @param string $url Local upload URL. | |
| 533 | + * @return int Attachment ID, or 0. | |
| 534 | + */ | |
| 535 | + private static function attachment_id_from_local_url( $url ) { | |
| 536 | + $id = (int) attachment_url_to_postid( $url ); | |
| 537 | + | |
| 538 | + if ( $id ) { | |
| 539 | + return $id; | |
| 540 | + } | |
| 541 | + | |
| 542 | + $original = preg_replace( '/-scaled(\.[a-z0-9]+)$/i', '$1', $url ); | |
| 543 | + $original = preg_replace( '/-\d+x\d+(\.[a-z0-9]+)$/i', '$1', (string) $original ); | |
| 544 | + | |
| 545 | + if ( $original && $original !== $url ) { | |
| 546 | + return (int) attachment_url_to_postid( $original ); | |
| 547 | + } | |
| 548 | + | |
| 549 | + return 0; | |
| 550 | + } | |
| 551 | + | |
| 552 | + /** | |
| 553 | + * Does this downloaded body actually contain an image? | |
| 554 | + * | |
| 555 | + * A server answering with an HTML error page, a login redirect or a JSON error still | |
| 556 | + * returns a perfectly valid HTTP body. Written to disk under the requested name it | |
| 557 | + * becomes an attachment WordPress believes is an image, which then renders broken and is | |
| 558 | + * far harder to trace back than a refused import. | |
| 559 | + * | |
| 560 | + * SVG is checked separately because it is markup: getimagesize() cannot read it. | |
| 561 | + * | |
| 562 | + * @since 2.6.2 | |
| 563 | + * | |
| 564 | + * @param string $body Downloaded file contents. | |
| 565 | + * @param string $file_name Name the file will be stored under. | |
| 566 | + * @return bool True when the body looks like the image it claims to be. | |
| 567 | + */ | |
| 568 | + private static function is_image_payload( $body, $file_name ) { | |
| 569 | + | |
| 570 | + if ( preg_match( '/\.svg$/i', $file_name ) ) { | |
| 571 | + // Cheap structural check: an SVG has to contain an <svg> element. | |
| 572 | + return (bool) preg_match( '/<svg[\s>]/i', substr( $body, 0, 4096 ) ); | |
| 573 | + } | |
| 574 | + | |
| 575 | + return false !== @getimagesizefromstring( $body ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- malformed input is the case being detected. | |
| 576 | + } | |
| 577 | + | |
| 578 | + /** | |
| 52 | 579 | * Media Import image. |
| 53 | 580 | * |
| 54 | 581 | * Import a single image from a remote server, upload the image WordPress |
| 55 | 582 | * uploads folder, create a new attachment in the database and updates the |
| @@ -55,44 +582,163 @@ | ||
| 55 | 582 | * uploads folder, create a new attachment in the database and updates the |
| 56 | 583 | * attachment metadata. |
| 57 | 584 | * |
| 58 | 585 | * @since 1.0.0 |
| 59 | - * @access public | |
| 60 | 586 | * |
| 61 | - * @param array $attachment The attachment. | |
| 587 | + * @param array $attachment The attachment. | |
| 588 | + * @param bool $defer_media Record the URL for the cron sideload instead of downloading | |
| 589 | + * now. Video always defers regardless; SVG never does, because | |
| 590 | + * the cron has no user and so no `svg` mime type. | |
| 62 | 591 | */ |
| 63 | - public static function wdkit_Import_media( $attachment ) { | |
| 592 | + public static function wdkit_Import_media( $attachment, $defer_media = false ) { | |
| 64 | 593 | $stored_image = self::get_store_image_saved( $attachment ); |
| 65 | 594 | |
| 66 | 595 | if ( $stored_image ) { |
| 596 | + self::remember_url( $attachment['url'] ?? '', $stored_image['url'] ?? '' ); | |
| 67 | 597 | return $stored_image; |
| 68 | 598 | } |
| 69 | 599 | |
| 70 | - // Extract the file name and extension from the url. | |
| 71 | - $file_name = basename( $attachment['url'] ); | |
| 600 | + if ( empty( $attachment['url'] ) || ! is_string( $attachment['url'] ) ) { | |
| 601 | + return $attachment; | |
| 602 | + } | |
| 72 | 603 | |
| 73 | - $file_content = wp_remote_retrieve_body( wp_safe_remote_get( $attachment['url'] ) ); | |
| 604 | + // Deferred imports skip the network fetch for anything that safely tolerates staying | |
| 605 | + // remote until the background pass localises it - except SVGs. Nexter's animated icon | |
| 606 | + // blocks embed an SVG via <object data="..."> and read object.contentDocument in JS to | |
| 607 | + // drive the draw-on-scroll effect; a cross-origin <object> cannot be read that way, so | |
| 608 | + // the icon never reveals itself and the block renders as an empty box until the | |
| 609 | + // background pass catches up. SVGs are also the cheapest node to import synchronously | |
| 610 | + // (no raster decode - see the processing_resize timing on any .svg in this importer's | |
| 611 | + // own investigation log), so keeping them eager here does not reintroduce the cost this | |
| 612 | + // was written to remove. | |
| 613 | + if ( ! class_exists( 'Wdkit_Image_Guard' ) ) { | |
| 614 | + require_once WDKIT_INCLUDES . 'admin/class-wdkit-image-guard.php'; | |
| 615 | + } | |
| 74 | 616 | |
| 75 | - if ( empty( $file_content ) ) { | |
| 76 | - return false; | |
| 617 | + $media_path = (string) wp_parse_url( $attachment['url'], PHP_URL_PATH ); | |
| 618 | + $is_svg = (bool) preg_match( '/\.svg$/i', $media_path ); | |
| 619 | + | |
| 620 | + // Video defers whether or not the caller asked for deferral. It is the largest thing | |
| 621 | + // a kit references and this function runs inside the request the user is waiting on, | |
| 622 | + // so fetching one here is exactly the cost the deferred path exists to avoid. The | |
| 623 | + // background pass handles it (see wdkit_sideload_deferred_media()); a caller that | |
| 624 | + // never schedules that pass leaves the video remote, which is what it did before | |
| 625 | + // video was importable at all. | |
| 626 | + $is_video = (bool) preg_match( '/\.(?:' . Wdkit_Image_Guard::VIDEO_EXTENSIONS . ')$/i', $media_path ); | |
| 627 | + | |
| 628 | + if ( ( $defer_media || $is_video ) && ! $is_svg ) { | |
| 629 | + self::$deferred_urls[] = $attachment['url']; | |
| 630 | + return $attachment; | |
| 77 | 631 | } |
| 78 | 632 | |
| 633 | + // Take the file name from the URL *path*. basename() on the whole URL keeps any query | |
| 634 | + // string, and "photo.jpeg?w=1920&auto=compress" is not a filename WordPress will | |
| 635 | + // accept - wp_upload_bits() rejects it and every step after that worked on nothing. | |
| 636 | + $file_name = Wdkit_Image_Guard::filename_from_url( $attachment['url'] ); | |
| 637 | + | |
| 638 | + if ( '' === $file_name ) { | |
| 639 | + return $attachment; | |
| 640 | + } | |
| 641 | + | |
| 642 | + // Already one of ours: resolve the ID from the URL instead of fetching it. | |
| 643 | + // | |
| 644 | + // Media can arrive local for good reasons - the generated site logo, and stock images | |
| 645 | + // the wizard copies in before the import starts - and downloading our own URL is both | |
| 646 | + // pointless and fragile: it needs a working loopback request, which plenty of hosts | |
| 647 | + // block, and on failure this used to hand back false and wipe the attribute, taking | |
| 648 | + // the image off the page. It also stored a second copy of a file already on disk. | |
| 649 | + $uploads = wp_get_upload_dir(); | |
| 650 | + | |
| 651 | + if ( ! Wdkit_Image_Guard::is_foreign_media( $attachment['url'], isset( $uploads['baseurl'] ) ? $uploads['baseurl'] : '' ) ) { | |
| 652 | + $local_id = self::attachment_id_from_local_url( $attachment['url'] ); | |
| 653 | + | |
| 654 | + if ( $local_id ) { | |
| 655 | + return array( | |
| 656 | + 'id' => $local_id, | |
| 657 | + 'url' => $attachment['url'], | |
| 658 | + ); | |
| 659 | + } | |
| 660 | + | |
| 661 | + // A local URL we cannot match to an attachment is left exactly as it is; there is | |
| 662 | + // nothing to import and the URL itself still works. | |
| 663 | + return $attachment; | |
| 664 | + } | |
| 665 | + | |
| 666 | + // wdesignkit_safe_remote_get(), not wp_safe_remote_get(): this importer ingests the | |
| 667 | + // largest volume of externally supplied URLs in the plugin, and wp_safe_remote_get() | |
| 668 | + // never resolves the hostname — it only rejects a literal-IP host, so a name that | |
| 669 | + // resolves to 127.0.0.1 or 169.254.169.254 passes straight through (SSRF, CWE-918, | |
| 670 | + // ClickUp 86d41ce9w). The wrapper resolves the host and blocks the reserved ranges. | |
| 671 | + // Already pulled down by the concurrent warm-up (see prefetch_remote_files())? Use | |
| 672 | + // that copy. Only a validated, non-redirected, size-capped 200 ever lands there, so | |
| 673 | + // this is the same payload this request would have fetched - just without paying for | |
| 674 | + // the round trip a second time. A miss simply falls through to the request below, | |
| 675 | + // which is what happens for every URL when curl_multi is unavailable. | |
| 676 | + if ( isset( self::$prefetched[ $attachment['url'] ] ) ) { | |
| 677 | + $file_content = self::$prefetched[ $attachment['url'] ]; | |
| 678 | + } else { | |
| 679 | + $response = wdesignkit_safe_remote_get( $attachment['url'] ); | |
| 680 | + | |
| 681 | + // A 404 body is still a body: without this check the error page was written to disk | |
| 682 | + // as a .png and inserted as a real attachment, giving a broken image rather than an | |
| 683 | + // honest failure. | |
| 684 | + if ( is_wp_error( $response ) || 200 !== (int) wp_remote_retrieve_response_code( $response ) ) { | |
| 685 | + self::log_failed_download( | |
| 686 | + $attachment, | |
| 687 | + 'fetch_failed', | |
| 688 | + is_wp_error( $response ) ? $response->get_error_message() : 'HTTP ' . wp_remote_retrieve_response_code( $response ) | |
| 689 | + ); | |
| 690 | + | |
| 691 | + return $attachment; | |
| 692 | + } | |
| 693 | + | |
| 694 | + $file_content = wp_remote_retrieve_body( $response ); | |
| 695 | + } | |
| 696 | + | |
| 697 | + if ( empty( $file_content ) || ! self::is_image_payload( $file_content, $file_name ) ) { | |
| 698 | + // Returning the attachment unchanged keeps the remote URL working. Returning | |
| 699 | + // false - as this used to - made the caller overwrite the whole media attribute, | |
| 700 | + // so a single failed download removed the image from the page entirely. | |
| 701 | + self::log_failed_download( $attachment, 'not_an_image', 'empty body or unrecognised image payload' ); | |
| 702 | + | |
| 703 | + return $attachment; | |
| 704 | + } | |
| 705 | + | |
| 79 | 706 | $upload_data = wp_upload_bits( $file_name, null, $file_content ); |
| 80 | 707 | |
| 81 | - $post_image = array( | |
| 82 | - 'post_title' => $file_name, | |
| 83 | - 'guid' => $upload_data['url'], | |
| 84 | - ); | |
| 708 | + if ( ! empty( $upload_data['error'] ) || empty( $upload_data['file'] ) || empty( $upload_data['url'] ) ) { | |
| 709 | + self::log_failed_download( $attachment, 'upload_failed', ! empty( $upload_data['error'] ) ? (string) $upload_data['error'] : 'wp_upload_bits() returned no file' ); | |
| 85 | 710 | |
| 711 | + return $attachment; | |
| 712 | + } | |
| 713 | + | |
| 86 | 714 | $info = wp_check_filetype( $upload_data['file'] ); |
| 87 | - if ( ! empty( $info ) ) { | |
| 88 | - $post_image['post_mime_type'] = $info['type']; | |
| 89 | - } else { | |
| 715 | + | |
| 716 | + // wp_check_filetype() always returns an array - array( 'ext' => false, 'type' => false ) | |
| 717 | + // when it recognises nothing - so testing the array itself never failed and a file of | |
| 718 | + // unknown type was inserted with an empty post_mime_type. | |
| 719 | + if ( empty( $info['type'] ) ) { | |
| 720 | + wp_delete_file( $upload_data['file'] ); | |
| 721 | + | |
| 722 | + self::log_failed_download( $attachment, 'unrecognised_filetype', $file_name ); | |
| 723 | + | |
| 90 | 724 | return $attachment; |
| 91 | 725 | } |
| 92 | 726 | |
| 727 | + $post_image = array( | |
| 728 | + 'post_title' => $file_name, | |
| 729 | + 'guid' => $upload_data['url'], | |
| 730 | + 'post_mime_type' => $info['type'], | |
| 731 | + ); | |
| 732 | + | |
| 93 | 733 | $post_id = wp_insert_attachment( $post_image, $upload_data['file'] ); |
| 94 | 734 | |
| 735 | + if ( is_wp_error( $post_id ) || ! $post_id ) { | |
| 736 | + self::log_failed_download( $attachment, 'insert_failed', is_wp_error( $post_id ) ? $post_id->get_error_message() : 'wp_insert_attachment() returned no id' ); | |
| 737 | + | |
| 738 | + return $attachment; | |
| 739 | + } | |
| 740 | + | |
| 95 | 741 | // On REST requests. |
| 96 | 742 | if ( ! function_exists( 'wp_generate_attachment_metadata' ) ) { |
| 97 | 743 | require_once ABSPATH . '/wp-admin/includes/image.php'; |
| 98 | 744 | } |
| @@ -112,12 +758,50 @@ | ||
| 112 | 758 | if ( isset( $attachment['id'] ) ) { |
| 113 | 759 | self::$new_image_ids[ $attachment['id'] ] = $new_attachment_img; |
| 114 | 760 | } |
| 115 | 761 | |
| 762 | + self::remember_url( $attachment['url'], $upload_data['url'] ); | |
| 763 | + | |
| 116 | 764 | return $new_attachment_img; |
| 117 | 765 | } |
| 118 | 766 | |
| 119 | 767 | /** |
| 768 | + * Record that this image could not be localised, so the failure is diagnosable instead | |
| 769 | + * of invisible. The caller still keeps the original remote URL on the page - that part is | |
| 770 | + * deliberate (see the callers' own comments) - this only makes sure someone can find out | |
| 771 | + * it happened, via `wp eval 'print_r( Wdkit_Import_Log::read() );'` or | |
| 772 | + * `Wdkit_Import_Log::report()`. See ClickUp 14ynqxywnae. | |
| 773 | + * | |
| 774 | + * @since 2.7.3 | |
| 775 | + * | |
| 776 | + * @param array $attachment The attachment that failed to import. | |
| 777 | + * @param string $code Short machine-readable reason. | |
| 778 | + * @param string $message Detail for a human reading the log. | |
| 779 | + * @return void | |
| 780 | + */ | |
| 781 | + private static function log_failed_download( $attachment, $code, $message = '' ) { | |
| 782 | + if ( ! class_exists( 'Wdkit_Import_Log' ) ) { | |
| 783 | + require_once WDKIT_INCLUDES . 'admin/import/class-wdkit-import-log.php'; | |
| 784 | + } | |
| 785 | + | |
| 786 | + Wdkit_Import_Log::add( | |
| 787 | + 'failure', | |
| 788 | + array( | |
| 789 | + 'step' => 'media_import', | |
| 790 | + 'code' => $code, | |
| 791 | + 'message' => $message, | |
| 792 | + 'url' => isset( $attachment['url'] ) ? (string) $attachment['url'] : '', | |
| 793 | + ) | |
| 794 | + ); | |
| 795 | + | |
| 796 | + self::$failed_downloads[] = array( | |
| 797 | + 'url' => isset( $attachment['url'] ) ? (string) $attachment['url'] : '', | |
| 798 | + 'code' => $code, | |
| 799 | + 'message' => $message, | |
| 800 | + ); | |
| 801 | + } | |
| 802 | + | |
| 803 | + /** | |
| 120 | 804 | * Get store saved image. |
| 121 | 805 | * |
| 122 | 806 | * Retrieve new image ID, if the image has a new ID after the import. |
| 123 | 807 | * |
| @@ -137,8 +821,19 @@ | ||
| 137 | 821 | $wpdb->prepare( 'SELECT `post_id` FROM `' . $wpdb->postmeta . '` WHERE `meta_key` = \'tpgb_source_image_key\' AND `meta_value` = %s;', self::get_attachment_url_hash_image( $attachment['url'] ) ) |
| 138 | 822 | ); |
| 139 | 823 | |
| 140 | 824 | if ( ! empty( $post_id ) ) { |
| 825 | + | |
| 826 | + // Only reuse a copy that is actually usable. This lookup is what makes importing | |
| 827 | + // the same image across many pages cheap, but it also means a single bad file is | |
| 828 | + // handed out for every later import of that URL - one empty SVG stored during a | |
| 829 | + // failed run kept a logo blank through every subsequent import. Rejecting it here | |
| 830 | + // lets the importer fetch the file again instead. | |
| 831 | + if ( ! self::attachment_file_is_usable( (int) $post_id ) ) { | |
| 832 | + | |
| 833 | + return false; | |
| 834 | + } | |
| 835 | + | |
| 141 | 836 | $new_attachment_img = array( |
| 142 | 837 | 'id' => $post_id, |
| 143 | 838 | 'url' => wp_get_attachment_url( $post_id ), |
| 144 | 839 | ); |
| @@ -156,9 +851,8 @@ | ||
| 156 | 851 | /** |
| 157 | 852 | * Import images Constructor. |
| 158 | 853 | * |
| 159 | 854 | * @since 1.0.0 |
| 160 | - * @access public | |
| 161 | 855 | */ |
| 162 | 856 | public function __construct() { |
| 163 | 857 | if ( ! function_exists( 'WP_Filesystem' ) ) { |
| 164 | 858 | require_once ABSPATH . 'wp-admin/includes/file.php'; |
