| 1 |
<?php |
| 2 |
|
| 3 |
use BerqWP\Cache; |
| 4 |
use BerqWP_Deps\voku\helper\HtmlDomParser; |
| 5 |
|
| 6 |
class berqUpload |
| 7 |
{ |
| 8 |
static $resources = []; |
| 9 |
|
| 10 |
public static function process_page($url, $html = '') |
| 11 |
{ |
| 12 |
global $berq_log; |
| 13 |
|
| 14 |
self::$resources = []; |
| 15 |
|
| 16 |
$berq_log->info("Processing $url"); |
| 17 |
|
| 18 |
if (empty($html)) { |
| 19 |
$start = microtime(true); |
| 20 |
$response = wp_remote_get($url . '?berqwpnocache=' . time(), [ |
| 21 |
// 'sslverify' => false, |
| 22 |
// 'httpversion' => '2.0', |
| 23 |
// 'cookies' => [], |
| 24 |
// 'headers' => [ |
| 25 |
// 'User-Agent' => 'Mozilla/5.0 (BerqWP)', |
| 26 |
// 'Cache-Control' => 'no-cache, no-store, must-revalidate', |
| 27 |
// 'Pragma' => 'no-cache', |
| 28 |
// ], |
| 29 |
'sslverify' => false, |
| 30 |
'httpversion' => '2.0', |
| 31 |
'timeout' => 30 |
| 32 |
]); |
| 33 |
|
| 34 |
$end = microtime(true); |
| 35 |
$timeMs = ($end - $start) * 1000; |
| 36 |
$statusCode = wp_remote_retrieve_response_code($response); |
| 37 |
|
| 38 |
if (is_wp_error($response) || $statusCode !== 200) { |
| 39 |
$berq_log->info("failed to download page html, status code: $statusCode"); |
| 40 |
return; |
| 41 |
} |
| 42 |
|
| 43 |
$berq_log->info("html download $timeMs ms"); |
| 44 |
|
| 45 |
$html = wp_remote_retrieve_body($response); |
| 46 |
|
| 47 |
if (empty($html)) { |
| 48 |
self::queue_remove_page($url); |
| 49 |
|
| 50 |
return [ |
| 51 |
"success" => true |
| 52 |
]; |
| 53 |
} |
| 54 |
|
| 55 |
$html = preg_replace('/\?berqwpnocache=\d+&/', '?', $html); |
| 56 |
$html = preg_replace('/\?berqwpnocache=\d+/', '', $html); |
| 57 |
$html = preg_replace('/berqwpnocache%3D\d+(%26)?/', '', $html); |
| 58 |
} |
| 59 |
|
| 60 |
// Parse HTML once and reuse the DOM across all asset extraction methods |
| 61 |
$dom = HtmlDomParser::str_get_html($html); |
| 62 |
|
| 63 |
$start = microtime(true); |
| 64 |
self::process_css($dom); |
| 65 |
$end = microtime(true); |
| 66 |
$timeMs = ($end - $start) * 1000; |
| 67 |
$berq_log->info("process css $timeMs ms"); |
| 68 |
|
| 69 |
$start = microtime(true); |
| 70 |
self::process_js($dom); |
| 71 |
$end = microtime(true); |
| 72 |
$timeMs = ($end - $start) * 1000; |
| 73 |
$berq_log->info("process js $timeMs ms"); |
| 74 |
|
| 75 |
$start = microtime(true); |
| 76 |
self::process_images($dom); |
| 77 |
$end = microtime(true); |
| 78 |
$timeMs = ($end - $start) * 1000; |
| 79 |
$berq_log->info("process images $timeMs ms"); |
| 80 |
|
| 81 |
$start = microtime(true); |
| 82 |
self::proccess_inline_css($dom); |
| 83 |
$end = microtime(true); |
| 84 |
$timeMs = ($end - $start) * 1000; |
| 85 |
$berq_log->info("process inline css $timeMs ms"); |
| 86 |
|
| 87 |
// $start = microtime(true); |
| 88 |
// self::proccess_external(); |
| 89 |
// $end = microtime(true); |
| 90 |
// $timeMs = ($end - $start) * 1000; |
| 91 |
// $berq_log->info("process external 1 $timeMs ms"); |
| 92 |
|
| 93 |
$start = microtime(true); |
| 94 |
self::proccess_css_ref(); |
| 95 |
$end = microtime(true); |
| 96 |
$timeMs = ($end - $start) * 1000; |
| 97 |
$berq_log->info("process css ref $timeMs ms"); |
| 98 |
|
| 99 |
$start = microtime(true); |
| 100 |
self::proccess_css_import(); |
| 101 |
$end = microtime(true); |
| 102 |
$timeMs = ($end - $start) * 1000; |
| 103 |
$berq_log->info("process css import $timeMs ms"); |
| 104 |
|
| 105 |
// $start = microtime(true); |
| 106 |
// self::proccess_external(); |
| 107 |
// $end = microtime(true); |
| 108 |
// $timeMs = ($end - $start) * 1000; |
| 109 |
// $berq_log->info("process external 2 $timeMs ms"); |
| 110 |
|
| 111 |
// $compressed_assets = array_map(function ($asset) { |
| 112 |
|
| 113 |
// if (!empty($asset['content'])) { |
| 114 |
// $asset['content'] = base64_encode(gzencode($asset['content'])); |
| 115 |
// } |
| 116 |
|
| 117 |
// return $asset; |
| 118 |
// }, self::$resources); |
| 119 |
|
| 120 |
// free memory |
| 121 |
$dom->clear(); |
| 122 |
unset($dom); |
| 123 |
|
| 124 |
// add mime type in resources |
| 125 |
self::$resources = array_map(function ($asset) { |
| 126 |
|
| 127 |
if ($asset['content'] !== null) { |
| 128 |
unset($asset['content']); |
| 129 |
} |
| 130 |
|
| 131 |
if (!empty($asset['path'])) { |
| 132 |
// // detect mime |
| 133 |
// $finfo = finfo_open(FILEINFO_MIME_TYPE); |
| 134 |
// $mime = finfo_file($finfo, $asset['path']); |
| 135 |
// finfo_close($finfo); |
| 136 |
|
| 137 |
$mime = self::mime_from_ext(self::get_extension($asset['url'])); |
| 138 |
|
| 139 |
$asset['type'] = $mime; |
| 140 |
// $asset['url'] = urldecode($asset['url']); |
| 141 |
} |
| 142 |
|
| 143 |
// $asset['url'] = self::clean_url($asset['url']); |
| 144 |
|
| 145 |
return $asset; |
| 146 |
}, self::$resources); |
| 147 |
|
| 148 |
// remove assets that doesn't exist or has invalid file path |
| 149 |
self::$resources = array_filter(self::$resources, function ($asset) { |
| 150 |
return !empty($asset['path']) && is_file($asset['path']) && is_readable($asset['path']); |
| 151 |
}); |
| 152 |
|
| 153 |
// Pre-compute SHA-256 hashes so we don't hash the same file multiple times |
| 154 |
self::$resources = array_map(function ($asset) { |
| 155 |
$asset['hash'] = hash_file('sha256', $asset['path']); |
| 156 |
return $asset; |
| 157 |
}, self::$resources); |
| 158 |
|
| 159 |
// Filter out already-uploaded assets |
| 160 |
$uploaded_assets = get_option('berqwp_uploaded_assets', []); |
| 161 |
|
| 162 |
self::$resources = array_filter(self::$resources, function ($asset) use ($uploaded_assets) { |
| 163 |
|
| 164 |
// return new/outdated assets for the upload |
| 165 |
return !self::asset_upload_verified( $asset, $uploaded_assets ); |
| 166 |
|
| 167 |
}); |
| 168 |
|
| 169 |
// Prepare metadata |
| 170 |
$meta = [ |
| 171 |
'page_url' => $url, |
| 172 |
'assets' => array_map(fn($a) => ['url' => $a['url']], self::$resources), |
| 173 |
]; |
| 174 |
|
| 175 |
// echo print_r(json_decode(json_encode($meta), true), true); |
| 176 |
// exit; |
| 177 |
|
| 178 |
$parsed_url = wp_parse_url(get_site_url()); |
| 179 |
$domain = $parsed_url['host']; |
| 180 |
$queue_failed = null; |
| 181 |
|
| 182 |
if (empty(self::$resources)) { |
| 183 |
|
| 184 |
$start = microtime(true); |
| 185 |
$meta = [ |
| 186 |
'assets' => [], |
| 187 |
]; |
| 188 |
|
| 189 |
$body = [ |
| 190 |
'action' => 'prepare_assets', |
| 191 |
'meta' => json_encode($meta), |
| 192 |
'domain' => $domain, |
| 193 |
]; |
| 194 |
|
| 195 |
$gzip_html = gzencode($html); |
| 196 |
|
| 197 |
if ($gzip_html === false) { |
| 198 |
throw new Error("GZip encoding failed"); |
| 199 |
} |
| 200 |
|
| 201 |
$body['html'] = base64_encode($gzip_html); |
| 202 |
$post_data = berqwp_get_page_params($url); |
| 203 |
|
| 204 |
if (self::is_forced($url)) { |
| 205 |
$post_data['force'] = 1; |
| 206 |
} |
| 207 |
|
| 208 |
$body['params'] = json_encode($post_data); |
| 209 |
|
| 210 |
$queue_failed = self::handle_upload($body, [], true); |
| 211 |
|
| 212 |
$end = microtime(true); |
| 213 |
$timeMs = ($end - $start) * 1000; |
| 214 |
$berq_log->info("page queue 1 $timeMs ms"); |
| 215 |
|
| 216 |
} else { |
| 217 |
|
| 218 |
$start = microtime(true); |
| 219 |
$chunks = array_chunk(self::$resources, ini_get('max_file_uploads') ?? 15); |
| 220 |
|
| 221 |
foreach ($chunks as $index => $chunk) { |
| 222 |
|
| 223 |
$meta = [ |
| 224 |
'assets' => array_map(fn($a) => ['url' => $a['url'], 'hash' => $a['hash']], $chunk), |
| 225 |
]; |
| 226 |
|
| 227 |
$is_last_chunk = $index === count($chunks) - 1; |
| 228 |
|
| 229 |
$body = [ |
| 230 |
'action' => 'prepare_assets', |
| 231 |
'meta' => json_encode($meta), |
| 232 |
'domain' => $domain, |
| 233 |
]; |
| 234 |
|
| 235 |
try { |
| 236 |
|
| 237 |
if ($is_last_chunk) { |
| 238 |
|
| 239 |
global $berq_log; |
| 240 |
$berq_log->info("Processing last chunk"); |
| 241 |
|
| 242 |
$gzip_html = gzencode($html); |
| 243 |
|
| 244 |
if ($gzip_html === false) { |
| 245 |
throw new Error("GZip encoding failed"); |
| 246 |
} |
| 247 |
|
| 248 |
$body['html'] = base64_encode($gzip_html); |
| 249 |
$post_data = berqwp_get_page_params($url); |
| 250 |
|
| 251 |
if (self::is_forced($url)) { |
| 252 |
$post_data['force'] = 1; |
| 253 |
} |
| 254 |
|
| 255 |
$body['params'] = json_encode($post_data); |
| 256 |
} |
| 257 |
|
| 258 |
// Attach files |
| 259 |
foreach ($chunk as $i => $asset) { |
| 260 |
try { |
| 261 |
$body["asset_$i"] = curl_file_create($asset['path'], $asset['type'], basename($asset['path'])); |
| 262 |
$berq_log->info("Successfully created curl file: " . $asset['url']); |
| 263 |
|
| 264 |
} catch (Exception $e) { |
| 265 |
|
| 266 |
$berq_log->info("Curl file create failed: " . $asset['url'] . $e); |
| 267 |
|
| 268 |
} catch (Throwable $e) { |
| 269 |
|
| 270 |
$berq_log->info("Curl file create failed: " . $asset['url'] . $e); |
| 271 |
|
| 272 |
} |
| 273 |
} |
| 274 |
|
| 275 |
} catch (Exception $e) { |
| 276 |
|
| 277 |
$berq_log->info("Chunk processing failed, continuing"); |
| 278 |
|
| 279 |
} catch (Throwable $e) { |
| 280 |
$berq_log->info("Chunk processing failed, continuing"); |
| 281 |
} |
| 282 |
|
| 283 |
$upload_result = self::handle_upload($body, $chunk, $is_last_chunk); |
| 284 |
|
| 285 |
if ($is_last_chunk) { |
| 286 |
$queue_failed = $upload_result; |
| 287 |
} |
| 288 |
|
| 289 |
unset($chunk); |
| 290 |
} |
| 291 |
|
| 292 |
self::$resources = []; |
| 293 |
|
| 294 |
$end = microtime(true); |
| 295 |
$timeMs = ($end - $start) * 1000; |
| 296 |
$berq_log->info("page queue 2 $timeMs ms"); |
| 297 |
} |
| 298 |
|
| 299 |
if ($queue_failed === false) { |
| 300 |
global $berq_log; |
| 301 |
$berq_log->info("Upload failed, adding back to queue"); |
| 302 |
return ["success" => false]; |
| 303 |
} |
| 304 |
|
| 305 |
return [ |
| 306 |
"success" => true |
| 307 |
]; |
| 308 |
} |
| 309 |
|
| 310 |
static function asset_upload_verified($asset, $uploaded_assets) { |
| 311 |
|
| 312 |
$is_valid = array_filter($uploaded_assets, function ($uploaded_file) use ($asset) { |
| 313 |
return $uploaded_file['url'] === $asset['url'] && $uploaded_file['hash'] === $asset['hash'] && isset($uploaded_file['verified']) && $uploaded_file['verified'] === true; |
| 314 |
}); |
| 315 |
|
| 316 |
return !empty($is_valid); |
| 317 |
} |
| 318 |
|
| 319 |
public static function is_forced($page_url) |
| 320 |
{ |
| 321 |
$queue = get_option('berqwp_optimize_queue', []); |
| 322 |
$key = md5($page_url); |
| 323 |
|
| 324 |
return !empty($queue[$key]) && !empty($queue[$key]['force']); |
| 325 |
} |
| 326 |
|
| 327 |
static function handle_upload($body, $chunk = [], $is_last = false) |
| 328 |
{ |
| 329 |
global $berq_log; |
| 330 |
|
| 331 |
$berq_log->info("Uploading chunk with " . count($chunk) . " assets using Guzzle"); |
| 332 |
|
| 333 |
if ($is_last) { |
| 334 |
// $berq_log->info("Last chunk: " . print_r($chunk, true)); |
| 335 |
} |
| 336 |
|
| 337 |
// Use scoped Guzzle client |
| 338 |
$client = new \BerqWP_Deps\GuzzleHttp\Client([ |
| 339 |
'timeout' => 60, |
| 340 |
'verify' => false, // Disable SSL verification |
| 341 |
]); |
| 342 |
|
| 343 |
|
| 344 |
try { |
| 345 |
// Build multipart array for Guzzle |
| 346 |
$multipart = []; |
| 347 |
|
| 348 |
// Add regular POST fields |
| 349 |
foreach ($body as $key => $value) { |
| 350 |
|
| 351 |
if (defined('BERQWP_DEV_MOCKS') && BERQWP_DEV_MOCKS) { |
| 352 |
continue; |
| 353 |
} |
| 354 |
|
| 355 |
if ($value instanceof CURLFile) { |
| 356 |
continue; // Skip files for now |
| 357 |
} |
| 358 |
|
| 359 |
$multipart[] = [ |
| 360 |
'name' => $key, |
| 361 |
'contents' => $value, |
| 362 |
]; |
| 363 |
} |
| 364 |
|
| 365 |
// Add file uploads |
| 366 |
foreach ($body as $key => $value) { |
| 367 |
|
| 368 |
if (defined('BERQWP_DEV_MOCKS') && BERQWP_DEV_MOCKS) { |
| 369 |
continue; |
| 370 |
} |
| 371 |
|
| 372 |
if (!($value instanceof CURLFile)) { |
| 373 |
continue; // Only process files |
| 374 |
} |
| 375 |
|
| 376 |
$file_path = $value->getFilename(); |
| 377 |
$file_name = $value->getPostFilename(); |
| 378 |
$mime_type = $value->getMimeType(); |
| 379 |
|
| 380 |
if (!is_file($file_path)) { |
| 381 |
$berq_log->warning("File not found: {$file_path}"); |
| 382 |
continue; |
| 383 |
} |
| 384 |
|
| 385 |
$fp = fopen($file_path, 'r'); |
| 386 |
if ($fp === false) { |
| 387 |
$berq_log->warning("Failed to open file: {$file_path}"); |
| 388 |
continue; |
| 389 |
} |
| 390 |
|
| 391 |
$multipart[] = [ |
| 392 |
'name' => $key, |
| 393 |
'contents' => $fp, |
| 394 |
'filename' => $file_name, |
| 395 |
'headers' => [ |
| 396 |
'Content-Type' => $mime_type, |
| 397 |
], |
| 398 |
]; |
| 399 |
} |
| 400 |
|
| 401 |
if (defined('BERQWP_DEV_MOCKS') && BERQWP_DEV_MOCKS && $is_last) { |
| 402 |
sleep(10); |
| 403 |
$gz_html = base64_decode( $body['html'] ); |
| 404 |
$params = json_decode($body['params'], true); |
| 405 |
$page_url = $params['page_url']; |
| 406 |
$mocks_dir = WP_CONTENT_DIR . '/mocks/'; |
| 407 |
$cache_filename = $mocks_dir . md5($page_url) . ".gz"; |
| 408 |
$download_queue_option = "berqwp_mock_photon_download_queue"; |
| 409 |
$download_queue = get_option($download_queue_option, []); |
| 410 |
$file_url = WP_CONTENT_URL . "/mocks/" . md5($page_url) . ".gz"; |
| 411 |
|
| 412 |
$file_url = str_replace("localhost:9000", "host.docker.internal:9000", $file_url); |
| 413 |
|
| 414 |
wp_mkdir_p($mocks_dir); |
| 415 |
|
| 416 |
$download_queue[md5($page_url)] = [ |
| 417 |
'url' => $page_url, |
| 418 |
'html' => $file_url |
| 419 |
]; |
| 420 |
|
| 421 |
update_option($download_queue_option, $download_queue); |
| 422 |
|
| 423 |
file_put_contents($cache_filename, $gz_html); |
| 424 |
|
| 425 |
return true; |
| 426 |
} |
| 427 |
|
| 428 |
$response = $client->request('POST', 'https://boost.berqwp.com/photon/', [ |
| 429 |
'multipart' => $multipart, |
| 430 |
]); |
| 431 |
|
| 432 |
$http_code = $response->getStatusCode(); |
| 433 |
$response_body = $response->getBody()->getContents(); |
| 434 |
$json = json_decode($response_body, true); |
| 435 |
|
| 436 |
if ($json === null) { |
| 437 |
$berq_log->info("Response body JSON failed: " . $response_body); |
| 438 |
} |
| 439 |
|
| 440 |
$berq_log->info("Upload completed with HTTP code: " . $http_code); |
| 441 |
|
| 442 |
// Update uploaded assets on success |
| 443 |
if ($http_code >= 200 && $http_code < 300 && !empty($chunk) && isset($json['received'])) { |
| 444 |
$uploaded_assets = get_option('berqwp_uploaded_assets', []); |
| 445 |
$received_files = $json['received']; |
| 446 |
|
| 447 |
$berq_log->info("Successfully uploaded files: " . print_r($received_files, true)); |
| 448 |
|
| 449 |
// Create associative array with URL as key for deduplication |
| 450 |
$assets_by_url = []; |
| 451 |
foreach ($uploaded_assets as $asset) { |
| 452 |
if (!empty($asset['url'])) { |
| 453 |
$assets_by_url[$asset['url']] = $asset; |
| 454 |
} |
| 455 |
} |
| 456 |
|
| 457 |
// Add/update new assets (newer entries overwrite older ones) |
| 458 |
foreach ($chunk as $asset) { |
| 459 |
if (!empty($asset['url']) && !empty($asset['hash']) && in_array($asset['url'], $received_files)) { |
| 460 |
$assets_by_url[$asset['url']] = [ |
| 461 |
'url' => $asset['url'], |
| 462 |
'hash' => $asset['hash'], |
| 463 |
'verified' => true, |
| 464 |
]; |
| 465 |
} |
| 466 |
} |
| 467 |
|
| 468 |
// Convert back to indexed array and save |
| 469 |
update_option('berqwp_uploaded_assets', array_values($assets_by_url), false); |
| 470 |
} |
| 471 |
|
| 472 |
// Handle last chunk response |
| 473 |
if ($is_last && isset($json['pending_download'])) { |
| 474 |
self::handle_download_cache($json['pending_download']); |
| 475 |
} |
| 476 |
|
| 477 |
// Add to server queue if last chunk |
| 478 |
if ($is_last && $http_code >= 200 && $http_code < 300) { |
| 479 |
$server_queue = get_option('berqwp_server_queue', []); |
| 480 |
$params = json_decode($body['params'], true); |
| 481 |
$url = $params['page_url']; |
| 482 |
|
| 483 |
if (!in_array($url, $server_queue)) { |
| 484 |
$server_queue[] = $url; |
| 485 |
} |
| 486 |
|
| 487 |
update_option('berqwp_server_queue', $server_queue, false); |
| 488 |
} |
| 489 |
|
| 490 |
if ($is_last && !is_array($json)) { |
| 491 |
return false; |
| 492 |
} |
| 493 |
|
| 494 |
// Throw exception on HTTP error |
| 495 |
if ($http_code < 200 || $http_code >= 300) { |
| 496 |
$berq_log->error('Upload failed with HTTP code: ' . $http_code); |
| 497 |
throw new \Exception('Could not upload page to BerqWP server'); |
| 498 |
} |
| 499 |
|
| 500 |
return true; |
| 501 |
} catch (\BerqWP_Deps\GuzzleHttp\Exception\RequestException $e) { |
| 502 |
$berq_log->error("Guzzle request failed: " . $e->getMessage()); |
| 503 |
throw new \Exception('Could not upload page to BerqWP server: ' . $e->getMessage()); |
| 504 |
} catch (\Exception $e) { |
| 505 |
$berq_log->error("Upload failed: " . $e->getMessage()); |
| 506 |
throw $e; |
| 507 |
} catch (\Throwable $e) { |
| 508 |
$berq_log->error("Upload failed: " . $e->getMessage()); |
| 509 |
throw $e; |
| 510 |
} |
| 511 |
} |
| 512 |
|
| 513 |
// static function handle_upload($body, $chunk = [], $is_last = false) |
| 514 |
// { |
| 515 |
// global $berq_log; |
| 516 |
|
| 517 |
// $ch = curl_init('https://boost.berqwp.com/photon/'); |
| 518 |
// curl_setopt_array($ch, [ |
| 519 |
// CURLOPT_POST => true, |
| 520 |
// CURLOPT_POSTFIELDS => $body, |
| 521 |
// CURLOPT_RETURNTRANSFER => true, |
| 522 |
// CURLOPT_TIMEOUT => 60, |
| 523 |
// CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1, |
| 524 |
// CURLOPT_TCP_KEEPALIVE => 1, |
| 525 |
|
| 526 |
// // Disable SSL verification |
| 527 |
// CURLOPT_SSL_VERIFYPEER => false, |
| 528 |
// CURLOPT_SSL_VERIFYHOST => false, |
| 529 |
// ]); |
| 530 |
|
| 531 |
// $response_body = curl_exec($ch); |
| 532 |
// $error = curl_error($ch); |
| 533 |
|
| 534 |
// if (empty($error)) { |
| 535 |
// $uploaded_assets = get_option('berqwp_uploaded_assets', []); |
| 536 |
// $new_assets = array_map(function ($asset) { |
| 537 |
// return [ |
| 538 |
// 'url' => $asset['url'], |
| 539 |
// 'hash' => hash_file('sha256', $asset['path']), |
| 540 |
// ]; |
| 541 |
// }, $chunk); |
| 542 |
|
| 543 |
// update_option('berqwp_uploaded_assets', array_merge($uploaded_assets, $new_assets), false); |
| 544 |
// } |
| 545 |
|
| 546 |
// $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); |
| 547 |
// curl_close($ch); |
| 548 |
|
| 549 |
// // echo $response_body; |
| 550 |
|
| 551 |
// // if ($index === count($chunks) - 1) { |
| 552 |
// if ($is_last) { |
| 553 |
// $json = json_decode($response_body, true); |
| 554 |
// self::handle_download_cache($json['pending_download']); |
| 555 |
// } |
| 556 |
|
| 557 |
// if ($is_last && empty($error)) { |
| 558 |
// $server_queue = get_option('berqwp_server_queue', []); |
| 559 |
// $url = json_decode($body['params'], true)['page_url']; |
| 560 |
|
| 561 |
// if (!in_array($url, $server_queue)) { |
| 562 |
// $server_queue[] = $url; |
| 563 |
// } |
| 564 |
|
| 565 |
// update_option('berqwp_server_queue', $server_queue, false); |
| 566 |
// } |
| 567 |
|
| 568 |
// if (!empty($error)) { |
| 569 |
// $berq_log->info('Failed to upload assets ' . $error . ' code: ' . $http_code); |
| 570 |
// throw new Exception('Could not upload page to BerqWP server'); |
| 571 |
// } |
| 572 |
// } |
| 573 |
|
| 574 |
static function request_pending_cache() |
| 575 |
{ |
| 576 |
if (defined('BERQWP_DEV_MOCKS') && BERQWP_DEV_MOCKS) { |
| 577 |
$download_queue_option = "berqwp_mock_photon_download_queue"; |
| 578 |
$download_queue = get_option($download_queue_option, []); |
| 579 |
|
| 580 |
|
| 581 |
// $download_queue = array_map(function ($item) { |
| 582 |
// return $item[0]; |
| 583 |
// }, $download_queue); |
| 584 |
|
| 585 |
$download_queue = array_values($download_queue); |
| 586 |
|
| 587 |
self::handle_download_cache($download_queue); |
| 588 |
return; |
| 589 |
} |
| 590 |
|
| 591 |
$server_queue = get_option('berqwp_server_queue', []); |
| 592 |
|
| 593 |
if (empty($server_queue)) { |
| 594 |
return; |
| 595 |
} |
| 596 |
|
| 597 |
$params = berqwp_get_page_params(home_url('/')); |
| 598 |
$site_id = $params['site_id']; |
| 599 |
$response = wp_remote_post('https://boost.berqwp.com/photon/', [ |
| 600 |
'body' => [ |
| 601 |
'action' => 'pending_cache_downloads', |
| 602 |
'site_id' => $site_id, |
| 603 |
], |
| 604 |
'timeout' => 30, |
| 605 |
'headers' => [ |
| 606 |
'Content-Type' => 'application/x-www-form-urlencoded' |
| 607 |
] |
| 608 |
]); |
| 609 |
|
| 610 |
if (is_wp_error($response)) { |
| 611 |
global $berq_log; |
| 612 |
$berq_log->info("request pending cache failed"); |
| 613 |
return; |
| 614 |
} |
| 615 |
|
| 616 |
$json = json_decode(wp_remote_retrieve_body($response), true); |
| 617 |
|
| 618 |
self::handle_download_cache($json['pending_download']); |
| 619 |
} |
| 620 |
|
| 621 |
static function queue_remove_page($pag_url) |
| 622 |
{ |
| 623 |
$queue = get_option('berqwp_optimize_queue', []); |
| 624 |
$key = md5($pag_url); |
| 625 |
unset($queue[$key]); |
| 626 |
update_option('berqwp_optimize_queue', $queue, false); |
| 627 |
|
| 628 |
if (defined('BERQWP_DEV_MOCKS') && BERQWP_DEV_MOCKS) { |
| 629 |
$download_queue_option = "berqwp_mock_photon_download_queue"; |
| 630 |
$download_queue = get_option($download_queue_option, []); |
| 631 |
unset($download_queue[$key]); |
| 632 |
update_option($download_queue_option, $download_queue, false); |
| 633 |
|
| 634 |
$file_url = WP_CONTENT_DIR . "/mocks/" . $key . ".gz"; |
| 635 |
unlink($file_url); |
| 636 |
} |
| 637 |
|
| 638 |
} |
| 639 |
|
| 640 |
static function handle_download_cache($pending_downloads) |
| 641 |
{ |
| 642 |
global $berq_log; |
| 643 |
|
| 644 |
if (!empty($pending_downloads)) { |
| 645 |
$optimized_pages = []; |
| 646 |
$params = berqwp_get_page_params(home_url('/')); |
| 647 |
|
| 648 |
foreach ($pending_downloads as $item) { |
| 649 |
|
| 650 |
$options = ['timeout' => 60, 'redirection' => 0]; |
| 651 |
|
| 652 |
// if (!empty($parsed['path']) && str_ends_with($parsed['path'], '.gz')) { |
| 653 |
// $options['headers'] = [ |
| 654 |
// 'Accept-Encoding' => 'identity' |
| 655 |
// ]; |
| 656 |
// } |
| 657 |
|
| 658 |
if (berqwp_is_page_url_excluded($item['url'])) { |
| 659 |
$server_queue = get_option('berqwp_server_queue', []); |
| 660 |
|
| 661 |
foreach ($server_queue as $index => $url) { |
| 662 |
if ($url == $item['url']) { |
| 663 |
unset($server_queue[$index]); |
| 664 |
} |
| 665 |
} |
| 666 |
|
| 667 |
$optimized_pages[] = $item['url']; |
| 668 |
continue; |
| 669 |
} |
| 670 |
|
| 671 |
// if (defined('BERQWP_DEV_MOCKS') && BERQWP_DEV_MOCKS) { |
| 672 |
// $item['html'] = str_replace("localhost:9000", "host.docker.internal:9000", $item['html']); |
| 673 |
// } |
| 674 |
|
| 675 |
// Download the HTML |
| 676 |
$response = wp_remote_get($item['html'], $options); |
| 677 |
$response_code = wp_remote_retrieve_response_code($response); |
| 678 |
|
| 679 |
// Check for errors |
| 680 |
if (is_wp_error($response) || $response_code !== 200) { |
| 681 |
continue; |
| 682 |
} |
| 683 |
|
| 684 |
$html = wp_remote_retrieve_body($response); |
| 685 |
|
| 686 |
$parsed = wp_parse_url($item['html']); |
| 687 |
|
| 688 |
if (!empty($parsed['path']) && str_ends_with($parsed['path'], '.gz')) { |
| 689 |
|
| 690 |
$html = gzdecode($html); |
| 691 |
// $tmp = wp_tempnam('cache.gz'); |
| 692 |
// file_put_contents($tmp, $html); |
| 693 |
|
| 694 |
// $gz = gzopen($tmp, 'rb'); |
| 695 |
// $html = ''; |
| 696 |
|
| 697 |
// while (!gzeof($gz)) { |
| 698 |
// $html .= gzread($gz, 8192); |
| 699 |
// } |
| 700 |
|
| 701 |
// gzclose($gz); |
| 702 |
// unlink($tmp); |
| 703 |
} |
| 704 |
|
| 705 |
// Allow other plugins to modify cache html |
| 706 |
$html = apply_filters('berqwp_cache_buffer', $html); |
| 707 |
|
| 708 |
$cache = new Cache(null, bwp_get_cache_dir()); |
| 709 |
$cache->store_cache($item['url'], $html); |
| 710 |
|
| 711 |
if (defined('BERQWP_DEV_MOCKS') && BERQWP_DEV_MOCKS) { |
| 712 |
self::queue_remove_page($item['url']); |
| 713 |
} |
| 714 |
|
| 715 |
$server_queue = get_option('berqwp_server_queue', []); |
| 716 |
|
| 717 |
foreach ($server_queue as $index => $url) { |
| 718 |
if ($url == $item['url']) { |
| 719 |
|
| 720 |
self::queue_remove_page($item['url']); |
| 721 |
|
| 722 |
unset($server_queue[$index]); |
| 723 |
} |
| 724 |
} |
| 725 |
|
| 726 |
$optimized_pages[] = $item['url']; |
| 727 |
|
| 728 |
update_option('berqwp_server_queue', $server_queue, false); |
| 729 |
|
| 730 |
$berq_log->info("Stored cache for " . $item['url']); |
| 731 |
|
| 732 |
$page_slug = bwp_url_into_path($item['url']); |
| 733 |
do_action('berqwp_stored_page_cache', $page_slug); |
| 734 |
} |
| 735 |
|
| 736 |
$site_id = $params['site_id']; |
| 737 |
$response = wp_remote_post('https://boost.berqwp.com/photon/', [ |
| 738 |
'body' => [ |
| 739 |
'action' => 'clean_page_cache', |
| 740 |
'site_id' => $site_id, |
| 741 |
'downloaded_cache' => $optimized_pages |
| 742 |
], |
| 743 |
'timeout' => 15, |
| 744 |
'headers' => [ |
| 745 |
'Content-Type' => 'application/x-www-form-urlencoded' |
| 746 |
] |
| 747 |
]); |
| 748 |
} |
| 749 |
} |
| 750 |
|
| 751 |
static function mime_from_ext($ext) |
| 752 |
{ |
| 753 |
static $map = [ |
| 754 |
'css' => 'text/css; charset=utf-8', |
| 755 |
'js' => 'application/javascript; charset=utf-8', |
| 756 |
'mjs' => 'application/javascript; charset=utf-8', |
| 757 |
'html' => 'text/html; charset=utf-8', |
| 758 |
'svg' => 'image/svg+xml', |
| 759 |
'png' => 'image/png', |
| 760 |
'jpg' => 'image/jpeg', |
| 761 |
'jpeg' => 'image/jpeg', |
| 762 |
'webp' => 'image/webp', |
| 763 |
'avif' => 'image/avif', |
| 764 |
'gif' => 'image/gif', |
| 765 |
'woff' => 'font/woff', |
| 766 |
'woff2' => 'font/woff2', |
| 767 |
'ttf' => 'font/ttf', |
| 768 |
'otf' => 'font/otf', |
| 769 |
'eot' => 'application/vnd.ms-fontobject', |
| 770 |
'json' => 'application/json; charset=utf-8', |
| 771 |
'xml' => 'application/xml; charset=utf-8', |
| 772 |
'ico' => 'image/x-icon', |
| 773 |
]; |
| 774 |
|
| 775 |
return $map[$ext] ?? 'text/plain'; |
| 776 |
} |
| 777 |
|
| 778 |
static function get_extension($url) |
| 779 |
{ |
| 780 |
$path = wp_parse_url($url, PHP_URL_PATH); |
| 781 |
if (!$path) { |
| 782 |
return ''; |
| 783 |
} |
| 784 |
|
| 785 |
return strtolower(pathinfo($path, PATHINFO_EXTENSION)); |
| 786 |
} |
| 787 |
|
| 788 |
static function is_valid_asset($url) |
| 789 |
{ |
| 790 |
return strpos($url, 'data:') === false; |
| 791 |
} |
| 792 |
|
| 793 |
static function get_cache_path($url) |
| 794 |
{ |
| 795 |
if (!self::has_cache($url)) { |
| 796 |
return false; |
| 797 |
} |
| 798 |
|
| 799 |
$path = bwp_get_static_cache_dir(); |
| 800 |
|
| 801 |
// $file_name = md5(self::clean_url($url)) . '.txt'; |
| 802 |
$file_name = md5($url) . '.txt'; |
| 803 |
|
| 804 |
return $path . $file_name; |
| 805 |
} |
| 806 |
|
| 807 |
static function is_valid_cache($url) |
| 808 |
{ |
| 809 |
$path = bwp_get_static_cache_dir(); |
| 810 |
|
| 811 |
// $file_name = md5(self::clean_url($url)) . '.txt'; |
| 812 |
$file_name = md5($url) . '.txt'; |
| 813 |
|
| 814 |
return is_file($path . $file_name) && (@filemtime($path . $file_name) + 60 * 60 > time()); |
| 815 |
} |
| 816 |
|
| 817 |
static function has_cache($url) |
| 818 |
{ |
| 819 |
$path = bwp_get_static_cache_dir(); |
| 820 |
|
| 821 |
// $file_name = md5(self::clean_url($url)) . '.txt'; |
| 822 |
$file_name = md5($url) . '.txt'; |
| 823 |
|
| 824 |
return is_file($path . $file_name); |
| 825 |
} |
| 826 |
|
| 827 |
static function get_cache($url) |
| 828 |
{ |
| 829 |
$path = bwp_get_static_cache_dir(); |
| 830 |
|
| 831 |
// $file_name = md5(self::clean_url($url)) . '.txt'; |
| 832 |
$file_name = md5($url) . '.txt'; |
| 833 |
|
| 834 |
return file_get_contents($path . $file_name); |
| 835 |
} |
| 836 |
|
| 837 |
static function cache_external_asset($url, $content) |
| 838 |
{ |
| 839 |
if (self::is_valid_cache($url)) { |
| 840 |
return $content; |
| 841 |
} |
| 842 |
|
| 843 |
$path = bwp_get_static_cache_dir(); |
| 844 |
|
| 845 |
// if (empty($content)) { |
| 846 |
// global $berq_log; |
| 847 |
// $berq_log->info("Empty file content, possibly file doesn't exist: $url"); |
| 848 |
// } |
| 849 |
|
| 850 |
// $file_name = md5(self::clean_url($url)) . '.txt'; |
| 851 |
$file_name = md5($url) . '.txt'; |
| 852 |
|
| 853 |
file_put_contents($path . $file_name, $content, LOCK_EX); |
| 854 |
|
| 855 |
return $content; |
| 856 |
} |
| 857 |
|
| 858 |
public static function proccess_css_ref() |
| 859 |
{ |
| 860 |
$css_assets = array_filter(self::$resources, function ($asset) { |
| 861 |
return $asset['type'] == 'css'; |
| 862 |
}); |
| 863 |
|
| 864 |
foreach ($css_assets as $content) { |
| 865 |
$urls = self::extractUrlsFromCss($content['content']); |
| 866 |
$asset_url = $content['url']; |
| 867 |
|
| 868 |
foreach ($urls as $url) { |
| 869 |
$url = trim($url); |
| 870 |
$url = explode('#', $url)[0]; |
| 871 |
|
| 872 |
if (empty($url)) { |
| 873 |
continue; |
| 874 |
} |
| 875 |
|
| 876 |
if (!self::is_valid_asset($url)) { |
| 877 |
continue; |
| 878 |
} |
| 879 |
|
| 880 |
// convert to abs |
| 881 |
$baseUrl = self::getBaseUrl($asset_url); |
| 882 |
$url = self::rel2abs($url, $baseUrl); |
| 883 |
|
| 884 |
// skip is if already exists |
| 885 |
if (!empty(self::resource_exists($url))) { |
| 886 |
continue; |
| 887 |
} |
| 888 |
|
| 889 |
if (self::is_self_hosted_url($url)) { |
| 890 |
$file_path = self::url_to_path($url); |
| 891 |
|
| 892 |
if (empty($file_path) || !is_file($file_path)) { |
| 893 |
|
| 894 |
self::cache_external_asset($url, ''); |
| 895 |
|
| 896 |
self::$resources[] = [ |
| 897 |
'url' => $url, |
| 898 |
'path' => self::get_cache_path($url), |
| 899 |
'type' => 'css_reff', |
| 900 |
'hosted' => true, |
| 901 |
'content' => '', |
| 902 |
'error' => '404', |
| 903 |
]; |
| 904 |
continue; |
| 905 |
} |
| 906 |
|
| 907 |
$file_content = file_get_contents($file_path); |
| 908 |
|
| 909 |
self::$resources[] = [ |
| 910 |
'url' => $url, |
| 911 |
'path' => $file_path, |
| 912 |
'type' => 'css_reff', |
| 913 |
'hosted' => true, |
| 914 |
'content' => $file_content, |
| 915 |
]; |
| 916 |
|
| 917 |
continue; |
| 918 |
} |
| 919 |
|
| 920 |
self::$resources[] = [ |
| 921 |
'url' => $url, |
| 922 |
'type' => 'css_reff', |
| 923 |
'hosted' => false, |
| 924 |
'content' => null, |
| 925 |
]; |
| 926 |
} |
| 927 |
} |
| 928 |
} |
| 929 |
public static function proccess_css_import() |
| 930 |
{ |
| 931 |
$css_assets = array_filter(self::$resources, function ($asset) { |
| 932 |
return $asset['type'] == 'css'; |
| 933 |
}); |
| 934 |
|
| 935 |
foreach ($css_assets as $content) { |
| 936 |
|
| 937 |
if (empty($content['content'])) { |
| 938 |
continue; |
| 939 |
} |
| 940 |
|
| 941 |
preg_match_all('/@import\s*[\'"]([^\'"]+)/', $content['content'], $match); |
| 942 |
$urls = $match[1]; |
| 943 |
$asset_url = $content['url']; |
| 944 |
|
| 945 |
foreach ($urls as $url) { |
| 946 |
$url = trim($url); |
| 947 |
|
| 948 |
if (empty($url)) { |
| 949 |
continue; |
| 950 |
} |
| 951 |
|
| 952 |
if (!self::is_valid_asset($url)) { |
| 953 |
continue; |
| 954 |
} |
| 955 |
|
| 956 |
// convert to abs |
| 957 |
$baseUrl = self::getBaseUrl($asset_url); |
| 958 |
$url = self::rel2abs($url, $baseUrl); |
| 959 |
|
| 960 |
// skip is if already exists |
| 961 |
if (!empty(self::resource_exists($url))) { |
| 962 |
continue; |
| 963 |
} |
| 964 |
|
| 965 |
if (self::is_self_hosted_url($url)) { |
| 966 |
$file_path = self::url_to_path($url); |
| 967 |
|
| 968 |
if (empty($file_path) || !is_file($file_path)) { |
| 969 |
|
| 970 |
self::cache_external_asset($url, ''); |
| 971 |
|
| 972 |
self::$resources[] = [ |
| 973 |
'url' => $url, |
| 974 |
'path' => self::get_cache_path($url), |
| 975 |
'type' => 'css_import', |
| 976 |
'hosted' => true, |
| 977 |
'content' => '', |
| 978 |
'error' => '404', |
| 979 |
]; |
| 980 |
continue; |
| 981 |
} |
| 982 |
|
| 983 |
$file_content = file_get_contents($file_path); |
| 984 |
|
| 985 |
self::$resources[] = [ |
| 986 |
'url' => $url, |
| 987 |
'path' => $file_path, |
| 988 |
'type' => 'css_import', |
| 989 |
'hosted' => true, |
| 990 |
'content' => $file_content, |
| 991 |
]; |
| 992 |
|
| 993 |
continue; |
| 994 |
} |
| 995 |
|
| 996 |
self::$resources[] = [ |
| 997 |
'url' => $url, |
| 998 |
'type' => 'css_reff', |
| 999 |
'hosted' => false, |
| 1000 |
'content' => null, |
| 1001 |
]; |
| 1002 |
} |
| 1003 |
} |
| 1004 |
} |
| 1005 |
|
| 1006 |
public static function proccess_external() |
| 1007 |
{ |
| 1008 |
$external = array_filter(self::$resources, function ($asset) { |
| 1009 |
return !$asset['hosted'] && $asset['content'] === null; |
| 1010 |
}); |
| 1011 |
|
| 1012 |
// get cache |
| 1013 |
$cached = array_filter($external, function ($asset) { |
| 1014 |
return self::has_cache($asset['url']) && self::is_valid_cache($asset['url']); |
| 1015 |
}); |
| 1016 |
|
| 1017 |
foreach ($cached as $cached_index => $cached_asset) { |
| 1018 |
$url = $cached_asset['url']; |
| 1019 |
|
| 1020 |
foreach (self::$resources as $resource_index => $resource) { |
| 1021 |
if ($resource['url'] == $url) { |
| 1022 |
self::$resources[$resource_index]['content'] = self::get_cache($url); |
| 1023 |
self::$resources[$resource_index]['path'] = self::get_cache_path($url); |
| 1024 |
|
| 1025 |
$external = array_filter($external, function ($asset) use ($url) { |
| 1026 |
return $asset['url'] !== $url; |
| 1027 |
}); |
| 1028 |
break; |
| 1029 |
} |
| 1030 |
} |
| 1031 |
} |
| 1032 |
|
| 1033 |
$responses = self::download_external_assets($external); |
| 1034 |
|
| 1035 |
foreach ($responses as $response_index => $response) { |
| 1036 |
$url = $external[$response_index]['url']; |
| 1037 |
|
| 1038 |
foreach (self::$resources as $resource_index => $resource) { |
| 1039 |
if ($resource['url'] == $url) { |
| 1040 |
|
| 1041 |
self::cache_external_asset($resource['url'], $response); |
| 1042 |
self::$resources[$resource_index]['content'] = $response; |
| 1043 |
self::$resources[$resource_index]['path'] = self::get_cache_path($resource['url']); |
| 1044 |
break; |
| 1045 |
} |
| 1046 |
} |
| 1047 |
} |
| 1048 |
} |
| 1049 |
|
| 1050 |
static function process_css($dom = null) |
| 1051 |
{ |
| 1052 |
if (empty($dom)) { |
| 1053 |
return; |
| 1054 |
} |
| 1055 |
|
| 1056 |
foreach ($dom->find('link[rel=stylesheet], link[as=style]') as $link) { |
| 1057 |
|
| 1058 |
$href = $link->getAttribute('href'); |
| 1059 |
|
| 1060 |
// convert to abs |
| 1061 |
$baseUrl = self::getBaseUrl(get_site_url()); |
| 1062 |
$href = self::rel2abs($href, $baseUrl); |
| 1063 |
|
| 1064 |
// skip is if already exists |
| 1065 |
if (!empty(self::resource_exists($href))) { |
| 1066 |
continue; |
| 1067 |
} |
| 1068 |
|
| 1069 |
if (!self::is_valid_asset($href)) { |
| 1070 |
continue; |
| 1071 |
} |
| 1072 |
|
| 1073 |
if (self::is_self_hosted_url($href)) { |
| 1074 |
$css_path = self::url_to_path($href); |
| 1075 |
|
| 1076 |
if (empty($css_path) || !is_file($css_path)) { |
| 1077 |
|
| 1078 |
self::cache_external_asset($href, ''); |
| 1079 |
|
| 1080 |
self::$resources[] = [ |
| 1081 |
'url' => $href, |
| 1082 |
'path' => self::get_cache_path($href), |
| 1083 |
'type' => 'css', |
| 1084 |
'hosted' => true, |
| 1085 |
'content' => '', |
| 1086 |
'error' => '404', |
| 1087 |
]; |
| 1088 |
continue; |
| 1089 |
} |
| 1090 |
|
| 1091 |
$local_css = file_get_contents($css_path); |
| 1092 |
|
| 1093 |
self::$resources[] = [ |
| 1094 |
'url' => $href, |
| 1095 |
'path' => $css_path, |
| 1096 |
'type' => 'css', |
| 1097 |
'hosted' => true, |
| 1098 |
'content' => $local_css, |
| 1099 |
]; |
| 1100 |
|
| 1101 |
continue; |
| 1102 |
} |
| 1103 |
|
| 1104 |
self::$resources[] = [ |
| 1105 |
'url' => $href, |
| 1106 |
'type' => 'css', |
| 1107 |
'hosted' => false, |
| 1108 |
'content' => null, |
| 1109 |
]; |
| 1110 |
} |
| 1111 |
} |
| 1112 |
|
| 1113 |
static function proccess_inline_css($dom = null) |
| 1114 |
{ |
| 1115 |
if (empty($dom)) { |
| 1116 |
return; |
| 1117 |
} |
| 1118 |
|
| 1119 |
foreach ($dom->find('style') as $style) { |
| 1120 |
|
| 1121 |
$urls = self::extractUrlsFromCss($style->innertext); |
| 1122 |
|
| 1123 |
foreach ($urls as $url) { |
| 1124 |
$url = trim($url); |
| 1125 |
|
| 1126 |
if (empty($url)) { |
| 1127 |
continue; |
| 1128 |
} |
| 1129 |
|
| 1130 |
if (!self::is_valid_asset($url)) { |
| 1131 |
continue; |
| 1132 |
} |
| 1133 |
|
| 1134 |
// convert to abs |
| 1135 |
$baseUrl = self::getBaseUrl(get_site_url()); |
| 1136 |
$url = self::rel2abs($url, $baseUrl); |
| 1137 |
|
| 1138 |
// skip is if already exists |
| 1139 |
if (!empty(self::resource_exists($url))) { |
| 1140 |
continue; |
| 1141 |
} |
| 1142 |
|
| 1143 |
if (self::is_self_hosted_url($url)) { |
| 1144 |
$file_path = self::url_to_path($url); |
| 1145 |
|
| 1146 |
if (empty($file_path) || !is_file($file_path)) { |
| 1147 |
|
| 1148 |
self::cache_external_asset($url, ''); |
| 1149 |
|
| 1150 |
self::$resources[] = [ |
| 1151 |
'url' => $url, |
| 1152 |
'path' => self::get_cache_path($url), |
| 1153 |
'type' => 'inline_css', |
| 1154 |
'hosted' => true, |
| 1155 |
'content' => '', |
| 1156 |
'error' => '404', |
| 1157 |
]; |
| 1158 |
continue; |
| 1159 |
} |
| 1160 |
|
| 1161 |
$file_content = file_get_contents($file_path); |
| 1162 |
|
| 1163 |
self::$resources[] = [ |
| 1164 |
'url' => $url, |
| 1165 |
'path' => $file_path, |
| 1166 |
'type' => 'inline_css', |
| 1167 |
'hosted' => true, |
| 1168 |
'content' => $file_content, |
| 1169 |
]; |
| 1170 |
|
| 1171 |
continue; |
| 1172 |
} |
| 1173 |
|
| 1174 |
self::$resources[] = [ |
| 1175 |
'url' => $url, |
| 1176 |
'type' => 'inline_css', |
| 1177 |
'hosted' => false, |
| 1178 |
'content' => null, |
| 1179 |
]; |
| 1180 |
} |
| 1181 |
} |
| 1182 |
} |
| 1183 |
|
| 1184 |
static function process_js($dom = null) |
| 1185 |
{ |
| 1186 |
if (empty($dom)) { |
| 1187 |
return; |
| 1188 |
} |
| 1189 |
|
| 1190 |
foreach ($dom->find('script') as $script) { |
| 1191 |
|
| 1192 |
if (empty($script->getAttribute('src'))) { |
| 1193 |
continue; |
| 1194 |
} |
| 1195 |
|
| 1196 |
$src = $script->getAttribute('src'); |
| 1197 |
|
| 1198 |
// convert to abs |
| 1199 |
$baseUrl = self::getBaseUrl(get_site_url()); |
| 1200 |
$src = self::rel2abs($src, $baseUrl); |
| 1201 |
|
| 1202 |
// skip is if already exists |
| 1203 |
if (!empty(self::resource_exists($src))) { |
| 1204 |
continue; |
| 1205 |
} |
| 1206 |
|
| 1207 |
if (!self::is_valid_asset($src)) { |
| 1208 |
continue; |
| 1209 |
} |
| 1210 |
|
| 1211 |
if (self::is_self_hosted_url($src)) { |
| 1212 |
$js_path = self::url_to_path($src); |
| 1213 |
|
| 1214 |
if (empty($js_path) || !is_file($js_path)) { |
| 1215 |
|
| 1216 |
self::cache_external_asset($src, ''); |
| 1217 |
|
| 1218 |
self::$resources[] = [ |
| 1219 |
'url' => $src, |
| 1220 |
'path' => self::get_cache_path($src), |
| 1221 |
'type' => 'js', |
| 1222 |
'hosted' => true, |
| 1223 |
'content' => '', |
| 1224 |
'error' => '404', |
| 1225 |
]; |
| 1226 |
continue; |
| 1227 |
} |
| 1228 |
|
| 1229 |
// $js = file_get_contents($js_path); |
| 1230 |
|
| 1231 |
self::$resources[] = [ |
| 1232 |
'url' => $src, |
| 1233 |
'path' => $js_path, |
| 1234 |
'type' => 'js', |
| 1235 |
'hosted' => true, |
| 1236 |
'content' => '', |
| 1237 |
]; |
| 1238 |
|
| 1239 |
continue; |
| 1240 |
} |
| 1241 |
|
| 1242 |
self::$resources[] = [ |
| 1243 |
'url' => $src, |
| 1244 |
'type' => 'js', |
| 1245 |
'hosted' => false, |
| 1246 |
'content' => null, |
| 1247 |
]; |
| 1248 |
} |
| 1249 |
} |
| 1250 |
|
| 1251 |
static function process_images($dom = null) |
| 1252 |
{ |
| 1253 |
if (empty($dom)) { |
| 1254 |
return; |
| 1255 |
} |
| 1256 |
|
| 1257 |
foreach ($dom->find('img') as $image) { |
| 1258 |
|
| 1259 |
if (empty($image->getAttribute('src'))) { |
| 1260 |
continue; |
| 1261 |
} |
| 1262 |
|
| 1263 |
$src = $image->getAttribute('src'); |
| 1264 |
|
| 1265 |
if (!empty($image->getAttribute('srcset'))) { |
| 1266 |
self::process_srcset($image->getAttribute('srcset')); |
| 1267 |
} |
| 1268 |
|
| 1269 |
// convert to abs |
| 1270 |
$baseUrl = self::getBaseUrl(get_site_url()); |
| 1271 |
$src = self::rel2abs($src, $baseUrl); |
| 1272 |
|
| 1273 |
// skip is if already exists |
| 1274 |
if (!empty(self::resource_exists($src))) { |
| 1275 |
continue; |
| 1276 |
} |
| 1277 |
|
| 1278 |
if (!self::is_valid_asset($src)) { |
| 1279 |
continue; |
| 1280 |
} |
| 1281 |
|
| 1282 |
if (self::is_self_hosted_url($src)) { |
| 1283 |
$img_path = self::url_to_path($src); |
| 1284 |
|
| 1285 |
if (empty($img_path) || !is_file($img_path)) { |
| 1286 |
|
| 1287 |
self::cache_external_asset($src, ''); |
| 1288 |
|
| 1289 |
self::$resources[] = [ |
| 1290 |
'url' => $src, |
| 1291 |
'path' => self::get_cache_path($src), |
| 1292 |
'type' => 'img', |
| 1293 |
'hosted' => true, |
| 1294 |
'content' => '', |
| 1295 |
'error' => '404', |
| 1296 |
]; |
| 1297 |
continue; |
| 1298 |
} |
| 1299 |
|
| 1300 |
// $img = file_get_contents($img_path); |
| 1301 |
|
| 1302 |
self::$resources[] = [ |
| 1303 |
'url' => $src, |
| 1304 |
'path' => $img_path, |
| 1305 |
'type' => 'img', |
| 1306 |
'hosted' => true, |
| 1307 |
'content' => '', |
| 1308 |
]; |
| 1309 |
|
| 1310 |
continue; |
| 1311 |
} |
| 1312 |
|
| 1313 |
self::$resources[] = [ |
| 1314 |
'url' => $src, |
| 1315 |
'type' => 'img', |
| 1316 |
'hosted' => false, |
| 1317 |
'content' => null, |
| 1318 |
]; |
| 1319 |
} |
| 1320 |
} |
| 1321 |
|
| 1322 |
static function clean_url($url) |
| 1323 |
{ |
| 1324 |
$parts = wp_parse_url($url); |
| 1325 |
|
| 1326 |
if (!$parts) { |
| 1327 |
return ''; |
| 1328 |
} |
| 1329 |
|
| 1330 |
$clean = ($parts['scheme'] ?? 'https') . '://'; |
| 1331 |
$clean .= $parts['host'] ?? ''; |
| 1332 |
|
| 1333 |
if (!empty($parts['port'])) { |
| 1334 |
$clean .= ':' . $parts['port']; |
| 1335 |
} |
| 1336 |
|
| 1337 |
$clean .= $parts['path'] ?? '/'; |
| 1338 |
|
| 1339 |
return $clean; |
| 1340 |
} |
| 1341 |
|
| 1342 |
static function resource_exists($url) |
| 1343 |
{ |
| 1344 |
|
| 1345 |
return array_filter(self::$resources, function ($asset) use ($url) { |
| 1346 |
return $asset['url'] == $url; |
| 1347 |
}); |
| 1348 |
} |
| 1349 |
|
| 1350 |
static function process_srcset($srcset) |
| 1351 |
{ |
| 1352 |
|
| 1353 |
$sources = explode(',', $srcset); |
| 1354 |
|
| 1355 |
foreach ($sources as $source) { |
| 1356 |
// Extract URL and remove leading/trailing whitespace |
| 1357 |
$srcset_img_url = trim(explode(' ', trim($source))[0]); |
| 1358 |
|
| 1359 |
// convert to abs |
| 1360 |
$baseUrl = self::getBaseUrl(get_site_url()); |
| 1361 |
$srcset_img_url = self::rel2abs($srcset_img_url, $baseUrl); |
| 1362 |
|
| 1363 |
// skip is if already exists |
| 1364 |
if (!empty(self::resource_exists($srcset_img_url))) { |
| 1365 |
continue; |
| 1366 |
} |
| 1367 |
|
| 1368 |
if (!self::is_valid_asset($srcset_img_url)) { |
| 1369 |
continue; |
| 1370 |
} |
| 1371 |
|
| 1372 |
if (self::is_self_hosted_url($srcset_img_url)) { |
| 1373 |
$img_path = self::url_to_path($srcset_img_url); |
| 1374 |
|
| 1375 |
if (empty($img_path) || !is_file($img_path)) { |
| 1376 |
|
| 1377 |
self::cache_external_asset($srcset_img_url, ''); |
| 1378 |
|
| 1379 |
self::$resources[] = [ |
| 1380 |
'url' => $srcset_img_url, |
| 1381 |
'path' => self::get_cache_path($srcset_img_url), |
| 1382 |
'type' => 'img', |
| 1383 |
'hosted' => true, |
| 1384 |
'content' => '', |
| 1385 |
'error' => '404', |
| 1386 |
]; |
| 1387 |
continue; |
| 1388 |
} |
| 1389 |
|
| 1390 |
// $img = file_get_contents($img_path); |
| 1391 |
|
| 1392 |
self::$resources[] = [ |
| 1393 |
'url' => $srcset_img_url, |
| 1394 |
'path' => $img_path, |
| 1395 |
'type' => 'img', |
| 1396 |
'hosted' => true, |
| 1397 |
'content' => '', |
| 1398 |
]; |
| 1399 |
|
| 1400 |
continue; |
| 1401 |
} |
| 1402 |
|
| 1403 |
self::$resources[] = [ |
| 1404 |
'url' => $srcset_img_url, |
| 1405 |
'type' => 'img', |
| 1406 |
'hosted' => false, |
| 1407 |
'content' => null, |
| 1408 |
]; |
| 1409 |
} |
| 1410 |
} |
| 1411 |
|
| 1412 |
static function is_self_hosted_url($url) |
| 1413 |
{ |
| 1414 |
if (empty($url)) { |
| 1415 |
return false; |
| 1416 |
} |
| 1417 |
|
| 1418 |
// Relative URLs are self-hosted |
| 1419 |
if (strpos($url, '//') === false) { |
| 1420 |
return true; |
| 1421 |
} |
| 1422 |
|
| 1423 |
$site_host = wp_parse_url(site_url(), PHP_URL_HOST); |
| 1424 |
$url_host = wp_parse_url($url, PHP_URL_HOST); |
| 1425 |
|
| 1426 |
return $site_host && $url_host && strtolower($site_host) === strtolower($url_host); |
| 1427 |
} |
| 1428 |
|
| 1429 |
static function url_to_path($url) |
| 1430 |
{ |
| 1431 |
|
| 1432 |
if (strpos($url, '?') !== false) { |
| 1433 |
$url = explode('?', $url)[0]; |
| 1434 |
} |
| 1435 |
|
| 1436 |
if (strpos($url, '#') !== false) { |
| 1437 |
$url = explode('#', $url)[0]; |
| 1438 |
} |
| 1439 |
|
| 1440 |
// Handle relative URLs |
| 1441 |
if (strpos($url, '//') === 0) { |
| 1442 |
return realpath(ABSPATH . ltrim($url, '/')); |
| 1443 |
} |
| 1444 |
|
| 1445 |
$content_url = content_url(); |
| 1446 |
$content_dir = WP_CONTENT_DIR; |
| 1447 |
|
| 1448 |
if (strpos($url, $content_url) === 0) { |
| 1449 |
// var_dump(realpath( |
| 1450 |
// str_replace($content_url, $content_dir, $url) |
| 1451 |
// )); |
| 1452 |
return str_replace($content_url, $content_dir, $url); |
| 1453 |
} |
| 1454 |
|
| 1455 |
// Fallback for site root files |
| 1456 |
$site_url = get_site_url('/'); |
| 1457 |
|
| 1458 |
if (strpos($url, $site_url) === 0) { |
| 1459 |
return realpath( |
| 1460 |
str_replace($site_url, ABSPATH, $url) |
| 1461 |
); |
| 1462 |
} |
| 1463 |
|
| 1464 |
return false; |
| 1465 |
} |
| 1466 |
|
| 1467 |
public static function rel2abs($rel, $base) |
| 1468 |
{ |
| 1469 |
/* return if already absolute URL */ |
| 1470 |
if (parse_url($rel, PHP_URL_SCHEME) != '') |
| 1471 |
return $rel; |
| 1472 |
|
| 1473 |
// Return with base scheme if protocol-relative (starts with //) |
| 1474 |
if (strpos($rel, '//') === 0) { |
| 1475 |
$scheme = parse_url($base, PHP_URL_SCHEME) ?: 'https'; |
| 1476 |
return $scheme . ':' . $rel; |
| 1477 |
} |
| 1478 |
|
| 1479 |
/* queries and anchors */ |
| 1480 |
if ($rel[0] == '#' || $rel[0] == '?') |
| 1481 |
return $base . $rel; |
| 1482 |
|
| 1483 |
/* parse base URL and convert to local variables: |
| 1484 |
$scheme, $host, $path */ |
| 1485 |
extract(parse_url($base)); |
| 1486 |
|
| 1487 |
/* remove non-directory element from path */ |
| 1488 |
$path = preg_replace('#/[^/]*$#', '', $path); |
| 1489 |
|
| 1490 |
/* destroy path if relative url points to root */ |
| 1491 |
if ($rel[0] == '/') |
| 1492 |
$path = ''; |
| 1493 |
|
| 1494 |
/* dirty absolute URL */ |
| 1495 |
$abs = "$host$path/$rel"; |
| 1496 |
|
| 1497 |
/* replace '//' or '/./' or '/foo/../' with '/' */ |
| 1498 |
$re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#'); |
| 1499 |
for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) { |
| 1500 |
} |
| 1501 |
|
| 1502 |
/* absolute URL is ready! */ |
| 1503 |
return $scheme . '://' . $abs; |
| 1504 |
} |
| 1505 |
|
| 1506 |
public static function getBaseUrl($fileUrl) |
| 1507 |
{ |
| 1508 |
// Parse the URL and get its components |
| 1509 |
$parsedUrl = parse_url($fileUrl); |
| 1510 |
$scheme = isset($parsedUrl['scheme']) ? $parsedUrl['scheme'] . '://' : ''; |
| 1511 |
$host = isset($parsedUrl['host']) ? $parsedUrl['host'] : ''; |
| 1512 |
$path = isset($parsedUrl['path']) ? $parsedUrl['path'] : ''; |
| 1513 |
|
| 1514 |
// Remove the file name from the path to get the base URL |
| 1515 |
$basePath = preg_replace('/\/[^\/]+$/', '/', $path); |
| 1516 |
|
| 1517 |
// Construct the base URL |
| 1518 |
return $scheme . $host . $basePath; |
| 1519 |
} |
| 1520 |
|
| 1521 |
public static function extractUrlsFromCss($cssContent) |
| 1522 |
{ |
| 1523 |
$urls = []; |
| 1524 |
$pattern = '/url\((.*?)\)/i'; |
| 1525 |
|
| 1526 |
if (empty($cssContent)) { |
| 1527 |
return []; |
| 1528 |
} |
| 1529 |
|
| 1530 |
preg_match_all($pattern, $cssContent, $matches); |
| 1531 |
|
| 1532 |
if (!empty($matches[1])) { |
| 1533 |
foreach ($matches[1] as $match) { |
| 1534 |
$urls[] = trim($match, '\'" '); |
| 1535 |
} |
| 1536 |
} |
| 1537 |
|
| 1538 |
return $urls; |
| 1539 |
} |
| 1540 |
|
| 1541 |
private static function download_external_assets($assets) |
| 1542 |
{ |
| 1543 |
$results = []; |
| 1544 |
$chunks = array_chunk($assets, 10, true); // Process 10 at a time |
| 1545 |
|
| 1546 |
foreach ($chunks as $chunk) { |
| 1547 |
$multi = curl_multi_init(); |
| 1548 |
$handles = []; |
| 1549 |
|
| 1550 |
// Initialize cURL handles |
| 1551 |
foreach ($chunk as $index => $asset) { |
| 1552 |
$ch = curl_init(); |
| 1553 |
curl_setopt_array($ch, [ |
| 1554 |
CURLOPT_URL => $asset['url'], |
| 1555 |
CURLOPT_RETURNTRANSFER => true, |
| 1556 |
CURLOPT_FOLLOWLOCATION => true, |
| 1557 |
CURLOPT_MAXREDIRS => 3, |
| 1558 |
CURLOPT_TIMEOUT => 15, |
| 1559 |
CURLOPT_CONNECTTIMEOUT => 5, |
| 1560 |
CURLOPT_SSL_VERIFYPEER => false, |
| 1561 |
CURLOPT_SSL_VERIFYHOST => false, |
| 1562 |
CURLOPT_USERAGENT => 'Mozilla/5.0 (compatible; BWP2/1.0)', |
| 1563 |
CURLOPT_MAXFILESIZE => 5 * 1024 * 1024, // 5 MB limit per asset |
| 1564 |
]); |
| 1565 |
|
| 1566 |
curl_multi_add_handle($multi, $ch); |
| 1567 |
$handles[$index] = $ch; |
| 1568 |
} |
| 1569 |
|
| 1570 |
// Execute parallel downloads |
| 1571 |
$running = null; |
| 1572 |
do { |
| 1573 |
curl_multi_exec($multi, $running); |
| 1574 |
curl_multi_select($multi); |
| 1575 |
} while ($running > 0); |
| 1576 |
|
| 1577 |
// Collect results |
| 1578 |
foreach ($handles as $index => $ch) { |
| 1579 |
$content = curl_multi_getcontent($ch); |
| 1580 |
$http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); |
| 1581 |
|
| 1582 |
$results[$index] = ($http_code === 200 && $content !== false) ? $content : false; |
| 1583 |
|
| 1584 |
curl_multi_remove_handle($multi, $ch); |
| 1585 |
curl_close($ch); |
| 1586 |
} |
| 1587 |
|
| 1588 |
curl_multi_close($multi); |
| 1589 |
} |
| 1590 |
|
| 1591 |
return $results; |
| 1592 |
} |
| 1593 |
} |
| 1594 |
|
