parsers
1 month ago
admin.php
1 month ago
buffers.php
2 weeks ago
core.php
2 weeks ago
engine.php
1 month ago
init.php
9 months ago
mcp.php
1 month ago
parsers.php
1 month ago
rest.php
2 weeks ago
runs.php
1 month ago
support.php
1 month ago
ui.php
1 month ago
buffers.php
681 lines
| 1 | <?php |
| 2 | |
| 3 | /** |
| 4 | * Sizes the scan buffers from what the server can actually do. |
| 5 | * |
| 6 | * Every buffer answers one question: how many items can a single request handle before PHP, |
| 7 | * the proxy or the memory limit cuts it off. Rather than asking the user to guess, the same |
| 8 | * work each phase really does — parsing content, resolving media paths, listing uploads, |
| 9 | * fingerprinting files, flushing references — is run for real and timed. |
| 10 | * |
| 11 | * It is measured the way it is used: as separate HTTP requests, at growing sample sizes. That |
| 12 | * matters, because a batch costs a fixed amount (bootstrap, parser init, the round trip) plus |
| 13 | * something per item, and those two only separate once the same work has been timed at two |
| 14 | * different sizes. One sample size can only ever produce an average that hides the fixed part. |
| 15 | * |
| 16 | * The dashboard drives it: plan() says what can be measured here, measure() runs one rung of |
| 17 | * one probe per request, and apply() fits a line through the collected rungs and stores the |
| 18 | * result. Buffers only decide how work is split across requests, never what the analysis |
| 19 | * finds, so getting one wrong costs time, not correctness. Settings that would change what the |
| 20 | * analysis sees (the document limit) are left alone, and so is the delay, which paces requests |
| 21 | * for the server's sake and cannot be derived from a benchmark. |
| 22 | * |
| 23 | * The probes are read-only. The only write is into a TEMPORARY copy of the references table, |
| 24 | * which never touches scan results and disappears with the connection. |
| 25 | */ |
| 26 | class Meow_WPMC_Buffers { |
| 27 | |
| 28 | // Only part of the request budget is spent on a batch, so an item slower than the ones |
| 29 | // sampled here still fits. The engine yields mid-batch anyway; this keeps that rare. |
| 30 | const BUDGET_TARGET = 0.5; |
| 31 | // A batch may hold at most this share of the PHP memory limit. |
| 32 | const MEMORY_TARGET = 0.25; |
| 33 | // A reference row is small; this is a deliberately generous estimate of its wire size, |
| 34 | // used to keep a flush well inside MySQL's max_allowed_packet. |
| 35 | const REF_ROW_BYTES = 512; |
| 36 | |
| 37 | private $core; |
| 38 | private $deadline = 0; |
| 39 | |
| 40 | public function __construct( $core ) { |
| 41 | $this->core = $core; |
| 42 | } |
| 43 | |
| 44 | // Every probe, the setting it sizes, and the sample sizes it is measured at. A rung is only |
| 45 | // offered when the install actually has that much to sample. |
| 46 | private function probes() { |
| 47 | return array( |
| 48 | 'content' => array( |
| 49 | 'option' => 'posts_buffer', |
| 50 | 'label' => __( 'Content', 'media-cleaner' ), |
| 51 | 'ladder' => array( 5, 25, 100 ), |
| 52 | ), |
| 53 | 'media' => array( |
| 54 | 'option' => 'medias_buffer', |
| 55 | 'label' => __( 'Media Library', 'media-cleaner' ), |
| 56 | 'ladder' => array( 5, 25, 100 ), |
| 57 | ), |
| 58 | 'analysis' => array( |
| 59 | 'option' => 'analysis_buffer', |
| 60 | 'label' => __( 'Analysis', 'media-cleaner' ), |
| 61 | 'ladder' => array( 5, 25, 100 ), |
| 62 | ), |
| 63 | 'files' => array( |
| 64 | 'option' => 'uploads_file_buffer', |
| 65 | 'label' => __( 'Filesystem', 'media-cleaner' ), |
| 66 | 'ladder' => array( 10, 50, 200 ), |
| 67 | ), |
| 68 | 'file_operation' => array( |
| 69 | 'option' => 'file_op_buffer', |
| 70 | 'label' => __( 'File Operations', 'media-cleaner' ), |
| 71 | 'ladder' => array( 5, 20 ), |
| 72 | ), |
| 73 | 'references' => array( |
| 74 | 'option' => 'refs_buffer', |
| 75 | 'label' => __( 'References', 'media-cleaner' ), |
| 76 | 'ladder' => array( 200, 1000 ), |
| 77 | ), |
| 78 | ); |
| 79 | } |
| 80 | |
| 81 | /** |
| 82 | * What can be measured on this install, and at which sample sizes. A phase with nothing to |
| 83 | * sample (no posts, no media, no filesystem scan) is left out entirely: its setting will |
| 84 | * keep its current value, because a guess is not an improvement over what the user has. |
| 85 | */ |
| 86 | public function plan() { |
| 87 | $blocked = $this->blocked(); |
| 88 | if ( is_wp_error( $blocked ) ) return $blocked; |
| 89 | |
| 90 | $available = $this->availability(); |
| 91 | $steps = array(); |
| 92 | foreach ( $this->probes() as $name => $probe ) { |
| 93 | $rungs = $this->ladder( $probe['ladder'], $available[ $name ] ); |
| 94 | if ( empty( $rungs ) ) continue; |
| 95 | $steps[] = array( |
| 96 | 'probe' => $name, |
| 97 | 'option' => $probe['option'], |
| 98 | 'label' => $probe['label'], |
| 99 | 'ladder' => $rungs, |
| 100 | ); |
| 101 | } |
| 102 | return array( 'environment' => $this->environment(), 'steps' => $steps ); |
| 103 | } |
| 104 | |
| 105 | /** |
| 106 | * One rung: run $items of one probe, in its own request, and report what it cost. The |
| 107 | * caller times the round trip; this reports the server side of it. |
| 108 | */ |
| 109 | public function measure( $name, $items ) { |
| 110 | $blocked = $this->blocked(); |
| 111 | if ( is_wp_error( $blocked ) ) return $blocked; |
| 112 | |
| 113 | $probes = $this->probes(); |
| 114 | if ( !isset( $probes[ $name ] ) ) { |
| 115 | return new WP_Error( 'wpmc_buffers_unknown_probe', |
| 116 | __( 'Unknown benchmark step.', 'media-cleaner' ), array( 'status' => 400 ) ); |
| 117 | } |
| 118 | // A rung must not outlast the request that carries it. Stopping early is not a failure: |
| 119 | // the number of items actually done is reported, and that is what gets fitted. |
| 120 | $this->deadline = microtime( true ) + $this->core->get_request_time_budget() * 0.8; |
| 121 | $items = max( 1, (int) $items ); |
| 122 | $result = call_user_func( array( $this, 'probe_' . $name ), $items ); |
| 123 | $result['probe'] = $name; |
| 124 | $result['requested'] = $items; |
| 125 | return $result; |
| 126 | } |
| 127 | |
| 128 | /** |
| 129 | * Fits the collected rungs and sizes every buffer it has evidence for. Nothing is stored, |
| 130 | * so this is what the dashboard shows before the user decides. |
| 131 | * |
| 132 | * $rounds is what the dashboard collected: probe name => list of rungs, each with the items |
| 133 | * done, the seconds the server spent, the bytes it used, and the round trip the browser saw. |
| 134 | */ |
| 135 | public function recommend( $rounds ) { |
| 136 | $rounds = $this->sanitize_rounds( $rounds ); |
| 137 | $environment = $this->environment(); |
| 138 | $environment['request_overhead'] = $this->request_overhead( $rounds ); |
| 139 | $environment['longest_request'] = $this->longest_request( $rounds ); |
| 140 | $budget = $environment['request_budget'] * self::BUDGET_TARGET; |
| 141 | |
| 142 | $options = $this->core->get_all_options(); |
| 143 | $report = array( |
| 144 | 'environment' => $environment, |
| 145 | 'buffers' => array(), |
| 146 | 'warnings' => $this->warnings( $rounds, $environment ), |
| 147 | 'applied' => array(), |
| 148 | ); |
| 149 | |
| 150 | foreach ( $this->probes() as $name => $probe ) { |
| 151 | $option = $probe['option']; |
| 152 | $fit = $this->fit( isset( $rounds[ $name ] ) ? $rounds[ $name ] : array() ); |
| 153 | $value = $fit === null ? null : $this->size( $name, $fit, $budget, $environment ); |
| 154 | $report['buffers'][ $option ] = array( |
| 155 | 'probe' => $name, |
| 156 | 'label' => $probe['label'], |
| 157 | 'from' => (int) $options[ $option ], |
| 158 | 'to' => $value === null ? null : $this->clamp( $option, $value ), |
| 159 | 'fit' => $fit, |
| 160 | ); |
| 161 | } |
| 162 | return $report; |
| 163 | } |
| 164 | |
| 165 | /** |
| 166 | * Stores what recommend() proposes. Buffers whose phase could not be measured keep their |
| 167 | * current value: a guess is not an improvement over what the user already has. |
| 168 | */ |
| 169 | public function apply( $rounds ) { |
| 170 | $blocked = $this->blocked(); |
| 171 | if ( is_wp_error( $blocked ) ) return $blocked; |
| 172 | |
| 173 | $report = $this->recommend( $rounds ); |
| 174 | $options = $this->core->get_all_options(); |
| 175 | $changes = array(); |
| 176 | foreach ( $report['buffers'] as $option => $buffer ) { |
| 177 | if ( $buffer['to'] === null ) continue; |
| 178 | $options[ $option ] = $buffer['to']; |
| 179 | $changes[] = $option; |
| 180 | } |
| 181 | |
| 182 | if ( !empty( $changes ) ) { |
| 183 | // update_options() clamps to option_ranges(), so the stored value is the truth. |
| 184 | $options = $this->core->update_options( $options ); |
| 185 | foreach ( $changes as $option ) { |
| 186 | $report['buffers'][ $option ]['to'] = (int) $options[ $option ]; |
| 187 | } |
| 188 | } |
| 189 | $report['applied'] = $changes; |
| 190 | $this->core->log( sprintf( 'Auto Buffer: %s.', empty( $changes ) |
| 191 | ? 'nothing could be measured, settings left as they are' |
| 192 | : implode( ', ', array_map( function( $option ) use ( $report ) { |
| 193 | return $option . ' ' . $report['buffers'][ $option ]['from'] . ' → ' . $report['buffers'][ $option ]['to']; |
| 194 | }, $changes ) ) ) ); |
| 195 | return $report; |
| 196 | } |
| 197 | |
| 198 | /** |
| 199 | * What the server allows. The request budget is the plugin's own: the time one REST call |
| 200 | * may spend before it has to hand control back, which is what a buffer is really sized by. |
| 201 | */ |
| 202 | public function environment() { |
| 203 | global $wpdb; |
| 204 | $packet = $wpdb->get_var( "SELECT @@max_allowed_packet" ); |
| 205 | return array( |
| 206 | 'max_execution_time' => $this->core->get_max_execution_time(), |
| 207 | 'memory_limit' => $this->core->parse_ini_bytes( ini_get( 'memory_limit' ) ), |
| 208 | 'request_budget' => $this->core->get_request_time_budget(), |
| 209 | 'max_allowed_packet' => $packet === null ? 0 : (int) $packet, |
| 210 | ); |
| 211 | } |
| 212 | |
| 213 | // A scan in flight would both distort the timings and be distorted by them. |
| 214 | private function blocked() { |
| 215 | if ( $this->core->runs && $this->core->runs->get_resumable() ) { |
| 216 | return new WP_Error( 'wpmc_buffers_scan_running', |
| 217 | __( 'Publish or cancel the staged scan before measuring the server.', 'media-cleaner' ), |
| 218 | array( 'status' => 409 ) ); |
| 219 | } |
| 220 | return null; |
| 221 | } |
| 222 | |
| 223 | #region Probes |
| 224 | |
| 225 | // Reading a post and pulling the URLs out of it is what every content parser triggers, |
| 226 | // shortcode rendering included. The parsers themselves are not run: they write references, |
| 227 | // and a benchmark must not leave any behind. |
| 228 | private function probe_content( $items ) { |
| 229 | return $this->run( $this->sample( 'posts', $items ), function( $post_id ) { |
| 230 | $html = get_post_field( 'post_content', $post_id ); |
| 231 | get_post_meta( $post_id ); |
| 232 | $this->core->get_urls_from_html( $html ); |
| 233 | } ); |
| 234 | } |
| 235 | |
| 236 | // Listing a media entry means resolving every file it owns, thumbnails included. |
| 237 | private function probe_media( $items ) { |
| 238 | return $this->run( $this->sample( 'medias', $items ), function( $media_id ) { |
| 239 | $this->core->get_paths_from_attachment( $media_id ); |
| 240 | } ); |
| 241 | } |
| 242 | |
| 243 | // Analysing one media is the same path resolution plus the reference lookups that decide |
| 244 | // whether it is used — the queries that dominate the matching step. |
| 245 | private function probe_analysis( $items ) { |
| 246 | return $this->run( $this->sample( 'medias', $items ), function( $media_id ) { |
| 247 | $paths = $this->core->get_paths_from_attachment( $media_id ); |
| 248 | foreach ( $paths as $path ) { |
| 249 | $this->core->reference_exists( $path, $media_id ); |
| 250 | } |
| 251 | } ); |
| 252 | } |
| 253 | |
| 254 | // The filesystem scan lists a directory page, then asks the same reference question about |
| 255 | // every file in it. Both halves are timed: the listing through the real iterator, the |
| 256 | // matching against real file paths. |
| 257 | private function probe_files( $items ) { |
| 258 | $listing = 0; |
| 259 | $entries = 0; |
| 260 | try { |
| 261 | $started = microtime( true ); |
| 262 | $found = $this->core->engine ? $this->core->engine->get_files( '', 0, $items ) : array(); |
| 263 | $listing = microtime( true ) - $started; |
| 264 | $entries = count( $found ); |
| 265 | } |
| 266 | catch ( Throwable $e ) { |
| 267 | return $this->unmeasured( __( 'The uploads directory could not be listed.', 'media-cleaner' ) ); |
| 268 | } |
| 269 | if ( $entries < 1 ) return $this->unmeasured( __( 'The filesystem scan is not available.', 'media-cleaner' ) ); |
| 270 | $probe = $this->run( $this->sample( 'files', $items ), function( $path ) { |
| 271 | $this->core->reference_exists( $path, null ); |
| 272 | } ); |
| 273 | if ( $probe['samples'] < 1 ) return $probe; |
| 274 | // Charge the listing to the files it produced, then to the files actually matched. |
| 275 | $probe['seconds'] += ( $listing / $entries ) * $probe['samples']; |
| 276 | $probe['note'] = __( 'Listing and matching.', 'media-cleaner' ); |
| 277 | return $probe; |
| 278 | } |
| 279 | |
| 280 | // Trashing or recovering an item verifies its fingerprint, then moves the file. The |
| 281 | // fingerprint reads real files; the move is measured through the round-trip test that |
| 282 | // already exists, and charged to every item, because every item is a real move. |
| 283 | private function probe_file_operation( $items ) { |
| 284 | $probe = $this->run( $this->sample( 'files', $items ), function( $path ) { |
| 285 | $absolute = $this->core->resolve_upload_path( $path ); |
| 286 | if ( is_wp_error( $absolute ) ) return; |
| 287 | $this->core->file_fingerprint( $absolute ); |
| 288 | } ); |
| 289 | if ( $probe['samples'] < 1 ) return $probe; |
| 290 | $started = microtime( true ); |
| 291 | $roundtrip = $this->core->test_quarantine_roundtrip(); |
| 292 | if ( !is_wp_error( $roundtrip ) ) { |
| 293 | $probe['seconds'] += ( microtime( true ) - $started ) * $probe['samples']; |
| 294 | $probe['note'] = __( 'Includes a quarantine round-trip per item.', 'media-cleaner' ); |
| 295 | } |
| 296 | return $probe; |
| 297 | } |
| 298 | |
| 299 | // Flushing references is one multi-row INSERT. It is timed against a temporary copy of the |
| 300 | // real table — same columns, same unique key, so the same insert cost — which is private to |
| 301 | // this connection and cannot touch a scan's results. |
| 302 | private function probe_references( $items ) { |
| 303 | global $wpdb; |
| 304 | $table = $wpdb->prefix . 'mclean_refs'; |
| 305 | $temporary = $table . '_benchmark'; |
| 306 | $suppressed = $wpdb->suppress_errors( true ); |
| 307 | $created = $wpdb->query( "CREATE TEMPORARY TABLE $temporary LIKE $table" ); |
| 308 | if ( $created === false ) { |
| 309 | $wpdb->suppress_errors( $suppressed ); |
| 310 | return $this->unmeasured( __( 'The database user cannot create temporary tables.', 'media-cleaner' ) ); |
| 311 | } |
| 312 | |
| 313 | $values = array(); |
| 314 | $placeholders = array(); |
| 315 | for ( $i = 0; $i < $items; $i++ ) { |
| 316 | $url = sprintf( '%d/%02d/media-cleaner-benchmark-%d.jpg', 2000 + ( $i % 25 ), 1 + ( $i % 12 ), $i ); |
| 317 | array_push( $values, 0, $url, hash( 'sha256', $url ), 'BENCHMARK', md5( $url ) ); |
| 318 | $placeholders[] = "('%d', NULL, '%s', '%s', '%s', NULL, NULL, '%s')"; |
| 319 | } |
| 320 | $query = "INSERT IGNORE INTO $temporary (run_id, mediaId, mediaUrl, mediaUrl_hash, originType, origin, parentId, ref_hash) VALUES " |
| 321 | . implode( ', ', $placeholders ); |
| 322 | |
| 323 | $baseline = $this->memory_baseline(); |
| 324 | $started = microtime( true ); |
| 325 | $inserted = $wpdb->query( $wpdb->prepare( $query, $values ) ); |
| 326 | $elapsed = microtime( true ) - $started; |
| 327 | $wpdb->query( "DROP TEMPORARY TABLE IF EXISTS $temporary" ); |
| 328 | $wpdb->suppress_errors( $suppressed ); |
| 329 | if ( $inserted === false ) return $this->unmeasured( __( 'The reference flush could not be timed.', 'media-cleaner' ) ); |
| 330 | |
| 331 | return array( |
| 332 | 'samples' => $items, |
| 333 | 'seconds' => $elapsed, |
| 334 | 'bytes' => $this->memory_cost( $baseline ), |
| 335 | // One INSERT is one item here: it either fitted or it did not. |
| 336 | 'truncated' => false, |
| 337 | 'slowest' => null, |
| 338 | 'note' => '', |
| 339 | ); |
| 340 | } |
| 341 | |
| 342 | #endregion |
| 343 | |
| 344 | #region Plumbing |
| 345 | |
| 346 | // How much of each kind there is to sample, so the plan never offers a rung the install |
| 347 | // cannot fill. Everything here is a count or a single-entry probe. |
| 348 | private function availability() { |
| 349 | $engine = $this->core->engine; |
| 350 | $posts = $engine ? $engine->count_posts_to_check() : 0; |
| 351 | $medias = $engine ? $engine->count_media_entries() : 0; |
| 352 | $files = 0; |
| 353 | if ( $engine && $medias > 0 ) { |
| 354 | try { |
| 355 | $files = count( $engine->get_files( '', 0, 1 ) ) > 0 ? $medias : 0; |
| 356 | } |
| 357 | catch ( Throwable $e ) { |
| 358 | $files = 0; |
| 359 | } |
| 360 | } |
| 361 | return array( |
| 362 | 'content' => $posts, |
| 363 | 'media' => $medias, |
| 364 | 'analysis' => $medias, |
| 365 | 'files' => $files, |
| 366 | 'file_operation' => $medias, |
| 367 | // A temporary table can hold any number of rows; the ladder decides. |
| 368 | 'references' => PHP_INT_MAX, |
| 369 | ); |
| 370 | } |
| 371 | |
| 372 | private function ladder( $rungs, $available ) { |
| 373 | if ( $available < 1 ) return array(); |
| 374 | $usable = array(); |
| 375 | foreach ( $rungs as $rung ) { |
| 376 | if ( $rung <= $available ) $usable[] = $rung; |
| 377 | } |
| 378 | // Too small for even the first rung: measure what there is, once. |
| 379 | return empty( $usable ) ? array( (int) $available ) : $usable; |
| 380 | } |
| 381 | |
| 382 | // Sampled across the whole library, not one end of it. The heaviest content is usually not |
| 383 | // the newest: it is the page some builder produced years ago and nobody has opened since. |
| 384 | // Reading only recent posts reports a server faster than the one the scan will meet. |
| 385 | private function sample( $kind, $items ) { |
| 386 | $engine = $this->core->engine; |
| 387 | if ( !$engine ) return array(); |
| 388 | if ( $kind === 'posts' ) { |
| 389 | $total = $engine->count_posts_to_check(); |
| 390 | $posts = array(); |
| 391 | foreach ( $this->slices( $total, $items ) as $slice ) { |
| 392 | $posts = array_merge( $posts, $engine->get_posts_to_check( $slice[0], $slice[1] ) ); |
| 393 | } |
| 394 | return $posts; |
| 395 | } |
| 396 | $total = $engine->count_media_entries(); |
| 397 | $medias = array(); |
| 398 | foreach ( $this->slices( $total, $items ) as $slice ) { |
| 399 | $medias = array_merge( $medias, $engine->get_media_entries( $slice[0], $slice[1] ) ); |
| 400 | } |
| 401 | if ( $kind === 'medias' ) return $medias; |
| 402 | |
| 403 | // Files: the paths of those media, which are real files in the uploads directory. |
| 404 | $files = array(); |
| 405 | foreach ( $medias as $media_id ) { |
| 406 | foreach ( $this->core->get_paths_from_attachment( $media_id ) as $path ) { |
| 407 | if ( $path !== '' ) $files[] = $path; |
| 408 | } |
| 409 | if ( count( $files ) >= $items ) break; |
| 410 | } |
| 411 | return array_slice( array_unique( $files ), 0, $items ); |
| 412 | } |
| 413 | |
| 414 | // Splits a sample into a few evenly spread windows over the whole range: the oldest, the |
| 415 | // middle and the newest, rather than one block at one end. |
| 416 | private function slices( $total, $items, $chunks = 3 ) { |
| 417 | $items = (int) min( $items, $total ); |
| 418 | if ( $items < 1 ) return array(); |
| 419 | $chunks = max( 1, min( $chunks, $items ) ); |
| 420 | $per_chunk = (int) ceil( $items / $chunks ); |
| 421 | $slices = array(); |
| 422 | $taken = 0; |
| 423 | for ( $i = 0; $i < $chunks && $taken < $items; $i++ ) { |
| 424 | $size = min( $per_chunk, $items - $taken ); |
| 425 | $offset = $chunks === 1 ? 0 : (int) floor( $i * ( $total - $size ) / ( $chunks - 1 ) ); |
| 426 | $slices[] = array( max( 0, $offset ), $size ); |
| 427 | $taken += $size; |
| 428 | } |
| 429 | return $slices; |
| 430 | } |
| 431 | |
| 432 | private function run( $items, $work ) { |
| 433 | if ( empty( $items ) ) return $this->unmeasured(); |
| 434 | $total = count( $items ); |
| 435 | $baseline = $this->memory_baseline(); |
| 436 | $started = microtime( true ); |
| 437 | $done = 0; |
| 438 | $truncated = false; |
| 439 | // The slowest single item is worth keeping: when one of them is heavy enough to fill a |
| 440 | // request on its own, that is the thing to tell the user about, and no buffer can fix it. |
| 441 | $slowest = null; |
| 442 | foreach ( $items as $item ) { |
| 443 | $item_started = microtime( true ); |
| 444 | call_user_func( $work, $item ); |
| 445 | $now = microtime( true ); |
| 446 | $done++; |
| 447 | if ( $slowest === null || $now - $item_started > $slowest['seconds'] ) { |
| 448 | $slowest = array( |
| 449 | 'item' => is_scalar( $item ) ? $item : '', |
| 450 | 'seconds' => $now - $item_started, |
| 451 | ); |
| 452 | } |
| 453 | if ( $now > $this->deadline ) { |
| 454 | $truncated = $done < $total; |
| 455 | break; |
| 456 | } |
| 457 | } |
| 458 | return array( |
| 459 | 'samples' => $done, |
| 460 | 'seconds' => microtime( true ) - $started, |
| 461 | 'bytes' => $this->memory_cost( $baseline ), |
| 462 | 'truncated' => $truncated, |
| 463 | 'slowest' => $slowest, |
| 464 | 'note' => '', |
| 465 | ); |
| 466 | } |
| 467 | |
| 468 | private function unmeasured( $note = null ) { |
| 469 | if ( $note === null ) $note = __( 'Nothing to sample here.', 'media-cleaner' ); |
| 470 | // Kept in the same shape as a real round so callers never special-case an empty probe. |
| 471 | return array( 'samples' => 0, 'seconds' => 0, 'bytes' => 0, |
| 472 | 'truncated' => false, 'slowest' => null, 'note' => $note ); |
| 473 | } |
| 474 | |
| 475 | // Peak memory only ever grows during a request, so a probe can measure its own peak only |
| 476 | // where PHP lets it be reset (8.2+). Below that no memory figure is produced and the |
| 477 | // buffers are sized on time alone: an invented number would be worse than none. |
| 478 | private function memory_baseline() { |
| 479 | if ( !function_exists( 'memory_reset_peak_usage' ) ) return null; |
| 480 | memory_reset_peak_usage(); |
| 481 | return memory_get_usage( true ); |
| 482 | } |
| 483 | |
| 484 | private function memory_cost( $baseline ) { |
| 485 | return $baseline === null ? 0 : max( 0, memory_get_peak_usage( true ) - $baseline ); |
| 486 | } |
| 487 | |
| 488 | #endregion |
| 489 | |
| 490 | #region Sizing |
| 491 | |
| 492 | private function sanitize_rounds( $rounds ) { |
| 493 | if ( !is_array( $rounds ) ) return array(); |
| 494 | $clean = array(); |
| 495 | foreach ( $this->probes() as $name => $probe ) { |
| 496 | if ( empty( $rounds[ $name ] ) || !is_array( $rounds[ $name ] ) ) continue; |
| 497 | foreach ( $rounds[ $name ] as $round ) { |
| 498 | if ( !is_array( $round ) ) continue; |
| 499 | $samples = isset( $round['samples'] ) ? (int) $round['samples'] : 0; |
| 500 | $seconds = isset( $round['seconds'] ) ? (float) $round['seconds'] : 0; |
| 501 | if ( $samples < 1 || $seconds <= 0 ) continue; |
| 502 | $slowest = null; |
| 503 | if ( isset( $round['slowest'] ) && is_array( $round['slowest'] ) ) { |
| 504 | $slowest = array( |
| 505 | 'item' => isset( $round['slowest']['item'] ) && is_scalar( $round['slowest']['item'] ) |
| 506 | ? sanitize_text_field( (string) $round['slowest']['item'] ) : '', |
| 507 | 'seconds' => isset( $round['slowest']['seconds'] ) ? max( 0, (float) $round['slowest']['seconds'] ) : 0, |
| 508 | ); |
| 509 | } |
| 510 | $clean[ $name ][] = array( |
| 511 | 'samples' => $samples, |
| 512 | 'seconds' => $seconds, |
| 513 | 'bytes' => isset( $round['bytes'] ) ? max( 0, (float) $round['bytes'] ) : 0, |
| 514 | 'roundtrip' => isset( $round['roundtrip'] ) ? max( 0, (float) $round['roundtrip'] ) : 0, |
| 515 | 'truncated' => !empty( $round['truncated'] ), |
| 516 | 'slowest' => $slowest, |
| 517 | ); |
| 518 | } |
| 519 | } |
| 520 | return $clean; |
| 521 | } |
| 522 | |
| 523 | /** |
| 524 | * Separates the fixed cost of a batch from the cost of one item, by least squares over the |
| 525 | * rungs. With a single rung there is nothing to separate, so all of it is charged per item — |
| 526 | * which is the pessimistic reading, and the safe one. |
| 527 | * |
| 528 | * Returns null when there is no usable evidence, meaning "leave this setting alone". |
| 529 | */ |
| 530 | private function fit( $rounds ) { |
| 531 | if ( empty( $rounds ) ) return null; |
| 532 | // A rung the server had to cut short measures the moment it ran out of time, not the |
| 533 | // cost of the work: a single pathological item inside it would otherwise drag every |
| 534 | // batch down to one item. It is reported as a warning instead — unless it is all the |
| 535 | // evidence there is, in which case this really is what the phase costs here. |
| 536 | $complete = array_values( array_filter( $rounds, function( $round ) { |
| 537 | return empty( $round['truncated'] ); |
| 538 | } ) ); |
| 539 | $rounds = empty( $complete ) ? array_values( $rounds ) : $complete; |
| 540 | $n = count( $rounds ); |
| 541 | $bytes = 0; |
| 542 | foreach ( $rounds as $round ) { |
| 543 | $bytes = max( $bytes, $round['bytes'] / $round['samples'] ); |
| 544 | } |
| 545 | if ( $n === 1 ) { |
| 546 | $round = $rounds[0]; |
| 547 | return array( |
| 548 | 'per_item' => $round['seconds'] / $round['samples'], |
| 549 | 'fixed' => 0.0, |
| 550 | 'bytes_per_item' => $bytes, |
| 551 | 'rungs' => $n, |
| 552 | ); |
| 553 | } |
| 554 | |
| 555 | $sum_x = $sum_y = $sum_xy = $sum_xx = 0; |
| 556 | foreach ( $rounds as $round ) { |
| 557 | $sum_x += $round['samples']; |
| 558 | $sum_y += $round['seconds']; |
| 559 | $sum_xy += $round['samples'] * $round['seconds']; |
| 560 | $sum_xx += $round['samples'] * $round['samples']; |
| 561 | } |
| 562 | $divisor = ( $n * $sum_xx ) - ( $sum_x * $sum_x ); |
| 563 | $slope = $divisor == 0 ? 0 : ( ( $n * $sum_xy ) - ( $sum_x * $sum_y ) ) / $divisor; |
| 564 | $intercept = $divisor == 0 ? 0 : ( $sum_y - ( $slope * $sum_x ) ) / $n; |
| 565 | |
| 566 | // A flat or falling line means the noise was larger than the signal — the items are too |
| 567 | // cheap to separate at these sizes. Fall back to the plain average, fixed cost included. |
| 568 | if ( $slope <= 0 ) { |
| 569 | return array( |
| 570 | 'per_item' => $sum_y / $sum_x, |
| 571 | 'fixed' => 0.0, |
| 572 | 'bytes_per_item' => $bytes, |
| 573 | 'rungs' => $n, |
| 574 | ); |
| 575 | } |
| 576 | return array( |
| 577 | 'per_item' => $slope, |
| 578 | 'fixed' => max( 0.0, $intercept ), |
| 579 | 'bytes_per_item' => $bytes, |
| 580 | 'rungs' => $n, |
| 581 | ); |
| 582 | } |
| 583 | |
| 584 | // How many items fit in one request, by time and by memory — and for references, by the |
| 585 | // packet MySQL accepts, since one INSERT carries every row at once and an oversized packet |
| 586 | // is a failed scan rather than a slow one. |
| 587 | private function size( $name, $fit, $budget, $environment ) { |
| 588 | if ( empty( $fit['per_item'] ) ) return null; |
| 589 | $for_items = $budget - $fit['fixed']; |
| 590 | // The fixed cost alone already fills the budget: one item at a time is all this server |
| 591 | // can promise. The engine will still yield if even that does not fit. |
| 592 | if ( $for_items <= 0 ) return 1; |
| 593 | |
| 594 | $size = max( 1, (int) floor( $for_items / $fit['per_item'] ) ); |
| 595 | if ( $environment['memory_limit'] > 0 && $fit['bytes_per_item'] > 0 ) { |
| 596 | $by_memory = (int) floor( ( $environment['memory_limit'] * self::MEMORY_TARGET ) / $fit['bytes_per_item'] ); |
| 597 | $size = max( 1, min( $size, $by_memory ) ); |
| 598 | } |
| 599 | if ( $name === 'references' && $environment['max_allowed_packet'] > 0 ) { |
| 600 | $by_packet = (int) floor( ( $environment['max_allowed_packet'] * self::MEMORY_TARGET ) / self::REF_ROW_BYTES ); |
| 601 | $size = max( 1, min( $size, $by_packet ) ); |
| 602 | } |
| 603 | return $size; |
| 604 | } |
| 605 | |
| 606 | private function clamp( $option, $value ) { |
| 607 | $ranges = $this->core->option_ranges(); |
| 608 | if ( !isset( $ranges[ $option ] ) ) return (int) $value; |
| 609 | return (int) max( $ranges[ $option ][0], min( $ranges[ $option ][1], (int) $value ) ); |
| 610 | } |
| 611 | |
| 612 | /** |
| 613 | * The things a buffer cannot fix. |
| 614 | * |
| 615 | * When a rung has to be cut short, the request ran out of time in the middle of it. If the |
| 616 | * slowest item in that rung fills a request on its own, no batch size will help: the scan |
| 617 | * will stop on that item every time it reaches it, whatever the buffers say. That is worth |
| 618 | * naming, with the item, because the fix is in the content or in the request budget. |
| 619 | */ |
| 620 | private function warnings( $rounds, $environment ) { |
| 621 | $warnings = array(); |
| 622 | foreach ( $this->probes() as $name => $probe ) { |
| 623 | if ( empty( $rounds[ $name ] ) ) continue; |
| 624 | foreach ( $rounds[ $name ] as $round ) { |
| 625 | if ( empty( $round['truncated'] ) ) continue; |
| 626 | $slowest = $round['slowest']; |
| 627 | $alone = $slowest && $slowest['seconds'] >= $environment['request_budget'] * self::BUDGET_TARGET; |
| 628 | $warnings[] = array( |
| 629 | 'probe' => $name, |
| 630 | 'label' => $probe['label'], |
| 631 | 'critical' => $alone, |
| 632 | 'message' => $alone |
| 633 | ? sprintf( |
| 634 | __( '%1$s: one item (#%2$s) took %3$s on its own, more than a single request can spend. A scan will stop there whatever the buffer is. That item needs looking at, or the request budget needs raising.', 'media-cleaner' ), |
| 635 | $probe['label'], $slowest['item'], $this->readable_seconds( $slowest['seconds'] ) ) |
| 636 | : sprintf( |
| 637 | __( '%1$s: %2$d items took %3$s, so the request ran out of time before that round finished. The buffer was sized from the rounds that did finish.', 'media-cleaner' ), |
| 638 | $probe['label'], $round['samples'], $this->readable_seconds( $round['seconds'] ) ), |
| 639 | ); |
| 640 | break; |
| 641 | } |
| 642 | } |
| 643 | return $warnings; |
| 644 | } |
| 645 | |
| 646 | private function readable_seconds( $seconds ) { |
| 647 | return $seconds >= 1 |
| 648 | ? sprintf( __( '%.1f s', 'media-cleaner' ), $seconds ) |
| 649 | : sprintf( __( '%d ms', 'media-cleaner' ), (int) round( $seconds * 1000 ) ); |
| 650 | } |
| 651 | |
| 652 | // What a request costs before any work happens: the round trip the browser saw, minus the |
| 653 | // time the server spent working. Reported for the user, not used for sizing — it argues for |
| 654 | // fewer, bigger batches, and the buffer ranges already decide how big those may get. |
| 655 | private function request_overhead( $rounds ) { |
| 656 | $overheads = array(); |
| 657 | foreach ( $rounds as $probe_rounds ) { |
| 658 | foreach ( $probe_rounds as $round ) { |
| 659 | if ( $round['roundtrip'] > 0 ) $overheads[] = max( 0, $round['roundtrip'] - $round['seconds'] ); |
| 660 | } |
| 661 | } |
| 662 | if ( empty( $overheads ) ) return null; |
| 663 | sort( $overheads ); |
| 664 | return $overheads[ (int) floor( count( $overheads ) / 2 ) ]; |
| 665 | } |
| 666 | |
| 667 | // The longest request that actually came back. It does not prove where the proxy cuts off, |
| 668 | // but it is a floor on it, and a useful thing to see next to the assumed budget. |
| 669 | private function longest_request( $rounds ) { |
| 670 | $longest = 0; |
| 671 | foreach ( $rounds as $probe_rounds ) { |
| 672 | foreach ( $probe_rounds as $round ) { |
| 673 | $longest = max( $longest, $round['roundtrip'], $round['seconds'] ); |
| 674 | } |
| 675 | } |
| 676 | return $longest > 0 ? $longest : null; |
| 677 | } |
| 678 | |
| 679 | #endregion |
| 680 | } |
| 681 |