← All changes
|
includes/seo/class-email-report-data-provider.php
+192
-10
2.9.0
→
2.14.1
View file →
| @@ -38,8 +38,32 @@ | ||
| 38 | 38 | */ |
| 39 | 39 | final class Email_Report_Data_Provider { |
| 40 | 40 | |
| 41 | 41 | /** |
| 42 | + * Rows fetched per window for the query and page comparisons. | |
| 43 | + * | |
| 44 | + * Search Console orders rows by clicks and stops here, so a list that | |
| 45 | + * comes back this long may be truncated: a row missing from it may sit | |
| 46 | + * just below the cut rather than have no traffic (#906). | |
| 47 | + */ | |
| 48 | + private const COMPARISON_ROW_LIMIT = 1000; | |
| 49 | + | |
| 50 | + /** | |
| 51 | + * How many apparent drop-outs are confirmed against the current window. | |
| 52 | + * | |
| 53 | + * The losing cards show five rows, so the biggest previous-period | |
| 54 | + * candidates are all that can reach them. Anything past this cap is left | |
| 55 | + * out of the comparison rather than presumed lost. | |
| 56 | + */ | |
| 57 | + private const DROPOUT_CONFIRM_LIMIT = 25; | |
| 58 | + | |
| 59 | + /** | |
| 60 | + * Longest regular expression sent in one confirmation request. Search | |
| 61 | + * Console rejects longer expressions, so candidates are batched under it. | |
| 62 | + */ | |
| 63 | + private const DROPOUT_REGEX_MAX_LENGTH = 3500; | |
| 64 | + | |
| 65 | + /** | |
| 42 | 66 | * Which data sources the report can draw on right now. |
| 43 | 67 | * |
| 44 | 68 | * Cheap on purpose — no dashboard fetch, no Search Console query. It |
| 45 | 69 | * runs on every hourly tick while a site is paused and on every load of |
| @@ -273,13 +297,20 @@ | ||
| 273 | 297 | // the current window opens. |
| 274 | 298 | $prev_end = gmdate('Y-m-d', strtotime('-1 day', strtotime($cur_start))); |
| 275 | 299 | $prev_start = gmdate('Y-m-d', strtotime('-' . ($frequency_days - 1) . ' days', strtotime($prev_end))); |
| 276 | 300 | |
| 277 | - $cur_q = $sc->get_search_performance_by_dates($site_url, $cur_start, $cur_end, 1000, ['query']); | |
| 278 | - $prev_q = $sc->get_search_performance_by_dates($site_url, $prev_start, $prev_end, 1000, ['query']); | |
| 279 | - $cur_p = $sc->get_search_performance_by_dates($site_url, $cur_start, $cur_end, 1000, ['page']); | |
| 280 | - $prev_p = $sc->get_search_performance_by_dates($site_url, $prev_start, $prev_end, 1000, ['page']); | |
| 301 | + $limit = self::COMPARISON_ROW_LIMIT; | |
| 302 | + $cur_q = $sc->get_search_performance_by_dates($site_url, $cur_start, $cur_end, $limit, ['query']); | |
| 303 | + $prev_q = $sc->get_search_performance_by_dates($site_url, $prev_start, $prev_end, $limit, ['query']); | |
| 304 | + $cur_p = $sc->get_search_performance_by_dates($site_url, $cur_start, $cur_end, $limit, ['page']); | |
| 305 | + $prev_p = $sc->get_search_performance_by_dates($site_url, $prev_start, $prev_end, $limit, ['page']); | |
| 281 | 306 | |
| 307 | + // A row missing from a full current list may only have fallen below | |
| 308 | + // the cut. Ask Search Console about the likely losers directly before | |
| 309 | + // any of them is reported as having lost everything (#906). | |
| 310 | + [$found_q, $dropouts_q] = $this->confirm_dropouts($sc, $site_url, $cur_start, $cur_end, $cur_q, $prev_q, 'query'); | |
| 311 | + [$found_p, $dropouts_p] = $this->confirm_dropouts($sc, $site_url, $cur_start, $cur_end, $cur_p, $prev_p, 'page'); | |
| 312 | + | |
| 282 | 313 | // Whole-property totals for both windows. A query with no |
| 283 | 314 | // dimensions returns one aggregated row, so the hero's clicks, |
| 284 | 315 | // impressions, CTR and position — and their change — are exact |
| 285 | 316 | // rather than summed from the 1,000-row query lists above. |
| @@ -287,10 +318,10 @@ | ||
| 287 | 318 | $prev_t = $sc->get_search_performance_by_dates($site_url, $prev_start, $prev_end, 1, []); |
| 288 | 319 | |
| 289 | 320 | return [ |
| 290 | 321 | 'available' => true, |
| 291 | - 'queries' => $this->merge_periods($cur_q, $prev_q, true), | |
| 292 | - 'pages' => $this->merge_periods($cur_p, $prev_p, false), | |
| 322 | + 'queries' => $this->merge_periods(array_merge($cur_q, $found_q), $prev_q, true, $dropouts_q), | |
| 323 | + 'pages' => $this->merge_periods(array_merge($cur_p, $found_p), $prev_p, false, $dropouts_p), | |
| 293 | 324 | 'totals' => [ |
| 294 | 325 | 'current' => $this->totals_row($cur_t), |
| 295 | 326 | 'previous' => $this->totals_row($prev_t), |
| 296 | 327 | ], |
| @@ -297,8 +328,149 @@ | ||
| 297 | 328 | ]; |
| 298 | 329 | } |
| 299 | 330 | |
| 300 | 331 | /** |
| 332 | + * Sort out which previous-window rows missing from the current list | |
| 333 | + * really lost all their traffic. | |
| 334 | + * | |
| 335 | + * Both windows are capped lists ordered by clicks. Absence from a | |
| 336 | + * current list shorter than the cap is real: Search Console returned | |
| 337 | + * everything it has. Absence from a full list is not evidence of | |
| 338 | + * anything, because growth elsewhere pushes unchanged rows below the | |
| 339 | + * cut. Those were synthesised as total losses and led the losing cards, | |
| 340 | + * labelled "No impressions this period", with the same clicks in both | |
| 341 | + * windows (#906). | |
| 342 | + * | |
| 343 | + * So when the current list is full, the biggest candidates by previous | |
| 344 | + * clicks are looked up in the current window by exact key. A candidate | |
| 345 | + * that comes back has its real current row returned, so its change is | |
| 346 | + * measured rather than presumed. Only one that comes back empty is a | |
| 347 | + * drop-out. Candidates past DROPOUT_CONFIRM_LIMIT, or in a batch whose | |
| 348 | + * request failed, are neither: they are left out of the comparison, | |
| 349 | + * since saying nothing is better than reporting a loss nobody measured. | |
| 350 | + * | |
| 351 | + * @since 2.15.0 | |
| 352 | + * | |
| 353 | + * @param object $sc Search Console client. | |
| 354 | + * @param string $site_url Property URL. | |
| 355 | + * @param string $cur_start Current window start (Y-m-d). | |
| 356 | + * @param string $cur_end Current window end (Y-m-d). | |
| 357 | + * @param array $current Current-window rows, as fetched. | |
| 358 | + * @param array $previous Previous-window rows, as fetched. | |
| 359 | + * @param string $dimension 'query' or 'page'. | |
| 360 | + * @return array{0: array, 1: ?array<string,bool>} Current rows found | |
| 361 | + * below the cut, and the confirmed drop-out keys, or null when | |
| 362 | + * the current list is complete and every absence is real. | |
| 363 | + */ | |
| 364 | + private function confirm_dropouts($sc, string $site_url, string $cur_start, string $cur_end, array $current, array $previous, string $dimension): array { | |
| 365 | + if (count($current) < self::COMPARISON_ROW_LIMIT) { | |
| 366 | + return [[], null]; | |
| 367 | + } | |
| 368 | + | |
| 369 | + $is_query = 'query' === $dimension; | |
| 370 | + | |
| 371 | + $present = []; | |
| 372 | + foreach ($current as $row) { | |
| 373 | + $raw = (string) ($row['keys'][0] ?? ''); | |
| 374 | + if ('' !== $raw) { | |
| 375 | + $present[$this->normalize_key($raw, $is_query)] = true; | |
| 376 | + } | |
| 377 | + } | |
| 378 | + | |
| 379 | + $candidates = []; | |
| 380 | + foreach ($previous as $row) { | |
| 381 | + $raw = (string) ($row['keys'][0] ?? ''); | |
| 382 | + if ('' === $raw) { | |
| 383 | + continue; | |
| 384 | + } | |
| 385 | + $key = $this->normalize_key($raw, $is_query); | |
| 386 | + if (!isset($present[$key]) && !isset($candidates[$key])) { | |
| 387 | + $candidates[$key] = ['raw' => $raw, 'clicks' => (int) ($row['clicks'] ?? 0)]; | |
| 388 | + } | |
| 389 | + } | |
| 390 | + | |
| 391 | + if ([] === $candidates) { | |
| 392 | + return [[], []]; | |
| 393 | + } | |
| 394 | + | |
| 395 | + uasort($candidates, static fn($a, $b) => $b['clicks'] <=> $a['clicks']); | |
| 396 | + $candidates = array_slice($candidates, 0, self::DROPOUT_CONFIRM_LIMIT, true); | |
| 397 | + | |
| 398 | + // Batch the exact-match lookups into as few requests as the | |
| 399 | + // expression length allows: one per dimension in practice. | |
| 400 | + $batches = []; | |
| 401 | + $batch = []; | |
| 402 | + $length = 0; | |
| 403 | + foreach ($candidates as $key => $candidate) { | |
| 404 | + $part = $this->re2_quote($candidate['raw']); | |
| 405 | + if ([] !== $batch && $length + strlen($part) + 1 > self::DROPOUT_REGEX_MAX_LENGTH) { | |
| 406 | + $batches[] = $batch; | |
| 407 | + $batch = []; | |
| 408 | + $length = 0; | |
| 409 | + } | |
| 410 | + $batch[$key] = $part; | |
| 411 | + $length += strlen($part) + 1; | |
| 412 | + } | |
| 413 | + $batches[] = $batch; | |
| 414 | + | |
| 415 | + $found = []; | |
| 416 | + $dropouts = []; | |
| 417 | + foreach ($batches as $batch) { | |
| 418 | + try { | |
| 419 | + $rows = $sc->get_search_performance_by_dates( | |
| 420 | + $site_url, | |
| 421 | + $cur_start, | |
| 422 | + $cur_end, | |
| 423 | + count($batch), | |
| 424 | + [$dimension], | |
| 425 | + [[ | |
| 426 | + 'dimension' => $dimension, | |
| 427 | + 'operator' => 'includingRegex', | |
| 428 | + 'expression' => '^(?:' . implode('|', $batch) . ')$', | |
| 429 | + ]] | |
| 430 | + ); | |
| 431 | + } catch (Throwable $e) { | |
| 432 | + // Unconfirmed: leave these out rather than call them lost. | |
| 433 | + continue; | |
| 434 | + } | |
| 435 | + | |
| 436 | + $seen = []; | |
| 437 | + foreach ((array) $rows as $row) { | |
| 438 | + $raw = (string) ($row['keys'][0] ?? ''); | |
| 439 | + $key = '' === $raw ? '' : $this->normalize_key($raw, $is_query); | |
| 440 | + if (isset($batch[$key])) { | |
| 441 | + $seen[$key] = true; | |
| 442 | + $found[] = $row; | |
| 443 | + } | |
| 444 | + } | |
| 445 | + | |
| 446 | + foreach (array_keys($batch) as $key) { | |
| 447 | + if (!isset($seen[$key])) { | |
| 448 | + $dropouts[$key] = true; | |
| 449 | + } | |
| 450 | + } | |
| 451 | + } | |
| 452 | + | |
| 453 | + return [$found, $dropouts]; | |
| 454 | + } | |
| 455 | + | |
| 456 | + /** | |
| 457 | + * Escape a literal for a Search Console (RE2) regular expression. | |
| 458 | + * | |
| 459 | + * RE2 accepts a backslash before any ASCII punctuation as that literal, | |
| 460 | + * so every punctuation character is escaped, not only the ones that are | |
| 461 | + * special today. | |
| 462 | + * | |
| 463 | + * @since 2.15.0 | |
| 464 | + * | |
| 465 | + * @param string $literal Text to match exactly. | |
| 466 | + * @return string | |
| 467 | + */ | |
| 468 | + private function re2_quote(string $literal): string { | |
| 469 | + return (string) preg_replace('/[[:punct:]]/', '\\\\$0', $literal); | |
| 470 | + } | |
| 471 | + | |
| 472 | + /** | |
| 301 | 473 | * Normalise the single aggregate row Search Console returns for a |
| 302 | 474 | * dimensionless query. An empty result (a property with no traffic in |
| 303 | 475 | * the window) yields zeroes, which the hero treats as "no comparison". |
| 304 | 476 | * |
| @@ -380,14 +552,21 @@ | ||
| 380 | 552 | * clicks has no current row at all — keying off `$current` alone would |
| 381 | 553 | * silently discard exactly the biggest losers the losing sections exist |
| 382 | 554 | * to surface. |
| 383 | 555 | * |
| 384 | - * @param array $current Current-window rows. | |
| 385 | - * @param array $previous Previous-window rows. | |
| 386 | - * @param bool $is_query True for query rows, false for page rows. | |
| 556 | + * Which absent rows count as drop-outs is up to the caller. Absence from | |
| 557 | + * a truncated list proves nothing, so build_comparison() passes the keys | |
| 558 | + * it confirmed with Search Console; any other absent row is left out | |
| 559 | + * (#906). Null means the current list was complete and every absence is | |
| 560 | + * a drop-out. | |
| 561 | + * | |
| 562 | + * @param array $current Current-window rows. | |
| 563 | + * @param array $previous Previous-window rows. | |
| 564 | + * @param bool $is_query True for query rows, false for page rows. | |
| 565 | + * @param array<string,bool>|null $dropout_keys Confirmed drop-out keys, or null for all. | |
| 387 | 566 | * @return array<string,array> |
| 388 | 567 | */ |
| 389 | - private function merge_periods(array $current, array $previous, bool $is_query): array { | |
| 568 | + private function merge_periods(array $current, array $previous, bool $is_query, ?array $dropout_keys = null): array { | |
| 390 | 569 | $prev_map = []; |
| 391 | 570 | foreach ($previous as $row) { |
| 392 | 571 | $key = (string) ($row['keys'][0] ?? ''); |
| 393 | 572 | if ($key === '') { |
| @@ -423,8 +602,11 @@ | ||
| 423 | 602 | // from the previous window with the current metrics zeroed. Position |
| 424 | 603 | // stays null rather than 0 — "no data" is not "ranked first". |
| 425 | 604 | foreach ($prev_map as $key => $prev_row) { |
| 426 | 605 | if (isset($merged[$key])) { |
| 606 | + continue; | |
| 607 | + } | |
| 608 | + if (null !== $dropout_keys && !isset($dropout_keys[$key])) { | |
| 427 | 609 | continue; |
| 428 | 610 | } |
| 429 | 611 | $raw = (string) ($prev_row['keys'][0] ?? ''); |
| 430 | 612 | if ($raw === '') { |