PluginProbe
ThinkRank AI SEO – AI SEO Plugin for WordPress: Schema, XML Sitemaps, Meta Tags, Search Console & Local SEO / 2.14.2
ThinkRank AI SEO – AI SEO Plugin for WordPress: Schema, XML Sitemaps, Meta Tags, Search Console & Local SEO v2.14.2
2.14.2 2.14.1 2.14.0 2.13.0 2.12.0 2.11.0 2.10.0 2.9.0 2.8.0 2.7.0 2.6.0 2.5.0 2.4.0 2.3.0 2.2.0 2.1.1 2.1.0 2.0.2 2.0.1 2.0.0 1.32.0 1.31.0 1.30.0 1.29.0 1.28.0 All 57 releases
← All changes | includes/seo/class-sitemap-generator.php +1521 -154 2.4.0 → 2.14.2 View file →
@@ -62,8 +62,25 @@
62 62 'include_tags' => 'tags',
63 63 ];
64 64
65 65 /**
66 + * Child sitemap type -> the object that actually supplies its entries.
67 + *
68 + * A child normally carries its object's own slug, but the sitemap presets UI
69 + * writes a display name for the WooCommerce taxonomy child
70 + * ('product_categories', not 'product_cat'), so a saved child list can name a
71 + * type no post type or taxonomy answers to. Resolving through here lets those
72 + * children take the same generic path as every other custom taxonomy instead
73 + * of needing a case of their own (#690).
74 + *
75 + * @since 2.7.0
76 + * @var array<string, string>
77 + */
78 + private const CHILD_TYPE_ALIASES = [
79 + 'product_categories' => 'product_cat',
80 + ];
81 +
82 + /**
66 83 * Supported sitemap types
67 84 *
68 85 * @since 1.0.0
69 86 * @var array
@@ -127,8 +144,98 @@
127 144 */
128 145 public const REGENERATION_ERROR_OPTION = 'thinkrank_sitemap_regeneration_error';
129 146
130 147 /**
148 + * Delivery modes accepted by the `delivery_mode` setting.
149 + *
150 + * Mirrors LLMs_Txt_Manager::DELIVERY_MODES, which solved the same problem
151 + * for llms.txt. `auto` is the only value a site should normally need.
152 + *
153 + * @since 2.9.0
154 + * @var string[]
155 + */
156 + public const DELIVERY_MODES = ['auto', 'static', 'dynamic'];
157 +
158 + /**
159 + * Cache group for documents rendered on the dynamic path.
160 + *
161 + * @since 2.9.0
162 + * @var string
163 + */
164 + private const DYNAMIC_CACHE_PREFIX = 'thinkrank_sitemap_doc_';
165 +
166 + /**
167 + * How long a dynamically rendered document is cached.
168 + *
169 + * Invalidated by content and settings changes through
170 + * {@see self::flush_dynamic_cache()}, so this is only the backstop for a
171 + * change nothing hooked.
172 + *
173 + * @since 2.9.0
174 + * @var int
175 + */
176 + private const DYNAMIC_CACHE_TTL = 12 * HOUR_IN_SECONDS;
177 +
178 + /**
179 + * How long the render lock is held before it is assumed abandoned.
180 + *
181 + * Long enough for a large site's full build, short enough that a request
182 + * killed mid-build does not lock the endpoint out for meaningfully long.
183 + *
184 + * @since 2.9.0
185 + * @var int
186 + */
187 + private const RENDER_LOCK_TTL = 60;
188 +
189 + /**
190 + * Cached stand-in for "this site does not publish that name".
191 + *
192 + * published_document_names() lists what the configuration *could* produce,
193 + * but a child whose type is excluded produces nothing. Without a negative
194 + * entry those names miss the cache forever, so every request for one
195 + * rebuilt the entire sitemap — the same cost the positive cache exists to
196 + * avoid, on a public endpoint (#754 review).
197 + *
198 + * @since 2.9.0
199 + * @var string
200 + */
201 + private const ABSENT_MARKER = "\0thinkrank-absent";
202 +
203 + /**
204 + * How many times, and how long, a losing request waits for the winner.
205 + *
206 + * Bounded at roughly a second in total: past that, building a second copy
207 + * costs less than making a crawler wait.
208 + *
209 + * @since 2.9.0
210 + * @var int
211 + */
212 + private const RENDER_LOCK_WAIT_ATTEMPTS = 4;
213 +
214 + /**
215 + * @since 2.9.0
216 + * @var int
217 + */
218 + private const RENDER_LOCK_WAIT_MICROSECONDS = 250000;
219 +
220 + /**
221 + * Where generated documents go instead of disk, when set.
222 + *
223 + * Every sitemap document this class produces — segments, the index, the
224 + * single flat file and local-sitemap.xml — is published through the one
225 + * writer, {@see self::save_sitemap_to_file()}. Swapping that writer for a
226 + * collector is therefore all it takes to render the same bytes without a
227 + * filesystem, which is what dynamic delivery needs (#752). Doing it here
228 + * rather than duplicating the build pipeline is deliberate: a second
229 + * pipeline would drift from this one, and the index in particular is
230 + * assembled from whatever the children actually produced.
231 + *
232 + * @since 2.9.0
233 + * @var callable|null
234 + */
235 + private $document_sink = null;
236 +
237 + /**
131 238 * Transient guarding against two generations running at once. Shared with
132 239 * Sitemap_Endpoint's manual generate route so an automatic rebuild and a
133 240 * manual one cannot write the same files concurrently.
134 241 *
@@ -144,8 +251,33 @@
144 251 * @var int
145 252 */
146 253 private const TERM_WALK_CHUNK = 1000;
147 254
255 + /**
256 + * Exception code for an automatic rebuild stopped short of the memory limit.
257 + *
258 + * @since 2.10.1
259 + * @var int
260 + */
261 + private const MEMORY_ABORT_CODE = 4290;
262 +
263 + /**
264 + * Whether the chunked walks should stop before the memory limit. Only on
265 + * for automatic rebuilds, whose failure is recorded and retried.
266 + *
267 + * @since 2.10.1
268 + * @var bool
269 + */
270 + private bool $memory_guard = false;
271 +
272 + /**
273 + * Largest memory cost of one walked chunk in this rebuild, in bytes.
274 + *
275 + * @since 2.10.1
276 + * @var int
277 + */
278 + private int $walk_chunk_cost = 0;
279 +
148 280 private array $sitemap_types = [
149 281 'posts' => [
150 282 'name' => 'Posts',
151 283 'post_types' => ['post'],
@@ -172,25 +304,29 @@
172 304 ]
173 305 ];
174 306
175 307 /**
308 + * The generator the content-change listeners share, built on the first
309 + * change of a request.
310 + *
311 + * @since 2.10.1
312 + * @var self|null
313 + */
314 + private static ?self $listener = null;
315 +
316 + /**
176 317 * Constructor
177 318 *
178 319 * @since 1.0.0
320 + * @since 2.10.1 Registers no hooks, whatever `$register_hooks` says. The
321 + * content-change listeners are registered once, at bootstrap,
322 + * by register_content_listeners().
179 323 *
180 - * @param bool $register_hooks Optional. Whether to register the auto-generation
181 - * hooks. Pass false for a read-only instance built
182 - * solely to query settings — the hooks are bound to
183 - * `$this`, so a second hook-registering instance
184 - * would run `handle_content_change()` twice per save.
324 + * @param bool $register_hooks Unused since 2.10.1; kept so existing callers,
325 + * Pro's included, keep working.
185 326 */
186 327 public function __construct(bool $register_hooks = true) {
187 328 parent::__construct('sitemap');
188 -
189 - // Initialize auto-generation hooks
190 - if ($register_hooks) {
191 - $this->init_auto_generation_hooks();
192 - }
193 329 }
194 330
195 331 /**
196 332 * Filter the args of a sitemap post query.
@@ -234,34 +370,154 @@
234 370 return (array) apply_filters('thinkrank_sitemap_term_query_args', $args);
235 371 }
236 372
237 373 /**
238 - * Initialize WordPress hooks for auto-generation
374 + * Register the content-change listeners that queue an automatic rebuild.
239 375 *
240 - * @since 1.0.0
376 + * Called once per request, at plugin bootstrap, next to the WP-Cron
377 + * listeners that run the rebuild these queue (both in
378 + * Plugin::register_sitemap_cron_listeners(), on plugins_loaded). The
379 + * constructor used to register them, so they existed only in a request
380 + * that happened to build a generator: the REST endpoint, the setup wizard,
381 + * an MCP ability. Block-editor saves go through REST and were heard. A
382 + * scheduled post published by WP-Cron, Quick Edit, the classic editor and
383 + * WP-CLI were not, so the post stayed out of the sitemap with nothing
384 + * pending to recover it (#824). Each instance also added its own set, so
385 + * one REST save ran the handlers once per generator built.
386 + *
387 + * The generator is built on the first change and shared for the rest of
388 + * the request, so a bulk edit does not construct one per post.
389 + *
390 + * @since 2.10.1
241 391 * @return void
242 392 */
243 - private function init_auto_generation_hooks(): void {
244 - // Content change hooks - use priority 20 to run after other plugins
245 - add_action('save_post', [$this, 'handle_content_change'], 20, 2);
246 - add_action('delete_post', [$this, 'handle_content_deletion'], 20);
247 - add_action('wp_trash_post', [$this, 'handle_content_deletion'], 20);
248 - add_action('untrash_post', [$this, 'handle_content_change_by_id'], 20);
393 + public static function register_content_listeners(): void {
394 + // Priority 20, to run after other plugins.
395 + add_action('save_post', static function (int $post_id, \WP_Post $post): void {
396 + self::listener()->handle_content_change($post_id, $post);
397 + }, 20, 2);
398 + add_action('delete_post', static function (int $post_id): void {
399 + self::listener()->handle_content_deletion($post_id);
400 + }, 20);
401 + add_action('wp_trash_post', static function (int $post_id): void {
402 + self::listener()->handle_content_deletion($post_id);
403 + }, 20);
404 + add_action('untrash_post', static function (int $post_id): void {
405 + self::listener()->handle_content_change_by_id($post_id);
406 + }, 20);
249 407
250 - // Taxonomy change hooks
251 - add_action('created_term', [$this, 'handle_taxonomy_change'], 20, 3);
252 - add_action('edited_term', [$this, 'handle_taxonomy_change'], 20, 3);
253 - add_action('delete_term', [$this, 'handle_taxonomy_change'], 20, 3);
408 + foreach (['created_term', 'edited_term', 'delete_term'] as $hook) {
409 + add_action($hook, static function (int $term_id, int $tt_id, string $taxonomy): void {
410 + self::listener()->handle_taxonomy_change($term_id, $tt_id, $taxonomy);
411 + }, 20, 3);
412 + }
254 413
255 - // NOTE: the WP-Cron regeneration listeners (thinkrank_regenerate_sitemap
256 - // and thinkrank_regenerate_sitemap_settings) are registered at plugin
257 - // bootstrap (Plugin::register_sitemap_cron_listeners(), on plugins_loaded)
258 - // rather than here. A cron run never builds this class via the REST
259 - // endpoint (no rest_api_init), so registering them in the constructor
260 - // would leave the scheduled events with no listener at cron time.
414 + // The sitemap leaves out what the robots tag noindexes and what
415 + // ThinkRank redirects (#911), so a change to either decides what the
416 + // published files should list. Neither is a post or term save, and
417 + // without these the files kept the old answer until unrelated content
418 + // changed.
419 + foreach (self::ROBOTS_SETTINGS_OPTIONS as $option) {
420 + add_action('update_option_' . $option, static function ($old_value, $value): void {
421 + self::handle_robots_settings_change($old_value, $value);
422 + }, 20, 2);
423 + add_action('add_option_' . $option, static function ($name, $value): void {
424 + self::handle_robots_settings_change([], $value);
425 + }, 20, 2);
426 + }
427 +
428 + add_action('thinkrank_object_redirect_saved', static function (): void {
429 + self::listener()->schedule_regeneration();
430 + }, 20, 0);
261 431 }
262 432
263 433 /**
434 + * Options holding robots directives the sitemap's item filter reads.
435 + *
436 + * @since 2.15.0
437 + * @var string[]
438 + */
439 + private const ROBOTS_SETTINGS_OPTIONS = [
440 + 'thinkrank_global_seo_settings',
441 + 'thinkrank_global_robot_meta_settings',
442 + ];
443 +
444 + /**
445 + * Queue a rebuild when a saved robots setting changes a noindex decision.
446 + *
447 + * Both options carry far more than robots directives (titles, schema,
448 + * feature switches), so only a change to a noindex outcome rebuilds.
449 + *
450 + * @since 2.15.0
451 + *
452 + * @param mixed $old_value Previous option value.
453 + * @param mixed $value New option value.
454 + * @return void
455 + */
456 + public static function handle_robots_settings_change($old_value, $value): void {
457 + if (self::noindex_fingerprint($old_value) === self::noindex_fingerprint($value)) {
458 + return;
459 + }
460 +
461 + // The matrix memoises the option for the request, and a rebuild that
462 + // runs in this request must read the value just saved.
463 + Content_Type_Settings::flush_cache();
464 +
465 + self::listener()->schedule_regeneration();
466 + }
467 +
468 + /**
469 + * The noindex decisions a robots option makes, keyed by what they apply to.
470 + *
471 + * Reads both shapes: the flat site-wide directives, and the per-entity
472 + * rows, where a row's directives apply only while its robots switch is on.
473 + * A row that is on but stores no `noindex` key changes nothing, matching
474 + * the array_merge() the robots tag does.
475 + *
476 + * @since 2.15.0
477 + *
478 + * @param mixed $value Option value.
479 + * @return array<string, bool|null>
480 + */
481 + private static function noindex_fingerprint($value): array {
482 + if (!is_array($value)) {
483 + return [];
484 + }
485 +
486 + $decisions = [];
487 +
488 + if (array_key_exists('noindex', $value) && !is_array($value['noindex'])) {
489 + $decisions['*'] = !empty($value['noindex']);
490 + }
491 +
492 + foreach ($value as $key => $row) {
493 + if (!is_array($row)) {
494 + continue;
495 + }
496 +
497 + $robots = $row['robots_meta'] ?? null;
498 +
499 + $decisions[(string) $key] = !empty($row['robots_meta_enabled']) && is_array($robots) && array_key_exists('noindex', $robots)
500 + ? !empty($robots['noindex'])
501 + : null;
502 + }
503 +
504 + ksort($decisions);
505 +
506 + return $decisions;
507 + }
508 +
509 + /**
510 + * The generator the content-change listeners share.
511 + *
512 + * @since 2.10.1
513 + * @return self
514 + */
515 + private static function listener(): self {
516 + return self::$listener ??= new self(false);
517 + }
518 +
519 + /**
264 520 * Generate XML sitemap
265 521 *
266 522 * @since 1.0.0
267 523 *
@@ -270,9 +526,9 @@
270 526 */
271 527 public function generate_sitemap(array $options = []): string {
272 528 $settings = $this->get_settings('site');
273 529
274 - $xml = $this->xml_prolog($settings, 'sitemap.xsl');
530 + $xml = $this->xml_prolog($settings, 'sitemap');
275 531
276 532 // Add image namespace if images are enabled
277 533 if (!empty($settings['include_images'])) {
278 534 $xml .= '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">' . "\n";
@@ -435,8 +691,15 @@
435 691 ]
436 692 ],
437 693 'use_sitemap_index' => false,
438 694
695 + // How the sitemap reaches crawlers. 'auto' keeps the historical
696 + // behaviour wherever the web root is writable, and only falls back
697 + // to serving the sitemap from PHP where writing a file is
698 + // impossible — previously a hard failure with nothing served
699 + // (#752).
700 + 'delivery_mode' => 'auto',
701 +
439 702 // General Settings
440 703 'links_per_sitemap' => 1000,
441 704 'include_images' => true,
442 705 'include_featured_images' => false,
@@ -458,8 +721,17 @@
458 721 // Advanced Options
459 722 'enable_styling' => true,
460 723 'custom_url_pattern' => 'sitemap-{type}.xml',
461 724
725 + // Stylesheet branding (#639). Both colours default to empty, not
726 + // to the stock hexes: empty means the stylesheet's own value
727 + // stands, so a site that never opens this screen renders exactly
728 + // as it did before the setting existed.
729 + 'styling_logo' => false,
730 + 'styling_logo_url' => '',
731 + 'styling_color_main' => '',
732 + 'styling_color_accent' => '',
733 +
462 734 // Generation tracking
463 735 'last_generated' => ''
464 736 ];
465 737 }
@@ -464,8 +736,49 @@
464 736 ];
465 737 }
466 738
467 739 /**
740 + * Normalize the stylesheet branding values on the way into the store.
741 + *
742 + * The generic sanitizer only runs `sanitize_text_field()` over a string,
743 + * which happily keeps "red" or "rebeccapurple" as a colour. Nothing
744 + * downstream can use those — {@see Sitemap_Stylesheet::render()} skips any
745 + * value it cannot read as a hex colour — so storing them would report a
746 + * successful save of a setting that changes nothing, and `get-sitemap-
747 + * settings` would hand an agent back a colour the sitemap does not use.
748 + * Reducing here instead keeps the store and the rendering in agreement.
749 + *
750 + * @since 2.7.0
751 + *
752 + * @param array $settings Settings to sanitize.
753 + * @param string $context_type Context the save is for.
754 + * @return array
755 + */
756 + protected function sanitize_settings(array $settings, string $context_type = 'site'): array {
757 + $sanitized = parent::sanitize_settings($settings, $context_type);
758 +
759 + foreach (['styling_color_main', 'styling_color_accent'] as $key) {
760 + if (array_key_exists($key, $sanitized)) {
761 + $sanitized[$key] = Sitemap_Stylesheet::hex($sanitized[$key]);
762 + }
763 + }
764 +
765 + if (array_key_exists('styling_logo_url', $sanitized)) {
766 + $sanitized['styling_logo_url'] = esc_url_raw((string) $sanitized['styling_logo_url']);
767 + }
768 +
769 + // A mode this build cannot act on has to be stored as the fallback
770 + // rather than kept verbatim, or get-sitemap-settings reports a delivery
771 + // mode the site does not actually apply.
772 + if (array_key_exists('delivery_mode', $sanitized)) {
773 + $mode = sanitize_key((string) $sanitized['delivery_mode']);
774 + $sanitized['delivery_mode'] = in_array($mode, self::DELIVERY_MODES, true) ? $mode : 'auto';
775 + }
776 +
777 + return $sanitized;
778 + }
779 +
780 + /**
468 781 * Get settings schema definition (implements interface)
469 782 *
470 783 * @since 1.0.0
471 784 *
@@ -479,8 +792,15 @@
479 792 'title' => 'Enable Sitemap',
480 793 'description' => 'Generate XML sitemap for search engines',
481 794 'default' => true
482 795 ],
796 + 'delivery_mode' => [
797 + 'type' => 'string',
798 + 'title' => 'Sitemap Delivery',
799 + 'description' => 'How the sitemap is served: auto picks static when the WordPress root is writable and dynamic when it is not, static writes files to the web root, dynamic serves the sitemap from WordPress with no files written',
800 + 'enum' => self::DELIVERY_MODES,
801 + 'default' => 'auto'
802 + ],
483 803 'include_posts' => [
484 804 'type' => 'boolean',
485 805 'title' => 'Include Posts',
486 806 'description' => 'Include blog posts in sitemap',
@@ -521,8 +841,32 @@
521 841 'type' => 'string',
522 842 'title' => 'Last Generated',
523 843 'description' => 'Timestamp of last sitemap generation',
524 844 'default' => ''
845 + ],
846 + 'styling_logo' => [
847 + 'type' => 'boolean',
848 + 'title' => 'Show Logo On Sitemap',
849 + 'description' => 'Show a logo above the sitemap heading',
850 + 'default' => false
851 + ],
852 + 'styling_logo_url' => [
853 + 'type' => 'string',
854 + 'title' => 'Sitemap Logo',
855 + 'description' => 'Logo image URL. Empty falls back to the site icon',
856 + 'default' => ''
857 + ],
858 + 'styling_color_main' => [
859 + 'type' => 'string',
860 + 'title' => 'Sitemap Main Color',
861 + 'description' => 'Hex color for the sitemap header, links and table head. Empty keeps the stock palette',
862 + 'default' => ''
863 + ],
864 + 'styling_color_accent' => [
865 + 'type' => 'string',
866 + 'title' => 'Sitemap Accent Color',
867 + 'description' => 'Hex color for the header gradient and link hovers. Empty keeps the stock palette',
868 + 'default' => ''
525 869 ]
526 870 ];
527 871 }
528 872
@@ -539,9 +883,14 @@
539 883 * @return string XML URL entry
540 884 */
541 885 private function generate_url_entry(string $url, string $lastmod, float $priority, string $changefreq, array $images = []): string {
542 886 $xml = " <url>\n";
543 - $xml .= " <loc>" . esc_url($url) . "</loc>\n";
887 + // Every <loc> in every sitemap passes through here, which is why the
888 + // scheme preference is applied at this one point rather than at each
889 + // of the dozen collectors that build URLs (#638). An http sitemap on
890 + // an https site hands search engines the wrong address for the whole
891 + // site at once.
892 + $xml .= " <loc>" . esc_url(Url_Scheme::apply($url)) . "</loc>\n";
544 893 // Omit <lastmod> when unknown (empty) — a fabricated timestamp is worse
545 894 // than no timestamp, and an absent lastmod is valid per the spec.
546 895 if (!empty($lastmod)) {
547 896 $xml .= " <lastmod>" . esc_html($lastmod) . "</lastmod>\n";
@@ -551,9 +900,12 @@
551 900
552 901 // Add image entries if provided
553 902 foreach ($images as $image) {
554 903 $xml .= " <image:image>\n";
555 - $xml .= " <image:loc>" . esc_url($image['url']) . "</image:loc>\n";
904 + // The sitemap protocol wants an escaped URL, and WordPress hands back
905 + // attachment URLs with non-ASCII filenames unencoded (esc_url() does
906 + // not encode them either), so write the percent-encoded form (#924).
907 + $xml .= " <image:loc>" . esc_url(\ThinkRank\Core\Url_Validator::to_ascii(Url_Scheme::apply((string) $image['url']))) . "</image:loc>\n";
556 908
557 909 if (!empty($image['title'])) {
558 910 $xml .= " <image:title>" . esc_html($image['title']) . "</image:title>\n";
559 911 }
@@ -674,8 +1026,11 @@
674 1026 // well-bounded routine with no ceiling (#402).
675 1027 $total = count($all_ids);
676 1028
677 1029 for ($offset = 0; $offset < $total; $offset += self::ID_WALK_CHUNK) {
1030 + $this->assert_memory_headroom();
1031 + $chunk_start = memory_get_usage(true);
1032 +
678 1033 $chunk = array_slice($all_ids, $offset, self::ID_WALK_CHUNK);
679 1034
680 1035 $posts = get_posts($this->filter_query_args([
681 1036 'post_type' => $post_types,
@@ -684,8 +1039,12 @@
684 1039 'post__in' => $chunk,
685 1040 'orderby' => 'post__in', // preserve the resolved order
686 1041 ]));
687 1042
1043 + // get_featured_image() hydrates each featured image under the
1044 + // attachment's own ID, which the chunk's post IDs do not reach.
1045 + $attachment_ids = [];
1046 +
688 1047 foreach ($posts as $post) {
689 1048 if ($this->should_include_in_sitemap($post, $settings)) {
690 1049 /**
691 1050 * Filter a sitemap entry's permalink.
@@ -706,14 +1065,25 @@
706 1065 $priority = $this->calculate_intelligent_priority($post, $post->post_type);
707 1066 $changefreq = $this->calculate_change_frequency($post, $post->post_type);
708 1067 $images = $this->extract_post_images($post, $settings);
709 1068
1069 + $thumbnail_id = (int) get_post_thumbnail_id($post);
1070 + if ($thumbnail_id > 0) {
1071 + $attachment_ids[] = $thumbnail_id;
1072 + }
1073 +
710 1074 yield $this->generate_url_entry($url, $lastmod, $priority, $changefreq, $images);
711 1075 }
712 1076 }
713 1077
714 - // Free the hydrated chunk before loading the next one.
1078 + // Free the hydrated chunk before loading the next one — including
1079 + // the copies get_posts() left in the runtime object cache.
715 1080 unset($posts);
1081 + $this->release_walk_memory(
1082 + $chunk_start,
1083 + $this->chunk_post_cache_groups($post_types),
1084 + array_merge($chunk, $attachment_ids)
1085 + );
716 1086 }
717 1087 }
718 1088
719 1089 /**
@@ -773,8 +1143,11 @@
773 1143 // Same moving window as the post walk above, for the same reason.
774 1144 $total = count($all_ids);
775 1145
776 1146 for ($offset = 0; $offset < $total; $offset += self::TERM_WALK_CHUNK) {
1147 + $this->assert_memory_headroom();
1148 + $chunk_start = memory_get_usage(true);
1149 +
777 1150 $chunk = array_slice($all_ids, $offset, self::TERM_WALK_CHUNK);
778 1151
779 1152 $terms = get_terms($this->filter_term_query_args([
780 1153 'taxonomy' => $taxonomy,
@@ -787,12 +1160,12 @@
787 1160 continue;
788 1161 }
789 1162
790 1163 foreach ($terms as $term) {
791 - // A term the user marked noindex must not be advertised in the
792 - // sitemap: the robots tag now honours term meta, so listing it
793 - // here would have the sitemap contradict the page's own tag.
794 - if ($this->term_is_noindexed((int) $term->term_id)) {
1164 + // A term whose archive says noindex (its own override or its
1165 + // taxonomy's), or that redirects, must not be advertised: the
1166 + // sitemap would contradict the page's own signal (#911).
1167 + if (!Indexability::is_indexable_term($term)) {
795 1168 continue;
796 1169 }
797 1170
798 1171 $url = get_term_link($term);
@@ -803,8 +1176,9 @@
803 1176 }
804 1177 }
805 1178
806 1179 unset($terms);
1180 + $this->release_walk_memory($chunk_start, ['terms', 'term_meta'], $chunk);
807 1181 }
808 1182 }
809 1183
810 1184 /**
@@ -819,19 +1193,25 @@
819 1193 * kept a shadowing file behind (#510) or had a competitor's deleted.
820 1194 *
821 1195 * @since 2.1.1
822 1196 *
823 - * @param array $settings Sitemap settings (read for `enable_styling`).
824 - * @param string $stylesheet Stylesheet basename in static/xsl/.
1197 + * The stylesheet URL is served by {@see Sitemap_Stylesheet}, not read off
1198 + * disk by the web server, because a static file cannot carry the site's own
1199 + * logo and colours (#639). It is a fixed URL: the palette is applied per
1200 + * request, so changing a brand colour needs no regeneration and shows up on
1201 + * sitemaps published long before.
1202 + *
1203 + * @param array $settings Sitemap settings (read for `enable_styling`).
1204 + * @param string $variant Stylesheet variant, `sitemap` or `index`.
825 1205 * @return string Prolog lines, newline-terminated.
826 1206 */
827 - private function xml_prolog(array $settings, string $stylesheet): string {
1207 + private function xml_prolog(array $settings, string $variant): string {
828 1208 $xml = '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
829 1209 $xml .= THINKRANK_SITEMAP_MARKER . "\n";
830 1210
831 1211 // The stylesheet is presentation only, so it stays opt-in.
832 1212 if (!empty($settings['enable_styling'])) {
833 - $xml .= '<?xml-stylesheet type="text/xsl" href="' . home_url('/wp-content/plugins/thinkrank/static/xsl/' . $stylesheet) . '"?>' . "\n";
1213 + $xml .= '<?xml-stylesheet type="text/xsl" href="' . esc_url(Sitemap_Stylesheet::url($variant)) . '"?>' . "\n";
834 1214 }
835 1215
836 1216 return $xml;
837 1217 }
@@ -846,9 +1226,9 @@
846 1226 * @param bool $with_image_ns Include the image sitemap namespace
847 1227 * @return string Full sitemap XML
848 1228 */
849 1229 private function wrap_urlset(array $entries, array $settings, bool $with_image_ns): string {
850 - $xml = $this->xml_prolog($settings, 'sitemap.xsl');
1230 + $xml = $this->xml_prolog($settings, 'sitemap');
851 1231
852 1232 if ($with_image_ns && !empty($settings['include_images'])) {
853 1233 $xml .= '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">' . "\n";
854 1234 } else {
@@ -973,9 +1353,9 @@
973 1353 if (preg_match('/src=["\']([^"\']+)["\']/', $img_tag, $src_match)) {
974 1354 $image_url = $src_match[1];
975 1355
976 1356 // Skip if not a valid URL or external image
977 - if (!filter_var($image_url, FILTER_VALIDATE_URL)) {
1357 + if (!\ThinkRank\Core\Url_Validator::is_valid($image_url)) {
978 1358 continue;
979 1359 }
980 1360
981 1361 // Extract title and alt attributes
@@ -1026,9 +1406,16 @@
1026 1406 '_builtin' => false
1027 1407 ], 'names');
1028 1408
1029 1409 foreach ($custom_taxonomies as $taxonomy) {
1030 - if ($this->should_include_taxonomy($taxonomy)) {
1410 + if (!$this->should_include_taxonomy($taxonomy)) {
1411 + continue;
1412 + }
1413 +
1414 + // Same as the post-type walk above: an explicit per-taxonomy flag
1415 + // now decides, and an unset flag keeps the previous "included"
1416 + // behaviour (#660).
1417 + if (\ThinkRank\SEO\Content_Type_Settings::is_included_in_sitemap('taxonomy', $taxonomy, $settings)) {
1031 1418 $taxonomies[] = $taxonomy;
1032 1419 }
1033 1420 }
1034 1421
@@ -1083,14 +1470,15 @@
1083 1470 if (is_wp_error($all_terms)) {
1084 1471 return [];
1085 1472 }
1086 1473
1087 - // Drop terms the user marked noindex. This path feeds the single general
1088 - // sitemap while collect_taxonomy_entries_iter() feeds the segmented ones,
1089 - // so both need the filter or the two disagree about the same term.
1474 + // Drop terms that are not indexable destinations. This path feeds the
1475 + // single general sitemap while collect_taxonomy_entries_iter() feeds
1476 + // the segmented ones, so both need the filter or the two disagree about
1477 + // the same term.
1090 1478 $all_terms = array_values(array_filter(
1091 1479 $all_terms,
1092 - fn($term) => !$this->term_is_noindexed((int) $term->term_id)
1480 + static fn($term) => $term instanceof \WP_Term && Indexability::is_indexable_term($term)
1093 1481 ));
1094 1482
1095 1483 // Group terms by taxonomy
1096 1484 return $this->group_terms_by_taxonomy($all_terms);
@@ -1272,17 +1660,16 @@
1272 1660 if (!in_array($post->post_status, ['publish', 'private'], true)) {
1273 1661 return false;
1274 1662 }
1275 1663
1276 - // Check if post overrides robots and sets noindex.
1277 - if ((bool) get_post_meta($post->ID, '_thinkrank_robots_meta_enabled', true)) {
1278 - $raw = get_post_meta($post->ID, '_thinkrank_robots_meta', true);
1279 - if (is_string($raw) && $raw !== '') {
1280 - $robots = json_decode($raw, true);
1281 - if (is_array($robots) && !empty($robots['noindex'])) {
1282 - return false;
1283 - }
1284 - }
1664 + // A URL whose page says noindex, or that ThinkRank redirects, is not a
1665 + // destination. Only the per-post noindex used to be read here, so a
1666 + // post type set to No-index still had every item listed, and redirected
1667 + // posts were submitted as "Page with redirect" (#911). The password
1668 + // check stays with the setting above: listing protected posts is a
1669 + // choice this sitemap has always offered.
1670 + if (Indexability::is_post_noindexed($post) || Indexability::is_post_redirected($post)) {
1671 + return false;
1285 1672 }
1286 1673
1287 1674 return true;
1288 1675 }
@@ -1321,35 +1708,8 @@
1321 1708 return $ids;
1322 1709 }
1323 1710
1324 1711 /**
1325 - * Whether a term carries an explicit noindex override.
1326 - *
1327 - * Mirrors the post-side check in should_include_post(); terms store the same
1328 - * `_thinkrank_robots_meta_enabled` / `_thinkrank_robots_meta` keys, written
1329 - * by the update-term-seo ability and by the SEO importer.
1330 - *
1331 - * @since 1.31.0
1332 - *
1333 - * @param int $term_id Term to test.
1334 - * @return bool True when the term is marked noindex.
1335 - */
1336 - private function term_is_noindexed(int $term_id): bool {
1337 - if (!(bool) get_term_meta($term_id, '_thinkrank_robots_meta_enabled', true)) {
1338 - return false;
1339 - }
1340 -
1341 - $raw = get_term_meta($term_id, '_thinkrank_robots_meta', true);
1342 - if (!is_string($raw) || $raw === '') {
1343 - return false;
1344 - }
1345 -
1346 - $robots = json_decode($raw, true);
1347 -
1348 - return is_array($robots) && !empty($robots['noindex']);
1349 - }
1350 -
1351 - /**
1352 1712 * Count total URLs in sitemap
1353 1713 *
1354 1714 * @since 1.0.0
1355 1715 *
@@ -1512,9 +1872,15 @@
1512 1872 if (!empty($settings['include_pages'])) {
1513 1873 $post_types[] = 'page';
1514 1874 }
1515 1875
1516 - // Auto-detect public custom post types that should be included
1876 + // Auto-detect public custom post types that should be included.
1877 + //
1878 + // A custom type's `include_<slug>` / `exclude_<slug>` flag is honoured
1879 + // here (#660). It was previously stored — additional_setting_keys()
1880 + // has always let those keys through — but never read, so a CPT was in
1881 + // the sitemap whatever the setting said. Unset still means included, so
1882 + // a site that never touched the flag is unaffected.
1517 1883 $custom_post_types = get_post_types([
1518 1884 'public' => true,
1519 1885 '_builtin' => false
1520 1886 ], 'names');
@@ -1519,9 +1885,13 @@
1519 1885 '_builtin' => false
1520 1886 ], 'names');
1521 1887
1522 1888 foreach ($custom_post_types as $post_type) {
1523 - if ($this->should_include_post_type($post_type)) {
1889 + if (!$this->should_include_post_type($post_type)) {
1890 + continue;
1891 + }
1892 +
1893 + if (\ThinkRank\SEO\Content_Type_Settings::is_included_in_sitemap('post_type', $post_type, $settings)) {
1524 1894 $post_types[] = $post_type;
1525 1895 }
1526 1896 }
1527 1897
@@ -1542,18 +1912,11 @@
1542 1912 // Templately's internal `templately_library` store — which are template
1543 1913 // records, not standalone indexable URLs. BetterDocs `docs` and
1544 1914 // WooCommerce `product` register exclude_from_search => false, so they
1545 1915 // remain included.
1546 - if (!is_post_type_viewable($post_type)) {
1547 - return false;
1548 - }
1549 -
1550 - $post_type_obj = get_post_type_object($post_type);
1551 - if (!$post_type_obj || !empty($post_type_obj->exclude_from_search)) {
1552 - return false;
1553 - }
1554 -
1555 - return true;
1916 + // The predicate lives in Content_Type_Settings so the matrix can ask
1917 + // the same question before offering a switch for this post type.
1918 + return \ThinkRank\SEO\Content_Type_Settings::sitemap_accepts_post_type($post_type);
1556 1919 }
1557 1920
1558 1921 /**
1559 1922 * Check if taxonomy should trigger regeneration
@@ -1562,11 +1925,13 @@
1562 1925 * @param string $taxonomy Taxonomy slug
1563 1926 * @return bool True if taxonomy should trigger regeneration
1564 1927 */
1565 1928 private function should_include_taxonomy(string $taxonomy): bool {
1566 - // Include all public taxonomies (presets control which sitemaps are created)
1567 - $taxonomy_obj = get_taxonomy($taxonomy);
1568 - return $taxonomy_obj && $taxonomy_obj->public;
1929 + // Public taxonomies only, and only the ones this generator can actually
1930 + // emit — the same predicate the content-type matrix asks before it
1931 + // offers a sitemap switch for one (presets still control which
1932 + // sitemaps are created).
1933 + return \ThinkRank\SEO\Content_Type_Settings::sitemap_accepts_taxonomy($taxonomy);
1569 1934 }
1570 1935
1571 1936 /**
1572 1937 * Schedule debounced sitemap regeneration
@@ -1623,8 +1988,14 @@
1623 1988 * @param string $source Either 'content' or 'settings'.
1624 1989 * @return void
1625 1990 */
1626 1991 private function mark_regeneration_pending(string $source): void {
1992 + // Whatever made the static files stale made the rendered ones stale
1993 + // too. Invalidating here rather than only on the rebuild keeps the two
1994 + // delivery modes reacting to exactly the same triggers, which is the
1995 + // only way a dynamic site stays as fresh as a static one (#752).
1996 + $this->flush_dynamic_cache();
1997 +
1627 1998 $pending = get_option(self::REGENERATION_PENDING_OPTION, []);
1628 1999 $pending = is_array($pending) ? $pending : [];
1629 2000
1630 2001 $since = !empty($pending['since']) ? (int) $pending['since'] : time();
@@ -1630,11 +2001,13 @@
1630 2001 $since = !empty($pending['since']) ? (int) $pending['since'] : time();
1631 2002 $current = isset($pending['source']) ? (string) $pending['source'] : '';
1632 2003 $source = ($current === 'settings' || $source === 'settings') ? 'settings' : 'content';
1633 2004
2005 + // Merged so the bookkeeping a rebuild keeps on the marker (`started`,
2006 + // `memory_limit`) survives an edit made while it is outstanding.
1634 2007 update_option(
1635 2008 self::REGENERATION_PENDING_OPTION,
1636 - [
2009 + array_merge($pending, [
1637 2010 'since' => $since,
1638 2011 'source' => $source,
1639 2012 'attempts' => !empty($pending['attempts']) ? (int) $pending['attempts'] : 0,
1640 2013 'next_attempt' => !empty($pending['next_attempt'])
@@ -1642,9 +2015,9 @@
1642 2015 : time() + self::REGENERATION_TAKEOVER_GRACE,
1643 2016 // Bumped on every change so a rebuild can tell whether the edit
1644 2017 // it started for is still the newest one outstanding.
1645 2018 'revision' => (!empty($pending['revision']) ? (int) $pending['revision'] : 0) + 1,
1646 - ],
2019 + ]),
1647 2020 true
1648 2021 );
1649 2022 }
1650 2023
@@ -1690,8 +2063,18 @@
1690 2063 $revision !== null
1691 2064 && is_array($pending)
1692 2065 && (int) ($pending['revision'] ?? 0) !== $revision
1693 2066 ) {
2067 + // This attempt succeeded, so drop what it claimed: the newer change
2068 + // waits the grace a fresh edit gets, not a failure backoff it never
2069 + // earned. Not zero: that edit queued its own debounced event, and
2070 + // a marker due at once had the next admin request rebuild in its
2071 + // shutdown and the event rebuild again seconds later.
2072 + unset($pending['started'], $pending['memory_limit']);
2073 + $pending['attempts'] = 0;
2074 + $pending['next_attempt'] = time() + self::REGENERATION_TAKEOVER_GRACE;
2075 +
2076 + update_option(self::REGENERATION_PENDING_OPTION, $pending, true);
1694 2077 return;
1695 2078 }
1696 2079
1697 2080 delete_option(self::REGENERATION_PENDING_OPTION);
@@ -1697,8 +2080,69 @@
1697 2080 delete_option(self::REGENERATION_PENDING_OPTION);
1698 2081 }
1699 2082
1700 2083 /**
2084 + * Record the attempt that is about to run before it runs.
2085 + *
2086 + * A PHP fatal — the memory limit or max_execution_time — is not a
2087 + * Throwable, so no catch or finally around the generation runs when the
2088 + * process dies, and a failure recorded afterwards was never recorded at
2089 + * all: `attempts` stayed 0, the backoff never applied, and the next request
2090 + * started the same doomed rebuild again (a fatal every few minutes for as
2091 + * long as an admin was logged in). Claiming the attempt up front makes the
2092 + * backoff hold even when nothing after this line gets to run, and leaves a
2093 + * `started` stamp the next attempt can recognise as an interrupted one.
2094 + *
2095 + * @since 2.10.1
2096 + * @return void
2097 + */
2098 + private function claim_regeneration_attempt(): void {
2099 + $pending = get_option(self::REGENERATION_PENDING_OPTION, null);
2100 +
2101 + // Nothing outstanding (e.g. a manual generation already satisfied it):
2102 + // there is no marker to retry from, so nothing to claim.
2103 + if (!is_array($pending) || empty($pending['since'])) {
2104 + return;
2105 + }
2106 +
2107 + if (!empty($pending['started'])) {
2108 + // The previous attempt claimed itself and never reported back.
2109 + update_option(
2110 + self::REGENERATION_ERROR_OPTION,
2111 + [
2112 + 'message' => __('The previous automatic sitemap rebuild stopped before it finished, most likely because PHP ran out of memory or time. It is retried with a growing delay. If this keeps happening, raise the PHP memory_limit or max_execution_time, or run WP-Cron from a system cron.', 'thinkrank'),
2113 + 'source' => isset($pending['source']) ? (string) $pending['source'] : 'content',
2114 + 'attempts' => !empty($pending['attempts']) ? (int) $pending['attempts'] : 1,
2115 + 'time' => (int) $pending['started'],
2116 + ],
2117 + false
2118 + );
2119 + }
2120 +
2121 + $attempts = (!empty($pending['attempts']) ? (int) $pending['attempts'] : 0) + 1;
2122 +
2123 + $pending['attempts'] = $attempts;
2124 + $pending['next_attempt'] = time() + $this->regeneration_backoff($attempts);
2125 + $pending['started'] = time();
2126 +
2127 + update_option(self::REGENERATION_PENDING_OPTION, $pending, true);
2128 + }
2129 +
2130 + /**
2131 + * Delay before the next takeover after the given number of attempts.
2132 + *
2133 + * @since 2.10.1
2134 + * @param int $attempts Attempts made so far (1 or more).
2135 + * @return int Seconds.
2136 + */
2137 + private function regeneration_backoff(int $attempts): int {
2138 + return (int) min(
2139 + self::REGENERATION_TAKEOVER_GRACE * (2 ** min(max($attempts, 1), 10)),
2140 + self::REGENERATION_MAX_BACKOFF
2141 + );
2142 + }
2143 +
2144 + /**
1701 2145 * Record a failed regeneration instead of discarding it.
1702 2146 *
1703 2147 * Keeps the pending marker in place so the rebuild is retried, but backs the
1704 2148 * next attempt off exponentially (capped) so a persistently failing
@@ -1704,22 +2148,30 @@
1704 2148 * next attempt off exponentially (capped) so a persistently failing
1705 2149 * generation cannot run on every admin request.
1706 2150 *
1707 2151 * @since 2.2.1
1708 - * @param string $message Failure detail.
1709 - * @param string $source Either 'content' or 'settings'.
2152 + * @since 2.10.1 Accepts the memory limit a rebuild had to stop short of, and
2153 + * does not count an attempt claim_regeneration_attempt()
2154 + * already counted.
2155 + * @param string $message Failure detail.
2156 + * @param string $source Either 'content' or 'settings'.
2157 + * @param int|null $memory_limit Memory limit (bytes) the rebuild stopped
2158 + * short of, when that was the failure.
1710 2159 * @return void
1711 2160 */
1712 - private function record_regeneration_failure(string $message, string $source): void {
2161 + private function record_regeneration_failure(string $message, string $source, ?int $memory_limit = null): void {
1713 2162 $pending = get_option(self::REGENERATION_PENDING_OPTION, []);
1714 2163 $pending = is_array($pending) ? $pending : [];
1715 - $attempts = (!empty($pending['attempts']) ? (int) $pending['attempts'] : 0) + 1;
2164 + $attempts = !empty($pending['attempts']) ? (int) $pending['attempts'] : 0;
1716 2165
1717 - $backoff = min(
1718 - self::REGENERATION_TAKEOVER_GRACE * (2 ** min($attempts, 10)),
1719 - self::REGENERATION_MAX_BACKOFF
1720 - );
2166 + // An attempt that claimed itself up front has already been counted.
2167 + if (empty($pending['started'])) {
2168 + $attempts++;
2169 + }
2170 + $attempts = max($attempts, 1);
1721 2171
2172 + $backoff = $this->regeneration_backoff($attempts);
2173 +
1722 2174 // Same precedence mark_regeneration_pending() enforces: a settings
1723 2175 // rebuild outranks a content one and must not be downgraded by a failed
1724 2176 // attempt. Overwriting it routed the retry back through the content
1725 2177 // path, where should_auto_generate() can be false and the completion
@@ -1728,20 +2180,24 @@
1728 2180 // whichever attempt actually failed.
1729 2181 $current = isset($pending['source']) ? (string) $pending['source'] : '';
1730 2182 $pending_source = ($current === 'settings' || $source === 'settings') ? 'settings' : 'content';
1731 2183
1732 - update_option(
1733 - self::REGENERATION_PENDING_OPTION,
1734 - [
1735 - 'since' => !empty($pending['since']) ? (int) $pending['since'] : time(),
1736 - 'source' => $pending_source,
1737 - 'attempts' => $attempts,
1738 - 'next_attempt' => time() + $backoff,
1739 - 'revision' => !empty($pending['revision']) ? (int) $pending['revision'] : 0,
1740 - ],
1741 - true
1742 - );
2184 + $marker = [
2185 + 'since' => !empty($pending['since']) ? (int) $pending['since'] : time(),
2186 + 'source' => $pending_source,
2187 + 'attempts' => $attempts,
2188 + 'next_attempt' => time() + $backoff,
2189 + 'revision' => !empty($pending['revision']) ? (int) $pending['revision'] : 0,
2190 + ];
1743 2191
2192 + // Remembered so has_memory_for_retry() can keep requests with no more
2193 + // memory than this from repeating the same attempt.
2194 + if ($memory_limit !== null && $memory_limit > 0) {
2195 + $marker['memory_limit'] = $memory_limit;
2196 + }
2197 +
2198 + update_option(self::REGENERATION_PENDING_OPTION, $marker, true);
2199 +
1744 2200 update_option(
1745 2201 self::REGENERATION_ERROR_OPTION,
1746 2202 [
1747 2203 'message' => $message,
@@ -1796,16 +2252,34 @@
1796 2252 if (!self::has_overdue_regeneration()) {
1797 2253 return;
1798 2254 }
1799 2255
1800 - // Cron is running: it is about to do exactly this work.
2256 + $pending = get_option(self::REGENERATION_PENDING_OPTION, []);
2257 + $pending = is_array($pending) ? $pending : [];
2258 + $source = isset($pending['source']) ? (string) $pending['source'] : 'content';
2259 +
2260 + // Cron is running and its own event for this rebuild is still queued
2261 + // and due: it is about to do exactly this work. Only then — an event
2262 + // that has already been consumed (e.g. it fired while another process
2263 + // held the generation lock) is never re-queued, and returning here
2264 + // unconditionally left the rebuild to requests that could not finish it.
1801 2265 if (wp_doing_cron()) {
2266 + $hook = $source === 'settings' ? 'thinkrank_regenerate_sitemap_settings' : 'thinkrank_regenerate_sitemap';
2267 + $next = wp_next_scheduled($hook);
2268 +
2269 + if ($next !== false && $next <= time()) {
2270 + return;
2271 + }
2272 + }
2273 +
2274 + // The last attempt had to stop short of this process's memory limit.
2275 + // Retrying at the same (or a lower) limit only repeats that, so leave
2276 + // the rebuild to a process with more room — WP-CLI, a system cron, or a
2277 + // host with a higher limit — instead of burning it on every request.
2278 + if (!$this->has_memory_for_retry($pending)) {
1802 2279 return;
1803 2280 }
1804 2281
1805 - $pending = get_option(self::REGENERATION_PENDING_OPTION, []);
1806 - $source = (is_array($pending) && isset($pending['source'])) ? (string) $pending['source'] : 'content';
1807 -
1808 2282 if ($source === 'settings') {
1809 2283 $this->regenerate_sitemap_from_settings();
1810 2284 return;
1811 2285 }
@@ -1839,8 +2313,228 @@
1839 2313 delete_transient(self::GENERATION_LOCK_TRANSIENT);
1840 2314 }
1841 2315
1842 2316 /**
2317 + * This process's PHP memory limit in bytes.
2318 + *
2319 + * @since 2.10.1
2320 + * @return int Bytes, or -1 when unlimited (or unreadable).
2321 + */
2322 + private function current_memory_limit(): int {
2323 + $limit = (string) ini_get('memory_limit');
2324 +
2325 + if ($limit === '' || $limit === '-1') {
2326 + return -1;
2327 + }
2328 +
2329 + $bytes = (int) wp_convert_hr_to_bytes($limit);
2330 +
2331 + return $bytes > 0 ? $bytes : -1;
2332 + }
2333 +
2334 + /**
2335 + * May this process retry a rebuild that last stopped at the memory limit?
2336 + *
2337 + * Raises the limit the way wp-admin does first, so a request that can get
2338 + * more room than the failed attempt had is still allowed to try.
2339 + *
2340 + * @since 2.10.1
2341 + * @param array $pending The pending marker.
2342 + * @return bool True when there is no recorded memory failure, or this
2343 + * process has more memory than the attempt that failed.
2344 + */
2345 + private function has_memory_for_retry(array $pending): bool {
2346 + if (empty($pending['memory_limit'])) {
2347 + return true;
2348 + }
2349 +
2350 + wp_raise_memory_limit('admin');
2351 +
2352 + $limit = $this->current_memory_limit();
2353 +
2354 + return $limit === -1 || $limit > (int) $pending['memory_limit'];
2355 + }
2356 +
2357 + /**
2358 + * Release what one walked chunk left behind.
2359 + *
2360 + * Hydrating a chunk through get_posts()/get_terms() also stores every
2361 + * object and its meta in the in-process object cache, which nothing
2362 + * empties until the request ends. Unsetting the chunk therefore freed
2363 + * nothing, and the walk grew with the size of the site instead of the size
2364 + * of a chunk — about 1.3 GB on a 45k-post site.
2365 + *
2366 + * A persistent object cache that supports it drops only its in-process
2367 + * copy (`flush_runtime`); the shared store keeps its data. WordPress's
2368 + * default cache has no shared store, and flushing it would empty every
2369 + * group for the rest of the request (options, the queried object, other
2370 + * plugins' data), so there only the chunk's own entries are deleted. A
2371 + * persistent cache without `flush_runtime` is left alone: deleting from it
2372 + * would evict the objects for every other request too.
2373 + *
2374 + * @since 2.10.1
2375 + * @param int $chunk_start memory_get_usage(true) before the chunk was hydrated.
2376 + * @param array $groups Cache groups keyed by the chunk's object IDs.
2377 + * @param int[] $ids The chunk's object IDs, plus any objects it
2378 + * hydrated under their own (featured images).
2379 + * @return void
2380 + * @throws \Error See assert_memory_headroom().
2381 + */
2382 + private function release_walk_memory(int $chunk_start, array $groups, array $ids): void {
2383 + // What one chunk costs before it is released: the margin the next one
2384 + // needs. Measured in the same real allocated size assert_memory_headroom()
2385 + // compares against the limit, so the two are the same unit.
2386 + $this->walk_chunk_cost = max($this->walk_chunk_cost, memory_get_usage(true) - $chunk_start);
2387 +
2388 + if (wp_using_ext_object_cache()) {
2389 + if (
2390 + function_exists('wp_cache_supports')
2391 + && wp_cache_supports('flush_runtime')
2392 + && function_exists('wp_cache_flush_runtime')
2393 + ) {
2394 + wp_cache_flush_runtime();
2395 + }
2396 + } elseif (!empty($ids)) {
2397 + foreach ($groups as $group) {
2398 + wp_cache_delete_multiple($ids, $group);
2399 + }
2400 + }
2401 +
2402 + $this->assert_memory_headroom();
2403 + }
2404 +
2405 + /**
2406 + * Cache groups get_posts() fills per post for the given post types.
2407 + *
2408 + * The post, its meta, and one relationships group per taxonomy the post
2409 + * type uses (update_object_term_cache()). Term objects themselves are
2410 + * bounded by the number of terms, not posts, so they are left cached.
2411 + *
2412 + * @since 2.10.1
2413 + * @param string[] $post_types Post types being walked.
2414 + * @return string[] Cache groups keyed by post ID.
2415 + */
2416 + private function chunk_post_cache_groups(array $post_types): array {
2417 + $groups = ['posts', 'post_meta'];
2418 +
2419 + foreach (get_object_taxonomies($post_types) as $taxonomy) {
2420 + $groups[] = $taxonomy . '_relationships';
2421 + }
2422 +
2423 + return array_values(array_unique($groups));
2424 + }
2425 +
2426 + /**
2427 + * Stop an automatic rebuild before the memory limit rather than at it.
2428 + *
2429 + * A PHP memory fatal skips every catch and finally, so the lock, the
2430 + * failure record and the backoff are all lost with it, while stopping here
2431 + * is an ordinary, fully recorded failure. Checked before each chunk is
2432 + * hydrated, against a margin of at least the largest chunk seen so far.
2433 + *
2434 + * It throws an \Error, not an \Exception, on purpose: the per-segment
2435 + * catch (\Exception) blocks in generate_multiple_sitemaps() would otherwise
2436 + * swallow it and carry on — writing an index without the aborted segments
2437 + * and then pruning their files as orphans. Only the automatic rebuild's
2438 + * catch (\Throwable) is meant to see it.
2439 + *
2440 + * @since 2.10.1
2441 + * @return void
2442 + * @throws \Error When the automatic rebuild is close to the memory limit.
2443 + */
2444 + private function assert_memory_headroom(): void {
2445 + if (!$this->memory_guard) {
2446 + return;
2447 + }
2448 +
2449 + $limit = $this->current_memory_limit();
2450 + if ($limit === -1) {
2451 + return;
2452 + }
2453 +
2454 + // A fifth of the limit (at least 32 MB) for writing the files, or one
2455 + // and a half of the costliest chunk if that is more — and never more
2456 + // than half the limit either way. Without that outer cap a single
2457 + // anomalously expensive chunk (500 posts of serialised page-builder or
2458 + // ACF meta reaches hundreds of megabytes) puts the margin above the
2459 + // limit itself, so every later check aborts at any usage at all, the
2460 + // failure records this process's limit, and has_memory_for_retry()
2461 + // then refuses every process that has the same limit. A site that
2462 + // never actually ran out of memory would stop rebuilding until WP-CLI
2463 + // or a system cron happened to run.
2464 + $headroom = (int) min(
2465 + max(
2466 + min(max($limit * 0.2, 32 * MB_IN_BYTES), $limit * 0.5),
2467 + $this->walk_chunk_cost * 1.5
2468 + ),
2469 + $limit * 0.5
2470 + );
2471 +
2472 + // The real allocated size, which is what PHP enforces memory_limit
2473 + // against; memory_get_usage(false) reports only what is handed out of
2474 + // those allocations and so understates the margin by the allocator's
2475 + // slack.
2476 + $usage = memory_get_usage(true);
2477 +
2478 + if ($usage > $limit - $headroom) {
2479 + throw new \Error(
2480 + sprintf(
2481 + /* translators: 1: memory in use, 2: PHP memory limit. */
2482 + esc_html__('The sitemap rebuild was stopped at %1$s of the %2$s PHP memory limit, before PHP would have run out of memory. It will be retried by a process with more memory (WP-CLI or a system cron). To let it finish in the admin, raise the PHP memory_limit.', 'thinkrank'),
2483 + esc_html((string) size_format($usage)),
2484 + esc_html((string) size_format($limit))
2485 + ),
2486 + // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- an integer class constant, not output.
2487 + self::MEMORY_ABORT_CODE
2488 + );
2489 + }
2490 + }
2491 +
2492 + /**
2493 + * Run an automatic rebuild's generation with the fatal-safe bookkeeping.
2494 + *
2495 + * @since 2.10.1
2496 + * @param array $settings Sitemap settings.
2497 + * @return bool Whatever generate_and_save() returned.
2498 + * @throws \Throwable Whatever generation throws, after the memory guard is
2499 + * switched back off.
2500 + */
2501 + private function generate_for_regeneration(array $settings): bool {
2502 + // Same headroom wp-admin gives itself; a no-op when the limit is
2503 + // already higher or unlimited.
2504 + wp_raise_memory_limit('admin');
2505 +
2506 + $this->claim_regeneration_attempt();
2507 + $this->memory_guard = true;
2508 + $this->walk_chunk_cost = 0;
2509 +
2510 + try {
2511 + return $this->generate_and_save($settings);
2512 + } finally {
2513 + $this->memory_guard = false;
2514 + }
2515 + }
2516 +
2517 + /**
2518 + * Record a failure thrown by an automatic rebuild.
2519 + *
2520 + * @since 2.10.1
2521 + * @param \Throwable $e What was thrown.
2522 + * @param string $source Either 'content' or 'settings'.
2523 + * @return void
2524 + */
2525 + private function record_thrown_regeneration_failure(\Throwable $e, string $source): void {
2526 + $memory_limit = null;
2527 +
2528 + if ($e instanceof \Error && $e->getCode() === self::MEMORY_ABORT_CODE) {
2529 + $memory_limit = $this->current_memory_limit();
2530 + $memory_limit = $memory_limit > 0 ? $memory_limit : null;
2531 + }
2532 +
2533 + $this->record_regeneration_failure($e->getMessage(), $source, $memory_limit);
2534 + }
2535 +
2536 + /**
1843 2537 * Report how automatic regeneration is faring, for the admin UI.
1844 2538 *
1845 2539 * The feature used to fail invisibly: `last_generated` simply stopped
1846 2540 * advancing and nothing drew attention to it (#629).
@@ -1938,16 +2632,20 @@
1938 2632 * auto_generate setting: the user deliberately changed inclusion rules and
1939 2633 * expects the served file to reflect them even if content-triggered
1940 2634 * auto-generation is turned off. Still respects the master `enabled` flag.
1941 2635 *
1942 - * @return void
2636 + * @since 2.10.0 Reports whether the served sitemap was actually rebuilt, so
2637 + * a caller can say so rather than assume it (#764). Existing
2638 + * callers that ignore the return are unaffected.
2639 + *
2640 + * @return bool True when the served sitemap now reflects the settings.
1943 2641 */
1944 - public function regenerate_sitemap_from_settings(): void {
2642 + public function regenerate_sitemap_from_settings(): bool {
1945 2643 if (!$this->acquire_generation_lock()) {
1946 2644 // A manual generation (or another request's takeover) is already
1947 2645 // writing the files; the pending marker survives so this rebuild is
1948 2646 // retried rather than lost.
1949 - return;
2647 + return false;
1950 2648 }
1951 2649
1952 2650 try {
1953 2651 $settings = $this->get_settings('site');
@@ -1954,25 +2652,56 @@
1954 2652 if (empty($settings['enabled'])) {
1955 2653 // The sitemap was disabled: remove the previously generated static
1956 2654 // files so the web server stops serving a stale sitemap that
1957 2655 // crawlers would otherwise keep fetching.
1958 - $this->delete_published_sitemaps();
2656 + //
2657 + // A file that could not be removed is still being served, so
2658 + // this is not a success. Reporting one here would tell a caller
2659 + // the sitemap was gone while the web server kept answering with
2660 + // it, which is the failure this return value exists to prevent
2661 + // (#764).
2662 + $removal = $this->delete_published_sitemaps($settings);
2663 + $stuck = is_array($removal['failed'] ?? null) ? $removal['failed'] : [];
2664 +
2665 + if (!empty($stuck)) {
2666 + $this->record_regeneration_failure(
2667 + $this->stuck_files_message($stuck, true),
2668 + 'settings'
2669 + );
2670 +
2671 + return false;
2672 + }
2673 +
1959 2674 $this->mark_regeneration_complete();
1960 - return;
2675 +
2676 + return true;
1961 2677 }
1962 2678
1963 2679 $revision = $this->current_regeneration_revision();
1964 2680
1965 - if ($this->generate_and_save($settings)) {
2681 + if ('dynamic' === $this->resolve_delivery_mode($settings)) {
2682 + // Returns false when a static file is stuck in the web root:
2683 + // the server keeps serving that file in preference to WordPress,
2684 + // so the switch has not taken effect (#764).
2685 + return $this->switch_to_dynamic_delivery($settings, $revision, 'settings');
2686 + }
2687 +
2688 + if ($this->generate_for_regeneration($settings)) {
1966 2689 $this->mark_regeneration_complete($revision);
1967 - } else {
1968 - $this->record_regeneration_failure(
1969 - __('The sitemap files could not be written to the site root.', 'thinkrank'),
1970 - 'settings'
1971 - );
2690 +
2691 + return true;
1972 2692 }
2693 +
2694 + $this->record_regeneration_failure(
2695 + $this->write_failure_message(),
2696 + 'settings'
2697 + );
2698 +
2699 + return false;
1973 2700 } catch (\Throwable $e) {
1974 - $this->record_regeneration_failure($e->getMessage(), 'settings');
2701 + $this->record_thrown_regeneration_failure($e, 'settings');
2702 +
2703 + return false;
1975 2704 } finally {
1976 2705 $this->release_generation_lock();
1977 2706 }
1978 2707 }
@@ -1977,8 +2706,27 @@
1977 2706 }
1978 2707 }
1979 2708
1980 2709 /**
2710 + * When a rebuild has been outstanding since, or 0 when none is.
2711 + *
2712 + * Lets a caller report an honest "saved, but the served file has not caught
2713 + * up yet" instead of a bare success (#764).
2714 + *
2715 + * @since 2.10.0
2716 + * @return int Unix timestamp, or 0 when nothing is pending.
2717 + */
2718 + public static function regeneration_pending_since(): int {
2719 + $pending = get_option(self::REGENERATION_PENDING_OPTION, []);
2720 +
2721 + if (!is_array($pending) || empty($pending['since'])) {
2722 + return 0;
2723 + }
2724 +
2725 + return (int) $pending['since'];
2726 + }
2727 +
2728 + /**
1981 2729 * Remove every static sitemap file ThinkRank publishes to the web root.
1982 2730 *
1983 2731 * Called when the sitemap feature is disabled, by the cleanup route, and by
1984 2732 * both removal paths, so /sitemap.xml, /sitemap_index.xml, the segmented
@@ -2077,10 +2825,17 @@
2077 2825 return;
2078 2826 }
2079 2827
2080 2828 $revision = $this->current_regeneration_revision();
2829 + $settings = $this->get_settings('site');
2081 2830
2082 - if ($this->generate_and_save($this->get_settings('site'))) {
2831 + // See regenerate_sitemap_from_settings(): nothing to write.
2832 + if ('dynamic' === $this->resolve_delivery_mode($settings)) {
2833 + $this->switch_to_dynamic_delivery($settings, $revision, 'content');
2834 + return;
2835 + }
2836 +
2837 + if ($this->generate_for_regeneration($settings)) {
2083 2838 $this->mark_regeneration_complete($revision);
2084 2839 } else {
2085 2840 // Previously this returned quietly and last_generated simply
2086 2841 // stopped advancing, leaving the site owner with no way to learn
@@ -2085,14 +2840,14 @@
2085 2840 // Previously this returned quietly and last_generated simply
2086 2841 // stopped advancing, leaving the site owner with no way to learn
2087 2842 // the sitemap had stopped updating (#629).
2088 2843 $this->record_regeneration_failure(
2089 - __('The sitemap files could not be written to the site root.', 'thinkrank'),
2844 + $this->write_failure_message(),
2090 2845 'content'
2091 2846 );
2092 2847 }
2093 2848 } catch (\Throwable $e) {
2094 - $this->record_regeneration_failure($e->getMessage(), 'content');
2849 + $this->record_thrown_regeneration_failure($e, 'content');
2095 2850 } finally {
2096 2851 $this->release_generation_lock();
2097 2852 }
2098 2853 }
@@ -2109,8 +2864,26 @@
2109 2864 * @param array $settings Sitemap settings.
2110 2865 * @return bool True when the sitemap files were written.
2111 2866 */
2112 2867 public function generate_and_save(array $settings): bool {
2868 + // Dynamic delivery publishes no files, so writing them here would put a
2869 + // static copy back in the web root for the server to serve in place of
2870 + // the dynamic route. Guarding at each call site left gaps — the
2871 + // snapshot migrator's post-import regeneration had none — so the rule
2872 + // lives with the writing instead.
2873 + //
2874 + // `is_collecting()` is the exception that makes dynamic delivery work
2875 + // at all: render_document() and collect_documents() reach this same
2876 + // method with the writer swapped for a collector, and that is precisely
2877 + // the dynamic build. Only a real write is skipped.
2878 + if (!$this->is_collecting() && 'dynamic' === $this->resolve_delivery_mode($settings)) {
2879 + // Whatever prompted this call changed the sitemap's content, so the
2880 + // rendered copies must not outlive it.
2881 + $this->flush_dynamic_cache();
2882 +
2883 + return true;
2884 + }
2885 +
2113 2886 // Index mode is driven by the use_sitemap_index toggle (not merely by how
2114 2887 // many sitemap_urls happen to be configured). When the toggle is on but
2115 2888 // no child sitemaps are set up yet, synthesize the per-type segmented set
2116 2889 // so we emit a real <sitemapindex> with paginated children instead of a
@@ -2137,9 +2910,13 @@
2137 2910 // names is never touched (#515).
2138 2911 $this->prune_orphaned_segments($settings, [['filename' => basename($primary)]]);
2139 2912 }
2140 2913
2141 - if ($written) {
2914 + // last_generated describes what is on disk. A dynamic render publishes
2915 + // nothing, so advancing it would report a static publication that never
2916 + // happened and would let primary_sitemap_file_exists() callers believe
2917 + // there is a file to serve.
2918 + if ($written && !$this->is_collecting()) {
2142 2919 $settings['last_generated'] = gmdate('c');
2143 2920 $this->save_settings('site', null, $settings);
2144 2921 }
2145 2922
@@ -2146,8 +2923,433 @@
2146 2923 return $written;
2147 2924 }
2148 2925
2149 2926 /**
2927 + * Whether this instance is rendering documents rather than publishing them.
2928 + *
2929 + * @since 2.9.0
2930 + *
2931 + * @return bool
2932 + */
2933 + private function is_collecting(): bool {
2934 + return $this->document_sink !== null;
2935 + }
2936 +
2937 + /**
2938 + * Complete a regeneration that delivers dynamically, retiring stale files.
2939 + *
2940 + * Dynamic delivery renders nothing to disk, but that is only half the job.
2941 + * A web server hands back an existing `/sitemap.xml` without ever loading
2942 + * WordPress, so any file left over from a previous static generation goes on
2943 + * being served forever and {@see \ThinkRank\Frontend\SEO_Manager
2944 + * ::maybe_serve_sitemap()} is never reached. Switching to dynamic while
2945 + * leaving those files in place would therefore appear to do nothing at all.
2946 + *
2947 + * Both transitions matter and they differ:
2948 + *
2949 + * - An explicit switch to `dynamic` happens on a site whose root is usually
2950 + * still writable, so the files can simply be removed.
2951 + * - An `auto` site that becomes read-only cannot remove them, because
2952 + * deleting an entry needs write permission on the directory that holds
2953 + * it. There the stale sitemap really is stuck in front of us, and the
2954 + * honest outcome is a recorded failure naming it rather than a rebuild
2955 + * reported as complete (#754 review).
2956 + *
2957 + * Ownership is tested per file by the shared helper, so another plugin's
2958 + * sitemap at one of our names is never deleted (#515).
2959 + *
2960 + * @since 2.9.0
2961 + *
2962 + * @param array $settings Sitemap settings.
2963 + * @param int $revision Revision this rebuild is completing.
2964 + * @param string $source 'settings' or 'content', for the failure record.
2965 + * @return void
2966 + */
2967 + private function switch_to_dynamic_delivery(array $settings, int $revision, string $source): bool {
2968 + $this->flush_dynamic_cache();
2969 +
2970 + $removal = $this->delete_published_sitemaps($settings);
2971 + $stuck = is_array($removal['failed'] ?? null) ? $removal['failed'] : [];
2972 +
2973 + if (!empty($stuck)) {
2974 + $this->record_regeneration_failure($this->stuck_files_message($stuck), $source);
2975 +
2976 + return false;
2977 + }
2978 +
2979 + $this->mark_regeneration_complete($revision);
2980 +
2981 + return true;
2982 + }
2983 +
2984 + /**
2985 + * Why a stale file left in the web root means the change has not landed.
2986 + *
2987 + * Shared by every path that removes published files, so they cannot
2988 + * describe the same situation differently (#764).
2989 + *
2990 + * The two situations that reach it differ in what WordPress is doing, and
2991 + * the message has to say which. After a switch to dynamic delivery
2992 + * WordPress IS serving the sitemap and the files shadow it. After the
2993 + * sitemap is switched off WordPress serves nothing, so the one message
2994 + * used to tell a site owner who had just disabled the sitemap that it was
2995 + * "being served from WordPress", which is the opposite of what they did.
2996 + *
2997 + * @since 2.10.0
2998 + * @since 2.10.0 Public, so the REST endpoint uses it rather than a copy;
2999 + * takes $sitemap_disabled for the disabled path.
3000 + *
3001 + * @param string[] $stuck Basenames that could not be removed.
3002 + * @param bool $sitemap_disabled True when the files outlived disabling
3003 + * the sitemap rather than a switch to
3004 + * dynamic delivery.
3005 + * @return string
3006 + */
3007 + public function stuck_files_message(array $stuck, bool $sitemap_disabled = false): string {
3008 + if ($sitemap_disabled) {
3009 + return sprintf(
3010 + /* translators: 1: comma-separated file names, 2: absolute path to the WordPress root. */
3011 + __('The sitemap is disabled, but these files are still in the site root and your web server is still serving them: %1$s. They could not be removed because %2$s is not writable. Delete them, or ask your host to make the WordPress root writable.', 'thinkrank'),
3012 + implode(', ', $stuck),
3013 + untrailingslashit(ABSPATH)
3014 + );
3015 + }
3016 +
3017 + return sprintf(
3018 + /* translators: 1: comma-separated file names, 2: absolute path to the WordPress root. */
3019 + __('The sitemap is being served from WordPress, but these files are still in the site root and your web server will keep serving them instead: %1$s. They could not be removed because %2$s is not writable. Delete them, or ask your host to make the WordPress root writable.', 'thinkrank'),
3020 + implode(', ', $stuck),
3021 + untrailingslashit(ABSPATH)
3022 + );
3023 + }
3024 +
3025 + /**
3026 + * What to tell the site owner when publishing the files failed.
3027 + *
3028 + * The old wording stated the symptom and stopped there, so the reported
3029 + * cause was a guess and this reached support as a plugin fault rather than
3030 + * a folder permission (#752, #753). When the root is demonstrably
3031 + * unwritable, say that, and say what to do about it.
3032 + *
3033 + * @since 2.9.0
3034 + *
3035 + * @return string
3036 + */
3037 + private function write_failure_message(): string {
3038 + if (!wp_is_writable(ABSPATH)) {
3039 + return sprintf(
3040 + /* translators: %s: absolute path to the WordPress root. */
3041 + __('The sitemap could not be written because the folder %s is not writable by PHP. Ask your host to make the WordPress root writable, or set Sitemap Delivery to Dynamic to serve the sitemap without writing files.', 'thinkrank'),
3042 + untrailingslashit(ABSPATH)
3043 + );
3044 + }
3045 +
3046 + return __('The sitemap files could not be written to the site root.', 'thinkrank');
3047 + }
3048 +
3049 + /**
3050 + * How this site delivers its sitemap.
3051 + *
3052 + * `auto` is resolved on whether the web root can be written. That is the
3053 + * right signal here (unlike llms.txt, where the question is whether the
3054 + * server applies the .htaccess charset block): a site whose root is
3055 + * read-only cannot publish a sitemap file at all, and before this existed
3056 + * the feature simply failed with "The sitemap files could not be written to
3057 + * the site root." and served nothing (#752).
3058 + *
3059 + * @since 2.9.0
3060 + *
3061 + * @param array|null $settings Sitemap settings (falls back to saved ones).
3062 + * @return string One of 'static' or 'dynamic'. Never 'auto'.
3063 + */
3064 + public function resolve_delivery_mode(?array $settings = null): string {
3065 + $settings = $settings ?? $this->get_settings('site');
3066 + $mode = (string) ($settings['delivery_mode'] ?? 'auto');
3067 +
3068 + if ('static' === $mode || 'dynamic' === $mode) {
3069 + return $mode;
3070 + }
3071 +
3072 + return wp_is_writable(ABSPATH) ? 'static' : 'dynamic';
3073 + }
3074 +
3075 + /**
3076 + * Render one published sitemap document without touching the filesystem.
3077 + *
3078 + * Runs the ordinary build pipeline with the writer swapped for a collector,
3079 + * so the bytes returned here are the bytes the static path would have
3080 + * written. `SitemapDeliveryParityTest` asserts that equivalence rather than
3081 + * trusting it.
3082 + *
3083 + * The whole set is built to answer for one file, because the index can only
3084 + * be assembled from the children that were actually produced. The result is
3085 + * cached per document, so that cost is paid once per change and not once
3086 + * per crawler request.
3087 + *
3088 + * @since 2.9.0
3089 + *
3090 + * @param string $filename Published file name, e.g. 'sitemap.xml'.
3091 + * @param array|null $settings Sitemap settings (falls back to saved ones).
3092 + * @return string|null XML, or null when this site does not publish that name.
3093 + */
3094 + public function render_document(string $filename, ?array $settings = null): ?string {
3095 + $filename = basename($filename);
3096 + $settings = $settings ?? $this->get_settings('site');
3097 +
3098 + if (empty($settings['enabled'])) {
3099 + return null;
3100 + }
3101 +
3102 + $cached = get_transient($this->dynamic_cache_key($filename));
3103 + if (self::ABSENT_MARKER === $cached) {
3104 + return null;
3105 + }
3106 + if (is_string($cached) && '' !== $cached) {
3107 + return $cached;
3108 + }
3109 +
3110 + // A miss builds the whole set, because the index can only be assembled
3111 + // from the children that were actually produced. Caching only the
3112 + // requested document therefore made a crawler walking the index and its
3113 + // children rebuild the entire site's sitemap once per file — every post
3114 + // and taxonomy query repeated N times on a public endpoint (#754
3115 + // review). The set is built once and stored in full.
3116 + return $this->stream_documents($settings, $filename);
3117 + }
3118 +
3119 + /**
3120 + * Build every document, caching each as it is produced, keeping one.
3121 + *
3122 + * A miss has to build the whole set, because the index can only be
3123 + * assembled from the children that were actually produced. It does not have
3124 + * to *hold* the whole set: the static path never keeps more than one page
3125 + * in memory, writing each to disk as it goes, and buffering every
3126 + * document's XML to return one of them undid that on the request path,
3127 + * where a large site's entire sitemap corpus would sit in a single PHP
3128 + * process (#754 review).
3129 + *
3130 + * So the sink writes each document straight to its cache entry and lets it
3131 + * go, retaining only the one this request is answering. Peak retention is
3132 + * one document, whatever the site's size.
3133 + *
3134 + * Concurrency: the first request through takes a short lock and does the
3135 + * work. One that finds the lock held waits a bounded moment for the winner
3136 + * to publish, then builds anyway, because serving a correct sitemap late
3137 + * beats serving none.
3138 + *
3139 + * @since 2.9.0
3140 + *
3141 + * @param array $settings Sitemap settings.
3142 + * @param string $wanted Document this request is answering.
3143 + * @return string|null XML for $wanted, or null when the site does not publish it.
3144 + */
3145 + private function stream_documents(array $settings, string $wanted): ?string {
3146 + $lock = self::DYNAMIC_CACHE_PREFIX . 'lock';
3147 +
3148 + if (!$this->acquire_render_lock($lock)) {
3149 + for ($attempt = 0; $attempt < self::RENDER_LOCK_WAIT_ATTEMPTS; $attempt++) {
3150 + usleep(self::RENDER_LOCK_WAIT_MICROSECONDS);
3151 +
3152 + $cached = get_transient($this->dynamic_cache_key($wanted));
3153 + if (self::ABSENT_MARKER === $cached) {
3154 + return null;
3155 + }
3156 + if (is_string($cached) && '' !== $cached) {
3157 + return $cached;
3158 + }
3159 + }
3160 + }
3161 +
3162 + $kept = null;
3163 + // Names only. Keeping the bodies here would be the very retention this
3164 + // method exists to avoid.
3165 + $produced = [];
3166 +
3167 + $previous = $this->document_sink;
3168 + $this->document_sink = function (string $name, string $xml) use (&$kept, &$produced, $wanted): void {
3169 + $produced[$name] = true;
3170 + set_transient($this->dynamic_cache_key($name), $xml, self::DYNAMIC_CACHE_TTL);
3171 +
3172 + if ($name === $wanted) {
3173 + $kept = $xml;
3174 + }
3175 + };
3176 +
3177 + try {
3178 + $this->generate_and_save($settings);
3179 +
3180 + // Names the configuration lists but this build did not produce get
3181 + // a negative entry, so asking for one again is a cache hit rather
3182 + // than another full rebuild.
3183 + $absent = $this->published_document_names($settings);
3184 +
3185 + // Also the exact name this request asked for: a paginated page past
3186 + // the end of a stem is a legitimate request shape that the base
3187 + // list cannot enumerate, and without an entry it would rebuild on
3188 + // every hit.
3189 + $absent[] = $wanted;
3190 +
3191 + foreach (array_unique($absent) as $name) {
3192 + if (!isset($produced[$name])) {
3193 + set_transient($this->dynamic_cache_key($name), self::ABSENT_MARKER, self::DYNAMIC_CACHE_TTL);
3194 + }
3195 + }
3196 + } finally {
3197 + $this->document_sink = $previous;
3198 + delete_transient($lock);
3199 + }
3200 +
3201 + return $kept;
3202 + }
3203 +
3204 + /**
3205 + * Take the render lock, if it is free.
3206 + *
3207 + * Not atomic across processes, and deliberately so: the fallback for losing
3208 + * a race is duplicated work, never a wrong or missing sitemap, so a
3209 + * heavier primitive would buy nothing here.
3210 + *
3211 + * @since 2.9.0
3212 + *
3213 + * @param string $lock Lock transient name.
3214 + * @return bool True when this request holds the lock.
3215 + */
3216 + private function acquire_render_lock(string $lock): bool {
3217 + if (false !== get_transient($lock)) {
3218 + return false;
3219 + }
3220 +
3221 + set_transient($lock, time(), self::RENDER_LOCK_TTL);
3222 +
3223 + return true;
3224 + }
3225 +
3226 + /**
3227 + * Build every document this site publishes and return them all.
3228 + *
3229 + * Verification and tooling only. This retains the whole set in memory, so
3230 + * it must never be used to answer a request: {@see self::stream_documents()}
3231 + * is the serving path and keeps one document at a time regardless of site
3232 + * size (#754 review). `SitemapDeliveryParityTest` enforces that separation
3233 + * by failing if the request path routes back through here.
3234 + *
3235 + * @since 2.9.0
3236 + *
3237 + * @param array $settings Sitemap settings.
3238 + * @return array<string,string> Filename => XML.
3239 + */
3240 + public function collect_documents(array $settings): array {
3241 + $documents = [];
3242 +
3243 + $previous = $this->document_sink;
3244 + $this->document_sink = static function (string $name, string $xml) use (&$documents): void {
3245 + $documents[$name] = $xml;
3246 + };
3247 +
3248 + try {
3249 + $this->generate_and_save($settings);
3250 + } finally {
3251 + $this->document_sink = $previous;
3252 + }
3253 +
3254 + return $documents;
3255 + }
3256 +
3257 + /**
3258 + * The file names this site publishes, without building their contents.
3259 + *
3260 + * Used by the request router to decide whether a URL is ours before doing
3261 + * any work. Cheap: it reads the configured child list rather than querying
3262 + * for entries.
3263 + *
3264 + * @since 2.9.0
3265 + *
3266 + * @param array|null $settings Sitemap settings (falls back to saved ones).
3267 + * @return string[] File names, including paginated pages that may exist.
3268 + */
3269 + public function published_document_names(?array $settings = null): array {
3270 + $settings = $settings ?? $this->get_settings('site');
3271 + $resolved = $this->maybe_promote_to_index($settings);
3272 +
3273 + $names = [$this->get_primary_sitemap_filename($settings), 'local-sitemap.xml'];
3274 +
3275 + foreach ((array) ($resolved['sitemap_urls'] ?? []) as $child) {
3276 + if (!is_array($child) || empty($child['enabled'])) {
3277 + continue;
3278 + }
3279 +
3280 + $path = (string) wp_parse_url((string) ($child['url'] ?? ''), PHP_URL_PATH);
3281 + if ('' !== $path) {
3282 + $names[] = basename($path);
3283 + }
3284 + }
3285 +
3286 + return array_values(array_unique(array_filter($names)));
3287 + }
3288 +
3289 + /**
3290 + * Does this site publish a document under that name?
3291 + *
3292 + * Not a plain membership test against {@see self::published_document_names()}:
3293 + * that lists the configured children, and a child over the per-file URL cap
3294 + * is split into `<stem>-2.xml`, `<stem>-3.xml` and so on, with every page
3295 + * listed in the index. Gating the request router on the base list alone
3296 + * therefore 404'd exactly the pages the index points at, which is worse than
3297 + * not serving them at all.
3298 + *
3299 + * Page counts are not knowable without building, so the stem is what is
3300 + * matched; a page that does not exist is answered by the build finding
3301 + * nothing for it, and is then cached as absent.
3302 + *
3303 + * @since 2.9.0
3304 + *
3305 + * @param string $name Requested file name.
3306 + * @param array|null $settings Sitemap settings (falls back to saved ones).
3307 + * @return bool
3308 + */
3309 + public function publishes_document_name(string $name, ?array $settings = null): bool {
3310 + $names = $this->published_document_names($settings);
3311 +
3312 + if (in_array($name, $names, true)) {
3313 + return true;
3314 + }
3315 +
3316 + if (!preg_match('/^(.*)-\d+\.xml$/i', $name, $m)) {
3317 + return false;
3318 + }
3319 +
3320 + return in_array($m[1] . '.xml', $names, true);
3321 + }
3322 +
3323 + /**
3324 + * Transient key for a rendered document.
3325 + *
3326 + * @since 2.9.0
3327 + *
3328 + * @param string $filename Published file name.
3329 + * @return string
3330 + */
3331 + private function dynamic_cache_key(string $filename): string {
3332 + return self::DYNAMIC_CACHE_PREFIX . md5($filename);
3333 + }
3334 +
3335 + /**
3336 + * Drop every cached dynamic document.
3337 + *
3338 + * Called from the same places that mark the static files stale, so the two
3339 + * delivery modes invalidate on identical triggers.
3340 + *
3341 + * @since 2.9.0
3342 + *
3343 + * @return void
3344 + */
3345 + public function flush_dynamic_cache(): void {
3346 + foreach ($this->published_document_names() as $name) {
3347 + delete_transient($this->dynamic_cache_key($name));
3348 + }
3349 + }
3350 +
3351 + /**
2150 3352 * Resolve index-vs-single mode, synthesizing child sitemaps when needed.
2151 3353 *
2152 3354 * - When use_sitemap_index is on but no child sitemaps are configured, build
2153 3355 * the per-type segmented set so a real <sitemapindex> is produced (#127).
@@ -2342,8 +3544,18 @@
2342 3544 // File validation failed - error details available in exception
2343 3545 return false;
2344 3546 }
2345 3547
3548 + // Dynamic delivery: hand the document to the collector instead of the
3549 + // filesystem. Reported as published, because for this run it is — the
3550 + // caller's success/failure bookkeeping and the index assembly both key
3551 + // off this return value.
3552 + if ($this->document_sink !== null) {
3553 + ($this->document_sink)($filename, $sitemap_xml);
3554 +
3555 + return true;
3556 + }
3557 +
2346 3558 $sitemap_path = ABSPATH . $filename;
2347 3559
2348 3560 // Use WordPress filesystem API for better security
2349 3561 global $wp_filesystem;
@@ -2401,13 +3613,55 @@
2401 3613
2402 3614 // Public custom post types each get a child sitemap (parity with the
2403 3615 // "complete" preset and with Rank Math, which lists every public CPT).
2404 3616 foreach (get_post_types(['public' => true, '_builtin' => false], 'names') as $cpt) {
2405 - if ($this->should_include_post_type($cpt)) {
2406 - $urls[] = $this->build_child_sitemap_entry($cpt, $pattern);
3617 + if (!$this->should_include_post_type($cpt)) {
3618 + continue;
2407 3619 }
3620 +
3621 + // ...and the per-content-type sitemap switch (#660).
3622 + // get_enabled_post_types() already honours it, but this list is what
3623 + // index mode builds its children from — so without the same test a
3624 + // CPT the user had switched off still got its own child sitemap,
3625 + // created and streamed in full. The flags live in $inclusions, which
3626 + // is the settings array these children are derived from.
3627 + if (!\ThinkRank\SEO\Content_Type_Settings::is_included_in_sitemap('post_type', $cpt, $inclusions)) {
3628 + continue;
3629 + }
3630 +
3631 + $urls[] = $this->build_child_sitemap_entry($cpt, $pattern);
2408 3632 }
2409 3633
3634 + // Public custom taxonomies get the same treatment (#690). Flat mode has
3635 + // always walked them through get_enabled_taxonomies(); index mode built
3636 + // its children from the list above and never consulted a taxonomy at
3637 + // all, so every custom-taxonomy archive silently vanished from the
3638 + // sitemap the moment a site switched modes — and the per-taxonomy switch
3639 + // the matrix writes had nothing to act on. Same two tests the post-type
3640 + // walk applies, in the same order.
3641 + $taken = array_column($urls, 'type');
3642 +
3643 + foreach (get_taxonomies(['public' => true, '_builtin' => false], 'names') as $taxonomy) {
3644 + if (!$this->should_include_taxonomy($taxonomy)) {
3645 + continue;
3646 + }
3647 +
3648 + if (!\ThinkRank\SEO\Content_Type_Settings::is_included_in_sitemap('taxonomy', $taxonomy, $inclusions)) {
3649 + continue;
3650 + }
3651 +
3652 + // Post types and taxonomies are separate registries, so a site can
3653 + // hold both a `foo` post type and a `foo` taxonomy. They would
3654 + // resolve to one filename, and stream_type_entries() answers post
3655 + // types first, so the second child would list the first one's file
3656 + // twice in the index rather than adding anything.
3657 + if (in_array($taxonomy, $taken, true)) {
3658 + continue;
3659 + }
3660 +
3661 + $urls[] = $this->build_child_sitemap_entry($taxonomy, $pattern);
3662 + }
3663 +
2410 3664 return $urls;
2411 3665 }
2412 3666
2413 3667 /**
@@ -2554,8 +3808,48 @@
2554 3808 if (post_type_exists($type) && !$this->should_include_post_type($type)) {
2555 3809 continue;
2556 3810 }
2557 3811
3812 + // Same for the matrix switch: a child list saved before the user
3813 + // excluded this content type still names it, and regenerating from
3814 + // that list would rewrite the file they asked not to have. Built-in
3815 + // aggregates ('posts', 'pages', ...) are not post type names, so
3816 + // post_type_exists() keeps this to real custom post types.
3817 + if (post_type_exists($type)
3818 + && !\ThinkRank\SEO\Content_Type_Settings::is_included_in_sitemap('post_type', $type, $settings)) {
3819 + continue;
3820 + }
3821 +
3822 + // The taxonomy counterpart of the two guards above (#690). A child
3823 + // list saved while a taxonomy was still included keeps naming it, so
3824 + // without this, excluding one in the matrix would still rewrite and
3825 + // re-list the file the user asked not to have. The built-in
3826 + // aggregates are named 'categories'/'tags' rather than
3827 + // 'category'/'post_tag', so taxonomy_exists() leaves them to the
3828 + // inclusion-flag check below.
3829 + $child_taxonomy = self::CHILD_TYPE_ALIASES[$type] ?? $type;
3830 + if (taxonomy_exists($child_taxonomy)
3831 + && (!$this->should_include_taxonomy($child_taxonomy)
3832 + || !\ThinkRank\SEO\Content_Type_Settings::is_included_in_sitemap('taxonomy', $child_taxonomy, $settings))) {
3833 + continue;
3834 + }
3835 +
3836 + // The built-in aggregates carry their switch in the inclusion flag
3837 + // rather than under a post type name, and they are the four most
3838 + // people actually use. build_segmented_sitemap_urls() drops a child
3839 + // whose flag is empty; regenerating from a list saved while it was
3840 + // still on has to make the same decision, or turning Posts, Pages,
3841 + // Categories or Tags off in the matrix rewrites and re-lists the
3842 + // very file it was asked to remove.
3843 + // An absent flag means "not configured", which every other reader
3844 + // treats as included; only a flag that is present and off excludes.
3845 + $aggregate_flag = array_search($type, self::INCLUSION_CHILD_TYPES, true);
3846 + if ($aggregate_flag !== false
3847 + && array_key_exists($aggregate_flag, $settings)
3848 + && empty($settings[$aggregate_flag])) {
3849 + continue;
3850 + }
3851 +
2558 3852 try {
2559 3853 // The single "general"/"WordPress" sitemap is one un-paginated file
2560 3854 // (there is no index to reference extra pages); it no longer drops
2561 3855 // overflow URLs.
@@ -2591,12 +3885,30 @@
2591 3885 $buffer = [];
2592 3886 }
2593 3887 }
2594 3888
2595 - // Flush the trailing partial page, or a single empty page when the
2596 - // type had no entries at all (parity with the previous behavior of
2597 - // always writing at least one page per configured child).
2598 - if (!empty($buffer) || $page === 0) {
3889 + // Flush the trailing partial page.
3890 + //
3891 + // A type that produced nothing writes no page at all in index
3892 + // mode (#836). It used to write one empty urlset and list it in
3893 + // the index, so a crawler was asked to fetch a file that
3894 + // answers with no URLs — Search Console reports an empty
3895 + // sitemap referenced from an index as a warning, and the fetch
3896 + // is wasted on every pass. On this site three of eleven
3897 + // children were empty: a post type with nothing published and
3898 + // two taxonomies with no terms.
3899 + //
3900 + // Single-file mode still writes its one page even when empty,
3901 + // because that file IS the site's /sitemap.xml and a 404 there
3902 + // is worse than an empty urlset. Nothing lists it, so it costs
3903 + // no crawl budget.
3904 + //
3905 + // Skipping the write also keeps the filename out of
3906 + // $results['sitemaps_generated'], which is what
3907 + // prune_orphaned_segments() treats as "written this run" — so
3908 + // a type that empties after previously publishing has its
3909 + // stale file deleted rather than left serving.
3910 + if (!empty($buffer) || ($page === 0 && $index_config === null)) {
2599 3911 $page++;
2600 3912 $this->write_sitemap_page($this->paginate_url($sitemap_config['url'], $page), $this->wrap_urlset($buffer, $settings, $image_ns), $type, count($buffer), $results, $index_children);
2601 3913 }
2602 3914
@@ -2675,8 +3987,15 @@
2675 3987 * @param array $generated Entries from $results['sitemaps_generated'].
2676 3988 * @return string[] Basenames removed.
2677 3989 */
2678 3990 private function prune_orphaned_segments(array $settings, array $generated): array {
3991 + // Rendering for a request, not publishing: there is nothing on disk
3992 + // this run owns, and a dynamic render must never delete the files a
3993 + // site's previous static mode left behind.
3994 + if ($this->is_collecting()) {
3995 + return [];
3996 + }
3997 +
2679 3998 $kept = [];
2680 3999 foreach ($generated as $entry) {
2681 4000 if (!empty($entry['filename'])) {
2682 4001 $kept[strtolower((string) $entry['filename'])] = true;
@@ -2778,9 +4097,9 @@
2778 4097 // publishes under too (this method mirrors it deliberately), so on
2779 4098 // a migrated site the file at that path may never have been ours
2780 4099 // to delete (#515).
2781 4100 $path = ABSPATH . 'local-sitemap.xml';
2782 - if (file_exists($path) && $this->webroot_sitemap_is_ours($path, $settings)) {
4101 + if (!$this->is_collecting() && file_exists($path) && $this->webroot_sitemap_is_ours($path, $settings)) {
2783 4102 wp_delete_file($path);
2784 4103 }
2785 4104 return false;
2786 4105 }
@@ -2802,8 +4121,25 @@
2802 4121 *
2803 4122 * @since 1.15.x
2804 4123 * @return array Zero or one URL entry
2805 4124 */
4125 + /**
4126 + * Does this site publish a local business sitemap right now?
4127 + *
4128 + * The same gate {@see self::regenerate_local_sitemap()} applies, asked
4129 + * without writing anything. Callers that need to know whether the document
4130 + * exists must not test the filesystem: under dynamic delivery it is served
4131 + * from PHP and there is no file, which is how `local-sitemap.xml` came to be
4132 + * dropped from robots.txt on exactly those sites (#752).
4133 + *
4134 + * @since 2.9.0
4135 + *
4136 + * @return bool True when the local sitemap has content to publish.
4137 + */
4138 + public function publishes_local_sitemap(): bool {
4139 + return !empty($this->collect_local_entries());
4140 + }
4141 +
2806 4142 private function collect_local_entries(): array {
2807 4143 if (!class_exists('ThinkRank\\SEO\\Site_Identity_Manager')) {
2808 4144 require_once THINKRANK_PLUGIN_DIR . 'includes/seo/class-site-identity-manager.php';
2809 4145 }
@@ -2875,17 +4211,43 @@
2875 4211 case 'products':
2876 4212 $entries = post_type_exists('product') ? $this->collect_post_entries_iter(['product'], $settings) : [];
2877 4213 return ['entries' => $entries, 'image_ns' => true];
2878 4214
2879 - case 'product_categories':
2880 - $entries = taxonomy_exists('product_cat') ? $this->collect_taxonomy_entries_iter('product_cat', $settings) : [];
2881 - return ['entries' => $entries, 'image_ns' => false];
4215 + default:
4216 + // Resolve a preset's display name to the object it streams, so
4217 + // 'product_categories' is an ordinary taxonomy child rather than
4218 + // a case of its own (#690).
4219 + $alias = self::CHILD_TYPE_ALIASES[$type] ?? null;
4220 + $object = $alias ?? $type;
2882 4221
2883 - default:
2884 4222 $custom_post_types = get_post_types(['public' => true, '_builtin' => false], 'names');
2885 - if (in_array($type, $custom_post_types, true)) {
2886 - return ['entries' => $this->collect_post_entries_iter([$type], $settings), 'image_ns' => true];
4223 + if (in_array($object, $custom_post_types, true)) {
4224 + return ['entries' => $this->collect_post_entries_iter([$object], $settings), 'image_ns' => true];
2887 4225 }
4226 +
4227 + // Custom taxonomies reach index mode here, streamed through the
4228 + // same iterator flat mode uses so the two modes emit identical
4229 + // URLs for the same settings.
4230 + if (taxonomy_exists($object) && $this->should_include_taxonomy($object)) {
4231 + return ['entries' => $this->collect_taxonomy_entries_iter($object, $settings), 'image_ns' => false];
4232 + }
4233 +
4234 + // An aliased child whose object is gone (WooCommerce deactivated)
4235 + // keeps writing the empty file it always wrote. Returning null
4236 + // here would hand it the whole-site fallback instead, dumping
4237 + // every URL on the site into a file named for products.
4238 + //
4239 + // A registered taxonomy this generator will not emit
4240 + // (`post_format`, `nav_menu`, a non-public one) needs the same
4241 + // answer for the same reason. generate_multiple_sitemaps()
4242 + // skips those before they reach here, so nothing takes this
4243 + // path today — but it is the one branch where falling through
4244 + // to null is silently catastrophic rather than merely wrong,
4245 + // and the guard keeping it unreachable lives in another method.
4246 + if ($alias !== null || taxonomy_exists($object)) {
4247 + return ['entries' => [], 'image_ns' => false];
4248 + }
4249 +
2888 4250 return null;
2889 4251 }
2890 4252 }
2891 4253
@@ -2938,8 +4300,13 @@
2938 4300 * @param array $settings Sitemap settings, for the ownership test.
2939 4301 * @return void
2940 4302 */
2941 4303 private function cleanup_stale_pages(string $base_url, int $current_pages, array $settings): void {
4304 + // See prune_orphaned_segments(): a dynamic render deletes nothing.
4305 + if ($this->is_collecting()) {
4306 + return;
4307 + }
4308 +
2942 4309 $filename = basename(wp_parse_url($base_url, PHP_URL_PATH));
2943 4310 if (!preg_match('/^(.*)\.xml$/i', $filename, $m)) {
2944 4311 return;
2945 4312 }
@@ -3064,9 +4431,9 @@
3064 4431 * @param array $settings Sitemap settings
3065 4432 * @return string Sitemap index XML
3066 4433 */
3067 4434 private function generate_sitemap_index(array $children, array $settings): string {
3068 - $xml = $this->xml_prolog($settings, 'sitemap-index.xsl');
4435 + $xml = $this->xml_prolog($settings, 'index');
3069 4436 $xml .= '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' . "\n";
3070 4437
3071 4438 $site_url = home_url();
3072 4439
@@ -3079,9 +4446,9 @@
3079 4446 $sitemap_url = $site_url . $sitemap_url;
3080 4447 }
3081 4448
3082 4449 $xml .= " <sitemap>\n";
3083 - $xml .= " <loc>" . esc_url($sitemap_url) . "</loc>\n";
4450 + $xml .= " <loc>" . esc_url(Url_Scheme::apply($sitemap_url)) . "</loc>\n";
3084 4451 $xml .= " <lastmod>" . gmdate('c') . "</lastmod>\n";
3085 4452 $xml .= " </sitemap>\n";
3086 4453 }
3087 4454