← All changes | includes/seo/class-site-identity-manager.php +258 -3 2.13.0 → 2.14.1 View file →

		@@ -318,8 +318,36 @@
318	318	* @var bool
319	319	*/
320	320	private static bool $icon_sizes_listener_registered = false;
321	321
	322	+ /**
	323	+ * Whether the robots.txt resync listener is registered for this request.
	324	+ *
	325	+ * @since 2.14.0
	326	+ * @var bool
	327	+ */
	328	+ private static bool $robots_sync_listener_registered = false;
	329	+
	330	+ /**
	331	+ * Flag set when a plugin change may have altered the sitemap set.
	332	+ *
	333	+ * @since 2.14.0
	334	+ * @var string
	335	+ */
	336	+ public const ROBOTS_RESYNC_OPTION = 'thinkrank_robots_txt_resync_pending';
	337	+
	338	+ /**
	339	+ * Flag set when a plugin change may have altered the sitemap index.
	340	+ *
	341	+ * Separate from ROBOTS_RESYNC_OPTION because the two files exist
	342	+ * independently: that flag is only set when a physical robots.txt exists,
	343	+ * and the sitemap index needs rebuilding whether or not it does.
	344	+ *
	345	+ * @since 2.15.0
	346	+ * @var string
	347	+ */
	348	+ public const SITEMAP_RESYNC_OPTION = 'thinkrank_sitemap_contributors_changed';
	349	+
322	350	public function __construct() {
323	351	parent::__construct('site_identity');
324	352
325	353	if (!self::$icon_sizes_listener_registered) {
		@@ -328,11 +356,130 @@
328	356	// Admin only: resizing is not front-end work, and admin traffic is
329	357	// enough to run a one-time backfill promptly.
330	358	add_action('admin_init', [self::class, 'maybe_backfill_icon_sizes']);
331	359	}
	360	+
	361	+ if (!self::$robots_sync_listener_registered) {
	362	+ self::$robots_sync_listener_registered = true;
	363	+
	364	+ // A physical robots.txt bypasses PHP entirely, so composing the
	365	+ // Sitemap block at render time fixes the served output only on
	366	+ // sites with no file. Activating or deactivating a sitemap
	367	+ // contributor changes the set, and until #835 nothing rewrote the
	368	+ // file: the deactivated plugin's sitemap stayed advertised, serving
	369	+ // HTML to anything that followed it.
	370	+ add_action('activated_plugin', [self::class, 'flag_robots_txt_resync']);
	371	+ add_action('deactivated_plugin', [self::class, 'flag_robots_txt_resync']);
	372	+ add_action('init', [self::class, 'maybe_resync_robots_txt'], 99);
	373	+
	374	+ // The sitemap index is a second static file listing the same
	375	+ // contributors, with its own rebuild path. #835 / #859 resynced
	376	+ // robots.txt only, so after Pro was deactivated the index kept
	377	+ // advertising news-sitemap.xml, which then served the home page
	378	+ // as HTML (#920).
	379	+ add_action('activated_plugin', [self::class, 'flag_sitemap_resync']);
	380	+ add_action('deactivated_plugin', [self::class, 'flag_sitemap_resync']);
	381	+ add_action('init', [self::class, 'maybe_resync_sitemap'], 99);
	382	+ }
332	383	}
333	384
334	385	/**
	386	+ * Note that the set of sitemap contributors may have changed.
	387	+ *
	388	+ * Deliberately unconditional about which plugin: a contributor is anything
	389	+ * hooking `thinkrank_additional_sitemaps`, which is resolved at runtime and
	390	+ * cannot be inspected for a plugin that is on its way out.
	391	+ *
	392	+ * The rewrite is not done here. `deactivated_plugin` fires inside the
	393	+ * request that deactivated it, while that plugin's filters are still
	394	+ * attached, so rendering now still sees the sitemap that is going away —
	395	+ * measured, not assumed: the first version of this fix wrote the
	396	+ * deactivated plugin's sitemap straight back into the file. The next
	397	+ * request has the real plugin set loaded, so the work waits for it.
	398	+ *
	399	+ * @since 2.14.0
	400	+ * @return void
	401	+ */
	402	+ public static function flag_robots_txt_resync(): void {
	403	+ if (!file_exists(ABSPATH . 'robots.txt')) {
	404	+ return;
	405	+ }
	406	+
	407	+ update_option(self::ROBOTS_RESYNC_OPTION, 1, false);
	408	+ }
	409	+
	410	+ /**
	411	+ * Rewrite the physical robots.txt once, on the request after a change.
	412	+ *
	413	+ * @since 2.14.0
	414	+ * @return void
	415	+ */
	416	+ public static function maybe_resync_robots_txt(): void {
	417	+ if (!get_option(self::ROBOTS_RESYNC_OPTION)) {
	418	+ return;
	419	+ }
	420	+
	421	+ // Cleared first, so a render that fatals cannot retry on every request
	422	+ // for the rest of the site's life.
	423	+ delete_option(self::ROBOTS_RESYNC_OPTION);
	424	+
	425	+ if (!file_exists(ABSPATH . 'robots.txt')) {
	426	+ return;
	427	+ }
	428	+
	429	+ (new self())->sync_robots_txt_file();
	430	+ }
	431	+
	432	+ /**
	433	+ * Note that the set of sitemap contributors may have changed.
	434	+ *
	435	+ * Unconditional, unlike flag_robots_txt_resync(): the sitemap files exist
	436	+ * whether or not robots.txt does. The rebuild waits for the next request
	437	+ * for the same reason as the robots.txt one, since `deactivated_plugin`
	438	+ * still runs with the outgoing plugin's `thinkrank_additional_sitemaps`
	439	+ * callback attached.
	440	+ *
	441	+ * @since 2.15.0
	442	+ * @return void
	443	+ */
	444	+ public static function flag_sitemap_resync(): void {
	445	+ update_option(self::SITEMAP_RESYNC_OPTION, 1, false);
	446	+ }
	447	+
	448	+ /**
	449	+ * Queue a sitemap rebuild once, on the request after a contributor change.
	450	+ *
	451	+ * Goes through schedule_regeneration(), the debounced and lock-protected
	452	+ * path a sitemap settings save uses, so a burst of plugin changes (a bulk
	453	+ * deactivate, say) still produces one rebuild. That path also drops the
	454	+ * cached dynamic documents, so sites serving the sitemap from PHP drop the
	455	+ * entry as well.
	456	+ *
	457	+ * @since 2.15.0
	458	+ * @return void
	459	+ */
	460	+ public static function maybe_resync_sitemap(): void {
	461	+ if (!get_option(self::SITEMAP_RESYNC_OPTION)) {
	462	+ return;
	463	+ }
	464	+
	465	+ // Cleared first, so a rebuild that fatals cannot be retried on every
	466	+ // request for the rest of the site's life.
	467	+ delete_option(self::SITEMAP_RESYNC_OPTION);
	468	+
	469	+ $generator = new Sitemap_Generator(false);
	470	+ $settings = $generator->get_settings('site');
	471	+
	472	+ // A disabled sitemap has no files to correct. Enabling it later builds
	473	+ // from the contributors present at that time.
	474	+ if (empty($settings['enabled'])) {
	475	+ return;
	476	+ }
	477	+
	478	+ $generator->schedule_regeneration();
	479	+ }
	480	+
	481	+ /**
335	482	* Save settings, then refresh what a new canonical scheme invalidates.
336	483	*
337	484	* The static sitemap files are written with the scheme in force when they
338	485	* were built, and nothing else rebuilds them until a post or term changes.
		@@ -814,9 +961,20 @@
814	961	// here rather than stored: the textarea holds the user's body, with
815	962	// the fenced block stripped out of every read and re-applied on every
816	963	// render. A site-wide block already disallows everyone, so adding the
817	964	// per-agent group there would be noise restating the same refusal.
	965	+ // Composed here rather than read from storage, for the same reason as
	966	+ // the AI block below: the set of sitemaps an install publishes is a
	967	+ // runtime fact. `robots_txt_content` is a snapshot of it taken at the
	968	+ // last save, and nothing invalidated that snapshot, so deactivating a
	969	+ // sitemap provider left its URL advertised and serving HTML (#835).
	970	+ // Composing it on every render means the advertisement agrees with what
	971	+ // the install publishes, in both directions, with no cache to expire.
818	972	if (!$fully_blocked) {
	973	+ $body = $this->apply_sitemap_block($body);
	974	+ }
	975	+
	976	+ if (!$fully_blocked) {
819	977	$body = $this->apply_ai_crawler_block($body, $settings);
820	978	}
821	979
822	980	if ($body === '') {
		@@ -826,8 +984,87 @@
826	984	return $this->robots_txt_header() . $body . "\n";
827	985	}
828	986
829	987	/**
	988	+ * Replace the generated Sitemap block with the one this install publishes.
	989	+ *
	990	+ * @since 2.14.0
	991	+ * @param string $body Robots.txt body, without the header.
	992	+ * @return string
	993	+ */
	994	+ private function apply_sitemap_block(string $body): string {
	995	+ $stripped = $this->strip_generated_sitemap_block($body);
	996	+ $urls = $this->get_sitemap_urls_for_robots();
	997	+
	998	+ if (empty($urls)) {
	999	+ return $stripped;
	1000	+ }
	1001	+
	1002	+ $block = '';
	1003	+ foreach ($urls as $url) {
	1004	+ $block .= 'Sitemap: ' . $url . "\n";
	1005	+ }
	1006	+
	1007	+ if ('' === trim($stripped)) {
	1008	+ return trim($block);
	1009	+ }
	1010	+
	1011	+ // The grammar build_robots_txt_content() writes: one blank line before
	1012	+ // the block, none inside it. A blank line terminates a record in the
	1013	+ // robots.txt grammar, so a line between every directive is invalid.
	1014	+ return rtrim($stripped) . "\n\n" . trim($block);
	1015	+ }
	1016	+
	1017	+ /**
	1018	+ * Remove the plugin-written Sitemap block from a stored body.
	1019	+ *
	1020	+ * Only the trailing run of `Sitemap:` lines is removed, which is the exact
	1021	+ * shape `build_robots_txt_content()` writes: a blank line, then nothing but
	1022	+ * `Sitemap:` lines to the end of the body. A `Sitemap:` line anywhere else
	1023	+ * was typed by the site owner and is left exactly where they put it, which
	1024	+ * is why this cannot simply strip every matching line.
	1025	+ *
	1026	+ * @since 2.14.0
	1027	+ * @param string $body Robots.txt body.
	1028	+ * @return string
	1029	+ */
	1030	+ private function strip_generated_sitemap_block(string $body): string {
	1031	+ $lines = preg_split('/\R/', $body);
	1032	+
	1033	+ if (!is_array($lines)) {
	1034	+ return $body;
	1035	+ }
	1036	+
	1037	+ $cut = count($lines);
	1038	+
	1039	+ // Walk back over the trailing block: sitemap lines, and the blank lines
	1040	+ // that separate or pad it. Anything else ends the block.
	1041	+ for ($i = count($lines) - 1; $i >= 0; $i--) {
	1042	+ $line = trim($lines[$i]);
	1043	+
	1044	+ if ('' === $line) {
	1045	+ $cut = $i;
	1046	+ continue;
	1047	+ }
	1048	+
	1049	+ if (0 === stripos($line, 'sitemap:')) {
	1050	+ $cut = $i;
	1051	+ continue;
	1052	+ }
	1053	+
	1054	+ break;
	1055	+ }
	1056	+
	1057	+ if ($cut >= count($lines)) {
	1058	+ return $body;
	1059	+ }
	1060	+
	1061	+ // Nothing but sitemap lines in the whole body means there is no owner
	1062	+ // content to keep.
	1063	+ return rtrim(implode("\n", array_slice($lines, 0, $cut)));
	1064	+ }
	1065	+
	1066	+ /**
830	1067	* Resolve the robots.txt actually served to crawlers, with its origin.
831	1068	*
832	1069	* Lets an API/MCP consumer see the effective output without crawling the
833	1070	* URL. Mirrors serving precedence: a physical robots.txt in the web root is
		@@ -3753,11 +3990,29 @@
3753	3990	}
3754	3991	}
3755	3992
3756	3993	if ($index_url !== '') {
3757		- // The index alone — it covers the children and, on a segmented
3758		- // install, the local business sitemap too.
3759		- return [$index_url];
	3994	+ // The index covers the children and, on a segmented install,
	3995	+ // the local business sitemap too.
	3996	+ //
	3997	+ // It does not cover a sitemap contributed through
	3998	+ // `thinkrank_additional_sitemaps`: the index is built by this
	3999	+ // plugin's own generator and never lists them. Returning the
	4000	+ // index alone therefore left a contributed sitemap with no
	4001	+ // discovery path at all — absent from robots.txt and absent
	4002	+ // from the index — so Pro's News sitemap was unreachable on any
	4003	+ // install with the index enabled, which is the default (#835).
	4004	+ $contributed = [];
	4005	+
	4006	+ foreach (\ThinkRank\SEO\Sitemap_Generator::additional_sitemaps() as $path) {
	4007	+ $url = home_url($path);
	4008	+
	4009	+ if ($url !== $index_url && !in_array($url, $contributed, true)) {
	4010	+ $contributed[] = $url;
	4011	+ }
	4012	+ }
	4013	+
	4014	+ return array_merge([$index_url], $contributed);
3760	4015	}
3761	4016
3762	4017	// Fallback to default if no URLs found
3763	4018	if (empty($sitemap_urls)) {