PluginProbe
ThinkRank AI SEO – AI SEO Plugin for WordPress: Schema, XML Sitemaps, Meta Tags, Search Console & Local SEO / 2.14.1
ThinkRank AI SEO – AI SEO Plugin for WordPress: Schema, XML Sitemaps, Meta Tags, Search Console & Local SEO v2.14.1
2.14.2 2.14.1 2.14.0 2.13.0 2.12.0 2.11.0 2.10.0 2.9.0 2.8.0 2.7.0 2.6.0 2.5.0 2.4.0 2.3.0 2.2.0 2.1.1 2.1.0 2.0.2 2.0.1 2.0.0 1.32.0 1.31.0 1.30.0 1.29.0 1.28.0 All 57 releases
← All changes | includes/seo/class-site-identity-manager.php +1225 -61 1.27.0 → 2.14.1 View file →
@@ -15,8 +15,13 @@
15 15 declare(strict_types=1);
16 16
17 17 namespace ThinkRank\SEO;
18 18
19 +// Prevent direct access
20 +if (!defined('ABSPATH')) {
21 + exit;
22 +}
23 +
19 24 // Ensure dependencies are loaded
20 25 if (!class_exists('ThinkRank\\SEO\\Abstract_SEO_Manager')) {
21 26 require_once THINKRANK_PLUGIN_DIR . 'includes/seo/class-abstract-seo-manager.php';
22 27 }
@@ -35,8 +40,36 @@
35 40 */
36 41 class Site_Identity_Manager extends Abstract_SEO_Manager {
37 42
38 43 /**
44 + * The per-context title formats as ThinkRank ships them.
45 + *
46 + * These are not in get_default_settings(): the admin screen seeds them on
47 + * first save, so on a real install they are stored values, indistinguishable
48 + * from a template the user typed. The migration needs to tell those two
49 + * apart — it may overwrite a shipped default with an imported template, and
50 + * must never overwrite a choice the user made — so this is the record of
51 + * what "untouched" looks like.
52 + *
53 + * Keep in step with getDefaultSettings() in
54 + * src/admin/components/essential-seo/SiteIdentityTab.js. SiteIdentityTitleFormatDefaultsTest
55 + * fails when the two drift.
56 + *
57 + * @since 2.8.0
58 + * @var array<string, string>
59 + */
60 + public const TITLE_FORMAT_DEFAULTS = [
61 + 'homepage_title' => '%site_title% %sep% %site_description%',
62 + 'post_title' => '%post_title% %sep% %site_title%',
63 + 'page_title' => '%page_title% %sep% %site_title%',
64 + 'category_title' => '%category_title% %sep% %site_title%',
65 + 'tag_title' => '%tag_title% %sep% %site_title%',
66 + 'author_title' => '%author_name% %sep% %site_title%',
67 + 'search_title' => 'Search Results for "%search_term%" %sep% %site_title%',
68 + 'archive_title' => '%archive_title% %sep% %site_title%',
69 + ];
70 +
71 + /**
39 72 * WordPress filesystem instance
40 73 *
41 74 * @since 1.0.0
42 75 * @var \WP_Filesystem_Base|null
@@ -240,13 +273,494 @@
240 273 * Constructor
241 274 *
242 275 * @since 1.0.0
243 276 */
277 + /**
278 + * The square derivatives wp_site_icon() asks for.
279 + *
280 + * Core generates these only through its own Site Icon crop flow, so an
281 + * image chosen as a ThinkRank favicon straight from the media library has
282 + * none of them and every sizes="" declaration is a near miss (#571).
283 + *
284 + * @since 2.3.1
285 + * @var int[]
286 + */
287 + public const ICON_SIZES = [32, 180, 192, 270];
288 +
289 + /**
290 + * Transient holding resolved icon URLs, keyed by configured URL and size.
291 + *
292 + * The site-icon filter runs in wp_head on every FRONT-END request, and
293 + * resolving a URL to its attachment costs an uncached postmeta query. The
294 + * mapping only changes when the icon setting does, so it is cached here and
295 + * dropped on save.
296 + *
297 + * @since 2.3.1
298 + * @var string
299 + */
300 + public const ICON_URL_TRANSIENT = 'thinkrank_site_icon_urls';
301 +
302 + /**
303 + * Marker for the one-time derivative backfill on existing installs.
304 + *
305 + * @since 2.3.1
306 + * @var string
307 + */
308 + public const ICON_BACKFILL_OPTION = 'thinkrank_site_icon_sizes_backfilled';
309 +
310 + /**
311 + * Whether the icon-derivative listener has been registered this request.
312 + *
313 + * Static because `thinkrank_seo_settings_saved` is a global hook — one
314 + * listener serves every instance, and this class is constructed on the
315 + * front end as well as in admin.
316 + *
317 + * @since 2.3.1
318 + * @var bool
319 + */
320 + private static bool $icon_sizes_listener_registered = false;
321 +
322 + /**
323 + * Whether the robots.txt resync listener is registered for this request.
324 + *
325 + * @since 2.14.0
326 + * @var bool
327 + */
328 + private static bool $robots_sync_listener_registered = false;
329 +
330 + /**
331 + * Flag set when a plugin change may have altered the sitemap set.
332 + *
333 + * @since 2.14.0
334 + * @var string
335 + */
336 + public const ROBOTS_RESYNC_OPTION = 'thinkrank_robots_txt_resync_pending';
337 +
338 + /**
339 + * Flag set when a plugin change may have altered the sitemap index.
340 + *
341 + * Separate from ROBOTS_RESYNC_OPTION because the two files exist
342 + * independently: that flag is only set when a physical robots.txt exists,
343 + * and the sitemap index needs rebuilding whether or not it does.
344 + *
345 + * @since 2.15.0
346 + * @var string
347 + */
348 + public const SITEMAP_RESYNC_OPTION = 'thinkrank_sitemap_contributors_changed';
349 +
244 350 public function __construct() {
245 351 parent::__construct('site_identity');
352 +
353 + if (!self::$icon_sizes_listener_registered) {
354 + self::$icon_sizes_listener_registered = true;
355 + add_action('thinkrank_seo_settings_saved', [$this, 'generate_icon_sizes_on_save'], 10, 2);
356 + // Admin only: resizing is not front-end work, and admin traffic is
357 + // enough to run a one-time backfill promptly.
358 + add_action('admin_init', [self::class, 'maybe_backfill_icon_sizes']);
359 + }
360 +
361 + if (!self::$robots_sync_listener_registered) {
362 + self::$robots_sync_listener_registered = true;
363 +
364 + // A physical robots.txt bypasses PHP entirely, so composing the
365 + // Sitemap block at render time fixes the served output only on
366 + // sites with no file. Activating or deactivating a sitemap
367 + // contributor changes the set, and until #835 nothing rewrote the
368 + // file: the deactivated plugin's sitemap stayed advertised, serving
369 + // HTML to anything that followed it.
370 + add_action('activated_plugin', [self::class, 'flag_robots_txt_resync']);
371 + add_action('deactivated_plugin', [self::class, 'flag_robots_txt_resync']);
372 + add_action('init', [self::class, 'maybe_resync_robots_txt'], 99);
373 +
374 + // The sitemap index is a second static file listing the same
375 + // contributors, with its own rebuild path. #835 / #859 resynced
376 + // robots.txt only, so after Pro was deactivated the index kept
377 + // advertising news-sitemap.xml, which then served the home page
378 + // as HTML (#920).
379 + add_action('activated_plugin', [self::class, 'flag_sitemap_resync']);
380 + add_action('deactivated_plugin', [self::class, 'flag_sitemap_resync']);
381 + add_action('init', [self::class, 'maybe_resync_sitemap'], 99);
382 + }
246 383 }
247 384
248 385 /**
386 + * Note that the set of sitemap contributors may have changed.
387 + *
388 + * Deliberately unconditional about which plugin: a contributor is anything
389 + * hooking `thinkrank_additional_sitemaps`, which is resolved at runtime and
390 + * cannot be inspected for a plugin that is on its way out.
391 + *
392 + * The rewrite is not done here. `deactivated_plugin` fires inside the
393 + * request that deactivated it, while that plugin's filters are still
394 + * attached, so rendering now still sees the sitemap that is going away —
395 + * measured, not assumed: the first version of this fix wrote the
396 + * deactivated plugin's sitemap straight back into the file. The next
397 + * request has the real plugin set loaded, so the work waits for it.
398 + *
399 + * @since 2.14.0
400 + * @return void
401 + */
402 + public static function flag_robots_txt_resync(): void {
403 + if (!file_exists(ABSPATH . 'robots.txt')) {
404 + return;
405 + }
406 +
407 + update_option(self::ROBOTS_RESYNC_OPTION, 1, false);
408 + }
409 +
410 + /**
411 + * Rewrite the physical robots.txt once, on the request after a change.
412 + *
413 + * @since 2.14.0
414 + * @return void
415 + */
416 + public static function maybe_resync_robots_txt(): void {
417 + if (!get_option(self::ROBOTS_RESYNC_OPTION)) {
418 + return;
419 + }
420 +
421 + // Cleared first, so a render that fatals cannot retry on every request
422 + // for the rest of the site's life.
423 + delete_option(self::ROBOTS_RESYNC_OPTION);
424 +
425 + if (!file_exists(ABSPATH . 'robots.txt')) {
426 + return;
427 + }
428 +
429 + (new self())->sync_robots_txt_file();
430 + }
431 +
432 + /**
433 + * Note that the set of sitemap contributors may have changed.
434 + *
435 + * Unconditional, unlike flag_robots_txt_resync(): the sitemap files exist
436 + * whether or not robots.txt does. The rebuild waits for the next request
437 + * for the same reason as the robots.txt one, since `deactivated_plugin`
438 + * still runs with the outgoing plugin's `thinkrank_additional_sitemaps`
439 + * callback attached.
440 + *
441 + * @since 2.15.0
442 + * @return void
443 + */
444 + public static function flag_sitemap_resync(): void {
445 + update_option(self::SITEMAP_RESYNC_OPTION, 1, false);
446 + }
447 +
448 + /**
449 + * Queue a sitemap rebuild once, on the request after a contributor change.
450 + *
451 + * Goes through schedule_regeneration(), the debounced and lock-protected
452 + * path a sitemap settings save uses, so a burst of plugin changes (a bulk
453 + * deactivate, say) still produces one rebuild. That path also drops the
454 + * cached dynamic documents, so sites serving the sitemap from PHP drop the
455 + * entry as well.
456 + *
457 + * @since 2.15.0
458 + * @return void
459 + */
460 + public static function maybe_resync_sitemap(): void {
461 + if (!get_option(self::SITEMAP_RESYNC_OPTION)) {
462 + return;
463 + }
464 +
465 + // Cleared first, so a rebuild that fatals cannot be retried on every
466 + // request for the rest of the site's life.
467 + delete_option(self::SITEMAP_RESYNC_OPTION);
468 +
469 + $generator = new Sitemap_Generator(false);
470 + $settings = $generator->get_settings('site');
471 +
472 + // A disabled sitemap has no files to correct. Enabling it later builds
473 + // from the contributors present at that time.
474 + if (empty($settings['enabled'])) {
475 + return;
476 + }
477 +
478 + $generator->schedule_regeneration();
479 + }
480 +
481 + /**
482 + * Save settings, then refresh what a new canonical scheme invalidates.
483 + *
484 + * The static sitemap files are written with the scheme in force when they
485 + * were built, and nothing else rebuilds them until a post or term changes.
486 + * So a change of scheme left every `<loc>` on the old one while canonical
487 + * and og:url had already moved (#736). Every writer (the settings route,
488 + * the robots route, the MCP abilities, an import) lands here.
489 + *
490 + * @since 2.7.0
491 + *
492 + * @param string $context_type Context type.
493 + * @param int|null $context_id Context ID.
494 + * @param array $settings Settings to save.
495 + * @return bool
496 + */
497 + public function save_settings(string $context_type, ?int $context_id, array $settings): bool {
498 + if (!self::touches_canonical_scheme($context_type, $context_id, $settings)) {
499 + return parent::save_settings($context_type, $context_id, $settings);
500 + }
501 +
502 + $before = Url_Scheme::preference();
503 + $saved = parent::save_settings($context_type, $context_id, $settings);
504 +
505 + if ($saved) {
506 + $this->on_canonical_scheme_saved($before);
507 + }
508 +
509 + return $saved;
510 + }
511 +
512 + /**
513 + * Whether a save can change the site-wide canonical scheme.
514 + *
515 + * @since 2.7.0
516 + *
517 + * @param string $context_type Context type.
518 + * @param int|null $context_id Context ID.
519 + * @param array $settings Settings being saved.
520 + * @return bool
521 + */
522 + public static function touches_canonical_scheme(string $context_type, ?int $context_id, array $settings): bool {
523 + return 'site' === sanitize_key($context_type)
524 + && empty($context_id)
525 + && array_key_exists('canonical_scheme', $settings);
526 + }
527 +
528 + /**
529 + * Rebuild the static sitemaps when the effective scheme changed.
530 + *
531 + * Compares the effective preference, filter included, so a site whose
532 + * scheme is pinned by `thinkrank_canonical_scheme` does not rebuild on a
533 + * stored value that changes nothing it publishes.
534 + *
535 + * @since 2.7.0
536 + *
537 + * @param string $before Effective scheme before the save.
538 + * @return void
539 + */
540 + protected function on_canonical_scheme_saved(string $before): void {
541 + // The preference is cached for the request; the save just changed it.
542 + Url_Scheme::reset();
543 +
544 + if (Url_Scheme::preference() === $before) {
545 + return;
546 + }
547 +
548 + $this->schedule_sitemap_rebuild();
549 + }
550 +
551 + /**
552 + * Queue a settings-driven sitemap rebuild.
553 + *
554 + * Debounced and run after the response, like any other settings change
555 + * that alters what the sitemap publishes.
556 + *
557 + * @since 2.7.0
558 + * @return void
559 + */
560 + protected function schedule_sitemap_rebuild(): void {
561 + (new Sitemap_Generator(false))->schedule_regeneration();
562 + }
563 +
564 + /**
565 + * Build the icon derivatives for a newly chosen favicon.
566 + *
567 + * Runs on save, which is the only moment the choice changes and the only
568 + * place image work belongs — resolving a size on the front end must stay a
569 + * lookup. Failure is silent by design: a missing derivative degrades to the
570 + * next best file, so a site whose host cannot resize still renders an icon.
571 + *
572 + * @since 2.3.1
573 + *
574 + * @param string $manager_type Settings category that was saved.
575 + * @param array $settings The settings that were written.
576 + * @return void
577 + */
578 + public function generate_icon_sizes_on_save(string $manager_type, array $settings): void {
579 + if ('site_identity' !== $manager_type) {
580 + return;
581 + }
582 +
583 + // The choice, or the derivatives behind it, may have just changed.
584 + delete_transient(self::ICON_URL_TRANSIENT);
585 +
586 + foreach (['favicon_url', 'apple_touch_icon_url'] as $key) {
587 + if (empty($settings[$key]) || !is_string($settings[$key])) {
588 + continue;
589 + }
590 +
591 + $attachment_id = self::icon_attachment_id($settings[$key]);
592 +
593 + if ($attachment_id) {
594 + self::ensure_icon_sizes($attachment_id);
595 + }
596 + }
597 +
598 + // Dropped again after the resizes finish. Resizing is not instant, and a
599 + // front-end request arriving mid-generation would otherwise repopulate
600 + // the transient with the pre-derivative URLs and pin them for the full
601 + // TTL — leaving the sizes= declarations untrue until the next save.
602 + delete_transient(self::ICON_URL_TRANSIENT);
603 + }
604 +
605 + /**
606 + * Build the derivatives for a site that configured its icons before this
607 + * existed.
608 + *
609 + * generate_icon_sizes_on_save() only fires on a settings write, so every
610 + * site with an icon already chosen would keep serving whatever
611 + * wp_get_attachment_image_url() could find — in practice the 150x150
612 + * thumbnail behind a sizes="32x32" declaration — until someone happened to
613 + * re-save Site Identity. That is the bug this is meant to fix, so the
614 + * derivatives are built once on upgrade instead of waiting for a save.
615 + *
616 + * Guarded by its own option rather than the plugin version so it runs once
617 + * and stays cheap: the check is a single autoloaded read on requests after
618 + * the first.
619 + *
620 + * @since 2.3.1
621 + *
622 + * @return void
623 + */
624 + public static function maybe_backfill_icon_sizes(): void {
625 + if (get_option(self::ICON_BACKFILL_OPTION)) {
626 + return;
627 + }
628 +
629 + // Written before the work, not after: a host that cannot resize must
630 + // not retry on every admin request forever.
631 + update_option(self::ICON_BACKFILL_OPTION, time(), true);
632 +
633 + $settings = (new self())->get_settings('site');
634 +
635 + if (!is_array($settings)) {
636 + return;
637 + }
638 +
639 + foreach (['favicon_url', 'apple_touch_icon_url'] as $key) {
640 + if (empty($settings[$key]) || !is_string($settings[$key])) {
641 + continue;
642 + }
643 +
644 + $attachment_id = self::icon_attachment_id($settings[$key]);
645 +
646 + if ($attachment_id) {
647 + self::ensure_icon_sizes($attachment_id);
648 + }
649 + }
650 +
651 + delete_transient(self::ICON_URL_TRANSIENT);
652 + }
653 +
654 + /**
655 + * Attachment ID behind a configured icon URL, or 0 when it is not ours.
656 + *
657 + * attachment_url_to_postid() matches _wp_attached_file, which holds the
658 + * ORIGINAL upload path, so the URL of a generated derivative
659 + * (`logo-512x512.png`) returns 0 — and that is exactly what the media
660 + * picker hands back when the user chooses a size. Attachment_Lookup falls
661 + * back to the original behind it; the fallback started here and moved
662 + * there when every other image lookup turned out to need it (#847).
663 + *
664 + * Shared with SEO_Manager's site-icon filter so both sides of the feature
665 + * agree on which attachment a configured URL means.
666 + *
667 + * @since 2.3.1
668 + *
669 + * @param string $url Configured icon URL.
670 + * @return int Attachment ID, or 0.
671 + */
672 + public static function icon_attachment_id(string $url): int {
673 + return Attachment_Lookup::id_from_url($url);
674 + }
675 +
676 + /**
677 + * Which ICON_SIZES derivatives this attachment still needs.
678 + *
679 + * Split out from the generation so the decision can be asserted on its
680 + * own: whether a size is skipped because it already exists or because it
681 + * would upscale is invisible once both answers are "nothing was built".
682 + *
683 + * A source is measured by its SHORTER edge — a 400x40 banner cannot yield
684 + * a true 192x192 — and anything reporting no dimensions at all (SVGs) is
685 + * left alone.
686 + *
687 + * @since 2.3.1
688 + *
689 + * @param array $meta Attachment metadata.
690 + * @return array<string, array{width: int, height: int, crop: bool}> Sizes to build.
691 + */
692 + public static function missing_icon_sizes(array $meta): array {
693 + $source = min((int) ($meta['width'] ?? 0), (int) ($meta['height'] ?? 0));
694 +
695 + if ($source < 1) {
696 + return [];
697 + }
698 +
699 + $wanted = [];
700 + foreach (self::ICON_SIZES as $size) {
701 + // Never upscale: a stretched source behind an accurate sizes=""
702 + // label is worse than the honest near miss it would replace.
703 + if (isset($meta['sizes']["site_icon-{$size}"]) || $size > $source) {
704 + continue;
705 + }
706 +
707 + $wanted["site_icon-{$size}"] = ['width' => $size, 'height' => $size, 'crop' => true];
708 + }
709 +
710 + return $wanted;
711 + }
712 +
713 + /**
714 + * Generate whatever ICON_SIZES derivatives this attachment is missing.
715 + *
716 + * Only the missing ones, and never one larger than the source: upscaling a
717 + * small favicon would put a blurrier file behind an accurate sizes="" label
718 + * than the honest near-miss it replaced.
719 + *
720 + * @since 2.3.1
721 + *
722 + * @param int $attachment_id Attachment to build derivatives for.
723 + * @return string[] Size names generated, empty when there was nothing to do.
724 + */
725 + public static function ensure_icon_sizes(int $attachment_id): array {
726 + $meta = wp_get_attachment_metadata($attachment_id);
727 +
728 + if (!is_array($meta)) {
729 + return [];
730 + }
731 +
732 + $wanted = self::missing_icon_sizes($meta);
733 +
734 + if (empty($wanted)) {
735 + return [];
736 + }
737 +
738 + $file = get_attached_file($attachment_id);
739 +
740 + if (!$file || !file_exists($file)) {
741 + return [];
742 + }
743 +
744 + $editor = wp_get_image_editor($file);
745 +
746 + if (is_wp_error($editor)) {
747 + return [];
748 + }
749 +
750 + $generated = $editor->multi_resize($wanted);
751 +
752 + if (empty($generated)) {
753 + return [];
754 + }
755 +
756 + $meta['sizes'] = array_merge($meta['sizes'] ?? [], $generated);
757 + wp_update_attachment_metadata($attachment_id, $meta);
758 +
759 + return array_keys($generated);
760 + }
761 +
762 + /**
249 763 * Initialize WordPress filesystem
250 764 *
251 765 * @since 1.0.0
252 766 * @return bool True if filesystem is initialized, false otherwise
@@ -442,8 +956,28 @@
442 956 $body = ($custom !== '' && !$fully_blocked)
443 957 ? $this->strip_robots_header($custom)
444 958 : trim($this->generate_robots_txt()['content']);
445 959
960 + // The per-agent AI directives are machine-owned, so they are composed
961 + // here rather than stored: the textarea holds the user's body, with
962 + // the fenced block stripped out of every read and re-applied on every
963 + // render. A site-wide block already disallows everyone, so adding the
964 + // per-agent group there would be noise restating the same refusal.
965 + // Composed here rather than read from storage, for the same reason as
966 + // the AI block below: the set of sitemaps an install publishes is a
967 + // runtime fact. `robots_txt_content` is a snapshot of it taken at the
968 + // last save, and nothing invalidated that snapshot, so deactivating a
969 + // sitemap provider left its URL advertised and serving HTML (#835).
970 + // Composing it on every render means the advertisement agrees with what
971 + // the install publishes, in both directions, with no cache to expire.
972 + if (!$fully_blocked) {
973 + $body = $this->apply_sitemap_block($body);
974 + }
975 +
976 + if (!$fully_blocked) {
977 + $body = $this->apply_ai_crawler_block($body, $settings);
978 + }
979 +
446 980 if ($body === '') {
447 981 return '';
448 982 }
449 983
@@ -450,8 +984,87 @@
450 984 return $this->robots_txt_header() . $body . "\n";
451 985 }
452 986
453 987 /**
988 + * Replace the generated Sitemap block with the one this install publishes.
989 + *
990 + * @since 2.14.0
991 + * @param string $body Robots.txt body, without the header.
992 + * @return string
993 + */
994 + private function apply_sitemap_block(string $body): string {
995 + $stripped = $this->strip_generated_sitemap_block($body);
996 + $urls = $this->get_sitemap_urls_for_robots();
997 +
998 + if (empty($urls)) {
999 + return $stripped;
1000 + }
1001 +
1002 + $block = '';
1003 + foreach ($urls as $url) {
1004 + $block .= 'Sitemap: ' . $url . "\n";
1005 + }
1006 +
1007 + if ('' === trim($stripped)) {
1008 + return trim($block);
1009 + }
1010 +
1011 + // The grammar build_robots_txt_content() writes: one blank line before
1012 + // the block, none inside it. A blank line terminates a record in the
1013 + // robots.txt grammar, so a line between every directive is invalid.
1014 + return rtrim($stripped) . "\n\n" . trim($block);
1015 + }
1016 +
1017 + /**
1018 + * Remove the plugin-written Sitemap block from a stored body.
1019 + *
1020 + * Only the trailing run of `Sitemap:` lines is removed, which is the exact
1021 + * shape `build_robots_txt_content()` writes: a blank line, then nothing but
1022 + * `Sitemap:` lines to the end of the body. A `Sitemap:` line anywhere else
1023 + * was typed by the site owner and is left exactly where they put it, which
1024 + * is why this cannot simply strip every matching line.
1025 + *
1026 + * @since 2.14.0
1027 + * @param string $body Robots.txt body.
1028 + * @return string
1029 + */
1030 + private function strip_generated_sitemap_block(string $body): string {
1031 + $lines = preg_split('/\R/', $body);
1032 +
1033 + if (!is_array($lines)) {
1034 + return $body;
1035 + }
1036 +
1037 + $cut = count($lines);
1038 +
1039 + // Walk back over the trailing block: sitemap lines, and the blank lines
1040 + // that separate or pad it. Anything else ends the block.
1041 + for ($i = count($lines) - 1; $i >= 0; $i--) {
1042 + $line = trim($lines[$i]);
1043 +
1044 + if ('' === $line) {
1045 + $cut = $i;
1046 + continue;
1047 + }
1048 +
1049 + if (0 === stripos($line, 'sitemap:')) {
1050 + $cut = $i;
1051 + continue;
1052 + }
1053 +
1054 + break;
1055 + }
1056 +
1057 + if ($cut >= count($lines)) {
1058 + return $body;
1059 + }
1060 +
1061 + // Nothing but sitemap lines in the whole body means there is no owner
1062 + // content to keep.
1063 + return rtrim(implode("\n", array_slice($lines, 0, $cut)));
1064 + }
1065 +
1066 + /**
454 1067 * Resolve the robots.txt actually served to crawlers, with its origin.
455 1068 *
456 1069 * Lets an API/MCP consumer see the effective output without crawling the
457 1070 * URL. Mirrors serving precedence: a physical robots.txt in the web root is
@@ -468,8 +1081,9 @@
468 1081 $robots_file = ABSPATH . 'robots.txt';
469 1082 if (file_exists($robots_file) && is_readable($robots_file)) {
470 1083 // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents -- Reading a public web-root file; WP_Filesystem is not available on front-end requests.
471 1084 return [
1085 + // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents -- reads a local file the plugin just located; WP_Filesystem would need credentials on some hosts.
472 1086 'content' => (string) file_get_contents($robots_file),
473 1087 'is_default' => false,
474 1088 'source' => 'file',
475 1089 ];
@@ -496,8 +1110,68 @@
496 1110 ];
497 1111 }
498 1112
499 1113 /**
1114 + * Describe how /robots.txt is actually delivered, and whether that still
1115 + * matches the saved settings.
1116 + *
1117 + * The admin screen edits settings, but a physical robots.txt in the web root
1118 + * is served directly by the web server and bypasses the `robots_txt` filter
1119 + * entirely. When those two drift, the editor is showing content no crawler
1120 + * ever sees — the conflict this exists to surface.
1121 + *
1122 + * @since 1.31.0
1123 + *
1124 + * @return array{content: string, source: string, is_default: bool, in_sync: bool, out_of_sync_reason: string, url: string}
1125 + * The served content and its origin, whether it still reflects the
1126 + * body the editor is showing, why it does not when it does not
1127 + * ('file_drift' or 'crawl_blocked'), and the public URL it is
1128 + * served from.
1129 + */
1130 + public function get_robots_txt_delivery(): array {
1131 + $effective = $this->get_effective_robots_txt();
1132 + $settings = $this->get_settings('site');
1133 +
1134 + // Compare bodies, not raw strings: the auto-generated header carries a
1135 + // regeneration timestamp that always differs and means nothing here.
1136 + // The AI crawler block is composed at render time on both sides, so it
1137 + // is identical by construction and comparing it would only ever report
1138 + // a false drift the admin cannot act on.
1139 + $served = $this->strip_ai_crawler_block($this->strip_robots_header($effective['content']));
1140 +
1141 + // Measure against the body the editor is displaying — get_served_robots_body()
1142 + // — not against render_robots_txt(). Two things made the old comparison
1143 + // report "in sync" while the screen showed rules no crawler receives:
1144 + // a physical file was compared to a freshly rendered body rather than
1145 + // to the stored override the textarea shows, and a site-wide crawl
1146 + // block makes render_robots_txt() return the generated "Disallow: /"
1147 + // on both sides of the comparison, so it always matched.
1148 + $expected = $this->get_served_robots_body();
1149 +
1150 + // Management off: WordPress serves its own default and the editor is not
1151 + // claiming anything is live, so there is nothing to be out of sync with.
1152 + $managed = !empty($settings['robots_txt_enabled']);
1153 + $in_sync = !$managed || $served === $expected;
1154 +
1155 + $reason = '';
1156 + if (!$in_sync) {
1157 + // A crawl block is a deliberate override, not a stale file, and the
1158 + // admin needs to be told which of the two they are looking at.
1159 + $blocked = empty($settings['allow_search_engines'] ?? true) || !get_option('blog_public');
1160 + $reason = $blocked ? 'crawl_blocked' : 'file_drift';
1161 + }
1162 +
1163 + return [
1164 + 'content' => $effective['content'],
1165 + 'source' => $effective['source'],
1166 + 'is_default' => $effective['is_default'],
1167 + 'in_sync' => $in_sync,
1168 + 'out_of_sync_reason' => $reason,
1169 + 'url' => home_url('/robots.txt'),
1170 + ];
1171 + }
1172 +
1173 + /**
500 1174 * Keep the physical robots.txt file in step with the saved settings.
501 1175 *
502 1176 * When management is enabled the physical file is the source of truth the
503 1177 * web server serves, so this makes sure it exists and matches the effective
@@ -534,8 +1208,9 @@
534 1208 // do_robots()/robots_txt filter entirely, so without this those lines
535 1209 // are silently dropped. ThinkRank's own filter_robots_txt callback just
536 1210 // re-returns this same content (it calls render_robots_txt(), which does
537 1211 // not re-apply the filter), so there is no recursion or double-append.
1212 + // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- WPML/core hook, not ours to name.
538 1213 $content = (string) apply_filters('robots_txt', $content, (bool) get_option('blog_public'));
539 1214 if ($content === '') {
540 1215 return false;
541 1216 }
@@ -840,17 +1515,19 @@
840 1515 $home_text = $settings['breadcrumb_home_text'] ?? 'Home';
841 1516 if (empty($home_text)) {
842 1517 $optimization['warnings'][] = 'Empty home text reduces accessibility for screen readers';
843 1518 $optimization['score'] -= 15;
844 - } elseif (strlen($home_text) > 20) {
845 - $optimization['suggestions'][] = 'Keep home text concise (current: ' . strlen($home_text) . ' chars)';
1519 + } elseif (mb_strlen($home_text) > 20) {
1520 + // mb_strlen: this number is shown to the user as "chars" (#687).
1521 + $optimization['suggestions'][] = 'Keep home text concise (current: ' . mb_strlen($home_text) . ' chars)';
846 1522 $optimization['score'] -= 5;
847 1523 }
848 1524
849 1525 // Check prefix usage
850 1526 $prefix = $settings['breadcrumb_prefix'] ?? '';
851 - if (!empty($prefix) && strlen($prefix) > 50) {
852 - $optimization['suggestions'][] = 'Breadcrumb prefix is quite long (' . strlen($prefix) . ' chars) - consider shortening';
1527 + if (!empty($prefix) && mb_strlen($prefix) > 50) {
1528 + // mb_strlen: this number is shown to the user as "chars" (#687).
1529 + $optimization['suggestions'][] = 'Breadcrumb prefix is quite long (' . mb_strlen($prefix) . ' chars) - consider shortening';
853 1530 $optimization['score'] -= 5;
854 1531 }
855 1532
856 1533 // Current page display
@@ -981,13 +1658,16 @@
981 1658 }
982 1659
983 1660 // Additional logo analysis for local images
984 1661 if (!empty($logo_url) && filter_var($logo_url, FILTER_VALIDATE_URL)) {
985 - $attachment_id = attachment_url_to_postid($logo_url);
1662 + $attachment_id = Attachment_Lookup::id_from_url($logo_url);
986 1663 if ($attachment_id) {
987 1664 $image_meta = wp_get_attachment_metadata($attachment_id);
988 - $width = isset($image_meta['width']) ? (int) $image_meta['width'] : 0;
989 - $height = isset($image_meta['height']) ? (int) $image_meta['height'] : 0;
1665 + // The configured file's own size — a logo picked at a generated
1666 + // size is not as large as the upload behind it.
1667 + $logo_file = Attachment_Lookup::describe($attachment_id, $logo_url);
1668 + $width = $logo_file['width'];
1669 + $height = $logo_file['height'];
990 1670
991 1671 // SVG logos store 0x0 metadata — no dimension/ratio analysis
992 1672 // is possible (and dividing by 0 is fatal).
993 1673 if ($image_meta && $width > 0 && $height > 0) {
@@ -1090,11 +1770,12 @@
1090 1770 $optimization['score'] -= 15;
1091 1771 }
1092 1772 }
1093 1773
1094 - // Business type validation
1095 - if (empty($settings['business_type'])) {
1096 - $optimization['suggestions'][] = 'Select a specific business type for better schema markup';
1774 + // Business type validation (shared rule, one message — #622).
1775 + $business_type = $this->business_type_status($settings);
1776 + if ('suggestion' === $business_type['status']) {
1777 + $optimization['suggestions'][] = $business_type['message'];
1097 1778 $optimization['score'] -= 5;
1098 1779 }
1099 1780
1100 1781 // Email validation
@@ -1624,24 +2305,17 @@
1624 2305 'icon' => '✗'
1625 2306 ];
1626 2307 }
1627 2308
1628 - // Business Type validation
1629 - if (!empty($settings['business_type']) && $settings['business_type'] !== 'LocalBusiness') {
1630 - $field_details[] = [
1631 - 'field' => 'business_type',
1632 - 'label' => 'Business type is selected for proper schema markup.',
1633 - 'status' => 'valid',
1634 - 'icon' => '✓'
1635 - ];
1636 - } else {
1637 - $field_details[] = [
1638 - 'field' => 'business_type',
1639 - 'label' => 'Specific business type selection recommended for better schema markup.',
1640 - 'status' => 'suggestion',
1641 - 'icon' => '⚠'
1642 - ];
1643 - }
2309 + // Business Type validation — see business_type_status() for why there
2310 + // is exactly one rule here now (#622).
2311 + $business_type = $this->business_type_status($settings);
2312 + $field_details[] = [
2313 + 'field' => 'business_type',
2314 + 'label' => $business_type['message'],
2315 + 'status' => $business_type['status'],
2316 + 'icon' => 'valid' === $business_type['status'] ? '✓' : '⚠',
2317 + ];
1644 2318
1645 2319 // Address validation (NAP consistency)
1646 2320 $address_fields = ['business_address', 'business_city', 'business_state', 'business_country'];
1647 2321 $address_complete = true;
@@ -2260,12 +2934,15 @@
2260 2934 'warnings' => [],
2261 2935 'suggestions' => []
2262 2936 ];
2263 2937
2264 - // Validate business name (required for local SEO)
2938 + // Business name is what makes the LocalBusiness schema useful, but it
2939 + // cannot be a blocking error: the toggle is what reveals the business
2940 + // fields, so requiring the name up front makes enabling Local SEO
2941 + // impossible. The frontend already skips the output while the name is
2942 + // empty (see Seo_Manager::output_local_seo_meta_tags()).
2265 2943 if (empty($settings['business_name'])) {
2266 - $validation['errors'][] = 'Business name is required when local SEO is enabled';
2267 - $validation['valid'] = false;
2944 + $validation['warnings'][] = 'Business name is missing - required before local business schema is output';
2268 2945 } elseif (strlen($settings['business_name']) > 100) {
2269 2946 $validation['warnings'][] = 'Business name is very long, consider shortening for better display';
2270 2947 }
2271 2948
@@ -2322,11 +2999,15 @@
2322 2999 } else {
2323 3000 $validation['suggestions'][] = 'Add business hours to improve local search visibility';
2324 3001 }
2325 3002
2326 - // Validate business type
2327 - if (empty($settings['business_type'])) {
2328 - $validation['suggestions'][] = 'Select a specific business type for better schema markup';
3003 + // Business type, through the shared rule (#622). This is the only place
3004 + // it is reported on the generic path: validate_settings() with no tab
3005 + // context attaches basic-info field details, not business-info ones, so
3006 + // without this the setting would go unreported there entirely.
3007 + $business_type = $this->business_type_status($settings);
3008 + if ('suggestion' === $business_type['status']) {
3009 + $validation['suggestions'][] = $business_type['message'];
2329 3010 }
2330 3011
2331 3012 return $validation;
2332 3013 }
@@ -2406,8 +3087,201 @@
2406 3087 return $output;
2407 3088 }
2408 3089
2409 3090 /**
3091 + * Keys the Site Identity screens store beyond the 16 defaults.
3092 + *
3093 + * Title formats, breadcrumb configuration, the hero fields, the business
3094 + * block and the wizard's identity fields are all real settings written by
3095 + * this manager, none of which get_default_settings() names — it seeds only
3096 + * the values a fresh install needs. Gating on defaults alone would stop
3097 + * every one of them saving (#452).
3098 + *
3099 + * @since 2.0.1
3100 + *
3101 + * @return string[]
3102 + */
3103 + /**
3104 + * The stored alternate name(s), shaped for schema output.
3105 + *
3106 + * schema.org and Google both allow `alternateName` to carry one value or
3107 + * several, and the store already round-trips either shape, so this accepts
3108 + * both and normalises: null when there is nothing to publish, a bare string
3109 + * for one name, a list for more. Emitting a one-element array would be
3110 + * valid but noisier than it needs to be.
3111 + *
3112 + * Shared because both WebSite producers need it and must agree — a property
3113 + * added to one and not the other is how #688 happened.
3114 + *
3115 + * @since 2.7.0
3116 + *
3117 + * @param mixed $value Stored alternate_name value.
3118 + * @return string|string[]|null
3119 + */
3120 + public static function alternate_name_for_schema($value) {
3121 + $names = [];
3122 +
3123 + foreach ((array) $value as $name) {
3124 + if (!is_scalar($name)) {
3125 + continue;
3126 + }
3127 +
3128 + $name = trim((string) $name);
3129 +
3130 + if ('' !== $name && !in_array($name, $names, true)) {
3131 + $names[] = $name;
3132 + }
3133 + }
3134 +
3135 + if (empty($names)) {
3136 + return null;
3137 + }
3138 +
3139 + return 1 === count($names) ? $names[0] : $names;
3140 + }
3141 +
3142 + protected function additional_setting_keys(): array {
3143 + return [
3144 + // Title formats, one per context.
3145 + 'homepage_title', 'post_title', 'page_title', 'category_title',
3146 + 'tag_title', 'author_title', 'search_title', 'archive_title',
3147 + // The blog-index homepage's meta description (#897).
3148 + 'homepage_description',
3149 + // Breadcrumbs.
3150 + 'breadcrumb_prefix', 'show_current_page', 'breadcrumb_use_seo_title',
3151 + // Identity, as written by the setup wizard and the importers.
3152 + 'alternate_name', 'identity_type', 'represents',
3153 + 'default_meta_description', 'default_social_image',
3154 + 'social_media_accounts',
3155 + // Schema toggles that live on this screen.
3156 + 'organization_schema', 'knowledge_graph',
3157 + // Robots rules composed by the Robots.txt panel.
3158 + 'custom_robots_rules',
3159 + // Per-agent AI crawler allow/block map (#657).
3160 + 'ai_crawler_rules',
3161 + // Hero section.
3162 + 'hero_title', 'hero_subtitle', 'hero_cta_text', 'hero_cta_url',
3163 + 'hero_background_image',
3164 + // Local SEO / business details.
3165 + 'local_seo_enabled', 'business_type', 'business_name',
3166 + 'business_address', 'business_city', 'business_state',
3167 + 'business_postal_code', 'business_country', 'business_phone',
3168 + 'business_email', 'business_latitude', 'business_longitude',
3169 + 'business_price_range', 'business_hours',
3170 + ];
3171 + }
3172 +
3173 + /**
3174 + * Sanitize settings, normalising the AI crawler rule map.
3175 + *
3176 + * The generic array sanitizer keeps the shape but says nothing about the
3177 + * values: a payload could store `ai_crawler_rules[gptbot] = "maybe"`, or a
3178 + * slug no crawler answers to, and both would round-trip through every
3179 + * later response. Normalising here rather than in the REST handler puts it
3180 + * on the one path every writer shares — the settings route, the robots
3181 + * route and the MCP abilities all land in save_settings() (#657).
3182 + *
3183 + * @since 2.5.0
3184 + *
3185 + * @param array $settings Settings to sanitize.
3186 + * @param string $context_type Context type.
3187 + * @return array Sanitized settings.
3188 + */
3189 + protected function sanitize_settings(array $settings, string $context_type = 'site'): array {
3190 + $sanitized = parent::sanitize_settings($settings, $context_type);
3191 +
3192 + if (array_key_exists('ai_crawler_rules', $sanitized)) {
3193 + $sanitized['ai_crawler_rules'] = AI_Crawlers::normalize_rules($sanitized['ai_crawler_rules']);
3194 + }
3195 +
3196 + // Same reasoning one key up, for the scheme override (#638). Anything
3197 + // that is not one of the three modes means "follow WordPress", and is
3198 + // stored as that rather than kept verbatim — otherwise get-site-identity
3199 + // -settings would report a scheme the site does not actually publish.
3200 + if (array_key_exists('canonical_scheme', $sanitized)) {
3201 + $sanitized['canonical_scheme'] = in_array($sanitized['canonical_scheme'], Url_Scheme::MODES, true)
3202 + ? $sanitized['canonical_scheme']
3203 + : Url_Scheme::AUTOMATIC;
3204 + }
3205 +
3206 + // Same reasoning again for the business type. It goes straight into
3207 + // LocalBusiness schema, so a type that is not in the schema.org
3208 + // vocabulary is invalid structured data — and storing it verbatim would
3209 + // have get-site-identity-settings report a type the site cannot
3210 + // actually publish. An empty value keeps meaning "not set"; anything
3211 + // else unrecognised falls back to the general-purpose root (#623).
3212 + if (array_key_exists('business_type', $sanitized)) {
3213 + $type = (string) $sanitized['business_type'];
3214 +
3215 + if ('' !== $type && !\ThinkRank\Config\Local_Business_Types_Config::is_valid($type)) {
3216 + $type = \ThinkRank\Config\Local_Business_Types_Config::ROOT;
3217 + }
3218 +
3219 + $sanitized['business_type'] = $type;
3220 + }
3221 +
3222 + return $sanitized;
3223 + }
3224 +
3225 + /**
3226 + * schema.org's general-purpose LocalBusiness type.
3227 + *
3228 + * The default, the first option in the control, and a valid answer in its
3229 + * own right — which is the whole point of #622.
3230 + *
3231 + * @since 2.10.0
3232 + * @var string
3233 + */
3234 + private const GENERAL_BUSINESS_TYPE = 'LocalBusiness';
3235 +
3236 + /**
3237 + * The one rule for whether a business type needs the user's attention.
3238 + *
3239 + * There were three, with two wordings and two different conditions. Two
3240 + * fired when the value was empty; the third fired when it WAS
3241 + * `LocalBusiness` — which is the default, the first option in the control
3242 + * and a perfectly valid schema.org type. So the warning appeared out of the
3243 + * box for every site, could not be cleared without choosing a type that
3244 + * might be inaccurate, and on an empty value it appeared three times in two
3245 + * different phrasings, which is why it was reported as showing twice (#622).
3246 + *
3247 + * The rule now: a type is expected, and any type in the vocabulary is a
3248 + * correct answer. Only an unset value is worth prompting about.
3249 + * `LocalBusiness` is the general-purpose answer and is accepted as one —
3250 + * with a note that a more specific type sharpens the schema, phrased as the
3251 + * guidance it is rather than as a fault the user has to clear.
3252 + *
3253 + * @since 2.10.0
3254 + *
3255 + * @param array $settings Site identity settings.
3256 + * @return array{status:string,message:string} `valid` or `suggestion`.
3257 + */
3258 + private function business_type_status(array $settings): array {
3259 + $type = trim((string) ($settings['business_type'] ?? ''));
3260 +
3261 + if ('' === $type) {
3262 + return [
3263 + 'status' => 'suggestion',
3264 + 'message' => __('Select a business type so your local schema describes the right kind of business.', 'thinkrank'),
3265 + ];
3266 + }
3267 +
3268 + // The literal rather than a constant from the expanded type list (#623):
3269 + // that lands on its own branch, and this fix must not wait on it.
3270 + if (self::GENERAL_BUSINESS_TYPE === $type) {
3271 + return [
3272 + 'status' => 'valid',
3273 + 'message' => __('Business type is set to Local Business. A more specific type sharpens your schema, if one fits.', 'thinkrank'),
3274 + ];
3275 + }
3276 +
3277 + return [
3278 + 'status' => 'valid',
3279 + 'message' => __('Business type is selected for proper schema markup.', 'thinkrank'),
3280 + ];
3281 + }
3282 +
3283 + /**
2410 3284 * Get default settings for a context type (implements interface)
2411 3285 *
2412 3286 * @since 1.0.0
2413 3287 *
@@ -2427,9 +3301,32 @@
2427 3301 'breadcrumb_home_text' => 'Home',
2428 3302 'breadcrumb_separator' => '>',
2429 3303 'robots_txt_enabled' => true,
2430 3304 'allow_search_engines' => true,
3305 + // Answer 404 when a content selector in the URL resolved to
3306 + // nothing (#634). On by default, unlike the other new settings
3307 + // here: it changes no URL a visitor or a correct crawler uses, only
3308 + // ones where WordPress resolved nothing and served the blog listing
3309 + // at 200 anyway.
3310 + 'query_protection' => true,
3311 +
3312 + // Feed controls (#635). All three off, so an upgrade changes
3313 + // nothing about what an existing site already sends its
3314 + // subscribers; a brand-new install is seeded with the signature and
3315 + // the noindex on, in Activator::seed_feed_defaults().
3316 + 'feed_excerpt_only' => false,
3317 + 'feed_source_link' => false,
3318 + 'feed_noindex' => false,
3319 +
3320 + // The scheme self-referential URLs go out with (#638). 'automatic'
3321 + // means substitute nothing and follow WordPress, which is what
3322 + // every site did before the setting existed.
3323 + 'canonical_scheme' => Url_Scheme::AUTOMATIC,
2431 3324 'robots_txt_content' => '',
3325 + // Empty map = every AI crawler allowed. Defaults must stay
3326 + // permissive so an upgrade never starts blocking a crawler a site
3327 + // was happily serving (#657).
3328 + 'ai_crawler_rules' => [],
2432 3329 'logo_url' => '',
2433 3330 'favicon_url' => '',
2434 3331 'apple_touch_icon_url' => ''
2435 3332 ];
@@ -2548,10 +3445,13 @@
2548 3445 */
2549 3446 private function prepare_title_placeholders(array $data, string $context, array $settings): array {
2550 3447 $placeholders = [
2551 3448 '%title%' => $data['title'] ?? '',
2552 - '%sitename%' => $settings['site_name'] ?? get_bloginfo('name'),
2553 - '%tagline%' => $settings['tagline'] ?? get_bloginfo('description'),
3449 + // `?:` rather than `??`: these are persisted as '' rather than left
3450 + // unset, and '' is not null, so the null-coalesce never reached the
3451 + // WordPress fallback (#398).
3452 + '%sitename%' => ($settings['site_name'] ?? '') ?: get_bloginfo('name'),
3453 + '%tagline%' => ($settings['tagline'] ?? '') ?: get_bloginfo('description'),
2554 3454 '%separator%' => '', // Will be replaced with actual separator
2555 3455 '%category%' => '',
2556 3456 '%author%' => '',
2557 3457 '%date%' => '',
@@ -2635,16 +3535,17 @@
2635 3535 // Remove extra whitespace
2636 3536 $title = preg_replace('/\s+/', ' ', $title);
2637 3537 $title = trim($title);
2638 3538
2639 - // Ensure title is not too long (60 characters max for SEO)
2640 - if (strlen($title) > 60) {
2641 - // Try to truncate at word boundary
2642 - $title = wp_trim_words($title, 8, '...');
2643 - if (strlen($title) > 60) {
2644 - $title = substr($title, 0, 57) . '...';
2645 - }
2646 - }
3539 + // Ensure title is not too long (60 characters max for SEO).
3540 + // All three units here were wrong for non-Latin text: strlen() counts
3541 + // BYTES so the gate fired at 20 Thai characters, wp_trim_words() counts
3542 + // CHARACTERS on th/ja/zh_* so `8` cut the title to 8 of them, and
3543 + // substr() cuts bytes so it split a character mid-sequence (#687).
3544 + $title = \ThinkRank\Core\Seo_Text::trim_to_length(
3545 + $title,
3546 + \ThinkRank\Core\Seo_Text::TITLE_MAX_LENGTH
3547 + );
2647 3548
2648 3549 // Ensure title is not empty
2649 3550 if (empty($title)) {
2650 3551 $title = get_bloginfo('name');
@@ -3034,8 +3935,29 @@
3034 3935 * @since 1.0.0
3035 3936 * @return array Array of sitemap URLs
3036 3937 */
3037 3938 private function get_sitemap_urls_for_robots(): array {
3939 + // One wrapper over every return path below, including the #104 extras.
3940 + // The Sitemap: line is the only absolute URL of ours in robots.txt and
3941 + // the one a crawler follows to find everything else, so it has to carry
3942 + // the site's scheme preference (#638). Applied here rather than where
3943 + // the body is assembled, because that path also renders a robots.txt a
3944 + // site owner typed themselves, and their text is not ours to rewrite.
3945 + return array_map(
3946 + static function (string $url): string {
3947 + return Url_Scheme::apply($url);
3948 + },
3949 + $this->collect_sitemap_urls_for_robots()
3950 + );
3951 + }
3952 +
3953 + /**
3954 + * The sitemap URLs robots.txt advertises, before the scheme preference.
3955 + *
3956 + * @since 1.0.0
3957 + * @return array Array of sitemap URLs
3958 + */
3959 + private function collect_sitemap_urls_for_robots(): array {
3038 3960 try {
3039 3961 // Get sitemap settings
3040 3962 $sitemap_generator = new \ThinkRank\SEO\Sitemap_Generator();
3041 3963 $sitemap_settings = $sitemap_generator->get_settings('site');
@@ -3047,29 +3969,88 @@
3047 3969
3048 3970 $sitemap_urls = [];
3049 3971 $site_url = home_url();
3050 3972
3051 - // Extract enabled sitemap URLs
3973 + // Extract enabled sitemap URLs. When the index is enabled it is the
3974 + // only entry worth advertising: every child sitemap is already
3975 + // listed inside it, so naming them again in robots.txt is pure
3976 + // redundancy and drifts out of date as soon as a post type is added.
3977 + $index_url = '';
3052 3978 if (!empty($sitemap_settings['sitemap_urls']) && is_array($sitemap_settings['sitemap_urls'])) {
3053 3979 foreach ($sitemap_settings['sitemap_urls'] as $sitemap) {
3054 - if (!empty($sitemap['enabled']) && !empty($sitemap['url'])) {
3055 - $sitemap_urls[] = $site_url . $sitemap['url'];
3980 + if (empty($sitemap['enabled']) || empty($sitemap['url'])) {
3981 + continue;
3056 3982 }
3983 +
3984 + if (($sitemap['type'] ?? '') === 'index') {
3985 + $index_url = $site_url . $sitemap['url'];
3986 + continue;
3987 + }
3988 +
3989 + $sitemap_urls[] = $site_url . $sitemap['url'];
3057 3990 }
3058 3991 }
3059 3992
3993 + if ($index_url !== '') {
3994 + // The index covers the children and, on a segmented install,
3995 + // the local business sitemap too.
3996 + //
3997 + // It does not cover a sitemap contributed through
3998 + // `thinkrank_additional_sitemaps`: the index is built by this
3999 + // plugin's own generator and never lists them. Returning the
4000 + // index alone therefore left a contributed sitemap with no
4001 + // discovery path at all — absent from robots.txt and absent
4002 + // from the index — so Pro's News sitemap was unreachable on any
4003 + // install with the index enabled, which is the default (#835).
4004 + $contributed = [];
4005 +
4006 + foreach (\ThinkRank\SEO\Sitemap_Generator::additional_sitemaps() as $path) {
4007 + $url = home_url($path);
4008 +
4009 + if ($url !== $index_url && !in_array($url, $contributed, true)) {
4010 + $contributed[] = $url;
4011 + }
4012 + }
4013 +
4014 + return array_merge([$index_url], $contributed);
4015 + }
4016 +
3060 4017 // Fallback to default if no URLs found
3061 4018 if (empty($sitemap_urls)) {
3062 4019 $sitemap_urls[] = home_url('/sitemap.xml');
3063 4020 }
3064 4021
3065 - // Advertise the local business sitemap when it exists. In segmented
3066 - // mode it is already listed inside the sitemap index; in single mode
3067 - // there is no index, so robots.txt is its discovery path.
3068 - if (file_exists(ABSPATH . 'local-sitemap.xml')) {
3069 - $local_url = home_url('/local-sitemap.xml');
3070 - if (!in_array($local_url, $sitemap_urls, true)) {
3071 - $sitemap_urls[] = $local_url;
4022 + // No index on this install, so anything not already listed above has
4023 + // no other discovery path — advertise it directly. The local
4024 + // business sitemap and the sitemaps other plugins register both land
4025 + // here for the same reason, so they go through one list (#104).
4026 + $extra = [];
4027 +
4028 + // Not a file test. Under dynamic delivery the local sitemap is
4029 + // served from PHP and no file is ever written, so file_exists()
4030 + // silently dropped a sitemap the site really does publish (#752).
4031 + // On static sites the file is still what proves it, so both count.
4032 + $local_sitemap_published = file_exists(ABSPATH . 'local-sitemap.xml');
4033 +
4034 + if (!$local_sitemap_published && class_exists('ThinkRank\\SEO\\Sitemap_Generator')) {
4035 + $generator = new \ThinkRank\SEO\Sitemap_Generator(false);
4036 +
4037 + $local_sitemap_published = 'dynamic' === $generator->resolve_delivery_mode()
4038 + && $generator->publishes_local_sitemap();
4039 + }
4040 +
4041 + if ($local_sitemap_published) {
4042 + $extra[] = '/local-sitemap.xml';
4043 + }
4044 +
4045 + foreach (\ThinkRank\SEO\Sitemap_Generator::additional_sitemaps() as $path) {
4046 + $extra[] = $path;
4047 + }
4048 +
4049 + foreach ($extra as $path) {
4050 + $url = home_url($path);
4051 + if (!in_array($url, $sitemap_urls, true)) {
4052 + $sitemap_urls[] = $url;
3072 4053 }
3073 4054 }
3074 4055
3075 4056 return $sitemap_urls;
@@ -3159,8 +4140,9 @@
3159 4140 // timestamp on every update.
3160 4141 $content = '';
3161 4142
3162 4143 $current_user_agent = '';
4144 + $sitemap_started = false;
3163 4145
3164 4146 foreach ($rules as $rule) {
3165 4147 $directive = $rule['directive'] ?? '';
3166 4148 $value = $rule['value'] ?? '';
@@ -3181,9 +4163,17 @@
3181 4163 case 'crawl_delay':
3182 4164 $content .= "Crawl-delay: {$value}\n";
3183 4165 break;
3184 4166 case 'sitemap':
3185 - $content .= "\nSitemap: {$value}\n";
4167 + // One blank line separates the Sitemap block from the
4168 + // preceding group, and none appear inside it. A blank line
4169 + // terminates a record in the robots.txt grammar, so putting
4170 + // one between every directive was invalid formatting.
4171 + if (!$sitemap_started) {
4172 + $content .= "\n";
4173 + $sitemap_started = true;
4174 + }
4175 + $content .= "Sitemap: {$value}\n";
3186 4176 break;
3187 4177 }
3188 4178 }
3189 4179
@@ -3190,8 +4180,171 @@
3190 4180 return ltrim($content, "\n");
3191 4181 }
3192 4182
3193 4183 /**
4184 + * Parse a robots.txt body back into the {directive, value} rule shape.
4185 + *
4186 + * generate_robots_txt() returns `rules` alongside `content`, but callers
4187 + * replace `content` with the body actually being served (a stored override
4188 + * or a physical file). The generated rules then described something the
4189 + * response no longer contained. Re-deriving them from the served body keeps
4190 + * the two halves of the payload describing the same document.
4191 + *
4192 + * @since 2.0.1
4193 + *
4194 + * @param string $content Robots.txt body (header optional).
4195 + * @return array<int, array{directive: string, value: string}> Parsed rules.
4196 + */
4197 + public function parse_robots_txt_rules(string $content): array {
4198 + $map = [
4199 + 'user-agent' => 'user_agent',
4200 + 'disallow' => 'disallow',
4201 + 'allow' => 'allow',
4202 + 'crawl-delay' => 'crawl_delay',
4203 + 'sitemap' => 'sitemap',
4204 + ];
4205 +
4206 + $rules = [];
4207 +
4208 + foreach (preg_split('/\r\n|\r|\n/', $this->strip_robots_header($content)) as $line) {
4209 + $line = trim($line);
4210 +
4211 + // Blank lines separate groups and `#` starts a comment; neither is
4212 + // a rule.
4213 + if ($line === '' || str_starts_with($line, '#')) {
4214 + continue;
4215 + }
4216 +
4217 + $parts = explode(':', $line, 2);
4218 + if (count($parts) !== 2) {
4219 + continue;
4220 + }
4221 +
4222 + $field = strtolower(trim($parts[0]));
4223 + if (!isset($map[$field])) {
4224 + continue;
4225 + }
4226 +
4227 + $rules[] = [
4228 + 'directive' => $map[$field],
4229 + // Sitemap values are absolute URLs and contain the `:` the
4230 + // limited explode above deliberately preserved.
4231 + 'value' => trim($parts[1]),
4232 + ];
4233 + }
4234 +
4235 + return $rules;
4236 + }
4237 +
4238 + /**
4239 + * Opening fence of the machine-owned AI crawler region.
4240 + *
4241 + * @since 2.5.0
4242 + * @var string
4243 + */
4244 + public const AI_BLOCK_BEGIN = '# BEGIN ThinkRank AI crawlers';
4245 +
4246 + /**
4247 + * Closing fence of the machine-owned AI crawler region.
4248 + *
4249 + * @since 2.5.0
4250 + * @var string
4251 + */
4252 + public const AI_BLOCK_END = '# END ThinkRank AI crawlers';
4253 +
4254 + /**
4255 + * Render the fenced AI crawler region for the current settings.
4256 + *
4257 + * One `User-agent:` / `Disallow: /` record per blocked crawler. Allowed
4258 + * crawlers emit nothing at all: `Disallow:` with an empty value is the
4259 + * robots.txt way of saying "allow everything", but writing eighteen such
4260 + * records to say what silence already says would triple the file and
4261 + * invite the reading that an unlisted crawler is therefore refused.
4262 + *
4263 + * @since 2.5.0
4264 + *
4265 + * @param array $settings Site settings.
4266 + * @return string Fenced block, newline-terminated, or '' when nothing is blocked.
4267 + */
4268 + private function build_ai_crawler_block(array $settings): string {
4269 + $blocked = AI_Crawlers::blocked_slugs($settings['ai_crawler_rules'] ?? []);
4270 +
4271 + if (empty($blocked)) {
4272 + return '';
4273 + }
4274 +
4275 + $agents = AI_Crawlers::all();
4276 +
4277 + $lines = [
4278 + self::AI_BLOCK_BEGIN,
4279 + '# Managed by ThinkRank — edits between these lines are overwritten.',
4280 + ];
4281 +
4282 + foreach ($blocked as $slug) {
4283 + $lines[] = '';
4284 + $lines[] = 'User-agent: ' . $agents[$slug]['token'];
4285 + $lines[] = 'Disallow: /';
4286 + }
4287 +
4288 + $lines[] = self::AI_BLOCK_END;
4289 +
4290 + return implode("\n", $lines) . "\n";
4291 + }
4292 +
4293 + /**
4294 + * Remove the fenced AI crawler region from a robots.txt body.
4295 + *
4296 + * Tolerates a missing closing fence rather than leaving the rest of the
4297 + * file swallowed: a truncated write, or someone deleting the END line by
4298 + * hand, would otherwise make every subsequent read drop everything below
4299 + * the opening fence.
4300 + *
4301 + * @since 2.5.0
4302 + *
4303 + * @param string $body Robots.txt body.
4304 + * @return string Body with the region removed.
4305 + */
4306 + public function strip_ai_crawler_block(string $body): string {
4307 + if (false === strpos($body, self::AI_BLOCK_BEGIN)) {
4308 + return $body;
4309 + }
4310 +
4311 + $pattern = '/\R*' . preg_quote(self::AI_BLOCK_BEGIN, '/')
4312 + . '.*?(?:' . preg_quote(self::AI_BLOCK_END, '/') . '|\z)\R*/s';
4313 +
4314 + return trim((string) preg_replace($pattern, "\n\n", $body, 1));
4315 + }
4316 +
4317 + /**
4318 + * Put the current AI crawler region into a robots.txt body.
4319 + *
4320 + * Replaces an existing region in place so the block keeps its position in
4321 + * a hand-ordered file, and appends when there is none. Everything outside
4322 + * the fences is returned untouched — that is the whole point of fencing
4323 + * it, since the body is also a free-text field the user edits.
4324 + *
4325 + * @since 2.5.0
4326 + *
4327 + * @param string $body Robots.txt body (fences optional).
4328 + * @param array $settings Site settings.
4329 + * @return string Body carrying the current region.
4330 + */
4331 + private function apply_ai_crawler_block(string $body, array $settings): string {
4332 + $stripped = $this->strip_ai_crawler_block($body);
4333 + $block = $this->build_ai_crawler_block($settings);
4334 +
4335 + if ('' === $block) {
4336 + return $stripped;
4337 + }
4338 +
4339 + if ('' === trim($stripped)) {
4340 + return trim($block);
4341 + }
4342 +
4343 + return rtrim($stripped) . "\n\n" . trim($block);
4344 + }
4345 +
4346 + /**
3194 4347 * The auto-generated header prepended to the served robots.txt.
3195 4348 *
3196 4349 * Kept separate from the body so it is only ever added at render time with
3197 4350 * a fresh timestamp, never stored or shown in the editable textarea.
@@ -3228,22 +4381,28 @@
3228 4381 */
3229 4382 public function get_served_robots_body(): string {
3230 4383 $settings = $this->get_settings('site');
3231 4384
4385 + // The AI block is stripped from every one of these paths. A physical
4386 + // robots.txt we wrote carries it, and the stored override is whatever
4387 + // the textarea last held — so without this the block round-trips into
4388 + // the editor, gets saved as ordinary body text, and is then appended
4389 + // to a second time on the next render.
3232 4390 $custom = trim((string) ($settings['robots_txt_content'] ?? ''));
3233 4391 if ($custom !== '') {
3234 - return $this->strip_robots_header($custom);
4392 + return $this->strip_ai_crawler_block($this->strip_robots_header($custom));
3235 4393 }
3236 4394
3237 4395 $robots_file = ABSPATH . 'robots.txt';
3238 4396 if (file_exists($robots_file)) {
4397 + // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents, WordPress.PHP.NoSilencedErrors.Discouraged -- an unreadable robots.txt is an expected state answered with an empty string.
3239 4398 $raw = (string) @file_get_contents($robots_file);
3240 4399 if ($raw !== '') {
3241 - return $this->strip_robots_header($raw);
4400 + return $this->strip_ai_crawler_block($this->strip_robots_header($raw));
3242 4401 }
3243 4402 }
3244 4403
3245 - return trim($this->generate_robots_txt()['content']);
4404 + return $this->strip_ai_crawler_block(trim($this->generate_robots_txt()['content']));
3246 4405 }
3247 4406 private function get_site_identity_data(array $settings): array {
3248 4407 return [
3249 4408 'site_name' => $settings['site_name'] ?? get_bloginfo('name'),
@@ -3305,11 +4464,14 @@
3305 4464 $optimization['validation']['valid'] = false;
3306 4465 }
3307 4466
3308 4467 if (!empty($value) && isset($config['max_length'])) {
3309 - if (strlen($value) > $config['max_length']) {
4468 + // The warning says "characters", so measure and cut in characters:
4469 + // strlen()/substr() fired early on non-Latin values and the
4470 + // suggested replacement was cut mid-character (#687).
4471 + if (mb_strlen($value) > $config['max_length']) {
3310 4472 $optimization['validation']['warnings'][] = "{$element} exceeds maximum length of {$config['max_length']} characters";
3311 - $optimization['optimized_value'] = substr($value, 0, $config['max_length']);
4473 + $optimization['optimized_value'] = \ThinkRank\Core\Seo_Text::trim_to_length($value, (int) $config['max_length']);
3312 4474 }
3313 4475 }
3314 4476
3315 4477 // SEO-specific optimizations
@@ -3348,18 +4510,20 @@
3348 4510 return $optimization;
3349 4511 }
3350 4512
3351 4513 // Check if it's a local image
3352 - $attachment_id = attachment_url_to_postid($value);
4514 + $attachment_id = Attachment_Lookup::id_from_url($value);
3353 4515 if ($attachment_id) {
3354 4516 $image_meta = wp_get_attachment_metadata($attachment_id);
3355 4517
3356 4518 if ($image_meta && isset($image_meta['width'], $image_meta['height'])) {
3357 - // Check recommended size
4519 + // Check recommended size, against the configured file itself
4520 + // rather than the upload it may have been generated from.
3358 4521 if (isset($config['recommended_size'])) {
3359 4522 [$rec_width, $rec_height] = explode('x', $config['recommended_size']);
4523 + $image_file = Attachment_Lookup::describe($attachment_id, $value);
3360 4524
3361 - if ($image_meta['width'] != $rec_width || $image_meta['height'] != $rec_height) {
4525 + if ($image_file['width'] !== (int) $rec_width || $image_file['height'] !== (int) $rec_height) {
3362 4526 $optimization['suggestions'][] = "Consider using {$config['recommended_size']} size for optimal {$element}";
3363 4527 }
3364 4528 }
3365 4529