← All changes | src/integrations/front-end/robots-txt-integration.php +106 -49 19.0 → trunk View file →

		@@ -1,11 +1,14 @@
1	1	<?php
2	2
3	3	namespace Yoast\WP\SEO\Integrations\Front_End;
4	4
	5	+use WPSEO_Sitemaps_Router;
5	6	use Yoast\WP\SEO\Conditionals\Robots_Txt_Conditional;
6	7	use Yoast\WP\SEO\Helpers\Options_Helper;
	8	+use Yoast\WP\SEO\Helpers\Robots_Txt_Helper;
7	9	use Yoast\WP\SEO\Integrations\Integration_Interface;
	10	+use Yoast\WP\SEO\Presenters\Robots_Txt_Presenter;
8	11
9	12	/**
10	13	* Handles adding the sitemap to the `robots.txt`.
11	14	*/
		@@ -18,14 +21,32 @@
18	21	*/
19	22	protected $options_helper;
20	23
21	24	/**
	25	+ * Holds the robots txt helper.
	26	+ *
	27	+ * @var Robots_Txt_Helper
	28	+ */
	29	+ protected $robots_txt_helper;
	30	+
	31	+ /**
	32	+ * Holds the robots txt presenter.
	33	+ *
	34	+ * @var Robots_Txt_Presenter
	35	+ */
	36	+ protected $robots_txt_presenter;
	37	+
	38	+ /**
22	39	* Sets the helpers.
23	40	*
24		- * @param Options_Helper $options_helper Options helper.
	41	+ * @param Options_Helper $options_helper Options helper.
	42	+ * @param Robots_Txt_Helper $robots_txt_helper Robots txt helper.
	43	+ * @param Robots_Txt_Presenter $robots_txt_presenter Robots txt presenter.
25	44	*/
26		- public function __construct( Options_Helper $options_helper ) {
27		- $this->options_helper = $options_helper;
	45	+ public function __construct( Options_Helper $options_helper, Robots_Txt_Helper $robots_txt_helper, Robots_Txt_Presenter $robots_txt_presenter ) {
	46	+ $this->options_helper = $options_helper;
	47	+ $this->robots_txt_helper = $robots_txt_helper;
	48	+ $this->robots_txt_presenter = $robots_txt_presenter;
28	49	}
29	50
30	51	/**
31	52	* Returns the conditionals based in which this loadable should be active.
		@@ -43,9 +64,19 @@
43	64	*
44	65	* @return void
45	66	*/
46	67	public function register_hooks() {
47		- \add_filter( 'robots_txt', [ $this, 'filter_robots' ], 99999, 2 );
	68	+ \add_filter( 'robots_txt', [ $this, 'filter_robots' ], 99_999 );
	69	+
	70	+ if ( $this->options_helper->get( 'deny_search_crawling' ) && ! \is_multisite() ) {
	71	+ \add_action( 'Yoast\WP\SEO\register_robots_rules', [ $this, 'add_disallow_search_to_robots' ], 10, 1 );
	72	+ }
	73	+ if ( $this->options_helper->get( 'deny_wp_json_crawling' ) && ! \is_multisite() ) {
	74	+ \add_action( 'Yoast\WP\SEO\register_robots_rules', [ $this, 'add_disallow_wp_json_to_robots' ], 10, 1 );
	75	+ }
	76	+ if ( $this->options_helper->get( 'deny_adsbot_crawling' ) && ! \is_multisite() ) {
	77	+ \add_action( 'Yoast\WP\SEO\register_robots_rules', [ $this, 'add_disallow_adsbot' ], 10, 1 );
	78	+ }
48	79	}
49	80
50	81	/**
51	82	* Filters the robots.txt output.
		@@ -50,25 +81,73 @@
50	81	/**
51	82	* Filters the robots.txt output.
52	83	*
53	84	* @param string $robots_txt The robots.txt output from WordPress.
54		- * @param string $public Option that says whether the site is public or not.
55	85	*
56		- * @return string $output Filtered robots.txt output.
	86	+ * @return string Filtered robots.txt output.
57	87	*/
58		- public function filter_robots( $robots_txt, $public ) {
59		- // If the site isn't public, bail.
60		- if ( $public === '0' ) {
61		- return $robots_txt;
	88	+ public function filter_robots( $robots_txt ) {
	89	+ $robots_txt = $this->remove_default_robots( $robots_txt );
	90	+ $this->maybe_add_xml_sitemap();
	91	+
	92	+ /**
	93	+ * Filter: 'wpseo_should_add_subdirectory_multisite_xml_sitemaps' - Disabling this filter removes subdirectory sites from xml sitemaps.
	94	+ *
	95	+ * @since 19.8
	96	+ *
	97	+ * @param bool $show Whether to display multisites in the xml sitemaps.
	98	+ */
	99	+ if ( \apply_filters( 'wpseo_should_add_subdirectory_multisite_xml_sitemaps', true ) ) {
	100	+ $this->add_subdirectory_multisite_xml_sitemaps();
62	101	}
63	102
64		- $robots_txt = $this->change_default_robots( $robots_txt );
65		- $robots_txt = $this->add_xml_sitemap( $robots_txt );
	103	+ /**
	104	+ * Allow registering custom robots rules to be outputted within the Yoast content block in robots.txt.
	105	+ *
	106	+ * @param Robots_Txt_Helper $robots_txt_helper The Robots_Txt_Helper object.
	107	+ */
	108	+ \do_action( 'Yoast\WP\SEO\register_robots_rules', $this->robots_txt_helper );
66	109
67		- return $this->add_subdirectory_multisite_xml_sitemaps( $robots_txt );
	110	+ return \trim( $robots_txt . \PHP_EOL . $this->robots_txt_presenter->present() . \PHP_EOL );
68	111	}
69	112
70	113	/**
	114	+ * Add a disallow rule for search to robots.txt.
	115	+ *
	116	+ * @param Robots_Txt_Helper $robots_txt_helper The robots txt helper.
	117	+ *
	118	+ * @return void
	119	+ */
	120	+ public function add_disallow_search_to_robots( Robots_Txt_Helper $robots_txt_helper ) {
	121	+ $robots_txt_helper->add_disallow( '*', '/?s=' );
	122	+ $robots_txt_helper->add_disallow( '*', '/page/*/?s=' );
	123	+ $robots_txt_helper->add_disallow( '*', '/search/' );
	124	+ }
	125	+
	126	+ /**
	127	+ * Add a disallow rule for /wp-json/ to robots.txt.
	128	+ *
	129	+ * @param Robots_Txt_Helper $robots_txt_helper The robots txt helper.
	130	+ *
	131	+ * @return void
	132	+ */
	133	+ public function add_disallow_wp_json_to_robots( Robots_Txt_Helper $robots_txt_helper ) {
	134	+ $robots_txt_helper->add_disallow( '*', '/wp-json/' );
	135	+ $robots_txt_helper->add_disallow( '*', '/?rest_route=' );
	136	+ }
	137	+
	138	+ /**
	139	+ * Add a disallow rule for AdsBot agents to robots.txt.
	140	+ *
	141	+ * @param Robots_Txt_Helper $robots_txt_helper The robots txt helper.
	142	+ *
	143	+ * @return void
	144	+ */
	145	+ public function add_disallow_adsbot( Robots_Txt_Helper $robots_txt_helper ) {
	146	+ $robots_txt_helper->add_disallow( 'AdsBot', '/' );
	147	+ }
	148	+
	149	+ /**
71	150	* Replaces the default WordPress robots.txt output.
72	151	*
73	152	* @param string $robots_txt Input robots.txt.
74	153	*
		@@ -73,13 +152,13 @@
73	152	* @param string $robots_txt Input robots.txt.
74	153	*
75	154	* @return string
76	155	*/
77		- protected function change_default_robots( $robots_txt ) {
78		- return \str_replace(
79		- "User-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php",
80		- "User-agent: *\nDisallow:",
81		- $robots_txt
	156	+ protected function remove_default_robots( $robots_txt ) {
	157	+ return \preg_replace(
	158	+ '`User-agent: \*[\r\n]+Disallow: /wp-admin/[\r\n]+Allow: /wp-admin/admin-ajax\.php[\r\n]+`',
	159	+ '',
	160	+ $robots_txt,
82	161	);
83	162	}
84	163
85	164	/**
		@@ -84,59 +163,37 @@
84	163
85	164	/**
86	165	* Adds XML sitemap reference to robots.txt.
87	166	*
88		- * @param string $robots_txt Robots.txt input.
89		- *
90		- * @return string
	167	+ * @return void
91	168	*/
92		- protected function add_xml_sitemap( $robots_txt ) {
	169	+ protected function maybe_add_xml_sitemap() {
93	170	// If the XML sitemap is disabled, bail.
94	171	if ( ! $this->options_helper->get( 'enable_xml_sitemap', false ) ) {
95		- return $robots_txt;
	172	+ return;
96	173	}
97		-
98		- $sitemap = 'Sitemap: ' . \esc_url( \WPSEO_Sitemaps_Router::get_base_url( 'sitemap_index.xml' ) );
99		-
100		- // If our sitemap is already output, bail.
101		- if ( \strpos( $robots_txt, $sitemap ) !== false ) {
102		- return $robots_txt;
103		- };
104		-
105		- return \trim( $robots_txt ) . "\n\n" . $sitemap . "\n";
	174	+ $this->robots_txt_helper->add_sitemap( \esc_url( WPSEO_Sitemaps_Router::get_base_url( 'sitemap_index.xml' ) ) );
106	175	}
107	176
108	177	/**
109	178	* Adds subdomain multisite' XML sitemap references to robots.txt.
110	179	*
111		- * @param string $robots_txt Robots.txt input.
112		- *
113		- * @return string
	180	+ * @return void
114	181	*/
115		- protected function add_subdirectory_multisite_xml_sitemaps( $robots_txt ) {
	182	+ protected function add_subdirectory_multisite_xml_sitemaps() {
116	183	// If not on a multisite subdirectory, bail.
117	184	if ( ! \is_multisite() || \is_subdomain_install() ) {
118		- return $robots_txt;
	185	+ return;
119	186	}
120	187
121	188	$sitemaps_enabled = $this->get_xml_sitemaps_enabled();
122		- $sitemaps = "\n\n";
	189	+
123	190	foreach ( $sitemaps_enabled as $blog_id => $is_sitemap_enabled ) {
124	191	if ( ! $is_sitemap_enabled ) {
125	192	continue;
126	193	}
127		-
128		- $sitemap = 'Sitemap: ' . \esc_url( \get_home_url( $blog_id, 'sitemap_index.xml' ) );
129		-
130		- // If our sitemap is already output, bail.
131		- if ( \strpos( $robots_txt, $sitemap ) !== false ) {
132		- continue;
133		- };
134		-
135		- $sitemaps .= $sitemap . "\n";
	194	+ $this->robots_txt_helper->add_sitemap( \esc_url( \get_home_url( $blog_id, 'sitemap_index.xml' ) ) );
136	195	}
137		-
138		- return \trim( $robots_txt ) . $sitemaps;
139	196	}
140	197
141	198	/**
142	199	* Retrieves whether the XML sitemaps are enabled, keyed by blog ID.