wordpress-seo / src / integrations / front-end / robots-txt-integration.php

robots-txt-integration.php in Yoast SEO – Advanced SEO with real-time guidance and built-in AI trunk, at src/integrations/front-end/robots-txt-integration.php

293 lines 7.9 KB

Raw Download Zip

1	<?php
2
3	namespace Yoast\WP\SEO\Integrations\Front_End;
4
5	use WPSEO_Sitemaps_Router;
6	use Yoast\WP\SEO\Conditionals\Robots_Txt_Conditional;
7	use Yoast\WP\SEO\Helpers\Options_Helper;
8	use Yoast\WP\SEO\Helpers\Robots_Txt_Helper;
9	use Yoast\WP\SEO\Integrations\Integration_Interface;
10	use Yoast\WP\SEO\Presenters\Robots_Txt_Presenter;
11
12	/**
13	* Handles adding the sitemap to the `robots.txt`.
14	*/
15	class Robots_Txt_Integration implements Integration_Interface {
16
17	/**
18	* Holds the options helper.
19	*
20	* @var Options_Helper
21	*/
22	protected $options_helper;
23
24	/**
25	* Holds the robots txt helper.
26	*
27	* @var Robots_Txt_Helper
28	*/
29	protected $robots_txt_helper;
30
31	/**
32	* Holds the robots txt presenter.
33	*
34	* @var Robots_Txt_Presenter
35	*/
36	protected $robots_txt_presenter;
37
38	/**
39	* Sets the helpers.
40	*
41	* @param Options_Helper $options_helper Options helper.
42	* @param Robots_Txt_Helper $robots_txt_helper Robots txt helper.
43	* @param Robots_Txt_Presenter $robots_txt_presenter Robots txt presenter.
44	*/
45	public function __construct( Options_Helper $options_helper, Robots_Txt_Helper $robots_txt_helper, Robots_Txt_Presenter $robots_txt_presenter ) {
46	$this->options_helper = $options_helper;
47	$this->robots_txt_helper = $robots_txt_helper;
48	$this->robots_txt_presenter = $robots_txt_presenter;
49	}
50
51	/**
52	* Returns the conditionals based in which this loadable should be active.
53	*
54	* @return array
55	*/
56	public static function get_conditionals() {
57	return [ Robots_Txt_Conditional::class ];
58	}
59
60	/**
61	* Initializes the integration.
62	*
63	* This is the place to register hooks and filters.
64	*
65	* @return void
66	*/
67	public function register_hooks() {
68	\add_filter( 'robots_txt', [ $this, 'filter_robots' ], 99_999 );
69
70	if ( $this->options_helper->get( 'deny_search_crawling' ) && ! \is_multisite() ) {
71	\add_action( 'Yoast\WP\SEO\register_robots_rules', [ $this, 'add_disallow_search_to_robots' ], 10, 1 );
72	}
73	if ( $this->options_helper->get( 'deny_wp_json_crawling' ) && ! \is_multisite() ) {
74	\add_action( 'Yoast\WP\SEO\register_robots_rules', [ $this, 'add_disallow_wp_json_to_robots' ], 10, 1 );
75	}
76	if ( $this->options_helper->get( 'deny_adsbot_crawling' ) && ! \is_multisite() ) {
77	\add_action( 'Yoast\WP\SEO\register_robots_rules', [ $this, 'add_disallow_adsbot' ], 10, 1 );
78	}
79	}
80
81	/**
82	* Filters the robots.txt output.
83	*
84	* @param string $robots_txt The robots.txt output from WordPress.
85	*
86	* @return string Filtered robots.txt output.
87	*/
88	public function filter_robots( $robots_txt ) {
89	$robots_txt = $this->remove_default_robots( $robots_txt );
90	$this->maybe_add_xml_sitemap();
91
92	/**
93	* Filter: 'wpseo_should_add_subdirectory_multisite_xml_sitemaps' - Disabling this filter removes subdirectory sites from xml sitemaps.
94	*
95	* @since 19.8
96	*
97	* @param bool $show Whether to display multisites in the xml sitemaps.
98	*/
99	if ( \apply_filters( 'wpseo_should_add_subdirectory_multisite_xml_sitemaps', true ) ) {
100	$this->add_subdirectory_multisite_xml_sitemaps();
101	}
102
103	/**
104	* Allow registering custom robots rules to be outputted within the Yoast content block in robots.txt.
105	*
106	* @param Robots_Txt_Helper $robots_txt_helper The Robots_Txt_Helper object.
107	*/
108	\do_action( 'Yoast\WP\SEO\register_robots_rules', $this->robots_txt_helper );
109
110	return \trim( $robots_txt . \PHP_EOL . $this->robots_txt_presenter->present() . \PHP_EOL );
111	}
112
113	/**
114	* Add a disallow rule for search to robots.txt.
115	*
116	* @param Robots_Txt_Helper $robots_txt_helper The robots txt helper.
117	*
118	* @return void
119	*/
120	public function add_disallow_search_to_robots( Robots_Txt_Helper $robots_txt_helper ) {
121	$robots_txt_helper->add_disallow( '*', '/?s=' );
122	$robots_txt_helper->add_disallow( '', '/page//?s=' );
123	$robots_txt_helper->add_disallow( '*', '/search/' );
124	}
125
126	/**
127	* Add a disallow rule for /wp-json/ to robots.txt.
128	*
129	* @param Robots_Txt_Helper $robots_txt_helper The robots txt helper.
130	*
131	* @return void
132	*/
133	public function add_disallow_wp_json_to_robots( Robots_Txt_Helper $robots_txt_helper ) {
134	$robots_txt_helper->add_disallow( '*', '/wp-json/' );
135	$robots_txt_helper->add_disallow( '*', '/?rest_route=' );
136	}
137
138	/**
139	* Add a disallow rule for AdsBot agents to robots.txt.
140	*
141	* @param Robots_Txt_Helper $robots_txt_helper The robots txt helper.
142	*
143	* @return void
144	*/
145	public function add_disallow_adsbot( Robots_Txt_Helper $robots_txt_helper ) {
146	$robots_txt_helper->add_disallow( 'AdsBot', '/' );
147	}
148
149	/**
150	* Replaces the default WordPress robots.txt output.
151	*
152	* @param string $robots_txt Input robots.txt.
153	*
154	* @return string
155	*/
156	protected function remove_default_robots( $robots_txt ) {
157	return \preg_replace(
158	'`User-agent: \*[\r\n]+Disallow: /wp-admin/[\r\n]+Allow: /wp-admin/admin-ajax\.php[\r\n]+`',
159	'',
160	$robots_txt,
161	);
162	}
163
164	/**
165	* Adds XML sitemap reference to robots.txt.
166	*
167	* @return void
168	*/
169	protected function maybe_add_xml_sitemap() {
170	// If the XML sitemap is disabled, bail.
171	if ( ! $this->options_helper->get( 'enable_xml_sitemap', false ) ) {
172	return;
173	}
174	$this->robots_txt_helper->add_sitemap( \esc_url( WPSEO_Sitemaps_Router::get_base_url( 'sitemap_index.xml' ) ) );
175	}
176
177	/**
178	* Adds subdomain multisite' XML sitemap references to robots.txt.
179	*
180	* @return void
181	*/
182	protected function add_subdirectory_multisite_xml_sitemaps() {
183	// If not on a multisite subdirectory, bail.
184	if ( ! \is_multisite() \|\| \is_subdomain_install() ) {
185	return;
186	}
187
188	$sitemaps_enabled = $this->get_xml_sitemaps_enabled();
189
190	foreach ( $sitemaps_enabled as $blog_id => $is_sitemap_enabled ) {
191	if ( ! $is_sitemap_enabled ) {
192	continue;
193	}
194	$this->robots_txt_helper->add_sitemap( \esc_url( \get_home_url( $blog_id, 'sitemap_index.xml' ) ) );
195	}
196	}
197
198	/**
199	* Retrieves whether the XML sitemaps are enabled, keyed by blog ID.
200	*
201	* @return array
202	*/
203	protected function get_xml_sitemaps_enabled() {
204	$is_allowed = $this->is_sitemap_allowed();
205	$blog_ids = $this->get_blog_ids();
206	$is_enabled = [];
207	foreach ( $blog_ids as $blog_id ) {
208	$is_enabled[ $blog_id ] = $is_allowed && $this->is_sitemap_enabled_for( $blog_id );
209	}
210
211	return $is_enabled;
212	}
213
214	/**
215	* Retrieves whether the sitemap is allowed on a sub site.
216	*
217	* @return bool
218	*/
219	protected function is_sitemap_allowed() {
220	$options = \get_network_option( null, 'wpseo_ms' );
221	if ( ! $options \|\| ! isset( $options['allow_enable_xml_sitemap'] ) ) {
222	// Default is enabled.
223	return true;
224	}
225
226	return (bool) $options['allow_enable_xml_sitemap'];
227	}
228
229	/**
230	* Retrieves whether the sitemap is enabled on a site.
231	*
232	* @param int $blog_id The blog ID.
233	*
234	* @return bool
235	*/
236	protected function is_sitemap_enabled_for( $blog_id ) {
237	if ( ! $this->is_yoast_active_on( $blog_id ) ) {
238	return false;
239	}
240
241	$options = \get_blog_option( $blog_id, 'wpseo' );
242	if ( ! $options \|\| ! isset( $options['enable_xml_sitemap'] ) ) {
243	// Default is enabled.
244	return true;
245	}
246
247	return (bool) $options['enable_xml_sitemap'];
248	}
249
250	/**
251	* Determines whether Yoast SEO is active.
252	*
253	* @param int $blog_id The blog ID.
254	*
255	* @return bool
256	*/
257	protected function is_yoast_active_on( $blog_id ) {
258	return \in_array( 'wordpress-seo/wp-seo.php', (array) \get_blog_option( $blog_id, 'active_plugins', [] ), true ) \|\| $this->is_yoast_active_for_network();
259	}
260
261	/**
262	* Determines whether Yoast SEO is active for the entire network.
263	*
264	* @return bool
265	*/
266	protected function is_yoast_active_for_network() {
267	$plugins = \get_network_option( null, 'active_sitewide_plugins' );
268	if ( isset( $plugins['wordpress-seo/wp-seo.php'] ) ) {
269	return true;
270	}
271
272	return false;
273	}
274
275	/**
276	* Retrieves the blog IDs of public, "active" sites on the network.
277	*
278	* @return array
279	*/
280	protected function get_blog_ids() {
281	$criteria = [
282	'archived' => 0,
283	'deleted' => 0,
284	'public' => 1,
285	'spam' => 0,
286	'fields' => 'ids',
287	'network_id' => \get_current_network_id(),
288	];
289
290	return \get_sites( $criteria );
291	}
292	}
293