botwriter / includes / siterewriter.php

siterewriter.php in BotWriter – AI Writer & SEO Content Generator 3.2.8, at includes/siterewriter.php

376 lines 16.0 KB

Raw Download Zip

1	<?php
2	if ( ! defined( 'ABSPATH' ) ) {
3	exit;
4	}
5
6	/**
7	* Site Rewriter Page Handler
8	* 3-step wizard: Crawl Website → Select & Review Content → Create Rewrite Task
9	*/
10	function botwriter_siterewriter_page_handler() {
11	if (!current_user_can('manage_options')) {
12	return;
13	}
14
15	$dir_images = plugin_dir_url(dirname(__FILE__)) . '/assets/images/';
16	?>
17
18	<div class="wrap">
19	<h1><?php esc_html_e('Site Rewriter', 'botwriter'); ?></h1>
20	<p><?php esc_html_e('Crawl an entire website, select the pages you want, and rewrite them all with AI.', 'botwriter'); ?></p>
21
22	<!-- Step 1: Crawl Settings -->
23	<div id="siterewriter_step1" class="super-ia bw-flex-col">
24	<div class="bw-flex-row-center bw-mb-15">
25	<div class="super-ia-image bw-img-shrink">
26	<img src="<?php echo esc_url($dir_images . 'ai_cerebro.png'); ?>" alt="<?php echo esc_attr__('AI', 'botwriter'); ?>" class="bw-hue-rotate-200 bw-img-80">
27	</div>
28	<div>
29	<h2 class="super-title"><?php esc_html_e('Step 1: Crawl Website', 'botwriter'); ?></h2>
30	<p class="super-text"><?php esc_html_e('Enter the root URL of the website to crawl. The system will discover internal pages and extract their titles.', 'botwriter'); ?></p>
31	</div>
32	</div>
33
34	<div class="col-md-6">
35	<label for="siterewriter_root_url"><?php esc_html_e('Website URL:', 'botwriter'); ?></label>
36	<input type="url" id="siterewriter_root_url" name="siterewriter_root_url"
37	placeholder="https://example.com" class="siterewriter-root-url" />
38	</div>
39	<br>
40
41	<div class="siterewriter-settings-row">
42	<div>
43	<label for="siterewriter_depth"><?php esc_html_e('Crawl Depth:', 'botwriter'); ?></label>
44	<select id="siterewriter_depth" name="siterewriter_depth">
45	<option value="1" selected>1 — <?php esc_html_e('Pages linked from root', 'botwriter'); ?></option>
46	<option value="2">2 — <?php esc_html_e('+ Pages linked from those', 'botwriter'); ?></option>
47	<option value="3">3 — <?php esc_html_e('+ One more level deep', 'botwriter'); ?></option>
48	</select>
49	<p class="form-text"><?php esc_html_e('How many link levels to follow from the root URL.', 'botwriter'); ?></p>
50	</div>
51	<div>
52	<label for="siterewriter_max_urls"><?php esc_html_e('Max Pages:', 'botwriter'); ?></label>
53	<input type="number" id="siterewriter_max_urls" name="siterewriter_max_urls"
54	value="20" min="1" max="200" class="siterewriter-max-urls" />
55	<p class="form-text"><?php esc_html_e('Maximum number of pages to discover (1–200).', 'botwriter'); ?></p>
56	</div>
57	</div>
58	<br>
59
60	<div>
61	<button id="siterewriter_crawl_btn" class="button-primary"><?php esc_html_e('Crawl Website', 'botwriter'); ?></button>
62	<button id="siterewriter_stop_btn" class="button siterewriter-stop-btn"><?php esc_html_e('Stop', 'botwriter'); ?></button>
63	</div>
64	<div id="siterewriter_crawl_status" class="bw-mt-10"></div>
65	<div id="siterewriter_crawl_live" class="siterewriter-crawl-live"></div>
66	</div>
67
68	<!-- Step 2: Select Pages & Fetch Content -->
69	<div id="siterewriter_step2" class="bw-mt-20 siterewriter-step-hidden">
70	<div class="super-ia bw-flex-col">
71	<h2 class="super-title"><?php esc_html_e('Step 2: Select Pages', 'botwriter'); ?></h2>
72	<p class="super-text"><?php esc_html_e('Select which pages to rewrite, then click "Fetch Content" to extract the article text.', 'botwriter'); ?></p>
73
74	<div class="siterewriter-select-row">
75	<label class="siterewriter-select-all-label"><input type="checkbox" id="siterewriter_select_all" checked> <?php esc_html_e('Select / Deselect All', 'botwriter'); ?></label>
76	<span id="siterewriter_selected_count" class="siterewriter-selected-count"></span>
77	</div>
78
79	<div id="siterewriter_pages_list"></div>
80	<br>
81	<div>
82	<button id="siterewriter_fetch_btn" class="button-primary"><?php esc_html_e('Use Selected Pages', 'botwriter'); ?></button>
83	</div>
84	<div id="siterewriter_fetch_status" class="bw-mt-10"></div>
85	</div>
86	</div>
87
88	<!-- Step 3: Review Content & Create Task -->
89	<div id="siterewriter_step3" class="bw-mt-20 siterewriter-step-hidden">
90	<div class="super-ia bw-flex-col">
91	<h2 class="super-title"><?php esc_html_e('Step 3: Review & Create Task', 'botwriter'); ?></h2>
92	<p class="super-text"><?php esc_html_e('Review extracted content, set rewrite instructions, and configure the task.', 'botwriter'); ?></p>
93
94	<div id="siterewriter_articles_list"></div>
95	<br>
96
97	<div class="col-md-6">
98	<label for="siterewriter_prompt"><?php esc_html_e('Rewrite Instructions:', 'botwriter'); ?></label>
99	<textarea id="siterewriter_prompt" name="siterewriter_prompt" rows="4" cols="60"
100	placeholder="<?php esc_attr_e('e.g. Rewrite in a friendly conversational tone. Add practical examples. Make it 30% longer.', 'botwriter'); ?>"></textarea>
101	<p class="form-text"><?php esc_html_e('Global instructions applied to every article.', 'botwriter'); ?></p>
102	</div>
103	<br>
104
105	<!-- Category -->
106	<div class="bw-rewriter-props-extra">
107	<div class="col-md-6">
108	<label for="siterewriter_category" class="form-label"><?php esc_html_e('Category:', 'botwriter'); ?></label>
109	<select id="siterewriter_category" name="siterewriter_category" class="form-select">
110	<option value="0"><?php esc_html_e('— Default —', 'botwriter'); ?></option>
111	<?php
112	$categories = get_categories(array('orderby' => 'name', 'order' => 'ASC', 'hide_empty' => false));
113	foreach ($categories as $category) {
114	echo '<option value="' . esc_attr($category->term_id) . '">' . esc_html($category->name) . '</option>';
115	}
116	?>
117	</select>
118	<p class="form-text"><?php esc_html_e('Category for the rewritten posts.', 'botwriter'); ?></p>
119	</div>
120	</div>
121	<br>
122
123	<!-- Shared task properties form (writer, language, length, days…) -->
124	<?php
125	$siterewriter_default_item = array(
126	/* translators: %s: current date and time */
127	'task_name' => sprintf(__('Site Rewriter %s', 'botwriter'), wp_date('M j, Y H:i')),
128	'writer' => 'ai_cerebro',
129	'narration' => 'Descriptive',
130	'custom_style' => '',
131	'post_language' => substr(get_locale(), 0, 2),
132	'post_length' => '800',
133	'custom_post_length' => '',
134	'post_status' => 'draft',
135	'days' => 'Monday,Tuesday,Wednesday,Thursday,Friday,Saturday,Sunday',
136	'times_per_day' => 1,
137	'author_selection' => strval(get_current_user_id()),
138	'disable_ai_images' => 0,
139	'template_id' => null,
140	);
141	botwriter_super_form_meta_box_handler($siterewriter_default_item);
142	?>
143	<br>
144
145	<div>
146	<button id="siterewriter_create_btn" class="button-primary"><?php esc_html_e('Create Rewrite Task', 'botwriter'); ?></button>
147	</div>
148	<div id="siterewriter_create_status" class="bw-mt-10"></div>
149	</div>
150	</div>
151
152	</div>
153
154	<?php
155	}
156
157
158	// ========================================================================
159	// Crawl functions
160	// ========================================================================
161
162	/**
163	* Crawl a single page: fetch it, extract the title, and return internal links found.
164	* The BFS queue is managed client-side in JS for progressive/live UI updates.
165	*
166	* @param string $url The URL to fetch.
167	* @param string $base_domain The domain to restrict links to.
168	* @return array\|WP_Error [ 'url', 'title', 'links' => [...] ]
169	*/
170	function botwriter_siterewriter_crawl_page($url, $base_domain) {
171	$parsed = wp_parse_url($url);
172	$scheme = isset($parsed['scheme']) ? $parsed['scheme'] : 'https';
173	$base_url = $scheme . '://' . $base_domain;
174
175	$ssl_verify = get_option('botwriter_sslverify', 'yes') === 'yes';
176	$http_args = array(
177	'timeout' => 20,
178	'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
179	'sslverify' => $ssl_verify,
180	'redirection' => 5,
181	'headers' => array(
182	'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,/;q=0.8',
183	'Accept-Language' => 'en-US,en;q=0.9,es;q=0.8',
184	'Accept-Encoding' => 'identity',
185	'Cache-Control' => 'no-cache',
186	'Sec-Fetch-Dest' => 'document',
187	'Sec-Fetch-Mode' => 'navigate',
188	'Sec-Fetch-Site' => 'none',
189	'Sec-Fetch-User' => '?1',
190	'Upgrade-Insecure-Requests' => '1',
191	'Sec-Ch-Ua' => '"Chromium";v="131", "Not_A Brand";v="24"',
192	'Sec-Ch-Ua-Mobile' => '?0',
193	'Sec-Ch-Ua-Platform' => '"Windows"',
194	),
195	);
196
197	$response = wp_remote_get($url, $http_args);
198	if (is_wp_error($response)) {
199	return new WP_Error('fetch_error', $response->get_error_message());
200	}
201
202	$status_code = wp_remote_retrieve_response_code($response);
203	if ($status_code !== 200) {
204	return new WP_Error('http_error', sprintf('HTTP %d', $status_code));
205	}
206
207	$content_type = wp_remote_retrieve_header($response, 'content-type');
208	if ($content_type && strpos($content_type, 'text/html') === false) {
209	return new WP_Error('not_html', 'Not an HTML page.');
210	}
211
212	$html = wp_remote_retrieve_body($response);
213	if (empty($html)) {
214	return new WP_Error('empty', 'Empty response.');
215	}
216
217	$title = botwriter_siterewriter_extract_title($html);
218	$links = botwriter_siterewriter_extract_links($html, $base_url, $base_domain);
219
220	// Also extract article content so the second fetch step can be skipped
221	$content = '';
222	$content_warning = '';
223	if (function_exists('botwriter_rewriter_extract_content_from_html')) {
224	$extract = botwriter_rewriter_extract_content_from_html($html, $url);
225	if (!is_wp_error($extract)) {
226	$content = $extract['content'] ?? '';
227	$content_warning = $extract['content_warning'] ?? '';
228	}
229	}
230
231	return array(
232	'url' => $url,
233	'title' => $title,
234	'links' => $links,
235	'content' => $content,
236	'content_warning' => $content_warning,
237	);
238	}
239
240
241	/**
242	* Extract internal links from an HTML page.
243	*
244	* @param string $html Full page HTML.
245	* @param string $base_url Scheme + host of the target site (e.g. https://example.com).
246	* @param string $base_domain Host only (e.g. example.com).
247	* @return array Unique normalized URLs.
248	*/
249	function botwriter_siterewriter_extract_links($html, $base_url, $base_domain) {
250	libxml_use_internal_errors(true);
251	$doc = new DOMDocument();
252	$doc->loadHTML('<?xml encoding="UTF-8">' . $html, LIBXML_NOWARNING \| LIBXML_NOERROR);
253	libxml_clear_errors();
254
255	$anchors = $doc->getElementsByTagName('a');
256	$links = array();
257
258	$skip_extensions = array(
259	'jpg','jpeg','png','gif','svg','webp','pdf','css','js','zip','rar',
260	'mp3','mp4','avi','doc','docx','xls','xlsx','ico','woff','woff2','ttf','eot',
261	);
262
263	$skip_patterns = array(
264	'/wp-admin', '/wp-login', '/wp-content/', '/wp-includes/',
265	'/feed', '/tag/', '/author/', '/page/',
266	'/cart', '/checkout', '/my-account', '/search',
267	'#', 'javascript:', 'mailto:', 'tel:',
268	'?replytocom=', '?share=', '/xmlrpc.php', '/wp-json/',
269	'/category/', '/categories/', '/archivo/', '/archives/', '/etiqueta/',
270	);
271
272	foreach ($anchors as $anchor) {
273	$href = trim($anchor->getAttribute('href'));
274	if (empty($href) \|\| $href === '/' \|\| $href === '#') {
275	continue;
276	}
277
278	// Skip unwanted patterns
279	$skip = false;
280	foreach ($skip_patterns as $pattern) {
281	if (stripos($href, $pattern) !== false) {
282	$skip = true;
283	break;
284	}
285	}
286	if ($skip) {
287	continue;
288	}
289
290	// Resolve relative URLs
291	if (strpos($href, 'http') !== 0 && strpos($href, '//') !== 0) {
292	$href = (strpos($href, '/') === 0)
293	? $base_url . $href
294	: rtrim($base_url, '/') . '/' . $href;
295	}
296
297	// Protocol-relative
298	if (strpos($href, '//') === 0) {
299	$href = 'https:' . $href;
300	}
301
302	// Parse and validate domain
303	$parsed = wp_parse_url($href);
304	if (!isset($parsed['host'])) {
305	continue;
306	}
307
308	$link_domain = strtolower($parsed['host']);
309	$check_domain = strtolower($base_domain);
310	if (
311	$link_domain !== $check_domain &&
312	$link_domain !== 'www.' . $check_domain &&
313	'www.' . $link_domain !== $check_domain
314	) {
315	continue;
316	}
317
318	// Skip non-HTML extensions
319	$path = isset($parsed['path']) ? $parsed['path'] : '/';
320	$ext = strtolower(pathinfo($path, PATHINFO_EXTENSION));
321	if (!empty($ext) && in_array($ext, $skip_extensions)) {
322	continue;
323	}
324
325	// Normalize for dedup
326	$norm = (isset($parsed['scheme']) ? $parsed['scheme'] : 'https')
327	. '://' . $parsed['host'] . rtrim($path, '/');
328
329	$links[$norm] = true;
330	}
331
332	return array_keys($links);
333	}
334
335
336	/**
337	* Quick title extraction from raw HTML using regex (no full DOM parse needed).
338	*/
339	function botwriter_siterewriter_extract_title($html) {
340	// og:title (property before content)
341	if (preg_match('/<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']/i', $html, $m)) {
342	return sanitize_text_field(html_entity_decode($m[1], ENT_QUOTES, 'UTF-8'));
343	}
344	// og:title (content before property)
345	if (preg_match('/<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']og:title["\']/i', $html, $m)) {
346	return sanitize_text_field(html_entity_decode($m[1], ENT_QUOTES, 'UTF-8'));
347	}
348	// <title>
349	if (preg_match('/<title[^>]>(.?)<\/title>/is', $html, $m)) {
350	return sanitize_text_field(html_entity_decode(trim($m[1]), ENT_QUOTES, 'UTF-8'));
351	}
352	// <h1>
353	if (preg_match('/<h1[^>]>(.?)<\/h1>/is', $html, $m)) {
354	return sanitize_text_field(html_entity_decode(wp_strip_all_tags(trim($m[1])), ENT_QUOTES, 'UTF-8'));
355	}
356
357	return '';
358	}
359
360
361	/**
362	* Normalize a URL for deduplication (lowercased scheme+host, no trailing slash, no fragment).
363	*/
364	function botwriter_siterewriter_normalize_url($url) {
365	$parsed = wp_parse_url($url);
366	if (!$parsed \|\| !isset($parsed['host'])) {
367	return $url;
368	}
369
370	$scheme = isset($parsed['scheme']) ? strtolower($parsed['scheme']) : 'https';
371	$host = strtolower($parsed['host']);
372	$path = isset($parsed['path']) ? rtrim($parsed['path'], '/') : '';
373
374	return $scheme . '://' . $host . $path;
375	}
376