# wt-security/2.4.21/lib/modules/logs/Crawler.php

WebTotem Security, version 2.4.21. 297 lines.

- Page: https://pluginprobe.com/plugins/wt-security/2.4.21/code/lib/modules/logs/Crawler.php
- Raw: https://pluginprobe.com/plugins/wt-security/2.4.21/raw/lib/modules/logs/Crawler.php
- Modified: 2023-08-09T05:34:10+00:00

Line numbers below start at 1. Link to a line or a range by appending a fragment to the
page URL, for example `https://pluginprobe.com/plugins/wt-security/2.4.21/code/lib/modules/logs/Crawler.php#L10-L20`.

```php
<?php

if (!defined('WEBTOTEM_INIT') || WEBTOTEM_INIT !== true) {
	if (!headers_sent()) {
		header('HTTP/1.1 403 Forbidden');
	}
	die("Protected By WebTotem!");
}

/**
 * WebTotem page scan class for Wordpress.
 */
class WebTotemCrawler
{
	/**
	 * Running a single iteration
	 */
	public static function init($scan_temp) {

		$temp_data = json_decode(WebTotemOption::getOption('crawler_temp'), true) ?: [];

		$i = 1;
		if (!$temp_data) {

			$pre_scan = self::pre_scan();

			$temp_data['internal']['new'] = $pre_scan['internal'];
			$temp_data['external'] = WebTotem::arrayUniqueKey($pre_scan['external'], 'link');
			$temp_data['scripts'] = WebTotem::arrayUniqueKey($pre_scan['scripts'], 'link');
			$temp_data['iframes'] = WebTotem::arrayUniqueKey( $pre_scan['iframes'], 'link');
			$temp_data['exclude'] = array_unique($pre_scan['exclude']);

			$temp_data['internal']['new'] = WebTotem::arrayUniqueKey( array_merge($temp_data['internal']['new'], $scan_temp['links']), 'link');

			$i++;
		}

		foreach ($temp_data['internal']['new'] as $item) {

			if($result = self::explore_page($item['link'], $temp_data['exclude'])) {
				$temp_data['internal']['visited'][] = $item;

				$temp_data['internal']['new'] = WebTotem::arrayUniqueKey(array_merge($temp_data['internal']['new'], $result['internal']), 'link');
				$temp_data['external'] = WebTotem::arrayUniqueKey(array_merge($temp_data['external'], $result['external']), 'link');
				$temp_data['scripts'] = WebTotem::arrayUniqueKey(array_merge($temp_data['scripts'], $result['scripts']), 'link');
				$temp_data['iframes'] = WebTotem::arrayUniqueKey(array_merge($temp_data['iframes'], $result['iframes']), 'link');
				$temp_data['exclude'] = array_merge($temp_data['exclude'], $result['exclude']);
			}

			$key = array_search($item, $temp_data['internal']['new']);
			if ($key !== false) {
				unset($temp_data['internal']['new'][$key]);
			}

			if ($i >= 5) break;
			$i++;
		}

		if (empty($temp_data['internal']['new'])) {

			if($scan_temp['ready_to_save']){

				$data = [
						'links' => array_merge($temp_data['internal']['visited'], $temp_data['external']),
						'scripts' => $temp_data['scripts'],
						'iframes' => $temp_data['iframes'],
				];
				self::saveData($data);

				WebTotemOption::setOptions(['crawler_temp' => '']);
				WebTotemOption::setOptions(['scan_temp' => '']);
				WebTotemOption::setOptions(['scan_init' => 0]);

				// Resetting the task in the cron.
				wp_clear_scheduled_hook('webtotem_daily_cron');
				wp_schedule_event(time() + 86395, 'daily', 'webtotem_daily_cron');
			} else {
				WebTotemOption::setOptions([
						'scan_temp' => [
								'current_scan' => 'crawler',
								'links' => [],
								'ready_to_save' => true,
						]
				]);
			}

		} else {
			WebTotemOption::setOptions(['crawler_temp' => $temp_data]);
		}

	}

	private static function pre_scan() {
		$site_url = get_site_url();
		$internal = [];
		$exclude = [];

		// Сканируем файл robots.txt
		$robotsTxt = file_get_contents(ABSPATH . '/robots.txt');
		$lines = explode("\n", $robotsTxt);

		foreach ($lines as $line) {
			if (strpos($line, 'Disallow:') === 0 || strpos($line, 'Allow:') === 0) {
				$url = trim(substr($line, strpos($line, ':') + 1));
				$exclude[] = $url;
				$robots_urls[] = (string)$url->loc;
			}
		}

		foreach ($robots_urls as $url) {
			if (substr($url, 0, 1) == "#") {
				continue;
			}
			$internal[] = ['link' => $url, 'page' => $site_url . '/robots.txt', 'is_internal' => self::isInternal($url)];
		}

		// Добавляем ссылки из популярных sitemaps плагинов
		$sitemaps = [
				$site_url . '/sitemaps.xml',
				$site_url . '/index.php?xml_sitemap=params=.',
				$site_url . '/?sitemap=1',
				$site_url . '/sitemap_index.xml',
		];

		foreach ($sitemaps as $url) {
			$internal[] = ['link' => $url, 'page' => __('by sitemap plugins', 'wtotem'), 'is_internal' => true];
		}

		// Сканируем файл sitemap.xml
		$xml = simplexml_load_file(ABSPATH . '/sitemap.xml');

		$sitemap_urls = [];
		foreach ($xml->url as $url) {
			$exclude[] = (string)$url->loc;
			$sitemap_urls[] = (string)$url->loc;
		}

		foreach ($sitemap_urls as $url) {
			if (substr($url, 0, 1) == "#") {
				continue;
			}
			$internal[] = ['link' => $url, 'page' => $site_url . '/sitemap.xml', 'is_internal' => self::isInternal($url)];
		}

		// Сканируем главную страницу
		$result = self::explore_page($site_url);

		$internal = array_merge($internal, $result['internal']);
		$external = array_unique($result['external']);
		$exclude =  array_merge($exclude, $result['exclude']);


		return [
				'internal' => $internal ?: [],
				'external' => $external ?: [],
				'scripts' => $result['scripts'] ?: [],
				'iframes' => $result['iframes'] ?: [],
				'exclude' => $exclude ?: [],
		];

	}

	private static function explore_page($url, $exclude = []) {

		$headers = get_headers($url);

		if ($headers === false || strpos($headers[0], '200 OK') === false) {
			return false;
		}

		// Initializing the cURL session
		$curl = curl_init();

		// Setting the parameters of the cURL session
		curl_setopt($curl, CURLOPT_URL, $url); // Устанавливаем URL
		curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); // Возвращаем результат в виде строки
		curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); // Следуем за редиректами
		curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false); // Отключаем проверку SSL-сертификата

		// Execute the request and get the content of the page
		$content = curl_exec($curl);

		// Checking for errors when executing the request
		if (curl_errno($curl)) {
			WebTotemOption::setNotification('error', __('Request execution error: ', 'wtotem')) . curl_error($curl);
		}

		// Closing the cURL session
		curl_close($curl);

		// Checking the content for matches with the template, using regular expressions
		return self::getMatches($content, $url, $exclude);

	}

	private static function getMatches($content, $url, $exclude) {

		$pattern = '/(<a.*?href=["\'](([\da-z\.-\/]+)([\/\w\.-\?\%\&]*)*\/?)["\'].*?>|<script.*?src=["\'](.*?)["\'].*?>|<iframe.*?src=["\'](.*?)["\'].*?>|onclick="[^"]*location[^"][^\'"]+\'([^\']+)\')/i';
		preg_match_all($pattern, $content, $all_matches);

		$array = [
				'links' => [],
				'scripts' => [],
				'iframes' => [],
		];

		foreach ($all_matches[0] as $match) {
			preg_match_all('/<a.*?href=["\'](.*?)["\'].*?>/i', $match, $links_matches);
			if ($links_matches[1]) $array['links'] = array_merge($array['links'], $links_matches[1]);
			preg_match_all('/onclick="[^"]*location[^"][^\'"]+\'([^\']+)\'/i', $match, $links_2_matches);
			if ($links_2_matches[1]) $array['links'] = array_merge($array['links'], $links_2_matches[1]);
			preg_match_all('/<script.*?src=["\'](.*?)["\'].*?>/i', $match, $js_matches);
			if ($js_matches[1]) $array['scripts'] = array_merge($array['scripts'], $js_matches[1]);
			preg_match_all('/<iframe.*?src=["\'](.*?)["\'].*?>/i', $match, $iframe_matches);
			if ($iframe_matches[1]) $array['iframes'] = array_merge($array['iframes'], $iframe_matches[1]);
		}

		$matches = [
				'internal' => [],
				'external' => [],
				'exclude' => [],
				'scripts' => [],
				'iframe' => [],
		];

		foreach ($array['links'] as $link) {
				if (self::isInternal($link)) {
					if (substr($link, 0, 1) == "#") {
						continue;
					}
					if (in_array($link, $exclude)) {
						continue;
					}
					$matches['internal'][] = ['link' => $link, 'page' => $url, 'is_internal' => true];
					$matches['exclude'][] = $link;
				} else {
					$matches['external'][] = ['link' => $link, 'page' => $url, 'is_internal' => false];
				}
		}

		foreach (array_unique($array['scripts']) as $script) {
			$matches['scripts'][] = ['link' => $script, 'page' => $url, 'is_internal' => self::isInternal($script)];
		}
		foreach (array_unique($array['iframes']) as $iframe) {
			$matches['iframe'][] = ['link' => $iframe, 'page' => $url, 'is_internal' => self::isInternal($iframe)];
		}

		return $matches;
	}

	private static function isInternal($string): bool {
		$current_domain_parts = parse_url(get_home_url());
		$current_domain = $current_domain_parts['host'];

		if (substr($string, 0, 5) == "https"
				|| substr($string, 0, 4) == "http"
				|| substr($string, 0, 2) == "//") {

			if (strpos($string, $current_domain) === false) {
				return false;
			}
		}

		return true;
	}

	/**
	 * Save data.
	 *
	 * @param array $data
	 *    Array matches data.
	 */
	private static function saveData($data) {

		WebTotemDB::deleteData([], 'scan_logs');
		$values = '';
		foreach ($data as $data_type => $links) {
			foreach ($links as $datum) {
				$values .= sprintf("('%s','%s','%s','%s','%s'),",
						date("Y-m-d H:i:s"),
						$data_type,
						$datum['page'],
						$datum['link'],
						$datum['is_internal']
				);
			}
		}

		$values = substr_replace($values, ";", -1);

		$columns = '(created_at, data_type, source, content, is_internal)';

		WebTotemDB::setRows('scan_logs', $columns, $values);
	}

}

```