SiteContentDetector.php — Matomo Analytics – Powerful, Privacy-First Insights for WordPress 4.15.2

matomo / app / core / SiteContentDetector.php

matomo / app / core Last commit date

API 3 years ago Access 3 years ago Application 4 years ago Archive 3 years ago ArchiveProcessor 3 years ago Archiver 5 years ago AssetManager 3 years ago Auth 3 years ago Category 5 years ago Changes 3 years ago CliMulti 4 years ago Columns 3 years ago Concurrency 3 years ago Config 4 years ago Container 4 years ago CronArchive 3 years ago DataAccess 3 years ago DataFiles 5 years ago DataTable 3 years ago Db 3 years ago DeviceDetector 3 years ago Email 5 years ago Exception 3 years ago Http 4 years ago Intl 4 years ago Mail 3 years ago Measurable 5 years ago Menu 3 years ago Metrics 4 years ago Notification 4 years ago Period 4 years ago Plugin 2 years ago ProfessionalServices 4 years ago Report 5 years ago ReportRenderer 3 years ago Scheduler 4 years ago Segment 3 years ago Session 4 years ago Settings 3 years ago Tracker 3 years ago Translation 4 years ago UpdateCheck 5 years ago Updater 3 years ago Updates 3 years ago Validators 4 years ago View 4 years ago ViewDataTable 4 years ago Visualization 4 years ago Widget 5 years ago .htaccess 2 years ago Access.php 4 years ago Archive.php 3 years ago ArchiveProcessor.php 3 years ago AssetManager.php 4 years ago Auth.php 5 years ago AuthResult.php 5 years ago BaseFactory.php 5 years ago Cache.php 5 years ago CacheId.php 5 years ago CliMulti.php 4 years ago Common.php 3 years ago Config.php 3 years ago Console.php 3 years ago Context.php 5 years ago Cookie.php 3 years ago CronArchive.php 3 years ago DataArray.php 4 years ago DataTable.php 3 years ago Date.php 3 years ago Db.php 4 years ago DbHelper.php 3 years ago Development.php 5 years ago ErrorHandler.php 5 years ago EventDispatcher.php 3 years ago ExceptionHandler.php 3 years ago FileIntegrity.php 3 years ago Filechecks.php 4 years ago Filesystem.php 3 years ago FrontController.php 3 years ago Http.php 3 years ago IP.php 5 years ago Log.php 5 years ago LogDeleter.php 4 years ago Mail.php 3 years ago Metrics.php 3 years ago NoAccessException.php 5 years ago Nonce.php 3 years ago Notification.php 5 years ago NumberFormatter.php 4 years ago Option.php 5 years ago Period.php 5 years ago Piwik.php 3 years ago Plugin.php 4 years ago Profiler.php 5 years ago ProxyHeaders.php 5 years ago ProxyHttp.php 4 years ago QuickForm2.php 5 years ago RankingQuery.php 3 years ago ReportRenderer.php 4 years ago Segment.php 3 years ago Sequence.php 5 years ago Session.php 3 years ago SettingsPiwik.php 3 years ago SettingsServer.php 5 years ago Singleton.php 5 years ago Site.php 4 years ago SiteContentDetector.php 3 years ago SupportedBrowser.php 3 years ago TCPDF.php 5 years ago Theme.php 5 years ago Timer.php 5 years ago Tracker.php 4 years ago Twig.php 4 years ago Unzip.php 5 years ago UpdateCheck.php 5 years ago Updater.php 4 years ago UpdaterErrorException.php 5 years ago Updates.php 5 years ago Url.php 3 years ago UrlHelper.php 3 years ago Version.php 2 years ago View.php 3 years ago bootstrap.php 3 years ago dispatch.php 5 years ago testMinimumPhpVersion.php 3 years ago

SiteContentDetector.php

447 lines

1	<?php
2
3	/**
4	* Matomo - free/libre analytics platform
5	*
6	* @link https://matomo.org
7	* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
8	*
9	*/
10
11	namespace Piwik;
12
13	use Matomo\Cache\Lazy;
14	use Piwik\Config\GeneralConfig;
15	use Piwik\Plugins\SitesManager\GtmSiteTypeGuesser;
16	use Piwik\Plugins\SitesManager\SitesManager;
17
18	/**
19	* This class provides detection functions for specific content on a site. It can be used to easily detect the
20	* presence of known third party code.
21	*
22	* Note: Calling the detect() method will create a HTTP request to the site to retrieve data, only the main site URL
23	* will be checked
24	*
25	* Usage:
26	*
27	* $contentDetector = new SiteContentDetector();
28	* $contentDetector->detectContent([SiteContentDetector::GA3]);
29	* if ($contentDetector->ga3) {
30	* // site is using GA3
31	* }
32	*
33	*/
34	class SiteContentDetector
35	{
36	// Content types
37	const ALL_CONTENT = 1;
38	const CONSENT_MANAGER = 2;
39	const GA3 = 3;
40	const GA4 = 4;
41	const GTM = 5;
42	const CMS = 6;
43	const JS_FRAMEWORK = 7;
44
45	// Detection detail
46	public $consentManagerId; // Id of the detected consent manager, eg. 'osano'
47	public $consentManagerName; // Display name of the detected consent manager, eg. 'Osano'
48	public $consentManagerUrl; // Url for the configuration guide for the detected consent manager
49	public $isConnected = false; // True if the detected consent manager is already connected with Matomo
50	public $ga3; // True if GA3 was detected on the site
51	public $ga4; // True if GA4 was detected on the site
52	public $gtm; // True if GTM was detected on the site
53	public $cms; // The CMS that was detected on the site
54	public $cloudflare; // true if website is hosted on cloudflare
55	public $jsFramework; // The JS framework that was detected on the site
56
57	private $siteResponse = [
58	'data' => '',
59	'headers' => []
60	];
61
62	/** @var Lazy */
63	private $cache;
64
65	/**
66	* @var GtmSiteTypeGuesser
67	*/
68	private $siteGuesser;
69
70	public function __construct(?Lazy $cache = null)
71	{
72	if ($cache === null) {
73	$this->cache = Cache::getLazyCache();
74	} else {
75	$this->cache = $cache;
76	}
77	$this->siteGuesser = new GtmSiteTypeGuesser();
78	}
79
80	/**
81	* Reset the detection properties
82	*
83	* @return void
84	*/
85	private function resetDetectionProperties(): void
86	{
87	$this->consentManagerId = null;
88	$this->consentManagerUrl = null;
89	$this->consentManagerName = null;
90	$this->isConnected = false;
91	$this->ga3 = false;
92	$this->ga4 = false;
93	$this->gtm = false;
94	$this->cms = SitesManager::SITE_TYPE_UNKNOWN;
95	$this->cloudflare = false;
96	$this->jsFramework = SitesManager::JS_FRAMEWORK_UNKNOWN;
97	}
98
99	/**
100	* This will query the site and populate the class properties with
101	* the details of the detected content
102	*
103	* @param array $detectContent Array of content type for which to check, defaults to all, limiting this list
104	* will speed up the detection check
105	* @param ?int $idSite Override the site ID, will use the site from the current request if null
106	* @param ?array $siteResponse String containing the site data to search, if blank then data will be retrieved
107	* from the current request site via an http request
108	* @param int $timeOut How long to wait for the site to response, defaults to 5 seconds
109	* @return void
110	*/
111	public function detectContent(array $detectContent = [SiteContentDetector::ALL_CONTENT],
112	?int $idSite = null, ?array $siteResponse = null, int $timeOut = 5): void
113	{
114	$this->resetDetectionProperties();
115
116	// If site data was passed in, then just run the detection checks against it and return.
117	if ($siteResponse) {
118	$this->siteResponse = $siteResponse;
119	$this->detectionChecks($detectContent);
120	return;
121	}
122
123	// Get the site id from the request object if not explicitly passed
124	if ($idSite === null) {
125	if (!isset($_REQUEST['idSite'])) {
126	return;
127	}
128
129	$idSite = Common::getRequestVar('idSite', null, 'int');
130
131	if (!$idSite) {
132	return;
133	}
134	}
135
136	$url = Site::getMainUrlFor($idSite);
137
138	// Check and load previously cached site content detection data if it exists
139	$cacheKey = 'SiteContentDetector_' . md5($url);
140	$requiredProperties = $this->getRequiredProperties($detectContent);
141	$siteContentDetectionCache = $this->cache->fetch($cacheKey);
142
143	if ($siteContentDetectionCache !== false) {
144	if ($this->checkCacheHasRequiredProperties($requiredProperties, $siteContentDetectionCache)) {
145	$this->loadRequiredPropertiesFromCache($requiredProperties, $siteContentDetectionCache);
146	return;
147	}
148	}
149
150	// No cache hit, no passed data, so make a request for the site content
151	$siteResponse = $this->requestSiteResponse($url, $timeOut);
152
153	// Abort if still no site data
154	if (empty($siteResponse['data'])) {
155	return;
156	}
157
158	$this->siteResponse = $siteResponse;
159
160	// We now have site data to analyze, so run the detection checks
161	$this->detectionChecks($detectContent);
162
163	// A request was made to get this data and it isn't currently cached, so write it to the cache now
164	$cacheLife = (60 * 60 * 24 * 7);
165	$this->savePropertiesToCache($cacheKey, $requiredProperties, $cacheLife);
166	}
167
168	/**
169	* Returns an array of properties required by the detect content array
170	*
171	* @param array $detectContent
172	*
173	* @return array
174	*/
175	private function getRequiredProperties(array $detectContent): array
176	{
177	$requiredProperties = [];
178
179	if (in_array(SiteContentDetector::CONSENT_MANAGER, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
180	$requiredProperties = array_merge($requiredProperties, ['consentManagerId', 'consentManagerName', 'consentManagerUrl', 'isConnected']);
181	}
182
183	if (in_array(SiteContentDetector::GA3, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
184	$requiredProperties[] = 'ga3';
185	}
186
187	if (in_array(SiteContentDetector::GA4, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
188	$requiredProperties[] = 'ga4';
189	}
190
191	if (in_array(SiteContentDetector::GTM, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
192	$requiredProperties[] = 'gtm';
193	}
194
195	if (in_array(SiteContentDetector::CMS, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
196	$requiredProperties[] = 'cms';
197	}
198
199	if (in_array(SiteContentDetector::JS_FRAMEWORK, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
200	$requiredProperties[] = 'jsFramework';
201	}
202
203	return $requiredProperties;
204	}
205
206	/**
207	* Checks that all required properties are in the cache array
208	*
209	* @param array $properties
210	* @param array $cache
211	*
212	* @return bool
213	*/
214	private function checkCacheHasRequiredProperties(array $properties, array $cache): bool
215	{
216	foreach ($properties as $prop) {
217	if (!array_key_exists($prop, $cache)) {
218	return false;
219	}
220	}
221
222	return true;
223	}
224
225	/**
226	* Load object properties from the cache array
227	*
228	* @param array $properties
229	* @param array $cache
230	*
231	* @return void
232	*/
233	private function loadRequiredPropertiesFromCache(array $properties, array $cache): void
234	{
235	foreach ($properties as $prop) {
236	if (!array_key_exists($prop, $cache)) {
237	continue;
238	}
239
240	$this->{$prop} = $cache[$prop];
241	}
242	}
243
244	/**
245	* Save properties to the cache
246	*
247	* @param string $cacheKey
248	* @param array $properties
249	* @param int $cacheLife
250	*
251	* @return void
252	*/
253	private function savePropertiesToCache(string $cacheKey, array $properties, int $cacheLife): void
254	{
255	$cacheData = [];
256
257	// Load any existing cached values
258	$siteContentDetectionCache = $this->cache->fetch($cacheKey);
259
260	if (is_array($siteContentDetectionCache)) {
261	$cacheData = $siteContentDetectionCache;
262	}
263
264	foreach ($properties as $prop) {
265	$cacheData[$prop] = $this->{$prop};
266	}
267
268	$this->cache->save($cacheKey, $cacheData, $cacheLife);
269	}
270
271	/**
272	* Run various detection checks for site content
273	*
274	* @param array $detectContent Array of detection types used to filter the checks that are run
275	*
276	* @return void
277	*/
278	private function detectionChecks($detectContent): void
279	{
280	if (in_array(SiteContentDetector::CONSENT_MANAGER, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
281	$this->detectConsentManager();
282	}
283
284	if (in_array(SiteContentDetector::GA3, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
285	$this->ga3 = $this->siteGuesser->detectGA3FromResponse($this->siteResponse);
286	}
287
288	if (in_array(SiteContentDetector::GA4, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
289	$this->ga4 = $this->siteGuesser->detectGA4FromResponse($this->siteResponse);
290	}
291
292	if (in_array(SiteContentDetector::GTM, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
293	$this->gtm = $this->siteGuesser->guessGtmFromResponse($this->siteResponse);
294	}
295
296	if (in_array(SiteContentDetector::CMS, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
297	$this->cms = $this->siteGuesser->guessSiteTypeFromResponse($this->siteResponse);
298	}
299
300	if (in_array(SiteContentDetector::JS_FRAMEWORK, $detectContent) \|\| in_array(SiteContentDetector::ALL_CONTENT, $detectContent)) {
301	$this->jsFramework = $this->siteGuesser->guessJsFrameworkFromResponse($this->siteResponse);
302	}
303
304	if (
305	(!empty($this->siteResponse['headers']['server']) && stripos($this->siteResponse['headers']['server'], 'cloudflare') !== false) \|\|
306	(!empty($this->siteResponse['headers']['Server']) && stripos($this->siteResponse['headers']['Server'], 'cloudflare') !== false) \|\|
307	(!empty($this->siteResponse['headers']['SERVER']) && stripos($this->siteResponse['headers']['SERVER'], 'cloudflare') !== false) \|\|
308	!empty($this->siteResponse['headers']['cf-ray']) \|\|
309	!empty($this->siteResponse['headers']['Cf-Ray']) \|\|
310	!empty($this->siteResponse['headers']['CF-RAY'])
311	) {
312	$this->cloudflare = true;
313	}
314	}
315
316	/**
317	* Retrieve data from the specified site using an HTTP request
318	*
319	* @param string $url
320	* @param int $timeOut
321	*
322	* @return array
323	*/
324	private function requestSiteResponse(string $url, int $timeOut): array
325	{
326	if (!$url) {
327	return [];
328	}
329
330	// If internet features are disabled, we don't try to fetch any site content
331	if (0 === (int) GeneralConfig::getConfigValue('enable_internet_features')) {
332	return [];
333	}
334
335	$siteData = [];
336
337	try {
338	$siteData = Http::sendHttpRequestBy(Http::getTransportMethod(), $url, $timeOut, null, null,
339	null, 0, false, true, false, true);
340	} catch (\Exception $e) {
341	}
342
343	return $siteData;
344	}
345
346	/**
347	* Detect known consent managers in the site data
348	*
349	* Populate this object's properties with the results
350	*
351	* @return void
352	*/
353	private function detectConsentManager(): void
354	{
355	$defs = self::getConsentManagerDefinitions();
356
357	if (!$defs) {
358	return;
359	}
360
361	if (empty($this->siteResponse['data'])) {
362	return;
363	}
364
365	foreach ($defs as $consentManagerId => $consentManagerDef) {
366	foreach ($consentManagerDef['detectStrings'] as $dStr) {
367	if (strpos($this->siteResponse['data'], $dStr) !== false && array_key_exists($consentManagerId, $defs)) {
368	$this->consentManagerId = $consentManagerId;
369	$this->consentManagerName = $consentManagerDef['name'];
370	$this->consentManagerUrl = $consentManagerDef['url'];
371	break 2;
372	}
373	}
374	}
375
376	if (!isset($defs[$this->consentManagerId]['connectedStrings'])) {
377	return;
378	}
379
380	// If a consent manager was detected then perform an additional check to see if it has been connected to Matomo
381	foreach ($defs[$this->consentManagerId]['connectedStrings'] as $cStr) {
382	if (strpos($this->siteResponse['data'], $cStr) !== false) {
383	$this->isConnected = true;
384	break;
385	}
386	}
387	}
388
389	/**
390	* Return an array of consent manager definitions which can be used to detect their presence on the site and show
391	* the associated guide links
392	*
393	* @return array[]
394	*/
395	public static function getConsentManagerDefinitions(): array
396	{
397	return [
398
399	'osano' => [
400	'name' => 'Osano',
401	'detectStrings' => ['osano.com'],
402	'connectedStrings' => ["Osano.cm.addEventListener('osano-cm-consent-changed', (change) => { console.log('cm-change'); consentSet(change); });"],
403	'url' => 'https://matomo.org/faq/how-to/using-osano-consent-manager-with-matomo',
404	],
405
406	'cookiebot' => [
407	'name' => 'Cookiebot',
408	'detectStrings' => ['cookiebot.com'],
409	'connectedStrings' => ["typeof _paq === 'undefined' \|\| typeof Cookiebot === 'undefined'"],
410	'url' => 'https://matomo.org/faq/how-to/using-cookiebot-consent-manager-with-matomo',
411	],
412
413	'cookieyes' => [
414	'name' => 'CookieYes',
415	'detectStrings' => ['cookieyes.com'],
416	'connectedStrings' => ['document.addEventListener("cookieyes_consent_update", function (eventData)'],
417	'url' => 'https://matomo.org/faq/how-to/using-cookieyes-consent-manager-with-matomo',
418	],
419
420	// Note: tarte au citron pro is configured server side so we cannot tell if it has been connected by
421	// crawling the website, however setup of Matomo with the pro version is automatic, so displaying the guide
422	// link for pro isn't necessary. Only the open source version is detected by this definition.
423	'tarteaucitron' => [
424	'name' => 'Tarte au Citron',
425	'detectStrings' => ['tarteaucitron.js'],
426	'connectedStrings' => ['tarteaucitron.user.matomoHost'],
427	'url' => 'https://matomo.org/faq/how-to/using-tarte-au-citron-consent-manager-with-matomo',
428	],
429
430	'klaro' => [
431	'name' => 'Klaro',
432	'detectStrings' => ['klaro.js', 'kiprotect.com'],
433	'connectedStrings' => ['KlaroWatcher()', "title: 'Matomo',"],
434	'url' => 'https://matomo.org/faq/how-to/using-klaro-consent-manager-with-matomo',
435	],
436
437	'complianz' => [
438	'name' => 'Complianz',
439	'detectStrings' => ['complianz-gdpr'],
440	'connectedStrings' => ["if (!cmplz_in_array( 'statistics', consentedCategories )) {
441	_paq.push(['forgetCookieConsentGiven']);"],
442	'url' => 'https://matomo.org/faq/how-to/using-complianz-for-wordpress-consent-manager-with-matomo',
443	],
444	];
445	}
446	}
447