API
1 day ago
Access
1 month ago
Application
1 month ago
Archive
1 month ago
ArchiveProcessor
1 day ago
Archiver
2 years ago
AssetManager
1 month ago
Auth
1 month ago
Category
1 month ago
Changes
3 months ago
CliMulti
1 month ago
Columns
1 day ago
Concurrency
1 month ago
Config
1 month ago
Container
3 months ago
CronArchive
1 month ago
DataAccess
1 day ago
DataFiles
2 years ago
DataTable
1 month ago
Db
1 day ago
DeviceDetector
1 year ago
Email
2 years ago
Exception
6 months ago
Http
1 day ago
Intl
5 months ago
Log
2 years ago
Mail
1 year ago
Measurable
8 months ago
Menu
1 month ago
Metrics
1 month ago
Notification
8 months ago
Period
1 month ago
Plugin
1 day ago
Policy
3 months ago
ProfessionalServices
1 year ago
Report
1 year ago
ReportRenderer
1 day ago
Request
1 month ago
Scheduler
1 month ago
Segment
1 month ago
Session
1 day ago
Settings
1 day ago
Tracker
1 day ago
Translation
3 months ago
Twig
1 year ago
UpdateCheck
5 months ago
Updater
1 month ago
Updates
2 months ago
Validators
1 year ago
View
3 months ago
ViewDataTable
1 day ago
Visualization
1 day ago
Widget
1 month ago
.htaccess
2 years ago
Access.php
1 month ago
Archive.php
3 months ago
ArchiveProcessor.php
1 day ago
AssetManager.php
1 month ago
Auth.php
8 months ago
AuthResult.php
8 months ago
BaseFactory.php
2 years ago
Cache.php
1 month ago
CacheId.php
6 months ago
CliMulti.php
1 month ago
Common.php
1 month ago
Config.php
1 month ago
Console.php
5 months ago
Context.php
2 years ago
Cookie.php
1 month ago
CronArchive.php
1 month ago
DI.php
5 months ago
DataArray.php
1 month ago
DataTable.php
1 day ago
Date.php
3 months ago
Db.php
3 months ago
DbHelper.php
1 month ago
Development.php
1 month ago
ErrorHandler.php
8 months ago
EventDispatcher.php
3 months ago
ExceptionHandler.php
6 months ago
FileIntegrity.php
1 month ago
Filechecks.php
1 year ago
Filesystem.php
1 day ago
FrontController.php
1 month ago
Http.php
1 day ago
IP.php
1 year ago
Log.php
5 months ago
LogDeleter.php
1 month ago
Mail.php
1 year ago
Metrics.php
3 months ago
NoAccessException.php
2 years ago
Nonce.php
8 months ago
Notification.php
3 months ago
NumberFormatter.php
1 month ago
Option.php
1 month ago
Period.php
1 month ago
Piwik.php
1 month ago
Plugin.php
3 months ago
Process.php
3 months ago
Profiler.php
1 month ago
ProxyHeaders.php
6 months ago
ProxyHttp.php
1 month ago
QuickForm2.php
5 months ago
RankingQuery.php
3 months ago
ReportRenderer.php
1 day ago
Request.php
3 months ago
Segment.php
1 day ago
Sequence.php
1 month ago
Session.php
2 months ago
SettingsPiwik.php
1 month ago
SettingsServer.php
1 month ago
Singleton.php
2 years ago
Site.php
3 months ago
SiteContentDetector.php
1 day ago
SupportedBrowser.php
2 years ago
TCPDF.php
1 year ago
Theme.php
1 month ago
Timer.php
1 month ago
Tracker.php
1 month ago
Twig.php
1 day ago
Unzip.php
1 year ago
UpdateCheck.php
3 months ago
Updater.php
1 month ago
UpdaterErrorException.php
2 years ago
Updates.php
1 month ago
Url.php
1 month ago
UrlHelper.php
3 months ago
Version.php
1 day ago
View.php
1 month ago
bootstrap.php
1 year ago
dispatch.php
2 years ago
testMinimumPhpVersion.php
8 months ago
SiteContentDetector.php
342 lines
| 1 | <?php |
| 2 | |
| 3 | /** |
| 4 | * Matomo - free/libre analytics platform |
| 5 | * |
| 6 | * @link https://matomo.org |
| 7 | * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later |
| 8 | */ |
| 9 | namespace Piwik; |
| 10 | |
| 11 | use Matomo\Cache\Lazy; |
| 12 | use Piwik\Config\GeneralConfig; |
| 13 | use Piwik\Container\StaticContainer; |
| 14 | use Piwik\Http\EgressBlockedException; |
| 15 | use Piwik\Log\LoggerInterface; |
| 16 | use Piwik\Plugins\SitesManager\SiteContentDetection\ConsentManagerDetectionAbstract; |
| 17 | use Piwik\Plugins\SitesManager\SiteContentDetection\SiteContentDetectionAbstract; |
| 18 | /** |
| 19 | * This class provides detection functions for specific content on a site. It can be used to easily detect the |
| 20 | * presence of known third party code. |
| 21 | * |
| 22 | * Note: Calling the `detectContent()` method will create a HTTP request to the site to retrieve data, only the main site URL |
| 23 | * will be checked |
| 24 | * |
| 25 | * Usage: |
| 26 | * |
| 27 | * $contentDetector = new SiteContentDetector(); |
| 28 | * $contentDetector->detectContent([GoogleAnalytics3::getId()]); |
| 29 | * if ($contentDetector->ga3) { |
| 30 | * // site is using GA3 |
| 31 | * } |
| 32 | * |
| 33 | * @api |
| 34 | */ |
| 35 | class SiteContentDetector |
| 36 | { |
| 37 | /** |
| 38 | * @var array<string, array<string, SiteContentDetectionAbstract>> |
| 39 | */ |
| 40 | public $detectedContent = [SiteContentDetectionAbstract::TYPE_TRACKER => [], SiteContentDetectionAbstract::TYPE_CMS => [], SiteContentDetectionAbstract::TYPE_JS_FRAMEWORK => [], SiteContentDetectionAbstract::TYPE_CONSENT_MANAGER => [], SiteContentDetectionAbstract::TYPE_JS_CRASH_ANALYTICS => [], SiteContentDetectionAbstract::TYPE_OTHER => []]; |
| 41 | public $connectedConsentManagers = []; |
| 42 | private $siteResponse = ['data' => '', 'headers' => []]; |
| 43 | /** @var Lazy */ |
| 44 | private $cache; |
| 45 | public function __construct(?Lazy $cache = null) |
| 46 | { |
| 47 | if ($cache === null) { |
| 48 | $this->cache = \Piwik\Cache::getLazyCache(); |
| 49 | } else { |
| 50 | $this->cache = $cache; |
| 51 | } |
| 52 | } |
| 53 | /** |
| 54 | * @return array<string, SiteContentDetectionAbstract[]> |
| 55 | */ |
| 56 | public static function getSiteContentDetectionsByType() : array |
| 57 | { |
| 58 | $instancesByType = []; |
| 59 | $classes = self::getAllSiteContentDetectionClasses(); |
| 60 | foreach ($classes as $className) { |
| 61 | $instancesByType[$className::getContentType()][] = StaticContainer::get($className); |
| 62 | } |
| 63 | return $instancesByType; |
| 64 | } |
| 65 | /** |
| 66 | * Returns the site content detection object with the provided id, or null if it can't be found |
| 67 | */ |
| 68 | public function getSiteContentDetectionById(string $id) : ?SiteContentDetectionAbstract |
| 69 | { |
| 70 | $classes = $this->getAllSiteContentDetectionClasses(); |
| 71 | foreach ($classes as $className) { |
| 72 | if ($className::getId() === $id) { |
| 73 | return StaticContainer::get($className); |
| 74 | } |
| 75 | } |
| 76 | return null; |
| 77 | } |
| 78 | /** |
| 79 | * @return string[] |
| 80 | */ |
| 81 | protected static function getAllSiteContentDetectionClasses() : array |
| 82 | { |
| 83 | return \Piwik\Plugin\Manager::getInstance()->findMultipleComponents('SiteContentDetection', SiteContentDetectionAbstract::class); |
| 84 | } |
| 85 | /** |
| 86 | * Reset the detections |
| 87 | */ |
| 88 | private function resetDetections() : void |
| 89 | { |
| 90 | $this->detectedContent = [SiteContentDetectionAbstract::TYPE_TRACKER => [], SiteContentDetectionAbstract::TYPE_CMS => [], SiteContentDetectionAbstract::TYPE_JS_FRAMEWORK => [], SiteContentDetectionAbstract::TYPE_CONSENT_MANAGER => [], SiteContentDetectionAbstract::TYPE_JS_CRASH_ANALYTICS => [], SiteContentDetectionAbstract::TYPE_OTHER => []]; |
| 91 | $this->connectedConsentManagers = []; |
| 92 | } |
| 93 | /** |
| 94 | * This will query the site and populate the class properties with |
| 95 | * the details of the detected content |
| 96 | * |
| 97 | * @param array $detectContent Array of content type for which to check, defaults to all, limiting this list |
| 98 | * will speed up the detection check. |
| 99 | * Allowed values are: |
| 100 | * * empty array - to run all detections |
| 101 | * * an array containing ids of detections, e.g. Wordpress::getId() or any of the |
| 102 | * type constants, e.g. SiteContentDetectionAbstract::TYPE_TRACKER |
| 103 | * @param ?int $idSite Override the site ID, will use the site from the current request if null |
| 104 | * @param ?array $siteResponse String containing the site data to search, if blank then data will be retrieved |
| 105 | * from the current request site via an http request |
| 106 | * @param int $timeOut How long to wait for the site to response, defaults to 5 seconds |
| 107 | */ |
| 108 | public function detectContent(array $detectContent = [], ?int $idSite = null, ?array $siteResponse = null, int $timeOut = 5) : void |
| 109 | { |
| 110 | $this->resetDetections(); |
| 111 | // If site data was passed in, then just run the detection checks against it and return. |
| 112 | if ($siteResponse) { |
| 113 | $this->siteResponse = $siteResponse; |
| 114 | $this->detectionChecks($detectContent); |
| 115 | return; |
| 116 | } |
| 117 | // Get the site id from the request object if not explicitly passed |
| 118 | if ($idSite === null) { |
| 119 | $idSite = \Piwik\Request::fromRequest()->getIntegerParameter('idSite', 0); |
| 120 | if (!$idSite) { |
| 121 | return; |
| 122 | } |
| 123 | } |
| 124 | $url = \Piwik\Site::getMainUrlFor($idSite); |
| 125 | // Check and load previously cached site content detection data if it exists |
| 126 | $cacheKey = 'SiteContentDetection_' . md5($url); |
| 127 | $siteContentDetectionCache = $this->cache->fetch($cacheKey); |
| 128 | if ($siteContentDetectionCache !== \false) { |
| 129 | if ($this->checkCacheHasRequiredProperties($detectContent, $siteContentDetectionCache)) { |
| 130 | $this->detectedContent = $siteContentDetectionCache['detectedContent']; |
| 131 | $this->connectedConsentManagers = $siteContentDetectionCache['connectedConsentManagers']; |
| 132 | return; |
| 133 | } |
| 134 | } |
| 135 | // No cache hit, no passed data, so make a request for the site content |
| 136 | $siteResponse = $this->requestSiteResponse($url, $timeOut); |
| 137 | // Abort if still no site data |
| 138 | if (empty($siteResponse['data'])) { |
| 139 | return; |
| 140 | } |
| 141 | $this->siteResponse = $siteResponse; |
| 142 | // We now have site data to analyze, so run the detection checks |
| 143 | $this->detectionChecks($detectContent); |
| 144 | // A request was made to get this data and it isn't currently cached, so write it to the cache now |
| 145 | $cacheLife = 60 * 60 * 24 * 7; |
| 146 | $this->saveToCache($cacheKey, $cacheLife); |
| 147 | } |
| 148 | /** |
| 149 | * Returns if the detection with the provided id was detected or not |
| 150 | * |
| 151 | * Note: self::detectContent needs to be called before. |
| 152 | */ |
| 153 | public function wasDetected(string $detectionClassId) : bool |
| 154 | { |
| 155 | foreach ($this->detectedContent as $type => $detectedClassIds) { |
| 156 | if (array_key_exists($detectionClassId, $detectedClassIds)) { |
| 157 | return $detectedClassIds[$detectionClassId] ?? \false; |
| 158 | } |
| 159 | } |
| 160 | return \false; |
| 161 | } |
| 162 | /** |
| 163 | * Returns an array containing ids of all detected detections of the given type |
| 164 | * |
| 165 | * @param int $type One of the SiteContentDetectionAbstract::TYPE_* constants |
| 166 | * @return array |
| 167 | */ |
| 168 | public function getDetectsByType(int $type) : array |
| 169 | { |
| 170 | $detected = []; |
| 171 | foreach ($this->detectedContent[$type] as $objId => $wasDetected) { |
| 172 | if (\true === $wasDetected) { |
| 173 | $detected[] = $objId; |
| 174 | } |
| 175 | } |
| 176 | return $detected; |
| 177 | } |
| 178 | /** |
| 179 | * Checks that all required detections are in the cache array |
| 180 | * |
| 181 | * @param array $detectContent |
| 182 | * @param array $cache |
| 183 | */ |
| 184 | private function checkCacheHasRequiredProperties(array $detectContent, array $cache) : bool |
| 185 | { |
| 186 | if (empty($detectContent)) { |
| 187 | foreach (self::getSiteContentDetectionsByType() as $type => $entries) { |
| 188 | foreach ($entries as $entry) { |
| 189 | if (!isset($cache['detectedContent'][$type][$entry::getId()])) { |
| 190 | return \false; |
| 191 | // random detection missing |
| 192 | } |
| 193 | } |
| 194 | } |
| 195 | return \true; |
| 196 | } |
| 197 | foreach ($detectContent as $requestedDetection) { |
| 198 | if (is_string($requestedDetection)) { |
| 199 | // specific detection |
| 200 | $detectionObj = $this->getSiteContentDetectionById($requestedDetection); |
| 201 | if (null !== $detectionObj && !isset($cache['detectedContent'][$detectionObj::getContentType()][$detectionObj::getId()])) { |
| 202 | return \false; |
| 203 | // specific detection was run before |
| 204 | } |
| 205 | } elseif (is_int($requestedDetection)) { |
| 206 | // detection type requested |
| 207 | $detectionsByType = self::getSiteContentDetectionsByType(); |
| 208 | if (isset($detectionsByType[$requestedDetection])) { |
| 209 | foreach ($detectionsByType[$requestedDetection] as $detectionObj) { |
| 210 | if (!isset($cache['detectedContent'][$requestedDetection][$detectionObj::getId()])) { |
| 211 | return \false; |
| 212 | // random detection missing |
| 213 | } |
| 214 | } |
| 215 | } |
| 216 | } |
| 217 | } |
| 218 | return \true; |
| 219 | } |
| 220 | /** |
| 221 | * Save data to the cache |
| 222 | */ |
| 223 | private function saveToCache(string $cacheKey, int $cacheLife) : void |
| 224 | { |
| 225 | $cacheData = ['detectedContent' => [], 'connectedConsentManagers' => []]; |
| 226 | // Load any existing cached values |
| 227 | $siteContentDetectionCache = $this->cache->fetch($cacheKey); |
| 228 | if (is_array($siteContentDetectionCache)) { |
| 229 | $cacheData = $siteContentDetectionCache; |
| 230 | } |
| 231 | foreach ($this->detectedContent as $type => $detections) { |
| 232 | if (!isset($cacheData['detectedContent'][$type])) { |
| 233 | $cacheData['detectedContent'][$type] = []; |
| 234 | } |
| 235 | foreach ($detections as $detectionId => $wasDetected) { |
| 236 | if (null !== $wasDetected) { |
| 237 | $cacheData['detectedContent'][$type][$detectionId] = $wasDetected; |
| 238 | } |
| 239 | } |
| 240 | } |
| 241 | $cacheData['connectedConsentManagers'] = array_merge($cacheData['connectedConsentManagers'], $this->connectedConsentManagers); |
| 242 | $this->cache->save($cacheKey, $cacheData, $cacheLife); |
| 243 | } |
| 244 | /** |
| 245 | * Run various detection checks for site content |
| 246 | * |
| 247 | * @param array $detectContent Array of detection types used to filter the checks that are run |
| 248 | */ |
| 249 | private function detectionChecks(array $detectContent) : void |
| 250 | { |
| 251 | $detections = $this->getSiteContentDetectionsByType(); |
| 252 | foreach ($detections as $type => $typeDetections) { |
| 253 | foreach ($typeDetections as $typeDetection) { |
| 254 | $this->detectedContent[$type][$typeDetection::getId()] = null; |
| 255 | if (in_array($type, $detectContent) || in_array($typeDetection::getId(), $detectContent) || empty($detectContent)) { |
| 256 | $this->detectedContent[$type][$typeDetection::getId()] = \false; |
| 257 | if ($typeDetection->isDetected($this->siteResponse['data'], $this->siteResponse['headers'])) { |
| 258 | if ($typeDetection instanceof ConsentManagerDetectionAbstract && $typeDetection->checkIsConnected($this->siteResponse['data'], $this->siteResponse['headers'])) { |
| 259 | $this->connectedConsentManagers[] = $typeDetection::getId(); |
| 260 | } |
| 261 | $this->detectedContent[$type][$typeDetection::getId()] = \true; |
| 262 | } |
| 263 | } |
| 264 | } |
| 265 | } |
| 266 | } |
| 267 | /** |
| 268 | * Retrieve data from the specified site using an HTTP request |
| 269 | * |
| 270 | * @return array |
| 271 | */ |
| 272 | private function requestSiteResponse(string $url, int $timeOut) : array |
| 273 | { |
| 274 | if (!$url) { |
| 275 | return []; |
| 276 | } |
| 277 | // If internet features are disabled, we don't try to fetch any site content |
| 278 | if (0 === GeneralConfig::getIntegerConfigValue('enable_internet_features', 0)) { |
| 279 | return []; |
| 280 | } |
| 281 | $siteData = []; |
| 282 | try { |
| 283 | // @todo PHP 8.1 min (Matomo 6): use named arguments to drop the positional null/false filler. |
| 284 | $siteData = \Piwik\Http::sendHttpRequestBy( |
| 285 | 'curl', |
| 286 | $url, |
| 287 | $timeOut, |
| 288 | null, |
| 289 | null, |
| 290 | null, |
| 291 | 0, |
| 292 | \false, |
| 293 | \true, |
| 294 | // $acceptInvalidSslCertificate: detected sites may use self-signed certs |
| 295 | \false, |
| 296 | \true, |
| 297 | // $getExtendedInfo |
| 298 | 'GET', |
| 299 | null, |
| 300 | null, |
| 301 | null, |
| 302 | [], |
| 303 | null, |
| 304 | \true, |
| 305 | // $checkHostIsAllowed |
| 306 | \true |
| 307 | ); |
| 308 | } catch (EgressBlockedException $e) { |
| 309 | // admin-fixable rejection, not a transient network error, so it must clear the default WARN level |
| 310 | StaticContainer::get(LoggerInterface::class)->warning('Site content detection request for {url} was refused: {message}', [ |
| 311 | // host only, so a configured URL carrying userinfo keeps credentials out of the log |
| 312 | 'url' => \Piwik\UrlHelper::getHostFromUrl($url), |
| 313 | 'message' => $e->getMessage(), |
| 314 | ]); |
| 315 | } catch (\Exception $e) { |
| 316 | // intentionally fail closed, but leave a diagnostic trail |
| 317 | StaticContainer::get(LoggerInterface::class)->debug('Site content detection request for {url} failed: {message}', ['url' => \Piwik\UrlHelper::getHostFromUrl($url), 'message' => $e->getMessage()]); |
| 318 | } |
| 319 | return $siteData; |
| 320 | } |
| 321 | /** |
| 322 | * Return an array of consent manager definitions which can be used to detect their presence on the site and show |
| 323 | * the associated guide links |
| 324 | * |
| 325 | * Note: This list is also used to display the known / supported consent managers on the "Ask for Consent" page |
| 326 | * For adding a new consent manager to this page, it needs to be added here. If a consent manager can't be detected |
| 327 | * automatically, simply leave the detections empty. |
| 328 | * |
| 329 | * @return array[] |
| 330 | */ |
| 331 | public static function getKnownConsentManagers() : array |
| 332 | { |
| 333 | $detections = self::getSiteContentDetectionsByType(); |
| 334 | $cmDetections = $detections[SiteContentDetectionAbstract::TYPE_CONSENT_MANAGER]; |
| 335 | $consentManagers = []; |
| 336 | foreach ($cmDetections as $detection) { |
| 337 | $consentManagers[$detection::getId()] = ['name' => $detection::getName(), 'instructionUrl' => $detection::getInstructionUrl()]; |
| 338 | } |
| 339 | return $consentManagers; |
| 340 | } |
| 341 | } |
| 342 |