API
1 month ago
Access
3 months ago
Application
1 month ago
Archive
1 month ago
ArchiveProcessor
1 month ago
Archiver
2 years ago
AssetManager
1 month ago
Auth
6 months ago
Category
6 months ago
Changes
1 month ago
CliMulti
1 year ago
Columns
1 month ago
Concurrency
1 month ago
Config
1 month ago
Container
1 month ago
CronArchive
3 months ago
DataAccess
1 month ago
DataFiles
2 years ago
DataTable
2 weeks ago
Db
2 weeks ago
DeviceDetector
1 year ago
Email
2 years ago
Exception
4 months ago
Http
4 months ago
Intl
3 months ago
Log
2 years ago
Mail
1 year ago
Measurable
6 months ago
Menu
1 month ago
Metrics
3 months ago
Notification
6 months ago
Period
1 month ago
Plugin
2 weeks ago
Policy
1 month ago
ProfessionalServices
1 year ago
Report
1 year ago
ReportRenderer
3 months ago
Request
3 months ago
Scheduler
1 month ago
Segment
1 month ago
Session
2 weeks ago
Settings
1 month ago
Tracker
2 weeks ago
Translation
1 month ago
Twig
1 year ago
UpdateCheck
3 months ago
Updater
1 month ago
Updates
1 week ago
Validators
1 year ago
View
1 month ago
ViewDataTable
2 weeks ago
Visualization
1 year ago
Widget
1 month ago
.htaccess
2 years ago
Access.php
1 month ago
Archive.php
1 month ago
ArchiveProcessor.php
1 month ago
AssetManager.php
1 month ago
Auth.php
6 months ago
AuthResult.php
6 months ago
BaseFactory.php
2 years ago
Cache.php
2 years ago
CacheId.php
4 months ago
CliMulti.php
1 month ago
Common.php
2 weeks ago
Config.php
1 month ago
Console.php
3 months ago
Context.php
2 years ago
Cookie.php
1 year ago
CronArchive.php
1 month ago
DI.php
3 months ago
DataArray.php
1 month ago
DataTable.php
1 month ago
Date.php
1 month ago
Db.php
1 month ago
DbHelper.php
1 month ago
Development.php
1 year ago
ErrorHandler.php
6 months ago
EventDispatcher.php
1 month ago
ExceptionHandler.php
4 months ago
FileIntegrity.php
1 month ago
Filechecks.php
1 year ago
Filesystem.php
1 month ago
FrontController.php
4 months ago
Http.php
1 month ago
IP.php
1 year ago
Log.php
3 months ago
LogDeleter.php
1 year ago
Mail.php
1 year ago
Metrics.php
1 month ago
NoAccessException.php
2 years ago
Nonce.php
6 months ago
Notification.php
1 month ago
NumberFormatter.php
5 months ago
Option.php
5 months ago
Period.php
1 month ago
Piwik.php
1 month ago
Plugin.php
1 month ago
Process.php
1 month ago
Profiler.php
6 months ago
ProxyHeaders.php
4 months ago
ProxyHttp.php
5 months ago
QuickForm2.php
3 months ago
RankingQuery.php
1 month ago
ReportRenderer.php
1 month ago
Request.php
1 month ago
Segment.php
1 month ago
Sequence.php
6 months ago
Session.php
2 weeks ago
SettingsPiwik.php
1 month ago
SettingsServer.php
1 year ago
Singleton.php
2 years ago
Site.php
1 month ago
SiteContentDetector.php
1 month ago
SupportedBrowser.php
2 years ago
TCPDF.php
1 year ago
Theme.php
1 year ago
Timer.php
1 month ago
Tracker.php
1 month ago
Twig.php
1 month ago
Unzip.php
1 year ago
UpdateCheck.php
1 month ago
Updater.php
1 month ago
UpdaterErrorException.php
2 years ago
Updates.php
3 months ago
Url.php
3 months ago
UrlHelper.php
1 month ago
Version.php
1 week ago
View.php
1 month ago
bootstrap.php
1 year ago
dispatch.php
2 years ago
testMinimumPhpVersion.php
6 months ago
SiteContentDetector.php
307 lines
| 1 | <?php |
| 2 | |
| 3 | /** |
| 4 | * Matomo - free/libre analytics platform |
| 5 | * |
| 6 | * @link https://matomo.org |
| 7 | * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later |
| 8 | */ |
| 9 | namespace Piwik; |
| 10 | |
| 11 | use Matomo\Cache\Lazy; |
| 12 | use Piwik\Config\GeneralConfig; |
| 13 | use Piwik\Container\StaticContainer; |
| 14 | use Piwik\Plugins\SitesManager\SiteContentDetection\ConsentManagerDetectionAbstract; |
| 15 | use Piwik\Plugins\SitesManager\SiteContentDetection\SiteContentDetectionAbstract; |
| 16 | /** |
| 17 | * This class provides detection functions for specific content on a site. It can be used to easily detect the |
| 18 | * presence of known third party code. |
| 19 | * |
| 20 | * Note: Calling the `detectContent()` method will create a HTTP request to the site to retrieve data, only the main site URL |
| 21 | * will be checked |
| 22 | * |
| 23 | * Usage: |
| 24 | * |
| 25 | * $contentDetector = new SiteContentDetector(); |
| 26 | * $contentDetector->detectContent([GoogleAnalytics3::getId()]); |
| 27 | * if ($contentDetector->ga3) { |
| 28 | * // site is using GA3 |
| 29 | * } |
| 30 | * |
| 31 | * @api |
| 32 | */ |
| 33 | class SiteContentDetector |
| 34 | { |
| 35 | /** |
| 36 | * @var array<string, array<string, SiteContentDetectionAbstract>> |
| 37 | */ |
| 38 | public $detectedContent = [SiteContentDetectionAbstract::TYPE_TRACKER => [], SiteContentDetectionAbstract::TYPE_CMS => [], SiteContentDetectionAbstract::TYPE_JS_FRAMEWORK => [], SiteContentDetectionAbstract::TYPE_CONSENT_MANAGER => [], SiteContentDetectionAbstract::TYPE_JS_CRASH_ANALYTICS => [], SiteContentDetectionAbstract::TYPE_OTHER => []]; |
| 39 | public $connectedConsentManagers = []; |
| 40 | private $siteResponse = ['data' => '', 'headers' => []]; |
| 41 | /** @var Lazy */ |
| 42 | private $cache; |
| 43 | public function __construct(?Lazy $cache = null) |
| 44 | { |
| 45 | if ($cache === null) { |
| 46 | $this->cache = \Piwik\Cache::getLazyCache(); |
| 47 | } else { |
| 48 | $this->cache = $cache; |
| 49 | } |
| 50 | } |
| 51 | /** |
| 52 | * @return array<string, SiteContentDetectionAbstract[]> |
| 53 | */ |
| 54 | public static function getSiteContentDetectionsByType() : array |
| 55 | { |
| 56 | $instancesByType = []; |
| 57 | $classes = self::getAllSiteContentDetectionClasses(); |
| 58 | foreach ($classes as $className) { |
| 59 | $instancesByType[$className::getContentType()][] = StaticContainer::get($className); |
| 60 | } |
| 61 | return $instancesByType; |
| 62 | } |
| 63 | /** |
| 64 | * Returns the site content detection object with the provided id, or null if it can't be found |
| 65 | */ |
| 66 | public function getSiteContentDetectionById(string $id) : ?SiteContentDetectionAbstract |
| 67 | { |
| 68 | $classes = $this->getAllSiteContentDetectionClasses(); |
| 69 | foreach ($classes as $className) { |
| 70 | if ($className::getId() === $id) { |
| 71 | return StaticContainer::get($className); |
| 72 | } |
| 73 | } |
| 74 | return null; |
| 75 | } |
| 76 | /** |
| 77 | * @return string[] |
| 78 | */ |
| 79 | protected static function getAllSiteContentDetectionClasses() : array |
| 80 | { |
| 81 | return \Piwik\Plugin\Manager::getInstance()->findMultipleComponents('SiteContentDetection', SiteContentDetectionAbstract::class); |
| 82 | } |
| 83 | /** |
| 84 | * Reset the detections |
| 85 | */ |
| 86 | private function resetDetections() : void |
| 87 | { |
| 88 | $this->detectedContent = [SiteContentDetectionAbstract::TYPE_TRACKER => [], SiteContentDetectionAbstract::TYPE_CMS => [], SiteContentDetectionAbstract::TYPE_JS_FRAMEWORK => [], SiteContentDetectionAbstract::TYPE_CONSENT_MANAGER => [], SiteContentDetectionAbstract::TYPE_JS_CRASH_ANALYTICS => [], SiteContentDetectionAbstract::TYPE_OTHER => []]; |
| 89 | $this->connectedConsentManagers = []; |
| 90 | } |
| 91 | /** |
| 92 | * This will query the site and populate the class properties with |
| 93 | * the details of the detected content |
| 94 | * |
| 95 | * @param array $detectContent Array of content type for which to check, defaults to all, limiting this list |
| 96 | * will speed up the detection check. |
| 97 | * Allowed values are: |
| 98 | * * empty array - to run all detections |
| 99 | * * an array containing ids of detections, e.g. Wordpress::getId() or any of the |
| 100 | * type constants, e.g. SiteContentDetectionAbstract::TYPE_TRACKER |
| 101 | * @param ?int $idSite Override the site ID, will use the site from the current request if null |
| 102 | * @param ?array $siteResponse String containing the site data to search, if blank then data will be retrieved |
| 103 | * from the current request site via an http request |
| 104 | * @param int $timeOut How long to wait for the site to response, defaults to 5 seconds |
| 105 | */ |
| 106 | public function detectContent(array $detectContent = [], ?int $idSite = null, ?array $siteResponse = null, int $timeOut = 5) : void |
| 107 | { |
| 108 | $this->resetDetections(); |
| 109 | // If site data was passed in, then just run the detection checks against it and return. |
| 110 | if ($siteResponse) { |
| 111 | $this->siteResponse = $siteResponse; |
| 112 | $this->detectionChecks($detectContent); |
| 113 | return; |
| 114 | } |
| 115 | // Get the site id from the request object if not explicitly passed |
| 116 | if ($idSite === null) { |
| 117 | $idSite = \Piwik\Request::fromRequest()->getIntegerParameter('idSite', 0); |
| 118 | if (!$idSite) { |
| 119 | return; |
| 120 | } |
| 121 | } |
| 122 | $url = \Piwik\Site::getMainUrlFor($idSite); |
| 123 | // Check and load previously cached site content detection data if it exists |
| 124 | $cacheKey = 'SiteContentDetection_' . md5($url); |
| 125 | $siteContentDetectionCache = $this->cache->fetch($cacheKey); |
| 126 | if ($siteContentDetectionCache !== \false) { |
| 127 | if ($this->checkCacheHasRequiredProperties($detectContent, $siteContentDetectionCache)) { |
| 128 | $this->detectedContent = $siteContentDetectionCache['detectedContent']; |
| 129 | $this->connectedConsentManagers = $siteContentDetectionCache['connectedConsentManagers']; |
| 130 | return; |
| 131 | } |
| 132 | } |
| 133 | // No cache hit, no passed data, so make a request for the site content |
| 134 | $siteResponse = $this->requestSiteResponse($url, $timeOut); |
| 135 | // Abort if still no site data |
| 136 | if (empty($siteResponse['data'])) { |
| 137 | return; |
| 138 | } |
| 139 | $this->siteResponse = $siteResponse; |
| 140 | // We now have site data to analyze, so run the detection checks |
| 141 | $this->detectionChecks($detectContent); |
| 142 | // A request was made to get this data and it isn't currently cached, so write it to the cache now |
| 143 | $cacheLife = 60 * 60 * 24 * 7; |
| 144 | $this->saveToCache($cacheKey, $cacheLife); |
| 145 | } |
| 146 | /** |
| 147 | * Returns if the detection with the provided id was detected or not |
| 148 | * |
| 149 | * Note: self::detectContent needs to be called before. |
| 150 | */ |
| 151 | public function wasDetected(string $detectionClassId) : bool |
| 152 | { |
| 153 | foreach ($this->detectedContent as $type => $detectedClassIds) { |
| 154 | if (array_key_exists($detectionClassId, $detectedClassIds)) { |
| 155 | return $detectedClassIds[$detectionClassId] ?? \false; |
| 156 | } |
| 157 | } |
| 158 | return \false; |
| 159 | } |
| 160 | /** |
| 161 | * Returns an array containing ids of all detected detections of the given type |
| 162 | * |
| 163 | * @param int $type One of the SiteContentDetectionAbstract::TYPE_* constants |
| 164 | * @return array |
| 165 | */ |
| 166 | public function getDetectsByType(int $type) : array |
| 167 | { |
| 168 | $detected = []; |
| 169 | foreach ($this->detectedContent[$type] as $objId => $wasDetected) { |
| 170 | if (\true === $wasDetected) { |
| 171 | $detected[] = $objId; |
| 172 | } |
| 173 | } |
| 174 | return $detected; |
| 175 | } |
| 176 | /** |
| 177 | * Checks that all required detections are in the cache array |
| 178 | * |
| 179 | * @param array $detectContent |
| 180 | * @param array $cache |
| 181 | */ |
| 182 | private function checkCacheHasRequiredProperties(array $detectContent, array $cache) : bool |
| 183 | { |
| 184 | if (empty($detectContent)) { |
| 185 | foreach (self::getSiteContentDetectionsByType() as $type => $entries) { |
| 186 | foreach ($entries as $entry) { |
| 187 | if (!isset($cache['detectedContent'][$type][$entry::getId()])) { |
| 188 | return \false; |
| 189 | // random detection missing |
| 190 | } |
| 191 | } |
| 192 | } |
| 193 | return \true; |
| 194 | } |
| 195 | foreach ($detectContent as $requestedDetection) { |
| 196 | if (is_string($requestedDetection)) { |
| 197 | // specific detection |
| 198 | $detectionObj = $this->getSiteContentDetectionById($requestedDetection); |
| 199 | if (null !== $detectionObj && !isset($cache['detectedContent'][$detectionObj::getContentType()][$detectionObj::getId()])) { |
| 200 | return \false; |
| 201 | // specific detection was run before |
| 202 | } |
| 203 | } elseif (is_int($requestedDetection)) { |
| 204 | // detection type requested |
| 205 | $detectionsByType = self::getSiteContentDetectionsByType(); |
| 206 | if (isset($detectionsByType[$requestedDetection])) { |
| 207 | foreach ($detectionsByType[$requestedDetection] as $detectionObj) { |
| 208 | if (!isset($cache['detectedContent'][$requestedDetection][$detectionObj::getId()])) { |
| 209 | return \false; |
| 210 | // random detection missing |
| 211 | } |
| 212 | } |
| 213 | } |
| 214 | } |
| 215 | } |
| 216 | return \true; |
| 217 | } |
| 218 | /** |
| 219 | * Save data to the cache |
| 220 | */ |
| 221 | private function saveToCache(string $cacheKey, int $cacheLife) : void |
| 222 | { |
| 223 | $cacheData = ['detectedContent' => [], 'connectedConsentManagers' => []]; |
| 224 | // Load any existing cached values |
| 225 | $siteContentDetectionCache = $this->cache->fetch($cacheKey); |
| 226 | if (is_array($siteContentDetectionCache)) { |
| 227 | $cacheData = $siteContentDetectionCache; |
| 228 | } |
| 229 | foreach ($this->detectedContent as $type => $detections) { |
| 230 | if (!isset($cacheData['detectedContent'][$type])) { |
| 231 | $cacheData['detectedContent'][$type] = []; |
| 232 | } |
| 233 | foreach ($detections as $detectionId => $wasDetected) { |
| 234 | if (null !== $wasDetected) { |
| 235 | $cacheData['detectedContent'][$type][$detectionId] = $wasDetected; |
| 236 | } |
| 237 | } |
| 238 | } |
| 239 | $cacheData['connectedConsentManagers'] = array_merge($cacheData['connectedConsentManagers'], $this->connectedConsentManagers); |
| 240 | $this->cache->save($cacheKey, $cacheData, $cacheLife); |
| 241 | } |
| 242 | /** |
| 243 | * Run various detection checks for site content |
| 244 | * |
| 245 | * @param array $detectContent Array of detection types used to filter the checks that are run |
| 246 | */ |
| 247 | private function detectionChecks(array $detectContent) : void |
| 248 | { |
| 249 | $detections = $this->getSiteContentDetectionsByType(); |
| 250 | foreach ($detections as $type => $typeDetections) { |
| 251 | foreach ($typeDetections as $typeDetection) { |
| 252 | $this->detectedContent[$type][$typeDetection::getId()] = null; |
| 253 | if (in_array($type, $detectContent) || in_array($typeDetection::getId(), $detectContent) || empty($detectContent)) { |
| 254 | $this->detectedContent[$type][$typeDetection::getId()] = \false; |
| 255 | if ($typeDetection->isDetected($this->siteResponse['data'], $this->siteResponse['headers'])) { |
| 256 | if ($typeDetection instanceof ConsentManagerDetectionAbstract && $typeDetection->checkIsConnected($this->siteResponse['data'], $this->siteResponse['headers'])) { |
| 257 | $this->connectedConsentManagers[] = $typeDetection::getId(); |
| 258 | } |
| 259 | $this->detectedContent[$type][$typeDetection::getId()] = \true; |
| 260 | } |
| 261 | } |
| 262 | } |
| 263 | } |
| 264 | } |
| 265 | /** |
| 266 | * Retrieve data from the specified site using an HTTP request |
| 267 | * |
| 268 | * @return array |
| 269 | */ |
| 270 | private function requestSiteResponse(string $url, int $timeOut) : array |
| 271 | { |
| 272 | if (!$url) { |
| 273 | return []; |
| 274 | } |
| 275 | // If internet features are disabled, we don't try to fetch any site content |
| 276 | if (0 === GeneralConfig::getIntegerConfigValue('enable_internet_features', 0)) { |
| 277 | return []; |
| 278 | } |
| 279 | $siteData = []; |
| 280 | try { |
| 281 | $siteData = \Piwik\Http::sendHttpRequestBy(\Piwik\Http::getTransportMethod(), $url, $timeOut, null, null, null, 0, \false, \true, \false, \true); |
| 282 | } catch (\Exception $e) { |
| 283 | } |
| 284 | return $siteData; |
| 285 | } |
| 286 | /** |
| 287 | * Return an array of consent manager definitions which can be used to detect their presence on the site and show |
| 288 | * the associated guide links |
| 289 | * |
| 290 | * Note: This list is also used to display the known / supported consent managers on the "Ask for Consent" page |
| 291 | * For adding a new consent manager to this page, it needs to be added here. If a consent manager can't be detected |
| 292 | * automatically, simply leave the detections empty. |
| 293 | * |
| 294 | * @return array[] |
| 295 | */ |
| 296 | public static function getKnownConsentManagers() : array |
| 297 | { |
| 298 | $detections = self::getSiteContentDetectionsByType(); |
| 299 | $cmDetections = $detections[SiteContentDetectionAbstract::TYPE_CONSENT_MANAGER]; |
| 300 | $consentManagers = []; |
| 301 | foreach ($cmDetections as $detection) { |
| 302 | $consentManagers[$detection::getId()] = ['name' => $detection::getName(), 'instructionUrl' => $detection::getInstructionUrl()]; |
| 303 | } |
| 304 | return $consentManagers; |
| 305 | } |
| 306 | } |
| 307 |