Db
3 years ago
Handler
5 years ago
TableLogAction
5 years ago
Visit
5 years ago
Action.php
4 years ago
ActionPageview.php
5 years ago
Cache.php
4 years ago
Db.php
4 years ago
Failures.php
4 years ago
FingerprintSalt.php
4 years ago
GoalManager.php
4 years ago
Handler.php
5 years ago
IgnoreCookie.php
4 years ago
LogTable.php
5 years ago
Model.php
5 years ago
PageUrl.php
3 years ago
Request.php
3 years ago
RequestProcessor.php
5 years ago
RequestSet.php
4 years ago
Response.php
4 years ago
ScheduledTasksRunner.php
5 years ago
Settings.php
3 years ago
TableLogAction.php
5 years ago
TrackerCodeGenerator.php
3 years ago
TrackerConfig.php
3 years ago
Visit.php
4 years ago
VisitExcluded.php
3 years ago
VisitInterface.php
5 years ago
Visitor.php
5 years ago
VisitorNotFoundInDb.php
5 years ago
VisitorRecognizer.php
4 years ago
PageUrl.php
437 lines
| 1 | <?php |
| 2 | /** |
| 3 | * Matomo - free/libre analytics platform |
| 4 | * |
| 5 | * @link https://matomo.org |
| 6 | * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later |
| 7 | * |
| 8 | */ |
| 9 | |
| 10 | namespace Piwik\Tracker; |
| 11 | |
| 12 | use Piwik\Cache; |
| 13 | use Piwik\CacheId; |
| 14 | use Piwik\Tracker\Cache as TrackerCache; |
| 15 | use Piwik\Common; |
| 16 | use Piwik\Config; |
| 17 | use Piwik\Piwik; |
| 18 | use Piwik\Plugins\SitesManager\API as APISitesManager; |
| 19 | use Piwik\UrlHelper; |
| 20 | |
| 21 | class PageUrl |
| 22 | { |
| 23 | |
| 24 | /** |
| 25 | * Map URL prefixes to integers. |
| 26 | * @see self::normalizeUrl(), self::reconstructNormalizedUrl() |
| 27 | */ |
| 28 | public static $urlPrefixMap = array( |
| 29 | 'http://www.' => 1, |
| 30 | 'http://' => 0, |
| 31 | 'https://www.' => 3, |
| 32 | 'https://' => 2 |
| 33 | ); |
| 34 | |
| 35 | /** |
| 36 | * Given the Input URL, will exclude all query parameters set for this site |
| 37 | * |
| 38 | * @static |
| 39 | * @param string $originalUrl |
| 40 | * @param $idSite |
| 41 | * @param array $additionalParametersToExclude |
| 42 | * @return bool|string Returned URL is HTML entities decoded |
| 43 | */ |
| 44 | public static function excludeQueryParametersFromUrl($originalUrl, $idSite, $additionalParametersToExclude = []) |
| 45 | { |
| 46 | $originalUrl = self::cleanupUrl($originalUrl); |
| 47 | |
| 48 | $parsedUrl = @parse_url($originalUrl); |
| 49 | $parsedUrl = self::cleanupHostAndHashTag($parsedUrl, $idSite); |
| 50 | $parametersToExclude = array_merge(self::getQueryParametersToExclude($idSite), $additionalParametersToExclude); |
| 51 | |
| 52 | if (empty($parsedUrl['query'])) { |
| 53 | if (empty($parsedUrl['fragment'])) { |
| 54 | return UrlHelper::getParseUrlReverse($parsedUrl); |
| 55 | } |
| 56 | |
| 57 | // Exclude from the hash tag as well |
| 58 | $queryParameters = UrlHelper::getArrayFromQueryString($parsedUrl['fragment']); |
| 59 | $parsedUrl['fragment'] = UrlHelper::getQueryStringWithExcludedParameters($queryParameters, $parametersToExclude); |
| 60 | $url = UrlHelper::getParseUrlReverse($parsedUrl); |
| 61 | |
| 62 | return $url; |
| 63 | } |
| 64 | |
| 65 | $queryParameters = UrlHelper::getArrayFromQueryString($parsedUrl['query']); |
| 66 | $parsedUrl['query'] = UrlHelper::getQueryStringWithExcludedParameters($queryParameters, $parametersToExclude); |
| 67 | $url = UrlHelper::getParseUrlReverse($parsedUrl); |
| 68 | |
| 69 | return $url; |
| 70 | } |
| 71 | |
| 72 | /** |
| 73 | * Returns the array of parameters names that must be excluded from the Query String in all tracked URLs |
| 74 | * @static |
| 75 | * @param $idSite |
| 76 | * @return array |
| 77 | */ |
| 78 | public static function getQueryParametersToExclude($idSite) |
| 79 | { |
| 80 | $campaignTrackingParameters = Common::getCampaignParameters(); |
| 81 | |
| 82 | $campaignTrackingParameters = array_merge( |
| 83 | $campaignTrackingParameters[0], // campaign name parameters |
| 84 | $campaignTrackingParameters[1] // campaign keyword parameters |
| 85 | ); |
| 86 | |
| 87 | $website = TrackerCache::getCacheWebsiteAttributes($idSite); |
| 88 | $excludedParameters = self::getExcludedParametersFromWebsite($website); |
| 89 | |
| 90 | $parametersToExclude = array_merge( |
| 91 | ['ignore_referrer', 'ignore_referer'], |
| 92 | $excludedParameters, |
| 93 | self::getUrlParameterNamesToExcludeFromUrl(), |
| 94 | $campaignTrackingParameters |
| 95 | ); |
| 96 | |
| 97 | /** |
| 98 | * Triggered before setting the action url in Piwik\Tracker\Action so plugins can register |
| 99 | * parameters to be excluded from the tracking URL (e.g. campaign parameters). |
| 100 | * |
| 101 | * @param array &$parametersToExclude An array of parameters to exclude from the tracking url. |
| 102 | */ |
| 103 | Piwik::postEvent('Tracker.PageUrl.getQueryParametersToExclude', array(&$parametersToExclude)); |
| 104 | |
| 105 | if (!empty($parametersToExclude)) { |
| 106 | Common::printDebug('Excluding parameters "' . implode(',', $parametersToExclude) . '" from URL'); |
| 107 | } |
| 108 | |
| 109 | $parametersToExclude = array_map('strtolower', $parametersToExclude); |
| 110 | return $parametersToExclude; |
| 111 | } |
| 112 | |
| 113 | /** |
| 114 | * Returns the list of URL query parameters that should be removed from the tracked URL query string. |
| 115 | * |
| 116 | * @return array |
| 117 | */ |
| 118 | protected static function getUrlParameterNamesToExcludeFromUrl() |
| 119 | { |
| 120 | $paramsToExclude = Config::getInstance()->Tracker['url_query_parameter_to_exclude_from_url']; |
| 121 | $paramsToExclude = explode(",", $paramsToExclude); |
| 122 | $paramsToExclude = array_map('trim', $paramsToExclude); |
| 123 | return $paramsToExclude; |
| 124 | } |
| 125 | |
| 126 | /** |
| 127 | * Returns true if URL fragments should be removed for a specific site, |
| 128 | * false if otherwise. |
| 129 | * |
| 130 | * This function uses the Tracker cache and not the MySQL database. |
| 131 | * |
| 132 | * @param $idSite int The ID of the site to check for. |
| 133 | * @return bool |
| 134 | */ |
| 135 | public static function shouldRemoveURLFragmentFor($idSite) |
| 136 | { |
| 137 | $websiteAttributes = TrackerCache::getCacheWebsiteAttributes($idSite); |
| 138 | return empty($websiteAttributes['keep_url_fragment']); |
| 139 | } |
| 140 | |
| 141 | /** |
| 142 | * Cleans and/or removes the URL fragment of a URL. |
| 143 | * |
| 144 | * @param $urlFragment string The URL fragment to process. |
| 145 | * @param $idSite int|bool If not false, this function will check if URL fragments |
| 146 | * should be removed for the site w/ this ID and if so, |
| 147 | * the returned processed fragment will be empty. |
| 148 | * |
| 149 | * @return string The processed URL fragment. |
| 150 | */ |
| 151 | public static function processUrlFragment($urlFragment, $idSite = false) |
| 152 | { |
| 153 | // if we should discard the url fragment for this site, return an empty string as |
| 154 | // the processed url fragment |
| 155 | if ($idSite !== false |
| 156 | && PageUrl::shouldRemoveURLFragmentFor($idSite) |
| 157 | ) { |
| 158 | return ''; |
| 159 | } else { |
| 160 | // Remove trailing Hash tag in ?query#hash# |
| 161 | if (substr($urlFragment, -1) == '#') { |
| 162 | $urlFragment = substr($urlFragment, 0, strlen($urlFragment) - 1); |
| 163 | } |
| 164 | return $urlFragment; |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | /** |
| 169 | * Will cleanup the hostname (some browser do not strolower the hostname), |
| 170 | * and deal ith the hash tag on incoming URLs based on website setting. |
| 171 | * |
| 172 | * @param $parsedUrl |
| 173 | * @param $idSite int|bool The site ID of the current visit. This parameter is |
| 174 | * only used by the tracker to see if we should remove |
| 175 | * the URL fragment for this site. |
| 176 | * @return array |
| 177 | */ |
| 178 | protected static function cleanupHostAndHashTag($parsedUrl, $idSite = false) |
| 179 | { |
| 180 | if (empty($parsedUrl)) { |
| 181 | return $parsedUrl; |
| 182 | } |
| 183 | |
| 184 | if (!empty($parsedUrl['host'])) { |
| 185 | $parsedUrl['host'] = mb_strtolower($parsedUrl['host']); |
| 186 | } |
| 187 | |
| 188 | if (!empty($parsedUrl['fragment'])) { |
| 189 | $parsedUrl['fragment'] = PageUrl::processUrlFragment($parsedUrl['fragment'], $idSite); |
| 190 | } |
| 191 | |
| 192 | return $parsedUrl; |
| 193 | } |
| 194 | |
| 195 | /** |
| 196 | * Converts Matrix URL format |
| 197 | * from http://example.org/thing;paramA=1;paramB=6542 |
| 198 | * to http://example.org/thing?paramA=1¶mB=6542 |
| 199 | * |
| 200 | * @param string $originalUrl |
| 201 | * @return string |
| 202 | */ |
| 203 | public static function convertMatrixUrl($originalUrl) |
| 204 | { |
| 205 | $posFirstSemiColon = strpos($originalUrl, ";"); |
| 206 | |
| 207 | if (false === $posFirstSemiColon) { |
| 208 | return $originalUrl; |
| 209 | } |
| 210 | |
| 211 | $posQuestionMark = strpos($originalUrl, "?"); |
| 212 | $replace = (false === $posQuestionMark); |
| 213 | |
| 214 | if ($posQuestionMark > $posFirstSemiColon) { |
| 215 | $originalUrl = substr_replace($originalUrl, ";", $posQuestionMark, 1); |
| 216 | $replace = true; |
| 217 | } |
| 218 | |
| 219 | if ($replace) { |
| 220 | $originalUrl = substr_replace($originalUrl, "?", strpos($originalUrl, ";"), 1); |
| 221 | $originalUrl = str_replace(";", "&", $originalUrl); |
| 222 | } |
| 223 | |
| 224 | return $originalUrl; |
| 225 | } |
| 226 | |
| 227 | /** |
| 228 | * Clean up string contents (filter, truncate, ...) |
| 229 | * |
| 230 | * @param string $string Dirty string |
| 231 | * @return string |
| 232 | */ |
| 233 | public static function cleanupString($string) |
| 234 | { |
| 235 | $string = trim($string); |
| 236 | $string = str_replace(array("\n", "\r", "\0"), '', $string); |
| 237 | |
| 238 | $limit = Config::getInstance()->Tracker['page_maximum_length']; |
| 239 | $clean = substr($string, 0, $limit); |
| 240 | return $clean; |
| 241 | } |
| 242 | |
| 243 | protected static function reencodeParameterValue($value, $encoding) |
| 244 | { |
| 245 | if (is_string($value)) { |
| 246 | $decoded = urldecode($value); |
| 247 | try { |
| 248 | if (function_exists('mb_check_encoding') |
| 249 | && @mb_check_encoding($decoded, $encoding)) { |
| 250 | $value = urlencode(mb_convert_encoding($decoded, 'UTF-8', $encoding)); |
| 251 | } |
| 252 | } catch (\Error $e) { |
| 253 | // mb_check_encoding might throw an ValueError on PHP 8 if the given encoding does not exist |
| 254 | // we can't simply catch ValueError as it was introduced in PHP 8 |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | return $value; |
| 259 | } |
| 260 | |
| 261 | protected static function reencodeParametersArray($queryParameters, $encoding) |
| 262 | { |
| 263 | foreach ($queryParameters as &$value) { |
| 264 | if (is_array($value)) { |
| 265 | $value = self::reencodeParametersArray($value, $encoding); |
| 266 | } else { |
| 267 | $value = PageUrl::reencodeParameterValue($value, $encoding); |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | return $queryParameters; |
| 272 | } |
| 273 | |
| 274 | /** |
| 275 | * Checks if query parameters are of a non-UTF-8 encoding and converts the values |
| 276 | * from the specified encoding to UTF-8. |
| 277 | * This method is used to workaround browser/webapp bugs (see #3450). When |
| 278 | * browsers fail to encode query parameters in UTF-8, the tracker will send the |
| 279 | * charset of the page viewed and we can sometimes work around invalid data |
| 280 | * being stored. |
| 281 | * |
| 282 | * @param array $queryParameters Name/value mapping of query parameters. |
| 283 | * @param bool|string $encoding of the HTML page the URL is for. Used to workaround |
| 284 | * browser bugs & mis-coded webapps. See #3450. |
| 285 | * |
| 286 | * @return array |
| 287 | */ |
| 288 | public static function reencodeParameters(&$queryParameters, $encoding = false) |
| 289 | { |
| 290 | if (function_exists('mb_check_encoding')) { |
| 291 | // if query params are encoded w/ non-utf8 characters (due to browser bug or whatever), |
| 292 | // encode to UTF-8. |
| 293 | if (is_string($encoding) && |
| 294 | strtolower($encoding) !== 'utf-8' |
| 295 | ) { |
| 296 | Common::printDebug("Encoding page URL query parameters to $encoding."); |
| 297 | |
| 298 | $queryParameters = PageUrl::reencodeParametersArray($queryParameters, $encoding); |
| 299 | } |
| 300 | } else { |
| 301 | Common::printDebug("Page charset supplied in tracking request, but mbstring extension is not available."); |
| 302 | } |
| 303 | |
| 304 | return $queryParameters; |
| 305 | } |
| 306 | |
| 307 | public static function cleanupUrl($url) |
| 308 | { |
| 309 | $url = Common::unsanitizeInputValue($url); |
| 310 | $url = PageUrl::cleanupString($url); |
| 311 | $url = PageUrl::convertMatrixUrl($url); |
| 312 | |
| 313 | return $url; |
| 314 | } |
| 315 | |
| 316 | /** |
| 317 | * Build the full URL from the prefix ID and the rest. |
| 318 | * |
| 319 | * @param string $url |
| 320 | * @param integer $prefixId |
| 321 | * @return string |
| 322 | */ |
| 323 | public static function reconstructNormalizedUrl($url, $prefixId) |
| 324 | { |
| 325 | $map = array_flip(self::$urlPrefixMap); |
| 326 | |
| 327 | if ($prefixId !== null && isset($map[$prefixId])) { |
| 328 | $fullUrl = $map[$prefixId] . $url; |
| 329 | } else { |
| 330 | $fullUrl = $url; |
| 331 | } |
| 332 | |
| 333 | // Clean up host & hash tags, for URLs |
| 334 | $parsedUrl = @parse_url($fullUrl); |
| 335 | $parsedUrl = PageUrl::cleanupHostAndHashTag($parsedUrl); |
| 336 | $url = UrlHelper::getParseUrlReverse($parsedUrl); |
| 337 | |
| 338 | if (!empty($url)) { |
| 339 | return $url; |
| 340 | } |
| 341 | |
| 342 | return $fullUrl; |
| 343 | } |
| 344 | |
| 345 | /** |
| 346 | * Returns if the given host is also configured as https in page urls of given site |
| 347 | * |
| 348 | * @param $idSite |
| 349 | * @param $host |
| 350 | * @return false|mixed |
| 351 | * @throws \Exception |
| 352 | */ |
| 353 | public static function shouldUseHttpsHost($idSite, $host) |
| 354 | { |
| 355 | $cache = Cache::getTransientCache(); |
| 356 | |
| 357 | $cacheKeySiteUrls = CacheId::siteAware('siteurls', [$idSite]); |
| 358 | $cacheKeyHttpsForHost = CacheId::siteAware(sprintf('shouldusehttps-%s', $host), [$idSite]); |
| 359 | |
| 360 | $siteUrlCache = $cache->fetch($cacheKeySiteUrls); |
| 361 | |
| 362 | if (empty($siteUrlCache)) { |
| 363 | $siteUrlCache = APISitesManager::getInstance()->getSiteUrlsFromId($idSite); |
| 364 | $cache->save($cacheKeySiteUrls, $siteUrlCache); |
| 365 | } |
| 366 | |
| 367 | if (!$cache->contains($cacheKeyHttpsForHost)) { |
| 368 | $hostSiteCache = false; |
| 369 | |
| 370 | foreach ($siteUrlCache as $siteUrl) { |
| 371 | if (strpos(mb_strtolower($siteUrl), mb_strtolower('https://' . $host)) === 0) { |
| 372 | $hostSiteCache = true; |
| 373 | break; |
| 374 | } |
| 375 | } |
| 376 | |
| 377 | $cache->save($cacheKeyHttpsForHost, $hostSiteCache); |
| 378 | } |
| 379 | |
| 380 | return $cache->fetch($cacheKeyHttpsForHost); |
| 381 | } |
| 382 | |
| 383 | /** |
| 384 | * Extract the prefix from a URL. |
| 385 | * Return the prefix ID and the rest. |
| 386 | * |
| 387 | * @param string $url |
| 388 | * @return array |
| 389 | */ |
| 390 | public static function normalizeUrl($url) |
| 391 | { |
| 392 | foreach (self::$urlPrefixMap as $prefix => $id) { |
| 393 | if (strtolower(substr($url, 0, strlen($prefix))) == $prefix) { |
| 394 | return array( |
| 395 | 'url' => substr($url, strlen($prefix)), |
| 396 | 'prefixId' => $id |
| 397 | ); |
| 398 | } |
| 399 | } |
| 400 | |
| 401 | return array('url' => $url, 'prefixId' => null); |
| 402 | } |
| 403 | |
| 404 | public static function getUrlIfLookValid($url) |
| 405 | { |
| 406 | $url = PageUrl::cleanupString($url); |
| 407 | |
| 408 | if (!UrlHelper::isLookLikeUrl($url)) { |
| 409 | Common::printDebug("WARNING: URL looks invalid and is discarded"); |
| 410 | |
| 411 | return false; |
| 412 | } |
| 413 | |
| 414 | return $url; |
| 415 | } |
| 416 | |
| 417 | private static function getExcludedParametersFromWebsite($website) |
| 418 | { |
| 419 | if (isset($website['excluded_parameters'])) { |
| 420 | return $website['excluded_parameters']; |
| 421 | } |
| 422 | |
| 423 | return array(); |
| 424 | } |
| 425 | |
| 426 | public static function urldecodeValidUtf8($value) |
| 427 | { |
| 428 | $value = urldecode($value); |
| 429 | if (function_exists('mb_check_encoding') |
| 430 | && !@mb_check_encoding($value, 'utf-8') |
| 431 | ) { |
| 432 | return urlencode($value); |
| 433 | } |
| 434 | return $value; |
| 435 | } |
| 436 | } |
| 437 |