ArchivingStatus.php
4 years ago
Loader.php
3 years ago
LoaderLock.php
4 years ago
Parameters.php
4 years ago
PluginsArchiver.php
3 years ago
PluginsArchiverException.php
5 years ago
Rules.php
4 years ago
Rules.php
385 lines
| 1 | <?php |
| 2 | /** |
| 3 | * Matomo - free/libre analytics platform |
| 4 | * |
| 5 | * @link https://matomo.org |
| 6 | * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later |
| 7 | * |
| 8 | */ |
| 9 | namespace Piwik\ArchiveProcessor; |
| 10 | |
| 11 | use Exception; |
| 12 | use Piwik\Config; |
| 13 | use Piwik\Config\GeneralConfig; |
| 14 | use Piwik\DataAccess\ArchiveWriter; |
| 15 | use Piwik\Date; |
| 16 | use Piwik\Log; |
| 17 | use Piwik\Option; |
| 18 | use Piwik\Piwik; |
| 19 | use Piwik\Plugin\Manager; |
| 20 | use Piwik\Plugins\CoreAdminHome\Controller; |
| 21 | use Piwik\Segment; |
| 22 | use Piwik\SettingsPiwik; |
| 23 | use Piwik\SettingsServer; |
| 24 | use Piwik\Site; |
| 25 | use Piwik\Tracker\Cache; |
| 26 | |
| 27 | /** |
| 28 | * This class contains Archiving rules/logic which are used when creating and processing Archives. |
| 29 | * |
| 30 | */ |
| 31 | class Rules |
| 32 | { |
| 33 | const OPTION_TODAY_ARCHIVE_TTL = 'todayArchiveTimeToLive'; |
| 34 | |
| 35 | const OPTION_BROWSER_TRIGGER_ARCHIVING = 'enableBrowserTriggerArchiving'; |
| 36 | |
| 37 | const FLAG_TABLE_PURGED = 'lastPurge_'; |
| 38 | |
| 39 | /** Flag that will forcefully disable the archiving process (used in tests only) */ |
| 40 | public static $archivingDisabledByTests = false; |
| 41 | |
| 42 | /** To disable the Pure Outdated Archive set this to true will skip this task */ |
| 43 | public static $disablePureOutdatedArchive = false; |
| 44 | |
| 45 | |
| 46 | /** |
| 47 | * Returns the name of the archive field used to tell the status of an archive, (ie, |
| 48 | * whether the archive was created successfully or not). |
| 49 | * |
| 50 | * @param array $idSites |
| 51 | * @param Segment $segment |
| 52 | * @param string $periodLabel |
| 53 | * @param string $plugin |
| 54 | * @return string |
| 55 | */ |
| 56 | public static function getDoneStringFlagFor(array $idSites, $segment, $periodLabel, $plugin) |
| 57 | { |
| 58 | if (!empty($plugin) |
| 59 | && !self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel) |
| 60 | ) { |
| 61 | return self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin); |
| 62 | } |
| 63 | return self::getDoneFlagArchiveContainsAllPlugins($segment); |
| 64 | } |
| 65 | |
| 66 | public static function shouldProcessReportsAllPlugins(array $idSites, Segment $segment, $periodLabel) |
| 67 | { |
| 68 | if (self::isRequestingToAndAbleToForceArchiveSinglePlugin()) { |
| 69 | return false; |
| 70 | } |
| 71 | |
| 72 | if ($segment->isEmpty() && ($periodLabel != 'range' || SettingsServer::isArchivePhpTriggered())) { |
| 73 | return true; |
| 74 | } |
| 75 | |
| 76 | if ($periodLabel === 'range' && !SettingsServer::isArchivePhpTriggered()) { |
| 77 | return false; |
| 78 | } |
| 79 | |
| 80 | return self::isSegmentPreProcessed($idSites, $segment); |
| 81 | } |
| 82 | |
| 83 | /** |
| 84 | * @param $idSites |
| 85 | * @return array |
| 86 | */ |
| 87 | public static function getSegmentsToProcess($idSites) |
| 88 | { |
| 89 | $knownSegmentsToArchiveAllSites = SettingsPiwik::getKnownSegmentsToArchive(); |
| 90 | |
| 91 | $segmentsToProcess = $knownSegmentsToArchiveAllSites; |
| 92 | foreach ($idSites as $idSite) { |
| 93 | $segmentForThisWebsite = SettingsPiwik::getKnownSegmentsToArchiveForSite($idSite); |
| 94 | $segmentsToProcess = array_merge($segmentsToProcess, $segmentForThisWebsite); |
| 95 | } |
| 96 | $segmentsToProcess = array_unique($segmentsToProcess); |
| 97 | return $segmentsToProcess; |
| 98 | } |
| 99 | |
| 100 | public static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin) |
| 101 | { |
| 102 | return 'done' . $segment->getHash() . '.' . $plugin; |
| 103 | } |
| 104 | |
| 105 | public static function getDoneFlagArchiveContainsAllPlugins(Segment $segment) |
| 106 | { |
| 107 | return 'done' . $segment->getHash(); |
| 108 | } |
| 109 | |
| 110 | /** |
| 111 | * Return done flags used to tell how the archiving process for a specific archive was completed, |
| 112 | * |
| 113 | * @param array $plugins |
| 114 | * @param $segment |
| 115 | * @return array |
| 116 | */ |
| 117 | public static function getDoneFlags(array $plugins, Segment $segment) |
| 118 | { |
| 119 | $doneFlags = array(); |
| 120 | $doneAllPlugins = self::getDoneFlagArchiveContainsAllPlugins($segment); |
| 121 | $doneFlags[$doneAllPlugins] = $doneAllPlugins; |
| 122 | |
| 123 | $plugins = array_unique($plugins); |
| 124 | foreach ($plugins as $plugin) { |
| 125 | $doneOnePlugin = self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin); |
| 126 | $doneFlags[$plugin] = $doneOnePlugin; |
| 127 | } |
| 128 | return $doneFlags; |
| 129 | } |
| 130 | |
| 131 | public static function getMinTimeProcessedForInProgressArchive( |
| 132 | Date $dateStart, |
| 133 | \Piwik\Period $period, |
| 134 | Segment $segment, |
| 135 | Site $site |
| 136 | ) { |
| 137 | $todayArchiveTimeToLive = self::getPeriodArchiveTimeToLiveDefault($period->getLabel()); |
| 138 | |
| 139 | $now = time(); |
| 140 | $minimumArchiveTime = $now - $todayArchiveTimeToLive; |
| 141 | |
| 142 | $idSites = array($site->getId()); |
| 143 | $isArchivingDisabled = Rules::isArchivingDisabledFor($idSites, $segment, $period->getLabel()); |
| 144 | if ($isArchivingDisabled) { |
| 145 | if ($period->getNumberOfSubperiods() == 0 |
| 146 | && $dateStart->getTimestamp() <= $now |
| 147 | ) { |
| 148 | // Today: accept any recent enough archive |
| 149 | $minimumArchiveTime = false; |
| 150 | } else { |
| 151 | // This week, this month, this year: |
| 152 | // accept any archive that was processed today after 00:00:01 this morning |
| 153 | $timezone = $site->getTimezone(); |
| 154 | $minimumArchiveTime = Date::factory(Date::factory('now', |
| 155 | $timezone)->getDateStartUTC())->setTimezone($timezone)->getTimestamp(); |
| 156 | } |
| 157 | } |
| 158 | return $minimumArchiveTime; |
| 159 | } |
| 160 | |
| 161 | public static function setTodayArchiveTimeToLive($timeToLiveSeconds) |
| 162 | { |
| 163 | $timeToLiveSeconds = (int)$timeToLiveSeconds; |
| 164 | if ($timeToLiveSeconds <= 0) { |
| 165 | throw new Exception(Piwik::translate('General_ExceptionInvalidArchiveTimeToLive')); |
| 166 | } |
| 167 | Option::set(self::OPTION_TODAY_ARCHIVE_TTL, $timeToLiveSeconds, $autoLoad = true); |
| 168 | } |
| 169 | |
| 170 | public static function getTodayArchiveTimeToLive() |
| 171 | { |
| 172 | $uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled(); |
| 173 | |
| 174 | if ($uiSettingIsEnabled) { |
| 175 | $timeToLive = Option::get(self::OPTION_TODAY_ARCHIVE_TTL); |
| 176 | if ($timeToLive !== false) { |
| 177 | return $timeToLive; |
| 178 | } |
| 179 | } |
| 180 | return self::getTodayArchiveTimeToLiveDefault(); |
| 181 | } |
| 182 | |
| 183 | public static function getPeriodArchiveTimeToLiveDefault($periodLabel) |
| 184 | { |
| 185 | if (empty($periodLabel) || strtolower($periodLabel) === 'day') { |
| 186 | return self::getTodayArchiveTimeToLive(); |
| 187 | } |
| 188 | |
| 189 | $config = Config::getInstance(); |
| 190 | $general = $config->General; |
| 191 | |
| 192 | $key = sprintf('time_before_%s_archive_considered_outdated', $periodLabel); |
| 193 | if (isset($general[$key]) && is_numeric($general[$key]) && $general[$key] > 0) { |
| 194 | return $general[$key]; |
| 195 | } |
| 196 | |
| 197 | return self::getTodayArchiveTimeToLive(); |
| 198 | } |
| 199 | |
| 200 | public static function getTodayArchiveTimeToLiveDefault() |
| 201 | { |
| 202 | return Config::getInstance()->General['time_before_today_archive_considered_outdated']; |
| 203 | } |
| 204 | |
| 205 | public static function isBrowserArchivingAvailableForSegments() |
| 206 | { |
| 207 | $generalConfig = Config::getInstance()->General; |
| 208 | return !$generalConfig['browser_archiving_disabled_enforce']; |
| 209 | } |
| 210 | |
| 211 | public static function isArchivingEnabledFor(array $idSites, Segment $segment, $periodLabel) |
| 212 | { |
| 213 | $isArchivingEnabled = self::isRequestAuthorizedToArchive() && !self::$archivingDisabledByTests; |
| 214 | |
| 215 | $generalConfig = Config::getInstance()->General; |
| 216 | |
| 217 | if ($periodLabel === 'range') { |
| 218 | if (isset($generalConfig['archiving_range_force_on_browser_request']) |
| 219 | && $generalConfig['archiving_range_force_on_browser_request'] == false |
| 220 | ) { |
| 221 | Log::debug("Not forcing archiving for range period."); |
| 222 | return $isArchivingEnabled; |
| 223 | } |
| 224 | |
| 225 | return true; |
| 226 | } |
| 227 | |
| 228 | if ($segment->isEmpty()) { |
| 229 | // viewing "All Visits" |
| 230 | return $isArchivingEnabled; |
| 231 | } |
| 232 | |
| 233 | if (!$isArchivingEnabled |
| 234 | && (!self::isBrowserArchivingAvailableForSegments() || self::isSegmentPreProcessed($idSites, $segment)) |
| 235 | && !SettingsServer::isArchivePhpTriggered() // Only applies when we are not running core:archive command |
| 236 | ) { |
| 237 | Log::debug("Archiving is disabled because of config setting browser_archiving_disabled_enforce=1 or because the segment is selected to be pre-processed."); |
| 238 | return false; |
| 239 | } |
| 240 | |
| 241 | return true; |
| 242 | } |
| 243 | |
| 244 | public static function isArchivingDisabledFor(array $idSites, Segment $segment, $periodLabel) |
| 245 | { |
| 246 | return !self::isArchivingEnabledFor($idSites, $segment, $periodLabel); |
| 247 | } |
| 248 | |
| 249 | public static function isRequestAuthorizedToArchive(Parameters $params = null) |
| 250 | { |
| 251 | $isRequestAuthorizedToArchive = Rules::isBrowserTriggerEnabled() || SettingsServer::isArchivePhpTriggered(); |
| 252 | |
| 253 | if (!empty($params)) { |
| 254 | /** |
| 255 | * @ignore |
| 256 | * |
| 257 | * @params bool &$isRequestAuthorizedToArchive |
| 258 | * @params Parameters $params |
| 259 | */ |
| 260 | Piwik::postEvent('Archiving.isRequestAuthorizedToArchive', [&$isRequestAuthorizedToArchive, $params]); |
| 261 | } |
| 262 | |
| 263 | return $isRequestAuthorizedToArchive; |
| 264 | } |
| 265 | |
| 266 | public static function isBrowserTriggerEnabled() |
| 267 | { |
| 268 | $uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled(); |
| 269 | |
| 270 | if ($uiSettingIsEnabled) { |
| 271 | $browserArchivingEnabled = Option::get(self::OPTION_BROWSER_TRIGGER_ARCHIVING); |
| 272 | if ($browserArchivingEnabled !== false) { |
| 273 | return (bool)$browserArchivingEnabled; |
| 274 | } |
| 275 | } |
| 276 | return (bool)Config::getInstance()->General['enable_browser_archiving_triggering']; |
| 277 | } |
| 278 | |
| 279 | public static function setBrowserTriggerArchiving($enabled) |
| 280 | { |
| 281 | if (!is_bool($enabled)) { |
| 282 | throw new Exception('Browser trigger archiving must be set to true or false.'); |
| 283 | } |
| 284 | Option::set(self::OPTION_BROWSER_TRIGGER_ARCHIVING, (int)$enabled, $autoLoad = true); |
| 285 | Cache::clearCacheGeneral(); |
| 286 | } |
| 287 | |
| 288 | /** |
| 289 | * Returns true if the archiving process should skip the calculation of unique visitors |
| 290 | * across several sites. The `[General] enable_processing_unique_visitors_multiple_sites` |
| 291 | * INI config option controls the value of this variable. |
| 292 | * |
| 293 | * @return bool |
| 294 | */ |
| 295 | public static function shouldSkipUniqueVisitorsCalculationForMultipleSites() |
| 296 | { |
| 297 | return Config::getInstance()->General['enable_processing_unique_visitors_multiple_sites'] != 1; |
| 298 | } |
| 299 | |
| 300 | /** |
| 301 | * @param array $idSites |
| 302 | * @param Segment $segment |
| 303 | * @return bool |
| 304 | */ |
| 305 | public static function isSegmentPreProcessed(array $idSites, Segment $segment) |
| 306 | { |
| 307 | $segmentsToProcess = self::getSegmentsToProcess($idSites); |
| 308 | |
| 309 | if (empty($segmentsToProcess)) { |
| 310 | return false; |
| 311 | } |
| 312 | // If the requested segment is one of the segments to pre-process |
| 313 | // we ensure that any call to the API will trigger archiving of all reports for this segment |
| 314 | $segment = $segment->getString(); |
| 315 | |
| 316 | // Turns out the getString() above returns the URL decoded segment string |
| 317 | $segmentsToProcessUrlDecoded = array_map('urldecode', $segmentsToProcess); |
| 318 | |
| 319 | return in_array($segment, $segmentsToProcess) |
| 320 | || in_array($segment, $segmentsToProcessUrlDecoded); |
| 321 | } |
| 322 | |
| 323 | /** |
| 324 | * Returns done flag values allowed to be selected |
| 325 | * |
| 326 | * @return string[] |
| 327 | */ |
| 328 | public static function getSelectableDoneFlagValues( |
| 329 | $includeInvalidated = true, |
| 330 | Parameters $params = null, |
| 331 | $checkAuthorizedToArchive = true |
| 332 | ) { |
| 333 | $possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY); |
| 334 | |
| 335 | if ($includeInvalidated) { |
| 336 | if (!$checkAuthorizedToArchive || !Rules::isRequestAuthorizedToArchive($params)) { |
| 337 | //If request is not authorized to archive then fetch also invalidated archives |
| 338 | $possibleValues[] = ArchiveWriter::DONE_INVALIDATED; |
| 339 | $possibleValues[] = ArchiveWriter::DONE_PARTIAL; |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | return $possibleValues; |
| 344 | } |
| 345 | |
| 346 | public static function isRequestingToAndAbleToForceArchiveSinglePlugin() |
| 347 | { |
| 348 | if (!SettingsServer::isArchivePhpTriggered()) { |
| 349 | return false; |
| 350 | } |
| 351 | |
| 352 | return !empty($_GET['pluginOnly']) || !empty($_POST['pluginOnly']); |
| 353 | } |
| 354 | |
| 355 | public static function isActuallyForceArchivingSinglePlugin() |
| 356 | { |
| 357 | return Loader::getArchivingDepth() <= 1 && self::isRequestingToAndAbleToForceArchiveSinglePlugin(); |
| 358 | } |
| 359 | |
| 360 | public static function shouldProcessSegmentsWhenReArchivingReports() |
| 361 | { |
| 362 | return Config::getInstance()->General['rearchive_reports_in_past_exclude_segments'] != 1; |
| 363 | } |
| 364 | |
| 365 | public static function isSegmentPluginArchivingDisabled($pluginName, $siteId = null) |
| 366 | { |
| 367 | $pluginArchivingSetting = GeneralConfig::getConfigValue('disable_archiving_segment_for_plugins', $siteId); |
| 368 | |
| 369 | if (empty($pluginArchivingSetting)) { |
| 370 | return false; |
| 371 | } |
| 372 | |
| 373 | if (is_string($pluginArchivingSetting)) { |
| 374 | $pluginArchivingSetting = explode(",", $pluginArchivingSetting); |
| 375 | $pluginArchivingSetting = array_filter($pluginArchivingSetting, function($plugin){ |
| 376 | return Manager::getInstance()->isValidPluginName($plugin); |
| 377 | }); |
| 378 | } |
| 379 | |
| 380 | $pluginArchivingSetting = array_map('strtolower', $pluginArchivingSetting); |
| 381 | |
| 382 | return in_array(strtolower($pluginName), $pluginArchivingSetting); |
| 383 | } |
| 384 | } |
| 385 |