ArchiveInvalidator
5 years ago
ArchiveInvalidator.php
5 years ago
ArchivePurger.php
5 years ago
ArchiveQuery.php
5 years ago
ArchiveQueryFactory.php
5 years ago
Chunk.php
5 years ago
DataCollection.php
5 years ago
DataTableFactory.php
5 years ago
Parameters.php
5 years ago
ArchiveInvalidator.php
812 lines
| 1 | <?php |
| 2 | /** |
| 3 | * Matomo - free/libre analytics platform |
| 4 | * |
| 5 | * @link https://matomo.org |
| 6 | * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later |
| 7 | * |
| 8 | */ |
| 9 | |
| 10 | namespace Piwik\Archive; |
| 11 | |
| 12 | use Piwik\Access; |
| 13 | use Piwik\Archive\ArchiveInvalidator\InvalidationResult; |
| 14 | use Piwik\ArchiveProcessor\ArchivingStatus; |
| 15 | use Piwik\ArchiveProcessor\Loader; |
| 16 | use Piwik\ArchiveProcessor\Rules; |
| 17 | use Piwik\Config; |
| 18 | use Piwik\Container\StaticContainer; |
| 19 | use Piwik\CronArchive\ReArchiveList; |
| 20 | use Piwik\CronArchive\SegmentArchiving; |
| 21 | use Piwik\DataAccess\ArchiveTableCreator; |
| 22 | use Piwik\DataAccess\Model; |
| 23 | use Piwik\Date; |
| 24 | use Piwik\Db; |
| 25 | use Piwik\Option; |
| 26 | use Piwik\Common; |
| 27 | use Piwik\Piwik; |
| 28 | use Piwik\Plugin\Manager; |
| 29 | use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; |
| 30 | use Piwik\Plugins\PrivacyManager\PrivacyManager; |
| 31 | use Piwik\Period; |
| 32 | use Piwik\Segment; |
| 33 | use Piwik\SettingsServer; |
| 34 | use Piwik\Site; |
| 35 | use Piwik\Tracker\Cache; |
| 36 | use Piwik\Tracker\Model as TrackerModel; |
| 37 | use Psr\Log\LoggerInterface; |
| 38 | |
| 39 | /** |
| 40 | * Service that can be used to invalidate archives or add archive references to a list so they will |
| 41 | * be invalidated later. |
| 42 | * |
| 43 | * Archives are put in an "invalidated" state by setting the done flag to `ArchiveWriter::DONE_INVALIDATED`. |
| 44 | * This class also adds the archive's associated site to the a distributed list and adding the archive's year month to another |
| 45 | * distributed list. |
| 46 | * |
| 47 | * CronArchive will reprocess the archive data for all sites in the first list, and a scheduled task |
| 48 | * will purge the old, invalidated data in archive tables identified by the second list. |
| 49 | * |
| 50 | * Until CronArchive, or browser triggered archiving, re-processes data for an invalidated archive, the invalidated |
| 51 | * archive data will still be displayed in the UI and API. |
| 52 | * |
| 53 | * ### Deferred Invalidation |
| 54 | * |
| 55 | * Invalidating archives means running queries on one or more archive tables. In some situations, like during |
| 56 | * tracking, this is not desired. In such cases, archive references can be added to a list via the |
| 57 | * rememberToInvalidateArchivedReportsLater method, which will add the reference to a distributed list |
| 58 | * |
| 59 | * Later, during Piwik's normal execution, the list will be read and every archive it references will |
| 60 | * be invalidated. |
| 61 | */ |
| 62 | class ArchiveInvalidator |
| 63 | { |
| 64 | const TRACKER_CACHE_KEY = 'ArchiveInvalidator.rememberToInvalidate'; |
| 65 | |
| 66 | const INVALIDATION_STATUS_QUEUED = 0; |
| 67 | const INVALIDATION_STATUS_IN_PROGRESS = 1; |
| 68 | |
| 69 | private $rememberArchivedReportIdStart = 'report_to_invalidate_'; |
| 70 | |
| 71 | /** |
| 72 | * @var Model |
| 73 | */ |
| 74 | private $model; |
| 75 | |
| 76 | /** |
| 77 | * @var ArchivingStatus |
| 78 | */ |
| 79 | private $archivingStatus; |
| 80 | |
| 81 | /** |
| 82 | * @var SegmentArchiving |
| 83 | */ |
| 84 | private $segmentArchiving; |
| 85 | |
| 86 | /** |
| 87 | * @var LoggerInterface |
| 88 | */ |
| 89 | private $logger; |
| 90 | |
| 91 | /** |
| 92 | * @var int[] |
| 93 | */ |
| 94 | private $allIdSitesCache; |
| 95 | |
| 96 | public function __construct(Model $model, ArchivingStatus $archivingStatus, LoggerInterface $logger) |
| 97 | { |
| 98 | $this->model = $model; |
| 99 | $this->archivingStatus = $archivingStatus; |
| 100 | $this->segmentArchiving = null; |
| 101 | $this->logger = $logger; |
| 102 | } |
| 103 | |
| 104 | public function getAllRememberToInvalidateArchivedReportsLater() |
| 105 | { |
| 106 | // we do not really have to get the value first. we could simply always try to call set() and it would update or |
| 107 | // insert the record if needed but we do not want to lock the table (especially since there are still some |
| 108 | // MyISAM installations) |
| 109 | $values = Option::getLike('%' . $this->rememberArchivedReportIdStart . '%'); |
| 110 | |
| 111 | $all = []; |
| 112 | foreach ($values as $name => $value) { |
| 113 | $suffix = substr($name, strpos($name, $this->rememberArchivedReportIdStart)); |
| 114 | $suffix = str_replace($this->rememberArchivedReportIdStart, '', $suffix); |
| 115 | list($idSite, $dateStr) = explode('_', $suffix); |
| 116 | |
| 117 | $all[$idSite][$dateStr] = $value; |
| 118 | } |
| 119 | return $all; |
| 120 | } |
| 121 | |
| 122 | public function rememberToInvalidateArchivedReportsLater($idSite, Date $date) |
| 123 | { |
| 124 | if (SettingsServer::isTrackerApiRequest()) { |
| 125 | $value = $this->getRememberedArchivedReportsOptionFromTracker($idSite, $date->toString()); |
| 126 | } else { |
| 127 | // To support multiple transactions at once, look for any other process to have set (and committed) |
| 128 | // this report to be invalidated. |
| 129 | $key = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date->toString()); |
| 130 | |
| 131 | // we do not really have to get the value first. we could simply always try to call set() and it would update or |
| 132 | // insert the record if needed but we do not want to lock the table (especially since there are still some |
| 133 | // MyISAM installations) |
| 134 | $value = Option::getLike('%' . $key . '%'); |
| 135 | } |
| 136 | |
| 137 | // getLike() returns an empty array rather than 'false' |
| 138 | if (empty($value)) { |
| 139 | // In order to support multiple concurrent transactions, add our pid to the end of the key so that it will just insert |
| 140 | // rather than waiting on some other process to commit before proceeding.The issue is that with out this, more than |
| 141 | // one process is trying to add the exact same value to the table, which causes contention. With the pid suffixed to |
| 142 | // the value, each process can successfully enter its own row in the table. The net result will be the same. We could |
| 143 | // always just set this, but it would result in a lot of rows in the options table.. more than needed. With this |
| 144 | // change you'll have at most N rows per date/site, where N is the number of parallel requests on this same idsite/date |
| 145 | // that happen to run in overlapping transactions. |
| 146 | $mykey = $this->buildRememberArchivedReportIdProcessSafe($idSite, $date->toString()); |
| 147 | Option::set($mykey, '1'); |
| 148 | Cache::clearCacheGeneral(); |
| 149 | return $mykey; |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | private function getRememberedArchivedReportsOptionFromTracker($idSite, $dateStr) |
| 154 | { |
| 155 | $cacheKey = self::TRACKER_CACHE_KEY; |
| 156 | |
| 157 | $generalCache = Cache::getCacheGeneral(); |
| 158 | if (empty($generalCache[$cacheKey][$idSite][$dateStr])) { |
| 159 | return []; |
| 160 | } |
| 161 | |
| 162 | return $generalCache[$cacheKey][$idSite][$dateStr]; |
| 163 | } |
| 164 | |
| 165 | public function getRememberedArchivedReportsThatShouldBeInvalidated() |
| 166 | { |
| 167 | $reports = Option::getLike('%' . $this->rememberArchivedReportIdStart . '%_%'); |
| 168 | |
| 169 | $sitesPerDay = array(); |
| 170 | |
| 171 | foreach ($reports as $report => $value) { |
| 172 | $report = substr($report, strpos($report, $this->rememberArchivedReportIdStart)); |
| 173 | $report = str_replace($this->rememberArchivedReportIdStart, '', $report); |
| 174 | $report = explode('_', $report); |
| 175 | $siteId = (int) $report[0]; |
| 176 | $date = $report[1]; |
| 177 | |
| 178 | if (empty($siteId)) { |
| 179 | continue; |
| 180 | } |
| 181 | |
| 182 | if (empty($sitesPerDay[$date])) { |
| 183 | $sitesPerDay[$date] = array(); |
| 184 | } |
| 185 | |
| 186 | $sitesPerDay[$date][] = $siteId; |
| 187 | } |
| 188 | |
| 189 | return $sitesPerDay; |
| 190 | } |
| 191 | |
| 192 | private function buildRememberArchivedReportIdForSite($idSite) |
| 193 | { |
| 194 | return $this->rememberArchivedReportIdStart . (int) $idSite; |
| 195 | } |
| 196 | |
| 197 | private function buildRememberArchivedReportIdForSiteAndDate($idSite, $date) |
| 198 | { |
| 199 | $id = $this->buildRememberArchivedReportIdForSite($idSite); |
| 200 | $id .= '_' . trim($date); |
| 201 | |
| 202 | return $id; |
| 203 | } |
| 204 | |
| 205 | // This version is multi process safe on the insert of a new date to invalidate. |
| 206 | private function buildRememberArchivedReportIdProcessSafe($idSite, $date) |
| 207 | { |
| 208 | $id = Common::getRandomString(4, 'abcdefghijklmnoprstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ') . '_'; |
| 209 | $id .= $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date); |
| 210 | $id .= '_' . Common::getProcessId(); |
| 211 | |
| 212 | return $id; |
| 213 | } |
| 214 | |
| 215 | public function forgetRememberedArchivedReportsToInvalidateForSite($idSite) |
| 216 | { |
| 217 | $id = $this->buildRememberArchivedReportIdForSite($idSite) . '\_'; |
| 218 | $this->deleteOptionLike($id); |
| 219 | Cache::clearCacheGeneral(); |
| 220 | } |
| 221 | |
| 222 | /** |
| 223 | * @internal |
| 224 | * After calling this method, make sure to call Cache::clearCacheGeneral(); For performance reasons we don't call |
| 225 | * this here immediately in case there are multiple invalidations. |
| 226 | */ |
| 227 | public function forgetRememberedArchivedReportsToInvalidate($idSite, Date $date) |
| 228 | { |
| 229 | $id = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date->toString()); |
| 230 | |
| 231 | // The process pid is added to the end of the entry in order to support multiple concurrent transactions. |
| 232 | // So this must be a deleteLike call to get all the entries, where there used to only be one. |
| 233 | $this->deleteOptionLike($id); |
| 234 | Cache::clearCacheGeneral(); |
| 235 | } |
| 236 | |
| 237 | private function deleteOptionLike($id) |
| 238 | { |
| 239 | // we're not using deleteLike since it maybe could cause deadlocks see https://github.com/matomo-org/matomo/issues/15545 |
| 240 | // we want to reduce number of rows scanned and only delete specific primary key |
| 241 | $keys = Option::getLike('%' . $id . '%'); |
| 242 | |
| 243 | if (empty($keys)) { |
| 244 | return; |
| 245 | } |
| 246 | |
| 247 | $keys = array_keys($keys); |
| 248 | |
| 249 | $placeholders = Common::getSqlStringFieldsArray($keys); |
| 250 | |
| 251 | $table = Common::prefixTable('option'); |
| 252 | Db::query('DELETE FROM `' . $table . '` WHERE `option_name` IN (' . $placeholders . ')', $keys); |
| 253 | } |
| 254 | |
| 255 | /** |
| 256 | * @param $idSites int[] |
| 257 | * @param $dates Date[]|string[] |
| 258 | * @param $period string |
| 259 | * @param $segment Segment |
| 260 | * @param bool $cascadeDown |
| 261 | * @param bool $forceInvalidateNonexistantRanges set true to force inserting rows for ranges in archive_invalidations |
| 262 | * @param string $name null to make sure every plugin is archived when this invalidation is processed by core:archive, |
| 263 | * or a plugin name to only archive the specific plugin. |
| 264 | * @return InvalidationResult |
| 265 | * @throws \Exception |
| 266 | */ |
| 267 | public function markArchivesAsInvalidated(array $idSites, array $dates, $period, Segment $segment = null, $cascadeDown = false, |
| 268 | $forceInvalidateNonexistantRanges = false, $name = null, $ignorePurgeLogDataDate = false) |
| 269 | { |
| 270 | $plugin = null; |
| 271 | if ($name && strpos($name, '.') !== false) { |
| 272 | list($plugin) = explode('.', $name); |
| 273 | } |
| 274 | |
| 275 | if ($plugin |
| 276 | && !Manager::getInstance()->isPluginActivated($plugin) |
| 277 | ) { |
| 278 | throw new \Exception("Plugin is not activated: '$plugin'"); |
| 279 | } |
| 280 | |
| 281 | $invalidationInfo = new InvalidationResult(); |
| 282 | |
| 283 | // quick fix for #15086, if we're only invalidating today's date for a site, don't add the site to the list of sites |
| 284 | // to reprocess. |
| 285 | $hasMoreThanJustToday = []; |
| 286 | foreach ($idSites as $idSite) { |
| 287 | $hasMoreThanJustToday[$idSite] = true; |
| 288 | $tz = Site::getTimezoneFor($idSite); |
| 289 | |
| 290 | if (($period == 'day' || $period === false) |
| 291 | && count($dates) == 1 |
| 292 | && ((string)$dates[0]) == ((string)Date::factoryInTimezone('today', $tz)) |
| 293 | ) { |
| 294 | $hasMoreThanJustToday[$idSite] = false; |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | /** |
| 299 | * Triggered when a Matomo user requested the invalidation of some reporting archives. Using this event, plugin |
| 300 | * developers can automatically invalidate another site, when a site is being invalidated. A plugin may even |
| 301 | * remove an idSite from the list of sites that should be invalidated to prevent it from ever being |
| 302 | * invalidated. |
| 303 | * |
| 304 | * **Example** |
| 305 | * |
| 306 | * public function getIdSitesToMarkArchivesAsInvalidates(&$idSites) |
| 307 | * { |
| 308 | * if (in_array(1, $idSites)) { |
| 309 | * $idSites[] = 5; // when idSite 1 is being invalidated, also invalidate idSite 5 |
| 310 | * } |
| 311 | * } |
| 312 | * |
| 313 | * @param array &$idSites An array containing a list of site IDs which are requested to be invalidated. |
| 314 | */ |
| 315 | Piwik::postEvent('Archiving.getIdSitesToMarkArchivesAsInvalidated', array(&$idSites)); |
| 316 | // we trigger above event on purpose here and it is good that the segment was created like |
| 317 | // `new Segment($segmentString, $idSites)` because when a user adds a site via this event, the added idSite |
| 318 | // might not have this segment meaning we avoid a possible error. For the workflow to work, any added or removed |
| 319 | // idSite does not need to be added to $segment. |
| 320 | |
| 321 | $datesToInvalidate = $this->removeDatesThatHaveBeenPurged($dates, $period, $invalidationInfo, $ignorePurgeLogDataDate); |
| 322 | |
| 323 | $allPeriodsToInvalidate = $this->getAllPeriodsByYearMonth($period, $datesToInvalidate, $cascadeDown); |
| 324 | |
| 325 | $this->markArchivesInvalidated($idSites, $allPeriodsToInvalidate, $segment, $period != 'range', $forceInvalidateNonexistantRanges, $name); |
| 326 | |
| 327 | $isInvalidatingDays = $period == 'day' || $cascadeDown || empty($period); |
| 328 | $isNotInvalidatingSegment = empty($segment) || empty($segment->getString()); |
| 329 | if ($isInvalidatingDays |
| 330 | && $isNotInvalidatingSegment |
| 331 | ) { |
| 332 | foreach ($idSites as $idSite) { |
| 333 | foreach ($dates as $date) { |
| 334 | if (is_string($date)) { |
| 335 | $date = Date::factory($date); |
| 336 | } |
| 337 | |
| 338 | $this->forgetRememberedArchivedReportsToInvalidate($idSite, $date); |
| 339 | } |
| 340 | } |
| 341 | } |
| 342 | Cache::clearCacheGeneral(); |
| 343 | |
| 344 | return $invalidationInfo; |
| 345 | } |
| 346 | |
| 347 | private function getAllPeriodsByYearMonth($periodOrAll, $dates, $cascadeDown, &$result = []) |
| 348 | { |
| 349 | $periods = $periodOrAll ? [$periodOrAll] : ['day']; |
| 350 | foreach ($periods as $period) { |
| 351 | foreach ($dates as $date) { |
| 352 | $periodObj = $this->makePeriod($date, $period); |
| 353 | |
| 354 | $result[$this->getYearMonth($periodObj)][$this->getUniquePeriodId($periodObj)] = $periodObj; |
| 355 | |
| 356 | // cascade down |
| 357 | if ($cascadeDown |
| 358 | && $period != 'range' |
| 359 | ) { |
| 360 | $this->addChildPeriodsByYearMonth($result, $periodObj); |
| 361 | } |
| 362 | |
| 363 | // cascade up |
| 364 | // if the period spans multiple years or months, it won't be used when aggregating parent periods, so |
| 365 | // we can avoid invalidating it |
| 366 | if ($this->shouldPropagateUp($periodObj) |
| 367 | && $period != 'range' |
| 368 | ) { |
| 369 | $this->addParentPeriodsByYearMonth($result, $periodObj); |
| 370 | } |
| 371 | } |
| 372 | } |
| 373 | |
| 374 | return $result; |
| 375 | } |
| 376 | |
| 377 | private function shouldPropagateUp(Period $periodObj) |
| 378 | { |
| 379 | return $periodObj->getDateStart()->toString('Y') == $periodObj->getDateEnd()->toString('Y') |
| 380 | && $periodObj->getDateStart()->toString('m') == $periodObj->getDateEnd()->toString('m'); |
| 381 | } |
| 382 | |
| 383 | private function addChildPeriodsByYearMonth(&$result, Period $period) |
| 384 | { |
| 385 | if ($period->getLabel() == 'range') { |
| 386 | return; |
| 387 | } else if ($period->getLabel() == 'day' |
| 388 | && $this->shouldPropagateUp($period) |
| 389 | ) { |
| 390 | $this->addParentPeriodsByYearMonth($result, $period); |
| 391 | return; |
| 392 | } |
| 393 | |
| 394 | foreach ($period->getSubperiods() as $subperiod) { |
| 395 | $result[$this->getYearMonth($subperiod)][$this->getUniquePeriodId($subperiod)] = $subperiod; |
| 396 | $this->addChildPeriodsByYearMonth($result, $subperiod); |
| 397 | } |
| 398 | } |
| 399 | |
| 400 | private function addParentPeriodsByYearMonth(&$result, Period $period, Date $originalDate = null) |
| 401 | { |
| 402 | if ($period->getLabel() == 'year' |
| 403 | || $period->getLabel() == 'range' |
| 404 | ) { |
| 405 | return; |
| 406 | } |
| 407 | |
| 408 | $originalDate = $originalDate ?? $period->getDateStart(); |
| 409 | |
| 410 | $parentPeriod = Period\Factory::build($period->getParentPeriodLabel(), $originalDate); |
| 411 | $result[$this->getYearMonth($parentPeriod)][$this->getUniquePeriodId($parentPeriod)] = $parentPeriod; |
| 412 | $this->addParentPeriodsByYearMonth($result, $parentPeriod, $originalDate); |
| 413 | } |
| 414 | |
| 415 | /** |
| 416 | * @param $idSites int[] |
| 417 | * @param $dates Date[] |
| 418 | * @param $period string |
| 419 | * @param $segment Segment |
| 420 | * @param bool $cascadeDown |
| 421 | * @return InvalidationResult |
| 422 | * @throws \Exception |
| 423 | */ |
| 424 | public function markArchivesOverlappingRangeAsInvalidated(array $idSites, array $dates, Segment $segment = null) |
| 425 | { |
| 426 | $invalidationInfo = new InvalidationResult(); |
| 427 | |
| 428 | $ranges = array(); |
| 429 | foreach ($dates as $dateRange) { |
| 430 | $ranges[] = Period\Factory::build('range', $dateRange[0] . ',' . $dateRange[1]); |
| 431 | } |
| 432 | |
| 433 | $invalidatedMonths = array(); |
| 434 | $archiveNumericTables = ArchiveTableCreator::getTablesArchivesInstalled($type = ArchiveTableCreator::NUMERIC_TABLE); |
| 435 | foreach ($archiveNumericTables as $table) { |
| 436 | $tableDate = ArchiveTableCreator::getDateFromTableName($table); |
| 437 | |
| 438 | $rowsAffected = $this->model->updateArchiveAsInvalidated($table, $idSites, $ranges, $segment); |
| 439 | if ($rowsAffected > 0) { |
| 440 | $invalidatedMonths[] = $tableDate; |
| 441 | } |
| 442 | } |
| 443 | |
| 444 | foreach ($idSites as $idSite) { |
| 445 | foreach ($dates as $dateRange) { |
| 446 | $this->forgetRememberedArchivedReportsToInvalidate($idSite, $dateRange[0]); |
| 447 | $invalidationInfo->processedDates[] = $dateRange[0]; |
| 448 | } |
| 449 | } |
| 450 | |
| 451 | Cache::clearCacheGeneral(); |
| 452 | |
| 453 | return $invalidationInfo; |
| 454 | } |
| 455 | |
| 456 | /** |
| 457 | * Schedule rearchiving of reports for a single plugin or single report for N months in the past. The next time |
| 458 | * core:archive is run, they will be processed. |
| 459 | * |
| 460 | * @param int[]|string $idSites A list of idSites or 'all' |
| 461 | * @param string $plugin |
| 462 | * @param string|null $report |
| 463 | * @param Date|null $startDate |
| 464 | * @throws \Exception |
| 465 | * @api |
| 466 | */ |
| 467 | public function reArchiveReport($idSites, string $plugin = null, string $report = null, Date $startDate = null, Segment $segment = null) |
| 468 | { |
| 469 | $date2 = Date::yesterday(); |
| 470 | |
| 471 | $earliestDateToRearchive = $this->getEarliestDateToRearchive(); |
| 472 | if (empty($startDate)) { |
| 473 | if (empty($earliestDateToRearchive)) { |
| 474 | return null; // INI setting set to 0 months so no rearchiving |
| 475 | } |
| 476 | |
| 477 | $startDate = $earliestDateToRearchive; |
| 478 | } else if (!empty($earliestDateToRearchive)) { |
| 479 | // don't allow archiving further back than the rearchive_reports_in_past_last_n_months date allows |
| 480 | $startDate = $startDate->isEarlier($earliestDateToRearchive) ? $earliestDateToRearchive : $startDate; |
| 481 | } |
| 482 | |
| 483 | if ($idSites === 'all') { |
| 484 | $idSites = $this->getAllSitesId(); |
| 485 | } |
| 486 | |
| 487 | $dates = []; |
| 488 | $date = $startDate; |
| 489 | while ($date->isEarlier($date2)) { |
| 490 | $dates[] = $date; |
| 491 | $date = $date->addDay(1); |
| 492 | } |
| 493 | |
| 494 | if (empty($dates)) { |
| 495 | return; |
| 496 | } |
| 497 | |
| 498 | $name = $plugin; |
| 499 | if (!empty($report)) { |
| 500 | $name .= '.' . $report; |
| 501 | } |
| 502 | |
| 503 | $this->markArchivesAsInvalidated($idSites, $dates, 'day', $segment, $cascadeDown = false, $forceInvalidateRanges = false, $name); |
| 504 | if (empty($segment) |
| 505 | && Rules::shouldProcessSegmentsWhenReArchivingReports() |
| 506 | ) { |
| 507 | foreach ($idSites as $idSite) { |
| 508 | foreach (Rules::getSegmentsToProcess([$idSite]) as $segment) { |
| 509 | $this->markArchivesAsInvalidated($idSites, $dates, 'day', new Segment($segment, [$idSite]), |
| 510 | $cascadeDown = false, $forceInvalidateRanges = false, $name); |
| 511 | } |
| 512 | } |
| 513 | } |
| 514 | } |
| 515 | |
| 516 | /** |
| 517 | * Remove invalidations for a specific report or all invalidations for a specific plugin. If your plugin supports |
| 518 | * archiving data in the past, you may want to call this method to remove any pending invalidations if, for example, |
| 519 | * your plugin is deactivated or a report deleted. |
| 520 | * |
| 521 | * @param int|int[] $idSite one or more site IDs or 'all' for all site IDs |
| 522 | * @param string $string |
| 523 | * @param string|null $report |
| 524 | */ |
| 525 | public function removeInvalidations($idSite, $plugin, $report = null) |
| 526 | { |
| 527 | if (empty($report)) { |
| 528 | $this->model->removeInvalidationsLike($idSite, $plugin); |
| 529 | } else { |
| 530 | $this->model->removeInvalidations($idSite, $plugin, $report); |
| 531 | } |
| 532 | } |
| 533 | |
| 534 | /** |
| 535 | * Schedules a re-archiving reports without propagating exceptions. This is scheduled |
| 536 | * since adding invalidations can take a long time and delay UI response times. |
| 537 | * |
| 538 | * @param int|int[]|'all' $idSites |
| 539 | * @param string|int $pluginName |
| 540 | * @param string|null $report |
| 541 | * @param Date|null $startDate |
| 542 | */ |
| 543 | public function scheduleReArchiving($idSites, string $pluginName = null, $report = null, Date $startDate = null, |
| 544 | Segment $segment = null) |
| 545 | { |
| 546 | if (!empty($report)) { |
| 547 | $this->removeInvalidationsSafely($idSites, $pluginName, $report); |
| 548 | } |
| 549 | try { |
| 550 | $reArchiveList = new ReArchiveList($this->logger); |
| 551 | $reArchiveList->add(json_encode([ |
| 552 | 'idSites' => $idSites, |
| 553 | 'pluginName' => $pluginName, |
| 554 | 'report' => $report, |
| 555 | 'startDate' => $startDate ? $startDate->getTimestamp() : null, |
| 556 | 'segment' => $segment ? $segment->getOriginalString() : null, |
| 557 | ])); |
| 558 | } catch (\Throwable $ex) { |
| 559 | $this->logger->info("Failed to schedule rearchiving of past reports for $pluginName plugin."); |
| 560 | } |
| 561 | } |
| 562 | |
| 563 | /** |
| 564 | * Applies the queued archiving rearchiving entries. |
| 565 | */ |
| 566 | public function applyScheduledReArchiving() |
| 567 | { |
| 568 | $reArchiveList = new ReArchiveList($this->logger); |
| 569 | $items = $reArchiveList->getAll(); |
| 570 | |
| 571 | foreach ($items as $item) { |
| 572 | try { |
| 573 | $entry = @json_decode($item, true); |
| 574 | if (empty($entry)) { |
| 575 | continue; |
| 576 | } |
| 577 | |
| 578 | $idSites = Site::getIdSitesFromIdSitesString($entry['idSites']); |
| 579 | |
| 580 | $this->reArchiveReport( |
| 581 | $idSites, |
| 582 | $entry['pluginName'], |
| 583 | $entry['report'], |
| 584 | !empty($entry['startDate']) ? Date::factory((int) $entry['startDate']) : null, |
| 585 | !empty($entry['segment']) ? new Segment($entry['segment'], $idSites) : null |
| 586 | ); |
| 587 | } catch (\Throwable $ex) { |
| 588 | $this->logger->info("Failed to create invalidations for report re-archiving (idSites = {idSites}, pluginName = {pluginName}, report = {report}, startDate = {startDateTs}): {ex}", [ |
| 589 | 'idSites' => json_encode($entry['idSites']), |
| 590 | 'pluginName' => $entry['pluginName'], |
| 591 | 'report' => $entry['report'], |
| 592 | 'startDateTs' => $entry['startDate'], |
| 593 | 'ex' => $ex, |
| 594 | ]); |
| 595 | } finally { |
| 596 | $reArchiveList->remove([$item]); |
| 597 | } |
| 598 | } |
| 599 | } |
| 600 | |
| 601 | /** |
| 602 | * Calls removeInvalidations() without propagating exceptions. |
| 603 | * |
| 604 | * @param int|int[]|'all' $idSites |
| 605 | * @param string $pluginName |
| 606 | * @param string|null $report |
| 607 | */ |
| 608 | public function removeInvalidationsSafely($idSites, $pluginName, $report = null) |
| 609 | { |
| 610 | try { |
| 611 | $this->removeInvalidations($idSites, $pluginName, $report); |
| 612 | $this->removeInvalidationsFromDistributedList($idSites, $pluginName, $report); |
| 613 | } catch (\Throwable $ex) { |
| 614 | $logger = StaticContainer::get(LoggerInterface::class); |
| 615 | $logger->debug("Failed to remove invalidations the for $pluginName plugin."); |
| 616 | } |
| 617 | } |
| 618 | |
| 619 | public function removeInvalidationsFromDistributedList($idSites, $pluginName = null, $report = null) |
| 620 | { |
| 621 | $list = new ReArchiveList(); |
| 622 | $entries = $list->getAll(); |
| 623 | |
| 624 | if ($idSites === 'all') { |
| 625 | $idSites = $this->getAllSitesId(); |
| 626 | } |
| 627 | |
| 628 | foreach ($entries as $index => $entry) { |
| 629 | $entry = @json_decode($entry, true); |
| 630 | if (empty($entry)) { |
| 631 | unset($entries[$index]); |
| 632 | continue; |
| 633 | } |
| 634 | |
| 635 | $entryPluginName = $entry['pluginName']; |
| 636 | if (!empty($pluginName) |
| 637 | && $pluginName != $entryPluginName |
| 638 | ) { |
| 639 | continue; |
| 640 | } |
| 641 | |
| 642 | $entryReport = $entry['report']; |
| 643 | if (!empty($pluginName) |
| 644 | && !empty($report) |
| 645 | && $report != $entryReport |
| 646 | ) { |
| 647 | continue; |
| 648 | } |
| 649 | |
| 650 | $sitesInEntry = $entry['idSites']; |
| 651 | if ($sitesInEntry === 'all') { |
| 652 | $sitesInEntry = $this->getAllSitesId(); |
| 653 | } |
| 654 | |
| 655 | $diffSites = array_diff($sitesInEntry, $idSites); |
| 656 | if (empty($diffSites)) { |
| 657 | unset($entries[$index]); |
| 658 | continue; |
| 659 | } |
| 660 | |
| 661 | $entry['idSites'] = $diffSites; |
| 662 | |
| 663 | $entries[$index] = json_encode($entry); |
| 664 | } |
| 665 | |
| 666 | $list->setAll(array_values($entries)); |
| 667 | } |
| 668 | |
| 669 | /** |
| 670 | * @param int[] $idSites |
| 671 | * @param string[][][] $dates |
| 672 | * @throws \Exception |
| 673 | */ |
| 674 | private function markArchivesInvalidated($idSites, $dates, Segment $segment = null, $removeRanges = false, |
| 675 | $forceInvalidateNonexistantRanges = false, $name = null) |
| 676 | { |
| 677 | $idSites = array_map('intval', $idSites); |
| 678 | |
| 679 | $yearMonths = []; |
| 680 | |
| 681 | foreach ($dates as $tableDate => $datesForTable) { |
| 682 | $tableDateObj = Date::factory($tableDate); |
| 683 | |
| 684 | $table = ArchiveTableCreator::getNumericTable($tableDateObj); |
| 685 | $yearMonths[] = $tableDateObj->toString('Y_m'); |
| 686 | |
| 687 | $this->model->updateArchiveAsInvalidated($table, $idSites, $datesForTable, $segment, $forceInvalidateNonexistantRanges, $name); |
| 688 | |
| 689 | if ($removeRanges) { |
| 690 | $this->model->updateRangeArchiveAsInvalidated($table, $idSites, $datesForTable, $segment); |
| 691 | } |
| 692 | } |
| 693 | |
| 694 | $this->markInvalidatedArchivesForReprocessAndPurge($yearMonths); |
| 695 | } |
| 696 | |
| 697 | /** |
| 698 | * @param Date[] $dates |
| 699 | * @param InvalidationResult $invalidationInfo |
| 700 | * @return \Piwik\Date[] |
| 701 | */ |
| 702 | private function removeDatesThatHaveBeenPurged($dates, $period, InvalidationResult $invalidationInfo, $ignorePurgeLogDataDate) |
| 703 | { |
| 704 | $this->findOlderDateWithLogs($invalidationInfo); |
| 705 | |
| 706 | $result = array(); |
| 707 | foreach ($dates as $date) { |
| 708 | $periodObj = $this->makePeriod($date, $period ?: 'day'); |
| 709 | |
| 710 | // we should only delete reports for dates that are more recent than N days |
| 711 | if ($invalidationInfo->minimumDateWithLogs |
| 712 | && !$ignorePurgeLogDataDate |
| 713 | && ($periodObj->getDateEnd()->isEarlier($invalidationInfo->minimumDateWithLogs) |
| 714 | || $periodObj->getDateStart()->isEarlier($invalidationInfo->minimumDateWithLogs)) |
| 715 | ) { |
| 716 | $invalidationInfo->warningDates[] = $date; |
| 717 | continue; |
| 718 | } |
| 719 | |
| 720 | $result[] = $date; |
| 721 | $invalidationInfo->processedDates[] = $date; |
| 722 | } |
| 723 | return $result; |
| 724 | } |
| 725 | |
| 726 | private function findOlderDateWithLogs(InvalidationResult $info) |
| 727 | { |
| 728 | // If using the feature "Delete logs older than N days"... |
| 729 | $purgeDataSettings = PrivacyManager::getPurgeDataSettings(); |
| 730 | $logsDeletedWhenOlderThanDays = (int)$purgeDataSettings['delete_logs_older_than']; |
| 731 | $logsDeleteEnabled = $purgeDataSettings['delete_logs_enable']; |
| 732 | |
| 733 | if ($logsDeleteEnabled |
| 734 | && $logsDeletedWhenOlderThanDays |
| 735 | ) { |
| 736 | $info->minimumDateWithLogs = Date::factory('today')->subDay($logsDeletedWhenOlderThanDays); |
| 737 | } |
| 738 | } |
| 739 | |
| 740 | /** |
| 741 | * @param array $idSites |
| 742 | * @param array $yearMonths |
| 743 | */ |
| 744 | private function markInvalidatedArchivesForReprocessAndPurge($yearMonths) |
| 745 | { |
| 746 | $archivesToPurge = new ArchivesToPurgeDistributedList(); |
| 747 | $archivesToPurge->add($yearMonths); |
| 748 | } |
| 749 | |
| 750 | private function getYearMonth(Period $period) |
| 751 | { |
| 752 | return $period->getDateStart()->toString('Y-m-01'); |
| 753 | } |
| 754 | |
| 755 | private function getUniquePeriodId(Period $period) |
| 756 | { |
| 757 | return $period->getId() . '.' . $period->getRangeString(); |
| 758 | } |
| 759 | |
| 760 | private function makePeriod($date, $period) |
| 761 | { |
| 762 | if ($period === 'range' |
| 763 | && strpos($date, ',') === false |
| 764 | ) { |
| 765 | $date = $date . ',' . $date; |
| 766 | return new Period\Range('range', $date); |
| 767 | } else { |
| 768 | return Period\Factory::build($period, $date); |
| 769 | } |
| 770 | } |
| 771 | |
| 772 | private function getSegmentArchiving() |
| 773 | { |
| 774 | if (empty($this->segmentArchiving)) { |
| 775 | $this->segmentArchiving = new SegmentArchiving(StaticContainer::get('ini.General.process_new_segments_from')); |
| 776 | } |
| 777 | return $this->segmentArchiving; |
| 778 | } |
| 779 | |
| 780 | private function getAllSitesId() |
| 781 | { |
| 782 | if (isset($this->allIdSitesCache)) { |
| 783 | return $this->allIdSitesCache; |
| 784 | } |
| 785 | |
| 786 | $model = new \Piwik\Plugins\SitesManager\Model(); |
| 787 | $this->allIdSitesCache = $model->getSitesId(); |
| 788 | return $this->allIdSitesCache; |
| 789 | } |
| 790 | |
| 791 | private function getEarliestDateToRearchive() |
| 792 | { |
| 793 | $lastNMonthsToInvalidate = Config::getInstance()->General['rearchive_reports_in_past_last_n_months']; |
| 794 | if (empty($lastNMonthsToInvalidate)) { |
| 795 | return null; |
| 796 | } |
| 797 | |
| 798 | if (!is_numeric($lastNMonthsToInvalidate)) { |
| 799 | $lastNMonthsToInvalidate = (int)str_replace('last', '', $lastNMonthsToInvalidate); |
| 800 | if (empty($lastNMonthsToInvalidate)) { |
| 801 | return null; |
| 802 | } |
| 803 | } |
| 804 | |
| 805 | if ($lastNMonthsToInvalidate <= 0) { |
| 806 | return null; |
| 807 | } |
| 808 | |
| 809 | return Date::yesterday()->subMonth($lastNMonthsToInvalidate)->setDay(1); |
| 810 | } |
| 811 | } |
| 812 |