Performance
1 year ago
ArchiveFilter.php
3 months ago
FixedSiteIds.php
3 months ago
QueueConsumer.php
7 months ago
ReArchiveList.php
1 year ago
SegmentArchiving.php
7 months ago
SharedSiteIds.php
3 months ago
StopArchiverException.php
2 years ago
SegmentArchiving.php
197 lines
| 1 | <?php |
| 2 | |
| 3 | /** |
| 4 | * Matomo - free/libre analytics platform |
| 5 | * |
| 6 | * @link https://matomo.org |
| 7 | * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later |
| 8 | */ |
| 9 | namespace Piwik\CronArchive; |
| 10 | |
| 11 | use Doctrine\Common\Cache\Cache; |
| 12 | use Matomo\Cache\Transient; |
| 13 | use Piwik\Access; |
| 14 | use Piwik\Archive\ArchiveInvalidator; |
| 15 | use Piwik\ArchiveProcessor\Rules; |
| 16 | use Piwik\Cache as PiwikCache; |
| 17 | use Piwik\Container\StaticContainer; |
| 18 | use Piwik\Date; |
| 19 | use Piwik\Period\Range; |
| 20 | use Piwik\Plugins\SegmentEditor\Model; |
| 21 | use Piwik\Segment; |
| 22 | use Piwik\Site; |
| 23 | use Piwik\Log\LoggerInterface; |
| 24 | /** |
| 25 | * Provides URLs that initiate archiving during cron archiving for segments. |
| 26 | * |
| 27 | * Handles the `[General] process_new_segments_from` INI option. |
| 28 | */ |
| 29 | class SegmentArchiving |
| 30 | { |
| 31 | public const BEGINNING_OF_TIME = 'beginning_of_time'; |
| 32 | public const CREATION_TIME = 'segment_creation_time'; |
| 33 | public const LAST_EDIT_TIME = 'segment_last_edit_time'; |
| 34 | public const DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS = 7; |
| 35 | /** |
| 36 | * @var Model |
| 37 | */ |
| 38 | private $segmentEditorModel; |
| 39 | /** |
| 40 | * @var Transient |
| 41 | */ |
| 42 | private $segmentListCache; |
| 43 | /** |
| 44 | * @var Date |
| 45 | */ |
| 46 | private $now; |
| 47 | private $processNewSegmentsFrom; |
| 48 | /** |
| 49 | * @var LoggerInterface |
| 50 | */ |
| 51 | private $logger; |
| 52 | /** |
| 53 | * @var int |
| 54 | */ |
| 55 | private $beginningOfTimeLastNInYears; |
| 56 | /** |
| 57 | * @var bool |
| 58 | */ |
| 59 | private $forceArchiveAllSegments; |
| 60 | public function __construct($beginningOfTimeLastNInYears = self::DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS, ?Model $segmentEditorModel = null, ?Cache $segmentListCache = null, ?Date $now = null, ?LoggerInterface $logger = null) |
| 61 | { |
| 62 | $this->processNewSegmentsFrom = StaticContainer::get('ini.General.process_new_segments_from'); |
| 63 | $this->beginningOfTimeLastNInYears = $beginningOfTimeLastNInYears; |
| 64 | $this->segmentEditorModel = $segmentEditorModel ?: new Model(); |
| 65 | $this->segmentListCache = $segmentListCache ?: PiwikCache::getTransientCache(); |
| 66 | $this->now = $now ?: Date::factory('now'); |
| 67 | $this->logger = $logger ?: StaticContainer::get(LoggerInterface::class); |
| 68 | $this->forceArchiveAllSegments = self::getShouldForceArchiveAllSegments(); |
| 69 | } |
| 70 | public function findSegmentForHash($hash, $idSite) |
| 71 | { |
| 72 | foreach ($this->getAllSegments() as $segment) { |
| 73 | if (!$this->isAutoArchivingEnabledFor($segment) || !self::isSegmentForSite($segment, $idSite)) { |
| 74 | continue; |
| 75 | } |
| 76 | if (!Segment::isAvailable($segment['definition'], [$idSite])) { |
| 77 | $this->logger->debug("Could not process segment {$segment['definition']} for site {$idSite}. Segment should not exist for the site, but does."); |
| 78 | continue; |
| 79 | } |
| 80 | $segmentObj = new Segment($segment['definition'], [$idSite]); |
| 81 | if ($segmentObj->getHash() == $hash) { |
| 82 | return $segment; |
| 83 | } |
| 84 | } |
| 85 | return null; |
| 86 | } |
| 87 | public function getReArchiveSegmentStartDate($segmentInfo) |
| 88 | { |
| 89 | [$segmentCreatedTime, $segmentLastEditedTime] = $this->getCreatedTimeOfSegment($segmentInfo); |
| 90 | if ($this->processNewSegmentsFrom == \Piwik\CronArchive\SegmentArchiving::CREATION_TIME) { |
| 91 | if (empty($segmentCreatedTime)) { |
| 92 | return null; |
| 93 | } |
| 94 | $this->logger->debug("process_new_segments_from set to segment_creation_time, oldest date to process is {time}", array('time' => $segmentCreatedTime)); |
| 95 | return $segmentCreatedTime; |
| 96 | } elseif ($this->processNewSegmentsFrom == \Piwik\CronArchive\SegmentArchiving::LAST_EDIT_TIME) { |
| 97 | if (empty($segmentLastEditedTime)) { |
| 98 | return null; |
| 99 | } |
| 100 | $this->logger->debug("process_new_segments_from set to segment_last_edit_time, segment last edit time is {time}", array('time' => $segmentLastEditedTime)); |
| 101 | return $segmentLastEditedTime; |
| 102 | } elseif (preg_match("/^editLast([0-9]+)\$/", $this->processNewSegmentsFrom, $matches)) { |
| 103 | if (empty($segmentLastEditedTime)) { |
| 104 | return null; |
| 105 | } |
| 106 | $lastN = $matches[1]; |
| 107 | [$lastDate, $lastPeriod] = Range::getDateXPeriodsAgo($lastN, $segmentLastEditedTime, 'day'); |
| 108 | $result = Date::factory($lastDate); |
| 109 | $this->logger->debug("process_new_segments_from set to editLast{N}, oldest date to process is {time}", array('N' => $lastN, 'time' => $result)); |
| 110 | return $result; |
| 111 | } elseif (preg_match("/^last([0-9]+)\$/", $this->processNewSegmentsFrom, $matches)) { |
| 112 | if (empty($segmentCreatedTime)) { |
| 113 | return null; |
| 114 | } |
| 115 | $lastN = $matches[1]; |
| 116 | [$lastDate, $lastPeriod] = Range::getDateXPeriodsAgo($lastN, $segmentCreatedTime, 'day'); |
| 117 | $result = Date::factory($lastDate); |
| 118 | $this->logger->debug("process_new_segments_from set to last{N}, oldest date to process is {time}", array('N' => $lastN, 'time' => $result)); |
| 119 | return $result; |
| 120 | } else { |
| 121 | $this->logger->debug("process_new_segments_from set to beginning_of_time or cannot recognize value"); |
| 122 | $result = Date::factory('today')->subYear($this->beginningOfTimeLastNInYears); |
| 123 | $idSite = $segmentInfo['enable_only_idsite'] ?? null; |
| 124 | if (!empty($idSite)) { |
| 125 | $siteCreationDate = Date::factory(Site::getCreationDateFor($idSite)); |
| 126 | if ($result->isEarlier($siteCreationDate)) { |
| 127 | $result = $siteCreationDate; |
| 128 | } |
| 129 | } |
| 130 | return $result; |
| 131 | } |
| 132 | } |
| 133 | /** |
| 134 | * Retrieve the created and last edited time as date objects from the supplied segment array |
| 135 | * |
| 136 | * @param array $storedSegment |
| 137 | * |
| 138 | * @return array<Date|null> |
| 139 | */ |
| 140 | private function getCreatedTimeOfSegment(array $storedSegment) : array |
| 141 | { |
| 142 | // check for an earlier ts_created timestamp |
| 143 | $createdTime = empty($storedSegment['ts_created']) ? null : Date::factory($storedSegment['ts_created']); |
| 144 | // if there is no ts_last_edit timestamp, initialize it to ts_created |
| 145 | if (empty($storedSegment['ts_last_edit'])) { |
| 146 | $storedSegment['ts_last_edit'] = empty($storedSegment['ts_created']) ? null : $storedSegment['ts_created']; |
| 147 | } |
| 148 | // check for a later ts_last_edit timestamp |
| 149 | $lastEditTime = empty($storedSegment['ts_last_edit']) || $storedSegment['ts_last_edit'] === '0000-00-00 00:00:00' ? null : Date::factory($storedSegment['ts_last_edit']); |
| 150 | return [$createdTime, $lastEditTime]; |
| 151 | } |
| 152 | public function getAllSegments() |
| 153 | { |
| 154 | $cacheKey = 'SegmentArchiving_AllSegments'; |
| 155 | if (!$this->segmentListCache->contains($cacheKey)) { |
| 156 | $segments = $this->segmentEditorModel->getAllSegmentsAndIgnoreVisibility(); |
| 157 | $this->segmentListCache->save($cacheKey, $segments); |
| 158 | } |
| 159 | return $this->segmentListCache->fetch($cacheKey); |
| 160 | } |
| 161 | public function getAllSegmentsToArchive($idSite) |
| 162 | { |
| 163 | return Rules::getSegmentsToProcess([$idSite]); |
| 164 | } |
| 165 | public static function isSegmentForSite($segment, $idSite) |
| 166 | { |
| 167 | return $segment['enable_only_idsite'] == 0 || $segment['enable_only_idsite'] == $idSite; |
| 168 | } |
| 169 | public function isAutoArchivingEnabledFor($storedSegment) |
| 170 | { |
| 171 | return $this->forceArchiveAllSegments || !empty($storedSegment['auto_archive']); |
| 172 | } |
| 173 | public static function getShouldForceArchiveAllSegments() |
| 174 | { |
| 175 | return !Rules::isBrowserTriggerEnabled() && !Rules::isBrowserArchivingAvailableForSegments(); |
| 176 | } |
| 177 | public function reArchiveSegment($segmentInfo) |
| 178 | { |
| 179 | if (empty($segmentInfo['definition'])) { |
| 180 | // sanity check |
| 181 | return; |
| 182 | } |
| 183 | $definition = $segmentInfo['definition']; |
| 184 | $idSite = !empty($segmentInfo['enable_only_idsite']) ? $segmentInfo['enable_only_idsite'] : 'all'; |
| 185 | /* |
| 186 | * Done as super user, to ensure we are able to receive all site ids, even if the current user |
| 187 | * does not have access to all of them. |
| 188 | */ |
| 189 | $idSites = Access::doAsSuperUser(function () use($idSite) { |
| 190 | return Site::getIdSitesFromIdSitesString($idSite); |
| 191 | }); |
| 192 | $startDate = $this->getReArchiveSegmentStartDate($segmentInfo); |
| 193 | $invalidator = StaticContainer::get(ArchiveInvalidator::class); |
| 194 | $invalidator->scheduleReArchiving($idSites, null, null, $startDate, new Segment($definition, $idSites)); |
| 195 | } |
| 196 | } |
| 197 |