Performance
1 year ago
ArchiveFilter.php
1 year ago
FixedSiteIds.php
1 year ago
QueueConsumer.php
1 year ago
ReArchiveList.php
1 year ago
SegmentArchiving.php
1 year ago
SharedSiteIds.php
1 year ago
StopArchiverException.php
2 years ago
SegmentArchiving.php
211 lines
| 1 | <?php |
| 2 | |
| 3 | /** |
| 4 | * Matomo - free/libre analytics platform |
| 5 | * |
| 6 | * @link https://matomo.org |
| 7 | * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later |
| 8 | */ |
| 9 | namespace Piwik\CronArchive; |
| 10 | |
| 11 | use Doctrine\Common\Cache\Cache; |
| 12 | use Matomo\Cache\Transient; |
| 13 | use Piwik\Access; |
| 14 | use Piwik\Archive\ArchiveInvalidator; |
| 15 | use Piwik\ArchiveProcessor\Rules; |
| 16 | use Piwik\Common; |
| 17 | use Piwik\Container\StaticContainer; |
| 18 | use Piwik\Date; |
| 19 | use Piwik\Db; |
| 20 | use Piwik\Period\Range; |
| 21 | use Piwik\Plugins\SegmentEditor\Model; |
| 22 | use Piwik\Segment; |
| 23 | use Piwik\Site; |
| 24 | use Piwik\Log\LoggerInterface; |
| 25 | /** |
| 26 | * Provides URLs that initiate archiving during cron archiving for segments. |
| 27 | * |
| 28 | * Handles the `[General] process_new_segments_from` INI option. |
| 29 | */ |
| 30 | class SegmentArchiving |
| 31 | { |
| 32 | public const BEGINNING_OF_TIME = 'beginning_of_time'; |
| 33 | public const CREATION_TIME = 'segment_creation_time'; |
| 34 | public const LAST_EDIT_TIME = 'segment_last_edit_time'; |
| 35 | public const DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS = 7; |
| 36 | /** |
| 37 | * @var Model |
| 38 | */ |
| 39 | private $segmentEditorModel; |
| 40 | /** |
| 41 | * @var Transient |
| 42 | */ |
| 43 | private $segmentListCache; |
| 44 | /** |
| 45 | * @var Date |
| 46 | */ |
| 47 | private $now; |
| 48 | private $processNewSegmentsFrom; |
| 49 | /** |
| 50 | * @var LoggerInterface |
| 51 | */ |
| 52 | private $logger; |
| 53 | /** |
| 54 | * @var int |
| 55 | */ |
| 56 | private $beginningOfTimeLastNInYears; |
| 57 | /** |
| 58 | * @var bool |
| 59 | */ |
| 60 | private $forceArchiveAllSegments; |
| 61 | public function __construct($beginningOfTimeLastNInYears = self::DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS, ?Model $segmentEditorModel = null, ?Cache $segmentListCache = null, ?Date $now = null, ?LoggerInterface $logger = null) |
| 62 | { |
| 63 | $this->processNewSegmentsFrom = StaticContainer::get('ini.General.process_new_segments_from'); |
| 64 | $this->beginningOfTimeLastNInYears = $beginningOfTimeLastNInYears; |
| 65 | $this->segmentEditorModel = $segmentEditorModel ?: new Model(); |
| 66 | $this->segmentListCache = $segmentListCache ?: new Transient(); |
| 67 | $this->now = $now ?: Date::factory('now'); |
| 68 | $this->logger = $logger ?: StaticContainer::get(LoggerInterface::class); |
| 69 | $this->forceArchiveAllSegments = self::getShouldForceArchiveAllSegments(); |
| 70 | } |
| 71 | public function findSegmentForHash($hash, $idSite) |
| 72 | { |
| 73 | foreach ($this->getAllSegments() as $segment) { |
| 74 | if (!$this->isAutoArchivingEnabledFor($segment) || !self::isSegmentForSite($segment, $idSite)) { |
| 75 | continue; |
| 76 | } |
| 77 | try { |
| 78 | $segmentObj = new Segment($segment['definition'], [$idSite]); |
| 79 | } catch (\Exception $ex) { |
| 80 | $this->logger->debug("Could not process segment {$segment['definition']} for site {$idSite}. Segment should not exist for the site, but does."); |
| 81 | continue; |
| 82 | } |
| 83 | if ($segmentObj->getHash() == $hash) { |
| 84 | return $segment; |
| 85 | } |
| 86 | } |
| 87 | return null; |
| 88 | } |
| 89 | public function getReArchiveSegmentStartDate($segmentInfo) |
| 90 | { |
| 91 | /** |
| 92 | * @var Date $segmentCreatedTime |
| 93 | * @var Date $segmentLastEditedTime |
| 94 | */ |
| 95 | list($segmentCreatedTime, $segmentLastEditedTime) = $this->getCreatedTimeOfSegment($segmentInfo); |
| 96 | if ($this->processNewSegmentsFrom == \Piwik\CronArchive\SegmentArchiving::CREATION_TIME) { |
| 97 | if (empty($segmentCreatedTime)) { |
| 98 | return null; |
| 99 | } |
| 100 | $this->logger->debug("process_new_segments_from set to segment_creation_time, oldest date to process is {time}", array('time' => $segmentCreatedTime)); |
| 101 | return $segmentCreatedTime; |
| 102 | } elseif ($this->processNewSegmentsFrom == \Piwik\CronArchive\SegmentArchiving::LAST_EDIT_TIME) { |
| 103 | if (empty($segmentLastEditedTime)) { |
| 104 | return null; |
| 105 | } |
| 106 | $this->logger->debug("process_new_segments_from set to segment_last_edit_time, segment last edit time is {time}", array('time' => $segmentLastEditedTime)); |
| 107 | return $segmentLastEditedTime; |
| 108 | } elseif (preg_match("/^editLast([0-9]+)\$/", $this->processNewSegmentsFrom, $matches)) { |
| 109 | if (empty($segmentLastEditedTime)) { |
| 110 | return null; |
| 111 | } |
| 112 | $lastN = $matches[1]; |
| 113 | list($lastDate, $lastPeriod) = Range::getDateXPeriodsAgo($lastN, $segmentLastEditedTime, 'day'); |
| 114 | $result = Date::factory($lastDate); |
| 115 | $this->logger->debug("process_new_segments_from set to editLast{N}, oldest date to process is {time}", array('N' => $lastN, 'time' => $result)); |
| 116 | return $result; |
| 117 | } elseif (preg_match("/^last([0-9]+)\$/", $this->processNewSegmentsFrom, $matches)) { |
| 118 | if (empty($segmentCreatedTime)) { |
| 119 | return null; |
| 120 | } |
| 121 | $lastN = $matches[1]; |
| 122 | list($lastDate, $lastPeriod) = Range::getDateXPeriodsAgo($lastN, $segmentCreatedTime, 'day'); |
| 123 | $result = Date::factory($lastDate); |
| 124 | $this->logger->debug("process_new_segments_from set to last{N}, oldest date to process is {time}", array('N' => $lastN, 'time' => $result)); |
| 125 | return $result; |
| 126 | } else { |
| 127 | $this->logger->debug("process_new_segments_from set to beginning_of_time or cannot recognize value"); |
| 128 | $result = Date::factory('today')->subYear($this->beginningOfTimeLastNInYears); |
| 129 | $idSite = $segmentInfo['enable_only_idsite'] ?? null; |
| 130 | if (!empty($idSite)) { |
| 131 | $siteCreationDate = Date::factory(Site::getCreationDateFor($idSite)); |
| 132 | if ($result->isEarlier($siteCreationDate)) { |
| 133 | $result = $siteCreationDate; |
| 134 | } |
| 135 | } |
| 136 | $earliestVisitTime = $this->getEarliestVisitTimeFor($idSite); |
| 137 | if (!empty($earliestVisitTime) && $result->isEarlier($earliestVisitTime)) { |
| 138 | $result = $earliestVisitTime; |
| 139 | } |
| 140 | return $result; |
| 141 | } |
| 142 | } |
| 143 | /** |
| 144 | * Retrieve the created and last edited time as date objects from the supplied segment array |
| 145 | * |
| 146 | * @param array $storedSegment |
| 147 | * |
| 148 | * @return array |
| 149 | */ |
| 150 | private function getCreatedTimeOfSegment(array $storedSegment) : array |
| 151 | { |
| 152 | // check for an earlier ts_created timestamp |
| 153 | $createdTime = empty($storedSegment['ts_created']) ? null : Date::factory($storedSegment['ts_created']); |
| 154 | // if there is no ts_last_edit timestamp, initialize it to ts_created |
| 155 | if (empty($storedSegment['ts_last_edit'])) { |
| 156 | $storedSegment['ts_last_edit'] = empty($storedSegment['ts_created']) ? null : $storedSegment['ts_created']; |
| 157 | } |
| 158 | // check for a later ts_last_edit timestamp |
| 159 | $lastEditTime = empty($storedSegment['ts_last_edit']) || $storedSegment['ts_last_edit'] === '0000-00-00 00:00:00' ? null : Date::factory($storedSegment['ts_last_edit']); |
| 160 | return [$createdTime, $lastEditTime]; |
| 161 | } |
| 162 | private function getEarliestVisitTimeFor($idSite) |
| 163 | { |
| 164 | $earliestIdVisit = Db::fetchOne('SELECT idvisit FROM ' . Common::prefixTable('log_visit') . ' WHERE idsite = ? ORDER BY visit_last_action_time ASC LIMIT 1', [$idSite]); |
| 165 | $earliestStartTime = Db::fetchOne('SELECT visit_first_action_time FROM ' . Common::prefixTable('log_visit') . ' WHERE idvisit = ?', [$earliestIdVisit]); |
| 166 | if (empty($earliestStartTime)) { |
| 167 | return null; |
| 168 | } |
| 169 | return Date::factory($earliestStartTime); |
| 170 | } |
| 171 | public function getAllSegments() |
| 172 | { |
| 173 | if (!$this->segmentListCache->contains('all')) { |
| 174 | $segments = $this->segmentEditorModel->getAllSegmentsAndIgnoreVisibility(); |
| 175 | $this->segmentListCache->save('all', $segments); |
| 176 | } |
| 177 | return $this->segmentListCache->fetch('all'); |
| 178 | } |
| 179 | public function getAllSegmentsToArchive($idSite) |
| 180 | { |
| 181 | return Rules::getSegmentsToProcess([$idSite]); |
| 182 | } |
| 183 | public static function isSegmentForSite($segment, $idSite) |
| 184 | { |
| 185 | return $segment['enable_only_idsite'] == 0 || $segment['enable_only_idsite'] == $idSite; |
| 186 | } |
| 187 | public function isAutoArchivingEnabledFor($storedSegment) |
| 188 | { |
| 189 | return $this->forceArchiveAllSegments || !empty($storedSegment['auto_archive']); |
| 190 | } |
| 191 | public static function getShouldForceArchiveAllSegments() |
| 192 | { |
| 193 | return !Rules::isBrowserTriggerEnabled() && !Rules::isBrowserArchivingAvailableForSegments(); |
| 194 | } |
| 195 | public function reArchiveSegment($segmentInfo) |
| 196 | { |
| 197 | if (empty($segmentInfo['definition'])) { |
| 198 | // sanity check |
| 199 | return; |
| 200 | } |
| 201 | $definition = $segmentInfo['definition']; |
| 202 | $idSite = !empty($segmentInfo['enable_only_idsite']) ? $segmentInfo['enable_only_idsite'] : 'all'; |
| 203 | $idSites = Access::doAsSuperUser(function () use($idSite) { |
| 204 | return Site::getIdSitesFromIdSitesString($idSite); |
| 205 | }); |
| 206 | $startDate = $this->getReArchiveSegmentStartDate($segmentInfo); |
| 207 | $invalidator = StaticContainer::get(ArchiveInvalidator::class); |
| 208 | $invalidator->scheduleReArchiving($idSites, null, null, $startDate, new Segment($definition, $idSites)); |
| 209 | } |
| 210 | } |
| 211 |