Performance
5 years ago
ArchiveFilter.php
5 years ago
FixedSiteIds.php
5 years ago
QueueConsumer.php
3 years ago
ReArchiveList.php
5 years ago
SegmentArchiving.php
4 years ago
SharedSiteIds.php
5 years ago
StopArchiverException.php
5 years ago
SegmentArchiving.php
259 lines
| 1 | <?php |
| 2 | /** |
| 3 | * Matomo - free/libre analytics platform |
| 4 | * |
| 5 | * @link https://matomo.org |
| 6 | * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later |
| 7 | */ |
| 8 | namespace Piwik\CronArchive; |
| 9 | |
| 10 | use Doctrine\Common\Cache\Cache; |
| 11 | use Matomo\Cache\Transient; |
| 12 | use Piwik\Access; |
| 13 | use Piwik\Archive\ArchiveInvalidator; |
| 14 | use Piwik\ArchiveProcessor\Rules; |
| 15 | use Piwik\Common; |
| 16 | use Piwik\Container\StaticContainer; |
| 17 | use Piwik\Date; |
| 18 | use Piwik\Db; |
| 19 | use Piwik\Period\Range; |
| 20 | use Piwik\Plugins\SegmentEditor\Model; |
| 21 | use Piwik\Segment; |
| 22 | use Piwik\Site; |
| 23 | use Psr\Log\LoggerInterface; |
| 24 | |
| 25 | /** |
| 26 | * Provides URLs that initiate archiving during cron archiving for segments. |
| 27 | * |
| 28 | * Handles the `[General] process_new_segments_from` INI option. |
| 29 | */ |
| 30 | class SegmentArchiving |
| 31 | { |
| 32 | const BEGINNING_OF_TIME = 'beginning_of_time'; |
| 33 | const CREATION_TIME = 'segment_creation_time'; |
| 34 | const LAST_EDIT_TIME = 'segment_last_edit_time'; |
| 35 | const DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS = 7; |
| 36 | |
| 37 | /** |
| 38 | * @var Model |
| 39 | */ |
| 40 | private $segmentEditorModel; |
| 41 | |
| 42 | /** |
| 43 | * @var Transient |
| 44 | */ |
| 45 | private $segmentListCache; |
| 46 | |
| 47 | /** |
| 48 | * @var Date |
| 49 | */ |
| 50 | private $now; |
| 51 | |
| 52 | private $processNewSegmentsFrom; |
| 53 | |
| 54 | /** |
| 55 | * @var LoggerInterface |
| 56 | */ |
| 57 | private $logger; |
| 58 | |
| 59 | /** |
| 60 | * @var int |
| 61 | */ |
| 62 | private $beginningOfTimeLastNInYears; |
| 63 | |
| 64 | /** |
| 65 | * @var bool |
| 66 | */ |
| 67 | private $forceArchiveAllSegments; |
| 68 | |
| 69 | public function __construct($beginningOfTimeLastNInYears = self::DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS, |
| 70 | Model $segmentEditorModel = null, Cache $segmentListCache = null, Date $now = null, |
| 71 | LoggerInterface $logger = null) |
| 72 | { |
| 73 | $this->processNewSegmentsFrom = StaticContainer::get('ini.General.process_new_segments_from'); |
| 74 | $this->beginningOfTimeLastNInYears = $beginningOfTimeLastNInYears; |
| 75 | $this->segmentEditorModel = $segmentEditorModel ?: new Model(); |
| 76 | $this->segmentListCache = $segmentListCache ?: new Transient(); |
| 77 | $this->now = $now ?: Date::factory('now'); |
| 78 | $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface'); |
| 79 | $this->forceArchiveAllSegments = self::getShouldForceArchiveAllSegments(); |
| 80 | } |
| 81 | |
| 82 | public function findSegmentForHash($hash, $idSite) |
| 83 | { |
| 84 | foreach ($this->getAllSegments() as $segment) { |
| 85 | if (!$this->isAutoArchivingEnabledFor($segment) |
| 86 | || !self::isSegmentForSite($segment, $idSite) |
| 87 | ) { |
| 88 | continue; |
| 89 | } |
| 90 | |
| 91 | try { |
| 92 | $segmentObj = new Segment($segment['definition'], [$idSite]); |
| 93 | } catch (\Exception $ex) { |
| 94 | $this->logger->debug("Could not process segment {$segment['definition']} for site {$idSite}. Segment should not exist for the site, but does."); |
| 95 | continue; |
| 96 | } |
| 97 | |
| 98 | if ($segmentObj->getHash() == $hash) { |
| 99 | return $segment; |
| 100 | } |
| 101 | } |
| 102 | return null; |
| 103 | } |
| 104 | |
| 105 | public function getReArchiveSegmentStartDate($segmentInfo) |
| 106 | { |
| 107 | /** |
| 108 | * @var Date $segmentCreatedTime |
| 109 | * @var Date $segmentLastEditedTime |
| 110 | */ |
| 111 | list($segmentCreatedTime, $segmentLastEditedTime) = $this->getCreatedTimeOfSegment($segmentInfo); |
| 112 | |
| 113 | if ($this->processNewSegmentsFrom == SegmentArchiving::CREATION_TIME) { |
| 114 | if (empty($segmentCreatedTime)) { |
| 115 | return null; |
| 116 | } |
| 117 | $this->logger->debug("process_new_segments_from set to segment_creation_time, oldest date to process is {time}", array('time' => $segmentCreatedTime)); |
| 118 | |
| 119 | return $segmentCreatedTime; |
| 120 | } else if ($this->processNewSegmentsFrom == SegmentArchiving::LAST_EDIT_TIME) { |
| 121 | if (empty($segmentLastEditedTime)) { |
| 122 | return null; |
| 123 | } |
| 124 | $this->logger->debug("process_new_segments_from set to segment_last_edit_time, segment last edit time is {time}", |
| 125 | array('time' => $segmentLastEditedTime)); |
| 126 | |
| 127 | return $segmentLastEditedTime; |
| 128 | } else if (preg_match("/^editLast([0-9]+)$/", $this->processNewSegmentsFrom, $matches)) { |
| 129 | if (empty($segmentLastEditedTime)) { |
| 130 | return null; |
| 131 | } |
| 132 | $lastN = $matches[1]; |
| 133 | |
| 134 | list($lastDate, $lastPeriod) = Range::getDateXPeriodsAgo($lastN, $segmentLastEditedTime, 'day'); |
| 135 | $result = Date::factory($lastDate); |
| 136 | |
| 137 | $this->logger->debug("process_new_segments_from set to editLast{N}, oldest date to process is {time}", array('N' => $lastN, 'time' => $result)); |
| 138 | |
| 139 | return $result; |
| 140 | } else if (preg_match("/^last([0-9]+)$/", $this->processNewSegmentsFrom, $matches)) { |
| 141 | if (empty($segmentCreatedTime)) { |
| 142 | return null; |
| 143 | } |
| 144 | $lastN = $matches[1]; |
| 145 | |
| 146 | list($lastDate, $lastPeriod) = Range::getDateXPeriodsAgo($lastN, $segmentCreatedTime, 'day'); |
| 147 | $result = Date::factory($lastDate); |
| 148 | |
| 149 | $this->logger->debug("process_new_segments_from set to last{N}, oldest date to process is {time}", array('N' => $lastN, 'time' => $result)); |
| 150 | |
| 151 | return $result; |
| 152 | } else { |
| 153 | $this->logger->debug("process_new_segments_from set to beginning_of_time or cannot recognize value"); |
| 154 | |
| 155 | $result = Date::factory('today')->subYear($this->beginningOfTimeLastNInYears); |
| 156 | |
| 157 | $idSite = $segmentInfo['enable_only_idsite'] ?? null; |
| 158 | if (!empty($idSite)) { |
| 159 | $siteCreationDate = Date::factory(Site::getCreationDateFor($idSite)); |
| 160 | |
| 161 | if ($result->isEarlier($siteCreationDate)) { |
| 162 | $result = $siteCreationDate; |
| 163 | } |
| 164 | } |
| 165 | |
| 166 | $earliestVisitTime = $this->getEarliestVisitTimeFor($idSite); |
| 167 | if (!empty($earliestVisitTime) |
| 168 | && $result->isEarlier($earliestVisitTime) |
| 169 | ) { |
| 170 | $result = $earliestVisitTime; |
| 171 | } |
| 172 | |
| 173 | return $result; |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | private function getCreatedTimeOfSegment($storedSegment) |
| 178 | { |
| 179 | // check for an earlier ts_created timestamp |
| 180 | $createdTime = empty($storedSegment['ts_created']) ? null : Date::factory($storedSegment['ts_created']); |
| 181 | |
| 182 | // if there is no ts_last_edit timestamp, initialize it to ts_created |
| 183 | if (empty($storedSegment['ts_last_edit'])) { |
| 184 | $storedSegment['ts_last_edit'] = empty($storedSegment['ts_created']) ? null : $storedSegment['ts_created']; |
| 185 | } |
| 186 | |
| 187 | // check for a later ts_last_edit timestamp |
| 188 | $lastEditTime = empty($storedSegment['ts_last_edit']) ? null : Date::factory($storedSegment['ts_last_edit']); |
| 189 | |
| 190 | return array($createdTime, $lastEditTime); |
| 191 | } |
| 192 | |
| 193 | private function getEarliestVisitTimeFor($idSite) |
| 194 | { |
| 195 | $earliestIdVisit = Db::fetchOne('SELECT idvisit FROM ' . Common::prefixTable('log_visit') |
| 196 | . ' WHERE idsite = ? ORDER BY visit_last_action_time ASC LIMIT 1', [$idSite]); |
| 197 | |
| 198 | $earliestStartTime = Db::fetchOne('SELECT visit_first_action_time FROM ' . Common::prefixTable('log_visit') . ' WHERE idvisit = ?', [ |
| 199 | $earliestIdVisit, |
| 200 | ]); |
| 201 | |
| 202 | if (empty($earliestStartTime)) { |
| 203 | return null; |
| 204 | } |
| 205 | |
| 206 | return Date::factory($earliestStartTime); |
| 207 | } |
| 208 | |
| 209 | public function getAllSegments() |
| 210 | { |
| 211 | if (!$this->segmentListCache->contains('all')) { |
| 212 | $segments = $this->segmentEditorModel->getAllSegmentsAndIgnoreVisibility(); |
| 213 | |
| 214 | $this->segmentListCache->save('all', $segments); |
| 215 | } |
| 216 | |
| 217 | return $this->segmentListCache->fetch('all'); |
| 218 | } |
| 219 | |
| 220 | public function getAllSegmentsToArchive($idSite) |
| 221 | { |
| 222 | return Rules::getSegmentsToProcess([$idSite]); |
| 223 | } |
| 224 | |
| 225 | public static function isSegmentForSite($segment, $idSite) |
| 226 | { |
| 227 | return $segment['enable_only_idsite'] == 0 |
| 228 | || $segment['enable_only_idsite'] == $idSite; |
| 229 | } |
| 230 | |
| 231 | public function isAutoArchivingEnabledFor($storedSegment) |
| 232 | { |
| 233 | return $this->forceArchiveAllSegments || !empty($storedSegment['auto_archive']); |
| 234 | } |
| 235 | |
| 236 | public static function getShouldForceArchiveAllSegments() |
| 237 | { |
| 238 | return !Rules::isBrowserTriggerEnabled() && !Rules::isBrowserArchivingAvailableForSegments(); |
| 239 | } |
| 240 | |
| 241 | public function reArchiveSegment($segmentInfo) |
| 242 | { |
| 243 | if (empty($segmentInfo['definition'])) { // sanity check |
| 244 | return; |
| 245 | } |
| 246 | |
| 247 | $definition = $segmentInfo['definition']; |
| 248 | $idSite = !empty($segmentInfo['enable_only_idsite']) ? $segmentInfo['enable_only_idsite'] : 'all'; |
| 249 | |
| 250 | $idSites = Access::doAsSuperUser(function () use ($idSite) { |
| 251 | return Site::getIdSitesFromIdSitesString($idSite); |
| 252 | }); |
| 253 | $startDate = $this->getReArchiveSegmentStartDate($segmentInfo); |
| 254 | |
| 255 | $invalidator = StaticContainer::get(ArchiveInvalidator::class); |
| 256 | $invalidator->scheduleReArchiving($idSites, null, null, $startDate, new Segment($definition, $idSites)); |
| 257 | } |
| 258 | } |
| 259 |