PluginProbe ʕ •ᴥ•ʔ
Matomo Analytics – Powerful, Privacy-First Insights for WordPress / 4.3.0
Matomo Analytics – Powerful, Privacy-First Insights for WordPress v4.3.0
5.11.1 5.11.0 5.10.2 5.10.1 trunk 1.0.2 1.0.3 1.0.4 1.0.5 1.0.6 1.1.0 1.1.1 1.1.2 1.1.3 1.2.0 1.3.0 1.3.1 1.3.2 4.0.0 4.0.1 4.0.2 4.0.3 4.0.4 4.1.0 4.1.1 4.1.2 4.1.3 4.10.0 4.11.0 4.12.0 4.13.0 4.13.2 4.13.3 4.13.4 4.13.5 4.14.0 4.14.1 4.14.2 4.15.0 4.15.1 4.15.2 4.15.3 4.2.0 4.3.0 4.3.1 4.4.1 4.4.2 4.5.0 4.6.0 5.0.1 5.0.2 5.0.3 5.0.4 5.0.5 5.0.6 5.0.7 5.0.8 5.1.0 5.1.1 5.1.2 5.1.3 5.1.4 5.1.5 5.1.6 5.1.7 5.10.0 5.2.0 5.2.1 5.2.2 5.3.0 5.3.1 5.3.2 5.3.3 5.6.0 5.6.1 5.7.0 5.7.1 5.8.0 5.8.1 5.8.2
matomo / app / core / ArchiveProcessor / Loader.php
matomo / app / core / ArchiveProcessor Last commit date
ArchivingStatus.php 5 years ago Loader.php 5 years ago Parameters.php 5 years ago PluginsArchiver.php 5 years ago PluginsArchiverException.php 5 years ago Rules.php 5 years ago
Loader.php
527 lines
1 <?php
2 /**
3 * Matomo - free/libre analytics platform
4 *
5 * @link https://matomo.org
6 * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7 *
8 */
9 namespace Piwik\ArchiveProcessor;
10
11 use Piwik\Archive\ArchiveInvalidator;
12 use Piwik\Cache;
13 use Piwik\Common;
14 use Piwik\Config;
15 use Piwik\Container\StaticContainer;
16 use Piwik\Context;
17 use Piwik\DataAccess\ArchiveSelector;
18 use Piwik\DataAccess\ArchiveTableCreator;
19 use Piwik\DataAccess\ArchiveWriter;
20 use Piwik\DataAccess\Model;
21 use Piwik\DataAccess\RawLogDao;
22 use Piwik\Date;
23 use Piwik\Db;
24 use Piwik\Option;
25 use Piwik\Period;
26 use Piwik\Piwik;
27 use Piwik\SettingsServer;
28 use Piwik\Site;
29 use Psr\Log\LoggerInterface;
30 use Piwik\CronArchive\SegmentArchiving;
31
32 /**
33 * This class uses PluginsArchiver class to trigger data aggregation and create archives.
34 */
35 class Loader
36 {
37 private static $archivingDepth = 0;
38
39 /**
40 * @var Parameters
41 */
42 protected $params;
43
44 /**
45 * @var ArchiveInvalidator
46 */
47 private $invalidator;
48
49 /**
50 * @var \Matomo\Cache\Cache
51 */
52 private $cache;
53
54 /**
55 * @var LoggerInterface
56 */
57 private $logger;
58
59 /**
60 * @var RawLogDao
61 */
62 private $rawLogDao;
63
64 /**
65 * @var Model
66 */
67 private $dataAccessModel;
68
69 public function __construct(Parameters $params, $invalidateBeforeArchiving = false)
70 {
71 $this->params = $params;
72 $this->invalidateBeforeArchiving = $invalidateBeforeArchiving;
73 $this->invalidator = StaticContainer::get(ArchiveInvalidator::class);
74 $this->cache = Cache::getTransientCache();
75 $this->logger = StaticContainer::get(LoggerInterface::class);
76 $this->rawLogDao = new RawLogDao();
77 $this->dataAccessModel = new Model();
78 }
79
80 /**
81 * @return bool
82 */
83 protected function isThereSomeVisits($visits)
84 {
85 return $visits > 0;
86 }
87
88 /**
89 * @return bool
90 */
91 protected function mustProcessVisitCount($visits)
92 {
93 return $visits === false;
94 }
95
96 public function prepareArchive($pluginName)
97 {
98 return Context::changeIdSite($this->params->getSite()->getId(), function () use ($pluginName) {
99 try {
100 ++self::$archivingDepth;
101 return $this->prepareArchiveImpl($pluginName);
102 } finally {
103 --self::$archivingDepth;
104 }
105 });
106 }
107
108 private function prepareArchiveImpl($pluginName)
109 {
110 $this->params->setRequestedPlugin($pluginName);
111
112 if (SettingsServer::isArchivePhpTriggered()) {
113 $requestedReport = Common::getRequestVar('requestedReport', '', 'string');
114 if (!empty($requestedReport)) {
115 $this->params->setArchiveOnlyReport($requestedReport);
116 }
117 }
118
119 // invalidate existing archives before we start archiving in case data was tracked in the past. if the archive is
120 // made invalid, we will correctly re-archive below.
121 if ($this->invalidateBeforeArchiving
122 && Rules::isBrowserTriggerEnabled()
123 ) {
124 $this->invalidatedReportsIfNeeded();
125 }
126
127 // NOTE: $idArchives will contain the latest DONE_OK/DONE_INVALIDATED archive as well as any partial archives
128 // with a ts_archived >= the DONE_OK/DONE_INVALIDATED date.
129 list($idArchives, $visits, $visitsConverted, $isAnyArchiveExists, $tsArchived, $value) = $this->loadExistingArchiveIdFromDb();
130 if (!empty($idArchives)
131 && !Rules::isActuallyForceArchivingSinglePlugin()
132 && !$this->shouldForceInvalidatedArchive($value, $tsArchived)
133 ) {
134 // we have a usable idarchive (it's not invalidated and it's new enough), and we are not archiving
135 // a single report
136 return [$idArchives, $visits];
137 }
138
139 // NOTE: this optimization helps when archiving large periods. eg, if archiving a year w/ a segment where
140 // there are not visits in the entire year, we don't have to go through and do anything. but, w/o this
141 // code, we will end up launching archiving for each month, week and day, even though we don't have to.
142 //
143 // we don't create an archive in this case, because the archive may be in progress in some way, so a 0
144 // visits archive can be inaccurate in the long run.
145 if ($this->canSkipThisArchive()) {
146 if (!empty($idArchives)) {
147 return [$idArchives, $visits];
148 } else {
149 return [false, 0];
150 }
151 }
152
153 if (SettingsServer::isArchivePhpTriggered()) {
154 $this->logger->info("initiating archiving via core:archive for " . $this->params);
155 }
156
157 /** @var ArchivingStatus $archivingStatus */
158 $archivingStatus = StaticContainer::get(ArchivingStatus::class);
159 $locked = $archivingStatus->archiveStarted($this->params);
160
161 try {
162 list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted);
163 list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted);
164 } finally {
165 if ($locked) {
166 $archivingStatus->archiveFinished();
167 }
168 }
169
170 if ($this->isThereSomeVisits($visits) || PluginsArchiver::doesAnyPluginArchiveWithoutVisits()) {
171 return [[$idArchive], $visits];
172 }
173
174 return [false, false];
175 }
176
177 /**
178 * Prepares the core metrics if needed.
179 *
180 * @param $visits
181 * @return array
182 */
183 protected function prepareCoreMetricsArchive($visits, $visitsConverted)
184 {
185 $createSeparateArchiveForCoreMetrics = $this->mustProcessVisitCount($visits)
186 && !$this->doesRequestedPluginIncludeVisitsSummary();
187
188 if ($createSeparateArchiveForCoreMetrics) {
189 $requestedPlugin = $this->params->getRequestedPlugin();
190 $requestedReport = $this->params->getArchiveOnlyReport();
191 $isPartialArchive = $this->params->isPartialArchive();
192
193 $this->params->setRequestedPlugin('VisitsSummary');
194 $this->params->setArchiveOnlyReport(null);
195 $this->params->setIsPartialArchive(false);
196
197 $metrics = Context::executeWithQueryParameters(['requestedReport' => ''], function () {
198 $pluginsArchiver = new PluginsArchiver($this->params);
199 $metrics = $pluginsArchiver->callAggregateCoreMetrics();
200 $pluginsArchiver->finalizeArchive();
201 return $metrics;
202 });
203
204 $this->params->setRequestedPlugin($requestedPlugin);
205 $this->params->setArchiveOnlyReport($requestedReport);
206 $this->params->setIsPartialArchive($isPartialArchive);
207
208 $visits = $metrics['nb_visits'];
209 $visitsConverted = $metrics['nb_visits_converted'];
210 }
211
212 return array($visits, $visitsConverted);
213 }
214
215 protected function prepareAllPluginsArchive($visits, $visitsConverted)
216 {
217 $pluginsArchiver = new PluginsArchiver($this->params);
218
219 if ($this->mustProcessVisitCount($visits)
220 || $this->doesRequestedPluginIncludeVisitsSummary()
221 ) {
222 $metrics = $pluginsArchiver->callAggregateCoreMetrics();
223 $visits = $metrics['nb_visits'];
224 $visitsConverted = $metrics['nb_visits_converted'];
225 }
226
227 $forceArchivingWithoutVisits = !$this->isThereSomeVisits($visits) && $this->shouldArchiveForSiteEvenWhenNoVisits();
228 $pluginsArchiver->callAggregateAllPlugins($visits, $visitsConverted, $forceArchivingWithoutVisits);
229
230 $idArchive = $pluginsArchiver->finalizeArchive();
231
232 return array($idArchive, $visits);
233 }
234
235 protected function doesRequestedPluginIncludeVisitsSummary()
236 {
237 $processAllReportsIncludingVisitsSummary =
238 Rules::shouldProcessReportsAllPlugins(array($this->params->getSite()->getId()), $this->params->getSegment(), $this->params->getPeriod()->getLabel());
239 $doesRequestedPluginIncludeVisitsSummary = $processAllReportsIncludingVisitsSummary
240 || $this->params->getRequestedPlugin() == 'VisitsSummary';
241 return $doesRequestedPluginIncludeVisitsSummary;
242 }
243
244 protected function isArchivingForcedToTrigger()
245 {
246 $period = $this->params->getPeriod()->getLabel();
247 $debugSetting = 'always_archive_data_period'; // default
248
249 if ($period == 'day') {
250 $debugSetting = 'always_archive_data_day';
251 } elseif ($period == 'range') {
252 $debugSetting = 'always_archive_data_range';
253 }
254
255 return (bool) Config::getInstance()->Debug[$debugSetting];
256 }
257
258 /**
259 * Returns the idArchive if the archive is available in the database for the requested plugin.
260 * Returns false if the archive needs to be processed.
261 *
262 * (public for tests)
263 *
264 * @return array
265 */
266 public function loadExistingArchiveIdFromDb()
267 {
268 if ($this->isArchivingForcedToTrigger()) {
269 $this->logger->debug("Archiving forced to trigger for {$this->params}.");
270
271 // return no usable archive found, and no existing archive. this will skip invalidation, which should
272 // be fine since we just force archiving.
273 return [false, false, false, false, false, false];
274 }
275
276 $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();
277 $result = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
278 return $result;
279 }
280
281 /**
282 * Returns the minimum archive processed datetime to look at. Only public for tests.
283 *
284 * @return int|bool Datetime timestamp, or false if must look at any archive available
285 */
286 protected function getMinTimeArchiveProcessed()
287 {
288 // for range periods we can archive in a browser request request, make sure to check for the ttl no matter what
289 $isRangeArchiveAndArchivingEnabled = $this->params->getPeriod()->getLabel() == 'range'
290 && Rules::isArchivingEnabledFor([$this->params->getSite()->getId()], $this->params->getSegment(), $this->params->getPeriod()->getLabel());
291
292 if (!$isRangeArchiveAndArchivingEnabled) {
293 $endDateTimestamp = self::determineIfArchivePermanent($this->params->getDateEnd());
294 if ($endDateTimestamp) {
295 // past archive
296 return $endDateTimestamp;
297 }
298 }
299
300 $dateStart = $this->params->getDateStart();
301 $period = $this->params->getPeriod();
302 $segment = $this->params->getSegment();
303 $site = $this->params->getSite();
304 // in-progress archive
305 return Rules::getMinTimeProcessedForInProgressArchive($dateStart, $period, $segment, $site);
306 }
307
308 protected static function determineIfArchivePermanent(Date $dateEnd)
309 {
310 $now = time();
311 $endTimestampUTC = strtotime($dateEnd->getDateEndUTC());
312
313 if ($endTimestampUTC <= $now) {
314 // - if the period we are looking for is finished, we look for a ts_archived that
315 // is greater than the last day of the archive
316 return $endTimestampUTC;
317 }
318
319 return false;
320 }
321
322 private function shouldArchiveForSiteEvenWhenNoVisits()
323 {
324 $idSitesToArchive = $this->getIdSitesToArchiveWhenNoVisits();
325 return in_array($this->params->getSite()->getId(), $idSitesToArchive);
326 }
327
328 private function getIdSitesToArchiveWhenNoVisits()
329 {
330 $cache = Cache::getTransientCache();
331 $cacheKey = 'Archiving.getIdSitesToArchiveWhenNoVisits';
332
333 if (!$cache->contains($cacheKey)) {
334 $idSites = array();
335
336 // leaving undocumented unless decided otherwise
337 Piwik::postEvent('Archiving.getIdSitesToArchiveWhenNoVisits', array(&$idSites));
338
339 $cache->save($cacheKey, $idSites);
340 }
341
342 return $cache->fetch($cacheKey);
343 }
344
345 // public for tests
346 public function getReportsToInvalidate()
347 {
348 $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
349
350 foreach ($sitesPerDays as $dateStr => $siteIds) {
351 if (empty($siteIds)
352 || !in_array($this->params->getSite()->getId(), $siteIds)
353 ) {
354 unset($sitesPerDays[$dateStr]);
355 }
356
357 $date = Date::factory($dateStr);
358 if ($date->isEarlier($this->params->getPeriod()->getDateStart())
359 || $date->isLater($this->params->getPeriod()->getDateEnd())
360 ) { // date in list is not the current date, so ignore it
361 unset($sitesPerDays[$dateStr]);
362 }
363 }
364
365 return $sitesPerDays;
366 }
367
368 private function invalidatedReportsIfNeeded()
369 {
370 $sitesPerDays = $this->getReportsToInvalidate();
371 if (empty($sitesPerDays)) {
372 return;
373 }
374
375 foreach ($sitesPerDays as $date => $siteIds) {
376 try {
377 $this->invalidator->markArchivesAsInvalidated([$this->params->getSite()->getId()], array(Date::factory($date)), false, $this->params->getSegment());
378 } catch (\Exception $e) {
379 Site::clearCache();
380 throw $e;
381 }
382 }
383
384 Site::clearCache();
385 }
386
387 public function canSkipThisArchive()
388 {
389 $params = $this->params;
390 $idSite = $params->getSite()->getId();
391
392 $isWebsiteUsingTracker = $this->isWebsiteUsingTheTracker($idSite);
393 $isArchivingForcedWhenNoVisits = $this->shouldArchiveForSiteEvenWhenNoVisits();
394 $hasSiteVisitsBetweenTimeframe = $this->hasSiteVisitsBetweenTimeframe($idSite, $params->getPeriod());
395 $hasChildArchivesInPeriod = $this->dataAccessModel->hasChildArchivesInPeriod($idSite, $params->getPeriod());
396
397 if ($this->canSkipArchiveForSegment()) {
398 return true;
399 }
400
401 return $isWebsiteUsingTracker
402 && !$isArchivingForcedWhenNoVisits
403 && !$hasSiteVisitsBetweenTimeframe
404 && !$hasChildArchivesInPeriod;
405 }
406
407 public function canSkipArchiveForSegment()
408 {
409 $params = $this->params;
410
411 if ($params->getSegment()->isEmpty()) {
412 return false;
413 }
414
415 /** @var SegmentArchiving */
416 $segmentArchiving = StaticContainer::get(SegmentArchiving::class);
417 $segmentInfo = $segmentArchiving->findSegmentForHash($params->getSegment()->getHash(), $params->getSite()->getId());
418
419 if (!$segmentInfo) {
420 return false;
421 }
422
423 $segmentArchiveStartDate = $segmentArchiving->getReArchiveSegmentStartDate($segmentInfo);
424
425 if ($segmentArchiveStartDate !==null && $segmentArchiveStartDate->isLater($params->getPeriod()->getDateEnd()->getEndOfDay())) {
426 $doneFlag = Rules::getDoneStringFlagFor(
427 [$params->getSite()->getId()],
428 $params->getSegment(),
429 $params->getPeriod()->getLabel(),
430 $params->getRequestedPlugin()
431 );
432
433 // if there is no invalidation where the report is null, we can skip
434 // if we have invalidations for the period and name, but only for a specific reports, we can skip
435 // if the report is not null we only want to rearchive if we have invalidation for that report
436 // if we don't find invalidation for that report, we can skip
437 return !$this->dataAccessModel->hasInvalidationForPeriodAndName($params->getSite()->getId(), $params->getPeriod(), $doneFlag, $params->getArchiveOnlyReport());
438 }
439
440 return false;
441 }
442
443 private function isWebsiteUsingTheTracker($idSite)
444 {
445 $idSitesNotUsingTracker = self::getSitesNotUsingTracker();
446
447 $isUsingTracker = !in_array($idSite, $idSitesNotUsingTracker);
448
449 return $isUsingTracker;
450 }
451
452 public static function getSitesNotUsingTracker()
453 {
454 $cache = Cache::getTransientCache();
455
456 $cacheKey = 'Archiving.isWebsiteUsingTheTracker';
457 $idSitesNotUsingTracker = $cache->fetch($cacheKey);
458 if ($idSitesNotUsingTracker === false || !isset($idSitesNotUsingTracker)) {
459 // we want to trigger event only once
460 $idSitesNotUsingTracker = array();
461
462 /**
463 * This event is triggered when detecting whether there are sites that do not use the tracker.
464 *
465 * By default we only archive a site when there was actually any visit since the last archiving.
466 * However, some plugins do import data from another source instead of using the tracker and therefore
467 * will never have any visits for this site. To make sure we still archive data for such a site when
468 * archiving for this site is requested, you can listen to this event and add the idSite to the list of
469 * sites that do not use the tracker.
470 *
471 * @param bool $idSitesNotUsingTracker The list of idSites that rather import data instead of using the tracker
472 */
473 Piwik::postEvent('CronArchive.getIdSitesNotUsingTracker', array(&$idSitesNotUsingTracker));
474
475 $cache->save($cacheKey, $idSitesNotUsingTracker);
476 }
477 return $idSitesNotUsingTracker;
478 }
479
480 private function hasSiteVisitsBetweenTimeframe($idSite, Period $period)
481 {
482 $timezone = Site::getTimezoneFor($idSite);
483 list($date1, $date2) = $period->getBoundsInTimezone($timezone);
484
485 return $this->rawLogDao->hasSiteVisitsBetweenTimeframe($date1->getDatetime(), $date2->getDatetime(), $idSite);
486 }
487
488 public static function getArchivingDepth()
489 {
490 return self::$archivingDepth;
491 }
492
493 private function shouldForceInvalidatedArchive($value, $tsArchived)
494 {
495 $params = $this->params;
496
497 // the archive is invalidated and we are in a browser request that is allowed archive it
498 if ($value == ArchiveWriter::DONE_INVALIDATED
499 && Rules::isArchivingEnabledFor([$params->getSite()->getId()], $params->getSegment(), $params->getPeriod()->getLabel())
500 ) {
501 // if coming from core:archive, force rearchiving, since if we don't the entry will be removed from archive_invalidations
502 // w/o being rearchived
503 if (SettingsServer::isArchivePhpTriggered()) {
504 return true;
505 }
506
507 // if coming from a browser request, and period does not contain today, force rearchiving
508 $timezone = $params->getSite()->getTimezone();
509 if (!$params->getPeriod()->isDateInPeriod(Date::factoryInTimezone('today', $timezone))) {
510 return true;
511 }
512
513 // if coming from a browser request, and period does contain today, check the ttl for the period (done just below this)
514 $minDatetimeArchiveProcessedUTC = Rules::getMinTimeProcessedForInProgressArchive(
515 $params->getDateStart(), $params->getPeriod(), $params->getSegment(), $params->getSite());
516 $minDatetimeArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC);
517 if ($minDatetimeArchiveProcessedUTC
518 && Date::factory($tsArchived)->isEarlier($minDatetimeArchiveProcessedUTC)
519 ) {
520 return false;
521 }
522 }
523
524 return false;
525 }
526 }
527