PluginProbe ʕ •ᴥ•ʔ
Matomo Analytics – Powerful, Privacy-First Insights for WordPress / 4.4.2
Matomo Analytics – Powerful, Privacy-First Insights for WordPress v4.4.2
5.13.0 5.12.1 5.12.0 5.11.1 5.11.0 5.10.2 5.10.1 trunk 1.0.2 1.0.3 1.0.4 1.0.5 1.0.6 1.1.0 1.1.1 1.1.2 1.1.3 1.2.0 1.3.0 1.3.1 1.3.2 4.0.0 4.0.1 4.0.2 4.0.3 4.0.4 4.1.0 4.1.1 4.1.2 4.1.3 4.10.0 4.11.0 4.12.0 4.13.0 4.13.2 4.13.3 4.13.4 4.13.5 4.14.0 4.14.1 4.14.2 4.15.0 4.15.1 4.15.2 4.15.3 4.2.0 4.3.0 4.3.1 4.4.1 4.4.2 4.5.0 4.6.0 5.0.1 5.0.2 5.0.3 5.0.4 5.0.5 5.0.6 5.0.7 5.0.8 5.1.0 5.1.1 5.1.2 5.1.3 5.1.4 5.1.5 5.1.6 5.1.7 5.10.0 5.2.0 5.2.1 5.2.2 5.3.0 5.3.1 5.3.2 5.3.3 5.6.0 5.6.1 5.7.0 5.7.1 5.8.0 5.8.1 5.8.2
matomo / app / core / ArchiveProcessor / Loader.php
matomo / app / core / ArchiveProcessor Last commit date
ArchivingStatus.php 5 years ago Loader.php 5 years ago Parameters.php 5 years ago PluginsArchiver.php 5 years ago PluginsArchiverException.php 5 years ago Rules.php 5 years ago
Loader.php
517 lines
1 <?php
2 /**
3 * Matomo - free/libre analytics platform
4 *
5 * @link https://matomo.org
6 * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7 *
8 */
9 namespace Piwik\ArchiveProcessor;
10
11 use Piwik\Archive\ArchiveInvalidator;
12 use Piwik\Cache;
13 use Piwik\Common;
14 use Piwik\Config;
15 use Piwik\Container\StaticContainer;
16 use Piwik\Context;
17 use Piwik\DataAccess\ArchiveSelector;
18 use Piwik\DataAccess\ArchiveTableCreator;
19 use Piwik\DataAccess\ArchiveWriter;
20 use Piwik\DataAccess\Model;
21 use Piwik\DataAccess\RawLogDao;
22 use Piwik\Date;
23 use Piwik\Db;
24 use Piwik\Option;
25 use Piwik\Period;
26 use Piwik\Piwik;
27 use Piwik\SettingsServer;
28 use Piwik\Site;
29 use Psr\Log\LoggerInterface;
30 use Piwik\CronArchive\SegmentArchiving;
31
32 /**
33 * This class uses PluginsArchiver class to trigger data aggregation and create archives.
34 */
35 class Loader
36 {
37 private static $archivingDepth = 0;
38
39 /**
40 * @var Parameters
41 */
42 protected $params;
43
44 /**
45 * @var ArchiveInvalidator
46 */
47 private $invalidator;
48
49 /**
50 * @var \Matomo\Cache\Cache
51 */
52 private $cache;
53
54 /**
55 * @var LoggerInterface
56 */
57 private $logger;
58
59 /**
60 * @var RawLogDao
61 */
62 private $rawLogDao;
63
64 /**
65 * @var Model
66 */
67 private $dataAccessModel;
68
69 public function __construct(Parameters $params, $invalidateBeforeArchiving = false)
70 {
71 $this->params = $params;
72 $this->invalidateBeforeArchiving = $invalidateBeforeArchiving;
73 $this->invalidator = StaticContainer::get(ArchiveInvalidator::class);
74 $this->cache = Cache::getTransientCache();
75 $this->logger = StaticContainer::get(LoggerInterface::class);
76 $this->rawLogDao = new RawLogDao();
77 $this->dataAccessModel = new Model();
78 }
79
80 /**
81 * @return bool
82 */
83 protected function isThereSomeVisits($visits)
84 {
85 return $visits > 0;
86 }
87
88 /**
89 * @return bool
90 */
91 protected function mustProcessVisitCount($visits)
92 {
93 return $visits === false;
94 }
95
96 public function prepareArchive($pluginName)
97 {
98 return Context::changeIdSite($this->params->getSite()->getId(), function () use ($pluginName) {
99 try {
100 ++self::$archivingDepth;
101 return $this->prepareArchiveImpl($pluginName);
102 } finally {
103 --self::$archivingDepth;
104 }
105 });
106 }
107
108 private function prepareArchiveImpl($pluginName)
109 {
110 $this->params->setRequestedPlugin($pluginName);
111
112 if (SettingsServer::isArchivePhpTriggered()) {
113 $requestedReport = Common::getRequestVar('requestedReport', '', 'string');
114 if (!empty($requestedReport)) {
115 $this->params->setArchiveOnlyReport($requestedReport);
116 }
117 }
118
119 // invalidate existing archives before we start archiving in case data was tracked in the past. if the archive is
120 // made invalid, we will correctly re-archive below.
121 if ($this->invalidateBeforeArchiving
122 && Rules::isBrowserTriggerEnabled()
123 ) {
124 $this->invalidatedReportsIfNeeded();
125 }
126
127 // NOTE: $idArchives will contain the latest DONE_OK/DONE_INVALIDATED archive as well as any partial archives
128 // with a ts_archived >= the DONE_OK/DONE_INVALIDATED date.
129 list($idArchives, $visits, $visitsConverted, $isAnyArchiveExists, $tsArchived, $value) = $this->loadExistingArchiveIdFromDb();
130 if (!empty($idArchives)
131 && !Rules::isActuallyForceArchivingSinglePlugin()
132 && !$this->shouldForceInvalidatedArchive($value, $tsArchived)
133 ) {
134 // we have a usable idarchive (it's not invalidated and it's new enough), and we are not archiving
135 // a single report
136 return [$idArchives, $visits];
137 }
138
139 // NOTE: this optimization helps when archiving large periods. eg, if archiving a year w/ a segment where
140 // there are not visits in the entire year, we don't have to go through and do anything. but, w/o this
141 // code, we will end up launching archiving for each month, week and day, even though we don't have to.
142 //
143 // we don't create an archive in this case, because the archive may be in progress in some way, so a 0
144 // visits archive can be inaccurate in the long run.
145 if ($this->canSkipThisArchive()) {
146 if (!empty($idArchives)) {
147 return [$idArchives, $visits];
148 } else {
149 return [false, 0];
150 }
151 }
152
153 if (SettingsServer::isArchivePhpTriggered()) {
154 $this->logger->info("initiating archiving via core:archive for " . $this->params);
155 }
156
157 list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted);
158 list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted);
159
160 if ($this->isThereSomeVisits($visits) || PluginsArchiver::doesAnyPluginArchiveWithoutVisits()) {
161 return [[$idArchive], $visits];
162 }
163
164 return [false, false];
165 }
166
167 /**
168 * Prepares the core metrics if needed.
169 *
170 * @param $visits
171 * @return array
172 */
173 protected function prepareCoreMetricsArchive($visits, $visitsConverted)
174 {
175 $createSeparateArchiveForCoreMetrics = $this->mustProcessVisitCount($visits)
176 && !$this->doesRequestedPluginIncludeVisitsSummary();
177
178 if ($createSeparateArchiveForCoreMetrics) {
179 $requestedPlugin = $this->params->getRequestedPlugin();
180 $requestedReport = $this->params->getArchiveOnlyReport();
181 $isPartialArchive = $this->params->isPartialArchive();
182
183 $this->params->setRequestedPlugin('VisitsSummary');
184 $this->params->setArchiveOnlyReport(null);
185 $this->params->setIsPartialArchive(false);
186
187 $metrics = Context::executeWithQueryParameters(['requestedReport' => ''], function () {
188 $pluginsArchiver = new PluginsArchiver($this->params);
189 $metrics = $pluginsArchiver->callAggregateCoreMetrics();
190 $pluginsArchiver->finalizeArchive();
191 return $metrics;
192 });
193
194 $this->params->setRequestedPlugin($requestedPlugin);
195 $this->params->setArchiveOnlyReport($requestedReport);
196 $this->params->setIsPartialArchive($isPartialArchive);
197
198 $visits = $metrics['nb_visits'];
199 $visitsConverted = $metrics['nb_visits_converted'];
200 }
201
202 return array($visits, $visitsConverted);
203 }
204
205 protected function prepareAllPluginsArchive($visits, $visitsConverted)
206 {
207 $pluginsArchiver = new PluginsArchiver($this->params);
208
209 if ($this->mustProcessVisitCount($visits)
210 || $this->doesRequestedPluginIncludeVisitsSummary()
211 ) {
212 $metrics = $pluginsArchiver->callAggregateCoreMetrics();
213 $visits = $metrics['nb_visits'];
214 $visitsConverted = $metrics['nb_visits_converted'];
215 }
216
217 $forceArchivingWithoutVisits = !$this->isThereSomeVisits($visits) && $this->shouldArchiveForSiteEvenWhenNoVisits();
218 $pluginsArchiver->callAggregateAllPlugins($visits, $visitsConverted, $forceArchivingWithoutVisits);
219
220 $idArchive = $pluginsArchiver->finalizeArchive();
221
222 return array($idArchive, $visits);
223 }
224
225 protected function doesRequestedPluginIncludeVisitsSummary()
226 {
227 $processAllReportsIncludingVisitsSummary =
228 Rules::shouldProcessReportsAllPlugins(array($this->params->getSite()->getId()), $this->params->getSegment(), $this->params->getPeriod()->getLabel());
229 $doesRequestedPluginIncludeVisitsSummary = $processAllReportsIncludingVisitsSummary
230 || $this->params->getRequestedPlugin() == 'VisitsSummary';
231 return $doesRequestedPluginIncludeVisitsSummary;
232 }
233
234 protected function isArchivingForcedToTrigger()
235 {
236 $period = $this->params->getPeriod()->getLabel();
237 $debugSetting = 'always_archive_data_period'; // default
238
239 if ($period == 'day') {
240 $debugSetting = 'always_archive_data_day';
241 } elseif ($period == 'range') {
242 $debugSetting = 'always_archive_data_range';
243 }
244
245 return (bool) Config::getInstance()->Debug[$debugSetting];
246 }
247
248 /**
249 * Returns the idArchive if the archive is available in the database for the requested plugin.
250 * Returns false if the archive needs to be processed.
251 *
252 * (public for tests)
253 *
254 * @return array
255 */
256 public function loadExistingArchiveIdFromDb()
257 {
258 if ($this->isArchivingForcedToTrigger()) {
259 $this->logger->debug("Archiving forced to trigger for {$this->params}.");
260
261 // return no usable archive found, and no existing archive. this will skip invalidation, which should
262 // be fine since we just force archiving.
263 return [false, false, false, false, false, false];
264 }
265
266 $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();
267 $result = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
268 return $result;
269 }
270
271 /**
272 * Returns the minimum archive processed datetime to look at. Only public for tests.
273 *
274 * @return int|bool Datetime timestamp, or false if must look at any archive available
275 */
276 protected function getMinTimeArchiveProcessed()
277 {
278 // for range periods we can archive in a browser request request, make sure to check for the ttl no matter what
279 $isRangeArchiveAndArchivingEnabled = $this->params->getPeriod()->getLabel() == 'range'
280 && Rules::isArchivingEnabledFor([$this->params->getSite()->getId()], $this->params->getSegment(), $this->params->getPeriod()->getLabel());
281
282 if (!$isRangeArchiveAndArchivingEnabled) {
283 $endDateTimestamp = self::determineIfArchivePermanent($this->params->getDateEnd());
284 if ($endDateTimestamp) {
285 // past archive
286 return $endDateTimestamp;
287 }
288 }
289
290 $dateStart = $this->params->getDateStart();
291 $period = $this->params->getPeriod();
292 $segment = $this->params->getSegment();
293 $site = $this->params->getSite();
294 // in-progress archive
295 return Rules::getMinTimeProcessedForInProgressArchive($dateStart, $period, $segment, $site);
296 }
297
298 protected static function determineIfArchivePermanent(Date $dateEnd)
299 {
300 $now = time();
301 $endTimestampUTC = strtotime($dateEnd->getDateEndUTC());
302
303 if ($endTimestampUTC <= $now) {
304 // - if the period we are looking for is finished, we look for a ts_archived that
305 // is greater than the last day of the archive
306 return $endTimestampUTC;
307 }
308
309 return false;
310 }
311
312 private function shouldArchiveForSiteEvenWhenNoVisits()
313 {
314 $idSitesToArchive = $this->getIdSitesToArchiveWhenNoVisits();
315 return in_array($this->params->getSite()->getId(), $idSitesToArchive);
316 }
317
318 private function getIdSitesToArchiveWhenNoVisits()
319 {
320 $cache = Cache::getTransientCache();
321 $cacheKey = 'Archiving.getIdSitesToArchiveWhenNoVisits';
322
323 if (!$cache->contains($cacheKey)) {
324 $idSites = array();
325
326 // leaving undocumented unless decided otherwise
327 Piwik::postEvent('Archiving.getIdSitesToArchiveWhenNoVisits', array(&$idSites));
328
329 $cache->save($cacheKey, $idSites);
330 }
331
332 return $cache->fetch($cacheKey);
333 }
334
335 // public for tests
336 public function getReportsToInvalidate()
337 {
338 $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
339
340 foreach ($sitesPerDays as $dateStr => $siteIds) {
341 if (empty($siteIds)
342 || !in_array($this->params->getSite()->getId(), $siteIds)
343 ) {
344 unset($sitesPerDays[$dateStr]);
345 }
346
347 $date = Date::factory($dateStr);
348 if ($date->isEarlier($this->params->getPeriod()->getDateStart())
349 || $date->isLater($this->params->getPeriod()->getDateEnd())
350 ) { // date in list is not the current date, so ignore it
351 unset($sitesPerDays[$dateStr]);
352 }
353 }
354
355 return $sitesPerDays;
356 }
357
358 private function invalidatedReportsIfNeeded()
359 {
360 $sitesPerDays = $this->getReportsToInvalidate();
361 if (empty($sitesPerDays)) {
362 return;
363 }
364
365 foreach ($sitesPerDays as $date => $siteIds) {
366 try {
367 $this->invalidator->markArchivesAsInvalidated([$this->params->getSite()->getId()], array(Date::factory($date)), false, $this->params->getSegment());
368 } catch (\Exception $e) {
369 Site::clearCache();
370 throw $e;
371 }
372 }
373
374 Site::clearCache();
375 }
376
377 public function canSkipThisArchive()
378 {
379 $params = $this->params;
380 $idSite = $params->getSite()->getId();
381
382 $isWebsiteUsingTracker = $this->isWebsiteUsingTheTracker($idSite);
383 $isArchivingForcedWhenNoVisits = $this->shouldArchiveForSiteEvenWhenNoVisits();
384 $hasSiteVisitsBetweenTimeframe = $this->hasSiteVisitsBetweenTimeframe($idSite, $params->getPeriod());
385 $hasChildArchivesInPeriod = $this->dataAccessModel->hasChildArchivesInPeriod($idSite, $params->getPeriod());
386
387 if ($this->canSkipArchiveForSegment()) {
388 return true;
389 }
390
391 return $isWebsiteUsingTracker
392 && !$isArchivingForcedWhenNoVisits
393 && !$hasSiteVisitsBetweenTimeframe
394 && !$hasChildArchivesInPeriod;
395 }
396
397 public function canSkipArchiveForSegment()
398 {
399 $params = $this->params;
400
401 if ($params->getSegment()->isEmpty()) {
402 return false;
403 }
404
405 /** @var SegmentArchiving */
406 $segmentArchiving = StaticContainer::get(SegmentArchiving::class);
407 $segmentInfo = $segmentArchiving->findSegmentForHash($params->getSegment()->getHash(), $params->getSite()->getId());
408
409 if (!$segmentInfo) {
410 return false;
411 }
412
413 $segmentArchiveStartDate = $segmentArchiving->getReArchiveSegmentStartDate($segmentInfo);
414
415 if ($segmentArchiveStartDate !==null && $segmentArchiveStartDate->isLater($params->getPeriod()->getDateEnd()->getEndOfDay())) {
416 $doneFlag = Rules::getDoneStringFlagFor(
417 [$params->getSite()->getId()],
418 $params->getSegment(),
419 $params->getPeriod()->getLabel(),
420 $params->getRequestedPlugin()
421 );
422
423 // if there is no invalidation where the report is null, we can skip
424 // if we have invalidations for the period and name, but only for a specific reports, we can skip
425 // if the report is not null we only want to rearchive if we have invalidation for that report
426 // if we don't find invalidation for that report, we can skip
427 return !$this->dataAccessModel->hasInvalidationForPeriodAndName($params->getSite()->getId(), $params->getPeriod(), $doneFlag, $params->getArchiveOnlyReport());
428 }
429
430 return false;
431 }
432
433 private function isWebsiteUsingTheTracker($idSite)
434 {
435 $idSitesNotUsingTracker = self::getSitesNotUsingTracker();
436
437 $isUsingTracker = !in_array($idSite, $idSitesNotUsingTracker);
438
439 return $isUsingTracker;
440 }
441
442 public static function getSitesNotUsingTracker()
443 {
444 $cache = Cache::getTransientCache();
445
446 $cacheKey = 'Archiving.isWebsiteUsingTheTracker';
447 $idSitesNotUsingTracker = $cache->fetch($cacheKey);
448 if ($idSitesNotUsingTracker === false || !isset($idSitesNotUsingTracker)) {
449 // we want to trigger event only once
450 $idSitesNotUsingTracker = array();
451
452 /**
453 * This event is triggered when detecting whether there are sites that do not use the tracker.
454 *
455 * By default we only archive a site when there was actually any visit since the last archiving.
456 * However, some plugins do import data from another source instead of using the tracker and therefore
457 * will never have any visits for this site. To make sure we still archive data for such a site when
458 * archiving for this site is requested, you can listen to this event and add the idSite to the list of
459 * sites that do not use the tracker.
460 *
461 * @param bool $idSitesNotUsingTracker The list of idSites that rather import data instead of using the tracker
462 */
463 Piwik::postEvent('CronArchive.getIdSitesNotUsingTracker', array(&$idSitesNotUsingTracker));
464
465 $cache->save($cacheKey, $idSitesNotUsingTracker);
466 }
467 return $idSitesNotUsingTracker;
468 }
469
470 private function hasSiteVisitsBetweenTimeframe($idSite, Period $period)
471 {
472 $timezone = Site::getTimezoneFor($idSite);
473 list($date1, $date2) = $period->getBoundsInTimezone($timezone);
474
475 return $this->rawLogDao->hasSiteVisitsBetweenTimeframe($date1->getDatetime(), $date2->getDatetime(), $idSite);
476 }
477
478 public static function getArchivingDepth()
479 {
480 return self::$archivingDepth;
481 }
482
483 private function shouldForceInvalidatedArchive($value, $tsArchived)
484 {
485 $params = $this->params;
486
487 // the archive is invalidated and we are in a browser request that is allowed archive it
488 if ($value == ArchiveWriter::DONE_INVALIDATED
489 && Rules::isArchivingEnabledFor([$params->getSite()->getId()], $params->getSegment(), $params->getPeriod()->getLabel())
490 ) {
491 // if coming from core:archive, force rearchiving, since if we don't the entry will be removed from archive_invalidations
492 // w/o being rearchived
493 if (SettingsServer::isArchivePhpTriggered()) {
494 return true;
495 }
496
497 // if coming from a browser request, and period does not contain today, force rearchiving
498 $timezone = $params->getSite()->getTimezone();
499 if (!$params->getPeriod()->isDateInPeriod(Date::factoryInTimezone('today', $timezone))) {
500 return true;
501 }
502
503 // if coming from a browser request, and period does contain today, check the ttl for the period (done just below this)
504 $minDatetimeArchiveProcessedUTC = Rules::getMinTimeProcessedForInProgressArchive(
505 $params->getDateStart(), $params->getPeriod(), $params->getSegment(), $params->getSite());
506 $minDatetimeArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC);
507 if ($minDatetimeArchiveProcessedUTC
508 && Date::factory($tsArchived)->isEarlier($minDatetimeArchiveProcessedUTC)
509 ) {
510 return false;
511 }
512 }
513
514 return false;
515 }
516 }
517