PluginProbe ʕ •ᴥ•ʔ
Matomo Analytics – Powerful, Privacy-First Insights for WordPress / 4.3.0
Matomo Analytics – Powerful, Privacy-First Insights for WordPress v4.3.0
5.11.1 5.11.0 5.10.2 5.10.1 trunk 1.0.2 1.0.3 1.0.4 1.0.5 1.0.6 1.1.0 1.1.1 1.1.2 1.1.3 1.2.0 1.3.0 1.3.1 1.3.2 4.0.0 4.0.1 4.0.2 4.0.3 4.0.4 4.1.0 4.1.1 4.1.2 4.1.3 4.10.0 4.11.0 4.12.0 4.13.0 4.13.2 4.13.3 4.13.4 4.13.5 4.14.0 4.14.1 4.14.2 4.15.0 4.15.1 4.15.2 4.15.3 4.2.0 4.3.0 4.3.1 4.4.1 4.4.2 4.5.0 4.6.0 5.0.1 5.0.2 5.0.3 5.0.4 5.0.5 5.0.6 5.0.7 5.0.8 5.1.0 5.1.1 5.1.2 5.1.3 5.1.4 5.1.5 5.1.6 5.1.7 5.10.0 5.2.0 5.2.1 5.2.2 5.3.0 5.3.1 5.3.2 5.3.3 5.6.0 5.6.1 5.7.0 5.7.1 5.8.0 5.8.1 5.8.2
matomo / app / core / DataAccess / ArchiveSelector.php
matomo / app / core / DataAccess Last commit date
LogQueryBuilder 5 years ago Actions.php 5 years ago ArchiveSelector.php 5 years ago ArchiveTableCreator.php 5 years ago ArchiveTableDao.php 5 years ago ArchiveWriter.php 5 years ago ArchivingDbAdapter.php 5 years ago LogAggregator.php 5 years ago LogQueryBuilder.php 5 years ago LogTableTemporary.php 5 years ago Model.php 5 years ago RawLogDao.php 5 years ago TableMetadata.php 5 years ago
ArchiveSelector.php
491 lines
1 <?php
2 /**
3 * Matomo - free/libre analytics platform
4 *
5 * @link https://matomo.org
6 * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7 *
8 */
9 namespace Piwik\DataAccess;
10
11 use Exception;
12 use Piwik\Archive;
13 use Piwik\Archive\Chunk;
14 use Piwik\ArchiveProcessor;
15 use Piwik\ArchiveProcessor\Rules;
16 use Piwik\Common;
17 use Piwik\Container\StaticContainer;
18 use Piwik\Date;
19 use Piwik\Db;
20 use Piwik\Period;
21 use Piwik\Period\Range;
22 use Piwik\Segment;
23 use Piwik\SettingsServer;
24 use Psr\Log\LoggerInterface;
25
26 /**
27 * Data Access object used to query archives
28 *
29 * A record in the Database for a given report is defined by
30 * - idarchive = unique ID that is associated to all the data of this archive (idsite+period+date)
31 * - idsite = the ID of the website
32 * - date1 = starting day of the period
33 * - date2 = ending day of the period
34 * - period = integer that defines the period (day/week/etc.). @see period::getId()
35 * - ts_archived = timestamp when the archive was processed (UTC)
36 * - name = the name of the report (ex: uniq_visitors or search_keywords_by_search_engines)
37 * - value = the actual data (a numeric value, or a blob of compressed serialized data)
38 *
39 */
40 class ArchiveSelector
41 {
42 const NB_VISITS_RECORD_LOOKED_UP = "nb_visits";
43
44 const NB_VISITS_CONVERTED_RECORD_LOOKED_UP = "nb_visits_converted";
45
46 private static function getModel()
47 {
48 return new Model();
49 }
50
51 /**
52 * @param ArchiveProcessor\Parameters $params
53 * @param bool $minDatetimeArchiveProcessedUTC deprecated. Will be removed in Matomo 4.
54 * @return array An array with four values:
55 * - the latest archive ID or false if none
56 * - the latest visits value for the latest archive, regardless of whether the archive is invalidated or not
57 * - the latest visits converted value for the latest archive, regardless of whether the archive is invalidated or not
58 * - whether there is an archive that exists or not. if this is true and the latest archive is false, it means
59 * the archive found was not usable (for example, it was invalidated and we are not looking for invalidated archives)
60 * - the ts_archived for the latest usable archive
61 * @throws Exception
62 */
63 public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC = false, $includeInvalidated = null)
64 {
65 $idSite = $params->getSite()->getId();
66 $period = $params->getPeriod()->getId();
67 $dateStart = $params->getPeriod()->getDateStart();
68 $dateStartIso = $dateStart->toString('Y-m-d');
69 $dateEndIso = $params->getPeriod()->getDateEnd()->toString('Y-m-d');
70
71 $numericTable = ArchiveTableCreator::getNumericTable($dateStart);
72
73 $requestedPlugin = $params->getRequestedPlugin();
74 $segment = $params->getSegment();
75 $plugins = array("VisitsSummary", $requestedPlugin);
76 $plugins = array_filter($plugins);
77
78 $doneFlags = Rules::getDoneFlags($plugins, $segment);
79
80 $requestedPluginDoneFlags = empty($requestedPlugin) ? [] : Rules::getDoneFlags([$requestedPlugin], $segment);
81 $allPluginsDoneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($segment);
82
83 $doneFlagValues = Rules::getSelectableDoneFlagValues($includeInvalidated === null ? true : $includeInvalidated, $params, $includeInvalidated === null);
84
85 $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, null, $doneFlags);
86 if (empty($results)) { // no archive found
87 return [false, false, false, false, false, false];
88 }
89
90 $result = self::findArchiveDataWithLatestTsArchived($results, $requestedPluginDoneFlags, $allPluginsDoneFlag);
91
92 $tsArchived = isset($result['ts_archived']) ? $result['ts_archived'] : false;
93 $visits = isset($result['nb_visits']) ? $result['nb_visits'] : false;
94 $visitsConverted = isset($result['nb_visits_converted']) ? $result['nb_visits_converted'] : false;
95 $value = isset($result['value']) ? $result['value'] : false;
96
97 $result['idarchive'] = empty($result['idarchive']) ? [] : [$result['idarchive']];
98 if (isset($result['partial'])) {
99 $result['idarchive'] = array_merge($result['idarchive'], $result['partial']);
100 }
101
102 if (empty($result['idarchive'])
103 || (isset($result['value'])
104 && !in_array($result['value'], $doneFlagValues))
105 ) { // the archive cannot be considered valid for this request (has wrong done flag value)
106 return [false, $visits, $visitsConverted, true, $tsArchived, $value];
107 }
108
109 if (!empty($minDatetimeArchiveProcessedUTC) && !is_object($minDatetimeArchiveProcessedUTC)) {
110 $minDatetimeArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC);
111 }
112
113 // the archive is too old
114 if ($minDatetimeArchiveProcessedUTC
115 && !empty($result['idarchive'])
116 && Date::factory($tsArchived)->isEarlier($minDatetimeArchiveProcessedUTC)
117 ) {
118 return [false, $visits, $visitsConverted, true, $tsArchived, $value];
119 }
120
121 $idArchives = !empty($result['idarchive']) ? $result['idarchive'] : false;
122
123 return [$idArchives, $visits, $visitsConverted, true, $tsArchived, $value];
124 }
125
126 /**
127 * Queries and returns archive IDs for a set of sites, periods, and a segment.
128 *
129 * @param int[] $siteIds
130 * @param Period[] $periods
131 * @param Segment $segment
132 * @param string[] $plugins List of plugin names for which data is being requested.
133 * @param bool $includeInvalidated true to include archives that are DONE_INVALIDATED, false if only DONE_OK.
134 * @param bool $_skipSetGroupConcatMaxLen for tests
135 * @return array Archive IDs are grouped by archive name and period range, ie,
136 * array(
137 * 'VisitsSummary.done' => array(
138 * '2010-01-01' => array(1,2,3)
139 * )
140 * )
141 * @throws
142 */
143 public static function getArchiveIds($siteIds, $periods, $segment, $plugins, $includeInvalidated = true, $_skipSetGroupConcatMaxLen = false)
144 {
145 $logger = StaticContainer::get(LoggerInterface::class);
146 if (!$_skipSetGroupConcatMaxLen) {
147 try {
148 Db::get()->query('SET SESSION group_concat_max_len=' . (128 * 1024));
149 } catch (\Exception $ex) {
150 $logger->info("Could not set group_concat_max_len MySQL session variable.");
151 }
152 }
153
154 if (empty($siteIds)) {
155 throw new \Exception("Website IDs could not be read from the request, ie. idSite=");
156 }
157
158 foreach ($siteIds as $index => $siteId) {
159 $siteIds[$index] = (int) $siteId;
160 }
161
162 $getArchiveIdsSql = "SELECT idsite, date1, date2,
163 GROUP_CONCAT(CONCAT(idarchive,'|',`name`,'|',`value`) ORDER BY idarchive DESC SEPARATOR ',') AS archives
164 FROM %s
165 WHERE idsite IN (" . implode(',', $siteIds) . ")
166 AND " . self::getNameCondition($plugins, $segment, $includeInvalidated) . "
167 AND ts_archived IS NOT NULL
168 AND %s
169 GROUP BY idsite, date1, date2";
170
171 $monthToPeriods = array();
172 foreach ($periods as $period) {
173 /** @var Period $period */
174 if ($period->getDateStart()->isLater(Date::now()->addDay(2))) {
175 continue; // avoid creating any archive tables in the future
176 }
177 $table = ArchiveTableCreator::getNumericTable($period->getDateStart());
178 $monthToPeriods[$table][] = $period;
179 }
180
181 $db = Db::get();
182
183 // for every month within the archive query, select from numeric table
184 $result = array();
185 foreach ($monthToPeriods as $table => $periods) {
186 $firstPeriod = reset($periods);
187
188 $bind = array();
189
190 if ($firstPeriod instanceof Range) {
191 $dateCondition = "date1 = ? AND date2 = ?";
192 $bind[] = $firstPeriod->getDateStart()->toString('Y-m-d');
193 $bind[] = $firstPeriod->getDateEnd()->toString('Y-m-d');
194 } else {
195 // we assume there is no range date in $periods
196 $dateCondition = '(';
197
198 foreach ($periods as $period) {
199 if (strlen($dateCondition) > 1) {
200 $dateCondition .= ' OR ';
201 }
202
203 $dateCondition .= "(period = ? AND date1 = ? AND date2 = ?)";
204 $bind[] = $period->getId();
205 $bind[] = $period->getDateStart()->toString('Y-m-d');
206 $bind[] = $period->getDateEnd()->toString('Y-m-d');
207 }
208
209 $dateCondition .= ')';
210 }
211
212 $sql = sprintf($getArchiveIdsSql, $table, $dateCondition);
213 $archiveIds = $db->fetchAll($sql, $bind);
214
215 // get the archive IDs. we keep all archives until the first all plugins archive.
216 // everything older than that one is discarded.
217 foreach ($archiveIds as $row) {
218 $dateStr = $row['date1'] . ',' . $row['date2'];
219
220 $archives = $row['archives'];
221 $pairs = explode(',', $archives);
222 foreach ($pairs as $pair) {
223 $parts = explode('|', $pair);
224 if (count($parts) != 3) { // GROUP_CONCAT got cut off, have to ignore the rest
225 // note: in this edge case, we end up not selecting the all plugins archive because it will be older than the partials.
226 // not ideal, but it avoids an exception.
227 $logger->info("GROUP_CONCAT got cut off in ArchiveSelector." . __FUNCTION__ . ' for idsite = ' . $row['idsite'] . ', period = ' . $dateStr);
228 continue;
229 }
230
231 list($idarchive, $doneFlag, $value) = $parts;
232
233 $result[$doneFlag][$dateStr][] = $idarchive;
234 if (strpos($doneFlag, '.') === false // all plugins archive
235 // sanity check: DONE_PARTIAL shouldn't be used w/ done archives, but in case we see one,
236 // don't treat it like an all plugins archive
237 && $value != ArchiveWriter::DONE_PARTIAL
238 ) {
239 break; // found the all plugins archive, don't need to look in older archives since we have everything here
240 }
241 }
242 }
243 }
244
245 return $result;
246 }
247
248 /**
249 * Queries and returns archive data using a set of archive IDs.
250 *
251 * @param array $archiveIds The IDs of the archives to get data from.
252 * @param array $recordNames The names of the data to retrieve (ie, nb_visits, nb_actions, etc.).
253 * Note: You CANNOT pass multiple recordnames if $loadAllSubtables=true.
254 * @param string $archiveDataType The archive data type (either, 'blob' or 'numeric').
255 * @param int|null|string $idSubtable null if the root blob should be loaded, an integer if a subtable should be
256 * loaded and 'all' if all subtables should be loaded.
257 * @return array
258 *@throws Exception
259 */
260 public static function getArchiveData($archiveIds, $recordNames, $archiveDataType, $idSubtable)
261 {
262 $chunk = new Chunk();
263
264 $db = Db::get();
265
266 // create the SQL to select archive data
267 $loadAllSubtables = $idSubtable === Archive::ID_SUBTABLE_LOAD_ALL_SUBTABLES;
268 if ($loadAllSubtables) {
269 $name = reset($recordNames);
270
271 // select blobs w/ name like "$name_[0-9]+" w/o using RLIKE
272 $nameEnd = strlen($name) + 1;
273 $nameEndAppendix = $nameEnd + 1;
274 $appendix = $chunk->getAppendix();
275 $lenAppendix = strlen($appendix);
276
277 $checkForChunkBlob = "SUBSTRING(name, $nameEnd, $lenAppendix) = '$appendix'";
278 $checkForSubtableId = "(SUBSTRING(name, $nameEndAppendix, 1) >= '0'
279 AND SUBSTRING(name, $nameEndAppendix, 1) <= '9')";
280
281 $whereNameIs = "(name = ? OR (name LIKE ? AND ( $checkForChunkBlob OR $checkForSubtableId ) ))";
282 $bind = array($name, $name . '%');
283 } else {
284 if ($idSubtable === null) {
285 // select root table or specific record names
286 $bind = array_values($recordNames);
287 } else {
288 // select a subtable id
289 $bind = array();
290 foreach ($recordNames as $recordName) {
291 // to be backwards compatibe we need to look for the exact idSubtable blob and for the chunk
292 // that stores the subtables (a chunk stores many blobs in one blob)
293 $bind[] = $chunk->getRecordNameForTableId($recordName, $idSubtable);
294 $bind[] = self::appendIdSubtable($recordName, $idSubtable);
295 }
296 }
297
298 $inNames = Common::getSqlStringFieldsArray($bind);
299 $whereNameIs = "name IN ($inNames)";
300 }
301
302 $getValuesSql = "SELECT value, name, idsite, date1, date2, ts_archived
303 FROM %s
304 WHERE idarchive IN (%s)
305 AND " . $whereNameIs . "
306 ORDER BY ts_archived ASC"; // ascending order so we use the latest data found
307
308 // get data from every table we're querying
309 $rows = array();
310 foreach ($archiveIds as $period => $ids) {
311 if (empty($ids)) {
312 throw new Exception("Unexpected: id archive not found for period '$period' '");
313 }
314
315 // $period = "2009-01-04,2009-01-04",
316 $date = Date::factory(substr($period, 0, 10));
317
318 $isNumeric = $archiveDataType === 'numeric';
319 if ($isNumeric) {
320 $table = ArchiveTableCreator::getNumericTable($date);
321 } else {
322 $table = ArchiveTableCreator::getBlobTable($date);
323 }
324
325 $sql = sprintf($getValuesSql, $table, implode(',', $ids));
326 $dataRows = $db->fetchAll($sql, $bind);
327
328 foreach ($dataRows as $row) {
329 if ($isNumeric) {
330 $rows[] = $row;
331 } else {
332 $row['value'] = self::uncompress($row['value']);
333
334 if ($chunk->isRecordNameAChunk($row['name'])) {
335 self::moveChunkRowToRows($rows, $row, $chunk, $loadAllSubtables, $idSubtable);
336 } else {
337 $rows[] = $row;
338 }
339 }
340 }
341 }
342
343 return $rows;
344 }
345
346 private static function moveChunkRowToRows(&$rows, $row, Chunk $chunk, $loadAllSubtables, $idSubtable)
347 {
348 // $blobs = array([subtableID] = [blob of subtableId])
349 $blobs = Common::safe_unserialize($row['value']);
350
351 if (!is_array($blobs)) {
352 return;
353 }
354
355 // $rawName = eg 'PluginName_ArchiveName'
356 $rawName = $chunk->getRecordNameWithoutChunkAppendix($row['name']);
357
358 if ($loadAllSubtables) {
359 foreach ($blobs as $subtableId => $blob) {
360 $row['value'] = $blob;
361 $row['name'] = self::appendIdSubtable($rawName, $subtableId);
362 $rows[] = $row;
363 }
364 } elseif (array_key_exists($idSubtable, $blobs)) {
365 $row['value'] = $blobs[$idSubtable];
366 $row['name'] = self::appendIdSubtable($rawName, $idSubtable);
367 $rows[] = $row;
368 }
369 }
370
371 public static function appendIdSubtable($recordName, $id)
372 {
373 return $recordName . "_" . $id;
374 }
375
376 private static function uncompress($data)
377 {
378 return @gzuncompress($data);
379 }
380
381 /**
382 * Returns the SQL condition used to find successfully completed archives that
383 * this instance is querying for.
384 *
385 * @param array $plugins
386 * @param Segment $segment
387 * @param bool $includeInvalidated
388 * @return string
389 */
390 private static function getNameCondition(array $plugins, Segment $segment, $includeInvalidated = true)
391 {
392 // the flags used to tell how the archiving process for a specific archive was completed,
393 // if it was completed
394 $doneFlags = Rules::getDoneFlags($plugins, $segment);
395 $allDoneFlags = "'" . implode("','", $doneFlags) . "'";
396
397 $possibleValues = Rules::getSelectableDoneFlagValues($includeInvalidated, null, $checkAuthorizedToArchive = false);
398
399 // create the SQL to find archives that are DONE
400 return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))";
401 }
402
403 /**
404 * This method takes the output of Model::getArchiveIdAndVisits() and selects data from the
405 * latest archives.
406 *
407 * This includes:
408 * - the idarchive with the latest ts_archived ($results will be ordered by ts_archived desc)
409 * - the visits/converted visits of the latest archive, which includes archives for VisitsSummary alone
410 * ($requestedPluginDoneFlags will have the done flag for the overall archive plus a done flag for
411 * VisitsSummary by itself)
412 * - the ts_archived for the latest idarchive
413 * - the doneFlag value for the latest archive
414 *
415 * @param $results
416 * @param $doneFlags
417 * @return array
418 */
419 private static function findArchiveDataWithLatestTsArchived($results, $requestedPluginDoneFlags, $allPluginsDoneFlag)
420 {
421 $doneFlags = array_merge($requestedPluginDoneFlags, [$allPluginsDoneFlag]);
422
423 // find latest idarchive for each done flag
424 $idArchives = [];
425 $tsArchiveds = [];
426 foreach ($results as $row) {
427 $doneFlag = $row['name'];
428 if (!isset($idArchives[$doneFlag])) {
429 $idArchives[$doneFlag] = $row['idarchive'];
430 $tsArchiveds[$doneFlag] = $row['ts_archived'];
431 }
432 }
433
434 $archiveData = [
435 self::NB_VISITS_RECORD_LOOKED_UP => false,
436 self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP => false,
437 ];
438
439 foreach ($results as $result) {
440 if (in_array($result['name'], $doneFlags)
441 && in_array($result['idarchive'], $idArchives)
442 && $result['value'] != ArchiveWriter::DONE_PARTIAL
443 ) {
444 $archiveData = $result;
445 if (empty($archiveData[self::NB_VISITS_RECORD_LOOKED_UP])) {
446 $archiveData[self::NB_VISITS_RECORD_LOOKED_UP] = 0;
447 }
448 if (empty($archiveData[self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP])) {
449 $archiveData[self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP] = 0;
450 }
451 break;
452 }
453 }
454
455 foreach ([self::NB_VISITS_RECORD_LOOKED_UP, self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP] as $metric) {
456 foreach ($results as $result) {
457 if (!in_array($result['idarchive'], $idArchives)) {
458 continue;
459 }
460
461 if (empty($archiveData[$metric])) {
462 if (!empty($result[$metric]) || $result[$metric] === 0 || $result[$metric] === '0') {
463 $archiveData[$metric] = $result[$metric];
464 }
465 }
466 }
467 }
468
469 // add partial archives
470 $mainTsArchived = isset($tsArchiveds[$allPluginsDoneFlag]) ? $tsArchiveds[$allPluginsDoneFlag] : null;
471 foreach ($results as $row) {
472 if (!isset($idArchives[$row['name']])) {
473 continue;
474 }
475
476 $thisTsArchived = Date::factory($row['ts_archived']);
477 if ($row['value'] == ArchiveWriter::DONE_PARTIAL
478 && (empty($mainTsArchived) || !Date::factory($mainTsArchived)->isLater($thisTsArchived))
479 ) {
480 $archiveData['partial'][] = $row['idarchive'];
481
482 if (empty($archiveData['ts_archived'])) {
483 $archiveData['ts_archived'] = $row['ts_archived'];
484 }
485 }
486 }
487
488 return $archiveData;
489 }
490 }
491