API
2 years ago
Access
3 years ago
Application
4 years ago
Archive
3 years ago
ArchiveProcessor
2 years ago
Archiver
5 years ago
AssetManager
3 years ago
Auth
3 years ago
Category
5 years ago
Changes
3 years ago
CliMulti
4 years ago
Columns
3 years ago
Concurrency
3 years ago
Config
4 years ago
Container
4 years ago
CronArchive
3 years ago
DataAccess
2 years ago
DataFiles
5 years ago
DataTable
2 years ago
Db
3 years ago
DeviceDetector
3 years ago
Email
5 years ago
Exception
3 years ago
Http
4 years ago
Intl
4 years ago
Mail
3 years ago
Measurable
5 years ago
Menu
3 years ago
Metrics
4 years ago
Notification
4 years ago
Period
4 years ago
Plugin
2 years ago
ProfessionalServices
4 years ago
Report
5 years ago
ReportRenderer
2 years ago
Scheduler
4 years ago
Segment
3 years ago
Session
4 years ago
Settings
3 years ago
Tracker
3 years ago
Translation
4 years ago
UpdateCheck
5 years ago
Updater
3 years ago
Updates
3 years ago
Validators
4 years ago
View
4 years ago
ViewDataTable
3 years ago
Visualization
4 years ago
Widget
5 years ago
Access.php
4 years ago
Archive.php
3 years ago
ArchiveProcessor.php
2 years ago
AssetManager.php
4 years ago
Auth.php
5 years ago
AuthResult.php
5 years ago
BaseFactory.php
5 years ago
Cache.php
5 years ago
CacheId.php
5 years ago
CliMulti.php
4 years ago
Common.php
3 years ago
Config.php
3 years ago
Console.php
3 years ago
Context.php
5 years ago
Cookie.php
3 years ago
CronArchive.php
3 years ago
DataArray.php
4 years ago
DataTable.php
2 years ago
Date.php
3 years ago
Db.php
4 years ago
DbHelper.php
3 years ago
Development.php
5 years ago
ErrorHandler.php
5 years ago
EventDispatcher.php
3 years ago
ExceptionHandler.php
3 years ago
FileIntegrity.php
3 years ago
Filechecks.php
4 years ago
Filesystem.php
3 years ago
FrontController.php
3 years ago
Http.php
3 years ago
IP.php
4 years ago
Log.php
4 years ago
LogDeleter.php
4 years ago
Mail.php
3 years ago
Metrics.php
3 years ago
NoAccessException.php
5 years ago
Nonce.php
3 years ago
Notification.php
5 years ago
NumberFormatter.php
4 years ago
Option.php
4 years ago
Period.php
5 years ago
Piwik.php
3 years ago
Plugin.php
4 years ago
Profiler.php
4 years ago
ProxyHeaders.php
5 years ago
ProxyHttp.php
3 years ago
QuickForm2.php
5 years ago
RankingQuery.php
3 years ago
ReportRenderer.php
4 years ago
Segment.php
3 years ago
Sequence.php
5 years ago
Session.php
3 years ago
SettingsPiwik.php
3 years ago
SettingsServer.php
5 years ago
Singleton.php
5 years ago
Site.php
3 years ago
SiteContentDetector.php
2 years ago
SupportedBrowser.php
3 years ago
TCPDF.php
5 years ago
Theme.php
5 years ago
Timer.php
5 years ago
Tracker.php
4 years ago
Twig.php
4 years ago
Unzip.php
5 years ago
UpdateCheck.php
5 years ago
Updater.php
4 years ago
UpdaterErrorException.php
5 years ago
Updates.php
5 years ago
Url.php
2 years ago
UrlHelper.php
3 years ago
Version.php
2 years ago
View.php
3 years ago
bootstrap.php
3 years ago
dispatch.php
5 years ago
testMinimumPhpVersion.php
3 years ago
ArchiveProcessor.php
780 lines
| 1 | <?php |
| 2 | /** |
| 3 | * Matomo - free/libre analytics platform |
| 4 | * |
| 5 | * @link https://matomo.org |
| 6 | * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later |
| 7 | * |
| 8 | */ |
| 9 | namespace Piwik; |
| 10 | |
| 11 | use Exception; |
| 12 | use Piwik\Archive\DataTableFactory; |
| 13 | use Piwik\ArchiveProcessor\Parameters; |
| 14 | use Piwik\ArchiveProcessor\Rules; |
| 15 | use Piwik\Container\StaticContainer; |
| 16 | use Piwik\DataAccess\ArchiveWriter; |
| 17 | use Piwik\DataAccess\LogAggregator; |
| 18 | use Piwik\DataTable\Manager; |
| 19 | use Piwik\DataTable\Map; |
| 20 | use Piwik\DataTable\Row; |
| 21 | use Piwik\Segment\SegmentExpression; |
| 22 | use Psr\Log\LoggerInterface; |
| 23 | |
| 24 | /** |
| 25 | * Used by {@link Piwik\Plugin\Archiver} instances to insert and aggregate archive data. |
| 26 | * |
| 27 | * ### See also |
| 28 | * |
| 29 | * - **{@link Piwik\Plugin\Archiver}** - to learn how plugins should implement their own analytics |
| 30 | * aggregation logic. |
| 31 | * - **{@link Piwik\DataAccess\LogAggregator}** - to learn how plugins can perform data aggregation |
| 32 | * across Piwik's log tables. |
| 33 | * |
| 34 | * ### Examples |
| 35 | * |
| 36 | * **Inserting numeric data** |
| 37 | * |
| 38 | * // function in an Archiver descendant |
| 39 | * public function aggregateDayReport() |
| 40 | * { |
| 41 | * $archiveProcessor = $this->getProcessor(); |
| 42 | * |
| 43 | * $myFancyMetric = // ... calculate the metric value ... |
| 44 | * $archiveProcessor->insertNumericRecord('MyPlugin_myFancyMetric', $myFancyMetric); |
| 45 | * } |
| 46 | * |
| 47 | * **Inserting serialized DataTables** |
| 48 | * |
| 49 | * // function in an Archiver descendant |
| 50 | * public function aggregateDayReport() |
| 51 | * { |
| 52 | * $archiveProcessor = $this->getProcessor(); |
| 53 | * |
| 54 | * $maxRowsInTable = Config::getInstance()->General['datatable_archiving_maximum_rows_standard'];j |
| 55 | * |
| 56 | * $dataTable = // ... build by aggregating visits ... |
| 57 | * $serializedData = $dataTable->getSerialized($maxRowsInTable, $maxRowsInSubtable = $maxRowsInTable, |
| 58 | * $columnToSortBy = Metrics::INDEX_NB_VISITS); |
| 59 | * |
| 60 | * $archiveProcessor->insertBlobRecords('MyPlugin_myFancyReport', $serializedData); |
| 61 | * } |
| 62 | * |
| 63 | * **Aggregating archive data** |
| 64 | * |
| 65 | * // function in Archiver descendant |
| 66 | * public function aggregateMultipleReports() |
| 67 | * { |
| 68 | * $archiveProcessor = $this->getProcessor(); |
| 69 | * |
| 70 | * // aggregate a metric |
| 71 | * $archiveProcessor->aggregateNumericMetrics('MyPlugin_myFancyMetric'); |
| 72 | * $archiveProcessor->aggregateNumericMetrics('MyPlugin_mySuperFancyMetric', 'max'); |
| 73 | * |
| 74 | * // aggregate a report |
| 75 | * $archiveProcessor->aggregateDataTableRecords('MyPlugin_myFancyReport'); |
| 76 | * } |
| 77 | * |
| 78 | */ |
| 79 | class ArchiveProcessor |
| 80 | { |
| 81 | /** |
| 82 | * @var bool |
| 83 | */ |
| 84 | public static $isRootArchivingRequest = true; |
| 85 | |
| 86 | /** |
| 87 | * @var \Piwik\DataAccess\ArchiveWriter |
| 88 | */ |
| 89 | private $archiveWriter; |
| 90 | |
| 91 | /** |
| 92 | * @var \Piwik\DataAccess\LogAggregator |
| 93 | */ |
| 94 | private $logAggregator; |
| 95 | |
| 96 | /** |
| 97 | * @var Archive |
| 98 | */ |
| 99 | public $archive = null; |
| 100 | |
| 101 | /** |
| 102 | * @var Parameters |
| 103 | */ |
| 104 | private $params; |
| 105 | |
| 106 | /** |
| 107 | * @var int |
| 108 | */ |
| 109 | private $numberOfVisits = false; |
| 110 | |
| 111 | private $numberOfVisitsConverted = false; |
| 112 | |
| 113 | public function __construct(Parameters $params, ArchiveWriter $archiveWriter, LogAggregator $logAggregator) |
| 114 | { |
| 115 | $this->params = $params; |
| 116 | $this->logAggregator = $logAggregator; |
| 117 | $this->archiveWriter = $archiveWriter; |
| 118 | } |
| 119 | |
| 120 | protected function getArchive() |
| 121 | { |
| 122 | if (empty($this->archive)) { |
| 123 | $subPeriods = $this->params->getSubPeriods(); |
| 124 | $idSites = $this->params->getIdSites(); |
| 125 | $this->archive = Archive::factory($this->params->getSegment(), $subPeriods, $idSites); |
| 126 | |
| 127 | /** |
| 128 | * @internal |
| 129 | */ |
| 130 | Piwik::postEvent('ArchiveProcessor.getArchive', [$this->archive]); |
| 131 | } |
| 132 | |
| 133 | return $this->archive; |
| 134 | } |
| 135 | |
| 136 | public function setNumberOfVisits($visits, $visitsConverted) |
| 137 | { |
| 138 | $this->numberOfVisits = $visits; |
| 139 | $this->numberOfVisitsConverted = $visitsConverted; |
| 140 | } |
| 141 | |
| 142 | /** |
| 143 | * Returns the {@link Parameters} object containing the site, period and segment we're archiving |
| 144 | * data for. |
| 145 | * |
| 146 | * @return Parameters |
| 147 | * @api |
| 148 | */ |
| 149 | public function getParams() |
| 150 | { |
| 151 | return $this->params; |
| 152 | } |
| 153 | |
| 154 | /** |
| 155 | * Returns a `{@link Piwik\DataAccess\LogAggregator}` instance for the site, period and segment this |
| 156 | * ArchiveProcessor will insert archive data for. |
| 157 | * |
| 158 | * @return LogAggregator |
| 159 | * @api |
| 160 | */ |
| 161 | public function getLogAggregator() |
| 162 | { |
| 163 | return $this->logAggregator; |
| 164 | } |
| 165 | |
| 166 | /** |
| 167 | * Array of (column name before => column name renamed) of the columns for which sum operation is invalid. |
| 168 | * These columns will be renamed as per this mapping. |
| 169 | * @var array |
| 170 | */ |
| 171 | protected static $columnsToRenameAfterAggregation = array( |
| 172 | Metrics::INDEX_NB_UNIQ_VISITORS => Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS, |
| 173 | Metrics::INDEX_NB_USERS => Metrics::INDEX_SUM_DAILY_NB_USERS, |
| 174 | ); |
| 175 | |
| 176 | /** |
| 177 | * Sums records for every subperiod of the current period and inserts the result as the record |
| 178 | * for this period. |
| 179 | * |
| 180 | * DataTables are summed recursively so subtables will be summed as well. |
| 181 | * |
| 182 | * @param string|array $recordNames Name(s) of the report we are aggregating, eg, `'Referrers_type'`. |
| 183 | * @param int $maximumRowsInDataTableLevelZero Maximum number of rows allowed in the top level DataTable. |
| 184 | * @param int $maximumRowsInSubDataTable Maximum number of rows allowed in each subtable. |
| 185 | * @param string|null $defaultColumnToSortByBeforeTruncation The name of the column to sort by before truncating a DataTable. |
| 186 | * If not set, and the table contains nb_visits or INDEX_NB_VISITS, we will |
| 187 | * sort by visits. |
| 188 | * @param array $columnsAggregationOperation Operations for aggregating columns, see {@link Row::sumRow()}. |
| 189 | * @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names |
| 190 | * when summed because they cannot be summed, eg, |
| 191 | * `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`. |
| 192 | * @param bool|array $countRowsRecursive if set to true, will calculate the recursive rows count for all record names |
| 193 | * which makes it slower. If you only need it for some records pass an array of |
| 194 | * recordNames that defines for which ones you need a recursive row count. |
| 195 | * @return array Returns the row counts of each aggregated report before truncation, eg, |
| 196 | * |
| 197 | * array( |
| 198 | * 'report1' => array('level0' => $report1->getRowsCount, |
| 199 | * 'recursive' => $report1->getRowsCountRecursive()), |
| 200 | * 'report2' => array('level0' => $report2->getRowsCount, |
| 201 | * 'recursive' => $report2->getRowsCountRecursive()), |
| 202 | * ... |
| 203 | * ) |
| 204 | * @api |
| 205 | */ |
| 206 | public function aggregateDataTableRecords($recordNames, |
| 207 | $maximumRowsInDataTableLevelZero = null, |
| 208 | $maximumRowsInSubDataTable = null, |
| 209 | $defaultColumnToSortByBeforeTruncation = null, |
| 210 | &$columnsAggregationOperation = null, |
| 211 | $columnsToRenameAfterAggregation = null, |
| 212 | $countRowsRecursive = true) |
| 213 | { |
| 214 | /** @var LoggerInterface $logger */ |
| 215 | $logger = StaticContainer::get(LoggerInterface::class); |
| 216 | |
| 217 | if (!is_array($recordNames)) { |
| 218 | $recordNames = array($recordNames); |
| 219 | } |
| 220 | |
| 221 | $archiveDescription = $this->params . ''; |
| 222 | |
| 223 | $nameToCount = array(); |
| 224 | foreach ($recordNames as $recordName) { |
| 225 | $latestUsedTableId = Manager::getInstance()->getMostRecentTableId(); |
| 226 | |
| 227 | $logger->debug("aggregating record {record} [archive = {archive}]", [ |
| 228 | 'record' => $recordName, |
| 229 | 'archive' => $archiveDescription, |
| 230 | ]); |
| 231 | |
| 232 | $table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation); |
| 233 | |
| 234 | $nameToCount[$recordName]['level0'] = $table->getRowsCount(); |
| 235 | if ($countRowsRecursive === true || (is_array($countRowsRecursive) && in_array($recordName, $countRowsRecursive))) { |
| 236 | $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive(); |
| 237 | } |
| 238 | |
| 239 | $columnToSortByBeforeTruncation = $defaultColumnToSortByBeforeTruncation; |
| 240 | if (empty($columnToSortByBeforeTruncation)) { |
| 241 | $columns = $table->getColumns(); |
| 242 | if (in_array(Metrics::INDEX_NB_VISITS, $columns)) { |
| 243 | $columnToSortByBeforeTruncation = Metrics::INDEX_NB_VISITS; |
| 244 | } else if (in_array('nb_visits', $columns)) { |
| 245 | $columnToSortByBeforeTruncation = 'nb_visits'; |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation); |
| 250 | Common::destroy($table); |
| 251 | $this->insertBlobRecord($recordName, $blob); |
| 252 | |
| 253 | unset($blob); |
| 254 | DataTable\Manager::getInstance()->deleteAll($latestUsedTableId); |
| 255 | } |
| 256 | |
| 257 | return $nameToCount; |
| 258 | } |
| 259 | |
| 260 | /** |
| 261 | * Aggregates one or more metrics for every subperiod of the current period and inserts the results |
| 262 | * as metrics for the current period. |
| 263 | * |
| 264 | * @param array|string $columns Array of metric names to aggregate. |
| 265 | * @param bool|string $operationToApply The operation to apply to the metric. Either `'sum'`, `'max'` or `'min'`. |
| 266 | * @return array|int Returns the array of aggregate values. If only one metric was aggregated, |
| 267 | * the aggregate value will be returned as is, not in an array. |
| 268 | * For example, if `array('nb_visits', 'nb_hits')` is supplied for `$columns`, |
| 269 | * |
| 270 | * array( |
| 271 | * 'nb_visits' => 3040, |
| 272 | * 'nb_hits' => 405 |
| 273 | * ) |
| 274 | * |
| 275 | * could be returned. If `array('nb_visits')` or `'nb_visits'` is used for `$columns`, |
| 276 | * then `3040` would be returned. |
| 277 | * @api |
| 278 | */ |
| 279 | public function aggregateNumericMetrics($columns, $operationToApply = false) |
| 280 | { |
| 281 | $metrics = $this->getAggregatedNumericMetrics($columns, $operationToApply); |
| 282 | |
| 283 | foreach ($metrics as $column => $value) { |
| 284 | $this->insertNumericRecord($column, $value); |
| 285 | } |
| 286 | // if asked for only one field to sum |
| 287 | if (count($metrics) === 1) { |
| 288 | return reset($metrics); |
| 289 | } |
| 290 | |
| 291 | // returns the array of records once summed |
| 292 | return $metrics; |
| 293 | } |
| 294 | |
| 295 | public function getNumberOfVisits() |
| 296 | { |
| 297 | if ($this->numberOfVisits === false) { |
| 298 | throw new Exception("visits should have been set here"); |
| 299 | } |
| 300 | return $this->numberOfVisits; |
| 301 | } |
| 302 | |
| 303 | public function getNumberOfVisitsConverted() |
| 304 | { |
| 305 | return $this->numberOfVisitsConverted; |
| 306 | } |
| 307 | |
| 308 | /** |
| 309 | * Caches multiple numeric records in the archive for this processor's site, period |
| 310 | * and segment. |
| 311 | * |
| 312 | * @param array $numericRecords A name-value mapping of numeric values that should be |
| 313 | * archived, eg, |
| 314 | * |
| 315 | * array('Referrers_distinctKeywords' => 23, 'Referrers_distinctCampaigns' => 234) |
| 316 | * @api |
| 317 | */ |
| 318 | public function insertNumericRecords($numericRecords) |
| 319 | { |
| 320 | foreach ($numericRecords as $name => $value) { |
| 321 | $this->insertNumericRecord($name, $value); |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | /** |
| 326 | * Caches a single numeric record in the archive for this processor's site, period and |
| 327 | * segment. |
| 328 | * |
| 329 | * Numeric values are not inserted if they equal `0`. |
| 330 | * |
| 331 | * @param string $name The name of the numeric value, eg, `'Referrers_distinctKeywords'`. |
| 332 | * @param float $value The numeric value. |
| 333 | * @api |
| 334 | */ |
| 335 | public function insertNumericRecord($name, $value) |
| 336 | { |
| 337 | $value = round($value ?? 0, 2); |
| 338 | $value = Common::forceDotAsSeparatorForDecimalPoint($value); |
| 339 | |
| 340 | $this->archiveWriter->insertRecord($name, $value); |
| 341 | } |
| 342 | |
| 343 | /** |
| 344 | * Caches one or more blob records in the archive for this processor's site, period |
| 345 | * and segment. |
| 346 | * |
| 347 | * @param string $name The name of the record, eg, 'Referrers_type'. |
| 348 | * @param string|array $values A blob string or an array of blob strings. If an array |
| 349 | * is used, the first element in the array will be inserted |
| 350 | * with the `$name` name. The others will be inserted with |
| 351 | * `$name . '_' . $index` as the record name (where $index is |
| 352 | * the index of the blob record in `$values`). |
| 353 | * @api |
| 354 | */ |
| 355 | public function insertBlobRecord($name, $values) |
| 356 | { |
| 357 | $this->archiveWriter->insertBlobRecord($name, $values); |
| 358 | } |
| 359 | |
| 360 | /** |
| 361 | * This method selects all DataTables that have the name $name over the period. |
| 362 | * All these DataTables are then added together, and the resulting DataTable is returned. |
| 363 | * |
| 364 | * @param string $name |
| 365 | * @param array $columnsAggregationOperation Operations for aggregating columns, @see Row::sumRow() |
| 366 | * @param array $columnsToRenameAfterAggregation columns in the array (old name, new name) to be renamed as the sum operation is not valid on them (eg. nb_uniq_visitors->sum_daily_nb_uniq_visitors) |
| 367 | * @return DataTable |
| 368 | */ |
| 369 | protected function aggregateDataTableRecord($name, $columnsAggregationOperation = null, $columnsToRenameAfterAggregation = null) |
| 370 | { |
| 371 | try { |
| 372 | ErrorHandler::pushFatalErrorBreadcrumb(__CLASS__, ['name' => $name]); |
| 373 | |
| 374 | $blobs = $this->getArchive()->querySingleBlob($name); |
| 375 | $dataTable = $this->getAggregatedDataTableMapFromBlobs($blobs, $columnsAggregationOperation, $columnsToRenameAfterAggregation, $name); |
| 376 | } finally { |
| 377 | ErrorHandler::popFatalErrorBreadcrumb(); |
| 378 | } |
| 379 | |
| 380 | return $dataTable; |
| 381 | } |
| 382 | |
| 383 | protected function getAggregatedDataTableMapFromBlobs(\Iterator $dataTableBlobs, $columnsAggregationOperation, $columnsToRenameAfterAggregation, $name) |
| 384 | { |
| 385 | // maps period & subtable ID in database to the Row instance in $result that subtable should be added to when encountered |
| 386 | // [$row['date1'].','.$row['date2']][$tableId] = $row in $result |
| 387 | /** @var Row[][] */ |
| 388 | $tableIdToResultRowMapping = []; |
| 389 | |
| 390 | $result = new DataTable(); |
| 391 | |
| 392 | if (!empty($columnsAggregationOperation)) { |
| 393 | $result->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation); |
| 394 | } |
| 395 | |
| 396 | foreach ($dataTableBlobs as $archiveDataRow) { |
| 397 | $period = $archiveDataRow['date1'] . ',' . $archiveDataRow['date2']; |
| 398 | $tableId = $archiveDataRow['name'] == $name ? null : $this->getSubtableIdFromBlobName($archiveDataRow['name']); |
| 399 | |
| 400 | $blobTable = DataTable::fromSerializedArray($archiveDataRow['value']); |
| 401 | |
| 402 | // see https://github.com/piwik/piwik/issues/4377 |
| 403 | $blobTable->filter(function ($table) use ($columnsToRenameAfterAggregation, $name) { |
| 404 | if ($this->areColumnsNotAlreadyRenamed($table)) { |
| 405 | /** |
| 406 | * This makes archiving and range dates a lot faster. Imagine we archive a week, then we will |
| 407 | * rename all columns of each 7 day archives. Afterwards we know the columns will be replaced in a |
| 408 | * week archive. When generating month archives, which uses mostly week archives, we do not have |
| 409 | * to replace those columns for the week archives again since we can be sure they were already |
| 410 | * replaced. Same when aggregating year and range archives. This can save up 10% or more when |
| 411 | * aggregating Month, Year and Range archives. |
| 412 | */ |
| 413 | $this->renameColumnsAfterAggregation($table, $columnsToRenameAfterAggregation); |
| 414 | } |
| 415 | }); |
| 416 | |
| 417 | $tableToAddTo = null; |
| 418 | if ($tableId === null) { |
| 419 | $tableToAddTo = $result; |
| 420 | } else if (empty($tableIdToResultRowMapping[$period][$tableId])) { // sanity check |
| 421 | StaticContainer::get(LoggerInterface::class)->info( |
| 422 | 'Unexpected state when aggregating DataTable, unknown period/table ID combination encountered: {period} - {tableId}.' |
| 423 | . ' This either means the SQL to order blobs is behaving incorrectly or the blob data is corrupt in some way.', |
| 424 | [ |
| 425 | 'period' => $period, |
| 426 | 'tableId' => $tableId, |
| 427 | ] |
| 428 | ); |
| 429 | continue; |
| 430 | } else { |
| 431 | $rowToAddTo = $tableIdToResultRowMapping[$period][$tableId]; |
| 432 | |
| 433 | if (!$rowToAddTo->getIdSubDataTable()) { |
| 434 | $newTable = new DataTable(); |
| 435 | $newTable->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation); |
| 436 | $rowToAddTo->setSubtable($newTable); |
| 437 | } |
| 438 | |
| 439 | $tableToAddTo = $rowToAddTo->getSubtable(); |
| 440 | } |
| 441 | |
| 442 | $tableToAddTo->addDataTable($blobTable); |
| 443 | |
| 444 | // add subtable IDs for $blobTableRow to $tableIdToResultRowMapping |
| 445 | foreach ($blobTable->getRows() as $blobTableRow) { |
| 446 | $label = $blobTableRow->getColumn('label'); |
| 447 | $subtableId = $blobTableRow->getIdSubDataTable(); |
| 448 | if (empty($subtableId)) { |
| 449 | continue; |
| 450 | } |
| 451 | |
| 452 | $rowToAddTo = $tableToAddTo->getRowFromLabel($label); |
| 453 | $tableIdToResultRowMapping[$period][$subtableId] = $rowToAddTo; |
| 454 | } |
| 455 | |
| 456 | Common::destroy($blobTable); |
| 457 | unset($blobTable); |
| 458 | } |
| 459 | |
| 460 | return $result; |
| 461 | } |
| 462 | |
| 463 | private function getSubtableIdFromBlobName($recordName) |
| 464 | { |
| 465 | $parts = explode('_', $recordName); |
| 466 | $id = end($parts); |
| 467 | |
| 468 | if (is_numeric($id)) { |
| 469 | return $id; |
| 470 | } |
| 471 | |
| 472 | return null; |
| 473 | } |
| 474 | |
| 475 | /** |
| 476 | * Note: public only for use in closure in PHP 5.3. |
| 477 | * |
| 478 | * @param $table |
| 479 | * @return \Piwik\Period |
| 480 | */ |
| 481 | public function areColumnsNotAlreadyRenamed($table) |
| 482 | { |
| 483 | $period = $table->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX); |
| 484 | |
| 485 | return !$period || $period->getLabel() === 'day'; |
| 486 | } |
| 487 | |
| 488 | protected function getOperationForColumns($columns, $defaultOperation) |
| 489 | { |
| 490 | $operationForColumn = array(); |
| 491 | foreach ($columns as $name) { |
| 492 | $operation = $defaultOperation; |
| 493 | if (empty($operation)) { |
| 494 | $operation = $this->guessOperationForColumn($name); |
| 495 | } |
| 496 | $operationForColumn[$name] = $operation; |
| 497 | } |
| 498 | return $operationForColumn; |
| 499 | } |
| 500 | |
| 501 | protected function enrichWithUniqueVisitorsMetric(Row $row) |
| 502 | { |
| 503 | if ($row->getColumn('nb_uniq_visitors') === false |
| 504 | && $row->getColumn('nb_users') === false |
| 505 | ) { |
| 506 | return; |
| 507 | } |
| 508 | |
| 509 | $periodLabel = $this->getParams()->getPeriod()->getLabel(); |
| 510 | |
| 511 | if (!SettingsPiwik::isUniqueVisitorsEnabled($periodLabel)) { |
| 512 | $row->deleteColumn('nb_uniq_visitors'); |
| 513 | $row->deleteColumn('nb_users'); |
| 514 | return; |
| 515 | } |
| 516 | |
| 517 | $sites = $this->getIdSitesToComputeNbUniques(); |
| 518 | |
| 519 | if (count($sites) > 1 && Rules::shouldSkipUniqueVisitorsCalculationForMultipleSites()) { |
| 520 | if ($periodLabel != 'day') { |
| 521 | // for day we still keep the aggregated metric but for other periods we remove it as it becomes to |
| 522 | // inaccurate |
| 523 | $row->deleteColumn('nb_uniq_visitors'); |
| 524 | $row->deleteColumn('nb_users'); |
| 525 | } |
| 526 | return; |
| 527 | } |
| 528 | |
| 529 | if (empty($sites)) { |
| 530 | // a plugin disabled running below query by removing all sites. |
| 531 | $row->deleteColumn('nb_uniq_visitors'); |
| 532 | $row->deleteColumn('nb_users'); |
| 533 | return; |
| 534 | } |
| 535 | |
| 536 | if (count($sites) === 1) { |
| 537 | $uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_VISITORS; |
| 538 | } else { |
| 539 | if (!SettingsPiwik::isSameFingerprintAcrossWebsites()) { |
| 540 | throw new Exception("Processing unique visitors across websites is enabled for this instance, |
| 541 | but to process this metric you must first set enable_fingerprinting_across_websites=1 |
| 542 | in the config file, under the [Tracker] section."); |
| 543 | } |
| 544 | $uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_FINGERPRINTS; |
| 545 | } |
| 546 | |
| 547 | $metrics = array( |
| 548 | Metrics::INDEX_NB_USERS, |
| 549 | $uniqueVisitorsMetric |
| 550 | ); |
| 551 | |
| 552 | $uniques = $this->computeNbUniques($metrics, $sites); |
| 553 | |
| 554 | // see edge case as described in https://github.com/piwik/piwik/issues/9357 where uniq_visitors might be higher |
| 555 | // than visits because we archive / process it after nb_visits. Between archiving nb_visits and nb_uniq_visitors |
| 556 | // there could have been a new visit leading to a higher nb_unique_visitors than nb_visits which is not possible |
| 557 | // by definition. In this case we simply use the visits metric instead of unique visitors metric. |
| 558 | $visits = $row->getColumn('nb_visits'); |
| 559 | if ($visits !== false && $uniques[$uniqueVisitorsMetric] !== false) { |
| 560 | $uniques[$uniqueVisitorsMetric] = min($uniques[$uniqueVisitorsMetric], $visits); |
| 561 | } |
| 562 | |
| 563 | $row->setColumn('nb_uniq_visitors', $uniques[$uniqueVisitorsMetric]); |
| 564 | $row->setColumn('nb_users', $uniques[Metrics::INDEX_NB_USERS]); |
| 565 | } |
| 566 | |
| 567 | protected function guessOperationForColumn($column) |
| 568 | { |
| 569 | if (strpos($column, 'max_') === 0) { |
| 570 | return 'max'; |
| 571 | } |
| 572 | if (strpos($column, 'min_') === 0) { |
| 573 | return 'min'; |
| 574 | } |
| 575 | return 'sum'; |
| 576 | } |
| 577 | |
| 578 | private function getIdSitesToComputeNbUniques() |
| 579 | { |
| 580 | $params = $this->getParams(); |
| 581 | $sites = array($params->getSite()->getId()); |
| 582 | |
| 583 | /** |
| 584 | * Triggered to change which site ids should be looked at when processing unique visitors and users. |
| 585 | * |
| 586 | * @param array &$idSites An array with one idSite. This site is being archived currently. To cancel the query |
| 587 | * you can change this value to an empty array. To include other sites in the query you |
| 588 | * can add more idSites to this list of idSites. |
| 589 | * @param Period $period The period that is being requested to be archived. |
| 590 | * @param Segment $segment The segment that is request to be archived. |
| 591 | */ |
| 592 | Piwik::postEvent('ArchiveProcessor.ComputeNbUniques.getIdSites', array(&$sites, $params->getPeriod(), $params->getSegment())); |
| 593 | |
| 594 | return $sites; |
| 595 | } |
| 596 | |
| 597 | /** |
| 598 | * Processes number of unique visitors for the given period |
| 599 | * |
| 600 | * This is the only Period metric (ie. week/month/year/range) that we process from the logs directly, |
| 601 | * since unique visitors cannot be summed like other metrics. |
| 602 | * |
| 603 | * @param array $metrics Metrics Ids for which to aggregates count of values |
| 604 | * @param int[] $sites A list of idSites that should be included |
| 605 | * @return array|null An array of metrics, where the key is metricid and the value is the metric value or null if |
| 606 | * the query was cancelled and not executed. |
| 607 | */ |
| 608 | protected function computeNbUniques($metrics, $sites) |
| 609 | { |
| 610 | $logAggregator = $this->getLogAggregator(); |
| 611 | $sitesBackup = $logAggregator->getSites(); |
| 612 | |
| 613 | $logAggregator->setSites($sites); |
| 614 | try { |
| 615 | $query = $logAggregator->queryVisitsByDimension(array(), false, array(), $metrics); |
| 616 | } finally { |
| 617 | $logAggregator->setSites($sitesBackup); |
| 618 | } |
| 619 | $data = $query->fetch(); |
| 620 | return $data; |
| 621 | } |
| 622 | |
| 623 | /** |
| 624 | * If the DataTable is a Map, sums all DataTable in the map and return the DataTable. |
| 625 | * |
| 626 | * |
| 627 | * @param $data DataTable|DataTable\Map |
| 628 | * @param $columnsToRenameAfterAggregation array |
| 629 | * @return DataTable |
| 630 | */ |
| 631 | protected function getAggregatedDataTableMap($data, $columnsAggregationOperation) |
| 632 | { |
| 633 | $table = new DataTable(); |
| 634 | |
| 635 | if (!empty($columnsAggregationOperation)) { |
| 636 | $table->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation); |
| 637 | } |
| 638 | |
| 639 | if ($data instanceof DataTable\Map) { |
| 640 | // as $date => $tableToSum |
| 641 | $this->aggregatedDataTableMapsAsOne($data, $table); |
| 642 | } else { |
| 643 | $table->addDataTable($data); |
| 644 | } |
| 645 | |
| 646 | return $table; |
| 647 | } |
| 648 | |
| 649 | /** |
| 650 | * Aggregates the DataTable\Map into the destination $aggregated |
| 651 | * @param $map |
| 652 | * @param $aggregated |
| 653 | */ |
| 654 | protected function aggregatedDataTableMapsAsOne(Map $map, DataTable $aggregated) |
| 655 | { |
| 656 | foreach ($map->getDataTables() as $tableToAggregate) { |
| 657 | if ($tableToAggregate instanceof Map) { |
| 658 | $this->aggregatedDataTableMapsAsOne($tableToAggregate, $aggregated); |
| 659 | } else { |
| 660 | $aggregated->addDataTable($tableToAggregate); |
| 661 | } |
| 662 | } |
| 663 | } |
| 664 | |
| 665 | /** |
| 666 | * Note: public only for use in closure in PHP 5.3. |
| 667 | */ |
| 668 | public function renameColumnsAfterAggregation(DataTable $table, $columnsToRenameAfterAggregation = null) |
| 669 | { |
| 670 | // Rename columns after aggregation |
| 671 | if (is_null($columnsToRenameAfterAggregation)) { |
| 672 | $columnsToRenameAfterAggregation = self::$columnsToRenameAfterAggregation; |
| 673 | } |
| 674 | |
| 675 | if (empty($columnsToRenameAfterAggregation)) { |
| 676 | return; |
| 677 | } |
| 678 | |
| 679 | foreach ($table->getRows() as $row) { |
| 680 | foreach ($columnsToRenameAfterAggregation as $oldName => $newName) { |
| 681 | $row->renameColumn($oldName, $newName); |
| 682 | } |
| 683 | |
| 684 | $subTable = $row->getSubtable(); |
| 685 | if ($subTable) { |
| 686 | $this->renameColumnsAfterAggregation($subTable, $columnsToRenameAfterAggregation); |
| 687 | } |
| 688 | } |
| 689 | } |
| 690 | |
| 691 | protected function getAggregatedNumericMetrics($columns, $operationToApply) |
| 692 | { |
| 693 | if (!is_array($columns)) { |
| 694 | $columns = array($columns); |
| 695 | } |
| 696 | |
| 697 | $operationForColumn = $this->getOperationForColumns($columns, $operationToApply); |
| 698 | |
| 699 | $dataTable = $this->getArchive()->getDataTableFromNumeric($columns); |
| 700 | |
| 701 | $results = $this->getAggregatedDataTableMap($dataTable, $operationForColumn); |
| 702 | if ($results->getRowsCount() > 1) { |
| 703 | throw new Exception("A DataTable is an unexpected state:" . var_export($results, true)); |
| 704 | } |
| 705 | |
| 706 | $rowMetrics = $results->getFirstRow(); |
| 707 | if ($rowMetrics === false) { |
| 708 | $rowMetrics = new Row; |
| 709 | } |
| 710 | $this->enrichWithUniqueVisitorsMetric($rowMetrics); |
| 711 | $this->renameColumnsAfterAggregation($results, self::$columnsToRenameAfterAggregation); |
| 712 | |
| 713 | $metrics = $rowMetrics->getColumns(); |
| 714 | |
| 715 | foreach ($columns as $name) { |
| 716 | if (!isset($metrics[$name])) { |
| 717 | $metrics[$name] = 0; |
| 718 | } |
| 719 | } |
| 720 | |
| 721 | return $metrics; |
| 722 | } |
| 723 | |
| 724 | /** |
| 725 | * Initiate archiving for a plugin during an ongoing archiving. The plugin can be another |
| 726 | * plugin or the same plugin. |
| 727 | * |
| 728 | * This method should be called during archiving when one plugin uses the report of another |
| 729 | * plugin with a segment. It will ensure reports for that segment & plugin will be archived |
| 730 | * without initiating archiving for every plugin with that segment (which would be a performance |
| 731 | * killer). |
| 732 | * |
| 733 | * @param string $plugin |
| 734 | * @param string $segment |
| 735 | */ |
| 736 | public function processDependentArchive($plugin, $segment) |
| 737 | { |
| 738 | if (!self::$isRootArchivingRequest) { // prevent all recursion |
| 739 | return; |
| 740 | } |
| 741 | |
| 742 | $params = $this->getParams(); |
| 743 | $idSites = [$params->getSite()->getId()]; |
| 744 | |
| 745 | // important to use the original segment string when combining. As the API itself would combine the original string. |
| 746 | // this prevents a bug where the API would use the segment |
| 747 | // userId!@%2540matomo.org;userId!=hello%2540matomo.org;visitorType==new |
| 748 | // vs here we would use |
| 749 | // userId!@%40matomo.org;userId!=hello%40matomo.org;visitorType==new |
| 750 | // thus these would result in different segment hashes and therefore the reports would either show 0 or archive the data twice |
| 751 | $originSegmentString = $params->getSegment()->getOriginalString(); |
| 752 | $newSegment = Segment::combine($originSegmentString, SegmentExpression::AND_DELIMITER, $segment); |
| 753 | if (!empty($originSegmentString) && $newSegment === $segment && $params->getRequestedPlugin() === $plugin) { // being processed now |
| 754 | return; |
| 755 | } |
| 756 | |
| 757 | $newSegment = new Segment($newSegment, $idSites, $params->getDateTimeStart(), $params->getDateTimeEnd()); |
| 758 | if (ArchiveProcessor\Rules::isSegmentPreProcessed($idSites, $newSegment)) { |
| 759 | // will be processed anyway |
| 760 | return; |
| 761 | } |
| 762 | |
| 763 | self::$isRootArchivingRequest = false; |
| 764 | try { |
| 765 | $parameters = new ArchiveProcessor\Parameters($params->getSite(), $params->getPeriod(), $newSegment); |
| 766 | $parameters->onlyArchiveRequestedPlugin(); |
| 767 | |
| 768 | $archiveLoader = new ArchiveProcessor\Loader($parameters); |
| 769 | $archiveLoader->prepareArchive($plugin); |
| 770 | } finally { |
| 771 | self::$isRootArchivingRequest = true; |
| 772 | } |
| 773 | } |
| 774 | |
| 775 | public function getArchiveWriter() |
| 776 | { |
| 777 | return $this->archiveWriter; |
| 778 | } |
| 779 | } |
| 780 |