API
3 years ago
Access
3 years ago
Application
4 years ago
Archive
3 years ago
ArchiveProcessor
3 years ago
Archiver
5 years ago
AssetManager
3 years ago
Auth
3 years ago
Category
5 years ago
Changes
4 years ago
CliMulti
4 years ago
Columns
3 years ago
Concurrency
3 years ago
Config
4 years ago
Container
4 years ago
CronArchive
3 years ago
DataAccess
3 years ago
DataFiles
5 years ago
DataTable
3 years ago
Db
3 years ago
DeviceDetector
3 years ago
Email
5 years ago
Exception
3 years ago
Http
4 years ago
Intl
4 years ago
Mail
3 years ago
Measurable
5 years ago
Menu
3 years ago
Metrics
4 years ago
Notification
4 years ago
Period
4 years ago
Plugin
3 years ago
ProfessionalServices
4 years ago
Report
5 years ago
ReportRenderer
3 years ago
Scheduler
4 years ago
Segment
3 years ago
Session
4 years ago
Settings
3 years ago
Tracker
3 years ago
Translation
4 years ago
UpdateCheck
5 years ago
Updater
3 years ago
Updates
3 years ago
Validators
4 years ago
View
4 years ago
ViewDataTable
3 years ago
Visualization
4 years ago
Widget
5 years ago
.htaccess
6 years ago
Access.php
4 years ago
Archive.php
4 years ago
ArchiveProcessor.php
4 years ago
AssetManager.php
4 years ago
Auth.php
5 years ago
AuthResult.php
5 years ago
BaseFactory.php
5 years ago
Cache.php
5 years ago
CacheId.php
5 years ago
CliMulti.php
4 years ago
Common.php
3 years ago
Config.php
3 years ago
Console.php
3 years ago
Context.php
5 years ago
Cookie.php
3 years ago
CronArchive.php
3 years ago
DataArray.php
4 years ago
DataTable.php
3 years ago
Date.php
4 years ago
Db.php
4 years ago
DbHelper.php
3 years ago
Development.php
5 years ago
ErrorHandler.php
5 years ago
EventDispatcher.php
3 years ago
ExceptionHandler.php
3 years ago
FileIntegrity.php
5 years ago
Filechecks.php
4 years ago
Filesystem.php
3 years ago
FrontController.php
3 years ago
Http.php
3 years ago
IP.php
4 years ago
Log.php
4 years ago
LogDeleter.php
4 years ago
Mail.php
4 years ago
Metrics.php
3 years ago
NoAccessException.php
5 years ago
Nonce.php
3 years ago
Notification.php
5 years ago
NumberFormatter.php
4 years ago
Option.php
4 years ago
Period.php
5 years ago
Piwik.php
3 years ago
Plugin.php
4 years ago
Profiler.php
4 years ago
ProxyHeaders.php
5 years ago
ProxyHttp.php
3 years ago
QuickForm2.php
5 years ago
RankingQuery.php
3 years ago
ReportRenderer.php
4 years ago
Segment.php
3 years ago
Sequence.php
5 years ago
Session.php
3 years ago
SettingsPiwik.php
3 years ago
SettingsServer.php
5 years ago
Singleton.php
5 years ago
Site.php
3 years ago
SiteContentDetector.php
3 years ago
SupportedBrowser.php
3 years ago
TCPDF.php
5 years ago
Theme.php
5 years ago
Timer.php
5 years ago
Tracker.php
4 years ago
Twig.php
4 years ago
Unzip.php
5 years ago
UpdateCheck.php
5 years ago
Updater.php
4 years ago
UpdaterErrorException.php
5 years ago
Updates.php
5 years ago
Url.php
3 years ago
UrlHelper.php
3 years ago
Version.php
3 years ago
View.php
3 years ago
bootstrap.php
3 years ago
dispatch.php
5 years ago
testMinimumPhpVersion.php
3 years ago
ArchiveProcessor.php
707 lines
| 1 | <?php |
| 2 | /** |
| 3 | * Matomo - free/libre analytics platform |
| 4 | * |
| 5 | * @link https://matomo.org |
| 6 | * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later |
| 7 | * |
| 8 | */ |
| 9 | namespace Piwik; |
| 10 | |
| 11 | use Exception; |
| 12 | use Piwik\Archive\DataCollection; |
| 13 | use Piwik\Archive\DataTableFactory; |
| 14 | use Piwik\ArchiveProcessor\Parameters; |
| 15 | use Piwik\ArchiveProcessor\Rules; |
| 16 | use Piwik\Container\StaticContainer; |
| 17 | use Piwik\DataAccess\ArchiveWriter; |
| 18 | use Piwik\DataAccess\LogAggregator; |
| 19 | use Piwik\DataTable\Manager; |
| 20 | use Piwik\DataTable\Map; |
| 21 | use Piwik\DataTable\Row; |
| 22 | use Piwik\Segment\SegmentExpression; |
| 23 | use Psr\Log\LoggerInterface; |
| 24 | |
| 25 | /** |
| 26 | * Used by {@link Piwik\Plugin\Archiver} instances to insert and aggregate archive data. |
| 27 | * |
| 28 | * ### See also |
| 29 | * |
| 30 | * - **{@link Piwik\Plugin\Archiver}** - to learn how plugins should implement their own analytics |
| 31 | * aggregation logic. |
| 32 | * - **{@link Piwik\DataAccess\LogAggregator}** - to learn how plugins can perform data aggregation |
| 33 | * across Piwik's log tables. |
| 34 | * |
| 35 | * ### Examples |
| 36 | * |
| 37 | * **Inserting numeric data** |
| 38 | * |
| 39 | * // function in an Archiver descendant |
| 40 | * public function aggregateDayReport() |
| 41 | * { |
| 42 | * $archiveProcessor = $this->getProcessor(); |
| 43 | * |
| 44 | * $myFancyMetric = // ... calculate the metric value ... |
| 45 | * $archiveProcessor->insertNumericRecord('MyPlugin_myFancyMetric', $myFancyMetric); |
| 46 | * } |
| 47 | * |
| 48 | * **Inserting serialized DataTables** |
| 49 | * |
| 50 | * // function in an Archiver descendant |
| 51 | * public function aggregateDayReport() |
| 52 | * { |
| 53 | * $archiveProcessor = $this->getProcessor(); |
| 54 | * |
| 55 | * $maxRowsInTable = Config::getInstance()->General['datatable_archiving_maximum_rows_standard'];j |
| 56 | * |
| 57 | * $dataTable = // ... build by aggregating visits ... |
| 58 | * $serializedData = $dataTable->getSerialized($maxRowsInTable, $maxRowsInSubtable = $maxRowsInTable, |
| 59 | * $columnToSortBy = Metrics::INDEX_NB_VISITS); |
| 60 | * |
| 61 | * $archiveProcessor->insertBlobRecords('MyPlugin_myFancyReport', $serializedData); |
| 62 | * } |
| 63 | * |
| 64 | * **Aggregating archive data** |
| 65 | * |
| 66 | * // function in Archiver descendant |
| 67 | * public function aggregateMultipleReports() |
| 68 | * { |
| 69 | * $archiveProcessor = $this->getProcessor(); |
| 70 | * |
| 71 | * // aggregate a metric |
| 72 | * $archiveProcessor->aggregateNumericMetrics('MyPlugin_myFancyMetric'); |
| 73 | * $archiveProcessor->aggregateNumericMetrics('MyPlugin_mySuperFancyMetric', 'max'); |
| 74 | * |
| 75 | * // aggregate a report |
| 76 | * $archiveProcessor->aggregateDataTableRecords('MyPlugin_myFancyReport'); |
| 77 | * } |
| 78 | * |
| 79 | */ |
| 80 | class ArchiveProcessor |
| 81 | { |
| 82 | /** |
| 83 | * @var \Piwik\DataAccess\ArchiveWriter |
| 84 | */ |
| 85 | private $archiveWriter; |
| 86 | |
| 87 | /** |
| 88 | * @var \Piwik\DataAccess\LogAggregator |
| 89 | */ |
| 90 | private $logAggregator; |
| 91 | |
| 92 | /** |
| 93 | * @var Archive |
| 94 | */ |
| 95 | public $archive = null; |
| 96 | |
| 97 | /** |
| 98 | * @var Parameters |
| 99 | */ |
| 100 | private $params; |
| 101 | |
| 102 | /** |
| 103 | * @var int |
| 104 | */ |
| 105 | private $numberOfVisits = false; |
| 106 | |
| 107 | private $numberOfVisitsConverted = false; |
| 108 | |
| 109 | public function __construct(Parameters $params, ArchiveWriter $archiveWriter, LogAggregator $logAggregator) |
| 110 | { |
| 111 | $this->params = $params; |
| 112 | $this->logAggregator = $logAggregator; |
| 113 | $this->archiveWriter = $archiveWriter; |
| 114 | } |
| 115 | |
| 116 | protected function getArchive() |
| 117 | { |
| 118 | if (empty($this->archive)) { |
| 119 | $subPeriods = $this->params->getSubPeriods(); |
| 120 | $idSites = $this->params->getIdSites(); |
| 121 | $this->archive = Archive::factory($this->params->getSegment(), $subPeriods, $idSites); |
| 122 | |
| 123 | /** |
| 124 | * @internal |
| 125 | */ |
| 126 | Piwik::postEvent('ArchiveProcessor.getArchive', [$this->archive]); |
| 127 | } |
| 128 | |
| 129 | return $this->archive; |
| 130 | } |
| 131 | |
| 132 | public function setNumberOfVisits($visits, $visitsConverted) |
| 133 | { |
| 134 | $this->numberOfVisits = $visits; |
| 135 | $this->numberOfVisitsConverted = $visitsConverted; |
| 136 | } |
| 137 | |
| 138 | /** |
| 139 | * Returns the {@link Parameters} object containing the site, period and segment we're archiving |
| 140 | * data for. |
| 141 | * |
| 142 | * @return Parameters |
| 143 | * @api |
| 144 | */ |
| 145 | public function getParams() |
| 146 | { |
| 147 | return $this->params; |
| 148 | } |
| 149 | |
| 150 | /** |
| 151 | * Returns a `{@link Piwik\DataAccess\LogAggregator}` instance for the site, period and segment this |
| 152 | * ArchiveProcessor will insert archive data for. |
| 153 | * |
| 154 | * @return LogAggregator |
| 155 | * @api |
| 156 | */ |
| 157 | public function getLogAggregator() |
| 158 | { |
| 159 | return $this->logAggregator; |
| 160 | } |
| 161 | |
| 162 | /** |
| 163 | * Array of (column name before => column name renamed) of the columns for which sum operation is invalid. |
| 164 | * These columns will be renamed as per this mapping. |
| 165 | * @var array |
| 166 | */ |
| 167 | protected static $columnsToRenameAfterAggregation = array( |
| 168 | Metrics::INDEX_NB_UNIQ_VISITORS => Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS, |
| 169 | Metrics::INDEX_NB_USERS => Metrics::INDEX_SUM_DAILY_NB_USERS, |
| 170 | ); |
| 171 | |
| 172 | /** |
| 173 | * Sums records for every subperiod of the current period and inserts the result as the record |
| 174 | * for this period. |
| 175 | * |
| 176 | * DataTables are summed recursively so subtables will be summed as well. |
| 177 | * |
| 178 | * @param string|array $recordNames Name(s) of the report we are aggregating, eg, `'Referrers_type'`. |
| 179 | * @param int $maximumRowsInDataTableLevelZero Maximum number of rows allowed in the top level DataTable. |
| 180 | * @param int $maximumRowsInSubDataTable Maximum number of rows allowed in each subtable. |
| 181 | * @param string $columnToSortByBeforeTruncation The name of the column to sort by before truncating a DataTable. |
| 182 | * @param array $columnsAggregationOperation Operations for aggregating columns, see {@link Row::sumRow()}. |
| 183 | * @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names |
| 184 | * when summed because they cannot be summed, eg, |
| 185 | * `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`. |
| 186 | * @param bool|array $countRowsRecursive if set to true, will calculate the recursive rows count for all record names |
| 187 | * which makes it slower. If you only need it for some records pass an array of |
| 188 | * recordNames that defines for which ones you need a recursive row count. |
| 189 | * @return array Returns the row counts of each aggregated report before truncation, eg, |
| 190 | * |
| 191 | * array( |
| 192 | * 'report1' => array('level0' => $report1->getRowsCount, |
| 193 | * 'recursive' => $report1->getRowsCountRecursive()), |
| 194 | * 'report2' => array('level0' => $report2->getRowsCount, |
| 195 | * 'recursive' => $report2->getRowsCountRecursive()), |
| 196 | * ... |
| 197 | * ) |
| 198 | * @api |
| 199 | */ |
| 200 | public function aggregateDataTableRecords($recordNames, |
| 201 | $maximumRowsInDataTableLevelZero = null, |
| 202 | $maximumRowsInSubDataTable = null, |
| 203 | $columnToSortByBeforeTruncation = null, |
| 204 | &$columnsAggregationOperation = null, |
| 205 | $columnsToRenameAfterAggregation = null, |
| 206 | $countRowsRecursive = true) |
| 207 | { |
| 208 | /** @var LoggerInterface $logger */ |
| 209 | $logger = StaticContainer::get(LoggerInterface::class); |
| 210 | |
| 211 | if (!is_array($recordNames)) { |
| 212 | $recordNames = array($recordNames); |
| 213 | } |
| 214 | |
| 215 | $archiveDescription = $this->params . ''; |
| 216 | |
| 217 | $nameToCount = array(); |
| 218 | foreach ($recordNames as $recordName) { |
| 219 | $latestUsedTableId = Manager::getInstance()->getMostRecentTableId(); |
| 220 | |
| 221 | $logger->debug("aggregating record {record} [archive = {archive}]", [ |
| 222 | 'record' => $recordName, |
| 223 | 'archive' => $archiveDescription, |
| 224 | ]); |
| 225 | |
| 226 | $table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation); |
| 227 | |
| 228 | $nameToCount[$recordName]['level0'] = $table->getRowsCount(); |
| 229 | if ($countRowsRecursive === true || (is_array($countRowsRecursive) && in_array($recordName, $countRowsRecursive))) { |
| 230 | $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive(); |
| 231 | } |
| 232 | |
| 233 | $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation); |
| 234 | Common::destroy($table); |
| 235 | $this->insertBlobRecord($recordName, $blob); |
| 236 | |
| 237 | unset($blob); |
| 238 | DataTable\Manager::getInstance()->deleteAll($latestUsedTableId); |
| 239 | } |
| 240 | |
| 241 | return $nameToCount; |
| 242 | } |
| 243 | |
| 244 | /** |
| 245 | * Aggregates one or more metrics for every subperiod of the current period and inserts the results |
| 246 | * as metrics for the current period. |
| 247 | * |
| 248 | * @param array|string $columns Array of metric names to aggregate. |
| 249 | * @param bool|string $operationToApply The operation to apply to the metric. Either `'sum'`, `'max'` or `'min'`. |
| 250 | * @return array|int Returns the array of aggregate values. If only one metric was aggregated, |
| 251 | * the aggregate value will be returned as is, not in an array. |
| 252 | * For example, if `array('nb_visits', 'nb_hits')` is supplied for `$columns`, |
| 253 | * |
| 254 | * array( |
| 255 | * 'nb_visits' => 3040, |
| 256 | * 'nb_hits' => 405 |
| 257 | * ) |
| 258 | * |
| 259 | * could be returned. If `array('nb_visits')` or `'nb_visits'` is used for `$columns`, |
| 260 | * then `3040` would be returned. |
| 261 | * @api |
| 262 | */ |
| 263 | public function aggregateNumericMetrics($columns, $operationToApply = false) |
| 264 | { |
| 265 | $metrics = $this->getAggregatedNumericMetrics($columns, $operationToApply); |
| 266 | |
| 267 | foreach ($metrics as $column => $value) { |
| 268 | $this->insertNumericRecord($column, $value); |
| 269 | } |
| 270 | // if asked for only one field to sum |
| 271 | if (count($metrics) === 1) { |
| 272 | return reset($metrics); |
| 273 | } |
| 274 | |
| 275 | // returns the array of records once summed |
| 276 | return $metrics; |
| 277 | } |
| 278 | |
| 279 | public function getNumberOfVisits() |
| 280 | { |
| 281 | if ($this->numberOfVisits === false) { |
| 282 | throw new Exception("visits should have been set here"); |
| 283 | } |
| 284 | return $this->numberOfVisits; |
| 285 | } |
| 286 | |
| 287 | public function getNumberOfVisitsConverted() |
| 288 | { |
| 289 | return $this->numberOfVisitsConverted; |
| 290 | } |
| 291 | |
| 292 | /** |
| 293 | * Caches multiple numeric records in the archive for this processor's site, period |
| 294 | * and segment. |
| 295 | * |
| 296 | * @param array $numericRecords A name-value mapping of numeric values that should be |
| 297 | * archived, eg, |
| 298 | * |
| 299 | * array('Referrers_distinctKeywords' => 23, 'Referrers_distinctCampaigns' => 234) |
| 300 | * @api |
| 301 | */ |
| 302 | public function insertNumericRecords($numericRecords) |
| 303 | { |
| 304 | foreach ($numericRecords as $name => $value) { |
| 305 | $this->insertNumericRecord($name, $value); |
| 306 | } |
| 307 | } |
| 308 | |
| 309 | /** |
| 310 | * Caches a single numeric record in the archive for this processor's site, period and |
| 311 | * segment. |
| 312 | * |
| 313 | * Numeric values are not inserted if they equal `0`. |
| 314 | * |
| 315 | * @param string $name The name of the numeric value, eg, `'Referrers_distinctKeywords'`. |
| 316 | * @param float $value The numeric value. |
| 317 | * @api |
| 318 | */ |
| 319 | public function insertNumericRecord($name, $value) |
| 320 | { |
| 321 | $value = round($value ?? 0, 2); |
| 322 | $value = Common::forceDotAsSeparatorForDecimalPoint($value); |
| 323 | |
| 324 | $this->archiveWriter->insertRecord($name, $value); |
| 325 | } |
| 326 | |
| 327 | /** |
| 328 | * Caches one or more blob records in the archive for this processor's site, period |
| 329 | * and segment. |
| 330 | * |
| 331 | * @param string $name The name of the record, eg, 'Referrers_type'. |
| 332 | * @param string|array $values A blob string or an array of blob strings. If an array |
| 333 | * is used, the first element in the array will be inserted |
| 334 | * with the `$name` name. The others will be inserted with |
| 335 | * `$name . '_' . $index` as the record name (where $index is |
| 336 | * the index of the blob record in `$values`). |
| 337 | * @api |
| 338 | */ |
| 339 | public function insertBlobRecord($name, $values) |
| 340 | { |
| 341 | $this->archiveWriter->insertBlobRecord($name, $values); |
| 342 | } |
| 343 | |
| 344 | /** |
| 345 | * This method selects all DataTables that have the name $name over the period. |
| 346 | * All these DataTables are then added together, and the resulting DataTable is returned. |
| 347 | * |
| 348 | * @param string $name |
| 349 | * @param array $columnsAggregationOperation Operations for aggregating columns, @see Row::sumRow() |
| 350 | * @param array $columnsToRenameAfterAggregation columns in the array (old name, new name) to be renamed as the sum operation is not valid on them (eg. nb_uniq_visitors->sum_daily_nb_uniq_visitors) |
| 351 | * @return DataTable |
| 352 | */ |
| 353 | protected function aggregateDataTableRecord($name, $columnsAggregationOperation = null, $columnsToRenameAfterAggregation = null) |
| 354 | { |
| 355 | try { |
| 356 | ErrorHandler::pushFatalErrorBreadcrumb(__CLASS__, ['name' => $name]); |
| 357 | |
| 358 | // By default we shall aggregate all sub-tables. |
| 359 | $dataTableBlobs = $this->getArchive()->getBlob($name, Archive::ID_SUBTABLE_LOAD_ALL_SUBTABLES); |
| 360 | $dataTable = $this->getAggregatedDataTableMapFromBlobs($dataTableBlobs, $columnsAggregationOperation, $columnsToRenameAfterAggregation, $name); |
| 361 | } finally { |
| 362 | ErrorHandler::popFatalErrorBreadcrumb(); |
| 363 | } |
| 364 | |
| 365 | return $dataTable; |
| 366 | } |
| 367 | |
| 368 | protected function getAggregatedDataTableMapFromBlobs(DataCollection $dataTableBlobs, $columnsAggregationOperation, $columnsToRenameAfterAggregation, $name) |
| 369 | { |
| 370 | $result = new DataTable(); |
| 371 | |
| 372 | if (!empty($columnsAggregationOperation)) { |
| 373 | $result->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation); |
| 374 | } |
| 375 | |
| 376 | $dataTableBlobs->forEachBlobExpanded(function ($reportBlobs, DataTableFactory $factory, $tableMetadata) use ($name, $result, $columnsToRenameAfterAggregation) { |
| 377 | $latestUsedTableId = Manager::getInstance()->getMostRecentTableId(); |
| 378 | |
| 379 | $toSum = $factory->make($reportBlobs, $index = [], $tableMetadata); |
| 380 | |
| 381 | $latestUsedAfterCreatingToSum = Manager::getInstance()->getMostRecentTableId(); |
| 382 | |
| 383 | // see https://github.com/piwik/piwik/issues/4377 |
| 384 | $toSum->filter(function ($table) use ($columnsToRenameAfterAggregation, $name) { |
| 385 | if ($this->areColumnsNotAlreadyRenamed($table)) { |
| 386 | /** |
| 387 | * This makes archiving and range dates a lot faster. Imagine we archive a week, then we will |
| 388 | * rename all columns of each 7 day archives. Afterwards we know the columns will be replaced in a |
| 389 | * week archive. When generating month archives, which uses mostly week archives, we do not have |
| 390 | * to replace those columns for the week archives again since we can be sure they were already |
| 391 | * replaced. Same when aggregating year and range archives. This can save up 10% or more when |
| 392 | * aggregating Month, Year and Range archives. |
| 393 | */ |
| 394 | $this->renameColumnsAfterAggregation($table, $columnsToRenameAfterAggregation); |
| 395 | } |
| 396 | }); |
| 397 | |
| 398 | $result->addDataTable($toSum); |
| 399 | |
| 400 | DataTable\Manager::getInstance()->deleteAll($latestUsedTableId, $latestUsedAfterCreatingToSum); |
| 401 | }); |
| 402 | |
| 403 | return $result; |
| 404 | } |
| 405 | |
| 406 | /** |
| 407 | * Note: public only for use in closure in PHP 5.3. |
| 408 | * |
| 409 | * @param $table |
| 410 | * @return \Piwik\Period |
| 411 | */ |
| 412 | public function areColumnsNotAlreadyRenamed($table) |
| 413 | { |
| 414 | $period = $table->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX); |
| 415 | |
| 416 | return !$period || $period->getLabel() === 'day'; |
| 417 | } |
| 418 | |
| 419 | protected function getOperationForColumns($columns, $defaultOperation) |
| 420 | { |
| 421 | $operationForColumn = array(); |
| 422 | foreach ($columns as $name) { |
| 423 | $operation = $defaultOperation; |
| 424 | if (empty($operation)) { |
| 425 | $operation = $this->guessOperationForColumn($name); |
| 426 | } |
| 427 | $operationForColumn[$name] = $operation; |
| 428 | } |
| 429 | return $operationForColumn; |
| 430 | } |
| 431 | |
| 432 | protected function enrichWithUniqueVisitorsMetric(Row $row) |
| 433 | { |
| 434 | if ($row->getColumn('nb_uniq_visitors') === false |
| 435 | && $row->getColumn('nb_users') === false |
| 436 | ) { |
| 437 | return; |
| 438 | } |
| 439 | |
| 440 | $periodLabel = $this->getParams()->getPeriod()->getLabel(); |
| 441 | |
| 442 | if (!SettingsPiwik::isUniqueVisitorsEnabled($periodLabel)) { |
| 443 | $row->deleteColumn('nb_uniq_visitors'); |
| 444 | $row->deleteColumn('nb_users'); |
| 445 | return; |
| 446 | } |
| 447 | |
| 448 | $sites = $this->getIdSitesToComputeNbUniques(); |
| 449 | |
| 450 | if (count($sites) > 1 && Rules::shouldSkipUniqueVisitorsCalculationForMultipleSites()) { |
| 451 | if ($periodLabel != 'day') { |
| 452 | // for day we still keep the aggregated metric but for other periods we remove it as it becomes to |
| 453 | // inaccurate |
| 454 | $row->deleteColumn('nb_uniq_visitors'); |
| 455 | $row->deleteColumn('nb_users'); |
| 456 | } |
| 457 | return; |
| 458 | } |
| 459 | |
| 460 | if (empty($sites)) { |
| 461 | // a plugin disabled running below query by removing all sites. |
| 462 | $row->deleteColumn('nb_uniq_visitors'); |
| 463 | $row->deleteColumn('nb_users'); |
| 464 | return; |
| 465 | } |
| 466 | |
| 467 | if (count($sites) === 1) { |
| 468 | $uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_VISITORS; |
| 469 | } else { |
| 470 | if (!SettingsPiwik::isSameFingerprintAcrossWebsites()) { |
| 471 | throw new Exception("Processing unique visitors across websites is enabled for this instance, |
| 472 | but to process this metric you must first set enable_fingerprinting_across_websites=1 |
| 473 | in the config file, under the [Tracker] section."); |
| 474 | } |
| 475 | $uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_FINGERPRINTS; |
| 476 | } |
| 477 | |
| 478 | $metrics = array( |
| 479 | Metrics::INDEX_NB_USERS, |
| 480 | $uniqueVisitorsMetric |
| 481 | ); |
| 482 | |
| 483 | $uniques = $this->computeNbUniques($metrics, $sites); |
| 484 | |
| 485 | // see edge case as described in https://github.com/piwik/piwik/issues/9357 where uniq_visitors might be higher |
| 486 | // than visits because we archive / process it after nb_visits. Between archiving nb_visits and nb_uniq_visitors |
| 487 | // there could have been a new visit leading to a higher nb_unique_visitors than nb_visits which is not possible |
| 488 | // by definition. In this case we simply use the visits metric instead of unique visitors metric. |
| 489 | $visits = $row->getColumn('nb_visits'); |
| 490 | if ($visits !== false && $uniques[$uniqueVisitorsMetric] !== false) { |
| 491 | $uniques[$uniqueVisitorsMetric] = min($uniques[$uniqueVisitorsMetric], $visits); |
| 492 | } |
| 493 | |
| 494 | $row->setColumn('nb_uniq_visitors', $uniques[$uniqueVisitorsMetric]); |
| 495 | $row->setColumn('nb_users', $uniques[Metrics::INDEX_NB_USERS]); |
| 496 | } |
| 497 | |
| 498 | protected function guessOperationForColumn($column) |
| 499 | { |
| 500 | if (strpos($column, 'max_') === 0) { |
| 501 | return 'max'; |
| 502 | } |
| 503 | if (strpos($column, 'min_') === 0) { |
| 504 | return 'min'; |
| 505 | } |
| 506 | return 'sum'; |
| 507 | } |
| 508 | |
| 509 | private function getIdSitesToComputeNbUniques() |
| 510 | { |
| 511 | $params = $this->getParams(); |
| 512 | $sites = array($params->getSite()->getId()); |
| 513 | |
| 514 | /** |
| 515 | * Triggered to change which site ids should be looked at when processing unique visitors and users. |
| 516 | * |
| 517 | * @param array &$idSites An array with one idSite. This site is being archived currently. To cancel the query |
| 518 | * you can change this value to an empty array. To include other sites in the query you |
| 519 | * can add more idSites to this list of idSites. |
| 520 | * @param Period $period The period that is being requested to be archived. |
| 521 | * @param Segment $segment The segment that is request to be archived. |
| 522 | */ |
| 523 | Piwik::postEvent('ArchiveProcessor.ComputeNbUniques.getIdSites', array(&$sites, $params->getPeriod(), $params->getSegment())); |
| 524 | |
| 525 | return $sites; |
| 526 | } |
| 527 | |
| 528 | /** |
| 529 | * Processes number of unique visitors for the given period |
| 530 | * |
| 531 | * This is the only Period metric (ie. week/month/year/range) that we process from the logs directly, |
| 532 | * since unique visitors cannot be summed like other metrics. |
| 533 | * |
| 534 | * @param array $metrics Metrics Ids for which to aggregates count of values |
| 535 | * @param int[] $sites A list of idSites that should be included |
| 536 | * @return array|null An array of metrics, where the key is metricid and the value is the metric value or null if |
| 537 | * the query was cancelled and not executed. |
| 538 | */ |
| 539 | protected function computeNbUniques($metrics, $sites) |
| 540 | { |
| 541 | $logAggregator = $this->getLogAggregator(); |
| 542 | $sitesBackup = $logAggregator->getSites(); |
| 543 | |
| 544 | $logAggregator->setSites($sites); |
| 545 | try { |
| 546 | $query = $logAggregator->queryVisitsByDimension(array(), false, array(), $metrics); |
| 547 | } finally { |
| 548 | $logAggregator->setSites($sitesBackup); |
| 549 | } |
| 550 | $data = $query->fetch(); |
| 551 | return $data; |
| 552 | } |
| 553 | |
| 554 | /** |
| 555 | * If the DataTable is a Map, sums all DataTable in the map and return the DataTable. |
| 556 | * |
| 557 | * |
| 558 | * @param $data DataTable|DataTable\Map |
| 559 | * @param $columnsToRenameAfterAggregation array |
| 560 | * @return DataTable |
| 561 | */ |
| 562 | protected function getAggregatedDataTableMap($data, $columnsAggregationOperation) |
| 563 | { |
| 564 | $table = new DataTable(); |
| 565 | |
| 566 | if (!empty($columnsAggregationOperation)) { |
| 567 | $table->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation); |
| 568 | } |
| 569 | |
| 570 | if ($data instanceof DataTable\Map) { |
| 571 | // as $date => $tableToSum |
| 572 | $this->aggregatedDataTableMapsAsOne($data, $table); |
| 573 | } else { |
| 574 | $table->addDataTable($data); |
| 575 | } |
| 576 | |
| 577 | return $table; |
| 578 | } |
| 579 | |
| 580 | /** |
| 581 | * Aggregates the DataTable\Map into the destination $aggregated |
| 582 | * @param $map |
| 583 | * @param $aggregated |
| 584 | */ |
| 585 | protected function aggregatedDataTableMapsAsOne(Map $map, DataTable $aggregated) |
| 586 | { |
| 587 | foreach ($map->getDataTables() as $tableToAggregate) { |
| 588 | if ($tableToAggregate instanceof Map) { |
| 589 | $this->aggregatedDataTableMapsAsOne($tableToAggregate, $aggregated); |
| 590 | } else { |
| 591 | $aggregated->addDataTable($tableToAggregate); |
| 592 | } |
| 593 | } |
| 594 | } |
| 595 | |
| 596 | /** |
| 597 | * Note: public only for use in closure in PHP 5.3. |
| 598 | */ |
| 599 | public function renameColumnsAfterAggregation(DataTable $table, $columnsToRenameAfterAggregation = null) |
| 600 | { |
| 601 | // Rename columns after aggregation |
| 602 | if (is_null($columnsToRenameAfterAggregation)) { |
| 603 | $columnsToRenameAfterAggregation = self::$columnsToRenameAfterAggregation; |
| 604 | } |
| 605 | |
| 606 | if (empty($columnsToRenameAfterAggregation)) { |
| 607 | return; |
| 608 | } |
| 609 | |
| 610 | foreach ($table->getRows() as $row) { |
| 611 | foreach ($columnsToRenameAfterAggregation as $oldName => $newName) { |
| 612 | $row->renameColumn($oldName, $newName); |
| 613 | } |
| 614 | |
| 615 | $subTable = $row->getSubtable(); |
| 616 | if ($subTable) { |
| 617 | $this->renameColumnsAfterAggregation($subTable, $columnsToRenameAfterAggregation); |
| 618 | } |
| 619 | } |
| 620 | } |
| 621 | |
| 622 | protected function getAggregatedNumericMetrics($columns, $operationToApply) |
| 623 | { |
| 624 | if (!is_array($columns)) { |
| 625 | $columns = array($columns); |
| 626 | } |
| 627 | |
| 628 | $operationForColumn = $this->getOperationForColumns($columns, $operationToApply); |
| 629 | |
| 630 | $dataTable = $this->getArchive()->getDataTableFromNumeric($columns); |
| 631 | |
| 632 | $results = $this->getAggregatedDataTableMap($dataTable, $operationForColumn); |
| 633 | if ($results->getRowsCount() > 1) { |
| 634 | throw new Exception("A DataTable is an unexpected state:" . var_export($results, true)); |
| 635 | } |
| 636 | |
| 637 | $rowMetrics = $results->getFirstRow(); |
| 638 | if ($rowMetrics === false) { |
| 639 | $rowMetrics = new Row; |
| 640 | } |
| 641 | $this->enrichWithUniqueVisitorsMetric($rowMetrics); |
| 642 | $this->renameColumnsAfterAggregation($results, self::$columnsToRenameAfterAggregation); |
| 643 | |
| 644 | $metrics = $rowMetrics->getColumns(); |
| 645 | |
| 646 | foreach ($columns as $name) { |
| 647 | if (!isset($metrics[$name])) { |
| 648 | $metrics[$name] = 0; |
| 649 | } |
| 650 | } |
| 651 | |
| 652 | return $metrics; |
| 653 | } |
| 654 | |
| 655 | /** |
| 656 | * Initiate archiving for a plugin during an ongoing archiving. The plugin can be another |
| 657 | * plugin or the same plugin. |
| 658 | * |
| 659 | * This method should be called during archiving when one plugin uses the report of another |
| 660 | * plugin with a segment. It will ensure reports for that segment & plugin will be archived |
| 661 | * without initiating archiving for every plugin with that segment (which would be a performance |
| 662 | * killer). |
| 663 | * |
| 664 | * @param string $plugin |
| 665 | * @param string $segment |
| 666 | */ |
| 667 | public function processDependentArchive($plugin, $segment) |
| 668 | { |
| 669 | $params = $this->getParams(); |
| 670 | if (!$params->isRootArchiveRequest()) { // prevent all recursion |
| 671 | return; |
| 672 | } |
| 673 | |
| 674 | $idSites = [$params->getSite()->getId()]; |
| 675 | |
| 676 | // important to use the original segment string when combining. As the API itself would combine the original string. |
| 677 | // this prevents a bug where the API would use the segment |
| 678 | // userId!@%2540matomo.org;userId!=hello%2540matomo.org;visitorType==new |
| 679 | // vs here we would use |
| 680 | // userId!@%40matomo.org;userId!=hello%40matomo.org;visitorType==new |
| 681 | // thus these would result in different segment hashes and therefore the reports would either show 0 or archive the data twice |
| 682 | $originSegmentString = $params->getSegment()->getOriginalString(); |
| 683 | $newSegment = Segment::combine($originSegmentString, SegmentExpression::AND_DELIMITER, $segment); |
| 684 | if (!empty($originSegmentString) && $newSegment === $segment && $params->getRequestedPlugin() === $plugin) { // being processed now |
| 685 | return; |
| 686 | } |
| 687 | |
| 688 | $newSegment = new Segment($newSegment, $idSites, $params->getDateStart(), $params->getDateEnd()); |
| 689 | if (ArchiveProcessor\Rules::isSegmentPreProcessed($idSites, $newSegment)) { |
| 690 | // will be processed anyway |
| 691 | return; |
| 692 | } |
| 693 | |
| 694 | $parameters = new ArchiveProcessor\Parameters($params->getSite(), $params->getPeriod(), $newSegment); |
| 695 | $parameters->onlyArchiveRequestedPlugin(); |
| 696 | $parameters->setIsRootArchiveRequest(false); |
| 697 | |
| 698 | $archiveLoader = new ArchiveProcessor\Loader($parameters); |
| 699 | $archiveLoader->prepareArchive($plugin); |
| 700 | } |
| 701 | |
| 702 | public function getArchiveWriter() |
| 703 | { |
| 704 | return $this->archiveWriter; |
| 705 | } |
| 706 | } |
| 707 |