API
5 years ago
Access
5 years ago
Application
5 years ago
Archive
5 years ago
ArchiveProcessor
5 years ago
Archiver
5 years ago
AssetManager
5 years ago
Auth
5 years ago
Category
5 years ago
CliMulti
5 years ago
Columns
5 years ago
Composer
5 years ago
Concurrency
5 years ago
Config
5 years ago
Container
5 years ago
CronArchive
5 years ago
DataAccess
5 years ago
DataFiles
5 years ago
DataTable
5 years ago
Db
5 years ago
DeviceDetector
5 years ago
Email
5 years ago
Exception
5 years ago
Http
5 years ago
Intl
5 years ago
Mail
5 years ago
Measurable
5 years ago
Menu
5 years ago
Metrics
5 years ago
Notification
5 years ago
Period
5 years ago
Plugin
5 years ago
ProfessionalServices
5 years ago
Report
5 years ago
ReportRenderer
5 years ago
Scheduler
5 years ago
Segment
5 years ago
Session
5 years ago
Settings
5 years ago
Tracker
5 years ago
Translation
5 years ago
UpdateCheck
5 years ago
Updater
5 years ago
Updates
5 years ago
Validators
5 years ago
View
5 years ago
ViewDataTable
5 years ago
Visualization
5 years ago
Widget
5 years ago
.htaccess
6 years ago
Access.php
5 years ago
Archive.php
5 years ago
ArchiveProcessor.php
5 years ago
AssetManager.php
5 years ago
Auth.php
5 years ago
AuthResult.php
5 years ago
BaseFactory.php
5 years ago
Cache.php
5 years ago
CacheId.php
5 years ago
CliMulti.php
5 years ago
Common.php
5 years ago
Config.php
5 years ago
Console.php
5 years ago
Context.php
5 years ago
Cookie.php
5 years ago
CronArchive.php
5 years ago
DataArray.php
5 years ago
DataTable.php
5 years ago
Date.php
5 years ago
Db.php
5 years ago
DbHelper.php
5 years ago
Development.php
5 years ago
ErrorHandler.php
5 years ago
EventDispatcher.php
5 years ago
ExceptionHandler.php
5 years ago
FileIntegrity.php
5 years ago
Filechecks.php
5 years ago
Filesystem.php
5 years ago
FrontController.php
5 years ago
Http.php
5 years ago
IP.php
5 years ago
Log.php
5 years ago
LogDeleter.php
5 years ago
Mail.php
5 years ago
Metrics.php
5 years ago
NoAccessException.php
5 years ago
Nonce.php
5 years ago
Notification.php
5 years ago
NumberFormatter.php
5 years ago
Option.php
5 years ago
Period.php
5 years ago
Piwik.php
5 years ago
Plugin.php
5 years ago
Profiler.php
5 years ago
ProxyHeaders.php
5 years ago
ProxyHttp.php
5 years ago
QuickForm2.php
5 years ago
RankingQuery.php
5 years ago
ReportRenderer.php
5 years ago
Segment.php
5 years ago
Sequence.php
5 years ago
Session.php
5 years ago
SettingsPiwik.php
5 years ago
SettingsServer.php
5 years ago
Singleton.php
5 years ago
Site.php
5 years ago
SupportedBrowser.php
5 years ago
TCPDF.php
5 years ago
Theme.php
5 years ago
Timer.php
5 years ago
Tracker.php
5 years ago
Twig.php
5 years ago
Unzip.php
5 years ago
UpdateCheck.php
5 years ago
Updater.php
5 years ago
UpdaterErrorException.php
5 years ago
Updates.php
5 years ago
Url.php
5 years ago
UrlHelper.php
5 years ago
Version.php
5 years ago
View.php
5 years ago
bootstrap.php
5 years ago
dispatch.php
5 years ago
testMinimumPhpVersion.php
5 years ago
ArchiveProcessor.php
676 lines
| 1 | <?php |
| 2 | /** |
| 3 | * Matomo - free/libre analytics platform |
| 4 | * |
| 5 | * @link https://matomo.org |
| 6 | * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later |
| 7 | * |
| 8 | */ |
| 9 | namespace Piwik; |
| 10 | |
| 11 | use Exception; |
| 12 | use Piwik\Archive\DataTableFactory; |
| 13 | use Piwik\ArchiveProcessor\Parameters; |
| 14 | use Piwik\ArchiveProcessor\Rules; |
| 15 | use Piwik\DataAccess\ArchiveWriter; |
| 16 | use Piwik\DataAccess\LogAggregator; |
| 17 | use Piwik\DataTable\Manager; |
| 18 | use Piwik\DataTable\Map; |
| 19 | use Piwik\DataTable\Row; |
| 20 | use Piwik\Segment\SegmentExpression; |
| 21 | |
| 22 | /** |
| 23 | * Used by {@link Piwik\Plugin\Archiver} instances to insert and aggregate archive data. |
| 24 | * |
| 25 | * ### See also |
| 26 | * |
| 27 | * - **{@link Piwik\Plugin\Archiver}** - to learn how plugins should implement their own analytics |
| 28 | * aggregation logic. |
| 29 | * - **{@link Piwik\DataAccess\LogAggregator}** - to learn how plugins can perform data aggregation |
| 30 | * across Piwik's log tables. |
| 31 | * |
| 32 | * ### Examples |
| 33 | * |
| 34 | * **Inserting numeric data** |
| 35 | * |
| 36 | * // function in an Archiver descendant |
| 37 | * public function aggregateDayReport() |
| 38 | * { |
| 39 | * $archiveProcessor = $this->getProcessor(); |
| 40 | * |
| 41 | * $myFancyMetric = // ... calculate the metric value ... |
| 42 | * $archiveProcessor->insertNumericRecord('MyPlugin_myFancyMetric', $myFancyMetric); |
| 43 | * } |
| 44 | * |
| 45 | * **Inserting serialized DataTables** |
| 46 | * |
| 47 | * // function in an Archiver descendant |
| 48 | * public function aggregateDayReport() |
| 49 | * { |
| 50 | * $archiveProcessor = $this->getProcessor(); |
| 51 | * |
| 52 | * $maxRowsInTable = Config::getInstance()->General['datatable_archiving_maximum_rows_standard'];j |
| 53 | * |
| 54 | * $dataTable = // ... build by aggregating visits ... |
| 55 | * $serializedData = $dataTable->getSerialized($maxRowsInTable, $maxRowsInSubtable = $maxRowsInTable, |
| 56 | * $columnToSortBy = Metrics::INDEX_NB_VISITS); |
| 57 | * |
| 58 | * $archiveProcessor->insertBlobRecords('MyPlugin_myFancyReport', $serializedData); |
| 59 | * } |
| 60 | * |
| 61 | * **Aggregating archive data** |
| 62 | * |
| 63 | * // function in Archiver descendant |
| 64 | * public function aggregateMultipleReports() |
| 65 | * { |
| 66 | * $archiveProcessor = $this->getProcessor(); |
| 67 | * |
| 68 | * // aggregate a metric |
| 69 | * $archiveProcessor->aggregateNumericMetrics('MyPlugin_myFancyMetric'); |
| 70 | * $archiveProcessor->aggregateNumericMetrics('MyPlugin_mySuperFancyMetric', 'max'); |
| 71 | * |
| 72 | * // aggregate a report |
| 73 | * $archiveProcessor->aggregateDataTableRecords('MyPlugin_myFancyReport'); |
| 74 | * } |
| 75 | * |
| 76 | */ |
| 77 | class ArchiveProcessor |
| 78 | { |
| 79 | /** |
| 80 | * @var \Piwik\DataAccess\ArchiveWriter |
| 81 | */ |
| 82 | private $archiveWriter; |
| 83 | |
| 84 | /** |
| 85 | * @var \Piwik\DataAccess\LogAggregator |
| 86 | */ |
| 87 | private $logAggregator; |
| 88 | |
| 89 | /** |
| 90 | * @var Archive |
| 91 | */ |
| 92 | public $archive = null; |
| 93 | |
| 94 | /** |
| 95 | * @var Parameters |
| 96 | */ |
| 97 | private $params; |
| 98 | |
| 99 | /** |
| 100 | * @var int |
| 101 | */ |
| 102 | private $numberOfVisits = false; |
| 103 | |
| 104 | private $numberOfVisitsConverted = false; |
| 105 | |
| 106 | public function __construct(Parameters $params, ArchiveWriter $archiveWriter, LogAggregator $logAggregator) |
| 107 | { |
| 108 | $this->params = $params; |
| 109 | $this->logAggregator = $logAggregator; |
| 110 | $this->archiveWriter = $archiveWriter; |
| 111 | } |
| 112 | |
| 113 | protected function getArchive() |
| 114 | { |
| 115 | if (empty($this->archive)) { |
| 116 | $subPeriods = $this->params->getSubPeriods(); |
| 117 | $idSites = $this->params->getIdSites(); |
| 118 | $this->archive = Archive::factory($this->params->getSegment(), $subPeriods, $idSites); |
| 119 | |
| 120 | /** |
| 121 | * @internal |
| 122 | */ |
| 123 | Piwik::postEvent('ArchiveProcessor.getArchive', [$this->archive]); |
| 124 | } |
| 125 | |
| 126 | return $this->archive; |
| 127 | } |
| 128 | |
| 129 | public function setNumberOfVisits($visits, $visitsConverted) |
| 130 | { |
| 131 | $this->numberOfVisits = $visits; |
| 132 | $this->numberOfVisitsConverted = $visitsConverted; |
| 133 | } |
| 134 | |
| 135 | /** |
| 136 | * Returns the {@link Parameters} object containing the site, period and segment we're archiving |
| 137 | * data for. |
| 138 | * |
| 139 | * @return Parameters |
| 140 | * @api |
| 141 | */ |
| 142 | public function getParams() |
| 143 | { |
| 144 | return $this->params; |
| 145 | } |
| 146 | |
| 147 | /** |
| 148 | * Returns a `{@link Piwik\DataAccess\LogAggregator}` instance for the site, period and segment this |
| 149 | * ArchiveProcessor will insert archive data for. |
| 150 | * |
| 151 | * @return LogAggregator |
| 152 | * @api |
| 153 | */ |
| 154 | public function getLogAggregator() |
| 155 | { |
| 156 | return $this->logAggregator; |
| 157 | } |
| 158 | |
| 159 | /** |
| 160 | * Array of (column name before => column name renamed) of the columns for which sum operation is invalid. |
| 161 | * These columns will be renamed as per this mapping. |
| 162 | * @var array |
| 163 | */ |
| 164 | protected static $columnsToRenameAfterAggregation = array( |
| 165 | Metrics::INDEX_NB_UNIQ_VISITORS => Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS, |
| 166 | Metrics::INDEX_NB_USERS => Metrics::INDEX_SUM_DAILY_NB_USERS, |
| 167 | ); |
| 168 | |
| 169 | /** |
| 170 | * Sums records for every subperiod of the current period and inserts the result as the record |
| 171 | * for this period. |
| 172 | * |
| 173 | * DataTables are summed recursively so subtables will be summed as well. |
| 174 | * |
| 175 | * @param string|array $recordNames Name(s) of the report we are aggregating, eg, `'Referrers_type'`. |
| 176 | * @param int $maximumRowsInDataTableLevelZero Maximum number of rows allowed in the top level DataTable. |
| 177 | * @param int $maximumRowsInSubDataTable Maximum number of rows allowed in each subtable. |
| 178 | * @param string $columnToSortByBeforeTruncation The name of the column to sort by before truncating a DataTable. |
| 179 | * @param array $columnsAggregationOperation Operations for aggregating columns, see {@link Row::sumRow()}. |
| 180 | * @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names |
| 181 | * when summed because they cannot be summed, eg, |
| 182 | * `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`. |
| 183 | * @param bool|array $countRowsRecursive if set to true, will calculate the recursive rows count for all record names |
| 184 | * which makes it slower. If you only need it for some records pass an array of |
| 185 | * recordNames that defines for which ones you need a recursive row count. |
| 186 | * @return array Returns the row counts of each aggregated report before truncation, eg, |
| 187 | * |
| 188 | * array( |
| 189 | * 'report1' => array('level0' => $report1->getRowsCount, |
| 190 | * 'recursive' => $report1->getRowsCountRecursive()), |
| 191 | * 'report2' => array('level0' => $report2->getRowsCount, |
| 192 | * 'recursive' => $report2->getRowsCountRecursive()), |
| 193 | * ... |
| 194 | * ) |
| 195 | * @api |
| 196 | */ |
| 197 | public function aggregateDataTableRecords($recordNames, |
| 198 | $maximumRowsInDataTableLevelZero = null, |
| 199 | $maximumRowsInSubDataTable = null, |
| 200 | $columnToSortByBeforeTruncation = null, |
| 201 | &$columnsAggregationOperation = null, |
| 202 | $columnsToRenameAfterAggregation = null, |
| 203 | $countRowsRecursive = true) |
| 204 | { |
| 205 | if (!is_array($recordNames)) { |
| 206 | $recordNames = array($recordNames); |
| 207 | } |
| 208 | |
| 209 | $nameToCount = array(); |
| 210 | foreach ($recordNames as $recordName) { |
| 211 | $latestUsedTableId = Manager::getInstance()->getMostRecentTableId(); |
| 212 | |
| 213 | $table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation); |
| 214 | |
| 215 | $nameToCount[$recordName]['level0'] = $table->getRowsCount(); |
| 216 | if ($countRowsRecursive === true || (is_array($countRowsRecursive) && in_array($recordName, $countRowsRecursive))) { |
| 217 | $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive(); |
| 218 | } |
| 219 | |
| 220 | $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation); |
| 221 | Common::destroy($table); |
| 222 | $this->insertBlobRecord($recordName, $blob); |
| 223 | |
| 224 | unset($blob); |
| 225 | DataTable\Manager::getInstance()->deleteAll($latestUsedTableId); |
| 226 | } |
| 227 | |
| 228 | return $nameToCount; |
| 229 | } |
| 230 | |
| 231 | /** |
| 232 | * Aggregates one or more metrics for every subperiod of the current period and inserts the results |
| 233 | * as metrics for the current period. |
| 234 | * |
| 235 | * @param array|string $columns Array of metric names to aggregate. |
| 236 | * @param bool|string $operationToApply The operation to apply to the metric. Either `'sum'`, `'max'` or `'min'`. |
| 237 | * @return array|int Returns the array of aggregate values. If only one metric was aggregated, |
| 238 | * the aggregate value will be returned as is, not in an array. |
| 239 | * For example, if `array('nb_visits', 'nb_hits')` is supplied for `$columns`, |
| 240 | * |
| 241 | * array( |
| 242 | * 'nb_visits' => 3040, |
| 243 | * 'nb_hits' => 405 |
| 244 | * ) |
| 245 | * |
| 246 | * could be returned. If `array('nb_visits')` or `'nb_visits'` is used for `$columns`, |
| 247 | * then `3040` would be returned. |
| 248 | * @api |
| 249 | */ |
| 250 | public function aggregateNumericMetrics($columns, $operationToApply = false) |
| 251 | { |
| 252 | $metrics = $this->getAggregatedNumericMetrics($columns, $operationToApply); |
| 253 | |
| 254 | foreach ($metrics as $column => $value) { |
| 255 | $this->insertNumericRecord($column, $value); |
| 256 | } |
| 257 | // if asked for only one field to sum |
| 258 | if (count($metrics) === 1) { |
| 259 | return reset($metrics); |
| 260 | } |
| 261 | |
| 262 | // returns the array of records once summed |
| 263 | return $metrics; |
| 264 | } |
| 265 | |
| 266 | public function getNumberOfVisits() |
| 267 | { |
| 268 | if ($this->numberOfVisits === false) { |
| 269 | throw new Exception("visits should have been set here"); |
| 270 | } |
| 271 | return $this->numberOfVisits; |
| 272 | } |
| 273 | |
| 274 | public function getNumberOfVisitsConverted() |
| 275 | { |
| 276 | return $this->numberOfVisitsConverted; |
| 277 | } |
| 278 | |
| 279 | /** |
| 280 | * Caches multiple numeric records in the archive for this processor's site, period |
| 281 | * and segment. |
| 282 | * |
| 283 | * @param array $numericRecords A name-value mapping of numeric values that should be |
| 284 | * archived, eg, |
| 285 | * |
| 286 | * array('Referrers_distinctKeywords' => 23, 'Referrers_distinctCampaigns' => 234) |
| 287 | * @api |
| 288 | */ |
| 289 | public function insertNumericRecords($numericRecords) |
| 290 | { |
| 291 | foreach ($numericRecords as $name => $value) { |
| 292 | $this->insertNumericRecord($name, $value); |
| 293 | } |
| 294 | } |
| 295 | |
| 296 | /** |
| 297 | * Caches a single numeric record in the archive for this processor's site, period and |
| 298 | * segment. |
| 299 | * |
| 300 | * Numeric values are not inserted if they equal `0`. |
| 301 | * |
| 302 | * @param string $name The name of the numeric value, eg, `'Referrers_distinctKeywords'`. |
| 303 | * @param float $value The numeric value. |
| 304 | * @api |
| 305 | */ |
| 306 | public function insertNumericRecord($name, $value) |
| 307 | { |
| 308 | $value = round($value, 2); |
| 309 | $value = Common::forceDotAsSeparatorForDecimalPoint($value); |
| 310 | |
| 311 | $this->archiveWriter->insertRecord($name, $value); |
| 312 | } |
| 313 | |
| 314 | /** |
| 315 | * Caches one or more blob records in the archive for this processor's site, period |
| 316 | * and segment. |
| 317 | * |
| 318 | * @param string $name The name of the record, eg, 'Referrers_type'. |
| 319 | * @param string|array $values A blob string or an array of blob strings. If an array |
| 320 | * is used, the first element in the array will be inserted |
| 321 | * with the `$name` name. The others will be inserted with |
| 322 | * `$name . '_' . $index` as the record name (where $index is |
| 323 | * the index of the blob record in `$values`). |
| 324 | * @api |
| 325 | */ |
| 326 | public function insertBlobRecord($name, $values) |
| 327 | { |
| 328 | $this->archiveWriter->insertBlobRecord($name, $values); |
| 329 | } |
| 330 | |
| 331 | /** |
| 332 | * This method selects all DataTables that have the name $name over the period. |
| 333 | * All these DataTables are then added together, and the resulting DataTable is returned. |
| 334 | * |
| 335 | * @param string $name |
| 336 | * @param array $columnsAggregationOperation Operations for aggregating columns, @see Row::sumRow() |
| 337 | * @param array $columnsToRenameAfterAggregation columns in the array (old name, new name) to be renamed as the sum operation is not valid on them (eg. nb_uniq_visitors->sum_daily_nb_uniq_visitors) |
| 338 | * @return DataTable |
| 339 | */ |
| 340 | protected function aggregateDataTableRecord($name, $columnsAggregationOperation = null, $columnsToRenameAfterAggregation = null) |
| 341 | { |
| 342 | try { |
| 343 | ErrorHandler::pushFatalErrorBreadcrumb(__CLASS__, ['name' => $name]); |
| 344 | |
| 345 | // By default we shall aggregate all sub-tables. |
| 346 | $dataTable = $this->getArchive()->getDataTableExpanded($name, $idSubTable = null, $depth = null, $addMetadataSubtableId = false); |
| 347 | |
| 348 | $columnsRenamed = false; |
| 349 | |
| 350 | if ($dataTable instanceof Map) { |
| 351 | $columnsRenamed = true; |
| 352 | // see https://github.com/piwik/piwik/issues/4377 |
| 353 | $self = $this; |
| 354 | $dataTable->filter(function ($table) use ($self, $columnsToRenameAfterAggregation) { |
| 355 | |
| 356 | if ($self->areColumnsNotAlreadyRenamed($table)) { |
| 357 | /** |
| 358 | * This makes archiving and range dates a lot faster. Imagine we archive a week, then we will |
| 359 | * rename all columns of each 7 day archives. Afterwards we know the columns will be replaced in a |
| 360 | * week archive. When generating month archives, which uses mostly week archives, we do not have |
| 361 | * to replace those columns for the week archives again since we can be sure they were already |
| 362 | * replaced. Same when aggregating year and range archives. This can save up 10% or more when |
| 363 | * aggregating Month, Year and Range archives. |
| 364 | */ |
| 365 | $self->renameColumnsAfterAggregation($table, $columnsToRenameAfterAggregation); |
| 366 | } |
| 367 | }); |
| 368 | } |
| 369 | |
| 370 | $dataTable = $this->getAggregatedDataTableMap($dataTable, $columnsAggregationOperation); |
| 371 | |
| 372 | if (!$columnsRenamed) { |
| 373 | $this->renameColumnsAfterAggregation($dataTable, $columnsToRenameAfterAggregation); |
| 374 | } |
| 375 | } finally { |
| 376 | ErrorHandler::popFatalErrorBreadcrumb(); |
| 377 | } |
| 378 | |
| 379 | return $dataTable; |
| 380 | } |
| 381 | |
| 382 | /** |
| 383 | * Note: public only for use in closure in PHP 5.3. |
| 384 | * |
| 385 | * @param $table |
| 386 | * @return \Piwik\Period |
| 387 | */ |
| 388 | public function areColumnsNotAlreadyRenamed($table) |
| 389 | { |
| 390 | $period = $table->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX); |
| 391 | |
| 392 | return !$period || $period->getLabel() === 'day'; |
| 393 | } |
| 394 | |
| 395 | protected function getOperationForColumns($columns, $defaultOperation) |
| 396 | { |
| 397 | $operationForColumn = array(); |
| 398 | foreach ($columns as $name) { |
| 399 | $operation = $defaultOperation; |
| 400 | if (empty($operation)) { |
| 401 | $operation = $this->guessOperationForColumn($name); |
| 402 | } |
| 403 | $operationForColumn[$name] = $operation; |
| 404 | } |
| 405 | return $operationForColumn; |
| 406 | } |
| 407 | |
| 408 | protected function enrichWithUniqueVisitorsMetric(Row $row) |
| 409 | { |
| 410 | if ($row->getColumn('nb_uniq_visitors') === false |
| 411 | && $row->getColumn('nb_users') === false |
| 412 | ) { |
| 413 | return; |
| 414 | } |
| 415 | |
| 416 | $periodLabel = $this->getParams()->getPeriod()->getLabel(); |
| 417 | |
| 418 | if (!SettingsPiwik::isUniqueVisitorsEnabled($periodLabel)) { |
| 419 | $row->deleteColumn('nb_uniq_visitors'); |
| 420 | $row->deleteColumn('nb_users'); |
| 421 | return; |
| 422 | } |
| 423 | |
| 424 | $sites = $this->getIdSitesToComputeNbUniques(); |
| 425 | |
| 426 | if (count($sites) > 1 && Rules::shouldSkipUniqueVisitorsCalculationForMultipleSites()) { |
| 427 | if ($periodLabel != 'day') { |
| 428 | // for day we still keep the aggregated metric but for other periods we remove it as it becomes to |
| 429 | // inaccurate |
| 430 | $row->deleteColumn('nb_uniq_visitors'); |
| 431 | $row->deleteColumn('nb_users'); |
| 432 | } |
| 433 | return; |
| 434 | } |
| 435 | |
| 436 | if (empty($sites)) { |
| 437 | // a plugin disabled running below query by removing all sites. |
| 438 | $row->deleteColumn('nb_uniq_visitors'); |
| 439 | $row->deleteColumn('nb_users'); |
| 440 | return; |
| 441 | } |
| 442 | |
| 443 | if (count($sites) === 1) { |
| 444 | $uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_VISITORS; |
| 445 | } else { |
| 446 | if (!SettingsPiwik::isSameFingerprintAcrossWebsites()) { |
| 447 | throw new Exception("Processing unique visitors across websites is enabled for this instance, |
| 448 | but to process this metric you must first set enable_fingerprinting_across_websites=1 |
| 449 | in the config file, under the [Tracker] section."); |
| 450 | } |
| 451 | $uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_FINGERPRINTS; |
| 452 | } |
| 453 | |
| 454 | $metrics = array( |
| 455 | Metrics::INDEX_NB_USERS, |
| 456 | $uniqueVisitorsMetric |
| 457 | ); |
| 458 | |
| 459 | $uniques = $this->computeNbUniques($metrics, $sites); |
| 460 | |
| 461 | // see edge case as described in https://github.com/piwik/piwik/issues/9357 where uniq_visitors might be higher |
| 462 | // than visits because we archive / process it after nb_visits. Between archiving nb_visits and nb_uniq_visitors |
| 463 | // there could have been a new visit leading to a higher nb_unique_visitors than nb_visits which is not possible |
| 464 | // by definition. In this case we simply use the visits metric instead of unique visitors metric. |
| 465 | $visits = $row->getColumn('nb_visits'); |
| 466 | if ($visits !== false && $uniques[$uniqueVisitorsMetric] !== false) { |
| 467 | $uniques[$uniqueVisitorsMetric] = min($uniques[$uniqueVisitorsMetric], $visits); |
| 468 | } |
| 469 | |
| 470 | $row->setColumn('nb_uniq_visitors', $uniques[$uniqueVisitorsMetric]); |
| 471 | $row->setColumn('nb_users', $uniques[Metrics::INDEX_NB_USERS]); |
| 472 | } |
| 473 | |
| 474 | protected function guessOperationForColumn($column) |
| 475 | { |
| 476 | if (strpos($column, 'max_') === 0) { |
| 477 | return 'max'; |
| 478 | } |
| 479 | if (strpos($column, 'min_') === 0) { |
| 480 | return 'min'; |
| 481 | } |
| 482 | return 'sum'; |
| 483 | } |
| 484 | |
| 485 | private function getIdSitesToComputeNbUniques() |
| 486 | { |
| 487 | $params = $this->getParams(); |
| 488 | $sites = array($params->getSite()->getId()); |
| 489 | |
| 490 | /** |
| 491 | * Triggered to change which site ids should be looked at when processing unique visitors and users. |
| 492 | * |
| 493 | * @param array &$idSites An array with one idSite. This site is being archived currently. To cancel the query |
| 494 | * you can change this value to an empty array. To include other sites in the query you |
| 495 | * can add more idSites to this list of idSites. |
| 496 | * @param Period $period The period that is being requested to be archived. |
| 497 | * @param Segment $segment The segment that is request to be archived. |
| 498 | */ |
| 499 | Piwik::postEvent('ArchiveProcessor.ComputeNbUniques.getIdSites', array(&$sites, $params->getPeriod(), $params->getSegment())); |
| 500 | |
| 501 | return $sites; |
| 502 | } |
| 503 | |
| 504 | /** |
| 505 | * Processes number of unique visitors for the given period |
| 506 | * |
| 507 | * This is the only Period metric (ie. week/month/year/range) that we process from the logs directly, |
| 508 | * since unique visitors cannot be summed like other metrics. |
| 509 | * |
| 510 | * @param array $metrics Metrics Ids for which to aggregates count of values |
| 511 | * @param int[] $sites A list of idSites that should be included |
| 512 | * @return array|null An array of metrics, where the key is metricid and the value is the metric value or null if |
| 513 | * the query was cancelled and not executed. |
| 514 | */ |
| 515 | protected function computeNbUniques($metrics, $sites) |
| 516 | { |
| 517 | $logAggregator = $this->getLogAggregator(); |
| 518 | $sitesBackup = $logAggregator->getSites(); |
| 519 | |
| 520 | $logAggregator->setSites($sites); |
| 521 | try { |
| 522 | $query = $logAggregator->queryVisitsByDimension(array(), false, array(), $metrics); |
| 523 | } finally { |
| 524 | $logAggregator->setSites($sitesBackup); |
| 525 | } |
| 526 | $data = $query->fetch(); |
| 527 | return $data; |
| 528 | } |
| 529 | |
| 530 | /** |
| 531 | * If the DataTable is a Map, sums all DataTable in the map and return the DataTable. |
| 532 | * |
| 533 | * |
| 534 | * @param $data DataTable|DataTable\Map |
| 535 | * @param $columnsToRenameAfterAggregation array |
| 536 | * @return DataTable |
| 537 | */ |
| 538 | protected function getAggregatedDataTableMap($data, $columnsAggregationOperation) |
| 539 | { |
| 540 | $table = new DataTable(); |
| 541 | |
| 542 | if (!empty($columnsAggregationOperation)) { |
| 543 | $table->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation); |
| 544 | } |
| 545 | |
| 546 | if ($data instanceof DataTable\Map) { |
| 547 | // as $date => $tableToSum |
| 548 | $this->aggregatedDataTableMapsAsOne($data, $table); |
| 549 | } else { |
| 550 | $table->addDataTable($data); |
| 551 | } |
| 552 | |
| 553 | return $table; |
| 554 | } |
| 555 | |
| 556 | /** |
| 557 | * Aggregates the DataTable\Map into the destination $aggregated |
| 558 | * @param $map |
| 559 | * @param $aggregated |
| 560 | */ |
| 561 | protected function aggregatedDataTableMapsAsOne(Map $map, DataTable $aggregated) |
| 562 | { |
| 563 | foreach ($map->getDataTables() as $tableToAggregate) { |
| 564 | if ($tableToAggregate instanceof Map) { |
| 565 | $this->aggregatedDataTableMapsAsOne($tableToAggregate, $aggregated); |
| 566 | } else { |
| 567 | $aggregated->addDataTable($tableToAggregate); |
| 568 | } |
| 569 | } |
| 570 | } |
| 571 | |
| 572 | /** |
| 573 | * Note: public only for use in closure in PHP 5.3. |
| 574 | */ |
| 575 | public function renameColumnsAfterAggregation(DataTable $table, $columnsToRenameAfterAggregation = null) |
| 576 | { |
| 577 | // Rename columns after aggregation |
| 578 | if (is_null($columnsToRenameAfterAggregation)) { |
| 579 | $columnsToRenameAfterAggregation = self::$columnsToRenameAfterAggregation; |
| 580 | } |
| 581 | |
| 582 | if (empty($columnsToRenameAfterAggregation)) { |
| 583 | return; |
| 584 | } |
| 585 | |
| 586 | foreach ($table->getRows() as $row) { |
| 587 | foreach ($columnsToRenameAfterAggregation as $oldName => $newName) { |
| 588 | $row->renameColumn($oldName, $newName); |
| 589 | } |
| 590 | |
| 591 | $subTable = $row->getSubtable(); |
| 592 | if ($subTable) { |
| 593 | $this->renameColumnsAfterAggregation($subTable, $columnsToRenameAfterAggregation); |
| 594 | } |
| 595 | } |
| 596 | } |
| 597 | |
| 598 | protected function getAggregatedNumericMetrics($columns, $operationToApply) |
| 599 | { |
| 600 | if (!is_array($columns)) { |
| 601 | $columns = array($columns); |
| 602 | } |
| 603 | |
| 604 | $operationForColumn = $this->getOperationForColumns($columns, $operationToApply); |
| 605 | |
| 606 | $dataTable = $this->getArchive()->getDataTableFromNumeric($columns); |
| 607 | |
| 608 | $results = $this->getAggregatedDataTableMap($dataTable, $operationForColumn); |
| 609 | if ($results->getRowsCount() > 1) { |
| 610 | throw new Exception("A DataTable is an unexpected state:" . var_export($results, true)); |
| 611 | } |
| 612 | |
| 613 | $rowMetrics = $results->getFirstRow(); |
| 614 | if ($rowMetrics === false) { |
| 615 | $rowMetrics = new Row; |
| 616 | } |
| 617 | $this->enrichWithUniqueVisitorsMetric($rowMetrics); |
| 618 | $this->renameColumnsAfterAggregation($results, self::$columnsToRenameAfterAggregation); |
| 619 | |
| 620 | $metrics = $rowMetrics->getColumns(); |
| 621 | |
| 622 | foreach ($columns as $name) { |
| 623 | if (!isset($metrics[$name])) { |
| 624 | $metrics[$name] = 0; |
| 625 | } |
| 626 | } |
| 627 | |
| 628 | return $metrics; |
| 629 | } |
| 630 | |
| 631 | /** |
| 632 | * Initiate archiving for a plugin during an ongoing archiving. The plugin can be another |
| 633 | * plugin or the same plugin. |
| 634 | * |
| 635 | * This method should be called during archiving when one plugin uses the report of another |
| 636 | * plugin with a segment. It will ensure reports for that segment & plugin will be archived |
| 637 | * without initiating archiving for every plugin with that segment (which would be a performance |
| 638 | * killer). |
| 639 | * |
| 640 | * @param string $plugin |
| 641 | * @param string $segment |
| 642 | */ |
| 643 | public function processDependentArchive($plugin, $segment) |
| 644 | { |
| 645 | $params = $this->getParams(); |
| 646 | if (!$params->isRootArchiveRequest()) { // prevent all recursion |
| 647 | return; |
| 648 | } |
| 649 | |
| 650 | $idSites = [$params->getSite()->getId()]; |
| 651 | |
| 652 | $newSegment = Segment::combine($params->getSegment()->getString(), SegmentExpression::AND_DELIMITER, $segment); |
| 653 | if ($newSegment === $segment && $params->getRequestedPlugin() === $plugin) { // being processed now |
| 654 | return; |
| 655 | } |
| 656 | |
| 657 | $newSegment = new Segment($newSegment, $idSites, $params->getDateStart(), $params->getDateEnd()); |
| 658 | if (ArchiveProcessor\Rules::isSegmentPreProcessed($idSites, $newSegment)) { |
| 659 | // will be processed anyway |
| 660 | return; |
| 661 | } |
| 662 | |
| 663 | $parameters = new ArchiveProcessor\Parameters($params->getSite(), $params->getPeriod(), $newSegment); |
| 664 | $parameters->onlyArchiveRequestedPlugin(); |
| 665 | $parameters->setIsRootArchiveRequest(false); |
| 666 | |
| 667 | $archiveLoader = new ArchiveProcessor\Loader($parameters); |
| 668 | $archiveLoader->prepareArchive($plugin); |
| 669 | } |
| 670 | |
| 671 | public function getArchiveWriter() |
| 672 | { |
| 673 | return $this->archiveWriter; |
| 674 | } |
| 675 | } |
| 676 |