PluginProbe ʕ •ᴥ•ʔ
Matomo Analytics – Powerful, Privacy-First Insights for WordPress / 5.8.2
Matomo Analytics – Powerful, Privacy-First Insights for WordPress v5.8.2
5.11.1 5.11.0 5.10.2 5.10.1 trunk 1.0.2 1.0.3 1.0.4 1.0.5 1.0.6 1.1.0 1.1.1 1.1.2 1.1.3 1.2.0 1.3.0 1.3.1 1.3.2 4.0.0 4.0.1 4.0.2 4.0.3 4.0.4 4.1.0 4.1.1 4.1.2 4.1.3 4.10.0 4.11.0 4.12.0 4.13.0 4.13.2 4.13.3 4.13.4 4.13.5 4.14.0 4.14.1 4.14.2 4.15.0 4.15.1 4.15.2 4.15.3 4.2.0 4.3.0 4.3.1 4.4.1 4.4.2 4.5.0 4.6.0 5.0.1 5.0.2 5.0.3 5.0.4 5.0.5 5.0.6 5.0.7 5.0.8 5.1.0 5.1.1 5.1.2 5.1.3 5.1.4 5.1.5 5.1.6 5.1.7 5.10.0 5.2.0 5.2.1 5.2.2 5.3.0 5.3.1 5.3.2 5.3.3 5.6.0 5.6.1 5.7.0 5.7.1 5.8.0 5.8.1 5.8.2
matomo / app / core / RankingQuery.php
matomo / app / core Last commit date
API 3 months ago Access 3 months ago Application 3 months ago Archive 3 months ago ArchiveProcessor 3 months ago Archiver 2 years ago AssetManager 3 months ago Auth 6 months ago Category 6 months ago Changes 3 months ago CliMulti 1 year ago Columns 3 months ago Concurrency 3 months ago Config 3 months ago Container 3 months ago CronArchive 3 months ago DataAccess 3 months ago DataFiles 2 years ago DataTable 3 months ago Db 3 months ago DeviceDetector 1 year ago Email 2 years ago Exception 4 months ago Http 4 months ago Intl 3 months ago Log 2 years ago Mail 1 year ago Measurable 6 months ago Menu 3 months ago Metrics 3 months ago Notification 6 months ago Period 3 months ago Plugin 3 months ago Policy 3 months ago ProfessionalServices 1 year ago Report 1 year ago ReportRenderer 3 months ago Request 3 months ago Scheduler 3 months ago Segment 3 months ago Session 3 months ago Settings 3 months ago Tracker 3 months ago Translation 3 months ago Twig 1 year ago UpdateCheck 3 months ago Updater 4 months ago Updates 3 months ago Validators 1 year ago View 6 months ago ViewDataTable 3 months ago Visualization 1 year ago Widget 3 months ago .htaccess 2 years ago Access.php 3 months ago Archive.php 3 months ago ArchiveProcessor.php 4 months ago AssetManager.php 3 months ago Auth.php 6 months ago AuthResult.php 6 months ago BaseFactory.php 2 years ago Cache.php 2 years ago CacheId.php 4 months ago CliMulti.php 3 months ago Common.php 3 months ago Config.php 3 months ago Console.php 3 months ago Context.php 2 years ago Cookie.php 1 year ago CronArchive.php 3 months ago DI.php 3 months ago DataArray.php 5 months ago DataTable.php 3 months ago Date.php 3 months ago Db.php 3 months ago DbHelper.php 3 months ago Development.php 1 year ago ErrorHandler.php 6 months ago EventDispatcher.php 1 year ago ExceptionHandler.php 4 months ago FileIntegrity.php 3 months ago Filechecks.php 1 year ago Filesystem.php 3 months ago FrontController.php 4 months ago Http.php 4 months ago IP.php 1 year ago Log.php 3 months ago LogDeleter.php 1 year ago Mail.php 1 year ago Metrics.php 3 months ago NoAccessException.php 2 years ago Nonce.php 6 months ago Notification.php 6 months ago NumberFormatter.php 5 months ago Option.php 5 months ago Period.php 3 months ago Piwik.php 3 months ago Plugin.php 3 months ago Process.php 1 year ago Profiler.php 6 months ago ProxyHeaders.php 4 months ago ProxyHttp.php 5 months ago QuickForm2.php 3 months ago RankingQuery.php 5 months ago ReportRenderer.php 3 months ago Request.php 3 months ago Segment.php 3 months ago Sequence.php 6 months ago Session.php 3 months ago SettingsPiwik.php 3 months ago SettingsServer.php 1 year ago Singleton.php 2 years ago Site.php 4 months ago SiteContentDetector.php 3 months ago SupportedBrowser.php 2 years ago TCPDF.php 1 year ago Theme.php 1 year ago Timer.php 2 years ago Tracker.php 3 months ago Twig.php 3 months ago Unzip.php 1 year ago UpdateCheck.php 3 months ago Updater.php 3 months ago UpdaterErrorException.php 2 years ago Updates.php 3 months ago Url.php 3 months ago UrlHelper.php 3 months ago Version.php 3 months ago View.php 3 months ago bootstrap.php 1 year ago dispatch.php 2 years ago testMinimumPhpVersion.php 6 months ago
RankingQuery.php
393 lines
1 <?php
2
3 /**
4 * Matomo - free/libre analytics platform
5 *
6 * @link https://matomo.org
7 * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
8 */
9 namespace Piwik;
10
11 use Exception;
12 use Piwik\Db\Schema;
13 /**
14 * The ranking query class wraps an arbitrary SQL query with more SQL that limits
15 * the number of results while aggregating the rest in an a new "Others" row. It also
16 * allows for some more fancy things that can be configured via method calls of this
17 * class. The advanced use cases are explained in the doc comments of the methods.
18 *
19 * The general use case looks like this:
20 *
21 * // limit to 500 rows + "Others"
22 * $rankingQuery = new RankingQuery();
23 * $rankingQuery->setLimit(500);
24 *
25 * // idaction_url will be "Others" in the row that contains the aggregated rest
26 * $rankingQuery->addLabelColumn('idaction_url');
27 *
28 * // the actual query. it's important to sort it before the limit is applied
29 * $sql = 'SELECT idaction_url, COUNT(*) AS nb_hits
30 * FROM log_link_visit_action
31 * GROUP BY idaction_url
32 * ORDER BY nb_hits DESC';
33 *
34 * // execute the query
35 * $rankingQuery->execute($sql);
36 *
37 * For more examples, see RankingQueryTest.php
38 *
39 * @api
40 */
41 class RankingQuery
42 {
43 // a special label used to mark the 'Others' row in a ranking query result set. this is mapped to the
44 // datatable summary row during archiving.
45 public const LABEL_SUMMARY_ROW = '__mtm_ranking_query_others__';
46 /**
47 * Contains the labels of the inner query.
48 * Format: "label" => true (to make sure labels don't appear twice)
49 * @var array
50 */
51 private $labelColumns = array();
52 /**
53 * The columns of the inner query that are not labels
54 * Format: "label" => "aggregation function" or false for no aggregation
55 * @var array
56 */
57 private $additionalColumns = array();
58 /**
59 * The limit for each group
60 * @var int
61 */
62 private $limit = 5;
63 /**
64 * The name of the columns that marks rows to be excluded from the limit
65 * @var string
66 */
67 private $columnToMarkExcludedRows = \false;
68 /**
69 * The column that is used to partition the result
70 * @var bool|string
71 */
72 private $partitionColumn = \false;
73 /**
74 * The possible values for the column $this->partitionColumn
75 * @var array
76 */
77 private $partitionColumnValues = array();
78 /**
79 * The value to use in the label of the 'Others' row.
80 * @var string
81 */
82 private $othersLabelValue = self::LABEL_SUMMARY_ROW;
83 /**
84 * Constructor.
85 *
86 * @param int|false $limit The result row limit. See {@link setLimit()}.
87 */
88 public function __construct($limit = \false)
89 {
90 if ($limit !== \false) {
91 $this->setLimit($limit);
92 }
93 }
94 /**
95 * Set the limit after which everything is grouped to "Others".
96 *
97 * @param int $limit
98 */
99 public function setLimit($limit)
100 {
101 $this->limit = $limit;
102 }
103 /**
104 * Set the value to use for the label in the 'Others' row.
105 *
106 * @param string $value
107 */
108 public function setOthersLabel($value)
109 {
110 $this->othersLabelValue = $value;
111 }
112 /**
113 * Add a label column.
114 * Labels are the columns that are replaced with "Others" after the limit.
115 *
116 * @param string|array $labelColumn
117 */
118 public function addLabelColumn($labelColumn)
119 {
120 if (is_array($labelColumn)) {
121 foreach ($labelColumn as $label) {
122 $this->addLabelColumn($label);
123 }
124 return;
125 }
126 $this->labelColumns[$labelColumn] = \true;
127 }
128 /**
129 * @return array
130 */
131 public function getLabelColumns()
132 {
133 return $this->labelColumns;
134 }
135 /**
136 * Add a column that has be added to the outer queries.
137 *
138 * @param $column
139 * @param string|bool $aggregationFunction If set, this function is used to aggregate the values of "Others",
140 * eg, `'min'`, `'max'` or `'sum'`.
141 */
142 public function addColumn($column, $aggregationFunction = \false)
143 {
144 if (is_array($column)) {
145 foreach ($column as $c) {
146 $this->addColumn($c, $aggregationFunction);
147 }
148 return;
149 }
150 $this->additionalColumns[$column] = $aggregationFunction;
151 }
152 /**
153 * Sets a column that will be used to filter the result into two categories.
154 * Rows where this column has a value > 0 will be removed from the result and put
155 * into another array. Both the result and the array of excluded rows are returned
156 * by {@link execute()}.
157 *
158 * @param $column string Name of the column.
159 * @throws Exception if method is used more than once.
160 */
161 public function setColumnToMarkExcludedRows($column)
162 {
163 if ($this->columnToMarkExcludedRows !== \false) {
164 throw new Exception("setColumnToMarkExcludedRows can only be used once");
165 }
166 $this->columnToMarkExcludedRows = $column;
167 $this->addColumn($this->columnToMarkExcludedRows);
168 }
169 /**
170 * This method can be used to partition the result based on the possible values of one
171 * table column. This means the query will split the result set into other sets of rows
172 * for each possible value you provide (where the rows of each set have a column value
173 * that equals a possible value). Each of these new sets of rows will be individually
174 * limited resulting in several limited result sets.
175 *
176 * For example, you can run a query aggregating some data on the log_action table and
177 * partition by log_action.type with the possible values of {@link Piwik\Tracker\Action::TYPE_PAGE_URL},
178 * {@link Piwik\Tracker\Action::TYPE_OUTLINK}, {@link Piwik\Tracker\Action::TYPE_DOWNLOAD}.
179 * The result will be three separate result sets that are aggregated the same ways, but for rows
180 * where `log_action.type = TYPE_OUTLINK`, for rows where `log_action.type = TYPE_ACTION_URL` and for
181 * rows `log_action.type = TYPE_DOWNLOAD`.
182 *
183 * @param $partitionColumn string The column name to partition by.
184 * @param $possibleValues Array of possible column values.
185 * @throws Exception if method is used more than once.
186 */
187 public function partitionResultIntoMultipleGroups($partitionColumn, $possibleValues)
188 {
189 if ($this->partitionColumn !== \false) {
190 throw new Exception("partitionResultIntoMultipleGroups can only be used once");
191 }
192 $this->partitionColumn = $partitionColumn;
193 $this->partitionColumnValues = $possibleValues;
194 $this->addColumn($partitionColumn);
195 }
196 /**
197 * Executes the query.
198 * The object has to be configured first using the other methods.
199 *
200 * @param $innerQuery string The "payload" query that does the actual data aggregation. The ordering
201 * has to be specified in this query. {@link RankingQuery} cannot apply ordering
202 * itself.
203 * @param $bind array Bindings for the inner query.
204 * @param int $timeLimit Adds a MAX_EXECUTION_TIME query hint to the query if $timeLimit > 0
205 * for more details see {@link DbHelper::addMaxExecutionTimeHintToQuery}
206 * @return array The format depends on which methods have been used
207 * to configure the ranking query.
208 */
209 public function execute($innerQuery, $bind = array(), $timeLimit = 0)
210 {
211 $query = $this->generateRankingQuery($innerQuery);
212 $query = \Piwik\DbHelper::addMaxExecutionTimeHintToQuery($query, $timeLimit);
213 $data = \Piwik\Db::getReader()->fetchAll($query, $bind);
214 if ($this->columnToMarkExcludedRows !== \false) {
215 // split the result into the regular result and the rows with special treatment
216 $excludedFromLimit = array();
217 $result = array();
218 foreach ($data as &$row) {
219 if ($row[$this->columnToMarkExcludedRows] != 0) {
220 $excludedFromLimit[] = $row;
221 } else {
222 $result[] = $row;
223 }
224 }
225 $data = array('result' => &$result, 'excludedFromLimit' => &$excludedFromLimit);
226 }
227 if ($this->partitionColumn !== \false) {
228 if ($this->columnToMarkExcludedRows !== \false) {
229 $data['result'] = $this->splitPartitions($data['result']);
230 } else {
231 $data = $this->splitPartitions($data);
232 }
233 }
234 return $data;
235 }
236 private function splitPartitions(&$data)
237 {
238 $result = array();
239 foreach ($data as &$row) {
240 $partition = $row[$this->partitionColumn];
241 if (!isset($result[$partition])) {
242 $result[$partition] = array();
243 }
244 $result[$partition][] =& $row;
245 }
246 return $result;
247 }
248 /**
249 * Generate the SQL code that does the magic.
250 * If you want to get the result, use execute() instead. If you want to run the query
251 * yourself, use this method.
252 *
253 * @param $innerQuery string The "payload" query that does the actual data aggregation. The ordering
254 * has to be specified in this query. {@link RankingQuery} cannot apply ordering
255 * itself.
256 * @param $withRollup bool A flag which determines whether to generate the SQL query using ROLLUP
257 * @return string The entire ranking query SQL.
258 */
259 public function generateRankingQuery($innerQuery, bool $withRollup = \false)
260 {
261 // +1 to include "Others"
262 $limit = $this->limit + 1;
263 $counterExpression = $this->getCounterExpression($limit, $withRollup);
264 // generate select clauses for label columns
265 $labelColumnsString = '`' . implode('`, `', array_keys($this->labelColumns)) . '`';
266 $labelColumnsOthersSwitch = [];
267 $withRollupColumns = [];
268 $withRollupOthersGroupBy = [];
269 foreach (array_keys($this->labelColumns) as $column) {
270 if ($withRollup) {
271 if ([] === $withRollupColumns) {
272 // support "Others" row for first label column
273 $rollupWhen = "\n WHEN counterRollup = {$limit} THEN '" . $this->othersLabelValue . "'\n WHEN counterRollup > 0 THEN `{$column}`\n WHEN counter = {$limit} AND counterRollup = 0 THEN `{$column}`\n WHEN counter = {$limit} THEN '" . $this->othersLabelValue . "'\n ";
274 } else {
275 // support "Others" row for secondary label columns
276 $rollupWhen = "\n WHEN `{$column}` IS NULL THEN NULL\n WHEN counter = {$limit} AND counterRollup = 0 THEN '" . $this->othersLabelValue . "'\n ";
277 }
278 $switch = "\n CASE\n {$rollupWhen}\n ELSE `{$column}`\n END\n ";
279 $labelColumnsOthersSwitch[] = "{$switch} AS `{$column}`";
280 $withRollupColumns[] = $column;
281 $withRollupOthersGroupBy[] = $switch;
282 } else {
283 $labelColumnsOthersSwitch[] = "\n CASE\n WHEN counter = {$limit} THEN '" . $this->othersLabelValue . "'\n ELSE `{$column}`\n END AS `{$column}`\n ";
284 }
285 }
286 $labelColumnsOthersSwitch = implode(', ', $labelColumnsOthersSwitch);
287 // generate select clauses for additional columns
288 $additionalColumnsString = '';
289 $additionalColumnsAggregatedString = '';
290 foreach ($this->additionalColumns as $additionalColumn => $aggregation) {
291 $additionalColumnsString .= ', `' . $additionalColumn . '`';
292 if ($aggregation !== \false) {
293 $additionalColumnsAggregatedString .= ', ' . $aggregation . '(`' . $additionalColumn . '`) AS `' . $additionalColumn . '`';
294 } else {
295 $additionalColumnsAggregatedString .= ', `' . $additionalColumn . '`';
296 }
297 }
298 // initialize the counters
299 if ($this->partitionColumn !== \false) {
300 $initCounter = '';
301 foreach ($this->partitionColumnValues as $value) {
302 $initCounter .= '( SELECT @counter' . intval($value) . ':=0 ) initCounter' . intval($value) . ', ';
303 }
304 } else {
305 $initCounter = '( SELECT @counter:=0 ) initCounter,';
306 }
307 $counterRollupExpression = '';
308 if ($withRollup) {
309 $initCounter .= ' ( SELECT @counterRollup:=0 ) initCounterRollup,';
310 $counterRollupWhen = '';
311 if (count($withRollupColumns) >= 2) {
312 $counterRollupWhen = "\n WHEN `" . implode('` IS NULL AND `', $withRollupColumns) . "` IS NULL THEN -1\n ";
313 }
314 foreach ($withRollupColumns as $withRollupColumn) {
315 $counterRollupWhen .= "\n WHEN `{$withRollupColumn}` IS NULL AND @counterRollup = {$limit} THEN {$limit}\n WHEN `{$withRollupColumn}` IS NULL THEN @counterRollup := @counterRollup + 1\n ";
316 }
317 $counterRollupExpression = "\n , CASE\n {$counterRollupWhen}\n ELSE 0\n END AS counterRollup\n ";
318 }
319 if (\false === strpos($innerQuery, ' LIMIT ') && !Schema::getInstance()->supportsSortingInSubquery()) {
320 // Setting a limit for the inner query forces the optimizer to use a temporary table, which uses the sorting
321 $innerQuery .= ' LIMIT 18446744073709551615';
322 }
323 // add a counter to the query
324 // we rely on the sorting of the inner query
325 $withCounter = "\n\t\t\tSELECT\n\t\t\t\t{$labelColumnsString},\n\t\t\t\t{$counterExpression} AS counter\n\t\t\t\t{$counterRollupExpression}\n\t\t\t\t{$additionalColumnsString}\n\t\t\tFROM\n\t\t\t\t{$initCounter}\n\t\t\t\t( {$innerQuery} ) actualQuery\n\t\t";
326 if ($withRollup && !Schema::getInstance()->supportsRankingRollupWithoutExtraSorting()) {
327 // MariaDB requires an additional sorting layer to return
328 // the counter/counterRollup values we expect
329 $rollupColumnSorts = [];
330 foreach ($withRollupColumns as $withRollupColumn) {
331 $rollupColumnSorts[] = "`{$withRollupColumn}` IS NULL";
332 }
333 $withCounter .= ' ORDER BY ' . implode(', ', $rollupColumnSorts);
334 $innerQueryOrderBy = \Piwik\DbHelper::extractOrderByFromQuery($innerQuery);
335 if (null !== $innerQueryOrderBy) {
336 // copy ORDER BY from inner query to rollup sorting
337 $withCounter .= ', ' . $innerQueryOrderBy;
338 }
339 }
340 // group by the counter - this groups "Others" because the counter stops at $limit
341 $groupBy = 'counter';
342 if ($withRollup) {
343 // group rollups additionally by the rollup counter and the
344 // full "Others" switch to ensure correct secondary level "Others" calculation
345 $groupBy .= ', counterRollup, ' . implode(', ', $withRollupOthersGroupBy);
346 }
347 if ($this->partitionColumn !== \false) {
348 $groupBy .= ', `' . $this->partitionColumn . '`';
349 }
350 $groupOthers = "\n\t\t\tSELECT\n\t\t\t\t{$labelColumnsOthersSwitch}\n\t\t\t\t{$additionalColumnsAggregatedString}\n\t\t\tFROM ( {$withCounter} ) AS withCounter\n\t\t\tGROUP BY {$groupBy}\n\t\t";
351 if ($withRollup) {
352 // Sort the final result if a rollup was used
353 // to ensure rollup values are returned first, and "Others" last
354 $groupOthers .= " ORDER BY counter, counterRollup";
355 } elseif (!Schema::getInstance()->supportsSortingInSubquery()) {
356 // When subqueries aren't sorted, we need to sort the result manually again
357 $groupOthers .= " ORDER BY counter";
358 }
359 return $groupOthers;
360 }
361 private function getCounterExpression($limit, bool $withRollup = \false)
362 {
363 $whens = array();
364 if ($this->columnToMarkExcludedRows !== \false) {
365 // when a row has been specified that marks which records should be excluded
366 // from limiting, we don't give those rows the normal counter but -1 times the
367 // value they had before. this way, they have a separate number space (i.e. negative
368 // integers).
369 $whens[] = "WHEN {$this->columnToMarkExcludedRows} != 0 THEN -1 * {$this->columnToMarkExcludedRows}";
370 }
371 if ($withRollup) {
372 foreach (array_keys($this->labelColumns) as $column) {
373 $whens[] = "WHEN `{$column}` IS NULL THEN -1";
374 }
375 }
376 if ($this->partitionColumn !== \false) {
377 // partition: one counter per possible value
378 foreach ($this->partitionColumnValues as $value) {
379 $isValue = '`' . $this->partitionColumn . '` = ' . intval($value);
380 $counter = '@counter' . intval($value);
381 $whens[] = "WHEN {$isValue} AND {$counter} = {$limit} THEN {$limit}";
382 $whens[] = "WHEN {$isValue} THEN {$counter}:={$counter}+1";
383 }
384 $whens[] = "ELSE 0";
385 } else {
386 // no partitioning: add a single counter
387 $whens[] = "WHEN @counter = {$limit} THEN {$limit}";
388 $whens[] = "ELSE @counter:=@counter+1";
389 }
390 return "\n\t\t\tCASE\n\t\t\t\t" . implode("\n\t\t\t\t", $whens) . "\n\t\t\tEND\n\t\t";
391 }
392 }
393