CliMulti.php — Matomo Analytics – Powerful, Privacy-First Insights for WordPress 5.10.0

matomo / app / core / CliMulti.php

matomo / app / core Last commit date

API 3 months ago Access 5 months ago Application 3 months ago Archive 3 months ago ArchiveProcessor 3 months ago Archiver 2 years ago AssetManager 3 months ago Auth 8 months ago Category 8 months ago Changes 3 months ago CliMulti 1 year ago Columns 3 months ago Concurrency 3 months ago Config 3 months ago Container 3 months ago CronArchive 5 months ago DataAccess 3 months ago DataFiles 2 years ago DataTable 3 months ago Db 3 months ago DeviceDetector 1 year ago Email 2 years ago Exception 6 months ago Http 6 months ago Intl 5 months ago Log 2 years ago Mail 1 year ago Measurable 8 months ago Menu 3 months ago Metrics 5 months ago Notification 8 months ago Period 3 months ago Plugin 3 months ago Policy 3 months ago ProfessionalServices 1 year ago Report 1 year ago ReportRenderer 5 months ago Request 5 months ago Scheduler 3 months ago Segment 3 months ago Session 3 months ago Settings 3 months ago Tracker 3 months ago Translation 3 months ago Twig 1 year ago UpdateCheck 5 months ago Updater 3 months ago Updates 3 months ago Validators 1 year ago View 3 months ago ViewDataTable 3 months ago Visualization 1 year ago Widget 3 months ago .htaccess 2 years ago Access.php 3 months ago Archive.php 3 months ago ArchiveProcessor.php 3 months ago AssetManager.php 3 months ago Auth.php 8 months ago AuthResult.php 8 months ago BaseFactory.php 2 years ago Cache.php 2 years ago CacheId.php 6 months ago CliMulti.php 3 months ago Common.php 3 months ago Config.php 3 months ago Console.php 5 months ago Context.php 2 years ago Cookie.php 1 year ago CronArchive.php 3 months ago DI.php 5 months ago DataArray.php 3 months ago DataTable.php 3 months ago Date.php 3 months ago Db.php 3 months ago DbHelper.php 3 months ago Development.php 1 year ago ErrorHandler.php 8 months ago EventDispatcher.php 3 months ago ExceptionHandler.php 6 months ago FileIntegrity.php 3 months ago Filechecks.php 1 year ago Filesystem.php 3 months ago FrontController.php 6 months ago Http.php 3 months ago IP.php 1 year ago Log.php 5 months ago LogDeleter.php 1 year ago Mail.php 1 year ago Metrics.php 3 months ago NoAccessException.php 2 years ago Nonce.php 8 months ago Notification.php 3 months ago NumberFormatter.php 7 months ago Option.php 7 months ago Period.php 3 months ago Piwik.php 3 months ago Plugin.php 3 months ago Process.php 3 months ago Profiler.php 8 months ago ProxyHeaders.php 6 months ago ProxyHttp.php 7 months ago QuickForm2.php 5 months ago RankingQuery.php 3 months ago ReportRenderer.php 3 months ago Request.php 3 months ago Segment.php 3 months ago Sequence.php 8 months ago Session.php 5 months ago SettingsPiwik.php 3 months ago SettingsServer.php 1 year ago Singleton.php 2 years ago Site.php 3 months ago SiteContentDetector.php 3 months ago SupportedBrowser.php 2 years ago TCPDF.php 1 year ago Theme.php 1 year ago Timer.php 3 months ago Tracker.php 3 months ago Twig.php 3 months ago Unzip.php 1 year ago UpdateCheck.php 3 months ago Updater.php 3 months ago UpdaterErrorException.php 2 years ago Updates.php 5 months ago Url.php 5 months ago UrlHelper.php 3 months ago Version.php 3 months ago View.php 3 months ago bootstrap.php 1 year ago dispatch.php 2 years ago testMinimumPhpVersion.php 8 months ago

CliMulti.php

543 lines

1	<?php
2
3	/**
4	* Matomo - free/libre analytics platform
5	*
6	* @link https://matomo.org
7	* @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
8	*/
9	namespace Piwik;
10
11	use Piwik\Archiver\Request;
12	use Piwik\CliMulti\CliPhp;
13	use Piwik\CliMulti\Output;
14	use Piwik\CliMulti\OutputInterface;
15	use Piwik\CliMulti\Process;
16	use Piwik\CliMulti\ProcessSymfony;
17	use Piwik\CliMulti\StaticOutput;
18	use Piwik\Container\StaticContainer;
19	use Piwik\Log\LoggerInterface;
20	use Piwik\Log\NullLogger;
21	use Piwik\Plugins\CoreConsole\FeatureFlags\CliMultiProcessSymfony;
22	use Piwik\Plugins\FeatureFlags\FeatureFlagManager;
23	use Throwable;
24	/**
25	* Class CliMulti.
26	*/
27	class CliMulti
28	{
29	public const BASE_WAIT_TIME = 250000;
30	// 250 * 1000 = 250ms
31	/**
32	* If set to true or false it will overwrite whether async is supported or not.
33	*
34	* @var null\|bool
35	*/
36	public $supportsAsync = null;
37	/**
38	* If set to true or false it will overwrite whether async using Symfony\Process is supported or not.
39	*
40	* Will only be checked if the default $supportsAsync is true.
41	*
42	* @var null\|bool
43	*/
44	public $supportsAsyncSymfony = null;
45	/**
46	* @var Process[]\|ProcessSymfony[]
47	*/
48	private $processes = [];
49	/**
50	* If set it will issue at most concurrentProcessesLimit requests
51	* @var int
52	*/
53	private $concurrentProcessesLimit = null;
54	/**
55	* @var OutputInterface[]
56	*/
57	private $outputs = [];
58	private $acceptInvalidSSLCertificate = \false;
59	/**
60	* @var bool
61	*/
62	private $runAsSuperUser = \false;
63	/**
64	* Only used when doing synchronous curl requests.
65	*
66	* @var string
67	*/
68	private $urlToPiwik = null;
69	private $phpCliOptions = '';
70	/**
71	* @var callable
72	*/
73	private $onProcessFinish = null;
74	/**
75	* @var Timer[]
76	*/
77	protected $timers = [];
78	protected $isTimingRequests = \false;
79	/**
80	* @var LoggerInterface
81	*/
82	private $logger;
83	/**
84	* @var int\|null
85	*/
86	private $signal = null;
87	public function __construct(?LoggerInterface $logger = null)
88	{
89	$this->supportsAsync = $this->supportsAsync();
90	$this->supportsAsyncSymfony = $this->supportsAsyncSymfony();
91	$this->logger = $logger ?: new NullLogger();
92	}
93	public function handleSignal(int $signal) : void
94	{
95	$this->signal = $signal;
96	if (\SIGTERM !== $signal) {
97	return;
98	}
99	foreach ($this->processes as $process) {
100	if ($process instanceof ProcessSymfony) {
101	$this->logger->debug('Aborting command: {command} [method = asyncCliSymfony]', ['command' => $process->getCommandLine()]);
102	$process->stop(0);
103	}
104	}
105	}
106	/**
107	* It will request all given URLs in parallel (async) using the CLI and wait until all requests are finished.
108	* If multi cli is not supported (eg windows) it will initiate an HTTP request instead (not async).
109	*
110	* @param string[] $piwikUrls An array of urls, for instance:
111	*
112	* `array('https://www.example.com/matomo?module=API...')`
113	*
114	* Make sure query parameter values are properly encoded in the URLs.
115	*
116	* @return array The response of each URL in the same order as the URLs. The array can contain null values in case
117	* there was a problem with a request, for instance if the process died unexpected.
118	*/
119	public function request(array $piwikUrls)
120	{
121	if ($this->isTimingRequests) {
122	foreach ($piwikUrls as $url) {
123	$this->timers[] = new \Piwik\Timer();
124	}
125	}
126	$chunks = [$piwikUrls];
127	if ($this->concurrentProcessesLimit) {
128	$chunks = array_chunk($piwikUrls, $this->concurrentProcessesLimit);
129	}
130	$results = [];
131	foreach ($chunks as $urlsChunk) {
132	if (null !== $this->signal) {
133	$this->logSkippedRequests($urlsChunk);
134	continue;
135	}
136	$results = array_merge($results, $this->requestUrls($urlsChunk));
137	}
138	return $results;
139	}
140	/**
141	* Forwards the given configuration options to the PHP cli command.
142	* @param string $phpCliOptions eg "-d memory_limit=8G -c=path/to/php.ini"
143	*/
144	public function setPhpCliConfigurationOptions($phpCliOptions)
145	{
146	$this->phpCliOptions = (string) $phpCliOptions;
147	}
148	/**
149	* Ok, this sounds weird. Why should we care about ssl certificates when we are in CLI mode? It is needed for
150	* our simple fallback mode for Windows where we initiate HTTP requests instead of CLI.
151	* @param $acceptInvalidSSLCertificate
152	*/
153	public function setAcceptInvalidSSLCertificate($acceptInvalidSSLCertificate)
154	{
155	$this->acceptInvalidSSLCertificate = $acceptInvalidSSLCertificate;
156	}
157	/**
158	* @param $limit int Maximum count of requests to issue in parallel
159	*/
160	public function setConcurrentProcessesLimit($limit)
161	{
162	$this->concurrentProcessesLimit = $limit;
163	}
164	public function runAsSuperUser($runAsSuperUser = \true)
165	{
166	$this->runAsSuperUser = $runAsSuperUser;
167	}
168	private function start($piwikUrls)
169	{
170	$numUrls = count($piwikUrls);
171	foreach ($piwikUrls as $index => $url) {
172	$shouldStart = null;
173	if ($url instanceof Request) {
174	$shouldStart = $url->start();
175	}
176	$cmdId = $this->generateCommandId($url) . $index;
177	if ($shouldStart === Request::ABORT) {
178	// output is needed to ensure same order of url to response
179	$output = new StaticOutput($cmdId);
180	$output->write(serialize(array('aborted' => '1')));
181	$this->outputs[] = $output;
182	} else {
183	$this->executeUrlCommand($cmdId, $url, $numUrls);
184	}
185	}
186	}
187	private function executeUrlCommand($cmdId, $url, $numUrls)
188	{
189	if ($this->supportsAsync) {
190	if ($this->supportsAsyncSymfony) {
191	$output = new StaticOutput($cmdId);
192	$this->executeAsyncCliSymfony($url, $cmdId);
193	} elseif ($numUrls === 1) {
194	$output = new StaticOutput($cmdId);
195	$this->executeSyncCli($url, $output);
196	} else {
197	$output = new Output($cmdId);
198	$this->executeAsyncCli($url, $output, $cmdId);
199	}
200	} else {
201	$output = new StaticOutput($cmdId);
202	$this->executeNotAsyncHttp($url, $output);
203	}
204	$this->outputs[] = $output;
205	}
206	private function buildCommand($hostname, $query, $outputFileIfAsync, $doEscapeArg = \true)
207	{
208	$bin = $this->findPhpBinary();
209	$superuserCommand = $this->runAsSuperUser ? "--superuser" : "";
210	$append = '2>&1';
211	if ($outputFileIfAsync) {
212	$append = sprintf(' > %s 2>&1 &', $outputFileIfAsync);
213	}
214	if ($doEscapeArg) {
215	$hostname = escapeshellarg($hostname);
216	$query = escapeshellarg($query);
217	}
218	return sprintf('%s %s %s/console climulti:request -q --matomo-domain=%s %s %s %s', $bin, $this->phpCliOptions, PIWIK_INCLUDE_PATH, $hostname, $superuserCommand, $query, $append);
219	}
220	private function getResponse()
221	{
222	$response = array();
223	foreach ($this->outputs as $output) {
224	$content = $output->get();
225	// Remove output that can be ignored in climulti . works around some WordPress setups where the hash bang may
226	// be printed
227	$search = '#!/usr/bin/env php';
228	if (!empty($content) && is_string($content) && mb_substr(trim($content), 0, strlen($search)) === $search) {
229	$content = trim(mb_substr(trim($content), strlen($search)));
230	}
231	$response[] = $content;
232	}
233	return $response;
234	}
235	private function hasFinished() : bool
236	{
237	$hasFinished = \true;
238	foreach ($this->processes as $index => $process) {
239	if ($process instanceof ProcessSymfony) {
240	$processFinished = $this->hasFinishedProcessSymfony($process);
241	} else {
242	$processFinished = $this->hasFinishedProcess($process);
243	}
244	if (\true === $processFinished) {
245	// prevent from checking this process over and over again
246	unset($this->processes[$index]);
247	if ($this->isTimingRequests) {
248	$this->timers[$index]->finish();
249	}
250	if ($this->onProcessFinish) {
251	$pid = $process->getPid();
252	$onProcessFinish = $this->onProcessFinish;
253	$onProcessFinish($pid);
254	}
255	}
256	$hasFinished = $hasFinished && $processFinished;
257	}
258	return $hasFinished;
259	}
260	private function hasFinishedProcess(Process $process) : bool
261	{
262	$hasStarted = $process->hasStarted();
263	if (!$hasStarted && 8 <= $process->getSecondsSinceCreation()) {
264	// if process was created more than 8 seconds ago but still not started there must be something wrong.
265	// ==> declare the process as finished
266	$process->finishProcess();
267	return \true;
268	} elseif (!$hasStarted) {
269	return \false;
270	}
271	if ($process->isRunning()) {
272	return \false;
273	}
274	$pid = $process->getPid();
275	foreach ($this->outputs as $output) {
276	if ($output->getOutputId() === $pid && $output->isAbnormal()) {
277	$process->finishProcess();
278	return \true;
279	}
280	}
281	return $process->hasFinished();
282	}
283	private function hasFinishedProcessSymfony(ProcessSymfony $process) : bool
284	{
285	$finished = $process->isStarted() && !$process->isRunning();
286	if ($finished) {
287	foreach ($this->outputs as $output) {
288	if ($output->getOutputId() !== $process->getCommandId()) {
289	continue;
290	}
291	$output->write($process->getOutput());
292	break;
293	}
294	}
295	return $finished;
296	}
297	private function generateCommandId($command)
298	{
299	return substr(\Piwik\Common::hash($command . microtime(\true) . rand(0, 99999)), 0, 100);
300	}
301	/**
302	* What is missing under windows? Detection whether a process is still running in Process::isProcessStillRunning
303	* and how to send a process into background in start()
304	*/
305	public function supportsAsync()
306	{
307	$supportsAsync = Process::isSupported() && !\Piwik\Common::isPhpCgiType() && $this->findPhpBinary();
308	/**
309	* Triggered to allow plugins to force the usage of async cli multi execution or to disable it.
310	*
311	* Example
312	*
313	* public function supportsAsync(&$supportsAsync)
314	* {
315	* $supportsAsync = false; // do not allow async climulti execution
316	* }
317	*
318	* @param bool &$supportsAsync Whether async is supported or not.
319	*/
320	\Piwik\Piwik::postEvent('CliMulti.supportsAsync', array(&$supportsAsync));
321	return $supportsAsync;
322	}
323	/**
324	* Returns whether Symfony\Process instead of the default Piwik\Process for
325	* async cli multi execution.
326	*
327	* Requirements:
328	* - supportsAsync has to be true
329	* - feature flag "CliMultiProcessSymfony" has to be active
330	* - proc_open has to be available
331	*
332	* Several of the regular supportsAsync requirements may be obsolete after
333	* a complete switch has been done.
334	*/
335	public function supportsAsyncSymfony() : bool
336	{
337	// When updating from an older version the required symfony component
338	// may not be available. We have to verify that outside of the
339	// CliMulti\ProcessSymfony wrapper because it extends the component.
340	if (!$this->supportsAsync \|\| Process::isMethodDisabled('proc_open') \|\| !class_exists(\Symfony\Component\Process\Process::class)) {
341	return \false;
342	}
343	// The required DI configuration may not be loaded during the update process.
344	// This can happen for an upgrade from a version that did not yet contain
345	// the feature flag plugin.
346	try {
347	$featureFlagManager = StaticContainer::get(FeatureFlagManager::class);
348	return $featureFlagManager->isFeatureActive(CliMultiProcessSymfony::class);
349	} catch (Throwable $e) {
350	return \false;
351	}
352	}
353	private function findPhpBinary()
354	{
355	$cliPhp = new CliPhp();
356	return $cliPhp->findPhpBinary();
357	}
358	private function cleanup()
359	{
360	foreach ($this->processes as $process) {
361	if ($process instanceof ProcessSymfony) {
362	$process->stop(0);
363	} else {
364	$process->finishProcess();
365	}
366	}
367	foreach ($this->outputs as $output) {
368	$output->destroy();
369	}
370	$this->processes = array();
371	$this->outputs = array();
372	}
373	/**
374	* Remove files older than one week. They should be cleaned up automatically after each request but for whatever
375	* reason there can be always some files left.
376	*/
377	public static function cleanupNotRemovedFiles()
378	{
379	$timeOneWeekAgo = strtotime('-1 week');
380	$files = _glob(self::getTmpPath() . '/*');
381	if (empty($files)) {
382	return;
383	}
384	foreach ($files as $file) {
385	if (file_exists($file)) {
386	$timeLastModified = filemtime($file);
387	if ($timeLastModified !== \false && $timeOneWeekAgo > $timeLastModified) {
388	unlink($file);
389	}
390	}
391	}
392	}
393	public static function getTmpPath()
394	{
395	return StaticContainer::get('path.tmp') . '/climulti';
396	}
397	private function executeAsyncCli($url, Output $output, $cmdId)
398	{
399	$this->processes[] = new Process($cmdId);
400	$url = $this->appendTestmodeParamToUrlIfNeeded($url);
401	$query = \Piwik\UrlHelper::getQueryFromUrl($url, ['pid' => $cmdId, 'runid' => \Piwik\Common::getProcessId()]);
402	$hostname = \Piwik\Url::getHost($checkIfTrusted = \false);
403	$command = $this->buildCommand($hostname, $query, $output->getPathToFile());
404	$this->logger->debug('Running command: {command} [method = {method}]', ['command' => $command, 'method' => 'asyncCli']);
405	shell_exec($command);
406	}
407	private function executeAsyncCliSymfony(string $url, string $cmdId) : void
408	{
409	$url = $this->appendTestmodeParamToUrlIfNeeded($url);
410	$query = \Piwik\UrlHelper::getQueryFromUrl($url, array());
411	$hostname = \Piwik\Url::getHost($checkIfTrusted = \false);
412	$command = $this->buildCommand($hostname, $query, '', \true);
413	$this->logger->debug('Running command: {command} [method = {method}]', ['command' => $command, 'method' => 'asyncCliSymfony']);
414	// Prepending "exec" is required to send signals to the process
415	// Not using array notation because $command can contain complex parameters
416	if ('\\' !== \DIRECTORY_SEPARATOR) {
417	$command = 'exec ' . $command;
418	}
419	$process = ProcessSymfony::fromShellCommandline($command);
420	$process->setTimeout(null);
421	$process->start();
422	$process->setCommandId($cmdId);
423	$this->processes[] = $process;
424	}
425	private function executeSyncCli($url, StaticOutput $output)
426	{
427	$url = $this->appendTestmodeParamToUrlIfNeeded($url);
428	$query = \Piwik\UrlHelper::getQueryFromUrl($url, array());
429	$hostname = \Piwik\Url::getHost($checkIfTrusted = \false);
430	$command = $this->buildCommand($hostname, $query, '', \true);
431	$this->logger->debug('Running command: {command} [method = {method}]', ['command' => $command, 'method' => 'syncCli']);
432	$result = shell_exec($command);
433	if ($result) {
434	$result = trim($result);
435	}
436	$output->write($result);
437	}
438	private function executeNotAsyncHttp($url, StaticOutput $output)
439	{
440	$piwikUrl = $this->urlToPiwik ?: \Piwik\SettingsPiwik::getPiwikUrl();
441	if (empty($piwikUrl)) {
442	$piwikUrl = 'http://' . \Piwik\Url::getHost() . '/';
443	}
444	$url = $piwikUrl . $url;
445	if (\Piwik\Config::getInstance()->General['force_ssl'] == 1) {
446	$url = str_replace("http://", "https://", $url);
447	}
448	$requestBody = null;
449	if ($this->runAsSuperUser) {
450	$tokenAuth = self::getSuperUserTokenAuth();
451	if (strpos($url, '?') === \false) {
452	$url .= '?';
453	} else {
454	$url .= '&';
455	}
456	$requestBody = 'token_auth=' . $tokenAuth;
457	}
458	$response = null;
459	try {
460	$this->logger->debug("Execute HTTP API request: " . $url);
461	$response = \Piwik\Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = \false, $this->acceptInvalidSSLCertificate, \false, \false, 'POST', null, null, $requestBody, [], $forcePost = \true);
462	$output->write($response);
463	} catch (\Exception $e) {
464	$message = "Got invalid response from API request: {$url}. ";
465	if (empty($response)) {
466	$message .= "The response was empty. This usually means a server error. This solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. Please check your Web server Error Log file for more details.";
467	} else {
468	$message .= "Response was '" . $e->getMessage() . "'";
469	}
470	$output->write($message);
471	$this->logger->debug($message, ['exception' => $e]);
472	}
473	}
474	private function appendTestmodeParamToUrlIfNeeded($url)
475	{
476	$isTestMode = defined('PIWIK_TEST_MODE');
477	if ($isTestMode && \false === strpos($url, '?')) {
478	$url .= "?testmode=1";
479	} elseif ($isTestMode) {
480	$url .= "&testmode=1";
481	}
482	return $url;
483	}
484	/**
485	* @param array $piwikUrls
486	* @return array
487	*/
488	private function requestUrls(array $piwikUrls)
489	{
490	$this->start($piwikUrls);
491	$startTime = time();
492	do {
493	$elapsed = time() - $startTime;
494	$timeToWait = $this->getTimeToWaitBeforeNextCheck($elapsed);
495	if (count($this->processes)) {
496	usleep($timeToWait);
497	}
498	} while (!$this->hasFinished());
499	$results = $this->getResponse();
500	$this->cleanup();
501	self::cleanupNotRemovedFiles();
502	return $results;
503	}
504	private function logSkippedRequests(array $urls) : void
505	{
506	foreach ($urls as $url) {
507	$this->logger->debug('Skipped climulti:request after abort signal received: {url}', ['url' => $url]);
508	}
509	}
510	private static function getSuperUserTokenAuth()
511	{
512	return \Piwik\Piwik::requestTemporarySystemAuthToken('CliMultiNonAsyncArchive', 36);
513	}
514	public function setUrlToPiwik($urlToPiwik)
515	{
516	$this->urlToPiwik = $urlToPiwik;
517	}
518	public function onProcessFinish(callable $callback)
519	{
520	$this->onProcessFinish = $callback;
521	}
522	// every minute that passes adds an extra 100ms to the wait time. so 5 minutes results in 500ms extra, 20mins results in 2s extra.
523	private function getTimeToWaitBeforeNextCheck($elapsed)
524	{
525	$minutes = floor($elapsed / 60);
526	return self::BASE_WAIT_TIME + $minutes * 100000;
527	// 100 * 1000 = 100ms
528	}
529	public static function isCliMultiRequest()
530	{
531	return \Piwik\Common::getRequestVar('pid', \false) !== \false;
532	}
533	public function timeRequests()
534	{
535	$this->timers = [];
536	$this->isTimingRequests = \true;
537	}
538	public function getTimers()
539	{
540	return $this->timers;
541	}
542	}
543