CliMulti.php — Matomo Analytics – Powerful, Privacy-First Insights for WordPress 1.3.1

matomo / app / core / CliMulti.php

matomo / app / core Last commit date

API 6 years ago Access 6 years ago Application 6 years ago Archive 6 years ago ArchiveProcessor 6 years ago Archiver 6 years ago AssetManager 6 years ago Auth 6 years ago Category 6 years ago CliMulti 6 years ago Columns 6 years ago Composer 6 years ago Concurrency 6 years ago Config 6 years ago Container 6 years ago CronArchive 6 years ago DataAccess 5 years ago DataFiles 6 years ago DataTable 6 years ago Db 6 years ago DeviceDetector 5 years ago Email 6 years ago Exception 6 years ago Http 6 years ago Intl 6 years ago Mail 6 years ago Measurable 6 years ago Menu 6 years ago Metrics 6 years ago Notification 6 years ago Period 6 years ago Plugin 6 years ago ProfessionalServices 6 years ago Report 6 years ago ReportRenderer 6 years ago Scheduler 6 years ago Segment 6 years ago Session 6 years ago Settings 6 years ago Tracker 5 years ago Translation 6 years ago UpdateCheck 6 years ago Updater 6 years ago Updates 6 years ago Validators 6 years ago View 6 years ago ViewDataTable 6 years ago Visualization 6 years ago Widget 6 years ago .htaccess 6 years ago Access.php 6 years ago Archive.php 6 years ago ArchiveProcessor.php 6 years ago AssetManager.php 6 years ago Auth.php 6 years ago BaseFactory.php 6 years ago Cache.php 6 years ago CacheId.php 6 years ago CliMulti.php 6 years ago Common.php 6 years ago Config.php 6 years ago Console.php 6 years ago Context.php 6 years ago Cookie.php 5 years ago CronArchive.php 5 years ago DataArray.php 6 years ago DataTable.php 6 years ago Date.php 6 years ago Db.php 6 years ago DbHelper.php 6 years ago Development.php 6 years ago DeviceDetectorFactory.php 6 years ago ErrorHandler.php 6 years ago EventDispatcher.php 6 years ago ExceptionHandler.php 6 years ago FileIntegrity.php 6 years ago Filechecks.php 6 years ago Filesystem.php 6 years ago FrontController.php 6 years ago Http.php 6 years ago IP.php 6 years ago Log.php 6 years ago LogDeleter.php 6 years ago Mail.php 6 years ago Metrics.php 6 years ago MetricsFormatter.php 6 years ago Nonce.php 5 years ago Notification.php 6 years ago NumberFormatter.php 6 years ago Option.php 5 years ago Period.php 6 years ago Piwik.php 6 years ago Plugin.php 6 years ago Profiler.php 6 years ago ProxyHeaders.php 6 years ago ProxyHttp.php 6 years ago QuickForm2.php 6 years ago RankingQuery.php 6 years ago Registry.php 6 years ago ReportRenderer.php 6 years ago ScheduledTask.php 6 years ago Segment.php 6 years ago Sequence.php 6 years ago Session.php 6 years ago SettingsPiwik.php 6 years ago SettingsServer.php 6 years ago Singleton.php 6 years ago Site.php 6 years ago TCPDF.php 6 years ago TaskScheduler.php 6 years ago Theme.php 6 years ago Timer.php 6 years ago Tracker.php 6 years ago Translate.php 6 years ago Twig.php 6 years ago Unzip.php 6 years ago UpdateCheck.php 6 years ago Updater.php 6 years ago Updates.php 6 years ago Url.php 6 years ago UrlHelper.php 6 years ago Version.php 5 years ago View.php 6 years ago bootstrap.php 6 years ago dispatch.php 6 years ago testMinimumPhpVersion.php 6 years ago

CliMulti.php

472 lines

1	<?php
2	/**
3	* Piwik - free/libre analytics platform
4	*
5	* @link https://matomo.org
6	* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	*/
8	namespace Piwik;
9
10	use Piwik\Archiver\Request;
11	use Piwik\CliMulti\CliPhp;
12	use Piwik\CliMulti\Output;
13	use Piwik\CliMulti\Process;
14	use Piwik\Container\StaticContainer;
15
16	/**
17	* Class CliMulti.
18	*/
19	class CliMulti
20	{
21	const BASE_WAIT_TIME = 250000; // 250 * 1000 = 250ms
22
23	/**
24	* If set to true or false it will overwrite whether async is supported or not.
25	*
26	* @var null\|bool
27	*/
28	public $supportsAsync = null;
29
30	/**
31	* @var Process[]
32	*/
33	private $processes = array();
34
35	/**
36	* If set it will issue at most concurrentProcessesLimit requests
37	* @var int
38	*/
39	private $concurrentProcessesLimit = null;
40
41	/**
42	* @var Output[]
43	*/
44	private $outputs = array();
45
46	private $acceptInvalidSSLCertificate = false;
47
48	/**
49	* @var bool
50	*/
51	private $runAsSuperUser = false;
52
53	/**
54	* Only used when doing synchronous curl requests.
55	*
56	* @var string
57	*/
58	private $urlToPiwik = null;
59
60	private $phpCliOptions = '';
61
62	/**
63	* @var callable
64	*/
65	private $onProcessFinish = null;
66
67	public function __construct()
68	{
69	$this->supportsAsync = $this->supportsAsync();
70	}
71
72	/**
73	* It will request all given URLs in parallel (async) using the CLI and wait until all requests are finished.
74	* If multi cli is not supported (eg windows) it will initiate an HTTP request instead (not async).
75	*
76	* @param string[] $piwikUrls An array of urls, for instance:
77	*
78	* `array('http://www.example.com/piwik?module=API...')`
79	*
80	* Make sure query parameter values are properly encoded in the URLs.
81	*
82	* @return array The response of each URL in the same order as the URLs. The array can contain null values in case
83	* there was a problem with a request, for instance if the process died unexpected.
84	*/
85	public function request(array $piwikUrls)
86	{
87	$chunks = array($piwikUrls);
88	if ($this->concurrentProcessesLimit) {
89	$chunks = array_chunk($piwikUrls, $this->concurrentProcessesLimit);
90	}
91
92	$results = array();
93	foreach ($chunks as $urlsChunk) {
94	$results = array_merge($results, $this->requestUrls($urlsChunk));
95	}
96
97	return $results;
98	}
99
100	/**
101	* Forwards the given configuration options to the PHP cli command.
102	* @param string $phpCliOptions eg "-d memory_limit=8G -c=path/to/php.ini"
103	*/
104	public function setPhpCliConfigurationOptions($phpCliOptions)
105	{
106	$this->phpCliOptions = (string) $phpCliOptions;
107	}
108
109	/**
110	* Ok, this sounds weird. Why should we care about ssl certificates when we are in CLI mode? It is needed for
111	* our simple fallback mode for Windows where we initiate HTTP requests instead of CLI.
112	* @param $acceptInvalidSSLCertificate
113	*/
114	public function setAcceptInvalidSSLCertificate($acceptInvalidSSLCertificate)
115	{
116	$this->acceptInvalidSSLCertificate = $acceptInvalidSSLCertificate;
117	}
118
119	/**
120	* @param $limit int Maximum count of requests to issue in parallel
121	*/
122	public function setConcurrentProcessesLimit($limit)
123	{
124	$this->concurrentProcessesLimit = $limit;
125	}
126
127	public function runAsSuperUser($runAsSuperUser = true)
128	{
129	$this->runAsSuperUser = $runAsSuperUser;
130	}
131
132	private function start($piwikUrls)
133	{
134	foreach ($piwikUrls as $index => $url) {
135	$shouldStart = null;
136	if ($url instanceof Request) {
137	$shouldStart = $url->start();
138	}
139
140	$cmdId = $this->generateCommandId($url) . $index;
141
142	if ($shouldStart === Request::ABORT) {
143	// output is needed to ensure same order of url to response
144	$output = new Output($cmdId);
145	$output->write(serialize(array('aborted' => '1')));
146	$this->outputs[] = $output;
147	} else {
148	$this->executeUrlCommand($cmdId, $url);
149	}
150	}
151	}
152
153	private function executeUrlCommand($cmdId, $url)
154	{
155	$output = new Output($cmdId);
156
157	if ($this->supportsAsync) {
158	$this->executeAsyncCli($url, $output, $cmdId);
159	} else {
160	$this->executeNotAsyncHttp($url, $output);
161	}
162
163	$this->outputs[] = $output;
164	}
165
166	private function buildCommand($hostname, $query, $outputFile, $doEsacpeArg = true)
167	{
168	$bin = $this->findPhpBinary();
169	$superuserCommand = $this->runAsSuperUser ? "--superuser" : "";
170
171	if ($doEsacpeArg) {
172	$hostname = escapeshellarg($hostname);
173	$query = escapeshellarg($query);
174	}
175
176	return sprintf('%s %s %s/console climulti:request -q --matomo-domain=%s %s %s > %s 2>&1 &',
177	$bin, $this->phpCliOptions, PIWIK_INCLUDE_PATH, $hostname, $superuserCommand, $query, $outputFile);
178	}
179
180	private function getResponse()
181	{
182	$response = array();
183
184	foreach ($this->outputs as $output) {
185	$response[] = $output->get();
186	}
187
188	return $response;
189	}
190
191	private function hasFinished()
192	{
193	foreach ($this->processes as $index => $process) {
194	$hasStarted = $process->hasStarted();
195
196	if (!$hasStarted && 8 <= $process->getSecondsSinceCreation()) {
197	// if process was created more than 8 seconds ago but still not started there must be something wrong.
198	// ==> declare the process as finished
199	$process->finishProcess();
200	continue;
201	} elseif (!$hasStarted) {
202	return false;
203	}
204
205	if ($process->isRunning()) {
206	return false;
207	}
208
209	$pid = $process->getPid();
210	foreach ($this->outputs as $output) {
211	if ($output->getOutputId() === $pid && $output->isAbnormal()) {
212	$process->finishProcess();
213	return true;
214	}
215	}
216
217	if ($process->hasFinished()) {
218	// prevent from checking this process over and over again
219	unset($this->processes[$index]);
220
221	if ($this->onProcessFinish) {
222	$onProcessFinish = $this->onProcessFinish;
223	$onProcessFinish($pid);
224	}
225	}
226	}
227
228	return true;
229	}
230
231	private function generateCommandId($command)
232	{
233	return substr(Common::hash($command . microtime(true) . rand(0, 99999)), 0, 100);
234	}
235
236	/**
237	* What is missing under windows? Detection whether a process is still running in Process::isProcessStillRunning
238	* and how to send a process into background in start()
239	*/
240	public function supportsAsync()
241	{
242	$supportsAsync = Process::isSupported() && !Common::isPhpCgiType() && $this->findPhpBinary();
243
244	/**
245	* Triggered to allow plugins to force the usage of async cli multi execution or to disable it.
246	*
247	* Example
248	*
249	* public function supportsAsync(&$supportsAsync)
250	* {
251	* $supportsAsync = false; // do not allow async climulti execution
252	* }
253	*
254	* @param bool &$supportsAsync Whether async is supported or not.
255	*/
256	Piwik::postEvent('CliMulti.supportsAsync', array(&$supportsAsync));
257
258	return $supportsAsync;
259	}
260
261	private function findPhpBinary()
262	{
263	$cliPhp = new CliPhp();
264	return $cliPhp->findPhpBinary();
265	}
266
267	private function cleanup()
268	{
269	foreach ($this->processes as $pid) {
270	$pid->finishProcess();
271	}
272
273	foreach ($this->outputs as $output) {
274	$output->destroy();
275	}
276
277	$this->processes = array();
278	$this->outputs = array();
279	}
280
281	/**
282	* Remove files older than one week. They should be cleaned up automatically after each request but for whatever
283	* reason there can be always some files left.
284	*/
285	public static function cleanupNotRemovedFiles()
286	{
287	$timeOneWeekAgo = strtotime('-1 week');
288
289	$files = _glob(self::getTmpPath() . '/*');
290	if (empty($files)) {
291	return;
292	}
293
294	foreach ($files as $file) {
295	if (file_exists($file)) {
296	$timeLastModified = filemtime($file);
297
298	if ($timeLastModified !== false && $timeOneWeekAgo > $timeLastModified) {
299	unlink($file);
300	}
301	}
302	}
303	}
304
305	public static function getTmpPath()
306	{
307	return StaticContainer::get('path.tmp') . '/climulti';
308	}
309
310	public function isCommandAlreadyRunning($url)
311	{
312	if (defined('PIWIK_TEST_MODE')) {
313	return false; // skip check in tests as it might result in random failures
314	}
315
316	if (!$this->supportsAsync) {
317	// we cannot detect if web archive is still running
318	return false;
319	}
320
321	$query = UrlHelper::getQueryFromUrl($url, array('pid' => 'removeme'));
322	$hostname = Url::getHost($checkIfTrusted = false);
323	$commandToCheck = $this->buildCommand($hostname, $query, $output = '', $escape = false);
324
325	$currentlyRunningJobs = `ps aux`;
326
327	$posStart = strpos($commandToCheck, 'console climulti');
328	$posPid = strpos($commandToCheck, '&pid='); // the pid is random each time so we need to ignore it.
329	$shortendCommand = substr($commandToCheck, $posStart, $posPid - $posStart);
330	// equals eg console climulti:request -q --matomo-domain= --superuser module=API&method=API.get&idSite=1&period=month&date=2018-04-08,2018-04-30&format=php&trigger=archivephp
331	$shortendCommand = preg_replace("/([&])date=.*?(&\|$)/", "", $shortendCommand);
332	$currentlyRunningJobs = preg_replace("/([&])date=.*?(&\|$)/", "", $currentlyRunningJobs);
333
334	if (strpos($currentlyRunningJobs, $shortendCommand) !== false) {
335	Log::debug($shortendCommand . ' is already running');
336	return true;
337	}
338
339	return false;
340	}
341
342	private function executeAsyncCli($url, Output $output, $cmdId)
343	{
344	$this->processes[] = new Process($cmdId);
345
346	$url = $this->appendTestmodeParamToUrlIfNeeded($url);
347	$query = UrlHelper::getQueryFromUrl($url, array('pid' => $cmdId, 'runid' => getmypid()));
348	$hostname = Url::getHost($checkIfTrusted = false);
349	$command = $this->buildCommand($hostname, $query, $output->getPathToFile());
350
351	Log::debug($command);
352	shell_exec($command);
353	}
354
355	private function executeNotAsyncHttp($url, Output $output)
356	{
357	$piwikUrl = $this->urlToPiwik ?: SettingsPiwik::getPiwikUrl();
358	if (empty($piwikUrl)) {
359	$piwikUrl = 'http://' . Url::getHost() . '/';
360	}
361
362	$url = $piwikUrl . $url;
363	if (Config::getInstance()->General['force_ssl'] == 1) {
364	$url = str_replace("http://", "https://", $url);
365	}
366
367	if ($this->runAsSuperUser) {
368	$tokenAuths = self::getSuperUserTokenAuths();
369	$tokenAuth = reset($tokenAuths);
370
371	if (strpos($url, '?') === false) {
372	$url .= '?';
373	} else {
374	$url .= '&';
375	}
376
377	$url .= 'token_auth=' . $tokenAuth;
378	}
379
380	try {
381	Log::debug("Execute HTTP API request: " . $url);
382	$response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate);
383	$output->write($response);
384	} catch (\Exception $e) {
385	$message = "Got invalid response from API request: $url. ";
386
387	if (isset($response) && empty($response)) {
388	$message .= "The response was empty. This usually means a server error. This solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. Please check your Web server Error Log file for more details.";
389	} else {
390	$message .= "Response was '" . $e->getMessage() . "'";
391	}
392
393	$output->write($message);
394
395	Log::debug($e);
396	}
397	}
398
399	private function appendTestmodeParamToUrlIfNeeded($url)
400	{
401	$isTestMode = defined('PIWIK_TEST_MODE');
402
403	if ($isTestMode && false === strpos($url, '?')) {
404	$url .= "?testmode=1";
405	} elseif ($isTestMode) {
406	$url .= "&testmode=1";
407	}
408
409	return $url;
410	}
411
412	/**
413	* @param array $piwikUrls
414	* @return array
415	*/
416	private function requestUrls(array $piwikUrls)
417	{
418	$this->start($piwikUrls);
419
420	$startTime = time();
421	do {
422	$elapsed = time() - $startTime;
423	$timeToWait = $this->getTimeToWaitBeforeNextCheck($elapsed);
424
425	usleep($timeToWait);
426	} while (!$this->hasFinished());
427
428	$results = $this->getResponse();
429	$this->cleanup();
430
431	self::cleanupNotRemovedFiles();
432
433	return $results;
434	}
435
436	private static function getSuperUserTokenAuths()
437	{
438	$tokens = array();
439
440	/**
441	* Used to be in CronArchive, moved to CliMulti.
442	*
443	* @ignore
444	*/
445	Piwik::postEvent('CronArchive.getTokenAuth', array(&$tokens));
446
447	return $tokens;
448	}
449
450	public function setUrlToPiwik($urlToPiwik)
451	{
452	$this->urlToPiwik = $urlToPiwik;
453	}
454
455	public function onProcessFinish(callable $callback)
456	{
457	$this->onProcessFinish = $callback;
458	}
459
460	// every minute that passes adds an extra 100ms to the wait time. so 5 minutes results in 500ms extra, 20mins results in 2s extra.
461	private function getTimeToWaitBeforeNextCheck($elapsed)
462	{
463	$minutes = floor($elapsed / 60);
464	return self::BASE_WAIT_TIME + $minutes * 100000; // 100 * 1000 = 100ms
465	}
466
467	public static function isCliMultiRequest()
468	{
469	return Common::getRequestVar('pid', false) !== false;
470	}
471	}
472