CliMulti.php — Matomo Analytics – Powerful, Privacy-First Insights for WordPress 4.3.0

matomo / app / core / CliMulti.php

matomo / app / core Last commit date

API 5 years ago Access 5 years ago Application 5 years ago Archive 5 years ago ArchiveProcessor 5 years ago Archiver 5 years ago AssetManager 5 years ago Auth 5 years ago Category 5 years ago CliMulti 5 years ago Columns 5 years ago Composer 5 years ago Concurrency 5 years ago Config 5 years ago Container 5 years ago CronArchive 5 years ago DataAccess 5 years ago DataFiles 5 years ago DataTable 5 years ago Db 5 years ago DeviceDetector 5 years ago Email 5 years ago Exception 5 years ago Http 5 years ago Intl 5 years ago Mail 5 years ago Measurable 5 years ago Menu 5 years ago Metrics 5 years ago Notification 5 years ago Period 5 years ago Plugin 5 years ago ProfessionalServices 5 years ago Report 5 years ago ReportRenderer 5 years ago Scheduler 5 years ago Segment 5 years ago Session 5 years ago Settings 5 years ago Tracker 5 years ago Translation 5 years ago UpdateCheck 5 years ago Updater 5 years ago Updates 5 years ago Validators 5 years ago View 5 years ago ViewDataTable 5 years ago Visualization 5 years ago Widget 5 years ago .htaccess 6 years ago Access.php 5 years ago Archive.php 5 years ago ArchiveProcessor.php 5 years ago AssetManager.php 5 years ago Auth.php 5 years ago AuthResult.php 5 years ago BaseFactory.php 5 years ago Cache.php 5 years ago CacheId.php 5 years ago CliMulti.php 5 years ago Common.php 5 years ago Config.php 5 years ago Console.php 5 years ago Context.php 5 years ago Cookie.php 5 years ago CronArchive.php 5 years ago DataArray.php 5 years ago DataTable.php 5 years ago Date.php 5 years ago Db.php 5 years ago DbHelper.php 5 years ago Development.php 5 years ago ErrorHandler.php 5 years ago EventDispatcher.php 5 years ago ExceptionHandler.php 5 years ago FileIntegrity.php 5 years ago Filechecks.php 5 years ago Filesystem.php 5 years ago FrontController.php 5 years ago Http.php 5 years ago IP.php 5 years ago Log.php 5 years ago LogDeleter.php 5 years ago Mail.php 5 years ago Metrics.php 5 years ago NoAccessException.php 5 years ago Nonce.php 5 years ago Notification.php 5 years ago NumberFormatter.php 5 years ago Option.php 5 years ago Period.php 5 years ago Piwik.php 5 years ago Plugin.php 5 years ago Profiler.php 5 years ago ProxyHeaders.php 5 years ago ProxyHttp.php 5 years ago QuickForm2.php 5 years ago RankingQuery.php 5 years ago ReportRenderer.php 5 years ago Segment.php 5 years ago Sequence.php 5 years ago Session.php 5 years ago SettingsPiwik.php 5 years ago SettingsServer.php 5 years ago Singleton.php 5 years ago Site.php 5 years ago SupportedBrowser.php 5 years ago TCPDF.php 5 years ago Theme.php 5 years ago Timer.php 5 years ago Tracker.php 5 years ago Twig.php 5 years ago Unzip.php 5 years ago UpdateCheck.php 5 years ago Updater.php 5 years ago UpdaterErrorException.php 5 years ago Updates.php 5 years ago Url.php 5 years ago UrlHelper.php 5 years ago Version.php 5 years ago View.php 5 years ago bootstrap.php 5 years ago dispatch.php 5 years ago testMinimumPhpVersion.php 5 years ago

CliMulti.php

467 lines

1	<?php
2	/**
3	* Matomo - free/libre analytics platform
4	*
5	* @link https://matomo.org
6	* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7	*/
8	namespace Piwik;
9
10	use Piwik\Archiver\Request;
11	use Piwik\CliMulti\CliPhp;
12	use Piwik\CliMulti\Output;
13	use Piwik\CliMulti\Process;
14	use Piwik\Container\StaticContainer;
15	use Psr\Log\LoggerInterface;
16	use Psr\Log\NullLogger;
17
18	/**
19	* Class CliMulti.
20	*/
21	class CliMulti
22	{
23	const BASE_WAIT_TIME = 250000; // 250 * 1000 = 250ms
24
25	/**
26	* If set to true or false it will overwrite whether async is supported or not.
27	*
28	* @var null\|bool
29	*/
30	public $supportsAsync = null;
31
32	/**
33	* @var Process[]
34	*/
35	private $processes = array();
36
37	/**
38	* If set it will issue at most concurrentProcessesLimit requests
39	* @var int
40	*/
41	private $concurrentProcessesLimit = null;
42
43	/**
44	* @var Output[]
45	*/
46	private $outputs = array();
47
48	private $acceptInvalidSSLCertificate = false;
49
50	/**
51	* @var bool
52	*/
53	private $runAsSuperUser = false;
54
55	/**
56	* Only used when doing synchronous curl requests.
57	*
58	* @var string
59	*/
60	private $urlToPiwik = null;
61
62	private $phpCliOptions = '';
63
64	/**
65	* @var callable
66	*/
67	private $onProcessFinish = null;
68
69	/**
70	* @var Timer[]
71	*/
72	protected $timers = [];
73
74	protected $isTimingRequests = false;
75
76	/**
77	* @var LoggerInterface
78	*/
79	private $logger;
80
81	public function __construct(LoggerInterface $logger = null)
82	{
83	$this->supportsAsync = $this->supportsAsync();
84	$this->logger = $logger ?: new NullLogger();
85	}
86
87	/**
88	* It will request all given URLs in parallel (async) using the CLI and wait until all requests are finished.
89	* If multi cli is not supported (eg windows) it will initiate an HTTP request instead (not async).
90	*
91	* @param string[] $piwikUrls An array of urls, for instance:
92	*
93	* `array('http://www.example.com/piwik?module=API...')`
94	*
95	* Make sure query parameter values are properly encoded in the URLs.
96	*
97	* @return array The response of each URL in the same order as the URLs. The array can contain null values in case
98	* there was a problem with a request, for instance if the process died unexpected.
99	*/
100	public function request(array $piwikUrls)
101	{
102	if ($this->isTimingRequests) {
103	foreach ($piwikUrls as $url) {
104	$this->timers[] = new Timer();
105	}
106	}
107
108	$chunks = array($piwikUrls);
109	if ($this->concurrentProcessesLimit) {
110	$chunks = array_chunk($piwikUrls, $this->concurrentProcessesLimit);
111	}
112
113	$results = array();
114	foreach ($chunks as $urlsChunk) {
115	$results = array_merge($results, $this->requestUrls($urlsChunk));
116	}
117
118	return $results;
119	}
120
121	/**
122	* Forwards the given configuration options to the PHP cli command.
123	* @param string $phpCliOptions eg "-d memory_limit=8G -c=path/to/php.ini"
124	*/
125	public function setPhpCliConfigurationOptions($phpCliOptions)
126	{
127	$this->phpCliOptions = (string) $phpCliOptions;
128	}
129
130	/**
131	* Ok, this sounds weird. Why should we care about ssl certificates when we are in CLI mode? It is needed for
132	* our simple fallback mode for Windows where we initiate HTTP requests instead of CLI.
133	* @param $acceptInvalidSSLCertificate
134	*/
135	public function setAcceptInvalidSSLCertificate($acceptInvalidSSLCertificate)
136	{
137	$this->acceptInvalidSSLCertificate = $acceptInvalidSSLCertificate;
138	}
139
140	/**
141	* @param $limit int Maximum count of requests to issue in parallel
142	*/
143	public function setConcurrentProcessesLimit($limit)
144	{
145	$this->concurrentProcessesLimit = $limit;
146	}
147
148	public function runAsSuperUser($runAsSuperUser = true)
149	{
150	$this->runAsSuperUser = $runAsSuperUser;
151	}
152
153	private function start($piwikUrls)
154	{
155	foreach ($piwikUrls as $index => $url) {
156	$shouldStart = null;
157	if ($url instanceof Request) {
158	$shouldStart = $url->start();
159	}
160
161	$cmdId = $this->generateCommandId($url) . $index;
162
163	if ($shouldStart === Request::ABORT) {
164	// output is needed to ensure same order of url to response
165	$output = new Output($cmdId);
166	$output->write(serialize(array('aborted' => '1')));
167	$this->outputs[] = $output;
168	} else {
169	$this->executeUrlCommand($cmdId, $url);
170	}
171	}
172	}
173
174	private function executeUrlCommand($cmdId, $url)
175	{
176	$output = new Output($cmdId);
177
178	if ($this->supportsAsync) {
179	$this->executeAsyncCli($url, $output, $cmdId);
180	} else {
181	$this->executeNotAsyncHttp($url, $output);
182	}
183
184	$this->outputs[] = $output;
185	}
186
187	private function buildCommand($hostname, $query, $outputFile, $doEsacpeArg = true)
188	{
189	$bin = $this->findPhpBinary();
190	$superuserCommand = $this->runAsSuperUser ? "--superuser" : "";
191
192	if ($doEsacpeArg) {
193	$hostname = escapeshellarg($hostname);
194	$query = escapeshellarg($query);
195	}
196
197	return sprintf('%s %s %s/console climulti:request -q --matomo-domain=%s %s %s > %s 2>&1 &',
198	$bin, $this->phpCliOptions, PIWIK_INCLUDE_PATH, $hostname, $superuserCommand, $query, $outputFile);
199	}
200
201	private function getResponse()
202	{
203	$response = array();
204
205	foreach ($this->outputs as $output) {
206	$response[] = $output->get();
207	}
208
209	return $response;
210	}
211
212	private function hasFinished()
213	{
214	foreach ($this->processes as $index => $process) {
215	$hasStarted = $process->hasStarted();
216
217	if (!$hasStarted && 8 <= $process->getSecondsSinceCreation()) {
218	// if process was created more than 8 seconds ago but still not started there must be something wrong.
219	// ==> declare the process as finished
220	$process->finishProcess();
221	continue;
222	} elseif (!$hasStarted) {
223	return false;
224	}
225
226	if ($process->isRunning()) {
227	return false;
228	}
229
230	$pid = $process->getPid();
231	foreach ($this->outputs as $output) {
232	if ($output->getOutputId() === $pid && $output->isAbnormal()) {
233	$process->finishProcess();
234	continue;
235	}
236	}
237
238	if ($process->hasFinished()) {
239	// prevent from checking this process over and over again
240	unset($this->processes[$index]);
241
242	if ($this->isTimingRequests) {
243	$this->timers[$index]->finish();
244	}
245
246	if ($this->onProcessFinish) {
247	$onProcessFinish = $this->onProcessFinish;
248	$onProcessFinish($pid);
249	}
250	}
251	}
252
253	return true;
254	}
255
256	private function generateCommandId($command)
257	{
258	return substr(Common::hash($command . microtime(true) . rand(0, 99999)), 0, 100);
259	}
260
261	/**
262	* What is missing under windows? Detection whether a process is still running in Process::isProcessStillRunning
263	* and how to send a process into background in start()
264	*/
265	public function supportsAsync()
266	{
267	$supportsAsync = Process::isSupported() && !Common::isPhpCgiType() && $this->findPhpBinary();
268
269	/**
270	* Triggered to allow plugins to force the usage of async cli multi execution or to disable it.
271	*
272	* Example
273	*
274	* public function supportsAsync(&$supportsAsync)
275	* {
276	* $supportsAsync = false; // do not allow async climulti execution
277	* }
278	*
279	* @param bool &$supportsAsync Whether async is supported or not.
280	*/
281	Piwik::postEvent('CliMulti.supportsAsync', array(&$supportsAsync));
282
283	return $supportsAsync;
284	}
285
286	private function findPhpBinary()
287	{
288	$cliPhp = new CliPhp();
289	return $cliPhp->findPhpBinary();
290	}
291
292	private function cleanup()
293	{
294	foreach ($this->processes as $pid) {
295	$pid->finishProcess();
296	}
297
298	foreach ($this->outputs as $output) {
299	$output->destroy();
300	}
301
302	$this->processes = array();
303	$this->outputs = array();
304	}
305
306	/**
307	* Remove files older than one week. They should be cleaned up automatically after each request but for whatever
308	* reason there can be always some files left.
309	*/
310	public static function cleanupNotRemovedFiles()
311	{
312	$timeOneWeekAgo = strtotime('-1 week');
313
314	$files = _glob(self::getTmpPath() . '/*');
315	if (empty($files)) {
316	return;
317	}
318
319	foreach ($files as $file) {
320	if (file_exists($file)) {
321	$timeLastModified = filemtime($file);
322
323	if ($timeLastModified !== false && $timeOneWeekAgo > $timeLastModified) {
324	unlink($file);
325	}
326	}
327	}
328	}
329
330	public static function getTmpPath()
331	{
332	return StaticContainer::get('path.tmp') . '/climulti';
333	}
334
335	private function executeAsyncCli($url, Output $output, $cmdId)
336	{
337	$this->processes[] = new Process($cmdId);
338
339	$url = $this->appendTestmodeParamToUrlIfNeeded($url);
340	$query = UrlHelper::getQueryFromUrl($url, array('pid' => $cmdId, 'runid' => getmypid()));
341	$hostname = Url::getHost($checkIfTrusted = false);
342	$command = $this->buildCommand($hostname, $query, $output->getPathToFile());
343
344	$this->logger->debug("Running command: {command}", ['command' => $command]);
345	shell_exec($command);
346	}
347
348	private function executeNotAsyncHttp($url, Output $output)
349	{
350	$piwikUrl = $this->urlToPiwik ?: SettingsPiwik::getPiwikUrl();
351	if (empty($piwikUrl)) {
352	$piwikUrl = 'http://' . Url::getHost() . '/';
353	}
354
355	$url = $piwikUrl . $url;
356	if (Config::getInstance()->General['force_ssl'] == 1) {
357	$url = str_replace("http://", "https://", $url);
358	}
359
360	$requestBody = null;
361	if ($this->runAsSuperUser) {
362	$tokenAuth = self::getSuperUserTokenAuth();
363
364	if (strpos($url, '?') === false) {
365	$url .= '?';
366	} else {
367	$url .= '&';
368	}
369
370	$requestBody = 'token_auth=' . $tokenAuth;
371	}
372
373	try {
374	$this->logger->debug("Execute HTTP API request: " . $url);
375	$response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate, false, false, 'POST', null, null, $requestBody, [], $forcePost = true);
376	$output->write($response);
377	} catch (\Exception $e) {
378	$message = "Got invalid response from API request: $url. ";
379
380	if (isset($response) && empty($response)) {
381	$message .= "The response was empty. This usually means a server error. This solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. Please check your Web server Error Log file for more details.";
382	} else {
383	$message .= "Response was '" . $e->getMessage() . "'";
384	}
385
386	$output->write($message);
387
388	$this->logger->debug($message, ['exception' => $e]);
389	}
390	}
391
392	private function appendTestmodeParamToUrlIfNeeded($url)
393	{
394	$isTestMode = defined('PIWIK_TEST_MODE');
395
396	if ($isTestMode && false === strpos($url, '?')) {
397	$url .= "?testmode=1";
398	} elseif ($isTestMode) {
399	$url .= "&testmode=1";
400	}
401
402	return $url;
403	}
404
405	/**
406	* @param array $piwikUrls
407	* @return array
408	*/
409	private function requestUrls(array $piwikUrls)
410	{
411	$this->start($piwikUrls);
412
413	$startTime = time();
414	do {
415	$elapsed = time() - $startTime;
416	$timeToWait = $this->getTimeToWaitBeforeNextCheck($elapsed);
417
418	usleep($timeToWait);
419	} while (!$this->hasFinished());
420
421	$results = $this->getResponse();
422	$this->cleanup();
423
424	self::cleanupNotRemovedFiles();
425
426	return $results;
427	}
428
429	private static function getSuperUserTokenAuth()
430	{
431	return Piwik::requestTemporarySystemAuthToken('CliMultiNonAsyncArchive', 36);
432	}
433
434	public function setUrlToPiwik($urlToPiwik)
435	{
436	$this->urlToPiwik = $urlToPiwik;
437	}
438
439	public function onProcessFinish(callable $callback)
440	{
441	$this->onProcessFinish = $callback;
442	}
443
444	// every minute that passes adds an extra 100ms to the wait time. so 5 minutes results in 500ms extra, 20mins results in 2s extra.
445	private function getTimeToWaitBeforeNextCheck($elapsed)
446	{
447	$minutes = floor($elapsed / 60);
448	return self::BASE_WAIT_TIME + $minutes * 100000; // 100 * 1000 = 100ms
449	}
450
451	public static function isCliMultiRequest()
452	{
453	return Common::getRequestVar('pid', false) !== false;
454	}
455
456	public function timeRequests()
457	{
458	$this->timers = [];
459	$this->isTimingRequests = true;
460	}
461
462	public function getTimers()
463	{
464	return $this->timers;
465	}
466	}
467