CliMulti.php — Matomo Analytics – Powerful, Privacy-First Insights for WordPress 5.1.4

matomo / app / core / CliMulti.php

matomo / app / core Last commit date

API 2 years ago Access 2 years ago Application 2 years ago Archive 2 years ago ArchiveProcessor 2 years ago Archiver 2 years ago AssetManager 2 years ago Auth 2 years ago Category 2 years ago Changes 2 years ago CliMulti 2 years ago Columns 2 years ago Concurrency 2 years ago Config 2 years ago Container 2 years ago CronArchive 2 years ago DataAccess 2 years ago DataFiles 2 years ago DataTable 2 years ago Db 1 year ago DeviceDetector 2 years ago Email 2 years ago Exception 2 years ago Http 2 years ago Intl 2 years ago Log 2 years ago Mail 2 years ago Measurable 2 years ago Menu 2 years ago Metrics 2 years ago Notification 2 years ago Period 1 year ago Plugin 2 years ago ProfessionalServices 2 years ago Report 2 years ago ReportRenderer 2 years ago Scheduler 2 years ago Segment 2 years ago Session 2 years ago Settings 2 years ago Tracker 1 year ago Translation 2 years ago Twig 2 years ago UpdateCheck 2 years ago Updater 1 year ago Updates 1 year ago Validators 2 years ago View 2 years ago ViewDataTable 2 years ago Visualization 2 years ago Widget 2 years ago .htaccess 2 years ago Access.php 2 years ago Archive.php 2 years ago ArchiveProcessor.php 2 years ago AssetManager.php 2 years ago Auth.php 2 years ago AuthResult.php 2 years ago BaseFactory.php 2 years ago Cache.php 2 years ago CacheId.php 2 years ago CliMulti.php 2 years ago Common.php 2 years ago Config.php 2 years ago Console.php 2 years ago Context.php 2 years ago Cookie.php 2 years ago CronArchive.php 2 years ago DI.php 2 years ago DataArray.php 2 years ago DataTable.php 2 years ago Date.php 2 years ago Db.php 1 year ago DbHelper.php 1 year ago Development.php 2 years ago ErrorHandler.php 2 years ago EventDispatcher.php 2 years ago ExceptionHandler.php 2 years ago FileIntegrity.php 2 years ago Filechecks.php 2 years ago Filesystem.php 2 years ago FrontController.php 2 years ago Http.php 2 years ago IP.php 2 years ago Log.php 2 years ago LogDeleter.php 2 years ago Mail.php 2 years ago Metrics.php 2 years ago NoAccessException.php 2 years ago Nonce.php 2 years ago Notification.php 2 years ago NumberFormatter.php 2 years ago Option.php 2 years ago Period.php 2 years ago Piwik.php 2 years ago Plugin.php 2 years ago Profiler.php 2 years ago ProxyHeaders.php 2 years ago ProxyHttp.php 2 years ago QuickForm2.php 2 years ago RankingQuery.php 2 years ago ReportRenderer.php 2 years ago Request.php 2 years ago Segment.php 2 years ago Sequence.php 2 years ago Session.php 2 years ago SettingsPiwik.php 2 years ago SettingsServer.php 2 years ago Singleton.php 2 years ago Site.php 2 years ago SiteContentDetector.php 2 years ago SupportedBrowser.php 2 years ago TCPDF.php 2 years ago Theme.php 2 years ago Timer.php 2 years ago Tracker.php 2 years ago Twig.php 2 years ago Unzip.php 2 years ago UpdateCheck.php 2 years ago Updater.php 2 years ago UpdaterErrorException.php 2 years ago Updates.php 2 years ago Url.php 2 years ago UrlHelper.php 1 year ago Version.php 1 year ago View.php 2 years ago bootstrap.php 2 years ago dispatch.php 2 years ago testMinimumPhpVersion.php 2 years ago

CliMulti.php

421 lines

1	<?php
2
3	/**
4	* Matomo - free/libre analytics platform
5	*
6	* @link https://matomo.org
7	* @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
8	*/
9	namespace Piwik;
10
11	use Piwik\Archiver\Request;
12	use Piwik\CliMulti\CliPhp;
13	use Piwik\CliMulti\Output;
14	use Piwik\CliMulti\OutputInterface;
15	use Piwik\CliMulti\Process;
16	use Piwik\CliMulti\StaticOutput;
17	use Piwik\Container\StaticContainer;
18	use Piwik\Log\LoggerInterface;
19	use Piwik\Log\NullLogger;
20	/**
21	* Class CliMulti.
22	*/
23	class CliMulti
24	{
25	public const BASE_WAIT_TIME = 250000;
26	// 250 * 1000 = 250ms
27	/**
28	* If set to true or false it will overwrite whether async is supported or not.
29	*
30	* @var null\|bool
31	*/
32	public $supportsAsync = null;
33	/**
34	* @var Process[]
35	*/
36	private $processes = array();
37	/**
38	* If set it will issue at most concurrentProcessesLimit requests
39	* @var int
40	*/
41	private $concurrentProcessesLimit = null;
42	/**
43	* @var OutputInterface[]
44	*/
45	private $outputs = array();
46	private $acceptInvalidSSLCertificate = false;
47	/**
48	* @var bool
49	*/
50	private $runAsSuperUser = false;
51	/**
52	* Only used when doing synchronous curl requests.
53	*
54	* @var string
55	*/
56	private $urlToPiwik = null;
57	private $phpCliOptions = '';
58	/**
59	* @var callable
60	*/
61	private $onProcessFinish = null;
62	/**
63	* @var Timer[]
64	*/
65	protected $timers = [];
66	protected $isTimingRequests = false;
67	/**
68	* @var LoggerInterface
69	*/
70	private $logger;
71	public function __construct(LoggerInterface $logger = null)
72	{
73	$this->supportsAsync = $this->supportsAsync();
74	$this->logger = $logger ?: new NullLogger();
75	}
76	/**
77	* It will request all given URLs in parallel (async) using the CLI and wait until all requests are finished.
78	* If multi cli is not supported (eg windows) it will initiate an HTTP request instead (not async).
79	*
80	* @param string[] $piwikUrls An array of urls, for instance:
81	*
82	* `array('http://www.example.com/piwik?module=API...')`
83	*
84	* Make sure query parameter values are properly encoded in the URLs.
85	*
86	* @return array The response of each URL in the same order as the URLs. The array can contain null values in case
87	* there was a problem with a request, for instance if the process died unexpected.
88	*/
89	public function request(array $piwikUrls)
90	{
91	if ($this->isTimingRequests) {
92	foreach ($piwikUrls as $url) {
93	$this->timers[] = new \Piwik\Timer();
94	}
95	}
96	$chunks = array($piwikUrls);
97	if ($this->concurrentProcessesLimit) {
98	$chunks = array_chunk($piwikUrls, $this->concurrentProcessesLimit);
99	}
100	$results = array();
101	foreach ($chunks as $urlsChunk) {
102	$results = array_merge($results, $this->requestUrls($urlsChunk));
103	}
104	return $results;
105	}
106	/**
107	* Forwards the given configuration options to the PHP cli command.
108	* @param string $phpCliOptions eg "-d memory_limit=8G -c=path/to/php.ini"
109	*/
110	public function setPhpCliConfigurationOptions($phpCliOptions)
111	{
112	$this->phpCliOptions = (string) $phpCliOptions;
113	}
114	/**
115	* Ok, this sounds weird. Why should we care about ssl certificates when we are in CLI mode? It is needed for
116	* our simple fallback mode for Windows where we initiate HTTP requests instead of CLI.
117	* @param $acceptInvalidSSLCertificate
118	*/
119	public function setAcceptInvalidSSLCertificate($acceptInvalidSSLCertificate)
120	{
121	$this->acceptInvalidSSLCertificate = $acceptInvalidSSLCertificate;
122	}
123	/**
124	* @param $limit int Maximum count of requests to issue in parallel
125	*/
126	public function setConcurrentProcessesLimit($limit)
127	{
128	$this->concurrentProcessesLimit = $limit;
129	}
130	public function runAsSuperUser($runAsSuperUser = true)
131	{
132	$this->runAsSuperUser = $runAsSuperUser;
133	}
134	private function start($piwikUrls)
135	{
136	$numUrls = count($piwikUrls);
137	foreach ($piwikUrls as $index => $url) {
138	$shouldStart = null;
139	if ($url instanceof Request) {
140	$shouldStart = $url->start();
141	}
142	$cmdId = $this->generateCommandId($url) . $index;
143	if ($shouldStart === Request::ABORT) {
144	// output is needed to ensure same order of url to response
145	$output = new StaticOutput($cmdId);
146	$output->write(serialize(array('aborted' => '1')));
147	$this->outputs[] = $output;
148	} else {
149	$this->executeUrlCommand($cmdId, $url, $numUrls);
150	}
151	}
152	}
153	private function executeUrlCommand($cmdId, $url, $numUrls)
154	{
155	if ($this->supportsAsync) {
156	if ($numUrls === 1) {
157	$output = new StaticOutput($cmdId);
158	$this->executeSyncCli($url, $output);
159	} else {
160	$output = new Output($cmdId);
161	$this->executeAsyncCli($url, $output, $cmdId);
162	}
163	} else {
164	$output = new StaticOutput($cmdId);
165	$this->executeNotAsyncHttp($url, $output);
166	}
167	$this->outputs[] = $output;
168	}
169	private function buildCommand($hostname, $query, $outputFileIfAsync, $doEsacpeArg = true)
170	{
171	$bin = $this->findPhpBinary();
172	$superuserCommand = $this->runAsSuperUser ? "--superuser" : "";
173	$append = '2>&1';
174	if ($outputFileIfAsync) {
175	$append = sprintf(' > %s 2>&1 &', $outputFileIfAsync);
176	}
177	if ($doEsacpeArg) {
178	$hostname = escapeshellarg($hostname);
179	$query = escapeshellarg($query);
180	}
181	return sprintf('%s %s %s/console climulti:request -q --matomo-domain=%s %s %s %s', $bin, $this->phpCliOptions, PIWIK_INCLUDE_PATH, $hostname, $superuserCommand, $query, $append);
182	}
183	private function getResponse()
184	{
185	$response = array();
186	foreach ($this->outputs as $output) {
187	$content = $output->get();
188	// Remove output that can be ignored in climulti . works around some worpdress setups where the hash bang may
189	// be printed
190	$search = '#!/usr/bin/env php';
191	if (!empty($content) && is_string($content) && mb_substr(trim($content), 0, strlen($search)) === $search) {
192	$content = trim(mb_substr(trim($content), strlen($search)));
193	}
194	$response[] = $content;
195	}
196	return $response;
197	}
198	private function hasFinished()
199	{
200	foreach ($this->processes as $index => $process) {
201	$hasStarted = $process->hasStarted();
202	if (!$hasStarted && 8 <= $process->getSecondsSinceCreation()) {
203	// if process was created more than 8 seconds ago but still not started there must be something wrong.
204	// ==> declare the process as finished
205	$process->finishProcess();
206	continue;
207	} elseif (!$hasStarted) {
208	return false;
209	}
210	if ($process->isRunning()) {
211	return false;
212	}
213	$pid = $process->getPid();
214	foreach ($this->outputs as $output) {
215	if ($output->getOutputId() === $pid && $output->isAbnormal()) {
216	$process->finishProcess();
217	continue;
218	}
219	}
220	if ($process->hasFinished()) {
221	// prevent from checking this process over and over again
222	unset($this->processes[$index]);
223	if ($this->isTimingRequests) {
224	$this->timers[$index]->finish();
225	}
226	if ($this->onProcessFinish) {
227	$onProcessFinish = $this->onProcessFinish;
228	$onProcessFinish($pid);
229	}
230	}
231	}
232	return true;
233	}
234	private function generateCommandId($command)
235	{
236	return substr(\Piwik\Common::hash($command . microtime(true) . rand(0, 99999)), 0, 100);
237	}
238	/**
239	* What is missing under windows? Detection whether a process is still running in Process::isProcessStillRunning
240	* and how to send a process into background in start()
241	*/
242	public function supportsAsync()
243	{
244	$supportsAsync = Process::isSupported() && !\Piwik\Common::isPhpCgiType() && $this->findPhpBinary();
245	/**
246	* Triggered to allow plugins to force the usage of async cli multi execution or to disable it.
247	*
248	* Example
249	*
250	* public function supportsAsync(&$supportsAsync)
251	* {
252	* $supportsAsync = false; // do not allow async climulti execution
253	* }
254	*
255	* @param bool &$supportsAsync Whether async is supported or not.
256	*/
257	\Piwik\Piwik::postEvent('CliMulti.supportsAsync', array(&$supportsAsync));
258	return $supportsAsync;
259	}
260	private function findPhpBinary()
261	{
262	$cliPhp = new CliPhp();
263	return $cliPhp->findPhpBinary();
264	}
265	private function cleanup()
266	{
267	foreach ($this->processes as $pid) {
268	$pid->finishProcess();
269	}
270	foreach ($this->outputs as $output) {
271	$output->destroy();
272	}
273	$this->processes = array();
274	$this->outputs = array();
275	}
276	/**
277	* Remove files older than one week. They should be cleaned up automatically after each request but for whatever
278	* reason there can be always some files left.
279	*/
280	public static function cleanupNotRemovedFiles()
281	{
282	$timeOneWeekAgo = strtotime('-1 week');
283	$files = _glob(self::getTmpPath() . '/*');
284	if (empty($files)) {
285	return;
286	}
287	foreach ($files as $file) {
288	if (file_exists($file)) {
289	$timeLastModified = filemtime($file);
290	if ($timeLastModified !== false && $timeOneWeekAgo > $timeLastModified) {
291	unlink($file);
292	}
293	}
294	}
295	}
296	public static function getTmpPath()
297	{
298	return StaticContainer::get('path.tmp') . '/climulti';
299	}
300	private function executeAsyncCli($url, Output $output, $cmdId)
301	{
302	$this->processes[] = new Process($cmdId);
303	$url = $this->appendTestmodeParamToUrlIfNeeded($url);
304	$query = \Piwik\UrlHelper::getQueryFromUrl($url, ['pid' => $cmdId, 'runid' => \Piwik\Common::getProcessId()]);
305	$hostname = \Piwik\Url::getHost($checkIfTrusted = false);
306	$command = $this->buildCommand($hostname, $query, $output->getPathToFile());
307	$this->logger->debug("Running command: {command}", ['command' => $command]);
308	shell_exec($command);
309	}
310	private function executeSyncCli($url, StaticOutput $output)
311	{
312	$url = $this->appendTestmodeParamToUrlIfNeeded($url);
313	$query = \Piwik\UrlHelper::getQueryFromUrl($url, array());
314	$hostname = \Piwik\Url::getHost($checkIfTrusted = false);
315	$command = $this->buildCommand($hostname, $query, '', true);
316	$this->logger->debug("Running command: {command}", ['command' => $command]);
317	$result = shell_exec($command);
318	if ($result) {
319	$result = trim($result);
320	}
321	$output->write($result);
322	}
323	private function executeNotAsyncHttp($url, StaticOutput $output)
324	{
325	$piwikUrl = $this->urlToPiwik ?: \Piwik\SettingsPiwik::getPiwikUrl();
326	if (empty($piwikUrl)) {
327	$piwikUrl = 'http://' . \Piwik\Url::getHost() . '/';
328	}
329	$url = $piwikUrl . $url;
330	if (\Piwik\Config::getInstance()->General['force_ssl'] == 1) {
331	$url = str_replace("http://", "https://", $url);
332	}
333	$requestBody = null;
334	if ($this->runAsSuperUser) {
335	$tokenAuth = self::getSuperUserTokenAuth();
336	if (strpos($url, '?') === false) {
337	$url .= '?';
338	} else {
339	$url .= '&';
340	}
341	$requestBody = 'token_auth=' . $tokenAuth;
342	}
343	try {
344	$this->logger->debug("Execute HTTP API request: " . $url);
345	$response = \Piwik\Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate, false, false, 'POST', null, null, $requestBody, [], $forcePost = true);
346	$output->write($response);
347	} catch (\Exception $e) {
348	$message = "Got invalid response from API request: {$url}. ";
349	if (isset($response) && empty($response)) {
350	$message .= "The response was empty. This usually means a server error. This solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. Please check your Web server Error Log file for more details.";
351	} else {
352	$message .= "Response was '" . $e->getMessage() . "'";
353	}
354	$output->write($message);
355	$this->logger->debug($message, ['exception' => $e]);
356	}
357	}
358	private function appendTestmodeParamToUrlIfNeeded($url)
359	{
360	$isTestMode = defined('PIWIK_TEST_MODE');
361	if ($isTestMode && false === strpos($url, '?')) {
362	$url .= "?testmode=1";
363	} elseif ($isTestMode) {
364	$url .= "&testmode=1";
365	}
366	return $url;
367	}
368	/**
369	* @param array $piwikUrls
370	* @return array
371	*/
372	private function requestUrls(array $piwikUrls)
373	{
374	$this->start($piwikUrls);
375	$startTime = time();
376	do {
377	$elapsed = time() - $startTime;
378	$timeToWait = $this->getTimeToWaitBeforeNextCheck($elapsed);
379	if (count($this->processes)) {
380	usleep($timeToWait);
381	}
382	} while (!$this->hasFinished());
383	$results = $this->getResponse();
384	$this->cleanup();
385	self::cleanupNotRemovedFiles();
386	return $results;
387	}
388	private static function getSuperUserTokenAuth()
389	{
390	return \Piwik\Piwik::requestTemporarySystemAuthToken('CliMultiNonAsyncArchive', 36);
391	}
392	public function setUrlToPiwik($urlToPiwik)
393	{
394	$this->urlToPiwik = $urlToPiwik;
395	}
396	public function onProcessFinish(callable $callback)
397	{
398	$this->onProcessFinish = $callback;
399	}
400	// every minute that passes adds an extra 100ms to the wait time. so 5 minutes results in 500ms extra, 20mins results in 2s extra.
401	private function getTimeToWaitBeforeNextCheck($elapsed)
402	{
403	$minutes = floor($elapsed / 60);
404	return self::BASE_WAIT_TIME + $minutes * 100000;
405	// 100 * 1000 = 100ms
406	}
407	public static function isCliMultiRequest()
408	{
409	return \Piwik\Common::getRequestVar('pid', false) !== false;
410	}
411	public function timeRequests()
412	{
413	$this->timers = [];
414	$this->isTimingRequests = true;
415	}
416	public function getTimers()
417	{
418	return $this->timers;
419	}
420	}
421