update Piwik to version 2.16 (fixes #91)
This commit is contained in:
oliver
parent
296343bf3b
commit
d885a4baa9
5833 changed files with 418860 additions and 226988 deletions
|
|
@ -1,19 +1,23 @@
|
|||
<?php
|
||||
/**
|
||||
* Piwik - Open source web analytics
|
||||
* Piwik - free/libre analytics platform
|
||||
*
|
||||
* @link http://piwik.org
|
||||
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
|
||||
*/
|
||||
namespace Piwik;
|
||||
|
||||
use Piwik\Archiver\Request;
|
||||
use Piwik\CliMulti\CliPhp;
|
||||
use Piwik\CliMulti\Output;
|
||||
use Piwik\CliMulti\Process;
|
||||
use Piwik\Container\StaticContainer;
|
||||
|
||||
/**
|
||||
* Class CliMulti.
|
||||
*/
|
||||
class CliMulti {
|
||||
class CliMulti
|
||||
{
|
||||
|
||||
/**
|
||||
* If set to true or false it will overwrite whether async is supported or not.
|
||||
|
|
@ -23,17 +27,37 @@ class CliMulti {
|
|||
public $supportsAsync = null;
|
||||
|
||||
/**
|
||||
* @var \Piwik\CliMulti\Process[]
|
||||
* @var Process[]
|
||||
*/
|
||||
private $processes = array();
|
||||
|
||||
/**
|
||||
* @var \Piwik\CliMulti\Output[]
|
||||
* If set it will issue at most concurrentProcessesLimit requests
|
||||
* @var int
|
||||
*/
|
||||
private $concurrentProcessesLimit = null;
|
||||
|
||||
/**
|
||||
* @var Output[]
|
||||
*/
|
||||
private $outputs = array();
|
||||
|
||||
private $acceptInvalidSSLCertificate = false;
|
||||
|
||||
/**
|
||||
* @var bool
|
||||
*/
|
||||
private $runAsSuperUser = false;
|
||||
|
||||
/**
|
||||
* Only used when doing synchronous curl requests.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
private $urlToPiwik = null;
|
||||
|
||||
private $phpCliOptions = '';
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->supportsAsync = $this->supportsAsync();
|
||||
|
|
@ -44,26 +68,38 @@ class CliMulti {
|
|||
* If multi cli is not supported (eg windows) it will initiate an HTTP request instead (not async).
|
||||
*
|
||||
* @param string[] $piwikUrls An array of urls, for instance:
|
||||
* array('http://www.example.com/piwik?module=API...')
|
||||
*
|
||||
* `array('http://www.example.com/piwik?module=API...')`
|
||||
*
|
||||
* **Make sure query parameter values are properly encoded in the URLs.**
|
||||
*
|
||||
* @return array The response of each URL in the same order as the URLs. The array can contain null values in case
|
||||
* there was a problem with a request, for instance if the process died unexpected.
|
||||
*/
|
||||
public function request(array $piwikUrls)
|
||||
{
|
||||
$this->start($piwikUrls);
|
||||
$chunks = array($piwikUrls);
|
||||
if ($this->concurrentProcessesLimit) {
|
||||
$chunks = array_chunk($piwikUrls, $this->concurrentProcessesLimit);
|
||||
}
|
||||
|
||||
do {
|
||||
usleep(100000); // 100 * 1000 = 100ms
|
||||
} while (!$this->hasFinished());
|
||||
|
||||
$results = $this->getResponse($piwikUrls);
|
||||
$this->cleanup();
|
||||
|
||||
self::cleanupNotRemovedFiles();
|
||||
$results = array();
|
||||
foreach ($chunks as $urlsChunk) {
|
||||
$results = array_merge($results, $this->requestUrls($urlsChunk));
|
||||
}
|
||||
|
||||
return $results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Forwards the given configuration options to the PHP cli command.
|
||||
* @param string $phpCliOptions eg "-d memory_limit=8G -c=path/to/php.ini"
|
||||
*/
|
||||
public function setPhpCliConfigurationOptions($phpCliOptions)
|
||||
{
|
||||
$this->phpCliOptions = (string) $phpCliOptions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ok, this sounds weird. Why should we care about ssl certificates when we are in CLI mode? It is needed for
|
||||
* our simple fallback mode for Windows where we initiate HTTP requests instead of CLI.
|
||||
|
|
@ -74,28 +110,51 @@ class CliMulti {
|
|||
$this->acceptInvalidSSLCertificate = $acceptInvalidSSLCertificate;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $limit int Maximum count of requests to issue in parallel
|
||||
*/
|
||||
public function setConcurrentProcessesLimit($limit)
|
||||
{
|
||||
$this->concurrentProcessesLimit = $limit;
|
||||
}
|
||||
|
||||
public function runAsSuperUser($runAsSuperUser = true)
|
||||
{
|
||||
$this->runAsSuperUser = $runAsSuperUser;
|
||||
}
|
||||
|
||||
private function start($piwikUrls)
|
||||
{
|
||||
foreach ($piwikUrls as $index => $url) {
|
||||
$cmdId = $this->generateCommandId($url) . $index;
|
||||
$output = new Output($cmdId);
|
||||
|
||||
if ($this->supportsAsync) {
|
||||
$this->executeAsyncCli($url, $output, $cmdId);
|
||||
} else {
|
||||
$this->executeNotAsyncHttp($url, $output);
|
||||
if ($url instanceof Request) {
|
||||
$url->start();
|
||||
}
|
||||
|
||||
$this->outputs[] = $output;
|
||||
$cmdId = $this->generateCommandId($url) . $index;
|
||||
$this->executeUrlCommand($cmdId, $url);
|
||||
}
|
||||
}
|
||||
|
||||
private function executeUrlCommand($cmdId, $url)
|
||||
{
|
||||
$output = new Output($cmdId);
|
||||
|
||||
if ($this->supportsAsync) {
|
||||
$this->executeAsyncCli($url, $output, $cmdId);
|
||||
} else {
|
||||
$this->executeNotAsyncHttp($url, $output);
|
||||
}
|
||||
|
||||
$this->outputs[] = $output;
|
||||
}
|
||||
|
||||
private function buildCommand($hostname, $query, $outputFile)
|
||||
{
|
||||
$bin = $this->findPhpBinary();
|
||||
$superuserCommand = $this->runAsSuperUser ? "--superuser" : "";
|
||||
|
||||
return sprintf('%s -q %s/console climulti:request --piwik-domain=%s %s > %s 2>&1 &',
|
||||
$bin, PIWIK_INCLUDE_PATH, escapeshellarg($hostname), escapeshellarg($query), $outputFile);
|
||||
return sprintf('%s %s %s/console climulti:request -q --piwik-domain=%s %s %s > %s 2>&1 &',
|
||||
$bin, $this->phpCliOptions, PIWIK_INCLUDE_PATH, escapeshellarg($hostname), $superuserCommand, escapeshellarg($query), $outputFile);
|
||||
}
|
||||
|
||||
private function getResponse()
|
||||
|
|
@ -119,7 +178,6 @@ class CliMulti {
|
|||
// ==> declare the process as finished
|
||||
$process->finishProcess();
|
||||
continue;
|
||||
|
||||
} elseif (!$hasStarted) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -128,6 +186,14 @@ class CliMulti {
|
|||
return false;
|
||||
}
|
||||
|
||||
$pid = $process->getPid();
|
||||
foreach ($this->outputs as $output) {
|
||||
if ($output->getOutputId() === $pid && $output->isAbnormal()) {
|
||||
$process->finishProcess();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if ($process->hasFinished()) {
|
||||
// prevent from checking this process over and over again
|
||||
unset($this->processes[$index]);
|
||||
|
|
@ -146,11 +212,15 @@ class CliMulti {
|
|||
* What is missing under windows? Detection whether a process is still running in Process::isProcessStillRunning
|
||||
* and how to send a process into background in start()
|
||||
*/
|
||||
private function supportsAsync()
|
||||
public function supportsAsync()
|
||||
{
|
||||
return !SettingsServer::isWindows()
|
||||
&& Process::isSupported()
|
||||
&& $this->findPhpBinary();
|
||||
return Process::isSupported() && !Common::isPhpCgiType() && $this->findPhpBinary();
|
||||
}
|
||||
|
||||
private function findPhpBinary()
|
||||
{
|
||||
$cliPhp = new CliPhp();
|
||||
return $cliPhp->findPhpBinary();
|
||||
}
|
||||
|
||||
private function cleanup()
|
||||
|
|
@ -176,61 +246,33 @@ class CliMulti {
|
|||
$timeOneWeekAgo = strtotime('-1 week');
|
||||
|
||||
$files = _glob(self::getTmpPath() . '/*');
|
||||
if(empty($files)) {
|
||||
if (empty($files)) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($files as $file) {
|
||||
$timeLastModified = filemtime($file);
|
||||
if (file_exists($file)) {
|
||||
$timeLastModified = filemtime($file);
|
||||
|
||||
if ($timeOneWeekAgo > $timeLastModified) {
|
||||
unlink($file);
|
||||
if ($timeLastModified !== false && $timeOneWeekAgo > $timeLastModified) {
|
||||
unlink($file);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static function getTmpPath()
|
||||
{
|
||||
$dir = PIWIK_INCLUDE_PATH . '/tmp/climulti';
|
||||
return SettingsPiwik::rewriteTmpPathWithHostname($dir);
|
||||
}
|
||||
|
||||
private function findPhpBinary()
|
||||
{
|
||||
if (defined('PHP_BINARY') && false === strpos(PHP_BINARY, 'fpm')) {
|
||||
return PHP_BINARY;
|
||||
}
|
||||
|
||||
$bin = '';
|
||||
|
||||
if (!empty($_SERVER['_']) && Common::isPhpCliMode()) {
|
||||
$bin = $this->getPhpCommandIfValid($_SERVER['_']);
|
||||
}
|
||||
|
||||
if (empty($bin) && !empty($_SERVER['argv'][0]) && Common::isPhpCliMode()) {
|
||||
$bin = $this->getPhpCommandIfValid($_SERVER['argv'][0]);
|
||||
}
|
||||
|
||||
if (empty($bin)) {
|
||||
$bin = shell_exec('which php');
|
||||
}
|
||||
|
||||
if (empty($bin)) {
|
||||
$bin = shell_exec('which php5');
|
||||
}
|
||||
|
||||
if (!empty($bin)) {
|
||||
return trim($bin);
|
||||
}
|
||||
return StaticContainer::get('path.tmp') . '/climulti';
|
||||
}
|
||||
|
||||
private function executeAsyncCli($url, Output $output, $cmdId)
|
||||
{
|
||||
$this->processes[] = new Process($cmdId);
|
||||
|
||||
$url = $this->appendTestmodeParamToUrlIfNeeded($url);
|
||||
$query = UrlHelper::getQueryFromUrl($url, array('pid' => $cmdId));
|
||||
$hostname = UrlHelper::getHostFromUrl($url);
|
||||
$url = $this->appendTestmodeParamToUrlIfNeeded($url);
|
||||
$query = UrlHelper::getQueryFromUrl($url, array('pid' => $cmdId));
|
||||
$hostname = Url::getHost($checkIfTrusted = false);
|
||||
$command = $this->buildCommand($hostname, $query, $output->getPathToFile());
|
||||
|
||||
Log::debug($command);
|
||||
|
|
@ -239,6 +281,29 @@ class CliMulti {
|
|||
|
||||
private function executeNotAsyncHttp($url, Output $output)
|
||||
{
|
||||
$piwikUrl = $this->urlToPiwik ?: SettingsPiwik::getPiwikUrl();
|
||||
if (empty($piwikUrl)) {
|
||||
$piwikUrl = 'http://' . Url::getHost() . '/';
|
||||
}
|
||||
|
||||
$url = $piwikUrl . $url;
|
||||
if (Config::getInstance()->General['force_ssl'] == 1) {
|
||||
$url = str_replace("http://", "https://", $url);
|
||||
}
|
||||
|
||||
if ($this->runAsSuperUser) {
|
||||
$tokenAuths = self::getSuperUserTokenAuths();
|
||||
$tokenAuth = reset($tokenAuths);
|
||||
|
||||
if (strpos($url, '?') === false) {
|
||||
$url .= '?';
|
||||
} else {
|
||||
$url .= '&';
|
||||
}
|
||||
|
||||
$url .= 'token_auth=' . $tokenAuth;
|
||||
}
|
||||
|
||||
try {
|
||||
Log::debug("Execute HTTP API request: " . $url);
|
||||
$response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate);
|
||||
|
|
@ -246,19 +311,21 @@ class CliMulti {
|
|||
} catch (\Exception $e) {
|
||||
$message = "Got invalid response from API request: $url. ";
|
||||
|
||||
if (empty($response)) {
|
||||
if (isset($response) && empty($response)) {
|
||||
$message .= "The response was empty. This usually means a server error. This solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. Please check your Web server Error Log file for more details.";
|
||||
} else {
|
||||
$message .= "Response was '" . $e->getMessage() . "'";
|
||||
}
|
||||
|
||||
$output->write($message);
|
||||
|
||||
Log::debug($e);
|
||||
}
|
||||
}
|
||||
|
||||
private function appendTestmodeParamToUrlIfNeeded($url)
|
||||
{
|
||||
$isTestMode = $url && false !== strpos($url, 'tests/PHPUnit/proxy');
|
||||
$isTestMode = defined('PIWIK_TEST_MODE');
|
||||
|
||||
if ($isTestMode && false === strpos($url, '?')) {
|
||||
$url .= "?testmode=1";
|
||||
|
|
@ -269,12 +336,42 @@ class CliMulti {
|
|||
return $url;
|
||||
}
|
||||
|
||||
private function getPhpCommandIfValid($path)
|
||||
/**
|
||||
* @param array $piwikUrls
|
||||
* @return array
|
||||
*/
|
||||
private function requestUrls(array $piwikUrls)
|
||||
{
|
||||
if (!empty($path) && is_executable($path)) {
|
||||
if (0 === strpos($path, PHP_BINDIR) && false === strpos($path, 'phpunit')) {
|
||||
return $path;
|
||||
}
|
||||
}
|
||||
$this->start($piwikUrls);
|
||||
|
||||
do {
|
||||
usleep(100000); // 100 * 1000 = 100ms
|
||||
} while (!$this->hasFinished());
|
||||
|
||||
$results = $this->getResponse($piwikUrls);
|
||||
$this->cleanup();
|
||||
|
||||
self::cleanupNotRemovedFiles();
|
||||
|
||||
return $results;
|
||||
}
|
||||
|
||||
private static function getSuperUserTokenAuths()
|
||||
{
|
||||
$tokens = array();
|
||||
|
||||
/**
|
||||
* Used to be in CronArchive, moved to CliMulti.
|
||||
*
|
||||
* @ignore
|
||||
*/
|
||||
Piwik::postEvent('CronArchive.getTokenAuth', array(&$tokens));
|
||||
|
||||
return $tokens;
|
||||
}
|
||||
|
||||
public function setUrlToPiwik($urlToPiwik)
|
||||
{
|
||||
$this->urlToPiwik = $urlToPiwik;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue