olli/questlab
Watch
1
0
Fork
You've already forked questlab
 0
Code Issues 15 Pull requests Projects Releases Wiki Activity

update Piwik to version 2.16 (fixes #91)

Browse source
This commit is contained in:
oliver 2016-04-10 18:55:57 +02:00
commit d885a4baa9
5833 changed files with 418860 additions and 226988 deletions
Download patch file Download diff file Expand all files Collapse all files

17
www/analytics/core/.htaccess
View file

@ -1,13 +1,8 @@
<Files "*">
<IfModule mod_access.c>
Deny from all
</IfModule>
<IfModule !mod_access_compat>
<IfModule mod_authz_host.c>
Deny from all
</IfModule>
</IfModule>
<IfModule mod_access_compat>
Deny from all
</IfModule>
<IfVersion < 2.4>
Deny from all
</IfVersion>
<IfVersion >= 2.4>
Require all denied
</IfVersion>
</Files>

131
www/analytics/core/API/ApiRenderer.php Normal file
View file

@ -0,0 +1,131 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\API;
use Exception;
use Piwik\Common;
use Piwik\DataTable\Renderer;
use Piwik\DataTable;
use Piwik\Piwik;
use Piwik\Plugin;
/**
* API renderer
*/
abstract class ApiRenderer
{
protected $request;
final public function __construct($request)
{
$this->request = $request;
$this->init();
}
protected function init()
{
}
abstract public function sendHeader();
public function renderSuccess($message)
{
return 'Success:' . $message;
}
public function renderException($message, \Exception $exception)
{
return $message;
}
public function renderScalar($scalar)
{
$dataTable = new DataTable\Simple();
$dataTable->addRowsFromArray(array($scalar));
return $this->renderDataTable($dataTable);
}
public function renderDataTable($dataTable)
{
$renderer = $this->buildDataTableRenderer($dataTable);
return $renderer->render();
}
public function renderArray($array)
{
$renderer = $this->buildDataTableRenderer($array);
return $renderer->render();
}
public function renderObject($object)
{
$exception = new Exception('The API cannot handle this data structure.');
return $this->renderException($exception->getMessage(), $exception);
}
public function renderResource($resource)
{
$exception = new Exception('The API cannot handle this data structure.');
return $this->renderException($exception->getMessage(), $exception);
}
/**
* @param $dataTable
* @return Renderer
*/
protected function buildDataTableRenderer($dataTable)
{
$format = self::getFormatFromClass(get_class($this));
if ($format == 'json2') {
$format = 'json';
}
$renderer = Renderer::factory($format);
$renderer->setTable($dataTable);
$renderer->setRenderSubTables(Common::getRequestVar('expanded', false, 'int', $this->request));
$renderer->setHideIdSubDatableFromResponse(Common::getRequestVar('hideIdSubDatable', false, 'int', $this->request));
return $renderer;
}
/**
* @param string $format
* @param array $request
* @return ApiRenderer
* @throws Exception
*/
public static function factory($format, $request)
{
$formatToCheck = '\\' . ucfirst(strtolower($format));
$rendererClassnames = Plugin\Manager::getInstance()->findMultipleComponents('Renderer', 'Piwik\\API\\ApiRenderer');
foreach ($rendererClassnames as $klassName) {
if (Common::stringEndsWith($klassName, $formatToCheck)) {
return new $klassName($request);
}
}
$availableRenderers = array();
foreach ($rendererClassnames as $rendererClassname) {
$availableRenderers[] = self::getFormatFromClass($rendererClassname);
}
$availableRenderers = implode(', ', $availableRenderers);
Common::sendHeader('Content-Type: text/plain; charset=utf-8');
throw new Exception(Piwik::translate('General_ExceptionInvalidRendererFormat', array($format, $availableRenderers)));
}
private static function getFormatFromClass($klassname)
{
$klass = explode('\\', $klassname);
return strtolower(end($klass));
}
}

41
www/analytics/core/API/CORSHandler.php Normal file
View file

@ -0,0 +1,41 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\API;
use Piwik\Url;
class CORSHandler
{
/**
* @var array
*/
protected $domains;
public function __construct()
{
$this->domains = Url::getCorsHostsFromConfig();
}
public function handle()
{
// allow Piwik to serve data to all domains
if (in_array("*", $this->domains)) {
header('Access-Control-Allow-Origin: *');
return;
}
// specifically allow if it is one of the whitelisted CORS domains
if (!empty($_SERVER['HTTP_ORIGIN'])) {
$origin = $_SERVER['HTTP_ORIGIN'];
if (in_array($origin, $this->domains, true)) {
header('Access-Control-Allow-Origin: ' . $_SERVER['HTTP_ORIGIN']);
}
}
}
}

174
www/analytics/core/API/DataTableGenericFilter.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -11,20 +11,37 @@ namespace Piwik\API;
use Exception;
use Piwik\Common;
use Piwik\DataTable;
use Piwik\DataTable\Filter\AddColumnsProcessedMetricsGoal;
use Piwik\Plugin\ProcessedMetric;
use Piwik\Plugin\Report;
class DataTableGenericFilter
{
private static $genericFiltersInfo = null;
/**
* List of filter names not to run.
*
* @var string[]
*/
private $disabledFilters = array();
/**
* @var Report
*/
private $report;
/**
* @var array
*/
private $request;
/**
* Constructor
*
* @param $request
*/
function __construct($request)
public function __construct($request, $report)
{
$this->request = $request;
$this->report = $report;
}
/**
@ -37,6 +54,16 @@ class DataTableGenericFilter
$this->applyGenericFilters($table);
}
/**
* Makes sure a set of filters are not run.
*
* @param string[] $filterNames The name of each filter to disable.
*/
public function disableFilters($filterNames)
{
$this->disabledFilters = array_unique(array_merge($this->disabledFilters, $filterNames));
}
/**
* Returns an array containing the information of the generic Filter
* to be applied automatically to the data resulting from the API calls.
@ -51,43 +78,54 @@ class DataTableGenericFilter
*/
public static function getGenericFiltersInformation()
{
if (is_null(self::$genericFiltersInfo)) {
self::$genericFiltersInfo = array(
'Pattern' => array(
'filter_column' => array('string', 'label'),
'filter_pattern' => array('string'),
),
'PatternRecursive' => array(
'filter_column_recursive' => array('string', 'label'),
'filter_pattern_recursive' => array('string'),
),
'ExcludeLowPopulation' => array(
'filter_excludelowpop' => array('string'),
'filter_excludelowpop_value' => array('float', '0'),
),
'AddColumnsProcessedMetrics' => array(
'filter_add_columns_when_show_all_columns' => array('integer')
),
'AddColumnsProcessedMetricsGoal' => array(
'filter_update_columns_when_show_all_goals' => array('integer'),
'idGoal' => array('string', AddColumnsProcessedMetricsGoal::GOALS_OVERVIEW),
),
'Sort' => array(
'filter_sort_column' => array('string'),
'filter_sort_order' => array('string', 'desc'),
),
'Truncate' => array(
'filter_truncate' => array('integer'),
),
'Limit' => array(
'filter_offset' => array('integer', '0'),
'filter_limit' => array('integer'),
'keep_summary_row' => array('integer', '0'),
),
);
return array(
array('Pattern',
array(
'filter_column' => array('string', 'label'),
'filter_pattern' => array('string')
)),
array('PatternRecursive',
array(
'filter_column_recursive' => array('string', 'label'),
'filter_pattern_recursive' => array('string'),
)),
array('ExcludeLowPopulation',
array(
'filter_excludelowpop' => array('string'),
'filter_excludelowpop_value' => array('float', '0'),
)),
array('Sort',
array(
'filter_sort_column' => array('string'),
'filter_sort_order' => array('string', 'desc'),
)),
array('Truncate',
array(
'filter_truncate' => array('integer'),
)),
array('Limit',
array(
'filter_offset' => array('integer', '0'),
'filter_limit' => array('integer'),
'keep_summary_row' => array('integer', '0'),
))
);
}
private function getGenericFiltersHavingDefaultValues()
{
$filters = self::getGenericFiltersInformation();
if ($this->report && $this->report->getDefaultSortColumn()) {
foreach ($filters as $index => $filter) {
if ($filter[0] === 'Sort') {
$filters[$index][1]['filter_sort_column'] = array('string', $this->report->getDefaultSortColumn());
$filters[$index][1]['filter_sort_order'] = array('string', $this->report->getDefaultSortOrder());
}
}
}
return self::$genericFiltersInfo;
return $filters;
}
/**
@ -107,13 +145,20 @@ class DataTableGenericFilter
return;
}
$genericFilters = self::getGenericFiltersInformation();
$genericFilters = $this->getGenericFiltersHavingDefaultValues();
$filterApplied = false;
foreach ($genericFilters as $filterName => $parameters) {
foreach ($genericFilters as $filterMeta) {
$filterName = $filterMeta[0];
$filterParams = $filterMeta[1];
$filterParameters = array();
$exceptionRaised = false;
foreach ($parameters as $name => $info) {
if (in_array($filterName, $this->disabledFilters)) {
continue;
}
foreach ($filterParams as $name => $info) {
// parameter type to cast to
$type = $info[0];
@ -123,12 +168,6 @@ class DataTableGenericFilter
$defaultValue = $info[1];
}
// third element in the array, if it exists, overrides the name of the request variable
$varName = $name;
if (isset($info[2])) {
$varName = $info[2];
}
try {
$value = Common::getRequestVar($name, $defaultValue, $type, $this->request);
settype($value, $type);
@ -144,6 +183,45 @@ class DataTableGenericFilter
$filterApplied = true;
}
}
return $filterApplied;
}
public function areProcessedMetricsNeededFor($metrics)
{
$columnQueryParameters = array(
'filter_column',
'filter_column_recursive',
'filter_excludelowpop',
'filter_sort_column'
);
foreach ($columnQueryParameters as $queryParamName) {
$queryParamValue = Common::getRequestVar($queryParamName, false, $type = null, $this->request);
if (!empty($queryParamValue)
&& $this->containsProcessedMetric($metrics, $queryParamValue)
) {
return true;
}
}
return false;
}
/**
* @param ProcessedMetric[] $metrics
* @param string $name
* @return bool
*/
private function containsProcessedMetric($metrics, $name)
{
foreach ($metrics as $metric) {
if ($metric instanceof ProcessedMetric
&& $metric->getName() == $name
) {
return true;
}
}
return false;
}
}

44
www/analytics/core/API/DataTableManipulator.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -10,7 +10,6 @@ namespace Piwik\API;
use Exception;
use Piwik\Archive\DataTableFactory;
use Piwik\Common;
use Piwik\DataTable\Row;
use Piwik\DataTable;
use Piwik\Period\Range;
@ -63,7 +62,7 @@ abstract class DataTableManipulator
{
if ($dataTable instanceof DataTable\Map) {
return $this->manipulateDataTableMap($dataTable);
} else if ($dataTable instanceof DataTable) {
} elseif ($dataTable instanceof DataTable) {
return $this->manipulateDataTable($dataTable);
} else {
return $dataTable;
@ -90,7 +89,7 @@ abstract class DataTableManipulator
* Manipulates a single DataTable instance. Derived classes must define
* this function.
*/
protected abstract function manipulateDataTable($dataTable);
abstract protected function manipulateDataTable($dataTable);
/**
* Load the subtable for a row.
@ -124,7 +123,7 @@ abstract class DataTableManipulator
}
}
$method = $this->getApiMethodForSubtable();
$method = $this->getApiMethodForSubtable($request);
return $this->callApiAndReturnDataTable($this->apiModule, $method, $request);
}
@ -136,19 +135,34 @@ abstract class DataTableManipulator
* @param $request
* @return
*/
protected abstract function manipulateSubtableRequest($request);
abstract protected function manipulateSubtableRequest($request);
/**
* Extract the API method for loading subtables from the meta data
*
* @throws Exception
* @return string
*/
private function getApiMethodForSubtable()
private function getApiMethodForSubtable($request)
{
if (!$this->apiMethodForSubtable) {
$meta = API::getInstance()->getMetadata('all', $this->apiModule, $this->apiMethod);
if (!empty($request['idSite'])) {
$idSite = $request['idSite'];
} else {
$idSite = 'all';
}
if(empty($meta)) {
$apiParameters = array();
if (!empty($request['idDimension'])) {
$apiParameters['idDimension'] = $request['idDimension'];
}
if (!empty($request['idGoal'])) {
$apiParameters['idGoal'] = $request['idGoal'];
}
$meta = API::getInstance()->getMetadata($idSite, $this->apiModule, $this->apiMethod, $apiParameters);
if (empty($meta)) {
throw new Exception(sprintf(
"The DataTable cannot be manipulated: Metadata for report %s.%s could not be found. You can define the metadata in a hook, see example at: http://developer.piwik.org/api-reference/events#apigetreportmetadata",
$this->apiModule, $this->apiMethod
@ -171,6 +185,8 @@ abstract class DataTableManipulator
$request = $this->manipulateSubtableRequest($request);
$request['serialize'] = 0;
$request['expanded'] = 0;
$request['format'] = 'original';
$request['format_metrics'] = 0;
// don't want to run recursive filters on the subtables as they are loaded,
// otherwise the result will be empty in places (or everywhere). instead we
@ -179,14 +195,8 @@ abstract class DataTableManipulator
$dataTable = Proxy::getInstance()->call($class, $method, $request);
$response = new ResponseBuilder($format = 'original', $request);
$dataTable = $response->getResponse($dataTable);
if (Common::getRequestVar('disable_queued_filters', 0, 'int', $request) == 0) {
if (method_exists($dataTable, 'applyQueuedFilters')) {
$dataTable->applyQueuedFilters();
}
}
$response->disableSendHeader();
$dataTable = $response->getResponse($dataTable, $apiModule, $method);
return $dataTable;
}
}

44
www/analytics/core/API/DataTableManipulator/Flattener.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -38,17 +38,16 @@ class Flattener extends DataTableManipulator
* Separator for building recursive labels (or paths)
* @var string
*/
public $recursiveLabelSeparator = ' - ';
public $recursiveLabelSeparator = '';
/**
* @param DataTable $dataTable
* @param string $recursiveLabelSeparator
* @return DataTable|DataTable\Map
*/
public function flatten($dataTable)
public function flatten($dataTable, $recursiveLabelSeparator)
{
if ($this->apiModule == 'Actions' || $this->apiMethod == 'getWebsites') {
$this->recursiveLabelSeparator = '/';
}
$this->recursiveLabelSeparator = $recursiveLabelSeparator;
return $this->manipulate($dataTable);
}
@ -72,9 +71,10 @@ class Flattener extends DataTableManipulator
}
$newDataTable = $dataTable->getEmptyClone($keepFilters);
foreach ($dataTable->getRows() as $row) {
$this->flattenRow($row, $newDataTable);
foreach ($dataTable->getRows() as $rowId => $row) {
$this->flattenRow($row, $rowId, $newDataTable);
}
return $newDataTable;
}
@ -84,15 +84,21 @@ class Flattener extends DataTableManipulator
* @param string $labelPrefix
* @param bool $parentLogo
*/
private function flattenRow(Row $row, DataTable $dataTable,
private function flattenRow(Row $row, $rowId, DataTable $dataTable,
$labelPrefix = '', $parentLogo = false)
{
$label = $row->getColumn('label');
if ($label !== false) {
$label = trim($label);
if (substr($label, 0, 1) == '/' && $this->recursiveLabelSeparator == '/') {
$label = substr($label, 1);
if ($this->recursiveLabelSeparator == '/') {
if (substr($label, 0, 1) == '/') {
$label = substr($label, 1);
} elseif ($rowId === DataTable::ID_SUMMARY_ROW && $labelPrefix && $label != DataTable::LABEL_SUMMARY_ROW) {
$label = ' - ' . $label;
}
}
$label = $labelPrefix . $label;
$row->setColumn('label', $label);
}
@ -103,7 +109,16 @@ class Flattener extends DataTableManipulator
$row->setMetadata('logo', $logo);
}
$subTable = $this->loadSubtable($dataTable, $row);
/** @var DataTable $subTable */
$subTable = $row->getSubtable();
if ($subTable) {
$subTable->applyQueuedFilters();
$row->deleteMetadata('idsubdatatable_in_db');
} else {
$subTable = $this->loadSubtable($dataTable, $row);
}
$row->removeSubtable();
if ($subTable === null) {
@ -117,8 +132,8 @@ class Flattener extends DataTableManipulator
$dataTable->addRow($row);
}
$prefix = $label . $this->recursiveLabelSeparator;
foreach ($subTable->getRows() as $row) {
$this->flattenRow($row, $dataTable, $prefix, $logo);
foreach ($subTable->getRows() as $rowId => $row) {
$this->flattenRow($row, $rowId, $dataTable, $prefix, $logo);
}
}
}
@ -127,6 +142,7 @@ class Flattener extends DataTableManipulator
* Remove the flat parameter from the subtable request
*
* @param array $request
* @return array
*/
protected function manipulateSubtableRequest($request)
{

39
www/analytics/core/API/DataTableManipulator/LabelFilter.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -24,6 +24,7 @@ use Piwik\DataTable\Row;
class LabelFilter extends DataTableManipulator
{
const SEPARATOR_RECURSIVE_LABEL = '>';
const TERMINAL_OPERATOR = '@';
private $labels;
private $addLabelIndex;
@ -63,6 +64,10 @@ class LabelFilter extends DataTableManipulator
*/
private function doFilterRecursiveDescend($labelParts, $dataTable)
{
// we need to make sure to rebuild the index as some filters change the label column directly via
// $row->setColumn('label', '') which would not be noticed in the label index otherwise.
$dataTable->rebuildIndex();
// search for the first part of the tree search
$labelPart = array_shift($labelParts);
@ -101,6 +106,9 @@ class LabelFilter extends DataTableManipulator
protected function manipulateSubtableRequest($request)
{
unset($request['label']);
unset($request['flat']);
$request['totals'] = 0;
$request['filter_sort_column'] = ''; // do not sort, we only want to find a matching column
return $request;
}
@ -111,16 +119,22 @@ class LabelFilter extends DataTableManipulator
* Note: The HTML Encoded version must be tried first, since in ResponseBuilder the $label is unsanitized
* via Common::unsanitizeLabelParameter.
*
* @param string $label
* @param string $originalLabel
* @return array
*/
private function getLabelVariations($label)
private function getLabelVariations($originalLabel)
{
static $pageTitleReports = array('getPageTitles', 'getEntryPageTitles', 'getExitPageTitles');
$originalLabel = trim($originalLabel);
$isTerminal = substr($originalLabel, 0, 1) == self::TERMINAL_OPERATOR;
if ($isTerminal) {
$originalLabel = substr($originalLabel, 1);
}
$variations = array();
$label = urldecode($label);
$label = trim($label);
$label = trim(urldecode($originalLabel));
$sanitizedLabel = Common::sanitizeInputValue($label);
$variations[] = $sanitizedLabel;
@ -128,13 +142,20 @@ class LabelFilter extends DataTableManipulator
if ($this->apiModule == 'Actions'
&& in_array($this->apiMethod, $pageTitleReports)
) {
// special case: the Actions.getPageTitles report prefixes some labels with a blank.
// the blank might be passed by the user but is removed in Request::getRequestArrayFromString.
$variations[] = ' ' . $sanitizedLabel;
$variations[] = ' ' . $label;
if ($isTerminal) {
array_unshift($variations, ' ' . $sanitizedLabel);
array_unshift($variations, ' ' . $label);
} else {
// special case: the Actions.getPageTitles report prefixes some labels with a blank.
// the blank might be passed by the user but is removed in Request::getRequestArrayFromString.
$variations[] = ' ' . $sanitizedLabel;
$variations[] = ' ' . $label;
}
}
$variations[] = $label;
$variations = array_unique($variations);
return $variations;
}

189
www/analytics/core/API/DataTableManipulator/ReportTotalsCalculator.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -10,13 +10,9 @@ namespace Piwik\API\DataTableManipulator;
use Piwik\API\DataTableManipulator;
use Piwik\DataTable;
use Piwik\DataTable\Row;
use Piwik\DataTable\BaseFilter;
use Piwik\Period\Range;
use Piwik\Period;
use Piwik\Piwik;
use Piwik\Metrics;
use Piwik\Plugins\API\API;
use Piwik\Period;
use Piwik\Plugin\Report;
/**
* This class is responsible for setting the metadata property 'totals' on each dataTable if the report
@ -26,10 +22,29 @@ use Piwik\Plugins\API\API;
class ReportTotalsCalculator extends DataTableManipulator
{
/**
* Cached report metadata array.
* Array [readableMetric] => [summed value]
* @var array
*/
private static $reportMetadata = array();
private $totals = array();
/**
* @var Report
*/
private $report;
/**
* Constructor
*
* @param bool $apiModule
* @param bool $apiMethod
* @param array $request
* @param Report $report
*/
public function __construct($apiModule = false, $apiMethod = false, $request = array(), $report = null)
{
parent::__construct($apiModule, $apiMethod, $request);
$this->report = $report;
}
/**
* @param DataTable $table
@ -46,7 +61,7 @@ class ReportTotalsCalculator extends DataTableManipulator
try {
return $this->manipulate($table);
} catch(\Exception $e) {
} catch (\Exception $e) {
// eg. requests with idSubtable may trigger this exception
// (where idSubtable was removed in
// ?module=API&method=Events.getNameFromCategoryId&idSubtable=1&secondaryDimension=eventName&format=XML&idSite=1&period=day&date=yesterday&flat=0
@ -62,75 +77,32 @@ class ReportTotalsCalculator extends DataTableManipulator
*/
protected function manipulateDataTable($dataTable)
{
$report = $this->findCurrentReport();
if (!empty($report) && empty($report['dimension'])) {
if (!empty($this->report) && !$this->report->getDimension() && !$this->isAllMetricsReport()) {
// we currently do not calculate the total value for reports having no dimension
return $dataTable;
}
// Array [readableMetric] => [summed value]
$totalValues = array();
$this->totals = array();
$firstLevelTable = $this->makeSureToWorkOnFirstLevelDataTable($dataTable);
$metricsToCalculate = Metrics::getMetricIdsToProcessReportTotal();
$metricNames = array();
foreach ($metricsToCalculate as $metricId) {
if (!$this->hasDataTableMetric($firstLevelTable, $metricId)) {
continue;
}
$metricNames[$metricId] = Metrics::getReadableColumnName($metricId);
}
foreach ($firstLevelTable->getRows() as $row) {
$totalValues = $this->sumColumnValueToTotal($row, $metricId, $totalValues);
foreach ($firstLevelTable->getRows() as $row) {
$columns = $row->getColumns();
foreach ($metricNames as $metricId => $metricName) {
$this->sumColumnValueToTotal($columns, $metricId, $metricName);
}
}
$dataTable->setMetadata('totals', $totalValues);
$dataTable->setMetadata('totals', $this->totals);
return $dataTable;
}
private function hasDataTableMetric(DataTable $dataTable, $metricId)
{
$firstRow = $dataTable->getFirstRow();
if (empty($firstRow)) {
return false;
}
if (false === $this->getColumn($firstRow, $metricId)) {
return false;
}
return true;
}
/**
* Returns column from a given row.
* Will work with 2 types of datatable
* - raw datatables coming from the archive DB, which columns are int indexed
* - datatables processed resulting of API calls, which columns have human readable english names
*
* @param Row|array $row
* @param int $columnIdRaw see consts in Metrics::
* @return mixed Value of column, false if not found
*/
private function getColumn($row, $columnIdRaw)
{
$columnIdReadable = Metrics::getReadableColumnName($columnIdRaw);
if ($row instanceof Row) {
$raw = $row->getColumn($columnIdRaw);
if ($raw !== false) {
return $raw;
}
return $row->getColumn($columnIdReadable);
}
return false;
}
private function makeSureToWorkOnFirstLevelDataTable($table)
{
if (!array_key_exists('idSubtable', $this->request)) {
@ -144,8 +116,8 @@ class ReportTotalsCalculator extends DataTableManipulator
$module = $this->apiModule;
$action = $this->apiMethod;
} else {
$module = $firstLevelReport['module'];
$action = $firstLevelReport['action'];
$module = $firstLevelReport->getModule();
$action = $firstLevelReport->getAction();
}
$request = $this->request;
@ -164,33 +136,56 @@ class ReportTotalsCalculator extends DataTableManipulator
}
}
return $this->callApiAndReturnDataTable($module, $action, $request);
$table = $this->callApiAndReturnDataTable($module, $action, $request);
if ($table instanceof DataTable\Map) {
$table = $table->mergeChildren();
}
return $table;
}
private function sumColumnValueToTotal(Row $row, $metricId, $totalValues)
private function sumColumnValueToTotal($columns, $metricId, $metricName)
{
$value = $this->getColumn($row, $metricId);
if (false === $value) {
return $totalValues;
$value = false;
if (array_key_exists($metricId, $columns)) {
$value = $columns[$metricId];
}
$metricName = Metrics::getReadableColumnName($metricId);
if ($value === false) {
// we do not add $metricId to $possibleMetricNames for a small performance improvement since in most cases
// $metricId should be present in $columns so we avoid this foreach loop
$possibleMetricNames = array(
$metricName,
// TODO: this and below is a hack to get report totals to work correctly w/ MultiSites.getAll. can be corrected
// when all metrics are described by Metadata classes & internal naming quirks are handled by core system.
'Goal_' . $metricName,
'Actions_' . $metricName
);
foreach ($possibleMetricNames as $possibleMetricName) {
if (array_key_exists($possibleMetricName, $columns)) {
$value = $columns[$possibleMetricName];
break;
}
}
if (array_key_exists($metricName, $totalValues)) {
$totalValues[$metricName] += $value;
if ($value === false) {
return;
}
}
if (array_key_exists($metricName, $this->totals)) {
$this->totals[$metricName] += $value;
} else {
$totalValues[$metricName] = $value;
$this->totals[$metricName] = $value;
}
return $totalValues;
}
/**
* Make sure to get all rows of the first level table.
*
* @param array $request
* @return array
*/
protected function manipulateSubtableRequest($request)
{
@ -198,6 +193,7 @@ class ReportTotalsCalculator extends DataTableManipulator
$request['expanded'] = 0;
$request['filter_limit'] = -1;
$request['filter_offset'] = 0;
$request['filter_sort_column'] = '';
$parametersToRemove = array('flat');
@ -213,38 +209,21 @@ class ReportTotalsCalculator extends DataTableManipulator
return $request;
}
private function getReportMetadata()
{
if (!empty(static::$reportMetadata)) {
return static::$reportMetadata;
}
static::$reportMetadata = API::getInstance()->getReportMetadata();
return static::$reportMetadata;
}
private function findCurrentReport()
{
foreach ($this->getReportMetadata() as $report) {
if ($this->apiMethod == $report['action']
&& $this->apiModule == $report['module']) {
return $report;
}
}
}
private function findFirstLevelReport()
{
foreach ($this->getReportMetadata() as $report) {
if (!empty($report['actionToLoadSubTables'])
&& $this->apiMethod == $report['actionToLoadSubTables']
&& $this->apiModule == $report['module']
foreach (Report::getAllReports() as $report) {
$actionToLoadSubtables = $report->getActionToLoadSubTables();
if ($actionToLoadSubtables == $this->apiMethod
&& $this->apiModule == $report->getModule()
) {
return $report;
}
}
return null;
}
private function isAllMetricsReport()
{
return $this->report->getModule() == 'API' && $this->report->getAction() == 'get';
}
}

436
www/analytics/core/API/DataTablePostProcessor.php Normal file
View file

@ -0,0 +1,436 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\API;
use Exception;
use Piwik\API\DataTableManipulator\Flattener;
use Piwik\API\DataTableManipulator\LabelFilter;
use Piwik\API\DataTableManipulator\ReportTotalsCalculator;
use Piwik\Common;
use Piwik\DataTable;
use Piwik\DataTable\DataTableInterface;
use Piwik\DataTable\Filter\PivotByDimension;
use Piwik\Metrics\Formatter;
use Piwik\Plugin\ProcessedMetric;
use Piwik\Plugin\Report;
/**
* Processes DataTables that should be served through Piwik's APIs. This processing handles
* special query parameters and computes processed metrics. It does not included rendering to
* output formats (eg, 'xml').
*/
class DataTablePostProcessor
{
const PROCESSED_METRICS_COMPUTED_FLAG = 'processed_metrics_computed';
/**
* @var null|Report
*/
private $report;
/**
* @var string[]
*/
private $request;
/**
* @var string
*/
private $apiModule;
/**
* @var string
*/
private $apiMethod;
/**
* @var Inconsistencies
*/
private $apiInconsistencies;
/**
* @var Formatter
*/
private $formatter;
private $callbackBeforeGenericFilters;
private $callbackAfterGenericFilters;
/**
* Constructor.
*/
public function __construct($apiModule, $apiMethod, $request)
{
$this->apiModule = $apiModule;
$this->apiMethod = $apiMethod;
$this->setRequest($request);
$this->report = Report::factory($apiModule, $apiMethod);
$this->apiInconsistencies = new Inconsistencies();
$this->setFormatter(new Formatter());
}
public function setFormatter(Formatter $formatter)
{
$this->formatter = $formatter;
}
public function setRequest($request)
{
$this->request = $request;
}
public function setCallbackBeforeGenericFilters($callbackBeforeGenericFilters)
{
$this->callbackBeforeGenericFilters = $callbackBeforeGenericFilters;
}
public function setCallbackAfterGenericFilters($callbackAfterGenericFilters)
{
$this->callbackAfterGenericFilters = $callbackAfterGenericFilters;
}
/**
* Apply post-processing logic to a DataTable of a report for an API request.
*
* @param DataTableInterface $dataTable The data table to process.
* @return DataTableInterface A new data table.
*/
public function process(DataTableInterface $dataTable)
{
// TODO: when calculating metrics before hand, only calculate for needed metrics, not all. NOTE:
// this is non-trivial since it will require, eg, to make sure processed metrics aren't added
// after pivotBy is handled.
$dataTable = $this->applyPivotByFilter($dataTable);
$dataTable = $this->applyTotalsCalculator($dataTable);
$dataTable = $this->applyFlattener($dataTable);
if ($this->callbackBeforeGenericFilters) {
call_user_func($this->callbackBeforeGenericFilters, $dataTable);
}
$dataTable = $this->applyGenericFilters($dataTable);
$this->applyComputeProcessedMetrics($dataTable);
if ($this->callbackAfterGenericFilters) {
call_user_func($this->callbackAfterGenericFilters, $dataTable);
}
// we automatically safe decode all datatable labels (against xss)
$dataTable->queueFilter('SafeDecodeLabel');
$dataTable = $this->convertSegmentValueToSegment($dataTable);
$dataTable = $this->applyQueuedFilters($dataTable);
$dataTable = $this->applyRequestedColumnDeletion($dataTable);
$dataTable = $this->applyLabelFilter($dataTable);
$dataTable = $this->applyMetricsFormatting($dataTable);
return $dataTable;
}
private function convertSegmentValueToSegment(DataTableInterface $dataTable)
{
$dataTable->filter('AddSegmentBySegmentValue', array($this->report));
$dataTable->filter('ColumnCallbackDeleteMetadata', array('segmentValue'));
return $dataTable;
}
/**
* @param DataTableInterface $dataTable
* @return DataTableInterface
*/
public function applyPivotByFilter(DataTableInterface $dataTable)
{
$pivotBy = Common::getRequestVar('pivotBy', false, 'string', $this->request);
if (!empty($pivotBy)) {
$this->applyComputeProcessedMetrics($dataTable);
$reportId = $this->apiModule . '.' . $this->apiMethod;
$pivotByColumn = Common::getRequestVar('pivotByColumn', false, 'string', $this->request);
$pivotByColumnLimit = Common::getRequestVar('pivotByColumnLimit', false, 'int', $this->request);
$dataTable->filter('ColumnCallbackDeleteMetadata', array('segmentValue'));
$dataTable->filter('ColumnCallbackDeleteMetadata', array('segment'));
$dataTable->filter('PivotByDimension', array($reportId, $pivotBy, $pivotByColumn, $pivotByColumnLimit,
PivotByDimension::isSegmentFetchingEnabledInConfig()));
}
return $dataTable;
}
/**
* @param DataTableInterface $dataTable
* @return DataTable|DataTableInterface|DataTable\Map
*/
public function applyFlattener($dataTable)
{
if (Common::getRequestVar('flat', '0', 'string', $this->request) == '1') {
$flattener = new Flattener($this->apiModule, $this->apiMethod, $this->request);
if (Common::getRequestVar('include_aggregate_rows', '0', 'string', $this->request) == '1') {
$flattener->includeAggregateRows();
}
$recursiveLabelSeparator = ' - ';
if ($this->report) {
$recursiveLabelSeparator = $this->report->getRecursiveLabelSeparator();
}
$dataTable = $flattener->flatten($dataTable, $recursiveLabelSeparator);
}
return $dataTable;
}
/**
* @param DataTableInterface $dataTable
* @return DataTableInterface
*/
public function applyTotalsCalculator($dataTable)
{
if (1 == Common::getRequestVar('totals', '1', 'integer', $this->request)) {
$calculator = new ReportTotalsCalculator($this->apiModule, $this->apiMethod, $this->request, $this->report);
$dataTable = $calculator->calculate($dataTable);
}
return $dataTable;
}
/**
* @param DataTableInterface $dataTable
* @return DataTableInterface
*/
public function applyGenericFilters($dataTable)
{
// if the flag disable_generic_filters is defined we skip the generic filters
if (0 == Common::getRequestVar('disable_generic_filters', '0', 'string', $this->request)) {
$this->applyProcessedMetricsGenericFilters($dataTable);
$genericFilter = new DataTableGenericFilter($this->request, $this->report);
$self = $this;
$report = $this->report;
$dataTable->filter(function (DataTable $table) use ($genericFilter, $report, $self) {
$processedMetrics = Report::getProcessedMetricsForTable($table, $report);
if ($genericFilter->areProcessedMetricsNeededFor($processedMetrics)) {
$self->computeProcessedMetrics($table);
}
});
$label = self::getLabelFromRequest($this->request);
if (!empty($label)) {
$genericFilter->disableFilters(array('Limit', 'Truncate'));
}
$genericFilter->filter($dataTable);
}
return $dataTable;
}
/**
* @param DataTableInterface $dataTable
* @return DataTableInterface
*/
public function applyProcessedMetricsGenericFilters($dataTable)
{
$addNormalProcessedMetrics = null;
try {
$addNormalProcessedMetrics = Common::getRequestVar(
'filter_add_columns_when_show_all_columns', null, 'integer', $this->request);
} catch (Exception $ex) {
// ignore
}
if ($addNormalProcessedMetrics !== null) {
$dataTable->filter('AddColumnsProcessedMetrics', array($addNormalProcessedMetrics));
}
$addGoalProcessedMetrics = null;
try {
$addGoalProcessedMetrics = Common::getRequestVar(
'filter_update_columns_when_show_all_goals', null, 'integer', $this->request);
} catch (Exception $ex) {
// ignore
}
if ($addGoalProcessedMetrics !== null) {
$idGoal = Common::getRequestVar(
'idGoal', DataTable\Filter\AddColumnsProcessedMetricsGoal::GOALS_OVERVIEW, 'string', $this->request);
$dataTable->filter('AddColumnsProcessedMetricsGoal', array($ignore = true, $idGoal));
}
return $dataTable;
}
/**
* @param DataTableInterface $dataTable
* @return DataTableInterface
*/
public function applyQueuedFilters($dataTable)
{
// if the flag disable_queued_filters is defined we skip the filters that were queued
if (Common::getRequestVar('disable_queued_filters', 0, 'int', $this->request) == 0) {
$dataTable->applyQueuedFilters();
}
return $dataTable;
}
/**
* @param DataTableInterface $dataTable
* @return DataTableInterface
*/
public function applyRequestedColumnDeletion($dataTable)
{
// use the ColumnDelete filter if hideColumns/showColumns is provided (must be done
// after queued filters are run so processed metrics can be removed, too)
$hideColumns = Common::getRequestVar('hideColumns', '', 'string', $this->request);
$showColumns = Common::getRequestVar('showColumns', '', 'string', $this->request);
$showRawMetrics = Common::getRequestVar('showRawMetrics', 0, 'int', $this->request);
if (!empty($hideColumns)
|| !empty($showColumns)
) {
$dataTable->filter('ColumnDelete', array($hideColumns, $showColumns));
} else if ($showRawMetrics !== 1) {
$this->removeTemporaryMetrics($dataTable);
}
return $dataTable;
}
/**
* @param DataTableInterface $dataTable
*/
public function removeTemporaryMetrics(DataTableInterface $dataTable)
{
$allColumns = !empty($this->report) ? $this->report->getAllMetrics() : array();
$report = $this->report;
$dataTable->filter(function (DataTable $table) use ($report, $allColumns) {
$processedMetrics = Report::getProcessedMetricsForTable($table, $report);
$allTemporaryMetrics = array();
foreach ($processedMetrics as $metric) {
$allTemporaryMetrics = array_merge($allTemporaryMetrics, $metric->getTemporaryMetrics());
}
if (!empty($allTemporaryMetrics)) {
$table->filter('ColumnDelete', array($allTemporaryMetrics));
}
});
}
/**
* @param DataTableInterface $dataTable
* @return DataTableInterface
*/
public function applyLabelFilter($dataTable)
{
$label = self::getLabelFromRequest($this->request);
// apply label filter: only return rows matching the label parameter (more than one if more than one label)
if (!empty($label)) {
$addLabelIndex = Common::getRequestVar('labelFilterAddLabelIndex', 0, 'int', $this->request) == 1;
$filter = new LabelFilter($this->apiModule, $this->apiMethod, $this->request);
$dataTable = $filter->filter($label, $dataTable, $addLabelIndex);
}
return $dataTable;
}
/**
* @param DataTableInterface $dataTable
* @return DataTableInterface
*/
public function applyMetricsFormatting($dataTable)
{
$formatMetrics = Common::getRequestVar('format_metrics', 0, 'string', $this->request);
if ($formatMetrics == '0') {
return $dataTable;
}
// in Piwik 2.X & below, metrics are not formatted in API responses except for percents.
// this code implements this inconsistency
$onlyFormatPercents = $formatMetrics === 'bc';
$metricsToFormat = null;
if ($onlyFormatPercents) {
$metricsToFormat = $this->apiInconsistencies->getPercentMetricsToFormat();
}
$dataTable->filter(array($this->formatter, 'formatMetrics'), array($this->report, $metricsToFormat));
return $dataTable;
}
/**
* Returns the value for the label query parameter which can be either a string
* (ie, label=...) or array (ie, label[]=...).
*
* @param array $request
* @return array
*/
public static function getLabelFromRequest($request)
{
$label = Common::getRequestVar('label', array(), 'array', $request);
if (empty($label)) {
$label = Common::getRequestVar('label', '', 'string', $request);
if (!empty($label)) {
$label = array($label);
}
}
$label = self::unsanitizeLabelParameter($label);
return $label;
}
public static function unsanitizeLabelParameter($label)
{
// this is needed because Proxy uses Common::getRequestVar which in turn
// uses Common::sanitizeInputValue. This causes the > that separates recursive labels
// to become &gt; and we need to undo that here.
$label = str_replace( htmlentities('>'), '>', $label);
return $label;
}
public function computeProcessedMetrics(DataTable $dataTable)
{
if ($dataTable->getMetadata(self::PROCESSED_METRICS_COMPUTED_FLAG)) {
return;
}
/** @var ProcessedMetric[] $processedMetrics */
$processedMetrics = Report::getProcessedMetricsForTable($dataTable, $this->report);
if (empty($processedMetrics)) {
return;
}
$dataTable->setMetadata(self::PROCESSED_METRICS_COMPUTED_FLAG, true);
foreach ($processedMetrics as $name => $processedMetric) {
if (!$processedMetric->beforeCompute($this->report, $dataTable)) {
continue;
}
foreach ($dataTable->getRows() as $row) {
if ($row->getColumn($name) === false) { // only compute the metric if it has not been computed already
$computedValue = $processedMetric->compute($row);
if ($computedValue !== false) {
$row->addColumn($name, $computedValue);
}
$subtable = $row->getSubtable();
if (!empty($subtable)) {
$this->computeProcessedMetrics($subtable);
}
}
}
}
}
public function applyComputeProcessedMetrics(DataTableInterface $dataTable)
{
$dataTable->filter(array($this, 'computeProcessedMetrics'));
}
}

242
www/analytics/core/API/DocumentationGenerator.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -12,10 +12,10 @@ use Exception;
use Piwik\Common;
use Piwik\Piwik;
use Piwik\Url;
use ReflectionClass;
class DocumentationGenerator
{
protected $modulesToHide = array('CoreAdminHome', 'DBStats');
protected $countPluginsLoaded = 0;
/**
@ -47,63 +47,152 @@ class DocumentationGenerator
if (!empty($prefixUrls)) {
$prefixUrls = 'http://demo.piwik.org/';
}
$str = $toc = '';
$token_auth = "&token_auth=" . Piwik::getCurrentUserTokenAuth();
$parametersToSet = array(
'idSite' => Common::getRequestVar('idSite', 1, 'int'),
'period' => Common::getRequestVar('period', 'day', 'string'),
'date' => Common::getRequestVar('date', 'today', 'string')
);
foreach (Proxy::getInstance()->getMetadata() as $class => $info) {
$moduleName = Proxy::getInstance()->getModuleNameFromClassName($class);
if (in_array($moduleName, $this->modulesToHide)) {
$rClass = new ReflectionClass($class);
if (!Piwik::hasUserSuperUserAccess() && $this->checkIfClassCommentContainsHideAnnotation($rClass)) {
continue;
}
$toc .= "<a href='#$moduleName'>$moduleName</a><br/>";
$str .= "\n<a name='$moduleName' id='$moduleName'></a><h2>Module " . $moduleName . "</h2>";
$str .= "<div class='apiDescription'> " . $info['__documentation'] . " </div>";
foreach ($info as $methodName => $infoMethod) {
if ($methodName == '__documentation') {
continue;
}
$params = $this->getParametersString($class, $methodName);
$str .= "\n <div class='apiMethod'>- <b>$moduleName.$methodName </b>" . $params . "";
$str .= '<small>';
if ($outputExampleUrls) {
// we prefix all URLs with $prefixUrls
// used when we include this output in the Piwik official documentation for example
$str .= "<span class=\"example\">";
$exampleUrl = $this->getExampleUrl($class, $methodName, $parametersToSet);
if ($exampleUrl !== false) {
$lastNUrls = '';
if (preg_match('/(&period)|(&date)/', $exampleUrl)) {
$exampleUrlRss1 = $prefixUrls . $this->getExampleUrl($class, $methodName, array('date' => 'last10', 'period' => 'day') + $parametersToSet);
$exampleUrlRss2 = $prefixUrls . $this->getExampleUrl($class, $methodName, array('date' => 'last5', 'period' => 'week',) + $parametersToSet);
$lastNUrls = ", RSS of the last <a target=_blank href='$exampleUrlRss1&format=rss$token_auth&translateColumnNames=1'>10 days</a>";
}
$exampleUrl = $prefixUrls . $exampleUrl;
$str .= " [ Example in
<a target=_blank href='$exampleUrl&format=xml$token_auth'>XML</a>,
<a target=_blank href='$exampleUrl&format=JSON$token_auth'>Json</a>,
<a target=_blank href='$exampleUrl&format=Tsv$token_auth&translateColumnNames=1'>Tsv (Excel)</a>
$lastNUrls
]";
} else {
$str .= " [ No example available ]";
}
$str .= "</span>";
}
$str .= '</small>';
$str .= "</div>\n";
$toDisplay = $this->prepareModulesAndMethods($info, $moduleName);
foreach ($toDisplay as $moduleName => $methods) {
$toc .= $this->prepareModuleToDisplay($moduleName);
$str .= $this->prepareMethodToDisplay($moduleName, $info, $methods, $class, $outputExampleUrls, $prefixUrls);
}
$str .= '<div style="margin:15px;"><a href="#topApiRef">↑ Back to top</a></div>';
}
$str = "<h2 id='topApiRef' name='topApiRef'>Quick access to APIs</h2>
$toc
$str";
return $str;
}
public function prepareModuleToDisplay($moduleName)
{
return "<a href='#$moduleName'>$moduleName</a><br/>";
}
public function prepareMethodToDisplay($moduleName, $info, $methods, $class, $outputExampleUrls, $prefixUrls)
{
$str = '';
$str .= "\n<a name='$moduleName' id='$moduleName'></a><h2>Module " . $moduleName . "</h2>";
$info['__documentation'] = $this->checkDocumentation($info['__documentation']);
$str .= "<div class='apiDescription'> " . $info['__documentation'] . " </div>";
foreach ($methods as $methodName) {
if (Proxy::getInstance()->isDeprecatedMethod($class, $methodName)) {
continue;
}
$params = $this->getParametersString($class, $methodName);
$str .= "\n <div class='apiMethod'>- <b>$moduleName.$methodName </b>" . $params . "";
$str .= '<small>';
if ($outputExampleUrls) {
$str .= $this->addExamples($class, $methodName, $prefixUrls);
}
$str .= '</small>';
$str .= "</div>\n";
}
return $str;
}
public function prepareModulesAndMethods($info, $moduleName)
{
$toDisplay = array();
foreach ($info as $methodName => $infoMethod) {
if ($methodName == '__documentation') {
continue;
}
$toDisplay[$moduleName][] = $methodName;
}
return $toDisplay;
}
public function addExamples($class, $methodName, $prefixUrls)
{
$token_auth = "&token_auth=" . Piwik::getCurrentUserTokenAuth();
$parametersToSet = array(
'idSite' => Common::getRequestVar('idSite', 1, 'int'),
'period' => Common::getRequestVar('period', 'day', 'string'),
'date' => Common::getRequestVar('date', 'today', 'string')
);
$str = '';
// used when we include this output in the Piwik official documentation for example
$str .= "<span class=\"example\">";
$exampleUrl = $this->getExampleUrl($class, $methodName, $parametersToSet);
if ($exampleUrl !== false) {
$lastNUrls = '';
if (preg_match('/(&period)|(&date)/', $exampleUrl)) {
$exampleUrlRss = $prefixUrls . $this->getExampleUrl($class, $methodName, array('date' => 'last10', 'period' => 'day') + $parametersToSet);
$lastNUrls = ", RSS of the last <a target='_blank' href='$exampleUrlRss&format=rss$token_auth&translateColumnNames=1'>10 days</a>";
}
$exampleUrl = $prefixUrls . $exampleUrl;
$str .= " [ Example in
<a target='_blank' href='$exampleUrl&format=xml$token_auth'>XML</a>,
<a target='_blank' href='$exampleUrl&format=JSON$token_auth'>Json</a>,
<a target='_blank' href='$exampleUrl&format=Tsv$token_auth&translateColumnNames=1'>Tsv (Excel)</a>
$lastNUrls
]";
} else {
$str .= " [ No example available ]";
}
$str .= "</span>";
return $str;
}
/**
* Check if Class contains @hide
*
* @param ReflectionClass $rClass instance of ReflectionMethod
* @return bool
*/
public function checkIfClassCommentContainsHideAnnotation(ReflectionClass $rClass)
{
return false !== strstr($rClass->getDocComment(), '@hide');
}
/**
* Check if documentation contains @hide annotation and deletes it
*
* @param $moduleToCheck
* @return mixed
*/
public function checkDocumentation($moduleToCheck)
{
if (strpos($moduleToCheck, '@hide') == true) {
$moduleToCheck = str_replace(strtok(strstr($moduleToCheck, '@hide'), "\n"), "", $moduleToCheck);
}
return $moduleToCheck;
}
private function getInterfaceString($moduleName, $class, $info, $parametersToSet, $outputExampleUrls, $prefixUrls)
{
$str = '';
$str .= "\n<a name='$moduleName' id='$moduleName'></a><h2>Module " . $moduleName . "</h2>";
$str .= "<div class='apiDescription'> " . $info['__documentation'] . " </div>";
foreach ($info as $methodName => $infoMethod) {
if ($methodName == '__documentation') {
continue;
}
if (Proxy::getInstance()->isDeprecatedMethod($class, $methodName)) {
continue;
}
$str .= $this->getMethodString($moduleName, $class, $parametersToSet, $outputExampleUrls, $prefixUrls, $methodName, $str);
}
$str .= '<div style="margin:15px;"><a href="#topApiRef">↑ Back to top</a></div>';
return $str;
}
@ -136,6 +225,7 @@ class DocumentationGenerator
'ip' => '194.57.91.215',
'idSites' => '1,2',
'idAlert' => '1',
'seconds' => '3600',
// 'segmentName' => 'browserCode',
);
@ -169,8 +259,8 @@ class DocumentationGenerator
$aParameters = Proxy::getInstance()->getParametersList($class, $methodName);
// Kindly force some known generic parameters to appear in the final list
// the parameter 'format' can be set to all API methods (used in tests)
// the parameter 'hideIdSubDatable' is used for integration tests only
// the parameter 'serialize' sets php outputs human readable, used in integration tests and debug
// the parameter 'hideIdSubDatable' is used for system tests only
// the parameter 'serialize' sets php outputs human readable, used in system tests and debug
// the parameter 'language' sets the language for the response (eg. country names)
// the parameter 'flat' reduces a hierarchical table to a single level by concatenating labels
// the parameter 'include_aggregate_rows' can be set to include inner nodes in flat reports
@ -183,12 +273,25 @@ class DocumentationGenerator
$aParameters['label'] = false;
$aParameters['flat'] = false;
$aParameters['include_aggregate_rows'] = false;
$aParameters['filter_limit'] = false; //@review without adding this, I can not set filter_limit in $otherRequestParameters integration tests
$aParameters['filter_sort_column'] = false; //@review without adding this, I can not set filter_sort_column in $otherRequestParameters integration tests
$aParameters['filter_offset'] = false; //@review without adding this, I can not set filter_offset in $otherRequestParameters system tests
$aParameters['filter_limit'] = false; //@review without adding this, I can not set filter_limit in $otherRequestParameters system tests
$aParameters['filter_sort_column'] = false; //@review without adding this, I can not set filter_sort_column in $otherRequestParameters system tests
$aParameters['filter_sort_order'] = false; //@review without adding this, I can not set filter_sort_order in $otherRequestParameters system tests
$aParameters['filter_excludelowpop'] = false; //@review without adding this, I can not set filter_sort_order in $otherRequestParameters system tests
$aParameters['filter_excludelowpop_value'] = false; //@review without adding this, I can not set filter_sort_order in $otherRequestParameters system tests
$aParameters['filter_column_recursive'] = false; //@review without adding this, I can not set filter_sort_order in $otherRequestParameters system tests
$aParameters['filter_pattern_recursive'] = false; //@review without adding this, I can not set filter_sort_order in $otherRequestParameters system tests
$aParameters['filter_truncate'] = false;
$aParameters['hideColumns'] = false;
$aParameters['showColumns'] = false;
$aParameters['filter_pattern_recursive'] = false;
$aParameters['pivotBy'] = false;
$aParameters['pivotByColumn'] = false;
$aParameters['pivotByColumnLimit'] = false;
$aParameters['disable_queued_filters'] = false;
$aParameters['disable_generic_filters'] = false;
$aParameters['expanded'] = false;
$aParameters['idDimenson'] = false;
$moduleName = Proxy::getInstance()->getModuleNameFromClassName($class);
$aParameters = array_merge(array('module' => 'API', 'method' => $moduleName . '.' . $methodName), $aParameters);
@ -235,4 +338,43 @@ class DocumentationGenerator
$sParameters = implode(", ", $asParameters);
return "($sParameters)";
}
private function getMethodString($moduleName, $class, $parametersToSet, $outputExampleUrls, $prefixUrls, $methodName)
{
$str = '';
$token_auth = "&token_auth=" . Piwik::getCurrentUserTokenAuth();
$params = $this->getParametersString($class, $methodName);
$str .= "\n <div class='apiMethod'>- <b>$moduleName.$methodName </b>" . $params . "";
$str .= '<small>';
if ($outputExampleUrls) {
// we prefix all URLs with $prefixUrls
// used when we include this output in the Piwik official documentation for example
$str .= "<span class=\"example\">";
$exampleUrl = $this->getExampleUrl($class, $methodName, $parametersToSet);
if ($exampleUrl !== false) {
$lastNUrls = '';
if (preg_match('/(&period)|(&date)/', $exampleUrl)) {
$exampleUrlRss = $prefixUrls . $this->getExampleUrl($class, $methodName, array('date' => 'last10', 'period' => 'day') + $parametersToSet);
$lastNUrls = ", RSS of the last <a target='_blank' href='$exampleUrlRss&format=rss$token_auth&translateColumnNames=1'>10 days</a>";
}
$exampleUrl = $prefixUrls . $exampleUrl;
$str .= " [ Example in
<a target='_blank' href='$exampleUrl&format=xml$token_auth'>XML</a>,
<a target='_blank' href='$exampleUrl&format=JSON$token_auth'>Json</a>,
<a target='_blank' href='$exampleUrl&format=Tsv$token_auth&translateColumnNames=1'>Tsv (Excel)</a>
$lastNUrls
]";
} else {
$str .= " [ No example available ]";
}
$str .= "</span>";
}
$str .= '</small>';
$str .= "</div>\n";
return $str;
}
}

42
www/analytics/core/API/Inconsistencies.php Normal file
View file

@ -0,0 +1,42 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\API;
/**
* Contains logic to replicate inconsistencies in Piwik's API. This class exists
* to provide a way to clean up existing Piwik code and behavior without breaking
* backwards compatibility immediately.
*
* Code that handles the case when the 'format_metrics' query parameter value is
* 'bc' should be removed as well. This code is in API\Request and DataTablePostProcessor.
*
* Should be removed before releasing Piwik 3.0.
*/
class Inconsistencies
{
/**
* In Piwik 2.X and below, the "raw" API would format percent values but no others.
* This method returns the list of percent metrics that were returned from the API
* formatted so we can maintain BC.
*
* Used by DataTablePostProcessor.
*/
public function getPercentMetricsToFormat()
{
return array(
'bounce_rate',
'conversion_rate',
'interaction_rate',
'exit_rate',
'bounce_rate_returning',
'nb_visits_percentage',
'/.*_evolution/',
'/goal_.*_conversion_rate/'
);
}
}

173
www/analytics/core/API/Proxy.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -24,7 +24,7 @@ use ReflectionMethod;
*
* It will also log the performance of API calls (time spent, parameter values, etc.) if logger available
*
* @method static \Piwik\API\Proxy getInstance()
* @method static Proxy getInstance()
*/
class Proxy extends Singleton
{
@ -37,10 +37,7 @@ class Proxy extends Singleton
// when a parameter doesn't have a default value we use this
private $noDefaultValue;
/**
* protected constructor
*/
protected function __construct()
public function __construct()
{
$this->noDefaultValue = new NoDefaultValue();
}
@ -78,12 +75,14 @@ class Proxy extends Singleton
$this->checkClassIsSingleton($className);
$rClass = new ReflectionClass($className);
foreach ($rClass->getMethods() as $method) {
$this->loadMethodMetadata($className, $method);
}
if (!$this->shouldHideAPIMethod($rClass->getDocComment())) {
foreach ($rClass->getMethods() as $method) {
$this->loadMethodMetadata($className, $method);
}
$this->setDocumentation($rClass, $className);
$this->alreadyRegistered[$className] = true;
$this->setDocumentation($rClass, $className);
$this->alreadyRegistered[$className] = true;
}
}
/**
@ -164,11 +163,11 @@ class Proxy extends Singleton
/**
* Triggered before an API request is dispatched.
*
*
* This event can be used to modify the arguments passed to one or more API methods.
*
*
* **Example**
*
*
* Piwik::addAction('API.Request.dispatch', function (&$parameters, $pluginName, $methodName) {
* if ($pluginName == 'Actions') {
* if ($methodName == 'getPageUrls') {
@ -178,7 +177,7 @@ class Proxy extends Singleton
* }
* }
* });
*
*
* @param array &$finalParameters List of parameters that will be passed to the API method.
* @param string $pluginName The name of the plugin the API method belongs to.
* @param string $methodName The name of the API method that will be called.
@ -187,20 +186,20 @@ class Proxy extends Singleton
/**
* Triggered before an API request is dispatched.
*
*
* This event exists for convenience and is triggered directly after the {@hook API.Request.dispatch}
* event is triggered. It can be used to modify the arguments passed to a **single** API method.
*
*
* _Note: This is can be accomplished with the {@hook API.Request.dispatch} event as well, however
* event handlers for that event will have to do more work._
*
*
* **Example**
*
*
* Piwik::addAction('API.Actions.getPageUrls', function (&$parameters) {
* // force use of a single website. for some reason.
* $parameters['idSite'] = 1;
* });
*
*
* @param array &$finalParameters List of parameters that will be passed to the API method.
*/
Piwik::postEvent(sprintf('API.%s.%s', $pluginName, $methodName), array(&$finalParameters));
@ -218,16 +217,16 @@ class Proxy extends Singleton
/**
* Triggered directly after an API request is dispatched.
*
*
* This event exists for convenience and is triggered immediately before the
* {@hook API.Request.dispatch.end} event. It can be used to modify the output of a **single**
* API method.
*
*
* _Note: This can be accomplished with the {@hook API.Request.dispatch.end} event as well,
* however event handlers for that event will have to do more work._
*
* **Example**
*
*
* // append (0 hits) to the end of row labels whose row has 0 hits
* Piwik::addAction('API.Actions.getPageUrls', function (&$returnValue, $info)) {
* $returnValue->filter('ColumnCallbackReplace', 'label', function ($label, $hits) {
@ -238,13 +237,13 @@ class Proxy extends Singleton
* }
* }, null, array('nb_hits'));
* }
*
*
* @param mixed &$returnedValue The API method's return value. Can be an object, such as a
* {@link Piwik\DataTable DataTable} instance.
* could be a {@link Piwik\DataTable DataTable}.
* @param array $extraInfo An array holding information regarding the API request. Will
* contain the following data:
*
*
* - **className**: The namespace-d class name of the API instance
* that's being called.
* - **module**: The name of the plugin the API request was
@ -257,20 +256,20 @@ class Proxy extends Singleton
/**
* Triggered directly after an API request is dispatched.
*
*
* This event can be used to modify the output of any API method.
*
*
* **Example**
*
*
* // append (0 hits) to the end of row labels whose row has 0 hits for any report that has the 'nb_hits' metric
* Piwik::addAction('API.Actions.getPageUrls', function (&$returnValue, $info)) {
* Piwik::addAction('API.Actions.getPageUrls.end', function (&$returnValue, $info)) {
* // don't process non-DataTable reports and reports that don't have the nb_hits column
* if (!($returnValue instanceof DataTableInterface)
* || in_array('nb_hits', $returnValue->getColumns())
* ) {
* return;
* }
*
*
* $returnValue->filter('ColumnCallbackReplace', 'label', function ($label, $hits) {
* if ($hits === 0) {
* return $label . " (0 hits)";
@ -279,12 +278,12 @@ class Proxy extends Singleton
* }
* }, null, array('nb_hits'));
* }
*
*
* @param mixed &$returnedValue The API method's return value. Can be an object, such as a
* {@link Piwik\DataTable DataTable} instance.
* @param array $extraInfo An array holding information regarding the API request. Will
* contain the following data:
*
*
* - **className**: The namespace-d class name of the API instance
* that's being called.
* - **module**: The name of the plugin the API request was
@ -323,6 +322,14 @@ class Proxy extends Singleton
return $this->metadataArray[$class][$name]['parameters'];
}
/**
* Check if given method name is deprecated or not.
*/
public function isDeprecatedMethod($class, $methodName)
{
return $this->metadataArray[$class][$methodName]['isDeprecated'];
}
/**
* Returns the 'moduleName' part of '\\Piwik\\Plugins\\moduleName\\API'
*
@ -378,7 +385,6 @@ class Proxy extends Singleton
$requestValue = Common::getRequestVar($name, null, null, $parametersRequest);
} else {
try {
if ($name == 'segment' && !empty($parametersRequest['segment'])) {
// segment parameter is an exception: we do not want to sanitize user input or it would break the segment encoding
$requestValue = ($parametersRequest['segment']);
@ -405,7 +411,7 @@ class Proxy extends Singleton
}
/**
* Includes the class API by looking up plugins/UserSettings/API.php
* Includes the class API by looking up plugins/xxx/API.php
*
* @param string $fileName api class name eg. "API"
* @throws Exception
@ -428,29 +434,27 @@ class Proxy extends Singleton
*/
private function loadMethodMetadata($class, $method)
{
if ($method->isPublic()
&& !$method->isConstructor()
&& $method->getName() != 'getInstance'
&& false === strstr($method->getDocComment(), '@deprecated')
&& (!$this->hideIgnoredFunctions || false === strstr($method->getDocComment(), '@ignore'))
) {
$name = $method->getName();
$parameters = $method->getParameters();
$aParameters = array();
foreach ($parameters as $parameter) {
$nameVariable = $parameter->getName();
$defaultValue = $this->noDefaultValue;
if ($parameter->isDefaultValueAvailable()) {
$defaultValue = $parameter->getDefaultValue();
}
$aParameters[$nameVariable] = $defaultValue;
}
$this->metadataArray[$class][$name]['parameters'] = $aParameters;
$this->metadataArray[$class][$name]['numberOfRequiredParameters'] = $method->getNumberOfRequiredParameters();
if (!$this->checkIfMethodIsAvailable($method)) {
return;
}
$name = $method->getName();
$parameters = $method->getParameters();
$docComment = $method->getDocComment();
$aParameters = array();
foreach ($parameters as $parameter) {
$nameVariable = $parameter->getName();
$defaultValue = $this->noDefaultValue;
if ($parameter->isDefaultValueAvailable()) {
$defaultValue = $parameter->getDefaultValue();
}
$aParameters[$nameVariable] = $defaultValue;
}
$this->metadataArray[$class][$name]['parameters'] = $aParameters;
$this->metadataArray[$class][$name]['numberOfRequiredParameters'] = $method->getNumberOfRequiredParameters();
$this->metadataArray[$class][$name]['isDeprecated'] = false !== strstr($docComment, '@deprecated');
}
/**
@ -468,15 +472,56 @@ class Proxy extends Singleton
}
/**
* Returns the number of required parameters (parameters without default values).
*
* @param string $class The class name
* @param string $name The method name
* @return int The number of required parameters
* @param $docComment
* @return bool
*/
private function getNumberOfRequiredParameters($class, $name)
public function shouldHideAPIMethod($docComment)
{
return $this->metadataArray[$class][$name]['numberOfRequiredParameters'];
$hideLine = strstr($docComment, '@hide');
if ($hideLine === false) {
return false;
}
$hideLine = trim($hideLine);
$hideLine .= ' ';
$token = trim(strtok($hideLine, " "), "\n");
$hide = false;
if (!empty($token)) {
/**
* This event exists for checking whether a Plugin API class or a Plugin API method tagged
* with a `@hideXYZ` should be hidden in the API listing.
*
* @param bool &$hide whether to hide APIs tagged with $token should be displayed.
*/
Piwik::postEvent(sprintf('API.DocumentationGenerator.%s', $token), array(&$hide));
}
return $hide;
}
/**
* @param ReflectionMethod $method
* @return bool
*/
protected function checkIfMethodIsAvailable(ReflectionMethod $method)
{
if (!$method->isPublic() || $method->isConstructor() || $method->getName() === 'getInstance') {
return false;
}
if ($this->hideIgnoredFunctions && false !== strstr($method->getDocComment(), '@ignore')) {
return false;
}
if ($this->shouldHideAPIMethod($method->getDocComment())) {
return false;
}
return true;
}
/**
@ -500,7 +545,7 @@ class Proxy extends Singleton
private function checkClassIsSingleton($className)
{
if (!method_exists($className, "getInstance")) {
throw new Exception("$className that provide an API must be Singleton and have a 'static public function getInstance()' method.");
throw new Exception("$className that provide an API must be Singleton and have a 'public static function getInstance()' method.");
}
}
}

274
www/analytics/core/API/Request.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -17,49 +17,49 @@ use Piwik\PluginDeactivatedException;
use Piwik\SettingsServer;
use Piwik\Url;
use Piwik\UrlHelper;
use Piwik\Log;
use Piwik\Plugin\Manager as PluginManager;
/**
* Dispatches API requests to the appropriate API method.
*
*
* The Request class is used throughout Piwik to call API methods. The difference
* between using Request and calling API methods directly is that Request
* will do more after calling the API including: applying generic filters, applying queued filters,
* and handling the **flat** and **label** query parameters.
*
*
* Additionally, the Request class will **forward current query parameters** to the request
* which is more convenient than calling {@link Piwik\Common::getRequestVar()} many times over.
*
*
* In most cases, using a Request object to query the API is the correct approach.
*
* ### Post-processing
*
*
* The return value of API methods undergo some extra processing before being returned by Request.
* To learn more about what happens to API results, read [this](/guides/piwiks-web-api#extra-report-processing).
*
* ### Output Formats
*
*
* The value returned by Request will be serialized to a certain format before being returned.
* To see the list of supported output formats, read [this](/guides/piwiks-web-api#output-formats).
*
*
* ### Examples
*
*
* **Basic Usage**
*
* $request = new Request('method=UserSettings.getWideScreen&idSite=1&date=yesterday&period=week'
*
* $request = new Request('method=UserLanguage.getLanguage&idSite=1&date=yesterday&period=week'
* . '&format=xml&filter_limit=5&filter_offset=0')
* $result = $request->process();
* echo $result;
*
*
* **Getting a unrendered DataTable**
*
*
* // use the convenience method 'processRequest'
* $dataTable = Request::processRequest('UserSettings.getWideScreen', array(
* $dataTable = Request::processRequest('UserLanguage.getLanguage', array(
* 'idSite' => 1,
* 'date' => 'yesterday',
* 'period' => 'week',
* 'filter_limit' => 5,
* 'filter_offset' => 0
*
*
* 'format' => 'original', // this is the important bit
* ));
* echo "This DataTable has " . $dataTable->getRowsCount() . " rows.";
@ -69,41 +69,46 @@ use Piwik\UrlHelper;
*/
class Request
{
protected $request = null;
private $request = null;
/**
* Converts the supplied request string into an array of query paramater name/value
* mappings. The current query parameters (everything in `$_GET` and `$_POST`) are
* forwarded to request array before it is returned.
*
* @param string|array $request The base request string or array, eg,
* `'module=UserSettings&action=getWidescreen'`.
* @param string|array|null $request The base request string or array, eg,
* `'module=UserLanguage&action=getLanguage'`.
* @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
* from this. Defaults to `$_GET + $_POST`.
* @return array
*/
static public function getRequestArrayFromString($request)
public static function getRequestArrayFromString($request, $defaultRequest = null)
{
$defaultRequest = $_GET + $_POST;
if ($defaultRequest === null) {
$defaultRequest = self::getDefaultRequest();
$requestRaw = self::getRequestParametersGET();
if (!empty($requestRaw['segment'])) {
$defaultRequest['segment'] = $requestRaw['segment'];
$requestRaw = self::getRequestParametersGET();
if (!empty($requestRaw['segment'])) {
$defaultRequest['segment'] = $requestRaw['segment'];
}
if (!isset($defaultRequest['format_metrics'])) {
$defaultRequest['format_metrics'] = 'bc';
}
}
$requestArray = $defaultRequest;
if (!is_null($request)) {
if (is_array($request)) {
$url = array();
foreach ($request as $key => $value) {
$url[] = $key . "=" . $value;
}
$request = implode("&", $url);
$requestParsed = $request;
} else {
$request = trim($request);
$request = str_replace(array("\n", "\t"), '', $request);
$requestParsed = UrlHelper::getArrayFromQueryString($request);
}
$request = trim($request);
$request = str_replace(array("\n", "\t"), '', $request);
$requestParsed = UrlHelper::getArrayFromQueryString($request);
$requestArray = $requestParsed + $defaultRequest;
}
@ -119,14 +124,17 @@ class Request
* Constructor.
*
* @param string|array $request Query string that defines the API call (must at least contain a **method** parameter),
* eg, `'method=UserSettings.getWideScreen&idSite=1&date=yesterday&period=week&format=xml'`
* eg, `'method=UserLanguage.getLanguage&idSite=1&date=yesterday&period=week&format=xml'`
* If a request is not provided, then we use the values in the `$_GET` and `$_POST`
* superglobals.
* @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
* from this. Defaults to `$_GET + $_POST`.
*/
public function __construct($request = null)
public function __construct($request = null, $defaultRequest = null)
{
$this->request = self::getRequestArrayFromString($request);
$this->request = self::getRequestArrayFromString($request, $defaultRequest);
$this->sanitizeRequest();
$this->renameModuleAndActionInRequest();
}
/**
@ -134,19 +142,23 @@ class Request
* we rewrite to correct renamed plugin: Referrers
*
* @param $module
* @return string
* @param $action
* @return array( $module, $action )
* @ignore
*/
public static function renameModule($module)
public static function getRenamedModuleAndAction($module, $action)
{
$moduleToRedirect = array(
'Referers' => 'Referrers',
'PDFReports' => 'ScheduledReports',
);
if (isset($moduleToRedirect[$module])) {
return $moduleToRedirect[$module];
}
return $module;
/**
* This event is posted in the Request dispatcher and can be used
* to overwrite the Module and Action to dispatch.
* This is useful when some Controller methods or API methods have been renamed or moved to another plugin.
*
* @param $module string
* @param $action string
*/
Piwik::postEvent('Request.getRenamedModuleAndAction', array(&$module, &$action));
return array($module, $action);
}
/**
@ -168,9 +180,9 @@ class Request
/**
* Dispatches the API request to the appropriate API method and returns the result
* after post-processing.
*
*
* Post-processing includes:
*
*
* - flattening if **flat** is 0
* - running generic filters unless **disable_generic_filters** is set to 1
* - URL decoding label column values
@ -178,10 +190,10 @@ class Request
* - removing columns based on the values of the **hideColumns** and **showColumns** query parameters
* - filtering rows if the **label** query parameter is set
* - converting the result to the appropriate format (ie, XML, JSON, etc.)
*
*
* If `'original'` is supplied for the output format, the result is returned as a PHP
* object.
*
*
* @throws PluginDeactivatedException if the module plugin is not activated.
* @throws Exception if the requested API method cannot be called, if required parameters for the
* API method are missing or if the API method throws an exception and the **format**
@ -196,42 +208,90 @@ class Request
// create the response
$response = new ResponseBuilder($outputFormat, $this->request);
$corsHandler = new CORSHandler();
$corsHandler->handle();
$tokenAuth = Common::getRequestVar('token_auth', '', 'string', $this->request);
$shouldReloadAuth = false;
try {
// read parameters
$moduleMethod = Common::getRequestVar('method', null, 'string', $this->request);
list($module, $method) = $this->extractModuleAndMethod($moduleMethod);
list($module, $method) = self::getRenamedModuleAndAction($module, $method);
PluginManager::getInstance()->checkIsPluginActivated($module);
$module = $this->renameModule($module);
$apiClassName = self::getClassNameAPI($module);
if (!\Piwik\Plugin\Manager::getInstance()->isPluginActivated($module)) {
throw new PluginDeactivatedException($module);
if ($shouldReloadAuth = self::shouldReloadAuthUsingTokenAuth($this->request)) {
$access = Access::getInstance();
$tokenAuthToRestore = $access->getTokenAuth();
$hadSuperUserAccess = $access->hasSuperUserAccess();
self::forceReloadAuthUsingTokenAuth($tokenAuth);
}
$apiClassName = $this->getClassNameAPI($module);
self::reloadAuthUsingTokenAuth($this->request);
// call the method
$returnedValue = Proxy::getInstance()->call($apiClassName, $method, $this->request);
$toReturn = $response->getResponse($returnedValue, $module, $method);
} catch (Exception $e) {
Log::debug($e);
$toReturn = $response->getResponseException($e);
}
if ($shouldReloadAuth) {
$this->restoreAuthUsingTokenAuth($tokenAuthToRestore, $hadSuperUserAccess);
}
return $toReturn;
}
private function restoreAuthUsingTokenAuth($tokenToRestore, $hadSuperUserAccess)
{
// if we would not make sure to unset super user access, the tokenAuth would be not authenticated and any
// token would just keep super user access (eg if the token that was reloaded before had super user access)
Access::getInstance()->setSuperUserAccess(false);
// we need to restore by reloading the tokenAuth as some permissions could have been removed in the API
// request etc. Otherwise we could just store a clone of Access::getInstance() and restore here
self::forceReloadAuthUsingTokenAuth($tokenToRestore);
if ($hadSuperUserAccess && !Access::getInstance()->hasSuperUserAccess()) {
// we are in context of `doAsSuperUser()` and need to restore this behaviour
Access::getInstance()->setSuperUserAccess(true);
}
}
/**
* Returns the name of a plugin's API class by plugin name.
*
*
* @param string $plugin The plugin name, eg, `'Referrers'`.
* @return string The fully qualified API class name, eg, `'\Piwik\Plugins\Referrers\API'`.
*/
static public function getClassNameAPI($plugin)
public static function getClassNameAPI($plugin)
{
return sprintf('\Piwik\Plugins\%s\API', $plugin);
}
/**
* Detect if request is an API request. Meaning the module is 'API' and an API method having a valid format was
* specified.
*
* @param array $request eg array('module' => 'API', 'method' => 'Test.getMethod')
* @return bool
* @throws Exception
*/
public static function isApiRequest($request)
{
$module = Common::getRequestVar('module', '', 'string', $request);
$method = Common::getRequestVar('method', '', 'string', $request);
return $module === 'API' && !empty($method) && (count(explode('.', $method)) === 2);
}
/**
* If the token_auth is found in the $request parameter,
* the current session will be authenticated using this token_auth.
@ -241,28 +301,60 @@ class Request
* @return void
* @ignore
*/
static public function reloadAuthUsingTokenAuth($request = null)
public static function reloadAuthUsingTokenAuth($request = null)
{
// if a token_auth is specified in the API request, we load the right permissions
$token_auth = Common::getRequestVar('token_auth', '', 'string', $request);
if ($token_auth) {
/**
* Triggered when authenticating an API request, but only if the **token_auth**
* query parameter is found in the request.
*
* Plugins that provide authentication capabilities should subscribe to this event
* and make sure the global authentication object (the object returned by `Registry::get('auth')`)
* is setup to use `$token_auth` when its `authenticate()` method is executed.
*
* @param string $token_auth The value of the **token_auth** query parameter.
*/
Piwik::postEvent('API.Request.authenticate', array($token_auth));
Access::getInstance()->reloadAccess();
SettingsServer::raiseMemoryLimitIfNecessary();
if (self::shouldReloadAuthUsingTokenAuth($request)) {
self::forceReloadAuthUsingTokenAuth($token_auth);
}
}
/**
* The current session will be authenticated using this token_auth.
* It will overwrite the previous Auth object.
*
* @param string $tokenAuth
* @return void
*/
private static function forceReloadAuthUsingTokenAuth($tokenAuth)
{
/**
* Triggered when authenticating an API request, but only if the **token_auth**
* query parameter is found in the request.
*
* Plugins that provide authentication capabilities should subscribe to this event
* and make sure the global authentication object (the object returned by `StaticContainer::get('Piwik\Auth')`)
* is setup to use `$token_auth` when its `authenticate()` method is executed.
*
* @param string $token_auth The value of the **token_auth** query parameter.
*/
Piwik::postEvent('API.Request.authenticate', array($tokenAuth));
Access::getInstance()->reloadAccess();
SettingsServer::raiseMemoryLimitIfNecessary();
}
private static function shouldReloadAuthUsingTokenAuth($request)
{
if (is_null($request)) {
$request = self::getDefaultRequest();
}
if (!isset($request['token_auth'])) {
// no token is given so we just keep the current loaded user
return false;
}
// a token is specified, we need to reload auth in case it is different than the current one, even if it is empty
$tokenAuth = Common::getRequestVar('token_auth', '', 'string', $request);
// not using !== is on purpose as getTokenAuth() might return null whereas $tokenAuth is '' . In this case
// we do not need to reload.
return $tokenAuth != Access::getInstance()->getTokenAuth();
}
/**
* Returns array($class, $method) from the given string $class.$method
*
@ -286,18 +378,23 @@ class Request
* @param string $method The API method to call, ie, `'Actions.getPageTitles'`.
* @param array $paramOverride The parameter name-value pairs to use instead of what's
* in `$_GET` & `$_POST`.
* @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
* from this. Defaults to `$_GET + $_POST`.
*
* To avoid using any parameters from $_GET or $_POST, set this to an empty `array()`.
* @return mixed The result of the API request. See {@link process()}.
*/
public static function processRequest($method, $paramOverride = array())
public static function processRequest($method, $paramOverride = array(), $defaultRequest = null)
{
$params = array();
$params['format'] = 'original';
$params['serialize'] = '0';
$params['module'] = 'API';
$params['method'] = $method;
$params = $paramOverride + $params;
// process request
$request = new Request($params);
$request = new Request($params, $defaultRequest);
return $request->process();
}
@ -305,7 +402,7 @@ class Request
* Returns the original request parameters in the current query string as an array mapping
* query parameter names with values. The result of this function will not be affected
* by any modifications to `$_GET` and will not include parameters in `$_POST`.
*
*
* @return array
*/
public static function getRequestParametersGET()
@ -343,7 +440,7 @@ class Request
// unless the filter param was in $queryParams
$genericFiltersInfo = DataTableGenericFilter::getGenericFiltersInformation();
foreach ($genericFiltersInfo as $filter) {
foreach ($filter as $queryParamName => $queryParamInfo) {
foreach ($filter[1] as $queryParamName => $queryParamInfo) {
if (!isset($params[$queryParamName])) {
$params[$queryParamName] = null;
}
@ -379,10 +476,10 @@ class Request
/**
* Returns the segment query parameter from the original request, without modifications.
*
*
* @return array|bool
*/
static public function getRawSegmentFromRequest()
public static function getRawSegmentFromRequest()
{
// we need the URL encoded segment parameter, we fetch it from _SERVER['QUERY_STRING'] instead of default URL decoded _GET
$segmentRaw = false;
@ -395,4 +492,23 @@ class Request
}
return $segmentRaw;
}
private function renameModuleAndActionInRequest()
{
if (empty($this->request['apiModule'])) {
return;
}
if (empty($this->request['apiAction'])) {
$this->request['apiAction'] = null;
}
list($this->request['apiModule'], $this->request['apiAction']) = $this->getRenamedModuleAndAction($this->request['apiModule'], $this->request['apiAction']);
}
/**
* @return array
*/
private static function getDefaultRequest()
{
return $_GET + $_POST;
}
}

506
www/analytics/core/API/ResponseBuilder.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,21 +9,22 @@
namespace Piwik\API;
use Exception;
use Piwik\API\DataTableManipulator\Flattener;
use Piwik\API\DataTableManipulator\LabelFilter;
use Piwik\API\DataTableManipulator\ReportTotalsCalculator;
use Piwik\Common;
use Piwik\DataTable\Renderer\Json;
use Piwik\DataTable\Renderer;
use Piwik\DataTable\Simple;
use Piwik\DataTable;
use Piwik\DataTable\Renderer;
use Piwik\DataTable\DataTableInterface;
use Piwik\DataTable\Filter\ColumnDelete;
use Piwik\DataTable\Filter\Pattern;
/**
*/
class ResponseBuilder
{
private $request = null;
private $outputFormat = null;
private $apiRenderer = null;
private $request = null;
private $sendHeader = true;
private $postProcessDataTable = true;
private $apiModule = false;
private $apiMethod = false;
@ -34,8 +35,19 @@ class ResponseBuilder
*/
public function __construct($outputFormat, $request = array())
{
$this->request = $request;
$this->outputFormat = $outputFormat;
$this->request = $request;
$this->apiRenderer = ApiRenderer::factory($outputFormat, $request);
}
public function disableSendHeader()
{
$this->sendHeader = false;
}
public function disableDataTablePostProcessor()
{
$this->postProcessDataTable = false;
}
/**
@ -70,61 +82,21 @@ class ResponseBuilder
$this->apiModule = $apiModule;
$this->apiMethod = $apiMethod;
if($this->outputFormat == 'original') {
@header('Content-Type: text/plain; charset=utf-8');
}
return $this->renderValue($value);
}
$this->sendHeaderIfEnabled();
/**
* Returns an error $message in the requested $format
*
* @param Exception $e
* @throws Exception
* @return string
*/
public function getResponseException(Exception $e)
{
$format = strtolower($this->outputFormat);
if ($format == 'original') {
throw $e;
}
try {
$renderer = Renderer::factory($format);
} catch (Exception $exceptionRenderer) {
return "Error: " . $e->getMessage() . " and: " . $exceptionRenderer->getMessage();
}
$e = $this->decorateExceptionWithDebugTrace($e);
$renderer->setException($e);
if ($format == 'php') {
$renderer->setSerialize($this->caseRendererPHPSerialize());
}
return $renderer->renderException();
}
/**
* @param $value
* @return string
*/
protected function renderValue($value)
{
// when null or void is returned from the api call, we handle it as a successful operation
if (!isset($value)) {
return $this->handleSuccess();
if (ob_get_contents()) {
return null;
}
return $this->apiRenderer->renderSuccess('ok');
}
// If the returned value is an object DataTable we
// apply the set of generic filters if asked in the URL
// and we render the DataTable according to the format specified in the URL
if ($value instanceof DataTable
|| $value instanceof DataTable\Map
) {
if ($value instanceof DataTableInterface) {
return $this->handleDataTable($value);
}
@ -137,26 +109,39 @@ class ResponseBuilder
return $this->handleArray($value);
}
// original data structure requested, we return without process
if ($this->outputFormat == 'original') {
return $value;
if (is_object($value)) {
return $this->apiRenderer->renderObject($value);
}
if (is_object($value)
|| is_resource($value)
) {
return $this->getResponseException(new Exception('The API cannot handle this data structure.'));
if (is_resource($value)) {
return $this->apiRenderer->renderResource($value);
}
// bool // integer // float // serialized object
return $this->handleScalar($value);
return $this->apiRenderer->renderScalar($value);
}
/**
* Returns an error $message in the requested $format
*
* @param Exception $e
* @throws Exception
* @return string
*/
public function getResponseException(Exception $e)
{
$e = $this->decorateExceptionWithDebugTrace($e);
$message = $this->formatExceptionMessage($e);
$this->sendHeaderIfEnabled();
return $this->apiRenderer->renderException($message, $e);
}
/**
* @param Exception $e
* @return Exception
*/
protected function decorateExceptionWithDebugTrace(Exception $e)
private function decorateExceptionWithDebugTrace(Exception $e)
{
// If we are in tests, show full backtrace
if (defined('PIWIK_PATH_TEST_TO_ROOT')) {
@ -165,314 +150,109 @@ class ResponseBuilder
} else {
$message = $e->getMessage() . "\n \n --> To temporarily debug this error further, set const PIWIK_PRINT_ERROR_BACKTRACE=true; in index.php";
}
return new Exception($message);
}
return $e;
}
/**
* Returns true if the user requested to serialize the output data (&serialize=1 in the request)
*
* @param mixed $defaultSerializeValue Default value in case the user hasn't specified a value
* @return bool
*/
protected function caseRendererPHPSerialize($defaultSerializeValue = 1)
private function formatExceptionMessage(Exception $exception)
{
$serialize = Common::getRequestVar('serialize', $defaultSerializeValue, 'int', $this->request);
if ($serialize) {
$message = $exception->getMessage();
if (\Piwik_ShouldPrintBackTraceWithMessage()) {
$message .= "\n" . $exception->getTraceAsString();
}
return Renderer::formatValueXml($message);
}
private function handleDataTable(DataTableInterface $datatable)
{
if ($this->postProcessDataTable) {
$postProcessor = new DataTablePostProcessor($this->apiModule, $this->apiMethod, $this->request);
$datatable = $postProcessor->process($datatable);
}
return $this->apiRenderer->renderDataTable($datatable);
}
private function handleArray($array)
{
$firstArray = null;
$firstKey = null;
if (!empty($array)) {
$firstArray = reset($array);
$firstKey = key($array);
}
$isAssoc = !empty($firstArray) && is_numeric($firstKey) && is_array($firstArray) && count(array_filter(array_keys($firstArray), 'is_string'));
if (is_numeric($firstKey)) {
$columns = Common::getRequestVar('filter_column', false, 'array', $this->request);
$pattern = Common::getRequestVar('filter_pattern', '', 'string', $this->request);
if ($columns != array(false) && $pattern !== '') {
$pattern = new Pattern(new DataTable(), $columns, $pattern);
$array = $pattern->filterArray($array);
}
$limit = Common::getRequestVar('filter_limit', -1, 'integer', $this->request);
$offset = Common::getRequestVar('filter_offset', '0', 'integer', $this->request);
if ($this->shouldApplyLimitOnArray($limit, $offset)) {
$array = array_slice($array, $offset, $limit, $preserveKeys = false);
}
}
if ($isAssoc) {
$hideColumns = Common::getRequestVar('hideColumns', '', 'string', $this->request);
$showColumns = Common::getRequestVar('showColumns', '', 'string', $this->request);
if ($hideColumns !== '' || $showColumns !== '') {
$columnDelete = new ColumnDelete(new DataTable(), $hideColumns, $showColumns);
$array = $columnDelete->filter($array);
}
}
return $this->apiRenderer->renderArray($array);
}
private function shouldApplyLimitOnArray($limit, $offset)
{
if ($limit === -1) {
// all fields are requested
return false;
}
if ($offset > 0) {
// an offset is specified, we have to apply the limit
return true;
}
return false;
// "api_datatable_default_limit" is set by API\Controller if no filter_limit is specified by the user.
// it holds the number of the configured default limit.
$limitSetBySystem = Common::getRequestVar('api_datatable_default_limit', -2, 'integer', $this->request);
// we ignore the limit if the datatable_default_limit was set by the system as this default filter_limit is
// only meant for dataTables but not for arrays. This way we stay BC as filter_limit was not applied pre
// Piwik 2.6 and some fixes were made in Piwik 2.13.
$wasFilterLimitSetBySystem = $limitSetBySystem !== -2;
// we check for "$limitSetBySystem === $limit" as an API method could request another API method with
// another limit. In this case we need to apply it again.
$isLimitStillDefaultLimit = $limitSetBySystem === $limit;
if ($wasFilterLimitSetBySystem && $isLimitStillDefaultLimit) {
return false;
}
return true;
}
/**
* Apply the specified renderer to the DataTable
*
* @param DataTable|array $dataTable
* @return string
*/
protected function getRenderedDataTable($dataTable)
private function sendHeaderIfEnabled()
{
$format = strtolower($this->outputFormat);
// if asked for original dataStructure
if ($format == 'original') {
// by default "original" data is not serialized
if ($this->caseRendererPHPSerialize($defaultSerialize = 0)) {
$dataTable = serialize($dataTable);
}
return $dataTable;
if ($this->sendHeader) {
$this->apiRenderer->sendHeader();
}
$method = Common::getRequestVar('method', '', 'string', $this->request);
$renderer = Renderer::factory($format);
$renderer->setTable($dataTable);
$renderer->setRenderSubTables(Common::getRequestVar('expanded', false, 'int', $this->request));
$renderer->setHideIdSubDatableFromResponse(Common::getRequestVar('hideIdSubDatable', false, 'int', $this->request));
if ($format == 'php') {
$renderer->setSerialize($this->caseRendererPHPSerialize());
$renderer->setPrettyDisplay(Common::getRequestVar('prettyDisplay', false, 'int', $this->request));
} else if ($format == 'html') {
$renderer->setTableId($this->request['method']);
} else if ($format == 'csv' || $format == 'tsv') {
$renderer->setConvertToUnicode(Common::getRequestVar('convertToUnicode', true, 'int', $this->request));
}
// prepare translation of column names
if ($format == 'html' || $format == 'csv' || $format == 'tsv' || $format = 'rss') {
$renderer->setApiMethod($method);
$renderer->setIdSite(Common::getRequestVar('idSite', false, 'int', $this->request));
$renderer->setTranslateColumnNames(Common::getRequestVar('translateColumnNames', false, 'int', $this->request));
}
return $renderer->render();
}
/**
* Returns a success $message in the requested $format
*
* @param string $message
* @return string
*/
protected function handleSuccess($message = 'ok')
{
// return a success message only if no content has already been buffered, useful when APIs return raw text or html content to the browser
if (!ob_get_contents()) {
switch ($this->outputFormat) {
case 'xml':
@header("Content-Type: text/xml;charset=utf-8");
$return =
"<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" .
"<result>\n" .
"\t<success message=\"" . $message . "\" />\n" .
"</result>";
break;
case 'json':
@header("Content-Type: application/json");
$return = '{"result":"success", "message":"' . $message . '"}';
break;
case 'php':
$return = array('result' => 'success', 'message' => $message);
if ($this->caseRendererPHPSerialize()) {
$return = serialize($return);
}
break;
case 'csv':
@header("Content-Type: application/vnd.ms-excel");
@header("Content-Disposition: attachment; filename=piwik-report-export.csv");
$return = "message\n" . $message;
break;
default:
$return = 'Success:' . $message;
break;
}
return $return;
}
}
/**
* Converts the given scalar to an data table
*
* @param mixed $scalar
* @return string
*/
protected function handleScalar($scalar)
{
$dataTable = new Simple();
$dataTable->addRowsFromArray(array($scalar));
return $this->getRenderedDataTable($dataTable);
}
/**
* Handles the given data table
*
* @param DataTable $datatable
* @return string
*/
protected function handleDataTable($datatable)
{
// if requested, flatten nested tables
if (Common::getRequestVar('flat', '0', 'string', $this->request) == '1') {
$flattener = new Flattener($this->apiModule, $this->apiMethod, $this->request);
if (Common::getRequestVar('include_aggregate_rows', '0', 'string', $this->request) == '1') {
$flattener->includeAggregateRows();
}
$datatable = $flattener->flatten($datatable);
}
if (1 == Common::getRequestVar('totals', '1', 'integer', $this->request)) {
$genericFilter = new ReportTotalsCalculator($this->apiModule, $this->apiMethod, $this->request);
$datatable = $genericFilter->calculate($datatable);
}
// if the flag disable_generic_filters is defined we skip the generic filters
if (0 == Common::getRequestVar('disable_generic_filters', '0', 'string', $this->request)) {
$genericFilter = new DataTableGenericFilter($this->request);
$genericFilter->filter($datatable);
}
// we automatically safe decode all datatable labels (against xss)
$datatable->queueFilter('SafeDecodeLabel');
// if the flag disable_queued_filters is defined we skip the filters that were queued
if (Common::getRequestVar('disable_queued_filters', 0, 'int', $this->request) == 0) {
$datatable->applyQueuedFilters();
}
// use the ColumnDelete filter if hideColumns/showColumns is provided (must be done
// after queued filters are run so processed metrics can be removed, too)
$hideColumns = Common::getRequestVar('hideColumns', '', 'string', $this->request);
$showColumns = Common::getRequestVar('showColumns', '', 'string', $this->request);
if ($hideColumns !== '' || $showColumns !== '') {
$datatable->filter('ColumnDelete', array($hideColumns, $showColumns));
}
// apply label filter: only return rows matching the label parameter (more than one if more than one label)
$label = $this->getLabelFromRequest($this->request);
if (!empty($label)) {
$addLabelIndex = Common::getRequestVar('labelFilterAddLabelIndex', 0, 'int', $this->request) == 1;
$filter = new LabelFilter($this->apiModule, $this->apiMethod, $this->request);
$datatable = $filter->filter($label, $datatable, $addLabelIndex);
}
return $this->getRenderedDataTable($datatable);
}
/**
* Converts the given simple array to a data table
*
* @param array $array
* @return string
*/
protected function handleArray($array)
{
if ($this->outputFormat == 'original') {
// we handle the serialization. Because some php array have a very special structure that
// couldn't be converted with the automatic DataTable->addRowsFromSimpleArray
// the user may want to request the original PHP data structure serialized by the API
// in case he has to setup serialize=1 in the URL
if ($this->caseRendererPHPSerialize($defaultSerialize = 0)) {
return serialize($array);
}
return $array;
}
$multiDimensional = $this->handleMultiDimensionalArray($array);
if ($multiDimensional !== false) {
return $multiDimensional;
}
return $this->getRenderedDataTable($array);
}
/**
* Is this a multi dimensional array?
* Multi dim arrays are not supported by the Datatable renderer.
* We manually render these.
*
* array(
* array(
* 1,
* 2 => array( 1,
* 2
* )
* ),
* array( 2,
* 3
* )
* );
*
* @param array $array
* @return string|bool false if it isn't a multidim array
*/
protected function handleMultiDimensionalArray($array)
{
$first = reset($array);
foreach ($array as $first) {
if (is_array($first)) {
foreach ($first as $key => $value) {
// Yes, this is a multi dim array
if (is_array($value)) {
switch ($this->outputFormat) {
case 'json':
@header("Content-Type: application/json");
return self::convertMultiDimensionalArrayToJson($array);
break;
case 'php':
if ($this->caseRendererPHPSerialize($defaultSerialize = 0)) {
return serialize($array);
}
return $array;
case 'xml':
@header("Content-Type: text/xml;charset=utf-8");
return $this->getRenderedDataTable($array);
default:
break;
}
}
}
}
}
return false;
}
/**
* Render a multidimensional array to Json
* Handle DataTable|Set elements in the first dimension only, following case does not work:
* array(
* array(
* DataTable,
* 2 => array(
* 1,
* 2
* ),
* ),
* );
*
* @param array $array can contain scalar, arrays, DataTable and Set
* @return string
*/
public static function convertMultiDimensionalArrayToJson($array)
{
$jsonRenderer = new Json();
$jsonRenderer->setTable($array);
$renderedReport = $jsonRenderer->render();
return $renderedReport;
}
/**
* Returns the value for the label query parameter which can be either a string
* (ie, label=...) or array (ie, label[]=...).
*
* @param array $request
* @return array
*/
static public function getLabelFromRequest($request)
{
$label = Common::getRequestVar('label', array(), 'array', $request);
if (empty($label)) {
$label = Common::getRequestVar('label', '', 'string', $request);
if (!empty($label)) {
$label = array($label);
}
}
$label = self::unsanitizeLabelParameter($label);
return $label;
}
static public function unsanitizeLabelParameter($label)
{
// this is needed because Proxy uses Common::getRequestVar which in turn
// uses Common::sanitizeInputValue. This causes the > that separates recursive labels
// to become &gt; and we need to undo that here.
$label = Common::unsanitizeInputValues($label);
return $label;
}
}

212
www/analytics/core/Access.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -8,55 +8,32 @@
*/
namespace Piwik;
use Piwik\Db;
use Piwik\Plugins\UsersManager\API as APIUsersManager;
use Exception;
use Piwik\Container\StaticContainer;
/**
* Singleton that manages user access to Piwik resources.
*
*
* To check whether a user has access to a resource, use one of the {@link Piwik Piwik::checkUser...}
* methods.
*
* In Piwik there are four different access levels:
*
*
* - **no access**: Users with this access level cannot view the resource.
* - **view access**: Users with this access level can view the resource, but cannot modify it.
* - **admin access**: Users with this access level can view and modify the resource.
* - **Super User access**: Only the Super User has this access level. It means the user can do
* whatever he/she wants.
*
*
* Super user access is required to set some configuration options.
* All other options are specific to the user or to a website.
*
* Access is granted per website. Uses with access for a website can view all
* data associated with that website.
*
*
*/
class Access
{
private static $instance = null;
/**
* Gets the singleton instance. Creates it if necessary.
*/
public static function getInstance()
{
if (self::$instance == null) {
self::$instance = new self;
Piwik::postEvent('Access.createAccessSingleton', array(&self::$instance));
}
return self::$instance;
}
/**
* Sets the singleton instance. For testing purposes.
*/
public static function setSingletonInstance($instance)
{
self::$instance = $instance;
}
/**
* Array of idsites available to the current user, indexed by permission level
* @see getSitesIdWith*()
@ -101,6 +78,16 @@ class Access
*/
private $auth = null;
/**
* Gets the singleton instance. Creates it if necessary.
*
* @return self
*/
public static function getInstance()
{
return StaticContainer::get('Piwik\Access');
}
/**
* Returns the list of the existing Access level.
* Useful when a given API method requests a given acccess Level.
@ -117,6 +104,11 @@ class Access
* Constructor
*/
public function __construct()
{
$this->resetSites();
}
private function resetSites()
{
$this->idsitesByAccess = array(
'view' => array(),
@ -138,15 +130,22 @@ class Access
*/
public function reloadAccess(Auth $auth = null)
{
if (!is_null($auth)) {
$this->resetSites();
if (isset($auth)) {
$this->auth = $auth;
}
// if the Auth wasn't set, we may be in the special case of setSuperUser(), otherwise we fail
if (is_null($this->auth)) {
if ($this->hasSuperUserAccess()) {
return $this->reloadAccessSuperUser();
}
if ($this->hasSuperUserAccess()) {
$this->makeSureLoginNameIsSet();
return true;
}
$this->token_auth = null;
$this->login = null;
// if the Auth wasn't set, we may be in the special case of setSuperUser(), otherwise we fail TODO: docs + review
if ($this->auth === null) {
return false;
}
@ -156,28 +155,23 @@ class Access
if (!$result->wasAuthenticationSuccessful()) {
return false;
}
$this->login = $result->getIdentity();
$this->token_auth = $result->getTokenAuth();
// case the superUser is logged in
if ($result->hasSuperUserAccess()) {
return $this->reloadAccessSuperUser();
$this->setSuperUserAccess(true);
}
// in case multiple calls to API using different tokens, we ensure we reset it as not SU
$this->setSuperUserAccess(false);
// we join with site in case there are rows in access for an idsite that doesn't exist anymore
// (backward compatibility ; before we deleted the site without deleting rows in _access table)
$accessRaw = $this->getRawSitesWithSomeViewAccess($this->login);
foreach ($accessRaw as $access) {
$this->idsitesByAccess[$access['access']][] = $access['idsite'];
}
return true;
}
public function getRawSitesWithSomeViewAccess($login)
{
return Db::fetchAll(self::getSqlAccessSite("access, t2.idsite"), $login);
$sql = self::getSqlAccessSite("access, t2.idsite");
return Db::fetchAll($sql, $login);
}
/**
@ -188,29 +182,50 @@ class Access
*/
public static function getSqlAccessSite($select)
{
return "SELECT " . $select . "
FROM " . Common::prefixTable('access') . " as t1
JOIN " . Common::prefixTable('site') . " as t2 USING (idsite) " .
" WHERE login = ?";
$access = Common::prefixTable('access');
$siteTable = Common::prefixTable('site');
return "SELECT " . $select . " FROM " . $access . " as t1
JOIN " . $siteTable . " as t2 USING (idsite) WHERE login = ?";
}
/**
* Reload Super User access
* Make sure a login name is set
*
* @return bool
* @return true
*/
protected function reloadAccessSuperUser()
protected function makeSureLoginNameIsSet()
{
$this->hasSuperUserAccess = true;
try {
$allSitesId = Plugins\SitesManager\API::getInstance()->getAllSitesId();
} catch (\Exception $e) {
$allSitesId = array();
if (empty($this->login)) {
// flag to force non empty login so Super User is not mistaken for anonymous
$this->login = 'super user was set';
}
$this->idsitesByAccess['superuser'] = $allSitesId;
}
return true;
protected function loadSitesIfNeeded()
{
if ($this->hasSuperUserAccess) {
if (empty($this->idsitesByAccess['superuser'])) {
try {
$allSitesId = Plugins\SitesManager\API::getInstance()->getAllSitesId();
} catch (\Exception $e) {
$allSitesId = array();
}
$this->idsitesByAccess['superuser'] = $allSitesId;
}
} elseif (isset($this->login)) {
if (empty($this->idsitesByAccess['view'])
&& empty($this->idsitesByAccess['admin'])) {
// we join with site in case there are rows in access for an idsite that doesn't exist anymore
// (backward compatibility ; before we deleted the site without deleting rows in _access table)
$accessRaw = $this->getRawSitesWithSomeViewAccess($this->login);
foreach ($accessRaw as $access) {
$this->idsitesByAccess[$access['access']][] = $access['idsite'];
}
}
}
}
/**
@ -221,12 +236,12 @@ class Access
*/
public function setSuperUserAccess($bool = true)
{
if ($bool) {
$this->reloadAccessSuperUser();
} else {
$this->hasSuperUserAccess = false;
$this->idsitesByAccess['superuser'] = array();
$this->hasSuperUserAccess = (bool) $bool;
if ($bool) {
$this->makeSureLoginNameIsSet();
} else {
$this->resetSites();
}
}
@ -269,6 +284,8 @@ class Access
*/
public function getSitesIdWithAtLeastViewAccess()
{
$this->loadSitesIfNeeded();
return array_unique(array_merge(
$this->idsitesByAccess['view'],
$this->idsitesByAccess['admin'],
@ -284,13 +301,14 @@ class Access
*/
public function getSitesIdWithAdminAccess()
{
$this->loadSitesIfNeeded();
return array_unique(array_merge(
$this->idsitesByAccess['admin'],
$this->idsitesByAccess['superuser'])
);
}
/**
* Returns an array of ID sites for which the user has a VIEW access only.
*
@ -300,6 +318,8 @@ class Access
*/
public function getSitesIdWithViewAccess()
{
$this->loadSitesIfNeeded();
return $this->idsitesByAccess['view'];
}
@ -315,6 +335,22 @@ class Access
}
}
/**
* Returns `true` if the current user has admin access to at least one site.
*
* @return bool
*/
public function isUserHasSomeAdminAccess()
{
if ($this->hasSuperUserAccess()) {
return true;
}
$idSitesAccessible = $this->getSitesIdWithAdminAccess();
return count($idSitesAccessible) > 0;
}
/**
* If the user doesn't have an ADMIN access for at least one website, throws an exception
*
@ -322,11 +358,7 @@ class Access
*/
public function checkUserHasSomeAdminAccess()
{
if ($this->hasSuperUserAccess()) {
return;
}
$idSitesAccessible = $this->getSitesIdWithAdminAccess();
if (count($idSitesAccessible) == 0) {
if (!$this->isUserHasSomeAdminAccess()) {
throw new NoAccessException(Piwik::translate('General_ExceptionPrivilegeAtLeastOneWebsite', array('admin')));
}
}
@ -341,7 +373,9 @@ class Access
if ($this->hasSuperUserAccess()) {
return;
}
$idSitesAccessible = $this->getSitesIdWithAtLeastViewAccess();
if (count($idSitesAccessible) == 0) {
throw new NoAccessException(Piwik::translate('General_ExceptionPrivilegeAtLeastOneWebsite', array('view')));
}
@ -359,8 +393,10 @@ class Access
if ($this->hasSuperUserAccess()) {
return;
}
$idSites = $this->getIdSites($idSites);
$idSitesAccessible = $this->getSitesIdWithAdminAccess();
foreach ($idSites as $idsite) {
if (!in_array($idsite, $idSitesAccessible)) {
throw new NoAccessException(Piwik::translate('General_ExceptionPrivilegeAccessWebsite', array("'admin'", $idsite)));
@ -380,8 +416,10 @@ class Access
if ($this->hasSuperUserAccess()) {
return;
}
$idSites = $this->getIdSites($idSites);
$idSitesAccessible = $this->getSitesIdWithAtLeastViewAccess();
foreach ($idSites as $idsite) {
if (!in_array($idsite, $idSitesAccessible)) {
throw new NoAccessException(Piwik::translate('General_ExceptionPrivilegeAccessWebsite', array("'view'", $idsite)));
@ -401,11 +439,41 @@ class Access
}
$idSites = Site::getIdSitesFromIdSitesString($idSites);
if (empty($idSites)) {
throw new NoAccessException("The parameter 'idSite=' is missing from the request.");
}
return $idSites;
}
/**
* Executes a callback with superuser privileges, making sure those privileges are rescinded
* before this method exits. Privileges will be rescinded even if an exception is thrown.
*
* @param callback $function The callback to execute. Should accept no arguments.
* @return mixed The result of `$function`.
* @throws Exception rethrows any exceptions thrown by `$function`.
* @api
*/
public static function doAsSuperUser($function)
{
$isSuperUser = self::getInstance()->hasSuperUserAccess();
self::getInstance()->setSuperUserAccess(true);
try {
$result = $function();
} catch (Exception $ex) {
self::getInstance()->setSuperUserAccess($isSuperUser);
throw $ex;
}
self::getInstance()->setSuperUserAccess($isSuperUser);
return $result;
}
}
/**

246
www/analytics/core/Application/Environment.php Normal file
View file

@ -0,0 +1,246 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Application;
use DI\Container;
use Piwik\Application\Kernel\EnvironmentValidator;
use Piwik\Application\Kernel\GlobalSettingsProvider;
use Piwik\Application\Kernel\PluginList;
use Piwik\Container\ContainerFactory;
use Piwik\Container\StaticContainer;
use Piwik\Piwik;
/**
* Encapsulates Piwik environment setup and access.
*
* The Piwik environment consists of two main parts: the kernel and the DI container.
*
* The 'kernel' is the core part of Piwik that cannot be modified / extended through the DI container.
* It includes components that are required to create the DI container.
*
* Currently the only objects in the 'kernel' are a GlobalSettingsProvider object and a
* PluginList object. The GlobalSettingsProvider object is required for the current PluginList
* implementation and for checking whether Development mode is enabled. The PluginList is
* needed in order to determine what plugins are activated, since plugins can provide their
* own DI configuration.
*
* The DI container contains every other Piwik object, including the Plugin\Manager,
* plugin API instances, dependent services, etc. Plugins and users can override/extend
* the objects in this container.
*
* NOTE: DI support in Piwik is currently a work in process; not everything is currently
* stored in the DI container, but we are working towards this.
*/
class Environment
{
/**
* @internal
* @var EnvironmentManipulator
*/
private static $globalEnvironmentManipulator = null;
/**
* @var string
*/
private $environment;
/**
* @var array
*/
private $definitions;
/**
* @var Container
*/
private $container;
/**
* @var GlobalSettingsProvider
*/
private $globalSettingsProvider;
/**
* @var PluginList
*/
private $pluginList;
/**
* @param string $environment
* @param array $definitions
*/
public function __construct($environment, array $definitions = array())
{
$this->environment = $environment;
$this->definitions = $definitions;
}
/**
* Initializes the kernel globals and DI container.
*/
public function init()
{
$this->invokeBeforeContainerCreatedHook();
$this->container = $this->createContainer();
StaticContainer::push($this->container);
$this->validateEnvironment();
$this->invokeEnvironmentBootstrappedHook();
Piwik::postEvent('Environment.bootstrapped'); // this event should be removed eventually
}
/**
* Destroys an environment. MUST be called when embedding environments.
*/
public function destroy()
{
StaticContainer::pop();
}
/**
* Returns the DI container. All Piwik objects for a specific Piwik instance should be stored
* in this container.
*
* @return Container
*/
public function getContainer()
{
return $this->container;
}
/**
* @link http://php-di.org/doc/container-configuration.html
*/
private function createContainer()
{
$pluginList = $this->getPluginListCached();
$settings = $this->getGlobalSettingsCached();
$extraDefinitions = $this->getExtraDefinitionsFromManipulators();
$definitions = array_merge(StaticContainer::getDefinitions(), $extraDefinitions, array($this->definitions));
$environments = array($this->environment);
$environments = array_merge($environments, $this->getExtraEnvironmentsFromManipulators());
$containerFactory = new ContainerFactory($pluginList, $settings, $environments, $definitions);
return $containerFactory->create();
}
protected function getGlobalSettingsCached()
{
if ($this->globalSettingsProvider === null) {
$original = $this->getGlobalSettings();
$globalSettingsProvider = $this->getGlobalSettingsProviderOverride($original);
$this->globalSettingsProvider = $globalSettingsProvider ?: $original;
}
return $this->globalSettingsProvider;
}
protected function getPluginListCached()
{
if ($this->pluginList === null) {
$pluginList = $this->getPluginListOverride();
$this->pluginList = $pluginList ?: $this->getPluginList();
}
return $this->pluginList;
}
/**
* Returns the kernel global GlobalSettingsProvider object. Derived classes can override this method
* to provide a different implementation.
*
* @return null|GlobalSettingsProvider
*/
protected function getGlobalSettings()
{
return new GlobalSettingsProvider();
}
/**
* Returns the kernel global PluginList object. Derived classes can override this method to
* provide a different implementation.
*
* @return PluginList
*/
protected function getPluginList()
{
// TODO: in tracker should only load tracker plugins. can't do properly until tracker entrypoint is encapsulated.
return new PluginList($this->getGlobalSettingsCached());
}
private function validateEnvironment()
{
/** @var EnvironmentValidator $validator */
$validator = $this->container->get('Piwik\Application\Kernel\EnvironmentValidator');
$validator->validate();
}
/**
* @param EnvironmentManipulator $manipulator
* @internal
*/
public static function setGlobalEnvironmentManipulator(EnvironmentManipulator $manipulator)
{
self::$globalEnvironmentManipulator = $manipulator;
}
private function getGlobalSettingsProviderOverride(GlobalSettingsProvider $original)
{
if (self::$globalEnvironmentManipulator) {
return self::$globalEnvironmentManipulator->makeGlobalSettingsProvider($original);
} else {
return null;
}
}
private function invokeBeforeContainerCreatedHook()
{
if (self::$globalEnvironmentManipulator) {
return self::$globalEnvironmentManipulator->beforeContainerCreated();
}
}
private function getExtraDefinitionsFromManipulators()
{
if (self::$globalEnvironmentManipulator) {
return self::$globalEnvironmentManipulator->getExtraDefinitions();
} else {
return array();
}
}
private function invokeEnvironmentBootstrappedHook()
{
if (self::$globalEnvironmentManipulator) {
self::$globalEnvironmentManipulator->onEnvironmentBootstrapped();
}
}
private function getExtraEnvironmentsFromManipulators()
{
if (self::$globalEnvironmentManipulator) {
return self::$globalEnvironmentManipulator->getExtraEnvironments();
} else {
return array();
}
}
private function getPluginListOverride()
{
if (self::$globalEnvironmentManipulator) {
return self::$globalEnvironmentManipulator->makePluginList($this->getGlobalSettingsCached());
} else {
return null;
}
}
}

59
www/analytics/core/Application/EnvironmentManipulator.php Normal file
View file

@ -0,0 +1,59 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Application;
use Piwik\Application\Kernel\GlobalSettingsProvider;
use Piwik\Application\Kernel\PluginList;
/**
* Used to manipulate Environment instances before the container is created.
* Only used by the testing environment setup code, shouldn't be used anywhere
* else.
*/
interface EnvironmentManipulator
{
/**
* Create a custom GlobalSettingsProvider kernel object, overriding the default behavior.
*
* @return GlobalSettingsProvider
*/
public function makeGlobalSettingsProvider(GlobalSettingsProvider $original);
/**
* Create a custom PluginList kernel object, overriding the default behavior.@deprecated
*
* @param GlobalSettingsProvider $globalSettingsProvider
* @return PluginList
*/
public function makePluginList(GlobalSettingsProvider $globalSettingsProvider);
/**
* Invoked before the container is created.
*/
public function beforeContainerCreated();
/**
* Return an array of definition arrays that override DI config specified in PHP config files.
*
* @return array[]
*/
public function getExtraDefinitions();
/**
* Invoked after the container is created and the environment is considered bootstrapped.
*/
public function onEnvironmentBootstrapped();
/**
* Return an array of environment names to apply after the normal environment.
*
* @return string[]
*/
public function getExtraEnvironments();
}

79
www/analytics/core/Application/Kernel/EnvironmentValidator.php Normal file
View file

@ -0,0 +1,79 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Application\Kernel;
use Piwik\Common;
use Piwik\Piwik;
use Piwik\SettingsServer;
use Piwik\Translation\Translator;
/**
* Validates the Piwik environment. This includes making sure the required config files
* are present, and triggering the correct behaviour if otherwise.
*/
class EnvironmentValidator
{
/**
* @var GlobalSettingsProvider
*/
protected $settingsProvider;
/**
* @var Translator
*/
protected $translator;
public function __construct(GlobalSettingsProvider $settingsProvider, Translator $translator)
{
$this->settingsProvider = $settingsProvider;
$this->translator = $translator;
}
public function validate()
{
$inTrackerRequest = SettingsServer::isTrackerApiRequest();
$inConsole = Common::isPhpCliMode();
$this->checkConfigFileExists($this->settingsProvider->getPathGlobal());
$this->checkConfigFileExists($this->settingsProvider->getPathLocal(), $startInstaller = !$inTrackerRequest && !$inConsole);
}
/**
* @param $path
* @param bool $startInstaller
* @throws \Exception
*/
private function checkConfigFileExists($path, $startInstaller = false)
{
if (is_readable($path)) {
return;
}
$message = $this->translator->translate('General_ExceptionConfigurationFileNotFound', array($path));
if (Common::isPhpCliMode()) {
$message .= "\n" . $this->translator->translate('General_ExceptionConfigurationFileNotFound2', array($path, get_current_user()));
}
$exception = new \Exception($message);
if ($startInstaller) {
/**
* Triggered when the configuration file cannot be found or read, which usually
* means Piwik is not installed yet.
*
* This event can be used to start the installation process or to display a custom error message.
*
* @param \Exception $exception The exception that was thrown by `Config::getInstance()`.
*/
Piwik::postEvent('Config.NoConfigurationFile', array($exception), $pending = true);
} else {
throw $exception;
}
}
}

111
www/analytics/core/Application/Kernel/GlobalSettingsProvider.php Normal file
View file

@ -0,0 +1,111 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Application\Kernel;
use Piwik\Config;
use Piwik\Config\IniFileChain;
/**
* Provides global settings. Global settings are organized in sections where
* each section contains a list of name => value pairs. Setting values can
* be primitive values or arrays of primitive values.
*
* Uses the config.ini.php, common.ini.php and global.ini.php files to provide global settings.
*
* At the moment a singleton instance of this class is used in order to get tests to pass.
*/
class GlobalSettingsProvider
{
/**
* @var IniFileChain
*/
protected $iniFileChain;
/**
* @var string
*/
protected $pathGlobal = null;
/**
* @var string
*/
protected $pathCommon = null;
/**
* @var string
*/
protected $pathLocal = null;
/**
* @param string|null $pathGlobal Path to the global.ini.php file. Or null to use the default.
* @param string|null $pathLocal Path to the config.ini.php file. Or null to use the default.
* @param string|null $pathCommon Path to the common.ini.php file. Or null to use the default.
*/
public function __construct($pathGlobal = null, $pathLocal = null, $pathCommon = null)
{
$this->pathGlobal = $pathGlobal ?: Config::getGlobalConfigPath();
$this->pathCommon = $pathCommon ?: Config::getCommonConfigPath();
$this->pathLocal = $pathLocal ?: Config::getLocalConfigPath();
$this->iniFileChain = new IniFileChain();
$this->reload();
}
public function reload($pathGlobal = null, $pathLocal = null, $pathCommon = null)
{
$this->pathGlobal = $pathGlobal ?: $this->pathGlobal;
$this->pathCommon = $pathCommon ?: $this->pathCommon;
$this->pathLocal = $pathLocal ?: $this->pathLocal;
$this->iniFileChain->reload(array($this->pathGlobal, $this->pathCommon), $this->pathLocal);
}
/**
* Returns a settings section.
*
* @param string $name
* @return array
*/
public function &getSection($name)
{
$section =& $this->iniFileChain->get($name);
return $section;
}
/**
* Sets a settings section.
*
* @param string $name
* @param array $value
*/
public function setSection($name, $value)
{
$this->iniFileChain->set($name, $value);
}
public function getIniFileChain()
{
return $this->iniFileChain;
}
public function getPathGlobal()
{
return $this->pathGlobal;
}
public function getPathLocal()
{
return $this->pathLocal;
}
public function getPathCommon()
{
return $this->pathCommon;
}
}

120
www/analytics/core/Application/Kernel/PluginList.php Normal file
View file

@ -0,0 +1,120 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Application\Kernel;
/**
* Lists the currently activated plugins. Used when setting up Piwik's environment before
* initializing the DI container.
*
* Uses the [Plugins] section in Piwik's INI config to get the activated plugins.
*
* Depends on GlobalSettingsProvider being used.
*
* TODO: parts of Plugin\Manager edit the plugin list; maybe PluginList implementations should be mutable?
*/
class PluginList
{
/**
* @var GlobalSettingsProvider
*/
private $settings;
/**
* Plugins bundled with core package, disabled by default
* @var array
*/
private $corePluginsDisabledByDefault = array(
'DBStats',
'ExampleCommand',
'ExampleSettingsPlugin',
'ExampleUI',
'ExampleVisualization',
'ExamplePluginTemplate',
'ExampleTracker',
'ExampleReport',
'MobileAppMeasurable',
'Provider'
);
// Themes bundled with core package, disabled by default
private $coreThemesDisabledByDefault = array(
'ExampleTheme'
);
public function __construct(GlobalSettingsProvider $settings)
{
$this->settings = $settings;
}
/**
* Returns the list of plugins that should be loaded. Used by the container factory to
* load plugin specific DI overrides.
*
* @return string[]
*/
public function getActivatedPlugins()
{
$section = $this->settings->getSection('Plugins');
return @$section['Plugins'] ?: array();
}
/**
* Returns the list of plugins that are bundled with Piwik.
*
* @return string[]
*/
public function getPluginsBundledWithPiwik()
{
$pathGlobal = $this->settings->getPathGlobal();
$section = $this->settings->getIniFileChain()->getFrom($pathGlobal, 'Plugins');
return $section['Plugins'];
}
/**
* Returns the plugins bundled with core package that are disabled by default.
*
* @return string[]
*/
public function getCorePluginsDisabledByDefault()
{
return array_merge($this->corePluginsDisabledByDefault, $this->coreThemesDisabledByDefault);
}
/**
* Sorts an array of plugins in the order they should be loaded.
*
* @params string[] $plugins
* @return \string[]
*/
public function sortPlugins(array $plugins)
{
$global = $this->getPluginsBundledWithPiwik();
if (empty($global)) {
return $plugins;
}
// we need to make sure a possibly disabled plugin will be still loaded before any 3rd party plugin
$global = array_merge($global, $this->corePluginsDisabledByDefault);
$global = array_values($global);
$plugins = array_values($plugins);
$defaultPluginsLoadedFirst = array_intersect($global, $plugins);
$otherPluginsToLoadAfterDefaultPlugins = array_diff($plugins, $defaultPluginsLoadedFirst);
// sort by name to have a predictable order for those extra plugins
sort($otherPluginsToLoadAfterDefaultPlugins);
$sorted = array_merge($defaultPluginsLoadedFirst, $otherPluginsToLoadAfterDefaultPlugins);
return $sorted;
}
}

421
www/analytics/core/Archive.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,57 +9,58 @@
namespace Piwik;
use Piwik\Archive\Parameters;
use Piwik\ArchiveProcessor\Rules;
use Piwik\Archive\ArchiveInvalidator;
use Piwik\Container\StaticContainer;
use Piwik\DataAccess\ArchiveSelector;
use Piwik\Period\Range;
use Piwik\Period\Factory as PeriodFactory;
/**
* The **Archive** class is used to query cached analytics statistics
* (termed "archive data").
*
*
* You can use **Archive** instances to get data that was archived for one or more sites,
* for one or more periods and one optional segment.
*
*
* If archive data is not found, this class will initiate the archiving process. [1](#footnote-1)
*
*
* **Archive** instances must be created using the {@link build()} factory method;
* they cannot be constructed.
*
*
* You can search for metrics (such as `nb_visits`) using the {@link getNumeric()} and
* {@link getDataTableFromNumeric()} methods. You can search for
* reports using the {@link getBlob()}, {@link getDataTable()} and {@link getDataTableExpanded()} methods.
*
*
* If you're creating an API that returns report data, you may want to use the
* {@link getDataTableFromArchive()} helper function.
*
*
* ### Learn more
*
*
* Learn more about _archiving_ [here](/guides/all-about-analytics-data).
*
*
* ### Limitations
*
*
* - You cannot get data for multiple range periods in a single query.
* - You cannot get data for periods of different types in a single query.
*
*
* ### Examples
*
*
* **_Querying metrics for an API method_**
*
*
* // one site and one period
* $archive = Archive::build($idSite = 1, $period = 'week', $date = '2013-03-08');
* return $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
*
*
* // all sites and multiple dates
* $archive = Archive::build($idSite = 'all', $period = 'month', $date = '2013-01-02,2013-03-08');
* return $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
*
*
* **_Querying and using metrics immediately_**
*
*
* // one site and one period
* $archive = Archive::build($idSite = 1, $period = 'week', $date = '2013-03-08');
* $data = $archive->getNumeric(array('nb_visits', 'nb_actions'));
*
*
* $visits = $data['nb_visits'];
* $actions = $data['nb_actions'];
*
@ -68,41 +69,40 @@ use Piwik\Period\Range;
* // multiple sites and multiple dates
* $archive = Archive::build($idSite = '1,2,3', $period = 'month', $date = '2013-01-02,2013-03-08');
* $data = $archive->getNumeric('nb_visits');
*
*
* $janSite1Visits = $data['1']['2013-01-01,2013-01-31']['nb_visits'];
* $febSite1Visits = $data['1']['2013-02-01,2013-02-28']['nb_visits'];
* // ... etc.
*
*
* **_Querying for reports_**
*
*
* $archive = Archive::build($idSite = 1, $period = 'week', $date = '2013-03-08');
* $dataTable = $archive->getDataTable('MyPlugin_MyReport');
* // ... manipulate $dataTable ...
* return $dataTable;
*
*
* **_Querying a report for an API method_**
*
*
* public function getMyReport($idSite, $period, $date, $segment = false, $expanded = false)
* {
* $dataTable = Archive::getDataTableFromArchive('MyPlugin_MyReport', $idSite, $period, $date, $segment, $expanded);
* $dataTable->queueFilter('ReplaceColumnNames');
* return $dataTable;
* }
*
*
* **_Querying data for multiple range periods_**
*
*
* // get data for first range
* $archive = Archive::build($idSite = 1, $period = 'range', $date = '2013-03-08,2013-03-12');
* $dataTable = $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
*
*
* // get data for second range
* $archive = Archive::build($idSite = 1, $period = 'range', $date = '2013-03-15,2013-03-20');
* $dataTable = $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
*
*
* <a name="footnote-1"></a>
* [1]: The archiving process will not be launched if browser archiving is disabled
* and the current request came from a browser (and not the **archive.php** cron
* script).
* and the current request came from a browser.
*
*
* @api
@ -162,6 +162,16 @@ class Archive
*/
private $params;
/**
* @var \Piwik\Cache\Cache
*/
private static $cache;
/**
* @var ArchiveInvalidator
*/
private $invalidator;
/**
* @param Parameters $params
* @param bool $forceIndexedBySite Whether to force index the result of a query by site ID.
@ -173,6 +183,8 @@ class Archive
$this->params = $params;
$this->forceIndexedBySite = $forceIndexedBySite;
$this->forceIndexedByDate = $forceIndexedByDate;
$this->invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator');
}
/**
@ -192,37 +204,42 @@ class Archive
* or date range (ie, 'YYYY-MM-DD,YYYY-MM-DD').
* @param bool|false|string $segment Segment definition or false if no segment should be used. {@link Piwik\Segment}
* @param bool|false|string $_restrictSitesToLogin Used only when running as a scheduled task.
* @param bool $skipAggregationOfSubTables Whether the archive, when it is processed, should also aggregate all sub-tables
* @return Archive
* @return static
*/
public static function build($idSites, $period, $strDate, $segment = false, $_restrictSitesToLogin = false, $skipAggregationOfSubTables = false)
public static function build($idSites, $period, $strDate, $segment = false, $_restrictSitesToLogin = false)
{
$websiteIds = Site::getIdSitesFromIdSitesString($idSites, $_restrictSitesToLogin);
$timezone = false;
if (count($websiteIds) == 1) {
$timezone = Site::getTimezoneFor($websiteIds[0]);
}
if (Period::isMultiplePeriod($strDate, $period)) {
$oPeriod = new Range($period, $strDate);
$oPeriod = PeriodFactory::build($period, $strDate, $timezone);
$allPeriods = $oPeriod->getSubperiods();
} else {
$timezone = count($websiteIds) == 1 ? Site::getTimezoneFor($websiteIds[0]) : false;
$oPeriod = Period::makePeriodFromQueryParams($timezone, $period, $strDate);
$oPeriod = PeriodFactory::makePeriodFromQueryParams($timezone, $period, $strDate);
$allPeriods = array($oPeriod);
}
$segment = new Segment($segment, $websiteIds);
$idSiteIsAll = $idSites == self::REQUEST_ALL_WEBSITES_FLAG;
$segment = new Segment($segment, $websiteIds);
$idSiteIsAll = $idSites == self::REQUEST_ALL_WEBSITES_FLAG;
$isMultipleDate = Period::isMultiplePeriod($strDate, $period);
return Archive::factory($segment, $allPeriods, $websiteIds, $idSiteIsAll, $isMultipleDate, $skipAggregationOfSubTables);
return static::factory($segment, $allPeriods, $websiteIds, $idSiteIsAll, $isMultipleDate);
}
/**
* Returns a new Archive instance that will query archive data for the given set of
* sites and periods, using an optional segment.
*
*
* This method uses an array of Period instances and a Segment instance, instead of strings
* like {@link build()}.
*
*
* If you want to create an Archive instance using data found in query parameters,
* use {@link build()}.
*
*
* @param Segment $segment The segment to use. For no segment, use `new Segment('', $idSites)`.
* @param array $periods An array of Period instances.
* @param array $idSites An array of site IDs (eg, `array(1, 2, 3)`).
@ -232,41 +249,42 @@ class Archive
* @param bool $isMultipleDate Whether multiple dates are being queried or not. If true, then
* the result of querying functions will be indexed by period,
* regardless of whether `count($periods) == 1`.
* @param bool $skipAggregationOfSubTables Whether the archive should skip aggregation of all sub-tables
*
* @return Archive
*/
public static function factory(Segment $segment, array $periods, array $idSites, $idSiteIsAll = false, $isMultipleDate = false, $skipAggregationOfSubTables = false)
public static function factory(Segment $segment, array $periods, array $idSites, $idSiteIsAll = false, $isMultipleDate = false)
{
$forceIndexedBySite = false;
$forceIndexedByDate = false;
if ($idSiteIsAll || count($idSites) > 1) {
$forceIndexedBySite = true;
}
if (count($periods) > 1 || $isMultipleDate) {
$forceIndexedByDate = true;
}
$params = new Parameters($idSites, $periods, $segment, $skipAggregationOfSubTables);
$params = new Parameters($idSites, $periods, $segment);
return new Archive($params, $forceIndexedBySite, $forceIndexedByDate);
return new static($params, $forceIndexedBySite, $forceIndexedByDate);
}
/**
* Queries and returns metric data in an array.
*
*
* If multiple sites were requested in {@link build()} or {@link factory()} the result will
* be indexed by site ID.
*
*
* If multiple periods were requested in {@link build()} or {@link factory()} the result will
* be indexed by period.
*
*
* The site ID index is always first, so if multiple sites & periods were requested, the result
* will be indexed by site ID first, then period.
*
*
* @param string|array $names One or more archive names, eg, `'nb_visits'`, `'Referrers_distinctKeywords'`,
* etc.
* @return false|numeric|array `false` if there is no data to return, a single numeric value if we're not querying
* @return false|integer|array `false` if there is no data to return, a single numeric value if we're not querying
* for multiple sites/periods, or an array if multiple sites, periods or names are
* queried for.
*/
@ -288,50 +306,22 @@ class Archive
return $result;
}
/**
* Queries and returns blob data in an array.
*
* Reports are stored in blobs as serialized arrays of {@link DataTable\Row} instances, but this
* data can technically be anything. In other words, you can store whatever you want
* as archive data blobs.
*
* If multiple sites were requested in {@link build()} or {@link factory()} the result will
* be indexed by site ID.
*
* If multiple periods were requested in {@link build()} or {@link factory()} the result will
* be indexed by period.
*
* The site ID index is always first, so if multiple sites & periods were requested, the result
* will be indexed by site ID first, then period.
*
* @param string|array $names One or more archive names, eg, `'Referrers_keywordBySearchEngine'`.
* @param null|string $idSubtable If we're returning serialized DataTable data, then this refers
* to the subtable ID to return. If set to 'all', all subtables
* of each requested report are returned.
* @return array An array of appropriately indexed blob data.
*/
public function getBlob($names, $idSubtable = null)
{
$data = $this->get($names, 'blob', $idSubtable);
return $data->getIndexedArray($this->getResultIndices());
}
/**
* Queries and returns metric data in a DataTable instance.
*
*
* If multiple sites were requested in {@link build()} or {@link factory()} the result will
* be a DataTable\Map that is indexed by site ID.
*
*
* If multiple periods were requested in {@link build()} or {@link factory()} the result will
* be a {@link DataTable\Map} that is indexed by period.
*
*
* The site ID index is always first, so if multiple sites & periods were requested, the result
* will be a {@link DataTable\Map} indexed by site ID which contains {@link DataTable\Map} instances that are
* indexed by period.
*
*
* _Note: Every DataTable instance returned will have at most one row in it. The contents of each
* row will be the requested metrics for the appropriate site and period._
*
*
* @param string|array $names One or more archive names, eg, 'nb_visits', 'Referrers_distinctKeywords',
* etc.
* @return DataTable|DataTable\Map A DataTable if multiple sites and periods were not requested.
@ -343,22 +333,40 @@ class Archive
return $data->getDataTable($this->getResultIndices());
}
/**
* Similar to {@link getDataTableFromNumeric()} but merges all children on the created DataTable.
*
* This is the same as doing `$this->getDataTableFromNumeric()->mergeChildren()` but this way it is much faster.
*
* @return DataTable|DataTable\Map
*
* @internal Currently only used by MultiSites.getAll plugin. Feel free to remove internal tag if needed somewhere
* else. If no longer needed by MultiSites.getAll please remove this method. If you need this to work in
* a bit different way feel free to refactor as always.
*/
public function getDataTableFromNumericAndMergeChildren($names)
{
$data = $this->get($names, 'numeric');
$resultIndexes = $this->getResultIndices();
return $data->getMergedDataTable($resultIndexes);
}
/**
* Queries and returns one or more reports as DataTable instances.
*
*
* This method will query blob data that is a serialized array of of {@link DataTable\Row}'s and
* unserialize it.
*
*
* If multiple sites were requested in {@link build()} or {@link factory()} the result will
* be a {@link DataTable\Map} that is indexed by site ID.
*
*
* If multiple periods were requested in {@link build()} or {@link factory()} the result will
* be a DataTable\Map that is indexed by period.
*
*
* The site ID index is always first, so if multiple sites & periods were requested, the result
* will be a {@link DataTable\Map} indexed by site ID which contains {@link DataTable\Map} instances that are
* indexed by period.
*
*
* @param string $name The name of the record to get. This method can only query one record at a time.
* @param int|string|null $idSubtable The ID of the subtable to get (if any).
* @return DataTable|DataTable\Map A DataTable if multiple sites and periods were not requested.
@ -372,13 +380,13 @@ class Archive
/**
* Queries and returns one report with all of its subtables loaded.
*
*
* If multiple sites were requested in {@link build()} or {@link factory()} the result will
* be a DataTable\Map that is indexed by site ID.
*
*
* If multiple periods were requested in {@link build()} or {@link factory()} the result will
* be a DataTable\Map that is indexed by period.
*
*
* The site ID index is always first, so if multiple sites & periods were requested, the result
* will be a {@link DataTable\Map indexed} by site ID which contains {@link DataTable\Map} instances that are
* indexed by period.
@ -400,16 +408,18 @@ class Archive
/**
* Returns the list of plugins that archive the given reports.
*
*
* @param array $archiveNames
* @return array
*/
private function getRequestedPlugins($archiveNames)
{
$result = array();
foreach ($archiveNames as $name) {
$result[] = self::getPluginForReport($name);
}
return array_unique($result);
}
@ -427,7 +437,7 @@ class Archive
/**
* Helper function that creates an Archive instance and queries for report data using
* query parameter data. API methods can use this method to reduce code redundancy.
*
*
* @param string $name The name of the report to return.
* @param int|string|array $idSite @see {@link build()}
* @param string $period @see {@link build()}
@ -435,21 +445,19 @@ class Archive
* @param string $segment @see {@link build()}
* @param bool $expanded If true, loads all subtables. See {@link getDataTableExpanded()}
* @param int|null $idSubtable See {@link getDataTableExpanded()}
* @param bool $skipAggregationOfSubTables Whether or not we should skip the aggregation of all sub-tables and only aggregate parent DataTable.
* @param int|null $depth See {@link getDataTableExpanded()}
* @throws \Exception
* @return DataTable|DataTable\Map See {@link getDataTable()} and
* {@link getDataTableExpanded()} for more
* information
* @deprecated Since Piwik 2.12.0 Use Archive::createDataTableFromArchive() instead
*/
public static function getDataTableFromArchive($name, $idSite, $period, $date, $segment, $expanded,
$idSubtable = null, $skipAggregationOfSubTables = false, $depth = null)
$idSubtable = null, $depth = null)
{
Piwik::checkUserHasViewAccess($idSite);
if($skipAggregationOfSubTables && ($expanded || $idSubtable)) {
throw new \Exception("Not expected to skipAggregationOfSubTables when expanded=1 or idSubtable is set.");
}
$archive = Archive::build($idSite, $period, $date, $segment, $_restrictSitesToLogin = false, $skipAggregationOfSubTables);
$archive = Archive::build($idSite, $period, $date, $segment, $_restrictSitesToLogin = false);
if ($idSubtable === false) {
$idSubtable = null;
}
@ -465,9 +473,102 @@ class Archive
return $dataTable;
}
private function appendIdSubtable($recordName, $id)
/**
* Helper function that creates an Archive instance and queries for report data using
* query parameter data. API methods can use this method to reduce code redundancy.
*
* @param string $recordName The name of the report to return.
* @param int|string|array $idSite @see {@link build()}
* @param string $period @see {@link build()}
* @param string $date @see {@link build()}
* @param string $segment @see {@link build()}
* @param bool $expanded If true, loads all subtables. See {@link getDataTableExpanded()}
* @param bool $flat If true, loads all subtables and disabled all recursive filters.
* @param int|null $idSubtable See {@link getDataTableExpanded()}
* @param int|null $depth See {@link getDataTableExpanded()}
* @return DataTable|DataTable\Map
*/
public static function createDataTableFromArchive($recordName, $idSite, $period, $date, $segment, $expanded = false, $flat = false, $idSubtable = null, $depth = null)
{
return $recordName . "_" . $id;
if ($flat && !$idSubtable) {
$expanded = true;
}
$dataTable = self::getDataTableFromArchive($recordName, $idSite, $period, $date, $segment, $expanded, $idSubtable, $depth);
$dataTable->queueFilter('ReplaceColumnNames');
if ($expanded) {
$dataTable->queueFilterSubtables('ReplaceColumnNames');
}
if ($flat) {
$dataTable->disableRecursiveFilters();
}
return $dataTable;
}
private function getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet()
{
if (is_null(self::$cache)) {
self::$cache = Cache::getTransientCache();
}
$id = 'Archive.SiteIdsOfRememberedReportsInvalidated';
if (!self::$cache->contains($id)) {
self::$cache->save($id, array());
}
$siteIdsAlreadyHandled = self::$cache->fetch($id);
$siteIdsRequested = $this->params->getIdSites();
foreach ($siteIdsRequested as $index => $siteIdRequested) {
$siteIdRequested = (int) $siteIdRequested;
if (in_array($siteIdRequested, $siteIdsAlreadyHandled)) {
unset($siteIdsRequested[$index]); // was already handled previously, do not do it again
} else {
$siteIdsAlreadyHandled[] = $siteIdRequested; // we will handle this id this time
}
}
self::$cache->save($id, $siteIdsAlreadyHandled);
return $siteIdsRequested;
}
private function invalidatedReportsIfNeeded()
{
$siteIdsRequested = $this->getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet();
if (empty($siteIdsRequested)) {
return; // all requested site ids were already handled
}
$sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
foreach ($sitesPerDays as $date => $siteIds) {
if (empty($siteIds)) {
continue;
}
$siteIdsToActuallyInvalidate = array_intersect($siteIds, $siteIdsRequested);
if (empty($siteIdsToActuallyInvalidate)) {
continue; // all site ids that should be handled are already handled
}
try {
$this->invalidator->markArchivesAsInvalidated($siteIdsToActuallyInvalidate, array(Date::factory($date)), false);
} catch (\Exception $e) {
Site::clearCache();
throw $e;
}
}
Site::clearCache();
}
/**
@ -477,7 +578,7 @@ class Archive
* @param null|int $idSubtable
* @return Archive\DataCollection
*/
private function get($archiveNames, $archiveDataType, $idSubtable = null)
protected function get($archiveNames, $archiveDataType, $idSubtable = null)
{
if (!is_array($archiveNames)) {
$archiveNames = array($archiveNames);
@ -487,35 +588,41 @@ class Archive
if ($idSubtable !== null
&& $idSubtable != self::ID_SUBTABLE_LOAD_ALL_SUBTABLES
) {
foreach ($archiveNames as &$name) {
$name = $this->appendIdsubtable($name, $idSubtable);
// this is also done in ArchiveSelector. It should be actually only done in ArchiveSelector but DataCollection
// does require to have the subtableId appended. Needs to be changed in refactoring to have it only in one
// place.
$dataNames = array();
foreach ($archiveNames as $name) {
$dataNames[] = ArchiveSelector::appendIdsubtable($name, $idSubtable);
}
} else {
$dataNames = $archiveNames;
}
$result = new Archive\DataCollection(
$archiveNames, $archiveDataType, $this->params->getIdSites(), $this->params->getPeriods(), $defaultRow = null);
$dataNames, $archiveDataType, $this->params->getIdSites(), $this->params->getPeriods(), $defaultRow = null);
$archiveIds = $this->getArchiveIds($archiveNames);
if (empty($archiveIds)) {
return $result;
}
$loadAllSubtables = $idSubtable == self::ID_SUBTABLE_LOAD_ALL_SUBTABLES;
$archiveData = ArchiveSelector::getArchiveData($archiveIds, $archiveNames, $archiveDataType, $loadAllSubtables);
$archiveData = ArchiveSelector::getArchiveData($archiveIds, $archiveNames, $archiveDataType, $idSubtable);
$isNumeric = $archiveDataType == 'numeric';
foreach ($archiveData as $row) {
// values are grouped by idsite (site ID), date1-date2 (date range), then name (field name)
$idSite = $row['idsite'];
$periodStr = $row['date1'] . "," . $row['date2'];
$periodStr = $row['date1'] . ',' . $row['date2'];
if ($archiveDataType == 'numeric') {
$value = $this->formatNumericValue($row['value']);
if ($isNumeric) {
$row['value'] = $this->formatNumericValue($row['value']);
} else {
$value = $this->uncompress($row['value']);
$result->addMetadata($idSite, $periodStr, 'ts_archived', $row['ts_archived']);
$result->addMetadata($row['idsite'], $periodStr, DataTable::ARCHIVED_DATE_METADATA_NAME, $row['ts_archived']);
}
$resultRow = & $result->get($idSite, $periodStr);
$resultRow[$row['name']] = $value;
$result->set($row['idsite'], $periodStr, $row['name'], $row['value']);
}
return $result;
@ -526,6 +633,9 @@ class Archive
* queried. This function will use the idarchive cache if it has the right data,
* query archive tables for IDs w/o launching archiving, or launch archiving and
* get the idarchive from ArchiveProcessor instances.
*
* @param string $archiveNames
* @return array
*/
private function getArchiveIds($archiveNames)
{
@ -533,7 +643,7 @@ class Archive
// figure out which archives haven't been processed (if an archive has been processed,
// then we have the archive IDs in $this->idarchives)
$doneFlags = array();
$doneFlags = array();
$archiveGroups = array();
foreach ($plugins as $plugin) {
$doneFlag = $this->getDoneStringForPlugin($plugin);
@ -542,7 +652,7 @@ class Archive
if (!isset($this->idarchives[$doneFlag])) {
$archiveGroup = $this->getArchiveGroupOfPlugin($plugin);
if($archiveGroup == self::ARCHIVE_ALL_PLUGINS_FLAG) {
if ($archiveGroup == self::ARCHIVE_ALL_PLUGINS_FLAG) {
$archiveGroup = reset($plugins);
}
$archiveGroups[] = $archiveGroup;
@ -560,19 +670,7 @@ class Archive
}
}
// order idarchives by the table month they belong to
$idArchivesByMonth = array();
foreach (array_keys($doneFlags) as $doneFlag) {
if (empty($this->idarchives[$doneFlag])) {
continue;
}
foreach ($this->idarchives[$doneFlag] as $dateRange => $idarchives) {
foreach ($idarchives as $id) {
$idArchivesByMonth[$dateRange][] = $id;
}
}
}
$idArchivesByMonth = $this->getIdArchivesByMonth($doneFlags);
return $idArchivesByMonth;
}
@ -587,6 +685,8 @@ class Archive
*/
private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins)
{
$this->invalidatedReportsIfNeeded();
$today = Date::today();
foreach ($this->params->getPeriods() as $period) {
@ -600,14 +700,14 @@ class Archive
// we already know there are no stats for this period
// we add one day to make sure we don't miss the day of the website creation
if ($twoDaysAfterPeriod->isEarlier($site->getCreationDate())) {
Log::verbose("Archive site %s, %s (%s) skipped, archive is before the website was created.",
Log::debug("Archive site %s, %s (%s) skipped, archive is before the website was created.",
$idSite, $period->getLabel(), $period->getPrettyString());
continue;
}
// if the starting date is in the future we know there is no visiidsite = ?t
if ($twoDaysBeforePeriod->isLater($today)) {
Log::verbose("Archive site %s, %s (%s) skipped, archive is after today.",
Log::debug("Archive site %s, %s (%s) skipped, archive is after today.",
$idSite, $period->getLabel(), $period->getPrettyString());
continue;
}
@ -627,7 +727,7 @@ class Archive
private function cacheArchiveIdsWithoutLaunching($plugins)
{
$idarchivesByReport = ArchiveSelector::getArchiveIds(
$this->params->getIdSites(), $this->params->getPeriods(), $this->params->getSegment(), $plugins, $this->params->isSkipAggregationOfSubTables());
$this->params->getIdSites(), $this->params->getPeriods(), $this->params->getSegment(), $plugins);
// initialize archive ID cache for each report
foreach ($plugins as $plugin) {
@ -655,8 +755,7 @@ class Archive
$this->params->getIdSites(),
$this->params->getSegment(),
$this->getPeriodLabel(),
$plugin,
$this->params->isSkipAggregationOfSubTables()
$plugin
);
}
@ -707,11 +806,6 @@ class Archive
return round((float)$value, 2);
}
private function uncompress($data)
{
return @gzuncompress($data);
}
/**
* Initializes the archive ID cache ($this->idarchives) for a particular 'done' flag.
*
@ -723,6 +817,8 @@ class Archive
* If this function is not called, then periods with no visits will not add
* entries to the cache. If the archive is used again, SQL will be executed to
* try and find the archive IDs even though we know there are none.
*
* @param string $doneFlag
*/
private function initializeArchiveIdCache($doneFlag)
{
@ -745,7 +841,10 @@ class Archive
*/
private function getArchiveGroupOfPlugin($plugin)
{
if ($this->getPeriodLabel() != 'range') {
$periods = $this->params->getPeriods();
$periodLabel = reset($periods)->getLabel();
if (Rules::shouldProcessReportsAllPlugins($this->params->getIdSites(), $this->params->getSegment(), $periodLabel)) {
return self::ARCHIVE_ALL_PLUGINS_FLAG;
}
@ -755,7 +854,7 @@ class Archive
/**
* Returns the name of the plugin that archives a given report.
*
* @param string $report Archive data name, eg, `'nb_visits'`, `'UserSettings_...'`, etc.
* @param string $report Archive data name, eg, `'nb_visits'`, `'DevicesDetection_...'`, etc.
* @return string Plugin name.
* @throws \Exception If a plugin cannot be found or if the plugin for the report isn't
* activated.
@ -766,9 +865,9 @@ class Archive
if (in_array($report, Metrics::getVisitsMetricNames())) {
$report = 'VisitsSummary_CoreMetrics';
} // Goal_* metrics are processed by the Goals plugin (HACK)
else if (strpos($report, 'Goal_') === 0) {
elseif (strpos($report, 'Goal_') === 0) {
$report = 'Goals_Metrics';
} else if (strrpos($report, '_returning') === strlen($report) - strlen('_returning')) { // HACK
} elseif (strrpos($report, '_returning') === strlen($report) - strlen('_returning')) { // HACK
$report = 'VisitFrequency_Metrics';
}
@ -789,7 +888,7 @@ class Archive
*/
private function prepareArchive(array $archiveGroups, Site $site, Period $period)
{
$parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment(), $this->params->isSkipAggregationOfSubTables());
$parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment());
$archiveLoader = new ArchiveProcessor\Loader($parameters);
$periodString = $period->getRangeString();
@ -801,9 +900,37 @@ class Archive
$idArchive = $archiveLoader->prepareArchive($plugin);
if($idArchive) {
if ($idArchive) {
$this->idarchives[$doneFlag][$periodString][] = $idArchive;
}
}
}
private function getIdArchivesByMonth($doneFlags)
{
// order idarchives by the table month they belong to
$idArchivesByMonth = array();
foreach (array_keys($doneFlags) as $doneFlag) {
if (empty($this->idarchives[$doneFlag])) {
continue;
}
foreach ($this->idarchives[$doneFlag] as $dateRange => $idarchives) {
foreach ($idarchives as $id) {
$idArchivesByMonth[$dateRange][] = $id;
}
}
}
return $idArchivesByMonth;
}
/**
* @internal
*/
public static function clearStaticCache()
{
self::$cache = null;
}
}

317
www/analytics/core/Archive/ArchiveInvalidator.php Normal file
View file

@ -0,0 +1,317 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archive;
use Piwik\Archive\ArchiveInvalidator\InvalidationResult;
use Piwik\CronArchive\SitesToReprocessDistributedList;
use Piwik\DataAccess\ArchiveTableCreator;
use Piwik\DataAccess\Model;
use Piwik\Date;
use Piwik\Option;
use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
use Piwik\Plugins\PrivacyManager\PrivacyManager;
use Piwik\Period;
use Piwik\Segment;
/**
* Service that can be used to invalidate archives or add archive references to a list so they will
* be invalidated later.
*
* Archives are put in an "invalidated" state by setting the done flag to `ArchiveWriter::DONE_INVALIDATED`.
* This class also adds the archive's associated site to the a distributed list and adding the archive's year month to another
* distributed list.
*
* CronArchive will reprocess the archive data for all sites in the first list, and a scheduled task
* will purge the old, invalidated data in archive tables identified by the second list.
*
* Until CronArchive, or browser triggered archiving, re-processes data for an invalidated archive, the invalidated
* archive data will still be displayed in the UI and API.
*
* ### Deferred Invalidation
*
* Invalidating archives means running queries on one or more archive tables. In some situations, like during
* tracking, this is not desired. In such cases, archive references can be added to a list via the
* rememberToInvalidateArchivedReportsLater method, which will add the reference to a distributed list
*
* Later, during Piwik's normal execution, the list will be read and every archive it references will
* be invalidated.
*/
class ArchiveInvalidator
{
private $rememberArchivedReportIdStart = 'report_to_invalidate_';
/**
* @var Model
*/
private $model;
public function __construct(Model $model)
{
$this->model = $model;
}
public function rememberToInvalidateArchivedReportsLater($idSite, Date $date)
{
$key = $this->buildRememberArchivedReportId($idSite, $date->toString());
$value = Option::get($key);
// we do not really have to get the value first. we could simply always try to call set() and it would update or
// insert the record if needed but we do not want to lock the table (especially since there are still some
// MyISAM installations)
if (false === $value) {
Option::set($key, '1');
}
}
public function getRememberedArchivedReportsThatShouldBeInvalidated()
{
$reports = Option::getLike($this->rememberArchivedReportIdStart . '%_%');
$sitesPerDay = array();
foreach ($reports as $report => $value) {
$report = str_replace($this->rememberArchivedReportIdStart, '', $report);
$report = explode('_', $report);
$siteId = (int) $report[0];
$date = $report[1];
if (empty($sitesPerDay[$date])) {
$sitesPerDay[$date] = array();
}
$sitesPerDay[$date][] = $siteId;
}
return $sitesPerDay;
}
private function buildRememberArchivedReportId($idSite, $date)
{
$id = $this->buildRememberArchivedReportIdForSite($idSite);
$id .= '_' . trim($date);
return $id;
}
private function buildRememberArchivedReportIdForSite($idSite)
{
return $this->rememberArchivedReportIdStart . (int) $idSite;
}
public function forgetRememberedArchivedReportsToInvalidateForSite($idSite)
{
$id = $this->buildRememberArchivedReportIdForSite($idSite) . '_%';
Option::deleteLike($id);
}
/**
* @internal
*/
public function forgetRememberedArchivedReportsToInvalidate($idSite, Date $date)
{
$id = $this->buildRememberArchivedReportId($idSite, $date->toString());
Option::delete($id);
}
/**
* @param $idSites int[]
* @param $dates Date[]
* @param $period string
* @param $segment Segment
* @param bool $cascadeDown
* @return InvalidationResult
* @throws \Exception
*/
public function markArchivesAsInvalidated(array $idSites, array $dates, $period, Segment $segment = null, $cascadeDown = false)
{
$invalidationInfo = new InvalidationResult();
$datesToInvalidate = $this->removeDatesThatHaveBeenPurged($dates, $invalidationInfo);
if (empty($period)) {
// if the period is empty, we don't need to cascade in any way, since we'll remove all periods
$periodDates = $this->getDatesByYearMonthAndPeriodType($dates);
} else {
$periods = $this->getPeriodsToInvalidate($datesToInvalidate, $period, $cascadeDown);
$periodDates = $this->getPeriodDatesByYearMonthAndPeriodType($periods);
}
$periodDates = $this->getUniqueDates($periodDates);
$this->markArchivesInvalidated($idSites, $periodDates, $segment);
$yearMonths = array_keys($periodDates);
$this->markInvalidatedArchivesForReprocessAndPurge($idSites, $yearMonths);
foreach ($idSites as $idSite) {
foreach ($dates as $date) {
$this->forgetRememberedArchivedReportsToInvalidate($idSite, $date);
}
}
return $invalidationInfo;
}
/**
* @param string[][][] $periodDates
* @return string[][][]
*/
private function getUniqueDates($periodDates)
{
$result = array();
foreach ($periodDates as $yearMonth => $periodsByYearMonth) {
foreach ($periodsByYearMonth as $periodType => $periods) {
$result[$yearMonth][$periodType] = array_unique($periods);
}
}
return $result;
}
/**
* @param Date[] $dates
* @param string $periodType
* @param bool $cascadeDown
* @return Period[]
*/
private function getPeriodsToInvalidate($dates, $periodType, $cascadeDown)
{
$periodsToInvalidate = array();
foreach ($dates as $date) {
if ($periodType == 'range') {
$date = $date . ',' . $date;
}
$period = Period\Factory::build($periodType, $date);
$periodsToInvalidate[] = $period;
if ($cascadeDown) {
$periodsToInvalidate = array_merge($periodsToInvalidate, $period->getAllOverlappingChildPeriods());
}
if ($periodType != 'year'
&& $periodType != 'range'
) {
$periodsToInvalidate[] = Period\Factory::build('year', $date);
}
}
return $periodsToInvalidate;
}
/**
* @param Period[] $periods
* @return string[][][]
*/
private function getPeriodDatesByYearMonthAndPeriodType($periods)
{
$result = array();
foreach ($periods as $period) {
$date = $period->getDateStart();
$periodType = $period->getId();
$yearMonth = ArchiveTableCreator::getTableMonthFromDate($date);
$result[$yearMonth][$periodType][] = $date->toString();
}
return $result;
}
/**
* Called when deleting all periods.
*
* @param Date[] $dates
* @return string[][][]
*/
private function getDatesByYearMonthAndPeriodType($dates)
{
$result = array();
foreach ($dates as $date) {
$yearMonth = ArchiveTableCreator::getTableMonthFromDate($date);
$result[$yearMonth][null][] = $date->toString();
// since we're removing all periods, we must make sure to remove year periods as well.
// this means we have to make sure the january table is processed.
$janYearMonth = $date->toString('Y') . '_01';
$result[$janYearMonth][null][] = $date->toString();
}
return $result;
}
/**
* @param int[] $idSites
* @param string[][][] $dates
* @throws \Exception
*/
private function markArchivesInvalidated($idSites, $dates, Segment $segment = null)
{
$archiveNumericTables = ArchiveTableCreator::getTablesArchivesInstalled($type = ArchiveTableCreator::NUMERIC_TABLE);
foreach ($archiveNumericTables as $table) {
$tableDate = ArchiveTableCreator::getDateFromTableName($table);
if (empty($dates[$tableDate])) {
continue;
}
$this->model->updateArchiveAsInvalidated($table, $idSites, $dates[$tableDate], $segment);
}
}
/**
* @param Date[] $dates
* @param InvalidationResult $invalidationInfo
* @return \Piwik\Date[]
*/
private function removeDatesThatHaveBeenPurged($dates, InvalidationResult $invalidationInfo)
{
$this->findOlderDateWithLogs($invalidationInfo);
$result = array();
foreach ($dates as $date) {
// we should only delete reports for dates that are more recent than N days
if ($invalidationInfo->minimumDateWithLogs
&& $date->isEarlier($invalidationInfo->minimumDateWithLogs)
) {
$invalidationInfo->warningDates[] = $date->toString();
continue;
}
$result[] = $date;
$invalidationInfo->processedDates[] = $date->toString();
}
return $result;
}
private function findOlderDateWithLogs(InvalidationResult $info)
{
// If using the feature "Delete logs older than N days"...
$purgeDataSettings = PrivacyManager::getPurgeDataSettings();
$logsDeletedWhenOlderThanDays = (int)$purgeDataSettings['delete_logs_older_than'];
$logsDeleteEnabled = $purgeDataSettings['delete_logs_enable'];
if ($logsDeleteEnabled
&& $logsDeletedWhenOlderThanDays
) {
$info->minimumDateWithLogs = Date::factory('today')->subDay($logsDeletedWhenOlderThanDays);
}
}
/**
* @param array $idSites
* @param array $yearMonths
*/
private function markInvalidatedArchivesForReprocessAndPurge(array $idSites, $yearMonths)
{
$store = new SitesToReprocessDistributedList();
$store->add($idSites);
$archivesToPurge = new ArchivesToPurgeDistributedList();
$archivesToPurge->add($yearMonths);
}
}

56
www/analytics/core/Archive/ArchiveInvalidator/InvalidationResult.php Normal file
View file

@ -0,0 +1,56 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Archive\ArchiveInvalidator;
use Piwik\Date;
/**
* Information about the result of an archive invalidation operation.
*/
class InvalidationResult
{
/**
* Dates that couldn't be invalidated because they are earlier than the configured log
* deletion limit.
*
* @var array
*/
public $warningDates = array();
/**
* Dates that were successfully invalidated.
*
* @var array
*/
public $processedDates = array();
/**
* The day of the oldest log entry.
*
* @var Date|bool
*/
public $minimumDateWithLogs = false;
/**
* @return string[]
*/
public function makeOutputLogs()
{
$output = array();
if ($this->warningDates) {
$output[] = 'Warning: the following Dates have not been invalidated, because they are earlier than your Log Deletion limit: ' .
implode(", ", $this->warningDates) .
"\n The last day with logs is " . $this->minimumDateWithLogs . ". " .
"\n Please disable 'Delete old Logs' or set it to a higher deletion threshold (eg. 180 days or 365 years).'.";
}
$output[] = "Success. The following dates were invalidated successfully: " . implode(", ", $this->processedDates);
return $output;
}
}

272
www/analytics/core/Archive/ArchivePurger.php Normal file
View file

@ -0,0 +1,272 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archive;
use Piwik\ArchiveProcessor\Rules;
use Piwik\Config;
use Piwik\Container\StaticContainer;
use Piwik\DataAccess\ArchiveTableCreator;
use Piwik\DataAccess\Model;
use Piwik\Date;
use Piwik\Piwik;
use Psr\Log\LoggerInterface;
use Psr\Log\LogLevel;
/**
* Service that purges temporary, error-ed, invalid and custom range archives from archive tables.
*
* Temporary archives are purged if they were archived before a specific time. The time is dependent
* on whether browser triggered archiving is enabled or not.
*
* Error-ed archives are purged w/o constraint.
*
* Invalid archives are purged if a new, valid, archive exists w/ the same site, date, period combination.
* Archives are marked as invalid via Piwik\Archive\ArchiveInvalidator.
*/
class ArchivePurger
{
/**
* @var Model
*/
private $model;
/**
* Date threshold for purging custom range archives. Archives that are older than this date
* are purged unconditionally from the requested archive table.
*
* @var Date
*/
private $purgeCustomRangesOlderThan;
/**
* Date to use for 'yesterday'. Exists so tests can override this value.
*
* @var Date
*/
private $yesterday;
/**
* Date to use for 'today'. Exists so tests can override this value.
*
* @var $today
*/
private $today;
/**
* Date to use for 'now'. Exists so tests can override this value.
*
* @var int
*/
private $now;
/**
* @var LoggerInterface
*/
private $logger;
public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null, LoggerInterface $logger = null)
{
$this->model = $model ?: new Model();
$this->purgeCustomRangesOlderThan = $purgeCustomRangesOlderThan ?: self::getDefaultCustomRangeToPurgeAgeThreshold();
$this->yesterday = Date::factory('yesterday');
$this->today = Date::factory('today');
$this->now = time();
$this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
}
/**
* Purge all invalidate archives for whom there are newer, valid archives from the archive
* table that stores data for `$date`.
*
* @param Date $date The date identifying the archive table.
* @return int The total number of archive rows deleted (from both the blog & numeric tables).
*/
public function purgeInvalidatedArchivesFrom(Date $date)
{
$numericTable = ArchiveTableCreator::getNumericTable($date);
// we don't want to do an INNER JOIN on every row in a archive table that can potentially have tens to hundreds of thousands of rows,
// so we first look for sites w/ invalidated archives, and use this as a constraint in getInvalidatedArchiveIdsSafeToDelete() below.
// the constraint will hit an INDEX and speed up the inner join that happens in getInvalidatedArchiveIdsSafeToDelete().
$idSites = $this->model->getSitesWithInvalidatedArchive($numericTable);
if (empty($idSites)) {
$this->logger->debug("No sites with invalidated archives found in {table}.", array('table' => $numericTable));
return 0;
}
$archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
if (empty($archiveIds)) {
$this->logger->debug("No invalidated archives found in {table} with newer, valid archives.", array('table' => $numericTable));
return 0;
}
$this->logger->info("Found {countArchiveIds} invalidated archives safe to delete in {table}.", array(
'table' => $numericTable, 'countArchiveIds' => count($archiveIds)
));
$deletedRowCount = $this->deleteArchiveIds($date, $archiveIds);
$this->logger->debug("Deleted {count} rows in {table} and its associated blob table.", array(
'table' => $numericTable, 'count' => $deletedRowCount
));
return $deletedRowCount;
}
/**
* Removes the outdated archives for the given month.
* (meaning they are marked with a done flag of ArchiveWriter::DONE_OK_TEMPORARY or ArchiveWriter::DONE_ERROR)
*
* @param Date $dateStart Only the month will be used
* @return int Returns the total number of rows deleted.
*/
public function purgeOutdatedArchives(Date $dateStart)
{
$purgeArchivesOlderThan = $this->getOldestTemporaryArchiveToKeepThreshold();
$deletedRowCount = 0;
$idArchivesToDelete = $this->getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan);
if (!empty($idArchivesToDelete)) {
$deletedRowCount = $this->deleteArchiveIds($dateStart, $idArchivesToDelete);
$this->logger->info("Deleted {count} rows in archive tables (numeric + blob) for {date}.", array(
'count' => $deletedRowCount,
'date' => $dateStart
));
} else {
$this->logger->debug("No outdated archives found in archive numeric table for {date}.", array('date' => $dateStart));
}
$this->logger->debug("Purging temporary archives: done [ purged archives older than {date} in {yearMonth} ] [Deleted IDs: {deletedIds}]", array(
'date' => $purgeArchivesOlderThan,
'yearMonth' => $dateStart->toString('Y-m'),
'deletedIds' => implode(',', $idArchivesToDelete)
));
return $deletedRowCount;
}
protected function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan)
{
$archiveTable = ArchiveTableCreator::getNumericTable($date);
$result = $this->model->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan);
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
}
return $idArchivesToDelete;
}
/**
* Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space.
*
* @param $date Date
* @return int The total number of rows deleted from both the numeric & blob table.
*/
public function purgeArchivesWithPeriodRange(Date $date)
{
$numericTable = ArchiveTableCreator::getNumericTable($date);
$blobTable = ArchiveTableCreator::getBlobTable($date);
$deletedCount = $this->model->deleteArchivesWithPeriod(
$numericTable, $blobTable, Piwik::$idPeriods['range'], $this->purgeCustomRangesOlderThan);
$level = $deletedCount == 0 ? LogLevel::DEBUG : LogLevel::INFO;
$this->logger->log($level, "Purged {count} range archive rows from {numericTable} & {blobTable}.", array(
'count' => $deletedCount,
'numericTable' => $numericTable,
'blobTable' => $blobTable
));
$this->logger->debug(" [ purged archives older than {threshold} ]", array('threshold' => $this->purgeCustomRangesOlderThan));
return $deletedCount;
}
/**
* Deletes by batches Archive IDs in the specified month,
*
* @param Date $date
* @param $idArchivesToDelete
* @return int Number of rows deleted from both numeric + blob table.
*/
protected function deleteArchiveIds(Date $date, $idArchivesToDelete)
{
$batches = array_chunk($idArchivesToDelete, 1000);
$numericTable = ArchiveTableCreator::getNumericTable($date);
$blobTable = ArchiveTableCreator::getBlobTable($date);
$deletedCount = 0;
foreach ($batches as $idsToDelete) {
$deletedCount += $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete);
}
return $deletedCount;
}
/**
* Returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged.
*
* @return int|bool Outdated archives older than this timestamp should be purged
*/
protected function getOldestTemporaryArchiveToKeepThreshold()
{
$temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive();
if (Rules::isBrowserTriggerEnabled()) {
// If Browser Archiving is enabled, it is likely there are many more temporary archives
// We delete more often which is safe, since reports are re-processed on demand
return Date::factory($this->now - 2 * $temporaryArchivingTimeout)->getDateTime();
}
// If cron core:archive command is building the reports, we should keep all temporary reports from today
return $this->yesterday->getDateTime();
}
private static function getDefaultCustomRangeToPurgeAgeThreshold()
{
$daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days'];
return Date::factory('today')->subDay($daysRangesValid)->getDateTime();
}
/**
* For tests.
*
* @param Date $yesterday
*/
public function setYesterdayDate(Date $yesterday)
{
$this->yesterday = $yesterday;
}
/**
* For tests.
*
* @param Date $today
*/
public function setTodayDate(Date $today)
{
$this->today = $today;
}
/**
* For tests.
*
* @param int $now
*/
public function setNow($now)
{
$this->now = $now;
}
}

144
www/analytics/core/Archive/Chunk.php Normal file
View file

@ -0,0 +1,144 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archive;
use Piwik\DataTable;
/**
* This class is used to split blobs of DataTables into chunks. Each blob used to be stored under one blob in the
* archive table. For better efficiency we do now combine multiple DataTable into one blob entry.
*
* Chunks are identified by having the recordName $recordName_chunk_0_99, $recordName_chunk_100_199 (this chunk stores
* the subtable 100-199).
*/
class Chunk
{
const ARCHIVE_APPENDIX_SUBTABLES = 'chunk';
const NUM_TABLES_IN_CHUNK = 100;
/**
* Get's the record name to use for a given tableId/subtableId.
*
* @param string $recordName eg 'Actions_ActionsUrl'
* @param int $tableId eg '5' for tableId '5'
* @return string eg 'Actions_ActionsUrl_chunk_0_99' as the table should be stored under this blob id.
*/
public function getRecordNameForTableId($recordName, $tableId)
{
$chunk = (floor($tableId / self::NUM_TABLES_IN_CHUNK));
$start = $chunk * self::NUM_TABLES_IN_CHUNK;
$end = $start + self::NUM_TABLES_IN_CHUNK - 1;
return $recordName . $this->getAppendix() . $start . '_' . $end;
}
/**
* Moves the given blobs into chunks and assigns a proper record name containing the chunk number.
*
* @param string $recordName The original archive record name, eg 'Actions_ActionsUrl'
* @param array $blobs An array containg a mapping of tableIds to blobs. Eg array(0 => 'blob', 1 => 'subtableBlob', ...)
* @return array An array where each blob is moved into a chunk, indexed by recordNames.
* eg array('Actions_ActionsUrl_chunk_0_99' => array(0 => 'blob', 1 => 'subtableBlob', ...),
* 'Actions_ActionsUrl_chunk_100_199' => array(...))
*/
public function moveArchiveBlobsIntoChunks($recordName, $blobs)
{
$chunks = array();
foreach ($blobs as $tableId => $blob) {
$name = $this->getRecordNameForTableId($recordName, $tableId);
if (!array_key_exists($name, $chunks)) {
$chunks[$name] = array();
}
$chunks[$name][$tableId] = $blob;
}
return $chunks;
}
/**
* Detects whether a recordName like 'Actions_ActionUrls_chunk_0_99' or 'Actions_ActionUrls' belongs to a
* chunk or not.
*
* To be a valid recordName that belongs to a chunk it must end with '_chunk_NUMERIC_NUMERIC'.
*
* @param string $recordName
* @return bool
*/
public function isRecordNameAChunk($recordName)
{
$posAppendix = $this->getEndPosOfChunkAppendix($recordName);
if (false === $posAppendix) {
return false;
}
// will contain "0_99" of "chunk_0_99"
$blobId = substr($recordName, $posAppendix);
return $this->isChunkRange($blobId);
}
private function isChunkRange($blobId)
{
$blobId = explode('_', $blobId);
return 2 === count($blobId) && is_numeric($blobId[0]) && is_numeric($blobId[1]);
}
/**
* When having a record like 'Actions_ActionUrls_chunk_0_99" it will return the raw recordName 'Actions_ActionUrls'.
*
* @param string $recordName
* @return string
*/
public function getRecordNameWithoutChunkAppendix($recordName)
{
if (!$this->isRecordNameAChunk($recordName)) {
return $recordName;
}
$posAppendix = $this->getStartPosOfChunkAppendix($recordName);
if (false === $posAppendix) {
return $recordName;
}
return substr($recordName, 0, $posAppendix);
}
/**
* Returns the string that is appended to the original record name. This appendix identifes a record name is a
* chunk.
* @return string
*/
public function getAppendix()
{
return '_' . self::ARCHIVE_APPENDIX_SUBTABLES . '_';
}
private function getStartPosOfChunkAppendix($recordName)
{
return strpos($recordName, $this->getAppendix());
}
private function getEndPosOfChunkAppendix($recordName)
{
$pos = strpos($recordName, $this->getAppendix());
if ($pos === false) {
return false;
}
return $pos + strlen($this->getAppendix());
}
}

49
www/analytics/core/Archive/DataCollection.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -137,6 +137,21 @@ class DataCollection
return $this->data[$idSite][$period];
}
/**
* Set data for a specific site & period. If there is no data for the given site ID & period,
* it is set to the default row.
*
* @param int $idSite
* @param string $period eg, '2012-01-01,2012-01-31'
* @param string $name eg 'nb_visits'
* @param string $value eg 5
*/
public function set($idSite, $period, $name, $value)
{
$row = & $this->get($idSite, $period);
$row[$name] = $value;
}
/**
* Adds a new metadata to the data for specific site & period. If there is no
* data for the given site ID & period, it is set to the default row.
@ -188,6 +203,7 @@ class DataCollection
$this->putRowInIndex($result, $indexKeys, $row, $idSite, $period);
}
}
return $result;
}
@ -208,9 +224,27 @@ class DataCollection
$this->dataNames, $this->dataType, $this->sitesId, $this->periods, $this->defaultRow);
$index = $this->getIndexedArray($resultIndices);
return $dataTableFactory->make($index, $resultIndices);
}
/**
* See {@link DataTableFactory::makeMerged()}
*
* @param array $resultIndices
* @return DataTable|DataTable\Map
* @throws Exception
*/
public function getMergedDataTable($resultIndices)
{
$dataTableFactory = new DataTableFactory(
$this->dataNames, $this->dataType, $this->sitesId, $this->periods, $this->defaultRow);
$index = $this->getIndexedArray($resultIndices);
return $dataTableFactory->makeMerged($index, $resultIndices);
}
/**
* Returns archive data as a DataTable indexed by metadata. Indexed data will
* be represented by Map instances. Each DataTable will have
@ -249,6 +283,7 @@ class DataCollection
$dataTableFactory->useSubtable($idSubTable);
$index = $this->getIndexedArray($resultIndices);
return $dataTableFactory->make($index, $resultIndices);
}
@ -296,12 +331,16 @@ class DataCollection
if ($metadataName == DataTableFactory::TABLE_METADATA_SITE_INDEX) {
$indexKeyValues = array_values($this->sitesId);
} else if ($metadataName == DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
} elseif ($metadataName == DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
$indexKeyValues = array_keys($this->periods);
}
foreach ($indexKeyValues as $key) {
$result[$key] = $this->createOrderedIndex($metadataNamesToIndexBy);
if (empty($metadataNamesToIndexBy)) {
$result = array_fill_keys($indexKeyValues, array());
} else {
foreach ($indexKeyValues as $key) {
$result[$key] = $this->createOrderedIndex($metadataNamesToIndexBy);
}
}
}
@ -318,7 +357,7 @@ class DataCollection
foreach ($metadataNamesToIndexBy as $metadataName) {
if ($metadataName == DataTableFactory::TABLE_METADATA_SITE_INDEX) {
$key = $idSite;
} else if ($metadataName == DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
} elseif ($metadataName == DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
$key = $period;
} else {
$key = $row[self::METADATA_CONTAINER_ROW_KEY][$metadataName];

243
www/analytics/core/Archive/DataTableFactory.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -95,6 +95,23 @@ class DataTableFactory
$this->defaultRow = $defaultRow;
}
/**
* Returns the ID of the site a table is related to based on the 'site' metadata entry,
* or null if there is none.
*
* @param DataTable $table
* @return int|null
*/
public static function getSiteIdFromMetadata(DataTable $table)
{
$site = $table->getMetadata('site');
if (empty($site)) {
return null;
} else {
return $site->getId();
}
}
/**
* Tells the factory instance to expand the DataTables that are created by
* creating subtables and setting the subtable IDs of rows w/ subtables correctly.
@ -128,6 +145,11 @@ class DataTableFactory
$this->idSubtable = $idSubtable;
}
private function isNumericDataType()
{
return $this->dataType == 'numeric';
}
/**
* Creates a DataTable|Set instance using an index of
* archive data.
@ -139,21 +161,63 @@ class DataTableFactory
*/
public function make($index, $resultIndices)
{
$keyMetadata = $this->getDefaultMetadata();
if (empty($resultIndices)) {
// for numeric data, if there's no index (and thus only 1 site & period in the query),
// we want to display every queried metric name
if (empty($index)
&& $this->dataType == 'numeric'
&& $this->isNumericDataType()
) {
$index = $this->defaultRow;
}
$dataTable = $this->createDataTable($index, $keyMetadata = array());
$dataTable = $this->createDataTable($index, $keyMetadata);
} else {
$dataTable = $this->createDataTableMapFromIndex($index, $resultIndices, $keyMetadata = array());
$dataTable = $this->createDataTableMapFromIndex($index, $resultIndices, $keyMetadata);
}
return $dataTable;
}
/**
* Creates a merged DataTable|Map instance using an index of archive data similar to {@link make()}.
*
* Whereas {@link make()} creates a Map for each result index (period and|or site), this will only create a Map
* for a period result index and move all site related indices into one dataTable. This is the same as doing
* `$dataTableFactory->make()->mergeChildren()` just much faster. It is mainly useful for reports across many sites
* eg `MultiSites.getAll`. Was done as part of https://github.com/piwik/piwik/issues/6809
*
* @param array $index @see DataCollection
* @param array $resultIndices an array mapping metadata names with pretty metadata labels.
*
* @return DataTable|DataTable\Map
* @throws \Exception
*/
public function makeMerged($index, $resultIndices)
{
if (!$this->isNumericDataType()) {
throw new \Exception('This method is supposed to work with non-numeric data types but it is not tested. To use it, remove this exception and write tests to be sure it works.');
}
$hasSiteIndex = isset($resultIndices[self::TABLE_METADATA_SITE_INDEX]);
$hasPeriodIndex = isset($resultIndices[self::TABLE_METADATA_PERIOD_INDEX]);
$isNumeric = $this->isNumericDataType();
// to be backwards compatible use a Simple table if needed as it will be formatted differently
$useSimpleDataTable = !$hasSiteIndex && $isNumeric;
if (!$hasSiteIndex) {
$firstIdSite = reset($this->sitesId);
$index = array($firstIdSite => $index);
}
if ($hasPeriodIndex) {
$dataTable = $this->makeMergedTableWithPeriodAndSiteIndex($index, $resultIndices, $useSimpleDataTable, $isNumeric);
} else {
$dataTable = $this->makeMergedWithSiteIndex($index, $useSimpleDataTable, $isNumeric);
}
$this->transformMetadata($dataTable);
return $dataTable;
}
@ -171,16 +235,16 @@ class DataTableFactory
* @param array $blobRow
* @return DataTable|DataTable\Map
*/
private function makeFromBlobRow($blobRow)
private function makeFromBlobRow($blobRow, $keyMetadata)
{
if ($blobRow === false) {
return new DataTable();
}
if (count($this->dataNames) === 1) {
return $this->makeDataTableFromSingleBlob($blobRow);
return $this->makeDataTableFromSingleBlob($blobRow, $keyMetadata);
} else {
return $this->makeIndexedByRecordNameDataTable($blobRow);
return $this->makeIndexedByRecordNameDataTable($blobRow, $keyMetadata);
}
}
@ -192,7 +256,7 @@ class DataTableFactory
* @param array $blobRow
* @return DataTable
*/
private function makeDataTableFromSingleBlob($blobRow)
private function makeDataTableFromSingleBlob($blobRow, $keyMetadata)
{
$recordName = reset($this->dataNames);
if ($this->idSubtable !== null) {
@ -206,7 +270,7 @@ class DataTableFactory
}
// set table metadata
$table->setMetadataValues(DataCollection::getDataRowMetadata($blobRow));
$table->setAllTableMetadata(array_merge(DataCollection::getDataRowMetadata($blobRow), $keyMetadata));
if ($this->expandDataTable) {
$table->enableRecursiveFilters();
@ -223,12 +287,12 @@ class DataTableFactory
* @param array $blobRow
* @return DataTable\Map
*/
private function makeIndexedByRecordNameDataTable($blobRow)
private function makeIndexedByRecordNameDataTable($blobRow, $keyMetadata)
{
$table = new DataTable\Map();
$table->setKeyName('recordName');
$tableMetadata = DataCollection::getDataRowMetadata($blobRow);
$tableMetadata = array_merge(DataCollection::getDataRowMetadata($blobRow), $keyMetadata);
foreach ($blobRow as $name => $blob) {
$newTable = DataTable::fromSerializedArray($blob);
@ -248,23 +312,23 @@ class DataTableFactory
* @param array $keyMetadata The metadata to add to the table when it's created.
* @return DataTable\Map
*/
private function createDataTableMapFromIndex($index, $resultIndices, $keyMetadata = array())
private function createDataTableMapFromIndex($index, $resultIndices, $keyMetadata)
{
$resultIndexLabel = reset($resultIndices);
$result = new DataTable\Map();
$result->setKeyName(reset($resultIndices));
$resultIndex = key($resultIndices);
array_shift($resultIndices);
$result = new DataTable\Map();
$result->setKeyName($resultIndexLabel);
$hasIndices = !empty($resultIndices);
foreach ($index as $label => $value) {
$keyMetadata[$resultIndex] = $label;
$keyMetadata[$resultIndex] = $this->createTableIndexMetadata($resultIndex, $label);
if (empty($resultIndices)) {
$newTable = $this->createDataTable($value, $keyMetadata);
} else {
if ($hasIndices) {
$newTable = $this->createDataTableMapFromIndex($value, $resultIndices, $keyMetadata);
} else {
$newTable = $this->createDataTable($value, $keyMetadata);
}
$result->addTable($newTable, $this->prettifyIndexLabel($resultIndex, $label));
@ -273,6 +337,15 @@ class DataTableFactory
return $result;
}
private function createTableIndexMetadata($resultIndex, $label)
{
if ($resultIndex === DataTableFactory::TABLE_METADATA_SITE_INDEX) {
return new Site($label);
} elseif ($resultIndex === DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
return $this->periods[$label];
}
}
/**
* Creates a DataTable instance from an index row.
*
@ -283,11 +356,11 @@ class DataTableFactory
private function createDataTable($data, $keyMetadata)
{
if ($this->dataType == 'blob') {
$result = $this->makeFromBlobRow($data);
$result = $this->makeFromBlobRow($data, $keyMetadata);
} else {
$result = $this->makeFromMetricsArray($data);
$result = $this->makeFromMetricsArray($data, $keyMetadata);
}
$this->setTableMetadata($keyMetadata, $result);
return $result;
}
@ -307,7 +380,7 @@ class DataTableFactory
&& $treeLevel >= $this->maxSubtableDepth
) {
// unset the subtables so DataTableManager doesn't throw
foreach ($dataTable->getRows() as $row) {
foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
$row->removeSubtable();
}
@ -316,7 +389,7 @@ class DataTableFactory
$dataName = reset($this->dataNames);
foreach ($dataTable->getRows() as $row) {
foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
$sid = $row->getIdSubDataTable();
if ($sid === null) {
continue;
@ -340,17 +413,12 @@ class DataTableFactory
}
}
/**
* Converts site IDs and period string ranges into Site instances and
* Period instances in DataTable metadata.
*/
private function transformMetadata($table)
private function getDefaultMetadata()
{
$periods = $this->periods;
$table->filter(function ($table) use ($periods) {
$table->setMetadata(DataTableFactory::TABLE_METADATA_SITE_INDEX, new Site($table->getMetadata(DataTableFactory::TABLE_METADATA_SITE_INDEX)));
$table->setMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX, $periods[$table->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX)]);
});
return array(
DataTableFactory::TABLE_METADATA_SITE_INDEX => new Site(reset($this->sitesId)),
DataTableFactory::TABLE_METADATA_PERIOD_INDEX => reset($this->periods),
);
}
/**
@ -368,39 +436,16 @@ class DataTableFactory
return $label;
}
/**
* @param $keyMetadata
* @param $result
*/
private function setTableMetadata($keyMetadata, $result)
{
if (!isset($keyMetadata[DataTableFactory::TABLE_METADATA_SITE_INDEX])) {
$keyMetadata[DataTableFactory::TABLE_METADATA_SITE_INDEX] = reset($this->sitesId);
}
if (!isset($keyMetadata[DataTableFactory::TABLE_METADATA_PERIOD_INDEX])) {
reset($this->periods);
$keyMetadata[DataTableFactory::TABLE_METADATA_PERIOD_INDEX] = key($this->periods);
}
// Note: $result can be a DataTable\Map
$result->filter(function ($table) use ($keyMetadata) {
foreach ($keyMetadata as $name => $value) {
$table->setMetadata($name, $value);
}
});
}
/**
* @param $data
* @return DataTable\Simple
*/
private function makeFromMetricsArray($data)
private function makeFromMetricsArray($data, $keyMetadata)
{
$table = new DataTable\Simple();
if (!empty($data)) {
$table->setAllTableMetadata(DataCollection::getDataRowMetadata($data));
$table->setAllTableMetadata(array_merge(DataCollection::getDataRowMetadata($data), $keyMetadata));
DataCollection::removeMetadataFromDataRow($data);
@ -412,15 +457,89 @@ class DataTableFactory
// w/o this code, an empty array would be created, and other parts of Piwik
// would break.
if (count($this->dataNames) == 1
&& $this->dataType == 'numeric'
&& $this->isNumericDataType()
) {
$name = reset($this->dataNames);
$table->addRow(new Row(array(Row::COLUMNS => array($name => 0))));
}
$table->setAllTableMetadata($keyMetadata);
}
$result = $table;
return $result;
}
}
private function makeMergedTableWithPeriodAndSiteIndex($index, $resultIndices, $useSimpleDataTable, $isNumeric)
{
$map = new DataTable\Map();
$map->setKeyName($resultIndices[self::TABLE_METADATA_PERIOD_INDEX]);
// we save all tables of the map in this array to be able to add rows fast
$tables = array();
foreach ($this->periods as $range => $period) {
// as the resulting table is "merged", we do only set Period metedata and no metadata for site. Instead each
// row will have an idsite metadata entry.
$metadata = array(self::TABLE_METADATA_PERIOD_INDEX => $period);
if ($useSimpleDataTable) {
$table = new DataTable\Simple();
} else {
$table = new DataTable();
}
$table->setAllTableMetadata($metadata);
$map->addTable($table, $this->prettifyIndexLabel(self::TABLE_METADATA_PERIOD_INDEX, $range));
$tables[$range] = $table;
}
foreach ($index as $idsite => $table) {
$rowMeta = array('idsite' => $idsite);
foreach ($table as $range => $row) {
if (!empty($row)) {
$tables[$range]->addRow(new Row(array(
Row::COLUMNS => $row,
Row::METADATA => $rowMeta)
));
} elseif ($isNumeric) {
$tables[$range]->addRow(new Row(array(
Row::COLUMNS => $this->defaultRow,
Row::METADATA => $rowMeta)
));
}
}
}
return $map;
}
private function makeMergedWithSiteIndex($index, $useSimpleDataTable, $isNumeric)
{
if ($useSimpleDataTable) {
$table = new DataTable\Simple();
} else {
$table = new DataTable();
}
$table->setAllTableMetadata(array(DataTableFactory::TABLE_METADATA_PERIOD_INDEX => reset($this->periods)));
foreach ($index as $idsite => $row) {
if (!empty($row)) {
$table->addRow(new Row(array(
Row::COLUMNS => $row,
Row::METADATA => array('idsite' => $idsite))
));
} elseif ($isNumeric) {
$table->addRow(new Row(array(
Row::COLUMNS => $this->defaultRow,
Row::METADATA => array('idsite' => $idsite))
));
}
}
return $table;
}
}

18
www/analytics/core/Archive/Parameters.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,7 +9,6 @@
namespace Piwik\Archive;
use Exception;
use Piwik\Period;
use Piwik\Segment;
@ -36,22 +35,16 @@ class Parameters
*/
private $segment;
/**
* @var bool
*/
private $skipAggregationOfSubTables;
public function getSegment()
{
return $this->segment;
}
public function __construct($idSites, $periods, Segment $segment, $skipAggregationOfSubTables)
public function __construct($idSites, $periods, Segment $segment)
{
$this->idSites = $idSites;
$this->periods = $periods;
$this->segment = $segment;
$this->skipAggregationOfSubTables = $skipAggregationOfSubTables;
}
public function getPeriods()
@ -63,11 +56,4 @@ class Parameters
{
return $this->idSites;
}
public function isSkipAggregationOfSubTables()
{
return $this->skipAggregationOfSubTables;
}
}

260
www/analytics/core/ArchiveProcessor.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,82 +9,80 @@
namespace Piwik;
use Exception;
use Piwik\Archive\DataTableFactory;
use Piwik\ArchiveProcessor\Parameters;
use Piwik\ArchiveProcessor\Rules;
use Piwik\DataAccess\ArchiveWriter;
use Piwik\DataAccess\LogAggregator;
use Piwik\DataTable\Manager;
use Piwik\DataTable\Map;
use Piwik\DataTable\Row;
use Piwik\Db;
use Piwik\Period;
/**
* Used by {@link Piwik\Plugin\Archiver} instances to insert and aggregate archive data.
*
*
* ### See also
*
*
* - **{@link Piwik\Plugin\Archiver}** - to learn how plugins should implement their own analytics
* aggregation logic.
* - **{@link Piwik\DataAccess\LogAggregator}** - to learn how plugins can perform data aggregation
* across Piwik's log tables.
*
*
* ### Examples
*
*
* **Inserting numeric data**
*
*
* // function in an Archiver descendant
* public function aggregateDayReport()
* {
* $archiveProcessor = $this->getProcessor();
*
*
* $myFancyMetric = // ... calculate the metric value ...
* $archiveProcessor->insertNumericRecord('MyPlugin_myFancyMetric', $myFancyMetric);
* }
*
*
* **Inserting serialized DataTables**
*
*
* // function in an Archiver descendant
* public function aggregateDayReport()
* {
* $archiveProcessor = $this->getProcessor();
*
*
* $maxRowsInTable = Config::getInstance()->General['datatable_archiving_maximum_rows_standard'];j
*
*
* $dataTable = // ... build by aggregating visits ...
* $serializedData = $dataTable->getSerialized($maxRowsInTable, $maxRowsInSubtable = $maxRowsInTable,
* $columnToSortBy = Metrics::INDEX_NB_VISITS);
*
*
* $archiveProcessor->insertBlobRecords('MyPlugin_myFancyReport', $serializedData);
* }
*
*
* **Aggregating archive data**
*
*
* // function in Archiver descendant
* public function aggregateMultipleReports()
* {
* $archiveProcessor = $this->getProcessor();
*
*
* // aggregate a metric
* $archiveProcessor->aggregateNumericMetrics('MyPlugin_myFancyMetric');
* $archiveProcessor->aggregateNumericMetrics('MyPlugin_mySuperFancyMetric', 'max');
*
* // aggregate a report
*
* // aggregate a report
* $archiveProcessor->aggregateDataTableRecords('MyPlugin_myFancyReport');
* }
*
*
*/
class ArchiveProcessor
{
/**
* @var \Piwik\DataAccess\ArchiveWriter
*/
protected $archiveWriter;
private $archiveWriter;
/**
* @var \Piwik\DataAccess\LogAggregator
*/
protected $logAggregator;
private $logAggregator;
/**
* @var Archive
@ -94,28 +92,41 @@ class ArchiveProcessor
/**
* @var Parameters
*/
protected $params;
private $params;
/**
* @var int
*/
protected $numberOfVisits = false;
protected $numberOfVisitsConverted = false;
private $numberOfVisits = false;
public function __construct(Parameters $params, ArchiveWriter $archiveWriter)
private $numberOfVisitsConverted = false;
/**
* If true, unique visitors are not calculated when we are aggregating data for multiple sites.
* The `[General] enable_processing_unique_visitors_multiple_sites` INI config option controls
* the value of this variable.
*
* @var bool
*/
private $skipUniqueVisitorsCalculationForMultipleSites = true;
public function __construct(Parameters $params, ArchiveWriter $archiveWriter, LogAggregator $logAggregator)
{
$this->params = $params;
$this->logAggregator = new LogAggregator($params);
$this->logAggregator = $logAggregator;
$this->archiveWriter = $archiveWriter;
$this->skipUniqueVisitorsCalculationForMultipleSites = Rules::shouldSkipUniqueVisitorsCalculationForMultipleSites();
}
protected function getArchive()
{
if(empty($this->archive)) {
if (empty($this->archive)) {
$subPeriods = $this->params->getSubPeriods();
$idSites = $this->params->getIdSites();
$idSites = $this->params->getIdSites();
$this->archive = Archive::factory($this->params->getSegment(), $subPeriods, $idSites);
}
return $this->archive;
}
@ -155,7 +166,8 @@ class ArchiveProcessor
* @var array
*/
protected static $columnsToRenameAfterAggregation = array(
Metrics::INDEX_NB_UNIQ_VISITORS => Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS
Metrics::INDEX_NB_UNIQ_VISITORS => Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS,
Metrics::INDEX_NB_USERS => Metrics::INDEX_SUM_DAILY_NB_USERS,
);
/**
@ -172,8 +184,11 @@ class ArchiveProcessor
* @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names
* when summed because they cannot be summed, eg,
* `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`.
* @param bool|array $countRowsRecursive if set to true, will calculate the recursive rows count for all record names
* which makes it slower. If you only need it for some records pass an array of
* recordNames that defines for which ones you need a recursive row count.
* @return array Returns the row counts of each aggregated report before truncation, eg,
*
*
* array(
* 'report1' => array('level0' => $report1->getRowsCount,
* 'recursive' => $report1->getRowsCountRecursive()),
@ -188,25 +203,23 @@ class ArchiveProcessor
$maximumRowsInSubDataTable = null,
$columnToSortByBeforeTruncation = null,
&$columnsAggregationOperation = null,
$columnsToRenameAfterAggregation = null)
$columnsToRenameAfterAggregation = null,
$countRowsRecursive = true)
{
if (!is_array($recordNames)) {
$recordNames = array($recordNames);
}
$nameToCount = array();
foreach ($recordNames as $recordName) {
$latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
$table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation);
$rowsCount = $table->getRowsCount();
$nameToCount[$recordName]['level0'] = $rowsCount;
$rowsCountRecursive = $rowsCount;
if($this->isAggregateSubTables()) {
$rowsCountRecursive = $table->getRowsCountRecursive();
$nameToCount[$recordName]['level0'] = $table->getRowsCount();
if ($countRowsRecursive === true || (is_array($countRowsRecursive) && in_array($recordName, $countRowsRecursive))) {
$nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
}
$nameToCount[$recordName]['recursive'] = $rowsCountRecursive;
$blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
Common::destroy($table);
@ -228,12 +241,12 @@ class ArchiveProcessor
* @return array|int Returns the array of aggregate values. If only one metric was aggregated,
* the aggregate value will be returned as is, not in an array.
* For example, if `array('nb_visits', 'nb_hits')` is supplied for `$columns`,
*
*
* array(
* 'nb_visits' => 3040,
* 'nb_hits' => 405
* )
*
*
* could be returned. If `array('nb_visits')` or `'nb_visits'` is used for `$columns`,
* then `3040` would be returned.
* @api
@ -242,7 +255,8 @@ class ArchiveProcessor
{
$metrics = $this->getAggregatedNumericMetrics($columns, $operationToApply);
foreach($metrics as $column => $value) {
foreach ($metrics as $column => $value) {
$value = Common::forceDotAsSeparatorForDecimalPoint($value);
$this->archiveWriter->insertRecord($column, $value);
}
// if asked for only one field to sum
@ -256,7 +270,7 @@ class ArchiveProcessor
public function getNumberOfVisits()
{
if($this->numberOfVisits === false) {
if ($this->numberOfVisits === false) {
throw new Exception("visits should have been set here");
}
return $this->numberOfVisits;
@ -273,7 +287,7 @@ class ArchiveProcessor
*
* @param array $numericRecords A name-value mapping of numeric values that should be
* archived, eg,
*
*
* array('Referrers_distinctKeywords' => 23, 'Referrers_distinctCampaigns' => 234)
* @api
*/
@ -297,6 +311,8 @@ class ArchiveProcessor
public function insertNumericRecord($name, $value)
{
$value = round($value, 2);
$value = Common::forceDotAsSeparatorForDecimalPoint($value);
$this->archiveWriter->insertRecord($name, $value);
}
@ -328,20 +344,53 @@ class ArchiveProcessor
*/
protected function aggregateDataTableRecord($name, $columnsAggregationOperation = null, $columnsToRenameAfterAggregation = null)
{
if($this->isAggregateSubTables()) {
// By default we shall aggregate all sub-tables.
$dataTable = $this->getArchive()->getDataTableExpanded($name, $idSubTable = null, $depth = null, $addMetadataSubtableId = false);
} else {
// In some cases (eg. Actions plugin when period=range),
// for better performance we will only aggregate the parent table
$dataTable = $this->getArchive()->getDataTable($name, $idSubTable = null);
// By default we shall aggregate all sub-tables.
$dataTable = $this->getArchive()->getDataTableExpanded($name, $idSubTable = null, $depth = null, $addMetadataSubtableId = false);
$columnsRenamed = false;
if ($dataTable instanceof Map) {
$columnsRenamed = true;
// see https://github.com/piwik/piwik/issues/4377
$self = $this;
$dataTable->filter(function ($table) use ($self, $columnsToRenameAfterAggregation) {
if ($self->areColumnsNotAlreadyRenamed($table)) {
/**
* This makes archiving and range dates a lot faster. Imagine we archive a week, then we will
* rename all columns of each 7 day archives. Afterwards we know the columns will be replaced in a
* week archive. When generating month archives, which uses mostly week archives, we do not have
* to replace those columns for the week archives again since we can be sure they were already
* replaced. Same when aggregating year and range archives. This can save up 10% or more when
* aggregating Month, Year and Range archives.
*/
$self->renameColumnsAfterAggregation($table, $columnsToRenameAfterAggregation);
}
});
}
$dataTable = $this->getAggregatedDataTableMap($dataTable, $columnsAggregationOperation);
$this->renameColumnsAfterAggregation($dataTable, $columnsToRenameAfterAggregation);
if (!$columnsRenamed) {
$this->renameColumnsAfterAggregation($dataTable, $columnsToRenameAfterAggregation);
}
return $dataTable;
}
/**
* Note: public only for use in closure in PHP 5.3.
*
* @param $table
* @return \Piwik\Period
*/
public function areColumnsNotAlreadyRenamed($table)
{
$period = $table->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX);
return !$period || $period->getLabel() === 'day';
}
protected function getOperationForColumns($columns, $defaultOperation)
{
$operationForColumn = array();
@ -357,18 +406,54 @@ class ArchiveProcessor
protected function enrichWithUniqueVisitorsMetric(Row $row)
{
if(!$this->getParams()->isSingleSite() ) {
// we only compute unique visitors for a single site
// skip unique visitors metrics calculation if calculating for multiple sites is disabled
if (!$this->getParams()->isSingleSite()
&& $this->skipUniqueVisitorsCalculationForMultipleSites
) {
return;
}
if ( $row->getColumn('nb_uniq_visitors') !== false) {
if (SettingsPiwik::isUniqueVisitorsEnabled($this->getParams()->getPeriod()->getLabel())) {
$uniqueVisitors = (float)$this->computeNbUniqVisitors();
$row->setColumn('nb_uniq_visitors', $uniqueVisitors);
} else {
$row->deleteColumn('nb_uniq_visitors');
}
if ($row->getColumn('nb_uniq_visitors') === false
&& $row->getColumn('nb_users') === false
) {
return;
}
if (!SettingsPiwik::isUniqueVisitorsEnabled($this->getParams()->getPeriod()->getLabel())) {
$row->deleteColumn('nb_uniq_visitors');
$row->deleteColumn('nb_users');
return;
}
$metrics = array(
Metrics::INDEX_NB_USERS
);
if ($this->getParams()->isSingleSite()) {
$uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_VISITORS;
} else {
if (!SettingsPiwik::isSameFingerprintAcrossWebsites()) {
throw new Exception("Processing unique visitors across websites is enabled for this instance,
but to process this metric you must first set enable_fingerprinting_across_websites=1
in the config file, under the [Tracker] section.");
}
$uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_FINGERPRINTS;
}
$metrics[] = $uniqueVisitorsMetric;
$uniques = $this->computeNbUniques($metrics);
// see edge case as described in https://github.com/piwik/piwik/issues/9357 where uniq_visitors might be higher
// than visits because we archive / process it after nb_visits. Between archiving nb_visits and nb_uniq_visitors
// there could have been a new visit leading to a higher nb_unique_visitors than nb_visits which is not possible
// by definition. In this case we simply use the visits metric instead of unique visitors metric.
$visits = $row->getColumn('nb_visits');
if ($visits !== false && $uniques[$uniqueVisitorsMetric] !== false) {
$uniques[$uniqueVisitorsMetric] = min($uniques[$uniqueVisitorsMetric], $visits);
}
$row->setColumn('nb_uniq_visitors', $uniques[$uniqueVisitorsMetric]);
$row->setColumn('nb_users', $uniques[Metrics::INDEX_NB_USERS]);
}
protected function guessOperationForColumn($column)
@ -388,14 +473,15 @@ class ArchiveProcessor
* This is the only Period metric (ie. week/month/year/range) that we process from the logs directly,
* since unique visitors cannot be summed like other metrics.
*
* @return int
* @param array Metrics Ids for which to aggregates count of values
* @return array of metrics, where the key is metricid and the value is the metric value
*/
protected function computeNbUniqVisitors()
protected function computeNbUniques($metrics)
{
$logAggregator = $this->getLogAggregator();
$query = $logAggregator->queryVisitsByDimension(array(), false, array(), array(Metrics::INDEX_NB_UNIQ_VISITORS));
$query = $logAggregator->queryVisitsByDimension(array(), false, array(), $metrics);
$data = $query->fetch();
return $data[Metrics::INDEX_NB_UNIQ_VISITORS];
return $data;
}
/**
@ -409,15 +495,18 @@ class ArchiveProcessor
protected function getAggregatedDataTableMap($data, $columnsAggregationOperation)
{
$table = new DataTable();
if (!empty($columnsAggregationOperation)) {
$table->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation);
}
if ($data instanceof DataTable\Map) {
// as $date => $tableToSum
$this->aggregatedDataTableMapsAsOne($data, $table);
} else {
$table->addDataTable($data, $this->isAggregateSubTables());
$table->addDataTable($data);
}
return $table;
}
@ -429,22 +518,33 @@ class ArchiveProcessor
protected function aggregatedDataTableMapsAsOne(Map $map, DataTable $aggregated)
{
foreach ($map->getDataTables() as $tableToAggregate) {
if($tableToAggregate instanceof Map) {
if ($tableToAggregate instanceof Map) {
$this->aggregatedDataTableMapsAsOne($tableToAggregate, $aggregated);
} else {
$aggregated->addDataTable($tableToAggregate, $this->isAggregateSubTables());
$aggregated->addDataTable($tableToAggregate);
}
}
}
protected function renameColumnsAfterAggregation(DataTable $table, $columnsToRenameAfterAggregation = null)
/**
* Note: public only for use in closure in PHP 5.3.
*/
public function renameColumnsAfterAggregation(DataTable $table, $columnsToRenameAfterAggregation = null)
{
// Rename columns after aggregation
if (is_null($columnsToRenameAfterAggregation)) {
$columnsToRenameAfterAggregation = self::$columnsToRenameAfterAggregation;
}
foreach ($columnsToRenameAfterAggregation as $oldName => $newName) {
$table->renameColumn($oldName, $newName, $this->isAggregateSubTables());
foreach ($table->getRows() as $row) {
foreach ($columnsToRenameAfterAggregation as $oldName => $newName) {
$row->renameColumn($oldName, $newName);
}
$subTable = $row->getSubtable();
if ($subTable) {
$this->renameColumnsAfterAggregation($subTable, $columnsToRenameAfterAggregation);
}
}
}
@ -453,6 +553,7 @@ class ArchiveProcessor
if (!is_array($columns)) {
$columns = array($columns);
}
$operationForColumn = $this->getOperationForColumns($columns, $operationToApply);
$dataTable = $this->getArchive()->getDataTableFromNumeric($columns);
@ -463,11 +564,11 @@ class ArchiveProcessor
}
$rowMetrics = $results->getFirstRow();
if($rowMetrics === false) {
if ($rowMetrics === false) {
$rowMetrics = new Row;
}
$this->enrichWithUniqueVisitorsMetric($rowMetrics);
$this->renameColumnsAfterAggregation($results);
$this->renameColumnsAfterAggregation($results, self::$columnsToRenameAfterAggregation);
$metrics = $rowMetrics->getColumns();
@ -476,14 +577,7 @@ class ArchiveProcessor
$metrics[$name] = 0;
}
}
return $metrics;
}
/**
* @return bool
*/
protected function isAggregateSubTables()
{
return !$this->getParams()->isSkipAggregationOfSubTables();
}
}

57
www/analytics/core/ArchiveProcessor/Loader.php
View file

@ -1,18 +1,20 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\ArchiveProcessor;
use Piwik\Archive;
use Piwik\ArchiveProcessor;
use Piwik\Cache;
use Piwik\Config;
use Piwik\DataAccess\ArchiveSelector;
use Piwik\Date;
use Piwik\Period;
use Piwik\Piwik;
/**
* This class uses PluginsArchiver class to trigger data aggregation and create archives.
@ -81,6 +83,7 @@ class Loader
* Prepares the core metrics if needed.
*
* @param $visits
* @return array
*/
protected function prepareCoreMetricsArchive($visits, $visitsConverted)
{
@ -101,12 +104,14 @@ class Loader
$visits = $metrics['nb_visits'];
$visitsConverted = $metrics['nb_visits_converted'];
}
return array($visits, $visitsConverted);
}
protected function prepareAllPluginsArchive($visits, $visitsConverted)
{
$pluginsArchiver = new PluginsArchiver($this->params, $this->isArchiveTemporary());
if ($this->mustProcessVisitCount($visits)
|| $this->doesRequestedPluginIncludeVisitsSummary()
) {
@ -114,14 +119,14 @@ class Loader
$visits = $metrics['nb_visits'];
$visitsConverted = $metrics['nb_visits_converted'];
}
if ($this->isThereSomeVisits($visits)) {
if ($this->isThereSomeVisits($visits)
|| $this->shouldArchiveForSiteEvenWhenNoVisits()
) {
$pluginsArchiver->callAggregateAllPlugins($visits, $visitsConverted);
}
$idArchive = $pluginsArchiver->finalizeArchive();
if (!$this->params->isSingleSiteDayArchive() && $visits) {
ArchiveSelector::purgeOutdatedArchives($this->params->getPeriod()->getDateStart());
}
$idArchive = $pluginsArchiver->finalizeArchive();
return array($idArchive, $visits);
}
@ -139,11 +144,13 @@ class Loader
{
$period = $this->params->getPeriod()->getLabel();
$debugSetting = 'always_archive_data_period'; // default
if ($period == 'day') {
$debugSetting = 'always_archive_data_day';
} elseif ($period == 'range') {
$debugSetting = 'always_archive_data_range';
}
return (bool) Config::getInstance()->Debug[$debugSetting];
}
@ -165,9 +172,11 @@ class Loader
}
$idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
if (!$idAndVisits) {
return $noArchiveFound;
}
return $idAndVisits;
}
@ -186,19 +195,27 @@ class Loader
// Permanent archive
return $endDateTimestamp;
}
$dateStart = $this->params->getDateStart();
$period = $this->params->getPeriod();
$segment = $this->params->getSegment();
$site = $this->params->getSite();
// Temporary archive
return Rules::getMinTimeProcessedForTemporaryArchive($this->params->getDateStart(), $this->params->getPeriod(), $this->params->getSegment(), $this->params->getSite());
return Rules::getMinTimeProcessedForTemporaryArchive($dateStart, $period, $segment, $site);
}
protected static function determineIfArchivePermanent(Date $dateEnd)
{
$now = time();
$endTimestampUTC = strtotime($dateEnd->getDateEndUTC());
if ($endTimestampUTC <= $now) {
// - if the period we are looking for is finished, we look for a ts_archived that
// is greater than the last day of the archive
return $endTimestampUTC;
}
return false;
}
@ -207,8 +224,30 @@ class Loader
if (is_null($this->temporaryArchive)) {
throw new \Exception("getMinTimeArchiveProcessed() should be called prior to isArchiveTemporary()");
}
return $this->temporaryArchive;
}
}
private function shouldArchiveForSiteEvenWhenNoVisits()
{
$idSitesToArchive = $this->getIdSitesToArchiveWhenNoVisits();
return in_array($this->params->getSite()->getId(), $idSitesToArchive);
}
private function getIdSitesToArchiveWhenNoVisits()
{
$cache = Cache::getTransientCache();
$cacheKey = 'Archiving.getIdSitesToArchiveWhenNoVisits';
if (!$cache->contains($cacheKey)) {
$idSites = array();
// leaving undocumented unless decided otherwise
Piwik::postEvent('Archiving.getIdSitesToArchiveWhenNoVisits', array(&$idSites));
$cache->save($cacheKey, $idSites);
}
return $cache->fetch($cacheKey);
}
}

16
www/analytics/core/ArchiveProcessor/Parameters.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -45,15 +45,14 @@ class Parameters
/**
* Constructor.
*
*
* @ignore
*/
public function __construct(Site $site, Period $period, Segment $segment, $skipAggregationOfSubTables = false)
public function __construct(Site $site, Period $period, Segment $segment)
{
$this->site = $site;
$this->period = $period;
$this->segment = $segment;
$this->skipAggregationOfSubTables = $skipAggregationOfSubTables;
}
/**
@ -91,7 +90,7 @@ class Parameters
*/
public function getSubPeriods()
{
if($this->getPeriod()->getLabel() == 'day') {
if ($this->getPeriod()->getLabel() == 'day') {
return array( $this->getPeriod() );
}
return $this->getPeriod()->getSubperiods();
@ -169,18 +168,13 @@ class Parameters
return count($this->getIdSites()) == 1;
}
public function isSkipAggregationOfSubTables()
{
return $this->skipAggregationOfSubTables;
}
public function logStatusDebug($isTemporary)
{
$temporary = 'definitive archive';
if ($isTemporary) {
$temporary = 'temporary archive';
}
Log::verbose(
Log::debug(
"%s archive, idSite = %d (%s), segment '%s', report = '%s', UTC datetime [%s -> %s]",
$this->getPeriod()->getLabel(),
$this->getSite()->getId(),

74
www/analytics/core/ArchiveProcessor/PluginsArchiver.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,13 +9,15 @@
namespace Piwik\ArchiveProcessor;
use Piwik\Archive;
use Piwik\ArchiveProcessor;
use Piwik\DataAccess\ArchiveSelector;
use Piwik\DataAccess\ArchiveWriter;
use Piwik\DataAccess\LogAggregator;
use Piwik\DataTable\Manager;
use Piwik\Metrics;
use Piwik\Plugin\Archiver;
use Piwik\Log;
use Piwik\Timer;
use Exception;
/**
* This class creates the Archiver objects found in plugins and will trigger aggregation,
@ -34,9 +36,16 @@ class PluginsArchiver
protected $params;
/**
* @var LogAggregator
*/
private $logAggregator;
/**
* Public only for tests. Won't be necessary after DI changes are complete.
*
* @var Archiver[] $archivers
*/
private static $archivers = array();
public static $archivers = array();
public function __construct(Parameters $params, $isTemporaryArchive)
{
@ -45,7 +54,9 @@ class PluginsArchiver
$this->archiveWriter = new ArchiveWriter($this->params, $isTemporaryArchive);
$this->archiveWriter->initNewArchive();
$this->archiveProcessor = new ArchiveProcessor($this->params, $this->archiveWriter);
$this->logAggregator = new LogAggregator($params);
$this->archiveProcessor = new ArchiveProcessor($this->params, $this->archiveWriter, $this->logAggregator);
$this->isSingleSiteDayArchive = $this->params->isSingleSiteDayArchive();
}
@ -57,7 +68,9 @@ class PluginsArchiver
*/
public function callAggregateCoreMetrics()
{
if($this->isSingleSiteDayArchive) {
$this->logAggregator->setQueryOriginHint('Core');
if ($this->isSingleSiteDayArchive) {
$metrics = $this->aggregateDayVisitsMetrics();
} else {
$metrics = $this->aggregateMultipleVisitsMetrics();
@ -81,27 +94,57 @@ class PluginsArchiver
*/
public function callAggregateAllPlugins($visits, $visitsConverted)
{
Log::debug("PluginsArchiver::%s: Initializing archiving process for all plugins [visits = %s, visits converted = %s]",
__FUNCTION__, $visits, $visitsConverted);
$this->archiveProcessor->setNumberOfVisits($visits, $visitsConverted);
$archivers = $this->getPluginArchivers();
foreach($archivers as $pluginName => $archiverClass) {
foreach ($archivers as $pluginName => $archiverClass) {
// We clean up below all tables created during this function call (and recursive calls)
$latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
/** @var Archiver $archiver */
$archiver = new $archiverClass($this->archiveProcessor);
if(!$archiver->isEnabled()) {
if (!$archiver->isEnabled()) {
Log::debug("PluginsArchiver::%s: Skipping archiving for plugin '%s'.", __FUNCTION__, $pluginName);
continue;
}
if($this->shouldProcessReportsForPlugin($pluginName)) {
if($this->isSingleSiteDayArchive) {
$archiver->aggregateDayReport();
} else {
$archiver->aggregateMultipleReports();
if ($this->shouldProcessReportsForPlugin($pluginName)) {
$this->logAggregator->setQueryOriginHint($pluginName);
try {
$timer = new Timer();
if ($this->isSingleSiteDayArchive) {
Log::debug("PluginsArchiver::%s: Archiving day reports for plugin '%s'.", __FUNCTION__, $pluginName);
$archiver->aggregateDayReport();
} else {
Log::debug("PluginsArchiver::%s: Archiving period reports for plugin '%s'.", __FUNCTION__, $pluginName);
$archiver->aggregateMultipleReports();
}
$this->logAggregator->setQueryOriginHint('');
Log::debug("PluginsArchiver::%s: %s while archiving %s reports for plugin '%s'.",
__FUNCTION__,
$timer->getMemoryLeak(),
$this->params->getPeriod()->getLabel(),
$pluginName
);
} catch (Exception $e) {
$className = get_class($e);
$exception = new $className($e->getMessage() . " - caused by plugin $pluginName", $e->getCode(), $e);
throw $exception;
}
} else {
Log::debug("PluginsArchiver::%s: Not archiving reports for plugin '%s'.", __FUNCTION__, $pluginName);
}
Manager::getInstance()->deleteAll($latestUsedTableId);
@ -111,7 +154,7 @@ class PluginsArchiver
public function finalizeArchive()
{
$this->params->logStatusDebug( $this->archiveWriter->isArchiveTemporary );
$this->params->logStatusDebug($this->archiveWriter->isArchiveTemporary);
$this->archiveWriter->finalizeArchive();
return $this->archiveWriter->getIdArchive();
}
@ -193,5 +236,4 @@ class PluginsArchiver
$metrics = $this->archiveProcessor->aggregateNumericMetrics($toSum);
return $metrics;
}
}

156
www/analytics/core/ArchiveProcessor/Rules.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,14 +9,13 @@
namespace Piwik\ArchiveProcessor;
use Exception;
use Piwik\Common;
use Piwik\Config;
use Piwik\DataAccess\ArchiveWriter;
use Piwik\Date;
use Piwik\Log;
use Piwik\Option;
use Piwik\Piwik;
use Piwik\Plugins\CoreAdminHome\Controller;
use Piwik\Plugins\CoreAdminHome\CoreAdminHome;
use Piwik\Segment;
use Piwik\SettingsPiwik;
use Piwik\SettingsServer;
@ -25,7 +24,7 @@ use Piwik\Tracker\Cache;
/**
* This class contains Archiving rules/logic which are used when creating and processing Archives.
*
*
*/
class Rules
{
@ -35,9 +34,6 @@ class Rules
const FLAG_TABLE_PURGED = 'lastPurge_';
/** Old Archives purge can be disabled (used in tests only) */
static public $purgeDisabledByTests = false;
/** Flag that will forcefully disable the archiving process (used in tests only) */
public static $archivingDisabledByTests = false;
@ -45,15 +41,16 @@ class Rules
* Returns the name of the archive field used to tell the status of an archive, (ie,
* whether the archive was created successfully or not).
*
* @param array $idSites
* @param Segment $segment
* @param string $periodLabel
* @param string $plugin
* @return string
*/
public static function getDoneStringFlagFor(array $idSites, $segment, $periodLabel, $plugin, $isSkipAggregationOfSubTables)
public static function getDoneStringFlagFor(array $idSites, $segment, $periodLabel, $plugin)
{
if (!self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel)) {
return self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin, $isSkipAggregationOfSubTables);
return self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
}
return self::getDoneFlagArchiveContainsAllPlugins($segment);
}
@ -84,39 +81,24 @@ class Rules
return $segmentsToProcess;
}
public static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin, $isSkipAggregationOfSubTables = false)
public static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin)
{
$partial = self::isFlagArchivePartial($plugin, $isSkipAggregationOfSubTables);
return 'done' . $segment->getHash() . '.' . $plugin . $partial ;
return 'done' . $segment->getHash() . '.' . $plugin ;
}
private static function getDoneFlagArchiveContainsAllPlugins(Segment $segment)
public static function getDoneFlagArchiveContainsAllPlugins(Segment $segment)
{
return 'done' . $segment->getHash();
}
/**
* @param $plugin
* @param $isSkipAggregationOfSubTables
* @return string
*/
private static function isFlagArchivePartial($plugin, $isSkipAggregationOfSubTables)
{
$partialArchive = '';
if ($plugin != "VisitsSummary" // VisitsSummary is always called when segmenting and should not have its own .partial archive
&& $isSkipAggregationOfSubTables
) {
$partialArchive = '.partial';
}
return $partialArchive;
}
/**
* Return done flags used to tell how the archiving process for a specific archive was completed,
*
* @param array $plugins
* @param $segment
* @return array
*/
public static function getDoneFlags(array $plugins, Segment $segment, $isSkipAggregationOfSubTables)
public static function getDoneFlags(array $plugins, Segment $segment)
{
$doneFlags = array();
$doneAllPlugins = self::getDoneFlagArchiveContainsAllPlugins($segment);
@ -124,58 +106,12 @@ class Rules
$plugins = array_unique($plugins);
foreach ($plugins as $plugin) {
$doneOnePlugin = self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin, $isSkipAggregationOfSubTables);
$doneOnePlugin = self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
$doneFlags[$plugin] = $doneOnePlugin;
}
return $doneFlags;
}
/**
* Given a monthly archive table, will delete all reports that are now outdated,
* or reports that ended with an error
*
* @param \Piwik\Date $date
* @return int|bool False, or timestamp indicating which archives to delete
*/
public static function shouldPurgeOutdatedArchives(Date $date)
{
if (self::$purgeDisabledByTests) {
return false;
}
$key = self::FLAG_TABLE_PURGED . "blob_" . $date->toString('Y_m');
$timestamp = Option::get($key);
// we shall purge temporary archives after their timeout is finished, plus an extra 6 hours
// in case archiving is disabled or run once a day, we give it this extra time to run
// and re-process more recent records...
$temporaryArchivingTimeout = self::getTodayArchiveTimeToLive();
$hoursBetweenPurge = 6;
$purgeEveryNSeconds = max($temporaryArchivingTimeout, $hoursBetweenPurge * 3600);
// we only delete archives if we are able to process them, otherwise, the browser might process reports
// when &segment= is specified (or custom date range) and would below, delete temporary archives that the
// browser is not able to process until next cron run (which could be more than 1 hour away)
if (self::isRequestAuthorizedToArchive()
&& (!$timestamp
|| $timestamp < time() - $purgeEveryNSeconds)
) {
Option::set($key, time());
if (self::isBrowserTriggerEnabled()) {
// If Browser Archiving is enabled, it is likely there are many more temporary archives
// We delete more often which is safe, since reports are re-processed on demand
$purgeArchivesOlderThan = Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime();
} else {
// If archive.php via Cron is building the reports, we should keep all temporary reports from today
$purgeArchivesOlderThan = Date::factory('today')->getDateTime();
}
return $purgeArchivesOlderThan;
}
Log::info("Purging temporary archives: skipped.");
return false;
}
public static function getMinTimeProcessedForTemporaryArchive(
Date $dateStart, \Piwik\Period $period, Segment $segment, Site $site)
{
@ -213,30 +149,45 @@ class Rules
{
$uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
if($uiSettingIsEnabled) {
if ($uiSettingIsEnabled) {
$timeToLive = Option::get(self::OPTION_TODAY_ARCHIVE_TTL);
if ($timeToLive !== false) {
return $timeToLive;
}
}
return self::getTodayArchiveTimeToLiveDefault();
}
public static function getTodayArchiveTimeToLiveDefault()
{
return Config::getInstance()->General['time_before_today_archive_considered_outdated'];
}
public static function isArchivingDisabledFor(array $idSites, Segment $segment, $periodLabel)
{
$generalConfig = Config::getInstance()->General;
if ($periodLabel == 'range') {
return false;
if (!isset($generalConfig['archiving_range_force_on_browser_request'])
|| $generalConfig['archiving_range_force_on_browser_request'] != false
) {
return false;
} else {
Log::debug("Not forcing archiving for range period.");
}
}
$processOneReportOnly = !self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel);
$isArchivingDisabled = !self::isRequestAuthorizedToArchive() || self::$archivingDisabledByTests;
if ($processOneReportOnly) {
if ($processOneReportOnly
&& $periodLabel != 'range'
) {
// When there is a segment, we disable archiving when browser_archiving_disabled_enforce applies
if (!$segment->isEmpty()
&& $isArchivingDisabled
&& Config::getInstance()->General['browser_archiving_disabled_enforce']
&& !SettingsServer::isArchivePhpTriggered() // Only applies when we are not running archive.php
&& $generalConfig['browser_archiving_disabled_enforce']
&& !SettingsServer::isArchivePhpTriggered() // Only applies when we are not running core:archive command
) {
Log::debug("Archiving is disabled because of config setting browser_archiving_disabled_enforce=1");
return true;
@ -248,7 +199,7 @@ class Rules
return $isArchivingDisabled;
}
protected static function isRequestAuthorizedToArchive()
public static function isRequestAuthorizedToArchive()
{
return Rules::isBrowserTriggerEnabled() || SettingsServer::isArchivePhpTriggered();
}
@ -257,7 +208,7 @@ class Rules
{
$uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
if($uiSettingIsEnabled) {
if ($uiSettingIsEnabled) {
$browserArchivingEnabled = Option::get(self::OPTION_BROWSER_TRIGGER_ARCHIVING);
if ($browserArchivingEnabled !== false) {
return (bool)$browserArchivingEnabled;
@ -275,6 +226,18 @@ class Rules
Cache::clearCacheGeneral();
}
/**
* Returns true if the archiving process should skip the calculation of unique visitors
* across several sites. The `[General] enable_processing_unique_visitors_multiple_sites`
* INI config option controls the value of this variable.
*
* @return bool
*/
public static function shouldSkipUniqueVisitorsCalculationForMultipleSites()
{
return Config::getInstance()->General['enable_processing_unique_visitors_multiple_sites'] != 1;
}
/**
* @param array $idSites
* @param Segment $segment
@ -294,11 +257,24 @@ class Rules
// Turns out the getString() above returns the URL decoded segment string
$segmentsToProcessUrlDecoded = array_map('urldecode', $segmentsToProcess);
if (in_array($segment, $segmentsToProcess)
|| in_array($segment, $segmentsToProcessUrlDecoded)
) {
return true;
return in_array($segment, $segmentsToProcess)
|| in_array($segment, $segmentsToProcessUrlDecoded);
}
/**
* Returns done flag values allowed to be selected
*
* @return string
*/
public static function getSelectableDoneFlagValues()
{
$possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
if (!Rules::isRequestAuthorizedToArchive()) {
//If request is not authorized to archive then fetch also invalidated archives
$possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
}
return false;
return $possibleValues;
}
}

48
www/analytics/core/Archiver/Request.php Normal file
View file

@ -0,0 +1,48 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archiver;
class Request
{
/**
* @var string
*/
private $url;
/**
* @var callable|null
*/
private $before;
/**
* @param string $url
*/
public function __construct($url)
{
$this->url = $url;
}
public function before($callable)
{
$this->before = $callable;
}
public function start()
{
if ($this->before) {
$callable = $this->before;
$callable();
}
}
public function __toString()
{
return $this->url;
}
}

64
www/analytics/core/AssetManager.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -19,9 +19,8 @@ use Piwik\AssetManager\UIAssetFetcher\StylesheetUIAssetFetcher;
use Piwik\AssetManager\UIAssetFetcher;
use Piwik\AssetManager\UIAssetMerger\JScriptUIAssetMerger;
use Piwik\AssetManager\UIAssetMerger\StylesheetUIAssetMerger;
use Piwik\Container\StaticContainer;
use Piwik\Plugin\Manager;
use Piwik\Translate;
use Piwik\Config as PiwikConfig;
/**
* AssetManager is the class used to manage the inclusion of UI assets:
@ -37,7 +36,7 @@ use Piwik\Config as PiwikConfig;
* the global option 'disable_merged_assets'. See the documentation in the global
* config for more information.
*
* @method static \Piwik\AssetManager getInstance()
* @method static AssetManager getInstance()
*/
class AssetManager extends Singleton
{
@ -66,13 +65,13 @@ class AssetManager extends Singleton
*/
private $theme;
function __construct()
public function __construct()
{
$this->cacheBuster = UIAssetCacheBuster::getInstance();
$this->minimalStylesheetFetcher = new StaticUIAssetFetcher(array('plugins/Zeitgeist/stylesheets/base.less'), array(), $this->theme);
$this->minimalStylesheetFetcher = new StaticUIAssetFetcher(array('plugins/Morpheus/stylesheets/base.less', 'plugins/Morpheus/stylesheets/general/_forms.less'), array(), $this->theme);
$theme = Manager::getInstance()->getThemeEnabled();
if(!empty($theme)) {
if (!empty($theme)) {
$this->theme = new Theme();
}
}
@ -121,14 +120,11 @@ class AssetManager extends Singleton
$result = "<script type=\"text/javascript\">\n" . Translate::getJavascriptTranslations() . "\n</script>";
if ($this->isMergedAssetsDisabled()) {
$this->getMergedCoreJSAsset()->delete();
$this->getMergedNonCoreJSAsset()->delete();
$result .= $this->getIndividualJsIncludes();
} else {
$result .= sprintf(self::JS_IMPORT_DIRECTIVE, self::GET_CORE_JS_MODULE_ACTION);
$result .= sprintf(self::JS_IMPORT_DIRECTIVE, self::GET_NON_CORE_JS_MODULE_ACTION);
}
@ -201,13 +197,13 @@ class AssetManager extends Singleton
{
$loadedPlugins = array();
foreach(Manager::getInstance()->getPluginsLoadedAndActivated() as $plugin) {
foreach (Manager::getInstance()->getPluginsLoadedAndActivated() as $plugin) {
$pluginName = $plugin->getPluginName();
$pluginIsCore = Manager::getInstance()->isPluginBundledWithCore($pluginName);
if(($pluginIsCore && $core) || (!$pluginIsCore && !$core))
if (($pluginIsCore && $core) || (!$pluginIsCore && !$core)) {
$loadedPlugins[] = $pluginName;
}
}
return $loadedPlugins;
@ -220,23 +216,15 @@ class AssetManager extends Singleton
{
$assetsToRemove = array($this->getMergedStylesheetAsset());
if($pluginName) {
if($this->pluginContainsJScriptAssets($pluginName)) {
PiwikConfig::getInstance()->init();
if(Manager::getInstance()->isPluginBundledWithCore($pluginName)) {
if ($pluginName) {
if ($this->pluginContainsJScriptAssets($pluginName)) {
if (Manager::getInstance()->isPluginBundledWithCore($pluginName)) {
$assetsToRemove[] = $this->getMergedCoreJSAsset();
} else {
$assetsToRemove[] = $this->getMergedNonCoreJSAsset();
}
}
} else {
$assetsToRemove[] = $this->getMergedCoreJSAsset();
$assetsToRemove[] = $this->getMergedNonCoreJSAsset();
}
@ -252,8 +240,7 @@ class AssetManager extends Singleton
*/
public function getAssetDirectory()
{
$mergedFileDirectory = PIWIK_USER_PATH . "/tmp/assets";
$mergedFileDirectory = SettingsPiwik::rewriteTmpPathWithHostname($mergedFileDirectory);
$mergedFileDirectory = StaticContainer::get('path.tmp') . '/assets';
if (!is_dir($mergedFileDirectory)) {
Filesystem::mkdir($mergedFileDirectory);
@ -273,7 +260,15 @@ class AssetManager extends Singleton
*/
public function isMergedAssetsDisabled()
{
return Config::getInstance()->Debug['disable_merged_assets'];
if (Config::getInstance()->Development['disable_merged_assets'] == 1) {
return true;
}
if (isset($_GET['disable_merged_assets']) && $_GET['disable_merged_assets'] == 1) {
return true;
}
return false;
}
/**
@ -311,7 +306,6 @@ class AssetManager extends Singleton
$jsIncludeString = '';
foreach ($assetFetcher->getCatalog()->getAssets() as $jsFile) {
$jsFile->validateFile();
$jsIncludeString = $jsIncludeString . sprintf(self::JS_IMPORT_DIRECTIVE, $jsFile->getRelativeLocation());
}
@ -339,7 +333,7 @@ class AssetManager extends Singleton
try {
$assets = $fetcher->getCatalog()->getAssets();
} catch(\Exception $e) {
} catch (\Exception $e) {
// This can happen when a plugin is not valid (eg. Piwik 1.x format)
// When posting the event to the plugin, it returns an exception "Plugin has not been loaded"
return false;
@ -347,14 +341,14 @@ class AssetManager extends Singleton
$plugin = Manager::getInstance()->getLoadedPlugin($pluginName);
if($plugin->isTheme()) {
if ($plugin->isTheme()) {
$theme = Manager::getInstance()->getTheme($pluginName);
$javaScriptFiles = $theme->getJavaScriptFiles();
if(!empty($javaScriptFiles))
if (!empty($javaScriptFiles)) {
$assets = array_merge($assets, $javaScriptFiles);
}
}
return !empty($assets);
@ -363,9 +357,9 @@ class AssetManager extends Singleton
/**
* @param UIAsset[] $uiAssets
*/
private function removeAssets($uiAssets)
public function removeAssets($uiAssets)
{
foreach($uiAssets as $uiAsset) {
foreach ($uiAssets as $uiAsset) {
$uiAsset->delete();
}
}
@ -373,7 +367,7 @@ class AssetManager extends Singleton
/**
* @return UIAsset
*/
private function getMergedStylesheetAsset()
public function getMergedStylesheetAsset()
{
return $this->getMergedUIAsset(self::MERGED_CSS_FILE);
}

2
www/analytics/core/AssetManager/UIAsset.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later

3
www/analytics/core/AssetManager/UIAsset/InMemoryUIAsset.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -45,7 +45,6 @@ class InMemoryUIAsset extends UIAsset
return false;
}
public function writeContent($content)
{
$this->content = $content;

23
www/analytics/core/AssetManager/UIAsset/OnDiskUIAsset.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -10,6 +10,7 @@ namespace Piwik\AssetManager\UIAsset;
use Exception;
use Piwik\AssetManager\UIAsset;
use Piwik\Filesystem;
class OnDiskUIAsset extends UIAsset
{
@ -27,7 +28,7 @@ class OnDiskUIAsset extends UIAsset
* @param string $baseDirectory
* @param string $fileLocation
*/
function __construct($baseDirectory, $fileLocation)
public function __construct($baseDirectory, $fileLocation)
{
$this->baseDirectory = $baseDirectory;
$this->relativeLocation = $fileLocation;
@ -50,20 +51,23 @@ class OnDiskUIAsset extends UIAsset
public function validateFile()
{
if (!$this->assetIsReadable())
if (!$this->assetIsReadable()) {
throw new Exception("The ui asset with 'href' = " . $this->getAbsoluteLocation() . " is not readable");
}
}
public function delete()
{
if ($this->exists()) {
if (!unlink($this->getAbsoluteLocation()))
try {
Filesystem::remove($this->getAbsoluteLocation());
} catch (Exception $e) {
throw new Exception("Unable to delete merged file : " . $this->getAbsoluteLocation() . ". Please delete the file and refresh");
}
// try to remove compressed version of the merged file.
@unlink($this->getAbsoluteLocation() . ".deflate");
@unlink($this->getAbsoluteLocation() . ".gz");
Filesystem::remove($this->getAbsoluteLocation() . ".deflate", true);
Filesystem::remove($this->getAbsoluteLocation() . ".gz", true);
}
}
@ -77,8 +81,9 @@ class OnDiskUIAsset extends UIAsset
$newFile = @fopen($this->getAbsoluteLocation(), "w");
if (!$newFile)
throw new Exception ("The file : " . $newFile . " can not be opened in write mode.");
if (!$newFile) {
throw new Exception("The file : " . $newFile . " can not be opened in write mode.");
}
fwrite($newFile, $content);

2
www/analytics/core/AssetManager/UIAssetCacheBuster.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later

21
www/analytics/core/AssetManager/UIAssetCatalog.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -20,10 +20,15 @@ class UIAssetCatalog
*/
private $catalogSorter;
/**
* @var string[] Absolute file locations
*/
private $existingAssetLocations = array();
/**
* @param UIAssetCatalogSorter $catalogSorter
*/
function __construct($catalogSorter)
public function __construct($catalogSorter)
{
$this->catalogSorter = $catalogSorter;
}
@ -33,8 +38,10 @@ class UIAssetCatalog
*/
public function addUIAsset($uiAsset)
{
if(!$this->assetAlreadyInCatalog($uiAsset)) {
$location = $uiAsset->getAbsoluteLocation();
if (!$this->assetAlreadyInCatalog($location)) {
$this->existingAssetLocations[] = $location;
$this->uiAssets[] = $uiAsset;
}
}
@ -59,12 +66,8 @@ class UIAssetCatalog
* @param UIAsset $uiAsset
* @return boolean
*/
private function assetAlreadyInCatalog($uiAsset)
private function assetAlreadyInCatalog($location)
{
foreach($this->uiAssets as $existingAsset)
if($uiAsset->getAbsoluteLocation() == $existingAsset->getAbsoluteLocation())
return true;
return false;
return in_array($location, $this->existingAssetLocations);
}
}

9
www/analytics/core/AssetManager/UIAssetCatalogSorter.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -18,7 +18,7 @@ class UIAssetCatalogSorter
/**
* @param string[] $priorityOrder
*/
function __construct($priorityOrder)
public function __construct($priorityOrder)
{
$this->priorityOrder = $priorityOrder;
}
@ -31,12 +31,11 @@ class UIAssetCatalogSorter
{
$sortedCatalog = new UIAssetCatalog($this);
foreach ($this->priorityOrder as $filePattern) {
$assetsMatchingPattern = array_filter($uiAssetCatalog->getAssets(), function($uiAsset) use ($filePattern) {
$assetsMatchingPattern = array_filter($uiAssetCatalog->getAssets(), function ($uiAsset) use ($filePattern) {
return preg_match('~^' . $filePattern . '~', $uiAsset->getRelativeLocation());
});
foreach($assetsMatchingPattern as $assetMatchingPattern) {
foreach ($assetsMatchingPattern as $assetMatchingPattern) {
$sortedCatalog->addUIAsset($assetMatchingPattern);
}
}

8
www/analytics/core/AssetManager/UIAssetFetcher.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -37,7 +37,7 @@ abstract class UIAssetFetcher
* @param string[] $plugins
* @param Theme $theme
*/
function __construct($plugins, $theme)
public function __construct($plugins, $theme)
{
$this->plugins = $plugins;
$this->theme = $theme;
@ -56,8 +56,9 @@ abstract class UIAssetFetcher
*/
public function getCatalog()
{
if($this->catalog == null)
if ($this->catalog == null) {
$this->createCatalog();
}
return $this->catalog;
}
@ -89,7 +90,6 @@ abstract class UIAssetFetcher
private function populateCatalog()
{
foreach ($this->fileLocations as $fileLocation) {
$newUIAsset = new OnDiskUIAsset($this->getBaseDirectory(), $fileLocation);
$this->catalog->addUIAsset($newUIAsset);
}

31
www/analytics/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -10,15 +10,13 @@ namespace Piwik\AssetManager\UIAssetFetcher;
use Piwik\AssetManager\UIAssetFetcher;
use Piwik\Piwik;
use string;
class JScriptUIAssetFetcher extends UIAssetFetcher
{
protected function retrieveFileLocations()
{
if(!empty($this->plugins)) {
if (!empty($this->plugins)) {
/**
* Triggered when gathering the list of all JavaScript files needed by Piwik
@ -31,7 +29,7 @@ class JScriptUIAssetFetcher extends UIAssetFetcher
* plugin's root directory.
*
* _Note: While you are developing your plugin you should enable the config setting
* `[Debug] disable_merged_assets` so JavaScript files will be reloaded immediately
* `[Development] disable_merged_assets` so JavaScript files will be reloaded immediately
* after every change._
*
* **Example**
@ -53,17 +51,14 @@ class JScriptUIAssetFetcher extends UIAssetFetcher
protected function addThemeFiles()
{
$theme = $this->getTheme();
if(!$theme) {
if (!$theme) {
return;
}
if(in_array($theme->getThemeName(), $this->plugins)) {
if (in_array($theme->getThemeName(), $this->plugins)) {
$jsInThemes = $this->getTheme()->getJavaScriptFiles();
if(!empty($jsInThemes)) {
foreach($jsInThemes as $jsFile) {
if (!empty($jsInThemes)) {
foreach ($jsInThemes as $jsFile) {
$this->fileLocations[] = $jsFile;
}
}
@ -73,13 +68,17 @@ class JScriptUIAssetFetcher extends UIAssetFetcher
protected function getPriorityOrder()
{
return array(
'libs/jquery/jquery.js',
'libs/jquery/jquery-ui.js',
'libs/bower_components/jquery/dist/jquery.min.js',
'libs/bower_components/jquery-ui/ui/minified/jquery-ui.min.js',
'libs/jquery/jquery.browser.js',
'libs/',
'js/',
'piwik.js',
'plugins/CoreHome/javascripts/require.js',
'plugins/Zeitgeist/javascripts/piwikHelper.js',
'plugins/Zeitgeist/javascripts/',
'plugins/Morpheus/javascripts/piwikHelper.js',
'plugins/Morpheus/javascripts/jquery.icheck.min.js',
'plugins/Morpheus/javascripts/morpheus.js',
'plugins/Morpheus/javascripts/',
'plugins/CoreHome/javascripts/uiControl.js',
'plugins/CoreHome/javascripts/broadcast.js',
'plugins/CoreHome/javascripts/', // load CoreHome JS before other plugins

5
www/analytics/core/AssetManager/UIAssetFetcher/StaticUIAssetFetcher.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -17,7 +17,7 @@ class StaticUIAssetFetcher extends UIAssetFetcher
*/
private $priorityOrder;
function __construct($fileLocations, $priorityOrder, $theme)
public function __construct($fileLocations, $priorityOrder, $theme)
{
parent::__construct(array(), $theme);
@ -27,7 +27,6 @@ class StaticUIAssetFetcher extends UIAssetFetcher
protected function retrieveFileLocations()
{
}
protected function getPriorityOrder()

35
www/analytics/core/AssetManager/UIAssetFetcher/StylesheetUIAssetFetcher.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -15,15 +15,30 @@ class StylesheetUIAssetFetcher extends UIAssetFetcher
{
protected function getPriorityOrder()
{
return array(
$theme = $this->getTheme();
$themeName = $theme->getThemeName();
$order = array(
'libs/',
'plugins/CoreHome/stylesheets/color_manager.css', // must be before other Piwik stylesheets
'plugins/Zeitgeist/stylesheets/base.less',
'plugins/Zeitgeist/stylesheets/',
'plugins/',
'plugins/Dashboard/stylesheets/dashboard.less',
'tests/',
'plugins/Morpheus/stylesheets/base.less',
);
if ($themeName === 'Morpheus') {
$order[] = 'plugins\/((?!Morpheus).)*\/';
} else {
$order[] = sprintf('plugins\/((?!(Morpheus)|(%s)).)*\/', $themeName);
}
$order = array_merge(
$order,
array(
'plugins/Dashboard/stylesheets/dashboard.less',
'tests/',
)
);
return $order;
}
protected function retrieveFileLocations()
@ -55,9 +70,13 @@ class StylesheetUIAssetFetcher extends UIAssetFetcher
protected function addThemeFiles()
{
$theme = $this->getTheme();
if (!$theme) {
return;
}
$themeStylesheet = $this->getTheme()->getStylesheet();
if($themeStylesheet) {
if ($themeStylesheet) {
$this->fileLocations[] = $themeStylesheet;
}
}

38
www/analytics/core/AssetManager/UIAssetMerger.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -8,9 +8,6 @@
*/
namespace Piwik\AssetManager;
use Piwik\AssetManager\PiwikLessCompiler;
use Piwik\AssetManager\UIAsset\StylesheetUIAsset;
use Piwik\AssetManager;
abstract class UIAssetMerger
{
@ -39,7 +36,7 @@ abstract class UIAssetMerger
* @param UIAssetFetcher $assetFetcher
* @param UIAssetCacheBuster $cacheBuster
*/
function __construct($mergedAsset, $assetFetcher, $cacheBuster)
public function __construct($mergedAsset, $assetFetcher, $cacheBuster)
{
$this->mergedAsset = $mergedAsset;
$this->assetFetcher = $assetFetcher;
@ -48,8 +45,9 @@ abstract class UIAssetMerger
public function generateFile()
{
if(!$this->shouldGenerate())
if (!$this->shouldGenerate()) {
return;
}
$this->mergedContent = $this->getMergedAssets();
@ -95,8 +93,9 @@ abstract class UIAssetMerger
protected function getConcatenatedAssets()
{
if(empty($this->mergedContent))
if (empty($this->mergedContent)) {
$this->concatenateAssets();
}
return $this->mergedContent;
}
@ -106,7 +105,6 @@ abstract class UIAssetMerger
$mergedContent = '';
foreach ($this->getAssetCatalog()->getAssets() as $uiAsset) {
$uiAsset->validateFile();
$content = $this->processFileContent($uiAsset);
@ -137,8 +135,9 @@ abstract class UIAssetMerger
*/
private function shouldGenerate()
{
if(!$this->mergedAsset->exists())
if (!$this->mergedAsset->exists()) {
return true;
}
return !$this->isFileUpToDate();
}
@ -161,19 +160,11 @@ abstract class UIAssetMerger
return false;
}
/**
* @return boolean
*/
private function isMergedAssetsDisabled()
{
return AssetManager::getInstance()->isMergedAssetsDisabled();
}
private function adjustPaths()
{
$theme = $this->assetFetcher->getTheme();
// During installation theme is not yet ready
if($theme) {
if ($theme) {
$this->mergedContent = $this->assetFetcher->getTheme()->rewriteAssetsPathToTheme($this->mergedContent);
}
}
@ -188,8 +179,9 @@ abstract class UIAssetMerger
*/
protected function getCacheBusterValue()
{
if(empty($this->cacheBusterValue))
if (empty($this->cacheBusterValue)) {
$this->cacheBusterValue = $this->generateCacheBuster();
}
return $this->cacheBusterValue;
}
@ -198,12 +190,4 @@ abstract class UIAssetMerger
{
$this->mergedContent = $this->getPreamble() . $this->mergedContent;
}
/**
* @return boolean
*/
private function shouldCompareExistingVersion()
{
return $this->isMergedAssetsDisabled();
}
}

17
www/analytics/core/AssetManager/UIAssetMerger/JScriptUIAssetMerger.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -28,7 +28,7 @@ class JScriptUIAssetMerger extends UIAssetMerger
* @param JScriptUIAssetFetcher $assetFetcher
* @param UIAssetCacheBuster $cacheBuster
*/
function __construct($mergedAsset, $assetFetcher, $cacheBuster)
public function __construct($mergedAsset, $assetFetcher, $cacheBuster)
{
parent::__construct($mergedAsset, $assetFetcher, $cacheBuster);
@ -37,15 +37,13 @@ class JScriptUIAssetMerger extends UIAssetMerger
protected function getMergedAssets()
{
$concatenatedAssets = $this->getConcatenatedAssets();
return str_replace("\n", "\r\n", $concatenatedAssets);
return $this->getConcatenatedAssets();
}
protected function generateCacheBuster()
{
$cacheBuster = $this->cacheBuster->piwikVersionBasedCacheBuster($this->getPlugins());
return "/* Piwik Javascript - cb=" . $cacheBuster . "*/\r\n";
return "/* Piwik Javascript - cb=" . $cacheBuster . "*/\n";
}
protected function getPreamble()
@ -57,7 +55,7 @@ class JScriptUIAssetMerger extends UIAssetMerger
{
$plugins = $this->getPlugins();
if(!empty($plugins)) {
if (!empty($plugins)) {
/**
* Triggered after all the JavaScript files Piwik uses are minified and merged into a
@ -74,15 +72,16 @@ class JScriptUIAssetMerger extends UIAssetMerger
public function getFileSeparator()
{
return PHP_EOL;
return "\n";
}
protected function processFileContent($uiAsset)
{
$content = $uiAsset->getContent();
if (!$this->assetMinifier->isMinifiedJs($content))
if (!$this->assetMinifier->isMinifiedJs($content)) {
$content = $this->assetMinifier->minifyJs($content);
}
return $content;
}

99
www/analytics/core/AssetManager/UIAssetMerger/StylesheetUIAssetMerger.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,10 +9,10 @@
namespace Piwik\AssetManager\UIAssetMerger;
use Exception;
use lessc;
use Piwik\AssetManager\UIAsset;
use Piwik\AssetManager\UIAssetMerger;
use Piwik\Piwik;
use lessc;
class StylesheetUIAssetMerger extends UIAssetMerger
{
@ -21,7 +21,7 @@ class StylesheetUIAssetMerger extends UIAssetMerger
*/
private $lessCompiler;
function __construct($mergedAsset, $assetFetcher, $cacheBuster)
public function __construct($mergedAsset, $assetFetcher, $cacheBuster)
{
parent::__construct($mergedAsset, $assetFetcher, $cacheBuster);
@ -30,16 +30,10 @@ class StylesheetUIAssetMerger extends UIAssetMerger
protected function getMergedAssets()
{
foreach($this->getAssetCatalog()->getAssets() as $uiAsset) {
$content = $uiAsset->getContent();
if (false !== strpos($content, '@import')) {
$this->lessCompiler->addImportDir(dirname($uiAsset->getAbsoluteLocation()));
}
}
return $this->lessCompiler->compile($this->getConcatenatedAssets());
// note: we're using setImportDir on purpose (not addImportDir)
$this->lessCompiler->setImportDir(PIWIK_USER_PATH);
$concatenatedAssets = $this->getConcatenatedAssets();
return $this->lessCompiler->compile($concatenatedAssets);
}
/**
@ -87,46 +81,71 @@ class StylesheetUIAssetMerger extends UIAssetMerger
protected function processFileContent($uiAsset)
{
return $this->rewriteCssPathsDirectives($uiAsset);
$pathsRewriter = $this->getCssPathsRewriter($uiAsset);
$content = $uiAsset->getContent();
$content = $this->rewriteCssImagePaths($content, $pathsRewriter);
$content = $this->rewriteCssImportPaths($content, $pathsRewriter);
return $content;
}
/**
* Rewrite css url directives
* Rewrite CSS url() directives
*
* @param string $content
* @param callable $pathsRewriter
* @return string
*/
private function rewriteCssImagePaths($content, $pathsRewriter)
{
$content = preg_replace_callback("/(url\(['\"]?)([^'\")]*)/", $pathsRewriter, $content);
return $content;
}
/**
* Rewrite CSS import directives
*
* @param string $content
* @param callable $pathsRewriter
* @return string
*/
private function rewriteCssImportPaths($content, $pathsRewriter)
{
$content = preg_replace_callback("/(@import \")([^\")]*)/", $pathsRewriter, $content);
return $content;
}
/**
* Rewrite CSS url directives
* - rewrites paths defined relatively to their css/less definition file
* - rewrite windows directory separator \\ to /
*
* @param UIAsset $uiAsset
* @return string
* @return \Closure
*/
private function rewriteCssPathsDirectives($uiAsset)
private function getCssPathsRewriter($uiAsset)
{
static $rootDirectoryLength = null;
if (is_null($rootDirectoryLength)) {
$rootDirectoryLength = self::countDirectoriesInPathToRoot($uiAsset);
}
$baseDirectory = dirname($uiAsset->getRelativeLocation());
$content = preg_replace_callback(
"/(url\(['\"]?)([^'\")]*)/",
function ($matches) use ($rootDirectoryLength, $baseDirectory) {
$absolutePath = realpath(PIWIK_USER_PATH . "/$baseDirectory/" . $matches[2]);
return function ($matches) use ($baseDirectory) {
$absolutePath = PIWIK_USER_PATH . "/$baseDirectory/" . $matches[2];
if($absolutePath) {
// Allow to import extension less file
if (strpos($matches[2], '.') === false) {
$absolutePath .= '.less';
}
$relativePath = substr($absolutePath, $rootDirectoryLength);
// Prevent from rewriting full path
$absolutePath = realpath($absolutePath);
if ($absolutePath) {
$relativePath = $baseDirectory . "/" . $matches[2];
$relativePath = str_replace('\\', '/', $relativePath);
$publicPath = $matches[1] . $relativePath;
} else {
$publicPath = $matches[1] . $matches[2];
}
$relativePath = str_replace('\\', '/', $relativePath);
return $matches[1] . $relativePath;
} else {
return $matches[1] . $matches[2];
}
},
$uiAsset->getContent()
);
return $content;
return $publicPath;
};
}
/**
@ -138,7 +157,7 @@ class StylesheetUIAssetMerger extends UIAssetMerger
$rootDirectory = realpath($uiAsset->getBaseDirectory());
if ($rootDirectory != PATH_SEPARATOR
&& substr_compare($rootDirectory, PATH_SEPARATOR, -1)) {
&& substr($rootDirectory, -strlen(PATH_SEPARATOR)) !== PATH_SEPARATOR) {
$rootDirectory .= PATH_SEPARATOR;
}
$rootDirectoryLen = strlen($rootDirectory);

12
www/analytics/core/AssetManager/UIAssetMinifier.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -10,8 +10,8 @@
namespace Piwik\AssetManager;
use Exception;
use Piwik\Singleton;
use JShrink\Minifier;
use Piwik\Singleton;
class UIAssetMinifier extends Singleton
{
@ -23,7 +23,6 @@ class UIAssetMinifier extends Singleton
parent::__construct();
}
/**
* Indicates if the provided JavaScript content has already been minified or not.
* The heuristic is based on a custom ratio : (size of file) / (number of lines).
@ -37,6 +36,7 @@ class UIAssetMinifier extends Singleton
public function isMinifiedJs($content)
{
$lineCount = substr_count($content, "\n");
if ($lineCount == 0) {
return true;
}
@ -59,8 +59,8 @@ class UIAssetMinifier extends Singleton
private static function validateDependency()
{
if (!class_exists("JShrink\\Minifier"))
throw new Exception("JShrink could not be found, maybe you are using Piwik from git and need to have update Composer. <br>php composer.phar update");
if (!class_exists("JShrink\\Minifier")) {
throw new Exception("JShrink could not be found, maybe you are using Piwik from git and need to update Composer. $ php composer.phar update");
}
}
}

114
www/analytics/core/Auth.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,49 +9,123 @@
namespace Piwik;
use Exception;
/**
* Base for authentication modules
* Base interface for authentication implementations.
*
* Plugins that provide Auth implementations must provide a class that implements
* this interface. Additionally, an instance of that class must be set in the
* container with the 'Piwik\Auth' key during the
* [Request.initAuthenticationObject](http://developer.piwik.org/api-reference/events#requestinitauthenticationobject)
* event.
*
* Authentication implementations must support authentication via username and
* clear-text password and authentication via username and token auth. They can
* additionally support authentication via username and an MD5 hash of a password. If
* they don't support it, then [formless authentication](http://piwik.org/faq/how-to/faq_30/) will fail.
*
* Derived implementations should favor authenticating by password over authenticating
* by token auth. That is to say, if a token auth and a password are set, password
* authentication should be used.
*
* ### Examples
*
* **How an Auth implementation will be used**
*
* // authenticating by password
* $auth = StaticContainer::get('Piwik\Auth');
* $auth->setLogin('user');
* $auth->setPassword('password');
* $result = $auth->authenticate();
*
* // authenticating by token auth
* $auth = StaticContainer::get('Piwik\Auth');
* $auth->setLogin('user');
* $auth->setTokenAuth('...');
* $result = $auth->authenticate();
*
* @api
*/
interface Auth
{
/**
* Authentication module's name, e.g., "Login"
* Must return the Authentication module's name, e.g., `"Login"`.
*
* @return string
*/
public function getName();
/**
* Authenticates user
*
* @return AuthResult
*/
public function authenticate();
/**
* Authenticates the user and initializes the session.
*/
public function initSession($login, $md5Password, $rememberMe);
/**
* Accessor to set authentication token. If set, you can authenticate the tokenAuth by calling the authenticate()
* method afterwards.
* Sets the authentication token to authenticate with.
*
* @param string $token_auth authentication token
*/
public function setTokenAuth($token_auth);
/**
* Accessor to set login name
* Returns the login of the user being authenticated.
*
* @param string $login user login
* @return string
*/
public function getLogin();
/**
* Returns the secret used to calculate a user's token auth.
*
* A users token auth is generated using the user's login and this secret. The secret
* should be specific to the user and not easily guessed. Piwik's default Auth implementation
* uses an MD5 hash of a user's password.
*
* @return string
* @throws Exception if the token auth secret does not exist or cannot be obtained.
*/
public function getTokenAuthSecret();
/**
* Sets the login name to authenticate with.
*
* @param string $login The username.
*/
public function setLogin($login);
/**
* Sets the password to authenticate with.
*
* @param string $password Password (not hashed).
*/
public function setPassword($password);
/**
* Sets the hash of the password to authenticate with. The hash will be an MD5 hash.
*
* @param string $passwordHash The hashed password.
* @throws Exception if authentication by hashed password is not supported.
*/
public function setPasswordHash($passwordHash);
/**
* Authenticates a user using the login and password set using the setters. Can also authenticate
* via token auth if one is set and no password is set.
*
* Note: this method must successfully authenticate if the token auth supplied is a special hash
* of the user's real token auth. This is because the SessionInitializer class stores a
* hash of the token auth in the session cookie. You can calculate the token auth hash using the
* {@link Piwik\Plugins\Login\SessionInitializer::getHashTokenAuth()} method.
*
* @return AuthResult
* @throws Exception if the Auth implementation has an invalid state (ie, no login
* was specified). Note: implementations are not **required** to throw
* exceptions for invalid state, but they are allowed to.
*/
public function authenticate();
}
/**
* Authentication result
* Authentication result. This is what is returned by authentication attempts using {@link Auth}
* implementations.
*
* @api
*/
class AuthResult
{

60
www/analytics/core/BaseFactory.php Normal file
View file

@ -0,0 +1,60 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik;
use Exception;
/**
* Base class for all factory types.
*
* Factory types are base classes that contain a **factory** method. This method is used to instantiate
* concrete instances by a specified string ID. Fatal errors do not occur if a class does not exist.
* Instead an exception is thrown.
*
* Derived classes should override the **getClassNameFromClassId** and **getInvalidClassIdExceptionMessage**
* static methods.
*/
abstract class BaseFactory
{
/**
* Creates a new instance of a class using a string ID.
*
* @param string $classId The ID of the class.
* @return BaseFactory
* @throws Exception if $classId is invalid.
*/
public static function factory($classId)
{
$className = static::getClassNameFromClassId($classId);
if (!class_exists($className)) {
Common::sendHeader('Content-Type: text/plain; charset=utf-8');
throw new Exception(static::getInvalidClassIdExceptionMessage($classId));
}
return new $className;
}
/**
* Should return a class name based on the class's associated string ID.
*/
protected static function getClassNameFromClassId($id)
{
return $id;
}
/**
* Should return a message to use in an Exception when an invalid class ID is supplied to
* {@link factory()}.
*/
protected static function getInvalidClassIdExceptionMessage($id)
{
return "Invalid class ID '$id' for " . get_called_class() . "::factory().";
}
}

117
www/analytics/core/Cache.php Normal file
View file

@ -0,0 +1,117 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik;
use Piwik\Cache\Backend;
use Piwik\Container\StaticContainer;
class Cache
{
/**
* This can be considered as the default cache to use in case you don't know which one to pick. It does not support
* the caching of any objects though. Only boolean, numbers, strings and arrays are supported. Whenever you request
* an entry from the cache it will fetch the entry. Cache entries might be persisted but not necessarily. It
* depends on the configured backend.
*
* @return Cache\Lazy
*/
public static function getLazyCache()
{
return StaticContainer::get('Piwik\Cache\Lazy');
}
/**
* This class is used to cache any data during one request. It won't be persisted between requests and it can
* cache all kind of data, even objects or resources. This cache is very fast.
*
* @return Cache\Transient
*/
public static function getTransientCache()
{
return StaticContainer::get('Piwik\Cache\Transient');
}
/**
* This cache stores all its cache entries under one "cache" entry in a configurable backend.
*
* This comes handy for things that you need very often, nearly in every request. For example plugin metadata, the
* list of tracker plugins, the list of available languages, ...
* Instead of having to read eg. a hundred cache entries from files (or any other backend) it only loads one cache
* entry which contains the hundred keys. Should be used only for things that you need very often and only for
* cache entries that are not too large to keep loading and parsing the single cache entry fast.
* All cache entries it contains have the same life time. For fast performance it won't validate any cache ids.
* It is not possible to cache any objects using this cache.
*
* @return Cache\Eager
*/
public static function getEagerCache()
{
return StaticContainer::get('Piwik\Cache\Eager');
}
public static function flushAll()
{
self::getLazyCache()->flushAll();
self::getTransientCache()->flushAll();
self::getEagerCache()->flushAll();
}
/**
* @param $type
* @return Cache\Backend
*/
public static function buildBackend($type)
{
$factory = new Cache\Backend\Factory();
$options = self::getOptions($type);
$backend = $factory->buildBackend($type, $options);
return $backend;
}
private static function getOptions($type)
{
$options = self::getBackendOptions($type);
switch ($type) {
case 'file':
$options = array('directory' => StaticContainer::get('path.cache'));
break;
case 'chained':
foreach ($options['backends'] as $backend) {
$options[$backend] = self::getOptions($backend);
}
break;
case 'redis':
if (!empty($options['timeout'])) {
$options['timeout'] = (float)Common::forceDotAsSeparatorForDecimalPoint($options['timeout']);
}
break;
}
return $options;
}
private static function getBackendOptions($backend)
{
$key = ucfirst($backend) . 'Cache';
$options = Config::getInstance()->$key;
return $options;
}
}

206
www/analytics/core/CacheFile.php
View file

@ -1,206 +0,0 @@
<?php
/**
* Piwik - Open source web analytics
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik;
use Exception;
/**
* This class is used to cache data on the filesystem.
*
* It is for example used by the Tracker process to cache various settings and websites attributes in tmp/cache/tracker/*
*
*/
class CacheFile
{
// for testing purposes since tests run on both CLI/FPM (changes in CLI can't invalidate
// opcache in FPM, so we have to invalidate before reading)
public static $invalidateOpCacheBeforeRead = false;
/**
* @var string
*/
protected $cachePath;
/**
* @var
*/
protected $cachePrefix;
/**
* Minimum enforced TTL in seconds
*/
const MINIMUM_TTL = 60;
/**
* @param string $directory directory to use
* @param int $timeToLiveInSeconds TTL
*/
public function __construct($directory, $timeToLiveInSeconds = 300)
{
$cachePath = PIWIK_USER_PATH . '/tmp/cache/' . $directory . '/';
$this->cachePath = SettingsPiwik::rewriteTmpPathWithHostname($cachePath);
if ($timeToLiveInSeconds < self::MINIMUM_TTL) {
$timeToLiveInSeconds = self::MINIMUM_TTL;
}
$this->ttl = $timeToLiveInSeconds;
}
/**
* Function to fetch a cache entry
*
* @param string $id The cache entry ID
* @return array|bool False on error, or array the cache content
*/
public function get($id)
{
if (empty($id)) {
return false;
}
$id = $this->cleanupId($id);
$cache_complete = false;
$content = '';
$expires_on = false;
// We are assuming that most of the time cache will exists
$cacheFilePath = $this->cachePath . $id . '.php';
if (self::$invalidateOpCacheBeforeRead) {
$this->opCacheInvalidate($cacheFilePath);
}
$ok = @include($cacheFilePath);
if ($ok && $cache_complete == true) {
if (empty($expires_on)
|| $expires_on < time()
) {
return false;
}
return $content;
}
return false;
}
private function getExpiresTime()
{
return time() + $this->ttl;
}
protected function cleanupId($id)
{
if (!Filesystem::isValidFilename($id)) {
throw new Exception("Invalid cache ID request $id");
}
return $id;
}
/**
* A function to store content a cache entry.
*
* @param string $id The cache entry ID
* @param array $content The cache content
* @throws \Exception
* @return bool True if the entry was succesfully stored
*/
public function set($id, $content)
{
if (empty($id)) {
return false;
}
if (!is_dir($this->cachePath)) {
Filesystem::mkdir($this->cachePath);
}
if (!is_writable($this->cachePath)) {
return false;
}
$id = $this->cleanupId($id);
$id = $this->cachePath . $id . '.php';
if (is_object($content)) {
throw new \Exception('You cannot use the CacheFile to cache an object, only arrays, strings and numbers.');
}
$cache_literal = "<" . "?php\n";
$cache_literal .= "$" . "content = " . var_export($content, true) . ";\n";
$cache_literal .= "$" . "expires_on = " . $this->getExpiresTime() . ";\n";
$cache_literal .= "$" . "cache_complete = true;\n";
$cache_literal .= "?" . ">";
// Write cache to a temp file, then rename it, overwriting the old cache
// On *nix systems this should guarantee atomicity
$tmp_filename = tempnam($this->cachePath, 'tmp_');
@chmod($tmp_filename, 0640);
if ($fp = @fopen($tmp_filename, 'wb')) {
@fwrite($fp, $cache_literal, strlen($cache_literal));
@fclose($fp);
if (!@rename($tmp_filename, $id)) {
// On some systems rename() doesn't overwrite destination
@unlink($id);
if (!@rename($tmp_filename, $id)) {
// Make sure that no temporary file is left over
// if the destination is not writable
@unlink($tmp_filename);
}
}
$this->opCacheInvalidate($id);
return true;
}
return false;
}
/**
* A function to delete a single cache entry
*
* @param string $id The cache entry ID
* @return bool True if the entry was succesfully deleted
*/
public function delete($id)
{
if (empty($id)) {
return false;
}
$id = $this->cleanupId($id);
$filename = $this->cachePath . $id . '.php';
if (file_exists($filename)) {
$this->opCacheInvalidate($filename);
@unlink($filename);
return true;
}
return false;
}
/**
* A function to delete all cache entries in the directory
*/
public function deleteAll()
{
$self = $this;
$beforeUnlink = function ($path) use ($self) {
$self->opCacheInvalidate($path);
};
Filesystem::unlinkRecursive($this->cachePath, $deleteRootToo = false, $beforeUnlink);
}
public function opCacheInvalidate($filepath)
{
if (function_exists('opcache_invalidate')
&& is_file($filepath)
) {
opcache_invalidate($filepath, $force = true);
}
}
}

29
www/analytics/core/CacheId.php Normal file
View file

@ -0,0 +1,29 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik;
use Piwik\Plugin\Manager;
class CacheId
{
public static function languageAware($cacheId)
{
return $cacheId . '-' . Translate::getLanguageLoaded();
}
public static function pluginAware($cacheId)
{
$pluginManager = Manager::getInstance();
$pluginNames = $pluginManager->getLoadedPluginsName();
$cacheId = $cacheId . '-' . md5(implode('', $pluginNames));
$cacheId = self::languageAware($cacheId);
return $cacheId;
}
}

247
www/analytics/core/CliMulti.php
View file

@ -1,19 +1,23 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik;
use Piwik\Archiver\Request;
use Piwik\CliMulti\CliPhp;
use Piwik\CliMulti\Output;
use Piwik\CliMulti\Process;
use Piwik\Container\StaticContainer;
/**
* Class CliMulti.
*/
class CliMulti {
class CliMulti
{
/**
* If set to true or false it will overwrite whether async is supported or not.
@ -23,17 +27,37 @@ class CliMulti {
public $supportsAsync = null;
/**
* @var \Piwik\CliMulti\Process[]
* @var Process[]
*/
private $processes = array();
/**
* @var \Piwik\CliMulti\Output[]
* If set it will issue at most concurrentProcessesLimit requests
* @var int
*/
private $concurrentProcessesLimit = null;
/**
* @var Output[]
*/
private $outputs = array();
private $acceptInvalidSSLCertificate = false;
/**
* @var bool
*/
private $runAsSuperUser = false;
/**
* Only used when doing synchronous curl requests.
*
* @var string
*/
private $urlToPiwik = null;
private $phpCliOptions = '';
public function __construct()
{
$this->supportsAsync = $this->supportsAsync();
@ -44,26 +68,38 @@ class CliMulti {
* If multi cli is not supported (eg windows) it will initiate an HTTP request instead (not async).
*
* @param string[] $piwikUrls An array of urls, for instance:
* array('http://www.example.com/piwik?module=API...')
*
* `array('http://www.example.com/piwik?module=API...')`
*
* **Make sure query parameter values are properly encoded in the URLs.**
*
* @return array The response of each URL in the same order as the URLs. The array can contain null values in case
* there was a problem with a request, for instance if the process died unexpected.
*/
public function request(array $piwikUrls)
{
$this->start($piwikUrls);
$chunks = array($piwikUrls);
if ($this->concurrentProcessesLimit) {
$chunks = array_chunk($piwikUrls, $this->concurrentProcessesLimit);
}
do {
usleep(100000); // 100 * 1000 = 100ms
} while (!$this->hasFinished());
$results = $this->getResponse($piwikUrls);
$this->cleanup();
self::cleanupNotRemovedFiles();
$results = array();
foreach ($chunks as $urlsChunk) {
$results = array_merge($results, $this->requestUrls($urlsChunk));
}
return $results;
}
/**
* Forwards the given configuration options to the PHP cli command.
* @param string $phpCliOptions eg "-d memory_limit=8G -c=path/to/php.ini"
*/
public function setPhpCliConfigurationOptions($phpCliOptions)
{
$this->phpCliOptions = (string) $phpCliOptions;
}
/**
* Ok, this sounds weird. Why should we care about ssl certificates when we are in CLI mode? It is needed for
* our simple fallback mode for Windows where we initiate HTTP requests instead of CLI.
@ -74,28 +110,51 @@ class CliMulti {
$this->acceptInvalidSSLCertificate = $acceptInvalidSSLCertificate;
}
/**
* @param $limit int Maximum count of requests to issue in parallel
*/
public function setConcurrentProcessesLimit($limit)
{
$this->concurrentProcessesLimit = $limit;
}
public function runAsSuperUser($runAsSuperUser = true)
{
$this->runAsSuperUser = $runAsSuperUser;
}
private function start($piwikUrls)
{
foreach ($piwikUrls as $index => $url) {
$cmdId = $this->generateCommandId($url) . $index;
$output = new Output($cmdId);
if ($this->supportsAsync) {
$this->executeAsyncCli($url, $output, $cmdId);
} else {
$this->executeNotAsyncHttp($url, $output);
if ($url instanceof Request) {
$url->start();
}
$this->outputs[] = $output;
$cmdId = $this->generateCommandId($url) . $index;
$this->executeUrlCommand($cmdId, $url);
}
}
private function executeUrlCommand($cmdId, $url)
{
$output = new Output($cmdId);
if ($this->supportsAsync) {
$this->executeAsyncCli($url, $output, $cmdId);
} else {
$this->executeNotAsyncHttp($url, $output);
}
$this->outputs[] = $output;
}
private function buildCommand($hostname, $query, $outputFile)
{
$bin = $this->findPhpBinary();
$superuserCommand = $this->runAsSuperUser ? "--superuser" : "";
return sprintf('%s -q %s/console climulti:request --piwik-domain=%s %s > %s 2>&1 &',
$bin, PIWIK_INCLUDE_PATH, escapeshellarg($hostname), escapeshellarg($query), $outputFile);
return sprintf('%s %s %s/console climulti:request -q --piwik-domain=%s %s %s > %s 2>&1 &',
$bin, $this->phpCliOptions, PIWIK_INCLUDE_PATH, escapeshellarg($hostname), $superuserCommand, escapeshellarg($query), $outputFile);
}
private function getResponse()
@ -119,7 +178,6 @@ class CliMulti {
// ==> declare the process as finished
$process->finishProcess();
continue;
} elseif (!$hasStarted) {
return false;
}
@ -128,6 +186,14 @@ class CliMulti {
return false;
}
$pid = $process->getPid();
foreach ($this->outputs as $output) {
if ($output->getOutputId() === $pid && $output->isAbnormal()) {
$process->finishProcess();
return true;
}
}
if ($process->hasFinished()) {
// prevent from checking this process over and over again
unset($this->processes[$index]);
@ -146,11 +212,15 @@ class CliMulti {
* What is missing under windows? Detection whether a process is still running in Process::isProcessStillRunning
* and how to send a process into background in start()
*/
private function supportsAsync()
public function supportsAsync()
{
return !SettingsServer::isWindows()
&& Process::isSupported()
&& $this->findPhpBinary();
return Process::isSupported() && !Common::isPhpCgiType() && $this->findPhpBinary();
}
private function findPhpBinary()
{
$cliPhp = new CliPhp();
return $cliPhp->findPhpBinary();
}
private function cleanup()
@ -176,61 +246,33 @@ class CliMulti {
$timeOneWeekAgo = strtotime('-1 week');
$files = _glob(self::getTmpPath() . '/*');
if(empty($files)) {
if (empty($files)) {
return;
}
foreach ($files as $file) {
$timeLastModified = filemtime($file);
if (file_exists($file)) {
$timeLastModified = filemtime($file);
if ($timeOneWeekAgo > $timeLastModified) {
unlink($file);
if ($timeLastModified !== false && $timeOneWeekAgo > $timeLastModified) {
unlink($file);
}
}
}
}
public static function getTmpPath()
{
$dir = PIWIK_INCLUDE_PATH . '/tmp/climulti';
return SettingsPiwik::rewriteTmpPathWithHostname($dir);
}
private function findPhpBinary()
{
if (defined('PHP_BINARY') && false === strpos(PHP_BINARY, 'fpm')) {
return PHP_BINARY;
}
$bin = '';
if (!empty($_SERVER['_']) && Common::isPhpCliMode()) {
$bin = $this->getPhpCommandIfValid($_SERVER['_']);
}
if (empty($bin) && !empty($_SERVER['argv'][0]) && Common::isPhpCliMode()) {
$bin = $this->getPhpCommandIfValid($_SERVER['argv'][0]);
}
if (empty($bin)) {
$bin = shell_exec('which php');
}
if (empty($bin)) {
$bin = shell_exec('which php5');
}
if (!empty($bin)) {
return trim($bin);
}
return StaticContainer::get('path.tmp') . '/climulti';
}
private function executeAsyncCli($url, Output $output, $cmdId)
{
$this->processes[] = new Process($cmdId);
$url = $this->appendTestmodeParamToUrlIfNeeded($url);
$query = UrlHelper::getQueryFromUrl($url, array('pid' => $cmdId));
$hostname = UrlHelper::getHostFromUrl($url);
$url = $this->appendTestmodeParamToUrlIfNeeded($url);
$query = UrlHelper::getQueryFromUrl($url, array('pid' => $cmdId));
$hostname = Url::getHost($checkIfTrusted = false);
$command = $this->buildCommand($hostname, $query, $output->getPathToFile());
Log::debug($command);
@ -239,6 +281,29 @@ class CliMulti {
private function executeNotAsyncHttp($url, Output $output)
{
$piwikUrl = $this->urlToPiwik ?: SettingsPiwik::getPiwikUrl();
if (empty($piwikUrl)) {
$piwikUrl = 'http://' . Url::getHost() . '/';
}
$url = $piwikUrl . $url;
if (Config::getInstance()->General['force_ssl'] == 1) {
$url = str_replace("http://", "https://", $url);
}
if ($this->runAsSuperUser) {
$tokenAuths = self::getSuperUserTokenAuths();
$tokenAuth = reset($tokenAuths);
if (strpos($url, '?') === false) {
$url .= '?';
} else {
$url .= '&';
}
$url .= 'token_auth=' . $tokenAuth;
}
try {
Log::debug("Execute HTTP API request: " . $url);
$response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate);
@ -246,19 +311,21 @@ class CliMulti {
} catch (\Exception $e) {
$message = "Got invalid response from API request: $url. ";
if (empty($response)) {
if (isset($response) && empty($response)) {
$message .= "The response was empty. This usually means a server error. This solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. Please check your Web server Error Log file for more details.";
} else {
$message .= "Response was '" . $e->getMessage() . "'";
}
$output->write($message);
Log::debug($e);
}
}
private function appendTestmodeParamToUrlIfNeeded($url)
{
$isTestMode = $url && false !== strpos($url, 'tests/PHPUnit/proxy');
$isTestMode = defined('PIWIK_TEST_MODE');
if ($isTestMode && false === strpos($url, '?')) {
$url .= "?testmode=1";
@ -269,12 +336,42 @@ class CliMulti {
return $url;
}
private function getPhpCommandIfValid($path)
/**
* @param array $piwikUrls
* @return array
*/
private function requestUrls(array $piwikUrls)
{
if (!empty($path) && is_executable($path)) {
if (0 === strpos($path, PHP_BINDIR) && false === strpos($path, 'phpunit')) {
return $path;
}
}
$this->start($piwikUrls);
do {
usleep(100000); // 100 * 1000 = 100ms
} while (!$this->hasFinished());
$results = $this->getResponse($piwikUrls);
$this->cleanup();
self::cleanupNotRemovedFiles();
return $results;
}
private static function getSuperUserTokenAuths()
{
$tokens = array();
/**
* Used to be in CronArchive, moved to CliMulti.
*
* @ignore
*/
Piwik::postEvent('CronArchive.getTokenAuth', array(&$tokens));
return $tokens;
}
public function setUrlToPiwik($urlToPiwik)
{
$this->urlToPiwik = $urlToPiwik;
}
}

101
www/analytics/core/CliMulti/CliPhp.php Normal file
View file

@ -0,0 +1,101 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\CliMulti;
use Piwik\Common;
class CliPhp
{
public function findPhpBinary()
{
if (defined('PHP_BINARY')) {
if ($this->isHhvmBinary(PHP_BINARY)) {
return PHP_BINARY . ' --php';
}
if ($this->isValidPhpType(PHP_BINARY)) {
return PHP_BINARY . ' -q';
}
}
$bin = '';
if (!empty($_SERVER['_']) && Common::isPhpCliMode()) {
$bin = $this->getPhpCommandIfValid($_SERVER['_']);
}
if (empty($bin) && !empty($_SERVER['argv'][0]) && Common::isPhpCliMode()) {
$bin = $this->getPhpCommandIfValid($_SERVER['argv'][0]);
}
if (!$this->isValidPhpType($bin)) {
$bin = shell_exec('which php');
}
if (!$this->isValidPhpType($bin)) {
$bin = shell_exec('which php5');
}
if (!$this->isValidPhpType($bin)) {
return false;
}
$bin = trim($bin);
if (!$this->isValidPhpVersion($bin)) {
return false;
}
$bin .= ' -q';
return $bin;
}
private function isHhvmBinary($bin)
{
return false !== strpos($bin, 'hhvm');
}
private function isValidPhpVersion($bin)
{
global $piwik_minimumPHPVersion;
$cliVersion = $this->getPhpVersion($bin);
$isCliVersionValid = version_compare($piwik_minimumPHPVersion, $cliVersion) <= 0;
return $isCliVersionValid;
}
private function isValidPhpType($path)
{
return !empty($path)
&& false === strpos($path, 'fpm')
&& false === strpos($path, 'cgi')
&& false === strpos($path, 'phpunit');
}
private function getPhpCommandIfValid($path)
{
if (!empty($path) && is_executable($path)) {
if (0 === strpos($path, PHP_BINDIR) && $this->isValidPhpType($path)) {
return $path;
}
}
return null;
}
/**
* @param string $bin PHP binary
* @return string
*/
private function getPhpVersion($bin)
{
$command = sprintf("%s -r 'echo phpversion();'", $bin);
$version = shell_exec($command);
return $version;
}
}

25
www/analytics/core/CliMulti/Output.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -10,9 +10,11 @@ namespace Piwik\CliMulti;
use Piwik\CliMulti;
use Piwik\Filesystem;
class Output {
class Output
{
private $tmpFile = '';
private $outputId = null;
public function __construct($outputId)
{
@ -21,8 +23,15 @@ class Output {
}
$dir = CliMulti::getTmpPath();
Filesystem::mkdir($dir, true);
$this->tmpFile = $dir . '/' . $outputId . '.output';
Filesystem::mkdir($dir);
$this->tmpFile = $dir . '/' . $outputId . '.output';
$this->outputId = $outputId;
}
public function getOutputId()
{
return $this->outputId;
}
public function write($content)
@ -35,6 +44,13 @@ class Output {
return $this->tmpFile;
}
public function isAbnormal()
{
$size = Filesystem::getFileSize($this->tmpFile, 'MB');
return $size !== null && $size >= 100;
}
public function exists()
{
return file_exists($this->tmpFile);
@ -49,5 +65,4 @@ class Output {
{
Filesystem::deleteFileIfExists($this->tmpFile);
}
}

84
www/analytics/core/CliMulti/Process.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -24,6 +24,7 @@ class Process
private $pidFile = '';
private $timeCreation = null;
private $isSupported = null;
private $pid = null;
public function __construct($pid)
{
@ -32,15 +33,21 @@ class Process
}
$pidDir = CliMulti::getTmpPath();
Filesystem::mkdir($pidDir, true);
Filesystem::mkdir($pidDir);
$this->isSupported = self::isSupported();
$this->pidFile = $pidDir . '/' . $pid . '.pid';
$this->timeCreation = time();
$this->pid = $pid;
$this->markAsNotStarted();
}
public function getPid()
{
return $this->pid;
}
private function markAsNotStarted()
{
$content = $this->getPidFileContent();
@ -97,6 +104,11 @@ class Process
return false;
}
if (!$this->pidFileSizeIsNormal()) {
$this->finishProcess();
return false;
}
if ($this->isProcessStillRunning($content)) {
return true;
}
@ -108,6 +120,13 @@ class Process
return false;
}
private function pidFileSizeIsNormal()
{
$size = Filesystem::getFileSize($this->pidFile);
return $size !== null && $size < 500;
}
public function finishProcess()
{
Filesystem::deleteFileIfExists($this->pidFile);
@ -125,7 +144,7 @@ class Process
}
$lockedPID = trim($content);
$runningPIDs = explode("\n", trim( `ps -e | awk '{print $1}'` ));
$runningPIDs = self::getRunningProcesses();
return !empty($lockedPID) && in_array($lockedPID, $runningPIDs);
}
@ -154,17 +173,30 @@ class Process
return false;
}
if (static::commandExists('ps') && self::returnsSuccessCode('ps') && self::commandExists('awk')) {
if (!self::commandExists('ps') || !self::returnsSuccessCode('ps') || !self::commandExists('awk')) {
return false;
}
if (count(self::getRunningProcesses()) > 0) {
return true;
}
return false;
if (!self::isProcFSMounted()) {
return false;
}
return true;
}
private static function isSystemNotSupported()
{
$uname = shell_exec('uname -a');
if(strpos($uname, 'synology') !== false) {
$uname = @shell_exec('uname -a 2> /dev/null');
if (empty($uname)) {
$uname = php_uname();
}
if (strpos($uname, 'synology') !== false) {
return true;
}
return false;
@ -175,12 +207,12 @@ class Process
$command = 'shell_exec';
$disabled = explode(',', ini_get('disable_functions'));
$disabled = array_map('trim', $disabled);
return in_array($command, $disabled);
return in_array($command, $disabled) || !function_exists($command);
}
private static function returnsSuccessCode($command)
{
$exec = $command . ' > /dev/null 2>&1 & echo $?';
$exec = $command . ' > /dev/null 2>&1; echo $?';
$returnCode = shell_exec($exec);
$returnCode = trim($returnCode);
return 0 == (int) $returnCode;
@ -188,8 +220,38 @@ class Process
private static function commandExists($command)
{
$result = shell_exec('which ' . escapeshellarg($command));
$result = @shell_exec('which ' . escapeshellarg($command) . ' 2> /dev/null');
return !empty($result);
}
}
/**
* ps -e requires /proc
* @return bool
*/
private static function isProcFSMounted()
{
if (is_resource(@fopen('/proc', 'r'))) {
return true;
}
// Testing if /proc is a resource with @fopen fails on systems with open_basedir set.
// by using stat we not only test the existance of /proc but also confirm it's a 'proc' filesystem
$type = @shell_exec('stat -f -c "%T" /proc 2>/dev/null');
return strpos($type, 'proc') === 0;
}
/**
* @return int[] The ids of the currently running processes
*/
public static function getRunningProcesses()
{
$ids = explode("\n", trim(`ps ex 2>/dev/null | awk '{print $1}' 2>/dev/null`));
$ids = array_map('intval', $ids);
$ids = array_filter($ids, function ($id) {
return $id > 0;
});
return $ids;
}
}

64
www/analytics/core/CliMulti/RequestCommand.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -8,6 +8,12 @@
namespace Piwik\CliMulti;
use Piwik\Application\Environment;
use Piwik\Access;
use Piwik\Container\StaticContainer;
use Piwik\Db;
use Piwik\Log;
use Piwik\Option;
use Piwik\Plugin\ConsoleCommand;
use Piwik\Url;
use Piwik\UrlHelper;
@ -15,33 +21,41 @@ use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Piwik\Config;
use Piwik\Common;
use Piwik\FrontController;
/**
* RequestCommand
*/
class RequestCommand extends ConsoleCommand
{
/**
* @var Environment
*/
private $environment;
protected function configure()
{
$this->setName('climulti:request');
$this->setDescription('Parses and executes the given query. See Piwik\CliMulti. Intended only for system usage.');
$this->addArgument('url-query', null, InputOption::VALUE_REQUIRED, 'Piwik URL query string, for instance: "module=API&method=API.getPiwikVersion&token_auth=123456789"');
$this->addArgument('url-query', InputArgument::REQUIRED, 'Piwik URL query string, for instance: "module=API&method=API.getPiwikVersion&token_auth=123456789"');
$this->addOption('superuser', null, InputOption::VALUE_NONE, 'If supplied, runs the code as superuser.');
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$this->recreateContainerWithWebEnvironment();
$this->initHostAndQueryString($input);
if ($this->isTestModeEnabled()) {
Config::getInstance()->setTestEnvironment();
$indexFile = '/tests/PHPUnit/proxy/index.php';
$indexFile = '/tests/PHPUnit/proxy/';
$this->resetDatabase();
} else {
$indexFile = '/index.php';
$indexFile = '/';
}
$indexFile .= 'index.php';
if (!empty($_GET['pid'])) {
$process = new Process($_GET['pid']);
@ -52,6 +66,16 @@ class RequestCommand extends ConsoleCommand
$process->startProcess();
}
if ($input->getOption('superuser')) {
StaticContainer::addDefinitions(array(
'observers.global' => \DI\add(array(
array('Environment.bootstrapped', function () {
Access::getInstance()->setSuperUserAccess(true);
})
)),
));
}
require_once PIWIK_INCLUDE_PATH . $indexFile;
if (!empty($process)) {
@ -75,10 +99,30 @@ class RequestCommand extends ConsoleCommand
Url::setHost($hostname);
$query = $input->getArgument('url-query');
$query = UrlHelper::getArrayFromQueryString($query);
$query = UrlHelper::getArrayFromQueryString($query); // NOTE: this method can create the StaticContainer now
foreach ($query as $name => $value) {
$_GET[$name] = $value;
}
}
}
/**
* We will be simulating an HTTP request here (by including index.php).
*
* To avoid weird side-effects (e.g. the logging output messing up the HTTP response on the CLI output)
* we need to recreate the container with the default environment instead of the CLI environment.
*/
private function recreateContainerWithWebEnvironment()
{
StaticContainer::clearContainer();
Log::unsetInstance();
$this->environment = new Environment(null);
$this->environment->init();
}
private function resetDatabase()
{
Option::clearCache();
Db::destroyDatabaseObject();
}
}

242
www/analytics/core/Columns/Dimension.php Normal file
View file

@ -0,0 +1,242 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Columns;
use Exception;
use Piwik\Plugin;
use Piwik\Plugin\ComponentFactory;
use Piwik\Plugin\Dimension\ActionDimension;
use Piwik\Plugin\Dimension\ConversionDimension;
use Piwik\Plugin\Dimension\VisitDimension;
use Piwik\Plugin\Segment;
/**
* @api
* @since 2.5.0
*/
abstract class Dimension
{
const COMPONENT_SUBNAMESPACE = 'Columns';
// TODO that we have quite a few @ignore in public methods might show we should maybe split some code into two
// classes.
/**
* This will be the name of the column in the database table if a $columnType is specified.
* @var string
* @api
*/
protected $columnName = '';
/**
* If a columnType is defined, we will create a column in the MySQL table having this type. Please make sure
* MySQL understands this type. Once you change the column type the Piwik platform will notify the user to
* perform an update which can sometimes take a long time so be careful when choosing the correct column type.
* @var string
* @api
*/
protected $columnType = '';
/**
* Holds an array of segment instances
* @var Segment[]
*/
protected $segments = array();
/**
* Overwrite this method to configure segments. To do so just create an instance of a {@link \Piwik\Plugin\Segment}
* class, configure it and call the {@link addSegment()} method. You can add one or more segments for this
* dimension. Example:
*
* ```
* $segment = new Segment();
* $segment->setSegment('exitPageUrl');
* $segment->setName('Actions_ColumnExitPageURL');
* $segment->setCategory('General_Visit');
* $this->addSegment($segment);
* ```
*/
protected function configureSegments()
{
}
/**
* Check whether a dimension has overwritten a specific method.
* @param $method
* @return bool
* @ignore
*/
public function hasImplementedEvent($method)
{
$method = new \ReflectionMethod($this, $method);
$declaringClass = $method->getDeclaringClass();
return 0 === strpos($declaringClass->name, 'Piwik\Plugins');
}
/**
* Adds a new segment. The segment type will be set to 'dimension' automatically if not already set.
* @param Segment $segment
* @api
*/
protected function addSegment(Segment $segment)
{
$type = $segment->getType();
if (empty($type)) {
$segment->setType(Segment::TYPE_DIMENSION);
}
$this->segments[] = $segment;
}
/**
* Get the list of configured segments.
* @return Segment[]
* @ignore
*/
public function getSegments()
{
if (empty($this->segments)) {
$this->configureSegments();
}
return $this->segments;
}
/**
* Get the name of the dimension column.
* @return string
* @ignore
*/
public function getColumnName()
{
return $this->columnName;
}
/**
* Check whether the dimension has a column type configured
* @return bool
* @ignore
*/
public function hasColumnType()
{
return !empty($this->columnType);
}
/**
* Get the translated name of the dimension. Defaults to an empty string.
* @return string
* @api
*/
public function getName()
{
return '';
}
/**
* Returns a unique string ID for this dimension. The ID is built using the namespaced class name
* of the dimension, but is modified to be more human readable.
*
* @return string eg, `"Referrers.Keywords"`
* @throws Exception if the plugin and simple class name of this instance cannot be determined.
* This would only happen if the dimension is located in the wrong directory.
* @api
*/
public function getId()
{
$className = get_class($this);
// parse plugin name & dimension name
$regex = "/Piwik\\\\Plugins\\\\([^\\\\]+)\\\\" . self::COMPONENT_SUBNAMESPACE . "\\\\([^\\\\]+)/";
if (!preg_match($regex, $className, $matches)) {
throw new Exception("'$className' is located in the wrong directory.");
}
$pluginName = $matches[1];
$dimensionName = $matches[2];
return $pluginName . '.' . $dimensionName;
}
/**
* Gets an instance of all available visit, action and conversion dimension.
* @return Dimension[]
*/
public static function getAllDimensions()
{
$dimensions = array();
foreach (VisitDimension::getAllDimensions() as $dimension) {
$dimensions[] = $dimension;
}
foreach (ActionDimension::getAllDimensions() as $dimension) {
$dimensions[] = $dimension;
}
foreach (ConversionDimension::getAllDimensions() as $dimension) {
$dimensions[] = $dimension;
}
return $dimensions;
}
public static function getDimensions(Plugin $plugin)
{
$dimensions = array();
foreach (VisitDimension::getDimensions($plugin) as $dimension) {
$dimensions[] = $dimension;
}
foreach (ActionDimension::getDimensions($plugin) as $dimension) {
$dimensions[] = $dimension;
}
foreach (ConversionDimension::getDimensions($plugin) as $dimension) {
$dimensions[] = $dimension;
}
return $dimensions;
}
/**
* Creates a Dimension instance from a string ID (see {@link getId()}).
*
* @param string $dimensionId See {@link getId()}.
* @return Dimension|null The created instance or null if there is no Dimension for
* $dimensionId or if the plugin that contains the Dimension is
* not loaded.
* @api
*/
public static function factory($dimensionId)
{
list($module, $dimension) = explode('.', $dimensionId);
return ComponentFactory::factory($module, $dimension, __CLASS__);
}
/**
* Returns the name of the plugin that contains this Dimension.
*
* @return string
* @throws Exception if the Dimension is not located within a Plugin module.
* @api
*/
public function getModule()
{
$id = $this->getId();
if (empty($id)) {
throw new Exception("Invalid dimension ID: '$id'.");
}
$parts = explode('.', $id);
return reset($parts);
}
}

372
www/analytics/core/Columns/Updater.php Normal file
View file

@ -0,0 +1,372 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Columns;
use Piwik\Common;
use Piwik\DbHelper;
use Piwik\Plugin\Dimension\ActionDimension;
use Piwik\Plugin\Dimension\VisitDimension;
use Piwik\Plugin\Dimension\ConversionDimension;
use Piwik\Db;
use Piwik\Updater as PiwikUpdater;
use Piwik\Filesystem;
use Piwik\Cache as PiwikCache;
/**
* Class that handles dimension updates
*/
class Updater extends \Piwik\Updates
{
private static $cacheId = 'AllDimensionModifyTime';
/**
* @var VisitDimension[]
*/
public $visitDimensions;
/**
* @var ActionDimension[]
*/
private $actionDimensions;
/**
* @var ConversionDimension[]
*/
private $conversionDimensions;
/**
* @param VisitDimension[]|null $visitDimensions
* @param ActionDimension[]|null $actionDimensions
* @param ConversionDimension[]|null $conversionDimensions
*/
public function __construct(array $visitDimensions = null, array $actionDimensions = null, array $conversionDimensions = null)
{
$this->visitDimensions = $visitDimensions;
$this->actionDimensions = $actionDimensions;
$this->conversionDimensions = $conversionDimensions;
}
public function getMigrationQueries(PiwikUpdater $updater)
{
$sqls = array();
$changingColumns = $this->getUpdates($updater);
foreach ($changingColumns as $table => $columns) {
if (empty($columns) || !is_array($columns)) {
continue;
}
$sqls["ALTER TABLE `" . Common::prefixTable($table) . "` " . implode(', ', $columns)] = array('1091', '1060');
}
return $sqls;
}
public function doUpdate(PiwikUpdater $updater)
{
$updater->executeMigrationQueries(__FILE__, $this->getMigrationQueries($updater));
}
private function getVisitDimensions()
{
// see eg https://github.com/piwik/piwik/issues/8399 we fetch them only on demand to improve performance
if (!isset($this->visitDimensions)) {
$this->visitDimensions = VisitDimension::getAllDimensions();
}
return $this->visitDimensions;
}
private function getActionDimensions()
{
// see eg https://github.com/piwik/piwik/issues/8399 we fetch them only on demand to improve performance
if (!isset($this->actionDimensions)) {
$this->actionDimensions = ActionDimension::getAllDimensions();
}
return $this->actionDimensions;
}
private function getConversionDimensions()
{
// see eg https://github.com/piwik/piwik/issues/8399 we fetch them only on demand to improve performance
if (!isset($this->conversionDimensions)) {
$this->conversionDimensions = ConversionDimension::getAllDimensions();
}
return $this->conversionDimensions;
}
private function getUpdates(PiwikUpdater $updater)
{
$visitColumns = DbHelper::getTableColumns(Common::prefixTable('log_visit'));
$actionColumns = DbHelper::getTableColumns(Common::prefixTable('log_link_visit_action'));
$conversionColumns = DbHelper::getTableColumns(Common::prefixTable('log_conversion'));
$allUpdatesToRun = array();
foreach ($this->getVisitDimensions() as $dimension) {
$updates = $this->getUpdatesForDimension($updater, $dimension, 'log_visit.', $visitColumns, $conversionColumns);
$allUpdatesToRun = $this->mixinUpdates($allUpdatesToRun, $updates);
}
foreach ($this->getActionDimensions() as $dimension) {
$updates = $this->getUpdatesForDimension($updater, $dimension, 'log_link_visit_action.', $actionColumns);
$allUpdatesToRun = $this->mixinUpdates($allUpdatesToRun, $updates);
}
foreach ($this->getConversionDimensions() as $dimension) {
$updates = $this->getUpdatesForDimension($updater, $dimension, 'log_conversion.', $conversionColumns);
$allUpdatesToRun = $this->mixinUpdates($allUpdatesToRun, $updates);
}
return $allUpdatesToRun;
}
/**
* @param ActionDimension|ConversionDimension|VisitDimension $dimension
* @param string $componentPrefix
* @param array $existingColumnsInDb
* @param array $conversionColumns
* @return array
*/
private function getUpdatesForDimension(PiwikUpdater $updater, $dimension, $componentPrefix, $existingColumnsInDb, $conversionColumns = array())
{
$column = $dimension->getColumnName();
$componentName = $componentPrefix . $column;
if (!$updater->hasNewVersion($componentName)) {
return array();
}
if (array_key_exists($column, $existingColumnsInDb)) {
if ($dimension instanceof VisitDimension) {
$sqlUpdates = $dimension->update($conversionColumns);
} else {
$sqlUpdates = $dimension->update();
}
} else {
$sqlUpdates = $dimension->install();
}
return $sqlUpdates;
}
private function mixinUpdates($allUpdatesToRun, $updatesFromDimension)
{
if (!empty($updatesFromDimension)) {
foreach ($updatesFromDimension as $table => $col) {
if (empty($allUpdatesToRun[$table])) {
$allUpdatesToRun[$table] = $col;
} else {
$allUpdatesToRun[$table] = array_merge($allUpdatesToRun[$table], $col);
}
}
}
return $allUpdatesToRun;
}
public function getAllVersions(PiwikUpdater $updater)
{
// to avoid having to load all dimensions on each request we check if there were any changes on the file system
// can easily save > 100ms for each request
$cachedTimes = self::getCachedDimensionFileChanges();
$currentTimes = self::getCurrentDimensionFileChanges();
$diff = array_diff_assoc($currentTimes, $cachedTimes);
if (empty($diff)) {
return array();
}
$versions = array();
$visitColumns = DbHelper::getTableColumns(Common::prefixTable('log_visit'));
$actionColumns = DbHelper::getTableColumns(Common::prefixTable('log_link_visit_action'));
$conversionColumns = DbHelper::getTableColumns(Common::prefixTable('log_conversion'));
foreach ($this->getVisitDimensions() as $dimension) {
$versions = $this->mixinVersions($updater, $dimension, VisitDimension::INSTALLER_PREFIX, $visitColumns, $versions);
}
foreach ($this->getActionDimensions() as $dimension) {
$versions = $this->mixinVersions($updater, $dimension, ActionDimension::INSTALLER_PREFIX, $actionColumns, $versions);
}
foreach ($this->getConversionDimensions() as $dimension) {
$versions = $this->mixinVersions($updater, $dimension, ConversionDimension::INSTALLER_PREFIX, $conversionColumns, $versions);
}
return $versions;
}
/**
* @param ActionDimension|ConversionDimension|VisitDimension $dimension
* @param string $componentPrefix
* @param array $columns
* @param array $versions
* @return array The modified versions array
*/
private function mixinVersions(PiwikUpdater $updater, $dimension, $componentPrefix, $columns, $versions)
{
$columnName = $dimension->getColumnName();
// dimensions w/o columns do not need DB updates
if (!$columnName || !$dimension->hasColumnType()) {
return $versions;
}
$component = $componentPrefix . $columnName;
$version = $dimension->getVersion();
// if the column exists in the table, but has no associated version, and was one of the core columns
// that was moved when the dimension refactor took place, then:
// - set the installed version in the DB to the current code version
// - and do not check for updates since we just set the version to the latest
if (array_key_exists($columnName, $columns)
&& false === $updater->getCurrentComponentVersion($component)
&& self::wasDimensionMovedFromCoreToPlugin($component, $version)
) {
$updater->markComponentSuccessfullyUpdated($component, $version);
return $versions;
}
$versions[$component] = $version;
return $versions;
}
public static function isDimensionComponent($name)
{
return 0 === strpos($name, 'log_visit.')
|| 0 === strpos($name, 'log_conversion.')
|| 0 === strpos($name, 'log_conversion_item.')
|| 0 === strpos($name, 'log_link_visit_action.');
}
public static function wasDimensionMovedFromCoreToPlugin($name, $version)
{
// maps names of core dimension columns that were part of the original dimension refactor with their
// initial "version" strings. The '1' that is sometimes appended to the end of the string (sometimes seen as
// NULL1) is from individual dimension "versioning" logic (eg, see VisitDimension::getVersion())
$initialCoreDimensionVersions = array(
'log_visit.config_resolution' => 'VARCHAR(9) NOT NULL',
'log_visit.config_device_brand' => 'VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL',
'log_visit.config_device_model' => 'VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL',
'log_visit.config_windowsmedia' => 'TINYINT(1) NOT NULL',
'log_visit.config_silverlight' => 'TINYINT(1) NOT NULL',
'log_visit.config_java' => 'TINYINT(1) NOT NULL',
'log_visit.config_gears' => 'TINYINT(1) NOT NULL',
'log_visit.config_pdf' => 'TINYINT(1) NOT NULL',
'log_visit.config_quicktime' => 'TINYINT(1) NOT NULL',
'log_visit.config_realplayer' => 'TINYINT(1) NOT NULL',
'log_visit.config_device_type' => 'TINYINT( 100 ) NULL DEFAULT NULL',
'log_visit.visitor_localtime' => 'TIME NOT NULL',
'log_visit.location_region' => 'char(2) DEFAULT NULL1',
'log_visit.visitor_days_since_last' => 'SMALLINT(5) UNSIGNED NOT NULL',
'log_visit.location_longitude' => 'float(10, 6) DEFAULT NULL1',
'log_visit.visit_total_events' => 'SMALLINT(5) UNSIGNED NOT NULL',
'log_visit.config_os_version' => 'VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL',
'log_visit.location_city' => 'varchar(255) DEFAULT NULL1',
'log_visit.location_country' => 'CHAR(3) NOT NULL1',
'log_visit.location_latitude' => 'float(10, 6) DEFAULT NULL1',
'log_visit.config_flash' => 'TINYINT(1) NOT NULL',
'log_visit.config_director' => 'TINYINT(1) NOT NULL',
'log_visit.visit_total_time' => 'SMALLINT(5) UNSIGNED NOT NULL',
'log_visit.visitor_count_visits' => 'SMALLINT(5) UNSIGNED NOT NULL1',
'log_visit.visit_entry_idaction_name' => 'INTEGER(11) UNSIGNED NOT NULL',
'log_visit.visit_entry_idaction_url' => 'INTEGER(11) UNSIGNED NOT NULL',
'log_visit.visitor_returning' => 'TINYINT(1) NOT NULL1',
'log_visit.visitor_days_since_order' => 'SMALLINT(5) UNSIGNED NOT NULL1',
'log_visit.visit_goal_buyer' => 'TINYINT(1) NOT NULL',
'log_visit.visit_first_action_time' => 'DATETIME NOT NULL',
'log_visit.visit_goal_converted' => 'TINYINT(1) NOT NULL',
'log_visit.visitor_days_since_first' => 'SMALLINT(5) UNSIGNED NOT NULL1',
'log_visit.visit_exit_idaction_name' => 'INTEGER(11) UNSIGNED NOT NULL',
'log_visit.visit_exit_idaction_url' => 'INTEGER(11) UNSIGNED NULL DEFAULT 0',
'log_visit.config_browser_version' => 'VARCHAR(20) NOT NULL',
'log_visit.config_browser_name' => 'VARCHAR(10) NOT NULL',
'log_visit.config_browser_engine' => 'VARCHAR(10) NOT NULL',
'log_visit.location_browser_lang' => 'VARCHAR(20) NOT NULL',
'log_visit.config_os' => 'CHAR(3) NOT NULL',
'log_visit.config_cookie' => 'TINYINT(1) NOT NULL',
'log_visit.referer_url' => 'TEXT NOT NULL',
'log_visit.visit_total_searches' => 'SMALLINT(5) UNSIGNED NOT NULL',
'log_visit.visit_total_actions' => 'SMALLINT(5) UNSIGNED NOT NULL',
'log_visit.referer_keyword' => 'VARCHAR(255) NULL1',
'log_visit.referer_name' => 'VARCHAR(70) NULL1',
'log_visit.referer_type' => 'TINYINT(1) UNSIGNED NULL1',
'log_visit.user_id' => 'VARCHAR(200) NULL',
'log_link_visit_action.idaction_name' => 'INTEGER(10) UNSIGNED',
'log_link_visit_action.idaction_url' => 'INTEGER(10) UNSIGNED DEFAULT NULL',
'log_link_visit_action.server_time' => 'DATETIME NOT NULL',
'log_link_visit_action.time_spent_ref_action' => 'INTEGER(10) UNSIGNED NOT NULL',
'log_link_visit_action.idaction_event_action' => 'INTEGER(10) UNSIGNED DEFAULT NULL',
'log_link_visit_action.idaction_event_category' => 'INTEGER(10) UNSIGNED DEFAULT NULL',
'log_conversion.revenue_discount' => 'float default NULL',
'log_conversion.revenue' => 'float default NULL',
'log_conversion.revenue_shipping' => 'float default NULL',
'log_conversion.revenue_subtotal' => 'float default NULL',
'log_conversion.revenue_tax' => 'float default NULL',
);
if (!array_key_exists($name, $initialCoreDimensionVersions)) {
return false;
}
return strtolower($initialCoreDimensionVersions[$name]) === strtolower($version);
}
public function onNoUpdateAvailable($versionsThatWereChecked)
{
if (!empty($versionsThatWereChecked)) {
// invalidate cache only if there were actually file changes before, otherwise we write the cache on each
// request. There were versions checked only if there was a file change but no update, meaning we can
// set the cache and declare this state as "no update available".
self::cacheCurrentDimensionFileChanges();
}
}
private static function getCurrentDimensionFileChanges()
{
$files = Filesystem::globr(PIWIK_INCLUDE_PATH . '/plugins/*/Columns', '*.php');
$times = array();
foreach ($files as $file) {
$times[$file] = filemtime($file);
}
return $times;
}
private static function cacheCurrentDimensionFileChanges()
{
$changes = self::getCurrentDimensionFileChanges();
$cache = self::buildCache();
$cache->save(self::$cacheId, $changes);
}
private static function buildCache()
{
return PiwikCache::getEagerCache();
}
private static function getCachedDimensionFileChanges()
{
$cache = self::buildCache();
if ($cache->contains(self::$cacheId)) {
return $cache->fetch(self::$cacheId);
}
return array();
}
}

557
www/analytics/core/Common.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,9 +9,11 @@
namespace Piwik;
use Exception;
use Piwik\Container\StaticContainer;
use Piwik\Intl\Data\Provider\LanguageDataProvider;
use Piwik\Intl\Data\Provider\RegionDataProvider;
use Piwik\Plugins\UserCountry\LocationProvider\DefaultProvider;
use Piwik\Tracker;
use Piwik\Tracker\Cache;
use Piwik\Tracker\Cache as TrackerCache;
/**
* Contains helper methods used by both Piwik Core and the Piwik Tracking engine.
@ -34,6 +36,7 @@ class Common
/*
* Database
*/
const LANGUAGE_CODE_INVALID = 'xx';
/**
* Hashes a string into an integer which should be very low collision risks
@ -48,7 +51,7 @@ class Common
/**
* Returns a prefixed table name.
*
*
* The table prefix is determined by the `[database] tables_prefix` INI config
* option.
*
@ -82,7 +85,7 @@ class Common
*
* The table prefix is determined by the `[database] tables_prefix` INI config
* option.
*
*
* @param string $table The prefixed table name, eg "piwik-production_log_visit".
* @return string The unprefixed table name, eg "log_visit".
* @api
@ -107,12 +110,12 @@ class Common
*/
public static function isGoalPluginEnabled()
{
return \Piwik\Plugin\Manager::getInstance()->isPluginActivated('Goals');
return Plugin\Manager::getInstance()->isPluginActivated('Goals');
}
public static function isActionsPluginEnabled()
{
return \Piwik\Plugin\Manager::getInstance()->isPluginActivated('Actions');
return Plugin\Manager::getInstance()->isPluginActivated('Actions');
}
/**
@ -127,21 +130,44 @@ class Common
return self::$isCliMode;
}
$remoteAddr = @$_SERVER['REMOTE_ADDR'];
return PHP_SAPI == 'cli' ||
(!strncmp(PHP_SAPI, 'cgi', 3) && empty($remoteAddr));
if(PHP_SAPI == 'cli'){
return true;
}
if(self::isPhpCgiType() && (!isset($_SERVER['REMOTE_ADDR']) || empty($_SERVER['REMOTE_ADDR']))){
return true;
}
return false;
}
/**
* Returns true if the current request is a console command, eg. ./console xx:yy
* Returns true if PHP is executed as CGI type.
*
* @since added in 0.4.4
* @return bool true if PHP invoked as a CGI
*/
public static function isPhpCgiType()
{
$sapiType = php_sapi_name();
return substr($sapiType, 0, 3) === 'cgi';
}
/**
* Returns true if the current request is a console command, eg.
* ./console xx:yy
* or
* php console xx:yy
*
* @return bool
*/
public static function isRunningConsoleCommand()
{
$searched = '/console';
$searched = 'console';
$consolePos = strpos($_SERVER['SCRIPT_NAME'], $searched);
$expectedConsolePos = strlen($_SERVER['SCRIPT_NAME']) - strlen($searched);
$isScriptIsConsole = $consolePos == $expectedConsolePos;
$isScriptIsConsole = ($consolePos === $expectedConsolePos);
return self::isPhpCliMode() && $isScriptIsConsole;
}
@ -151,7 +177,7 @@ class Common
/**
* Multi-byte substr() - works with UTF-8.
*
*
* Calls `mb_substr` if available and falls back to `substr` if it's not.
*
* @param string $string
@ -175,7 +201,7 @@ class Common
/**
* Multi-byte strlen() - works with UTF-8
*
*
* Calls `mb_substr` if available and falls back to `substr` if not.
*
* @param string $string
@ -193,9 +219,9 @@ class Common
/**
* Multi-byte strtolower() - works with UTF-8.
*
*
* Calls `mb_strtolower` if available and falls back to `strtolower` if not.
*
*
* @param string $string
* @return string
* @api
@ -215,18 +241,18 @@ class Common
/**
* Sanitizes a string to help avoid XSS vulnerabilities.
*
*
* This function is automatically called when {@link getRequestVar()} is called,
* so you should not normally have to use it.
*
*
* This function should be used when outputting data that isn't escaped and was
* obtained from the user (for example when using the `|raw` twig filter on goal names).
*
*
* _NOTE: Sanitized input should not be used directly in an SQL query; SQL placeholders
* should still be used._
*
*
* **Implementation Details**
*
*
* - [htmlspecialchars](http://php.net/manual/en/function.htmlspecialchars.php) is used to escape text.
* - Single quotes are not escaped so **Piwik's amazing community** will still be
* **Piwik's amazing community**.
@ -246,10 +272,11 @@ class Common
if (is_numeric($value)) {
return $value;
} elseif (is_string($value)) {
$value = self::sanitizeInputValue($value);
$value = self::sanitizeString($value);
if (!$alreadyStripslashed) {
// a JSON array was already stripslashed, don't do it again for each value
if (!$alreadyStripslashed) // a JSON array was already stripslashed, don't do it again for each value
{
$value = self::undoMagicQuotes($value);
}
} elseif (is_array($value)) {
@ -272,20 +299,32 @@ class Common
}
/**
* Sanitize a single input value
* Sanitize a single input value and removes line breaks, tabs and null characters.
*
* @param string $value
* @return string sanitized input
*/
public static function sanitizeInputValue($value)
{
$value = self::sanitizeLineBreaks($value);
$value = self::sanitizeString($value);
return $value;
}
/**
* Sanitize a single input value
*
* @param $value
* @return string
*/
private static function sanitizeString($value)
{
// $_GET and $_REQUEST already urldecode()'d
// decode
// note: before php 5.2.7, htmlspecialchars() double encodes &#x hex items
$value = html_entity_decode($value, self::HTML_ENCODING_QUOTE_STYLE, 'UTF-8');
// filter
$value = self::sanitizeLineBreaks($value);
$value = self::sanitizeNullBytes($value);
// escape
$tmp = @htmlspecialchars($value, self::HTML_ENCODING_QUOTE_STYLE, 'UTF-8');
@ -295,15 +334,17 @@ class Common
// convert and escape
$value = utf8_encode($value);
$tmp = htmlspecialchars($value, self::HTML_ENCODING_QUOTE_STYLE, 'UTF-8');
return $tmp;
}
return $tmp;
}
/**
* Unsanitizes a single input value and returns the result.
*
*
* @param string $value
* @return string unsanitized input
* @api
*/
public static function unsanitizeInputValue($value)
{
@ -315,10 +356,10 @@ class Common
*
* This method should be used when you need to unescape data that was obtained from
* the user.
*
*
* Some data in Piwik is stored sanitized (such as site name). In this case you may
* have to use this method to unsanitize it in order to, for example, output it in JSON.
*
*
* @param string|array $value The data to unsanitize. If an array is passed, the
* array is sanitized recursively. Key values are not unsanitized.
* @return string|array The unsanitized data.
@ -345,28 +386,42 @@ class Common
*/
private static function undoMagicQuotes($value)
{
return version_compare(PHP_VERSION, '5.4', '<')
&& get_magic_quotes_gpc()
? stripslashes($value)
: $value;
}
static $shouldUndo;
if (!isset($shouldUndo)) {
$shouldUndo = version_compare(PHP_VERSION, '5.4', '<') && get_magic_quotes_gpc();
}
if ($shouldUndo) {
$value = stripslashes($value);
}
/**
*
* @param string
* @return string Line breaks and line carriage removed
*/
public static function sanitizeLineBreaks($value)
{
$value = str_replace(array("\n", "\r", "\0"), '', $value);
return $value;
}
/**
* @param string $value
* @return string Line breaks and line carriage removed
*/
public static function sanitizeLineBreaks($value)
{
return str_replace(array("\n", "\r"), '', $value);
}
/**
* @param string $value
* @return string Null bytes removed
*/
public static function sanitizeNullBytes($value)
{
return str_replace(array("\0"), '', $value);
}
/**
* Gets a sanitized request parameter by name from the `$_GET` and `$_POST` superglobals.
*
*
* Use this function to get request parameter values. **_NEVER use `$_GET` and `$_POST` directly._**
*
*
* If the variable cannot be found, and a default value was not provided, an exception is raised.
*
* _See {@link sanitizeInputValues()} to learn more about sanitization._
@ -376,7 +431,7 @@ class Common
* @param string|null $varDefault The value to return if the request parameter cannot be found or has an empty value.
* @param string|null $varType Expected type of the request variable. This parameters value must be one of the following:
* `'array'`, `'int'`, `'integer'`, `'string'`, `'json'`.
*
*
* If `'json'`, the string value will be `json_decode`-d and then sanitized.
* @param array|null $requestArrayToUse The array to use instead of `$_GET` and `$_POST`.
* @throws Exception If the request parameter doesn't exist and there is no default value, or if the request parameter
@ -389,6 +444,7 @@ class Common
if (is_null($requestArrayToUse)) {
$requestArrayToUse = $_GET + $_POST;
}
$varDefault = self::sanitizeInputValues($varDefault);
if ($varType === 'int') {
// settype accepts only integer
@ -420,22 +476,36 @@ class Common
// we deal w/ json differently
if ($varType == 'json') {
$value = self::undoMagicQuotes($requestArrayToUse[$varName]);
$value = self::json_decode($value, $assoc = true);
$value = json_decode($value, $assoc = true);
return self::sanitizeInputValues($value, $alreadyStripslashed = true);
}
$value = self::sanitizeInputValues($requestArrayToUse[$varName]);
if (!is_null($varType)) {
if (isset($varType)) {
$ok = false;
if ($varType === 'string') {
if (is_string($value)) $ok = true;
if (is_string($value) || is_int($value)) {
$ok = true;
} elseif (is_float($value)) {
$value = Common::forceDotAsSeparatorForDecimalPoint($value);
$ok = true;
}
} elseif ($varType === 'integer') {
if ($value == (string)(int)$value) $ok = true;
if ($value == (string)(int)$value) {
$ok = true;
}
} elseif ($varType === 'float') {
if ($value == (string)(float)$value) $ok = true;
$valueToCompare = (string)(float)$value;
$valueToCompare = Common::forceDotAsSeparatorForDecimalPoint($valueToCompare);
if ($value == $valueToCompare) {
$ok = true;
}
} elseif ($varType === 'array') {
if (is_array($value)) $ok = true;
if (is_array($value)) {
$ok = true;
}
} else {
throw new Exception("\$varType specified is not known. It should be one of the following: array, int, integer, float, string");
}
@ -452,6 +522,7 @@ class Common
}
settype($value, $varType);
}
return $value;
}
@ -466,11 +537,17 @@ class Common
*/
public static function generateUniqId()
{
return md5(uniqid(rand(), true));
if (function_exists('mt_rand')) {
$rand = mt_rand();
} else {
$rand = rand();
}
return md5(uniqid($rand, true));
}
/**
* Configureable hash() algorithm (defaults to md5)
* Configurable hash() algorithm (defaults to md5)
*
* @param string $str String to be hashed
* @param bool $raw_output
@ -479,14 +556,16 @@ class Common
public static function hash($str, $raw_output = false)
{
static $hashAlgorithm = null;
if (is_null($hashAlgorithm)) {
$hashAlgorithm = @Config::getInstance()->General['hash_algorithm'];
}
if ($hashAlgorithm) {
$hash = @hash($hashAlgorithm, $str, $raw_output);
if ($hash !== false)
if ($hash !== false) {
return $hash;
}
}
return md5($str, $raw_output);
@ -503,16 +582,13 @@ class Common
public static function getRandomString($length = 16, $alphabet = "abcdefghijklmnoprstuvwxyz0123456789")
{
$chars = $alphabet;
$str = '';
list($usec, $sec) = explode(" ", microtime());
$seed = ((float)$sec + (float)$usec) * 100000;
mt_srand($seed);
$str = '';
for ($i = 0; $i < $length; $i++) {
$rand_key = mt_rand(0, strlen($chars) - 1);
$str .= substr($chars, $rand_key, 1);
}
return str_shuffle($str);
}
@ -554,27 +630,22 @@ class Common
) {
throw new Exception("visitorId is expected to be a " . Tracker::LENGTH_HEX_ID_STRING . " hex char string");
}
return self::hex2bin($id);
}
/**
* Convert IP address (in network address format) to presentation format.
* This is a backward compatibility function for code that only expects
* IPv4 addresses (i.e., doesn't support IPv6).
* Converts a User ID string to the Visitor ID Binary representation.
*
* @see IP::N2P()
*
* This function does not support the long (or its string representation)
* returned by the built-in ip2long() function, from Piwik 1.3 and earlier.
*
* @deprecated 1.4
*
* @param string $ip IP address in network address format
* @param $userId
* @return string
*/
public static function long2ip($ip)
public static function convertUserIdToVisitorIdBin($userId)
{
return IP::long2ip($ip);
require_once PIWIK_INCLUDE_PATH . '/libs/PiwikTracker/PiwikTracker.php';
$userIdHashed = \PiwikTracker::getUserIdHashed($userId);
return self::convertVisitorIdToBin($userIdHashed);
}
/**
@ -653,14 +724,14 @@ class Common
/**
* Returns the list of parent classes for the given class.
*
* @param string $klass A class name.
* @param string $class A class name.
* @return string[] The list of parent classes in order from highest ancestor to the descended class.
*/
public static function getClassLineage($klass)
public static function getClassLineage($class)
{
$klasses = array_merge(array($klass), array_values(class_parents($klass, $autoload = false)));
$classes = array_merge(array($class), array_values(class_parents($class, $autoload = false)));
return array_reverse($klasses);
return array_reverse($classes);
}
/*
@ -672,14 +743,16 @@ class Common
*
* @see core/DataFiles/Countries.php
*
* @return array Array of 3 letter continent codes
* @return array Array of 3 letter continent codes
*
* @deprecated Use Piwik\Intl\Data\Provider\RegionDataProvider instead.
* @see \Piwik\Intl\Data\Provider\RegionDataProvider::getContinentList()
*/
public static function getContinentsList()
{
require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/Countries.php';
$continentsList = $GLOBALS['Piwik_ContinentList'];
return $continentsList;
/** @var RegionDataProvider $dataProvider */
$dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\RegionDataProvider');
return $dataProvider->getContinentList();
}
/**
@ -688,19 +761,16 @@ class Common
* @see core/DataFiles/Countries.php
*
* @param bool $includeInternalCodes
* @return array Array of (2 letter ISO codes => 3 letter continent code)
* @return array Array of (2 letter ISO codes => 3 letter continent code)
*
* @deprecated Use Piwik\Intl\Data\Provider\RegionDataProvider instead.
* @see \Piwik\Intl\Data\Provider\RegionDataProvider::getCountryList()
*/
public static function getCountriesList($includeInternalCodes = false)
{
require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/Countries.php';
$countriesList = $GLOBALS['Piwik_CountryList'];
$extras = $GLOBALS['Piwik_CountryList_Extras'];
if ($includeInternalCodes) {
return array_merge($countriesList, $extras);
}
return $countriesList;
/** @var RegionDataProvider $dataProvider */
$dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\RegionDataProvider');
return $dataProvider->getCountryList($includeInternalCodes);
}
/**
@ -711,13 +781,15 @@ class Common
* @return array Array of two letter ISO codes mapped with their associated language names (in English). E.g.
* `array('en' => 'English', 'ja' => 'Japanese')`.
* @api
*
* @deprecated Use Piwik\Intl\Data\Provider\LanguageDataProvider instead.
* @see \Piwik\Intl\Data\Provider\LanguageDataProvider::getLanguageList()
*/
public static function getLanguagesList()
{
require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/Languages.php';
$languagesList = $GLOBALS['Piwik_LanguageList'];
return $languagesList;
/** @var LanguageDataProvider $dataProvider */
$dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\LanguageDataProvider');
return $dataProvider->getLanguageList();
}
/**
@ -728,70 +800,15 @@ class Common
* @return array Array of two letter ISO language codes mapped with two letter ISO country codes:
* `array('fr' => 'fr') // French => France`
* @api
*
* @deprecated Use Piwik\Intl\Data\Provider\LanguageDataProvider instead.
* @see \Piwik\Intl\Data\Provider\LanguageDataProvider::getLanguageToCountryList()
*/
public static function getLanguageToCountryList()
{
require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/LanguageToCountry.php';
$languagesList = $GLOBALS['Piwik_LanguageToCountry'];
return $languagesList;
}
/**
* Returns list of search engines by URL
*
* @see core/DataFiles/SearchEngines.php
*
* @return array Array of ( URL => array( searchEngineName, keywordParameter, path, charset ) )
*/
public static function getSearchEngineUrls()
{
require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/SearchEngines.php';
$searchEngines = $GLOBALS['Piwik_SearchEngines'];
Piwik::postEvent('Referrer.addSearchEngineUrls', array(&$searchEngines));
return $searchEngines;
}
/**
* Returns list of search engines by name
*
* @see core/DataFiles/SearchEngines.php
*
* @return array Array of ( searchEngineName => URL )
*/
public static function getSearchEngineNames()
{
$searchEngines = self::getSearchEngineUrls();
$nameToUrl = array();
foreach ($searchEngines as $url => $info) {
if (!isset($nameToUrl[$info[0]])) {
$nameToUrl[$info[0]] = $url;
}
}
return $nameToUrl;
}
/**
* Returns list of social networks by URL
*
* @see core/DataFiles/Socials.php
*
* @return array Array of ( URL => Social Network Name )
*/
public static function getSocialUrls()
{
require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php';
$socialUrls = $GLOBALS['Piwik_socialUrl'];
Piwik::postEvent('Referrer.addSocialUrls', array(&$socialUrls));
return $socialUrls;
/** @var LanguageDataProvider $dataProvider */
$dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\LanguageDataProvider');
return $dataProvider->getLanguageToCountryList();
}
/**
@ -840,7 +857,7 @@ class Common
}
}
if (is_null($browserLang)) {
if (empty($browserLang)) {
// a fallback might be to infer the language in HTTP_USER_AGENT (i.e., localized build)
$browserLang = "";
} else {
@ -872,11 +889,15 @@ class Common
*/
public static function getCountry($lang, $enableLanguageToCountryGuess, $ip)
{
if (empty($lang) || strlen($lang) < 2 || $lang == 'xx') {
return 'xx';
if (empty($lang) || strlen($lang) < 2 || $lang == self::LANGUAGE_CODE_INVALID) {
return self::LANGUAGE_CODE_INVALID;
}
$validCountries = self::getCountriesList();
/** @var RegionDataProvider $dataProvider */
$dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\RegionDataProvider');
$validCountries = $dataProvider->getCountryList();
return self::extractCountryCodeFromBrowserLanguage($lang, $validCountries, $enableLanguageToCountryGuess);
}
@ -890,7 +911,10 @@ class Common
*/
public static function extractCountryCodeFromBrowserLanguage($browserLanguage, $validCountries, $enableLanguageToCountryGuess)
{
$langToCountry = self::getLanguageToCountryList();
/** @var LanguageDataProvider $dataProvider */
$dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\LanguageDataProvider');
$langToCountry = $dataProvider->getLanguageToCountryList();
if ($enableLanguageToCountryGuess) {
if (preg_match('/^([a-z]{2,3})(?:,|;|$)/', $browserLanguage, $matches)) {
@ -909,51 +933,90 @@ class Common
}
}
}
return 'xx';
return self::LANGUAGE_CODE_INVALID;
}
/**
* Returns the visitor language based only on the Browser 'accepted language' information
* Returns the language and region string, based only on the Browser 'accepted language' information.
* * The language tag is defined by ISO 639-1
*
* @param string $browserLanguage Browser's accepted langauge header
* @param array $validLanguages array of valid language codes
* @return string 2 letter ISO 639 code
* @return string 2 letter ISO 639 code 'es' (Spanish)
*/
public static function extractLanguageCodeFromBrowserLanguage($browserLanguage, $validLanguages)
public static function extractLanguageCodeFromBrowserLanguage($browserLanguage, $validLanguages = array())
{
// assumes language preference is sorted;
// does not handle language-script-region tags or language range (*)
if (!empty($validLanguages) && preg_match_all('/(?:^|,)([a-z]{2,3})([-][a-z]{2})?/', $browserLanguage, $matches, PREG_SET_ORDER)) {
foreach ($matches as $parts) {
if (count($parts) == 3) {
// match locale (language and location)
if (in_array($parts[1] . $parts[2], $validLanguages)) {
return $parts[1] . $parts[2];
}
$validLanguages = self::checkValidLanguagesIsSet($validLanguages);
$languageRegionCode = self::extractLanguageAndRegionCodeFromBrowserLanguage($browserLanguage, $validLanguages);
if (strlen($languageRegionCode) == 2) {
$languageCode = $languageRegionCode;
} else {
$languageCode = substr($languageRegionCode, 0, 2);
}
if (in_array($languageCode, $validLanguages)) {
return $languageCode;
}
return self::LANGUAGE_CODE_INVALID;
}
/**
* Returns the language and region string, based only on the Browser 'accepted language' information.
* * The language tag is defined by ISO 639-1
* * The region tag is defined by ISO 3166-1
*
* @param string $browserLanguage Browser's accepted langauge header
* @param array $validLanguages array of valid language codes. Note that if the array includes "fr" then it will consider all regional variants of this language valid, such as "fr-ca" etc.
* @return string 2 letter ISO 639 code 'es' (Spanish) or if found, includes the region as well: 'es-ar'
*/
public static function extractLanguageAndRegionCodeFromBrowserLanguage($browserLanguage, $validLanguages = array())
{
$validLanguages = self::checkValidLanguagesIsSet($validLanguages);
if (!preg_match_all('/(?:^|,)([a-z]{2,3})([-][a-z]{2})?/', $browserLanguage, $matches, PREG_SET_ORDER)) {
return self::LANGUAGE_CODE_INVALID;
}
foreach ($matches as $parts) {
$langIso639 = $parts[1];
if (empty($langIso639)) {
continue;
}
// If a region tag is found eg. "fr-ca"
if (count($parts) == 3) {
$regionIso3166 = $parts[2]; // eg. "-ca"
if (in_array($langIso639 . $regionIso3166, $validLanguages)) {
return $langIso639 . $regionIso3166;
}
// match language only (where no region provided)
if (in_array($parts[1], $validLanguages)) {
return $parts[1];
if (in_array($langIso639, $validLanguages)) {
return $langIso639 . $regionIso3166;
}
}
// eg. "fr" or "es"
if (in_array($langIso639, $validLanguages)) {
return $langIso639;
}
}
return 'xx';
return self::LANGUAGE_CODE_INVALID;
}
/**
* Returns the continent of a given country
*
* @param string $country 2 letters isocode
* @param string $country 2 letters iso code
*
* @return string Continent (3 letters code : afr, asi, eur, amn, ams, oce)
*/
public static function getContinent($country)
{
$countryList = self::getCountriesList();
if (isset($countryList[$country])) {
return $countryList[$country];
}
return 'unk';
/** @var RegionDataProvider $dataProvider */
$dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\RegionDataProvider');
$countryList = $dataProvider->getCountryList();
return isset($countryList[$country]) ? $countryList[$country] : 'unk';
}
/*
@ -995,10 +1058,10 @@ class Common
/**
* Returns a string with a comma separated list of placeholders for use in an SQL query. Used mainly
* to fill the `IN (...)` part of a query.
*
*
* @param array|string $fields The names of the mysql table fields to bind, e.g.
* `array(fieldName1, fieldName2, fieldName3)`.
*
*
* _Note: The content of the array isn't important, just its length._
* @return string The placeholder string, e.g. `"?, ?, ?"`.
* @api
@ -1015,6 +1078,22 @@ class Common
return '?' . str_repeat(',?', $count - 1);
}
/**
* Force the separator for decimal point to be a dot. See https://github.com/piwik/piwik/issues/6435
* If for instance a German locale is used it would be a comma otherwise.
*
* @param float|string $value
* @return string
*/
public static function forceDotAsSeparatorForDecimalPoint($value)
{
if (null === $value || false === $value) {
return $value;
}
return str_replace(',', '.', $value);
}
/**
* Sets outgoing header.
*
@ -1024,23 +1103,62 @@ class Common
public static function sendHeader($header, $replace = true)
{
// don't send header in CLI mode
if(Common::isPhpCliMode()) {
return;
}
if (isset($GLOBALS['PIWIK_TRACKER_LOCAL_TRACKING']) && $GLOBALS['PIWIK_TRACKER_LOCAL_TRACKING']) {
@header($header, $replace);
} else {
if (!Common::isPhpCliMode() and !headers_sent()) {
header($header, $replace);
}
}
/**
* Sends the given response code if supported.
*
* @param int $code Eg 204
*
* @throws Exception
*/
public static function sendResponseCode($code)
{
$messages = array(
200 => 'Ok',
204 => 'No Response',
301 => 'Moved Permanently',
302 => 'Found',
304 => 'Not Modified',
400 => 'Bad Request',
401 => 'Unauthorized',
403 => 'Forbidden',
404 => 'Not Found',
500 => 'Internal Server Error',
503 => 'Service Unavailable',
);
if (!array_key_exists($code, $messages)) {
throw new Exception('Response code not supported: ' . $code);
}
if (strpos(PHP_SAPI, '-fcgi') === false) {
$key = 'HTTP/1.1';
if (array_key_exists('SERVER_PROTOCOL', $_SERVER)
&& strlen($_SERVER['SERVER_PROTOCOL']) < 15
&& strlen($_SERVER['SERVER_PROTOCOL']) > 1) {
$key = $_SERVER['SERVER_PROTOCOL'];
}
} else {
// FastCGI
$key = 'Status:';
}
$message = $messages[$code];
Common::sendHeader($key . ' ' . $code . ' ' . $message);
}
/**
* Returns the ID of the current LocationProvider (see UserCountry plugin code) from
* the Tracker cache.
*/
public static function getCurrentLocationProviderId()
{
$cache = Cache::getCacheGeneral();
$cache = TrackerCache::getCacheGeneral();
return empty($cache['currentLocationProviderId'])
? DefaultProvider::ID
: $cache['currentLocationProviderId'];
@ -1049,11 +1167,11 @@ class Common
/**
* Marks an orphaned object for garbage collection.
*
* For more information: {@link http://dev.piwik.org/trac/ticket/374}
* @param $var The object to destroy.
* For more information: {@link https://github.com/piwik/piwik/issues/374}
* @param mixed $var The object to destroy.
* @api
*/
static public function destroy(&$var)
public static function destroy(&$var)
{
if (is_object($var) && method_exists($var, '__destruct')) {
$var->__destruct();
@ -1062,27 +1180,62 @@ class Common
$var = null;
}
static public function printDebug($info = '')
/**
* @todo This method is weird, it's debugging statements but seem to only work for the tracker, maybe it
* should be moved elsewhere
*/
public static function printDebug($info = '')
{
if (isset($GLOBALS['PIWIK_TRACKER_DEBUG']) && $GLOBALS['PIWIK_TRACKER_DEBUG']) {
if (!headers_sent()) {
// prevent XSS in tracker debug output
header('Content-type: text/plain');
}
if (is_object($info)) {
$info = var_export($info, true);
}
Log::getInstance()->setLogLevel(Log::DEBUG);
$logger = StaticContainer::get('Psr\Log\LoggerInterface');
if (is_array($info) || is_object($info)) {
$info = Common::sanitizeInputValues($info);
$out = var_export($info, true);
foreach (explode("\n", $out) as $line) {
Log::debug($line);
$logger->debug($line);
}
} else {
foreach (explode("\n", $info) as $line) {
Log::debug(htmlspecialchars($line, ENT_QUOTES));
$logger->debug($line);
}
}
}
}
/**
* Returns true if the request is an AJAX request.
*
* @return bool
*/
public static function isXmlHttpRequest()
{
return isset($_SERVER['HTTP_X_REQUESTED_WITH'])
&& (strtolower($_SERVER['HTTP_X_REQUESTED_WITH']) === 'xmlhttprequest');
}
/**
* @param $validLanguages
* @return array
*/
protected static function checkValidLanguagesIsSet($validLanguages)
{
/** @var LanguageDataProvider $dataProvider */
$dataProvider = StaticContainer::get('Piwik\Intl\Data\Provider\LanguageDataProvider');
if (empty($validLanguages)) {
$validLanguages = array_keys($dataProvider->getLanguageList());
return $validLanguages;
}
return $validLanguages;
}
}

47
www/analytics/core/Composer/ScriptHandler.php Normal file
View file

@ -0,0 +1,47 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Composer;
/**
* Scripts executed before/after Composer install and update.
*
* We use this PHP class because setting the bash scripts directly in composer.json breaks
* Composer on Windows systems.
*/
class ScriptHandler
{
private static function isPhp7orLater()
{
return version_compare('7.0.0-dev', PHP_VERSION) < 1;
}
public static function cleanXhprof()
{
if (! is_dir('vendor/facebook/xhprof/extension')) {
return;
}
if (!self::isPhp7orLater()) {
// doesn't work with PHP 7 at the moment
passthru('misc/composer/clean-xhprof.sh');
}
}
public static function buildXhprof()
{
if (! is_dir('vendor/facebook/xhprof/extension')) {
return;
}
if (!self::isPhp7orLater()) {
passthru('misc/composer/clean-xhprof.sh');
}
}
}

170
www/analytics/core/Concurrency/DistributedList.php Normal file
View file

@ -0,0 +1,170 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Concurrency;
use Piwik\Container\StaticContainer;
use Piwik\Option;
use Psr\Log\LoggerInterface;
/**
* Manages a simple distributed list stored in an Option. No locking occurs, so the list
* is not thread safe, and should only be used for use cases where atomicity is not
* important.
*
* The list of items is serialized and stored in an Option. Items are converted to string
* before being persisted, so it is not expected to unserialize objects.
*/
class DistributedList
{
/**
* The name of the option to store the list in.
*
* @var string
*/
private $optionName;
/**
* @var LoggerInterface
*/
private $logger;
/**
* Constructor.
*
* @param string $optionName
*/
public function __construct($optionName, LoggerInterface $logger = null)
{
$this->optionName = $optionName;
$this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
}
/**
* Queries the option table and returns all items in this list.
*
* @return array
*/
public function getAll()
{
$result = $this->getListOptionValue();
foreach ($result as $key => $item) {
// remove non-array items (unexpected state, though can happen when upgrading from an old Piwik)
if (is_array($item)) {
$this->logger->info("Found array item in DistributedList option value '{name}': {data}", array(
'name' => $this->optionName,
'data' => var_export($result, true)
));
unset($result[$key]);
}
}
return $result;
}
/**
* Sets the contents of the list in the option table.
*
* @param string[] $items
*/
public function setAll($items)
{
foreach ($items as $key => &$item) {
if (is_array($item)) {
throw new \InvalidArgumentException("Array item encountered in DistributedList::setAll() [ key = $key ].");
} else {
$item = (string)$item;
}
}
Option::set($this->optionName, serialize($items));
}
/**
* Adds one or more items to the list in the option table.
*
* @param string|array $item
*/
public function add($item)
{
$allItems = $this->getAll();
if (is_array($item)) {
$allItems = array_merge($allItems, $item);
} else {
$allItems[] = $item;
}
$this->setAll($allItems);
}
/**
* Removes one or more items by value from the list in the option table.
*
* Does not preserve array keys.
*
* @param string|array $items
*/
public function remove($items)
{
if (!is_array($items)) {
$items = array($items);
}
$allItems = $this->getAll();
foreach ($items as $item) {
$existingIndex = array_search($item, $allItems);
if ($existingIndex === false) {
return;
}
unset($allItems[$existingIndex]);
}
$this->setAll(array_values($allItems));
}
/**
* Removes one or more items by index from the list in the option table.
*
* Does not preserve array keys.
*
* @param int[]|int $indices
*/
public function removeByIndex($indices)
{
if (!is_array($indices)) {
$indices = array($indices);
}
$indices = array_unique($indices);
$allItems = $this->getAll();
foreach ($indices as $index) {
unset($allItems[$index]);
}
$this->setAll(array_values($allItems));
}
protected function getListOptionValue()
{
Option::clearCachedOption($this->optionName);
$array = Option::get($this->optionName);
$result = array();
if ($array
&& ($array = unserialize($array))
&& count($array)
) {
$result = $array;
}
return $result;
}
}

561
www/analytics/core/Config.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -10,16 +10,18 @@
namespace Piwik;
use Exception;
use Piwik\Application\Kernel\GlobalSettingsProvider;
use Piwik\Container\StaticContainer;
/**
* Singleton that provides read & write access to Piwik's INI configuration.
*
*
* This class reads and writes to the `config/config.ini.php` file. If config
* options are missing from that file, this class will look for their default
* values in `config/global.ini.php`.
*
*
* ### Examples
*
*
* **Getting a value:**
*
* // read the minimum_memory_limit option under the [General] section
@ -30,47 +32,39 @@ use Exception;
* // set the minimum_memory_limit option
* Config::getInstance()->General['minimum_memory_limit'] = 256;
* Config::getInstance()->forceSave();
*
*
* **Setting an entire section:**
*
*
* Config::getInstance()->MySection = array('myoption' => 1);
* Config::getInstance()->forceSave();
*
* @method static \Piwik\Config getInstance()
*/
class Config extends Singleton
class Config
{
const DEFAULT_LOCAL_CONFIG_PATH = '/config/config.ini.php';
const DEFAULT_COMMON_CONFIG_PATH = '/config/common.config.ini.php';
const DEFAULT_GLOBAL_CONFIG_PATH = '/config/global.ini.php';
/**
* Contains configuration files values
*
* @var array
*/
protected $initialized = false;
protected $configGlobal = array();
protected $configLocal = array();
protected $configCommon = array();
protected $configCache = array();
protected $pathGlobal = null;
protected $pathCommon = null;
protected $pathLocal = null;
/**
* @var boolean
*/
protected $isTest = false;
protected $doNotWriteConfigInTests = false;
/**
* Constructor
* @var GlobalSettingsProvider
*/
public function __construct($pathGlobal = null, $pathLocal = null, $pathCommon = null)
protected $settings;
/**
* @return Config
*/
public static function getInstance()
{
$this->pathGlobal = $pathGlobal ?: self::getGlobalConfigPath();
$this->pathCommon = $pathCommon ?: self::getCommonConfigPath();
$this->pathLocal = $pathLocal ?: self::getLocalConfigPath();
return StaticContainer::get('Piwik\Config');
}
public function __construct(GlobalSettingsProvider $settings)
{
$this->settings = $settings;
}
/**
@ -80,7 +74,7 @@ class Config extends Singleton
*/
public function getLocalPath()
{
return $this->pathLocal;
return $this->settings->getPathLocal();
}
/**
@ -90,7 +84,7 @@ class Config extends Singleton
*/
public function getGlobalPath()
{
return $this->pathGlobal;
return $this->settings->getPathGlobal();
}
/**
@ -100,72 +94,7 @@ class Config extends Singleton
*/
public function getCommonPath()
{
return $this->pathCommon;
}
/**
* Enable test environment
*
* @param string $pathLocal
* @param string $pathGlobal
* @param string $pathCommon
*/
public function setTestEnvironment($pathLocal = null, $pathGlobal = null, $pathCommon = null, $allowSaving = false)
{
if (!$allowSaving) {
$this->isTest = true;
}
$this->clear();
$this->pathLocal = $pathLocal ?: Config::getLocalConfigPath();
$this->pathGlobal = $pathGlobal ?: Config::getGlobalConfigPath();
$this->pathCommon = $pathCommon ?: Config::getCommonConfigPath();
$this->init();
// this proxy will not record any data in the production database.
// this provides security for Piwik installs and tests were setup.
if (isset($this->configGlobal['database_tests'])
|| isset($this->configLocal['database_tests'])
) {
$this->__get('database_tests');
$this->configCache['database'] = $this->configCache['database_tests'];
}
// Ensure local mods do not affect tests
if (empty($pathGlobal)) {
$this->configCache['log'] = $this->configGlobal['log'];
$this->configCache['Debug'] = $this->configGlobal['Debug'];
$this->configCache['mail'] = $this->configGlobal['mail'];
$this->configCache['General'] = $this->configGlobal['General'];
$this->configCache['Segments'] = $this->configGlobal['Segments'];
$this->configCache['Tracker'] = $this->configGlobal['Tracker'];
$this->configCache['Deletelogs'] = $this->configGlobal['Deletelogs'];
$this->configCache['Deletereports'] = $this->configGlobal['Deletereports'];
}
// for unit tests, we set that no plugin is installed. This will force
// the test initialization to create the plugins tables, execute ALTER queries, etc.
$this->configCache['PluginsInstalled'] = array('PluginsInstalled' => array());
// DevicesDetection plugin is not yet enabled by default
if (isset($configGlobal['Plugins'])) {
$this->configCache['Plugins'] = $this->configGlobal['Plugins'];
$this->configCache['Plugins']['Plugins'][] = 'DevicesDetection';
}
if (isset($configGlobal['Plugins_Tracker'])) {
$this->configCache['Plugins_Tracker'] = $this->configGlobal['Plugins_Tracker'];
$this->configCache['Plugins_Tracker']['Plugins_Tracker'][] = 'DevicesDetection';
}
// to avoid weird session error in travis
if (empty($pathGlobal)) {
$configArray = &$this->configCache;
} else {
$configArray = &$this->configLocal;
}
$configArray['General']['session_save_handler'] = 'dbtables';
return $this->settings->getPathCommon();
}
/**
@ -173,7 +102,7 @@ class Config extends Singleton
*
* @return string
*/
protected static function getGlobalConfigPath()
public static function getGlobalConfigPath()
{
return PIWIK_USER_PATH . self::DEFAULT_GLOBAL_CONFIG_PATH;
}
@ -204,6 +133,8 @@ class Config extends Singleton
private static function getLocalConfigInfoForHostname($hostname)
{
// Remove any port number to get actual hostname
$hostname = Url::getHostSanitized($hostname);
$perHostFilename = $hostname . '.config.ini.php';
$pathDomainConfig = PIWIK_USER_PATH . '/config/' . $perHostFilename;
@ -225,11 +156,25 @@ class Config extends Singleton
return array(
'action_url_category_delimiter' => $general['action_url_category_delimiter'],
'autocomplete_min_sites' => $general['autocomplete_min_sites'],
'datatable_export_range_as_day' => $general['datatable_export_range_as_day']
'datatable_export_range_as_day' => $general['datatable_export_range_as_day'],
'datatable_row_limits' => $this->getDatatableRowLimits(),
'are_ads_enabled' => $general['piwik_pro_ads_enabled']
);
}
protected static function getByDomainConfigPath()
/**
* @param $general
* @return mixed
*/
private function getDatatableRowLimits()
{
$limits = $this->General['datatable_row_limits'];
$limits = explode(",", $limits);
$limits = array_map('trim', $limits);
return $limits;
}
public static function getByDomainConfigPath()
{
$host = self::getHostname();
$hostConfig = self::getLocalConfigInfoForHostname($host);
@ -242,9 +187,19 @@ class Config extends Singleton
return false;
}
protected static function getHostname()
/**
* Returns the hostname of the current request (without port number)
*
* @return string
*/
public static function getHostname()
{
$host = Url::getHost($checkIfTrusted = false); // Check trusted requires config file which is not ready yet
// Check trusted requires config file which is not ready yet
$host = Url::getHost($checkIfTrusted = false);
// Remove any port number to get actual hostname
$host = Url::getHostSanitized($host);
return $host;
}
@ -259,19 +214,26 @@ class Config extends Singleton
* @param string $hostname eg piwik.example.com
* @return string
* @throws \Exception In case the domain contains not allowed characters
* @internal
*/
public function forceUsageOfLocalHostnameConfig($hostname)
{
$hostConfig = static::getLocalConfigInfoForHostname($hostname);
$hostConfig = self::getLocalConfigInfoForHostname($hostname);
if (!Filesystem::isValidFilename($hostConfig['file'])) {
throw new Exception('Hostname is not valid');
$filename = $hostConfig['file'];
if (!Filesystem::isValidFilename($filename)) {
throw new Exception('Piwik domain is not a valid looking hostname (' . $filename . ').');
}
$this->pathLocal = $hostConfig['path'];
$this->configLocal = array();
$this->initialized = false;
return $this->pathLocal;
$pathLocal = $hostConfig['path'];
try {
$this->reload($pathLocal);
} catch (Exception $ex) {
// pass (not required for local file to exist at this point)
}
return $pathLocal;
}
/**
@ -281,99 +243,57 @@ class Config extends Singleton
*/
public function isFileWritable()
{
return is_writable($this->pathLocal);
return is_writable($this->settings->getPathLocal());
}
/**
* Clear in-memory configuration so it can be reloaded
* @deprecated since v2.12.0
*/
public function clear()
{
$this->configGlobal = array();
$this->configLocal = array();
$this->configCache = array();
$this->initialized = false;
$this->reload();
}
/**
* Read configuration from files into memory
*
* @throws Exception if local config file is not readable; exits for other errors
* @deprecated since v2.12.0
*/
public function init()
{
$this->initialized = true;
$reportError = SettingsServer::isTrackerApiRequest();
// read defaults from global.ini.php
if (!is_readable($this->pathGlobal) && $reportError) {
Piwik_ExitWithMessage(Piwik::translate('General_ExceptionConfigurationFileNotFound', array($this->pathGlobal)));
}
$this->configGlobal = _parse_ini_file($this->pathGlobal, true);
if (empty($this->configGlobal) && $reportError) {
Piwik_ExitWithMessage(Piwik::translate('General_ExceptionUnreadableFileDisabledMethod', array($this->pathGlobal, "parse_ini_file()")));
}
$this->configCommon = _parse_ini_file($this->pathCommon, true);
// Check config.ini.php last
$this->checkLocalConfigFound();
$this->configLocal = _parse_ini_file($this->pathLocal, true);
if (empty($this->configLocal) && $reportError) {
Piwik_ExitWithMessage(Piwik::translate('General_ExceptionUnreadableFileDisabledMethod', array($this->pathLocal, "parse_ini_file()")));
}
$this->reload();
}
/**
* Reloads config data from disk.
*
* @throws \Exception if the global config file is not found and this is a tracker request, or
* if the local config file is not found and this is NOT a tracker request.
*/
protected function reload($pathLocal = null, $pathGlobal = null, $pathCommon = null)
{
$this->settings->reload($pathGlobal, $pathLocal, $pathCommon);
}
/**
* @deprecated
*/
public function existsLocalConfig()
{
return is_readable($this->pathLocal);
return is_readable($this->getLocalPath());
}
public function checkLocalConfigFound()
public function deleteLocalConfig()
{
if (!$this->existsLocalConfig()) {
throw new Exception(Piwik::translate('General_ExceptionConfigurationFileNotFound', array($this->pathLocal)));
$configLocal = $this->getLocalPath();
if(file_exists($configLocal)){
@unlink($configLocal);
}
}
/**
* Decode HTML entities
*
* @param mixed $values
* @return mixed
*/
protected function decodeValues($values)
{
if (is_array($values)) {
foreach ($values as &$value) {
$value = $this->decodeValues($value);
}
return $values;
}
return html_entity_decode($values, ENT_COMPAT, 'UTF-8');
}
/**
* Encode HTML entities
*
* @param mixed $values
* @return mixed
*/
protected function encodeValues($values)
{
if (is_array($values)) {
foreach ($values as &$value) {
$value = $this->encodeValues($value);
}
} else {
$values = htmlentities($values, ENT_COMPAT, 'UTF-8');
}
return $values;
}
/**
* Returns a configuration value or section by name.
*
@ -385,85 +305,32 @@ class Config extends Singleton
*/
public function &__get($name)
{
if (!$this->initialized) {
$this->init();
// must be called here, not in init(), since setTestEnvironment() calls init(). (this avoids
// infinite recursion)
Piwik::postTestEvent('Config.createConfigSingleton',
array($this, &$this->configCache, &$this->configLocal));
}
// check cache for merged section
if (isset($this->configCache[$name])) {
$tmp =& $this->configCache[$name];
return $tmp;
}
$section = $this->getFromGlobalConfig($name);
$sectionCommon = $this->getFromCommonConfig($name);
if(empty($section) && !empty($sectionCommon)) {
$section = $sectionCommon;
} elseif(!empty($section) && !empty($sectionCommon)) {
$section = $this->array_merge_recursive_distinct($section, $sectionCommon);
}
if (isset($this->configLocal[$name])) {
// local settings override the global defaults
$section = $section
? array_merge($section, $this->configLocal[$name])
: $this->configLocal[$name];
}
if ($section === null && $name = 'superuser') {
$user = $this->getConfigSuperUserForBackwardCompatibility();
return $user;
} else if ($section === null) {
throw new Exception("Error while trying to read a specific config file entry <strong>'$name'</strong> from your configuration files.</b>If you just completed a Piwik upgrade, please check that the file config/global.ini.php was overwritten by the latest Piwik version.");
}
// cache merged section for later
$this->configCache[$name] = $this->decodeValues($section);
$tmp =& $this->configCache[$name];
return $tmp;
$section =& $this->settings->getIniFileChain()->get($name);
return $section;
}
/**
* @deprecated since version 2.0.4
* @api
*/
public function getConfigSuperUserForBackwardCompatibility()
{
try {
$db = Db::get();
$user = $db->fetchRow("SELECT login, email, password
FROM " . Common::prefixTable("user") . "
WHERE superuser_access = 1
ORDER BY date_registered ASC LIMIT 1");
if (!empty($user)) {
$user['bridge'] = 1;
return $user;
}
} catch (Exception $e) {}
return array();
}
public function getFromGlobalConfig($name)
{
if (isset($this->configGlobal[$name])) {
return $this->configGlobal[$name];
}
return null;
return $this->settings->getIniFileChain()->getFrom($this->getGlobalPath(), $name);
}
/**
* @api
*/
public function getFromCommonConfig($name)
{
if (isset($this->configCommon[$name])) {
return $this->configCommon[$name];
}
return null;
return $this->settings->getIniFileChain()->getFrom($this->getCommonPath(), $name);
}
/**
* @api
*/
public function getFromLocalConfig($name)
{
return $this->settings->getIniFileChain()->getFrom($this->getLocalPath(), $name);
}
/**
@ -475,154 +342,24 @@ class Config extends Singleton
*/
public function __set($name, $value)
{
$this->configCache[$name] = $value;
}
/**
* Comparison function
*
* @param mixed $elem1
* @param mixed $elem2
* @return int;
*/
public static function compareElements($elem1, $elem2)
{
if (is_array($elem1)) {
if (is_array($elem2)) {
return strcmp(serialize($elem1), serialize($elem2));
}
return 1;
}
if (is_array($elem2)) {
return -1;
}
if ((string)$elem1 === (string)$elem2) {
return 0;
}
return ((string)$elem1 > (string)$elem2) ? 1 : -1;
}
/**
* Compare arrays and return difference, such that:
*
* $modified = array_merge($original, $difference);
*
* @param array $original original array
* @param array $modified modified array
* @return array differences between original and modified
*/
public function array_unmerge($original, $modified)
{
// return key/value pairs for keys in $modified but not in $original
// return key/value pairs for keys in both $modified and $original, but values differ
// ignore keys that are in $original but not in $modified
return array_udiff_assoc($modified, $original, array(__CLASS__, 'compareElements'));
$this->settings->getIniFileChain()->set($name, $value);
}
/**
* Dump config
*
* @param array $configLocal
* @param array $configGlobal
* @param array $configCommon
* @param array $configCache
* @return string
* @return string|null
* @throws \Exception
*/
public function dumpConfig($configLocal, $configGlobal, $configCommon, $configCache)
public function dumpConfig()
{
$dirty = false;
$chain = $this->settings->getIniFileChain();
$output = "; <?php exit; ?> DO NOT REMOVE THIS LINE\n";
$output .= "; file automatically generated or modified by Piwik; you can manually override the default values in global.ini.php by redefining them in this file.\n";
if (!$configCache) {
return false;
}
// If there is a common.config.ini.php, this will ensure config.ini.php does not duplicate its values
if(!empty($configCommon)) {
$configGlobal = $this->array_merge_recursive_distinct($configGlobal, $configCommon);
}
if ($configLocal) {
foreach ($configLocal as $name => $section) {
if (!isset($configCache[$name])) {
$configCache[$name] = $this->decodeValues($section);
}
}
}
$sectionNames = array_unique(array_merge(array_keys($configGlobal), array_keys($configCache)));
foreach ($sectionNames as $section) {
if (!isset($configCache[$section])) {
continue;
}
// Only merge if the section exists in global.ini.php (in case a section only lives in config.ini.php)
// get local and cached config
$local = isset($configLocal[$section]) ? $configLocal[$section] : array();
$config = $configCache[$section];
// remove default values from both (they should not get written to local)
if (isset($configGlobal[$section])) {
$config = $this->array_unmerge($configGlobal[$section], $configCache[$section]);
$local = $this->array_unmerge($configGlobal[$section], $local);
}
// if either local/config have non-default values and the other doesn't,
// OR both have values, but different values, we must write to config.ini.php
if (empty($local) xor empty($config)
|| (!empty($local)
&& !empty($config)
&& self::compareElements($config, $configLocal[$section]))
) {
$dirty = true;
}
// no point in writing empty sections, so skip if the cached section is empty
if (empty($config)) {
continue;
}
$output .= "[$section]\n";
foreach ($config as $name => $value) {
$value = $this->encodeValues($value);
if (is_numeric($name)) {
$name = $section;
$value = array($value);
}
if (is_array($value)) {
foreach ($value as $currentValue) {
$output .= $name . "[] = \"$currentValue\"\n";
}
} else {
if (!is_numeric($value)) {
$value = "\"$value\"";
}
$output .= $name . ' = ' . $value . "\n";
}
}
$output .= "\n";
}
if ($dirty) {
return $output;
}
return false;
$header = "; <?php exit; ?> DO NOT REMOVE THIS LINE\n";
$header .= "; file automatically generated or modified by Piwik; you can manually override the default values in global.ini.php by redefining them in this file.\n";
return $chain->dumpChanges($header);
}
/**
* Write user configuration file
*
@ -635,22 +372,24 @@ class Config extends Singleton
*
* @throws \Exception if config file not writable
*/
protected function writeConfig($configLocal, $configGlobal, $configCommon, $configCache, $pathLocal, $clear = true)
protected function writeConfig($clear = true)
{
if ($this->isTest) {
if ($this->doNotWriteConfigInTests) {
return;
}
$output = $this->dumpConfig($configLocal, $configGlobal, $configCommon, $configCache);
if ($output !== false) {
$success = @file_put_contents($pathLocal, $output);
if (!$success) {
$output = $this->dumpConfig();
if ($output !== null
&& $output !== false
) {
$success = @file_put_contents($this->getLocalPath(), $output);
if ($success === false) {
throw $this->getConfigNotWritableException();
}
}
if ($clear) {
$this->clear();
$this->reload();
}
}
@ -662,7 +401,7 @@ class Config extends Singleton
*/
public function forceSave()
{
$this->writeConfig($this->configLocal, $this->configGlobal, $this->configCommon, $this->configCache, $this->pathLocal);
$this->writeConfig();
}
/**
@ -670,45 +409,7 @@ class Config extends Singleton
*/
public function getConfigNotWritableException()
{
$path = "config/" . basename($this->pathLocal);
$path = "config/" . basename($this->getLocalPath());
return new Exception(Piwik::translate('General_ConfigFileIsNotWritable', array("(" . $path . ")", "")));
}
/**
* array_merge_recursive does indeed merge arrays, but it converts values with duplicate
* keys to arrays rather than overwriting the value in the first array with the duplicate
* value in the second array, as array_merge does. I.e., with array_merge_recursive,
* this happens (documented behavior):
*
* array_merge_recursive(array('key' => 'org value'), array('key' => 'new value'));
* => array('key' => array('org value', 'new value'));
*
* array_merge_recursive_distinct does not change the datatypes of the values in the arrays.
* Matching keys' values in the second array overwrite those in the first array, as is the
* case with array_merge, i.e.:
*
* array_merge_recursive_distinct(array('key' => 'org value'), array('key' => 'new value'));
* => array('key' => array('new value'));
*
* Parameters are passed by reference, though only for performance reasons. They're not
* altered by this function.
*
* @param array $array1
* @param array $array2
* @return array
* @author Daniel <daniel (at) danielsmedegaardbuus (dot) dk>
* @author Gabriel Sobrinho <gabriel (dot) sobrinho (at) gmail (dot) com>
*/
function array_merge_recursive_distinct ( array &$array1, array &$array2 )
{
$merged = $array1;
foreach ( $array2 as $key => &$value ) {
if ( is_array ( $value ) && isset ( $merged [$key] ) && is_array ( $merged [$key] ) ) {
$merged [$key] = $this->array_merge_recursive_distinct ( $merged [$key], $value );
} else {
$merged [$key] = $value;
}
}
return $merged;
}
}

16
www/analytics/core/Config/ConfigNotFoundException.php Normal file
View file

@ -0,0 +1,16 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Config;
/**
* Exception thrown when the config file doesn't exist.
*/
class ConfigNotFoundException extends \Exception
{
}

474
www/analytics/core/Config/IniFileChain.php Normal file
View file

@ -0,0 +1,474 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Config;
use Piwik\Common;
use Piwik\Ini\IniReader;
use Piwik\Ini\IniReadingException;
use Piwik\Ini\IniWriter;
/**
* Manages a list of INI files where the settings in each INI file merge with or override the
* settings in the previous INI file.
*
* The IniFileChain class manages two types of INI files: multiple default setting files and one
* user settings file.
*
* The default setting files (for example, global.ini.php & common.ini.php) hold the default setting values.
* The settings in these files are merged recursively, however, array settings in one file will still
* overwrite settings in the previous file.
*
* Default settings files cannot be modified through the IniFileChain class.
*
* The user settings file (for example, config.ini.php) holds the actual setting values. Settings in the
* user settings files overwrite other settings. So array settings will not merge w/ previous values.
*
* HTML characters and dollar signs are stored as encoded HTML entities in INI files. This prevents
* several `parse_ini_file` issues, including one where parse_ini_file tries to insert a variable
* into a setting value if a string like `"$varname" is present.
*/
class IniFileChain
{
/**
* Maps INI file names with their parsed contents. The order of the files signifies the order
* in the chain. Files with lower index are overwritten/merged with files w/ a higher index.
*
* @var array
*/
protected $settingsChain = array();
/**
* The merged INI settings.
*
* @var array
*/
protected $mergedSettings = array();
/**
* Constructor.
*
* @param string[] $defaultSettingsFiles The list of paths to INI files w/ the default setting values.
* @param string|null $userSettingsFile The path to the user settings file.
*/
public function __construct(array $defaultSettingsFiles = array(), $userSettingsFile = null)
{
$this->reload($defaultSettingsFiles, $userSettingsFile);
}
/**
* Return setting section by reference.
*
* @param string $name
* @return mixed
*/
public function &get($name)
{
if (!isset($this->mergedSettings[$name])) {
$this->mergedSettings[$name] = array();
}
$result =& $this->mergedSettings[$name];
return $result;
}
/**
* Return setting section from a specific file, rather than the current merged settings.
*
* @param string $file The path of the file. Should be the path used in construction or reload().
* @param string $name The name of the section to access.
*/
public function getFrom($file, $name)
{
return @$this->settingsChain[$file][$name];
}
/**
* Sets a setting value.
*
* @param string $name
* @param mixed $value
*/
public function set($name, $value)
{
$this->mergedSettings[$name] = $value;
}
/**
* Returns all settings. Changes made to the array result will be reflected in the
* IniFileChain instance.
*
* @return array
*/
public function &getAll()
{
return $this->mergedSettings;
}
/**
* Dumps the current in-memory setting values to a string in INI format and returns it.
*
* @param string $header The header of the output INI file.
* @return string The dumped INI contents.
*/
public function dump($header = '')
{
return $this->dumpSettings($this->mergedSettings, $header);
}
/**
* Writes the difference of the in-memory setting values and the on-disk user settings file setting
* values to a string in INI format, and returns it.
*
* If a config section is identical to the default settings section (as computed by merging
* all default setting files), it is not written to the user settings file.
*
* @param string $header The header of the INI output.
* @return string The dumped INI contents.
*/
public function dumpChanges($header = '')
{
$userSettingsFile = $this->getUserSettingsFile();
$defaultSettings = $this->getMergedDefaultSettings();
$existingMutableSettings = $this->settingsChain[$userSettingsFile];
$dirty = false;
$configToWrite = array();
foreach ($this->mergedSettings as $sectionName => $changedSection) {
if(isset($existingMutableSettings[$sectionName])){
$existingMutableSection = $existingMutableSettings[$sectionName];
} else{
$existingMutableSection = array();
}
// remove default values from both (they should not get written to local)
if (isset($defaultSettings[$sectionName])) {
$changedSection = $this->arrayUnmerge($defaultSettings[$sectionName], $changedSection);
$existingMutableSection = $this->arrayUnmerge($defaultSettings[$sectionName], $existingMutableSection);
}
// if either local/config have non-default values and the other doesn't,
// OR both have values, but different values, we must write to config.ini.php
if (empty($changedSection) xor empty($existingMutableSection)
|| (!empty($changedSection)
&& !empty($existingMutableSection)
&& self::compareElements($changedSection, $existingMutableSection))
) {
$dirty = true;
}
$configToWrite[$sectionName] = $changedSection;
}
if ($dirty) {
// sort config sections by how early they appear in the file chain
$self = $this;
uksort($configToWrite, function ($sectionNameLhs, $sectionNameRhs) use ($self) {
$lhsIndex = $self->findIndexOfFirstFileWithSection($sectionNameLhs);
$rhsIndex = $self->findIndexOfFirstFileWithSection($sectionNameRhs);
if ($lhsIndex == $rhsIndex) {
$lhsIndexInFile = $self->getIndexOfSectionInFile($lhsIndex, $sectionNameLhs);
$rhsIndexInFile = $self->getIndexOfSectionInFile($rhsIndex, $sectionNameRhs);
if ($lhsIndexInFile == $rhsIndexInFile) {
return 0;
} elseif ($lhsIndexInFile < $rhsIndexInFile) {
return -1;
} else {
return 1;
}
} elseif ($lhsIndex < $rhsIndex) {
return -1;
} else {
return 1;
}
});
return $this->dumpSettings($configToWrite, $header);
} else {
return null;
}
}
/**
* Reloads settings from disk.
*/
public function reload($defaultSettingsFiles = array(), $userSettingsFile = null)
{
if (!empty($defaultSettingsFiles)
|| !empty($userSettingsFile)
) {
$this->resetSettingsChain($defaultSettingsFiles, $userSettingsFile);
}
$reader = new IniReader();
foreach ($this->settingsChain as $file => $ignore) {
if (is_readable($file)) {
try {
$contents = $reader->readFile($file);
$this->settingsChain[$file] = $this->decodeValues($contents);
} catch (IniReadingException $ex) {
throw new IniReadingException('Unable to read INI file {' . $file . '}: ' . $ex->getMessage() . "\n Your host may have disabled parse_ini_file().");
}
$this->decodeValues($this->settingsChain[$file]);
}
}
$merged = $this->mergeFileSettings();
// remove reference to $this->settingsChain... otherwise dump() or compareElements() will never notice a difference
// on PHP 7+ as they would be always equal
$this->mergedSettings = $this->copy($merged);
}
private function copy($merged)
{
$copy = array();
foreach ($merged as $index => $value) {
if (is_array($value)) {
$copy[$index] = $this->copy($value);
} else {
$copy[$index] = $value;
}
}
return $copy;
}
private function resetSettingsChain($defaultSettingsFiles, $userSettingsFile)
{
$this->settingsChain = array();
if (!empty($defaultSettingsFiles)) {
foreach ($defaultSettingsFiles as $file) {
$this->settingsChain[$file] = null;
}
}
if (!empty($userSettingsFile)) {
$this->settingsChain[$userSettingsFile] = null;
}
}
protected function mergeFileSettings()
{
$mergedSettings = $this->getMergedDefaultSettings();
$userSettings = end($this->settingsChain) ?: array();
foreach ($userSettings as $sectionName => $section) {
if (!isset($mergedSettings[$sectionName])) {
$mergedSettings[$sectionName] = $section;
} else {
// the last user settings file completely overwrites INI sections. the other files in the chain
// can add to array options
$mergedSettings[$sectionName] = array_merge($mergedSettings[$sectionName], $section);
}
}
return $mergedSettings;
}
protected function getMergedDefaultSettings()
{
$userSettingsFile = $this->getUserSettingsFile();
$mergedSettings = array();
foreach ($this->settingsChain as $file => $settings) {
if ($file == $userSettingsFile
|| empty($settings)
) {
continue;
}
foreach ($settings as $sectionName => $section) {
if (!isset($mergedSettings[$sectionName])) {
$mergedSettings[$sectionName] = $section;
} else {
$mergedSettings[$sectionName] = $this->array_merge_recursive_distinct($mergedSettings[$sectionName], $section);
}
}
}
return $mergedSettings;
}
protected function getUserSettingsFile()
{
// the user settings file is the last key in $settingsChain
end($this->settingsChain);
return key($this->settingsChain);
}
/**
* Comparison function
*
* @param mixed $elem1
* @param mixed $elem2
* @return int;
*/
public static function compareElements($elem1, $elem2)
{
if (is_array($elem1)) {
if (is_array($elem2)) {
return strcmp(serialize($elem1), serialize($elem2));
}
return 1;
}
if (is_array($elem2)) {
return -1;
}
if ((string)$elem1 === (string)$elem2) {
return 0;
}
return ((string)$elem1 > (string)$elem2) ? 1 : -1;
}
/**
* Compare arrays and return difference, such that:
*
* $modified = array_merge($original, $difference);
*
* @param array $original original array
* @param array $modified modified array
* @return array differences between original and modified
*/
public function arrayUnmerge($original, $modified)
{
// return key/value pairs for keys in $modified but not in $original
// return key/value pairs for keys in both $modified and $original, but values differ
// ignore keys that are in $original but not in $modified
return array_udiff_assoc($modified, $original, array(__CLASS__, 'compareElements'));
}
/**
* array_merge_recursive does indeed merge arrays, but it converts values with duplicate
* keys to arrays rather than overwriting the value in the first array with the duplicate
* value in the second array, as array_merge does. I.e., with array_merge_recursive,
* this happens (documented behavior):
*
* array_merge_recursive(array('key' => 'org value'), array('key' => 'new value'));
* => array('key' => array('org value', 'new value'));
*
* array_merge_recursive_distinct does not change the datatypes of the values in the arrays.
* Matching keys' values in the second array overwrite those in the first array, as is the
* case with array_merge, i.e.:
*
* array_merge_recursive_distinct(array('key' => 'org value'), array('key' => 'new value'));
* => array('key' => array('new value'));
*
* Parameters are passed by reference, though only for performance reasons. They're not
* altered by this function.
*
* @param array $array1
* @param array $array2
* @return array
* @author Daniel <daniel (at) danielsmedegaardbuus (dot) dk>
* @author Gabriel Sobrinho <gabriel (dot) sobrinho (at) gmail (dot) com>
*/
private function array_merge_recursive_distinct(array &$array1, array &$array2)
{
$merged = $array1;
foreach ($array2 as $key => &$value) {
if (is_array($value) && isset($merged [$key]) && is_array($merged [$key])) {
$merged [$key] = $this->array_merge_recursive_distinct($merged [$key], $value);
} else {
$merged [$key] = $value;
}
}
return $merged;
}
/**
* public for use in closure.
*/
public function findIndexOfFirstFileWithSection($sectionName)
{
$count = 0;
foreach ($this->settingsChain as $file => $settings) {
if (isset($settings[$sectionName])) {
break;
}
++$count;
}
return $count;
}
/**
* public for use in closure.
*/
public function getIndexOfSectionInFile($fileIndex, $sectionName)
{
reset($this->settingsChain);
for ($i = 0; $i != $fileIndex; ++$i) {
next($this->settingsChain);
}
$settingsData = current($this->settingsChain);
if (empty($settingsData)) {
return -1;
}
$settingsDataSectionNames = array_keys($settingsData);
return array_search($sectionName, $settingsDataSectionNames);
}
/**
* Encode HTML entities
*
* @param mixed $values
* @return mixed
*/
protected function encodeValues(&$values)
{
if (is_array($values)) {
foreach ($values as &$value) {
$value = $this->encodeValues($value);
}
} elseif (is_float($values)) {
$values = Common::forceDotAsSeparatorForDecimalPoint($values);
} elseif (is_string($values)) {
$values = htmlentities($values, ENT_COMPAT, 'UTF-8');
$values = str_replace('$', '&#36;', $values);
}
return $values;
}
/**
* Decode HTML entities
*
* @param mixed $values
* @return mixed
*/
protected function decodeValues(&$values)
{
if (is_array($values)) {
foreach ($values as &$value) {
$value = $this->decodeValues($value);
}
return $values;
} elseif (is_string($values)) {
return html_entity_decode($values, ENT_COMPAT, 'UTF-8');
}
return $values;
}
private function dumpSettings($values, $header)
{
$values = $this->encodeValues($values);
$writer = new IniWriter();
return $writer->writeToString($values, $header);
}
}

162
www/analytics/core/Console.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -8,19 +8,32 @@
*/
namespace Piwik;
use Symfony\Component\Console\Application;
use Symfony\Component\Console\Input\ArgvInput;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Piwik\Application\Environment;
use Piwik\Config\ConfigNotFoundException;
use Piwik\Container\StaticContainer;
use Piwik\Plugin\Manager as PluginManager;
use Symfony\Bridge\Monolog\Handler\ConsoleHandler;
use Symfony\Component\Console\Application;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class Console extends Application
{
public function __construct()
/**
* @var Environment
*/
private $environment;
public function __construct(Environment $environment = null)
{
$this->setServerArgsIfPhpCgi();
parent::__construct();
$this->environment = $environment;
$option = new InputOption('piwik-domain',
null,
InputOption::VALUE_OPTIONAL,
@ -28,42 +41,60 @@ class Console extends Application
);
$this->getDefinition()->addOption($option);
}
/**
* @deprecated
*/
public function init()
{
// TODO: remove
$option = new InputOption('xhprof',
null,
InputOption::VALUE_NONE,
'Enable profiling with XHProf'
);
$this->getDefinition()->addOption($option);
}
public function doRun(InputInterface $input, OutputInterface $output)
{
$this->initPiwikHost($input);
$this->initConfig($output);
try {
self::initPlugins();
} catch(\Exception $e) {
// Piwik not installed yet, no config file?
if ($input->hasParameterOption('--xhprof')) {
Profiler::setupProfilerXHProf(true, true);
}
Translate::reloadLanguage('en');
$this->initPiwikHost($input);
$this->initEnvironment($output);
$this->initLoggerOutput($output);
try {
self::initPlugins();
} catch (ConfigNotFoundException $e) {
// Piwik not installed yet, no config file?
Log::warning($e->getMessage());
}
$commands = $this->getAvailableCommands();
foreach ($commands as $command) {
if (!class_exists($command)) {
Log::warning(sprintf('Cannot add command %s, class does not exist', $command));
} elseif (!is_subclass_of($command, 'Piwik\Plugin\ConsoleCommand')) {
Log::warning(sprintf('Cannot add command %s, class does not extend Piwik\Plugin\ConsoleCommand', $command));
} else {
$this->add(new $command);
}
$this->addCommandIfExists($command);
}
return parent::doRun($input, $output);
$self = $this;
return Access::doAsSuperUser(function () use ($input, $output, $self) {
return call_user_func(array($self, 'Symfony\Component\Console\Application::doRun'), $input, $output);
});
}
private function addCommandIfExists($command)
{
if (!class_exists($command)) {
Log::warning(sprintf('Cannot add command %s, class does not exist', $command));
} elseif (!is_subclass_of($command, 'Piwik\Plugin\ConsoleCommand')) {
Log::warning(sprintf('Cannot add command %s, class does not extend Piwik\Plugin\ConsoleCommand', $command));
} else {
/** @var Command $commandInstance */
$commandInstance = new $command;
// do not add the command if it already exists; this way we can add the command ourselves in tests
if (!$this->has($commandInstance->getName())) {
$this->add($commandInstance);
}
}
}
/**
@ -74,11 +105,9 @@ class Console extends Application
private function getAvailableCommands()
{
$commands = $this->getDefaultPiwikCommands();
$detected = PluginManager::getInstance()->findMultipleComponents('Commands', 'Piwik\\Plugin\\ConsoleCommand');
$pluginNames = PluginManager::getInstance()->getLoadedPluginsName();
foreach ($pluginNames as $pluginName) {
$commands = array_merge($commands, $this->findCommandsInPlugin($pluginName));
}
$commands = array_merge($commands, $detected);
/**
* Triggered to filter / restrict console commands. Plugins that want to restrict commands
@ -103,52 +132,76 @@ class Console extends Application
return $commands;
}
private function findCommandsInPlugin($pluginName)
private function setServerArgsIfPhpCgi()
{
$commands = array();
if (Common::isPhpCgiType()) {
$_SERVER['argv'] = array();
foreach ($_GET as $name => $value) {
$argument = $name;
if (!empty($value)) {
$argument .= '=' . $value;
}
$files = Filesystem::globr(PIWIK_INCLUDE_PATH . '/plugins/' . $pluginName .'/Commands', '*.php');
foreach ($files as $file) {
$klassName = sprintf('Piwik\\Plugins\\%s\\Commands\\%s', $pluginName, basename($file, '.php'));
if (!class_exists($klassName) || !is_subclass_of($klassName, 'Piwik\\Plugin\\ConsoleCommand')) {
continue;
$_SERVER['argv'][] = $argument;
}
$klass = new \ReflectionClass($klassName);
if ($klass->isAbstract()) {
continue;
if (!defined('STDIN')) {
define('STDIN', fopen('php://stdin', 'r'));
}
$commands[] = $klassName;
}
}
return $commands;
public static function isSupported()
{
return Common::isPhpCliMode() && !Common::isPhpCgiType();
}
protected function initPiwikHost(InputInterface $input)
{
$piwikHostname = $input->getParameterOption('--piwik-domain');
if (empty($piwikHostname)) {
$piwikHostname = $input->getParameterOption('--url');
}
$piwikHostname = UrlHelper::getHostFromUrl($piwikHostname);
Url::setHost($piwikHostname);
}
protected function initConfig(OutputInterface $output)
protected function initEnvironment(OutputInterface $output)
{
$config = Config::getInstance();
try {
$config->checkLocalConfigFound();
if ($this->environment === null) {
$this->environment = new Environment('cli');
$this->environment->init();
}
$config = Config::getInstance();
return $config;
} catch (\Exception $e) {
$output->writeln($e->getMessage() . "\n");
}
}
/**
* Register the console output into the logger.
*
* Ideally, this should be done automatically with events:
* @see http://symfony.com/fr/doc/current/components/console/events.html
* @see Symfony\Bridge\Monolog\Handler\ConsoleHandler::onCommand()
* But it would require to install Symfony's Event Dispatcher.
*/
private function initLoggerOutput(OutputInterface $output)
{
/** @var ConsoleHandler $consoleLogHandler */
$consoleLogHandler = StaticContainer::get('Symfony\Bridge\Monolog\Handler\ConsoleHandler');
$consoleLogHandler->setOutput($output);
}
public static function initPlugins()
{
Plugin\Manager::getInstance()->loadActivatedPlugins();
Plugin\Manager::getInstance()->loadPluginTranslations();
}
private function getDefaultPiwikCommands()
@ -156,11 +209,12 @@ class Console extends Application
$commands = array(
'Piwik\CliMulti\RequestCommand'
);
if (class_exists('Piwik\Plugins\EnterpriseAdmin\EnterpriseAdmin')) {
$extra = new \Piwik\Plugins\EnterpriseAdmin\EnterpriseAdmin();
$extra->addConsoleCommands($commands);
}
return $commands;
}
}

18
www/analytics/core/Container/ContainerDoesNotExistException.php Normal file
View file

@ -0,0 +1,18 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Container;
use RuntimeException;
/**
* Thrown if the root container has not been created and set in StaticContainer.
*/
class ContainerDoesNotExistException extends RuntimeException
{
}

151
www/analytics/core/Container/ContainerFactory.php Normal file
View file

@ -0,0 +1,151 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Container;
use DI\Container;
use DI\ContainerBuilder;
use Doctrine\Common\Cache\ArrayCache;
use Piwik\Application\Kernel\GlobalSettingsProvider;
use Piwik\Application\Kernel\PluginList;
use Piwik\Plugin\Manager;
/**
* Creates a configured DI container.
*/
class ContainerFactory
{
/**
* @var PluginList
*/
private $pluginList;
/**
* @var GlobalSettingsProvider
*/
private $settings;
/**
* Optional environment configs to load.
*
* @var string[]
*/
private $environments;
/**
* @var array[]
*/
private $definitions;
/**
* @param PluginList $pluginList
* @param GlobalSettingsProvider $settings
* @param string[] $environment Optional environment configs to load.
* @param array[] $definitions
*/
public function __construct(PluginList $pluginList, GlobalSettingsProvider $settings, array $environments = array(), array $definitions = array())
{
$this->pluginList = $pluginList;
$this->settings = $settings;
$this->environments = $environments;
$this->definitions = $definitions;
}
/**
* @link http://php-di.org/doc/container-configuration.html
* @throws \Exception
* @return Container
*/
public function create()
{
$builder = new ContainerBuilder();
$builder->useAnnotations(false);
$builder->setDefinitionCache(new ArrayCache());
// INI config
$builder->addDefinitions(new IniConfigDefinitionSource($this->settings));
// Global config
$builder->addDefinitions(PIWIK_USER_PATH . '/config/global.php');
// Plugin configs
$this->addPluginConfigs($builder);
// Development config
if ($this->isDevelopmentModeEnabled()) {
$this->addEnvironmentConfig($builder, 'dev');
}
// Environment config
foreach ($this->environments as $environment) {
$this->addEnvironmentConfig($builder, $environment);
}
// User config
if (file_exists(PIWIK_USER_PATH . '/config/config.php')) {
$builder->addDefinitions(PIWIK_USER_PATH . '/config/config.php');
}
if (!empty($this->definitions)) {
foreach ($this->definitions as $definitionArray) {
$builder->addDefinitions($definitionArray);
}
}
$container = $builder->build();
$container->set('Piwik\Application\Kernel\PluginList', $this->pluginList);
$container->set('Piwik\Application\Kernel\GlobalSettingsProvider', $this->settings);
return $container;
}
private function addEnvironmentConfig(ContainerBuilder $builder, $environment)
{
if (!$environment) {
return;
}
$file = sprintf('%s/config/environment/%s.php', PIWIK_USER_PATH, $environment);
if (file_exists($file)) {
$builder->addDefinitions($file);
}
// add plugin environment configs
$plugins = $this->pluginList->getActivatedPlugins();
foreach ($plugins as $plugin) {
$baseDir = Manager::getPluginsDirectory() . $plugin;
$environmentFile = $baseDir . '/config/' . $environment . '.php';
if (file_exists($environmentFile)) {
$builder->addDefinitions($environmentFile);
}
}
}
private function addPluginConfigs(ContainerBuilder $builder)
{
$plugins = $this->pluginList->getActivatedPlugins();
foreach ($plugins as $plugin) {
$baseDir = Manager::getPluginsDirectory() . $plugin;
$file = $baseDir . '/config/config.php';
if (file_exists($file)) {
$builder->addDefinitions($file);
}
}
}
private function isDevelopmentModeEnabled()
{
$section = $this->settings->getSection('Development');
return (bool) @$section['enabled']; // TODO: code redundancy w/ Development. hopefully ok for now.
}
}

95
www/analytics/core/Container/IniConfigDefinitionSource.php Normal file
View file

@ -0,0 +1,95 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Container;
use DI\Definition\Exception\DefinitionException;
use DI\Definition\Source\DefinitionSource;
use DI\Definition\ValueDefinition;
use Piwik\Application\Kernel\GlobalSettingsProvider;
/**
* Expose the INI config into PHP-DI.
*
* The INI config can be used by prefixing `ini.` before the setting we want to get:
*
* $maintenanceMode = $container->get('ini.General.maintenance_mode');
*/
class IniConfigDefinitionSource implements DefinitionSource
{
/**
* @var GlobalSettingsProvider
*/
private $config;
/**
* @var string
*/
private $prefix;
/**
* @param GlobalSettingsProvider $config
* @param string $prefix Prefix for the container entries.
*/
public function __construct(GlobalSettingsProvider $config, $prefix = 'ini.')
{
$this->config = $config;
$this->prefix = $prefix;
}
/**
* {@inheritdoc}
*/
public function getDefinition($name)
{
if (strpos($name, $this->prefix) !== 0) {
return null;
}
list($sectionName, $configKey) = $this->parseEntryName($name);
$section = $this->getSection($sectionName);
if ($configKey === null) {
return new ValueDefinition($name, $section);
}
if (! array_key_exists($configKey, $section)) {
return null;
}
return new ValueDefinition($name, $section[$configKey]);
}
private function parseEntryName($name)
{
$parts = explode('.', $name, 3);
array_shift($parts);
if (! isset($parts[1])) {
$parts[1] = null;
}
return $parts;
}
private function getSection($sectionName)
{
$section = $this->config->getSection($sectionName);
if (!is_array($section)) {
throw new DefinitionException(sprintf(
'IniFileChain did not return an array for the config section %s',
$section
));
}
return $section;
}
}

87
www/analytics/core/Container/StaticContainer.php Normal file
View file

@ -0,0 +1,87 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Container;
use DI\Container;
/**
* This class provides a static access to the container.
*
* @deprecated This class is introduced only to keep BC with the current static architecture. It will be removed in 3.0.
* - it is global state (that class makes the container a global variable)
* - using the container directly is the "service locator" anti-pattern (which is not dependency injection)
*/
class StaticContainer
{
/**
* @var Container[]
*/
private static $containerStack = array();
/**
* Definitions to register in the container.
*
* @var array[]
*/
private static $definitions = array();
/**
* @return Container
*/
public static function getContainer()
{
if (empty(self::$containerStack)) {
throw new ContainerDoesNotExistException("The root container has not been created yet.");
}
return end(self::$containerStack);
}
public static function clearContainer()
{
self::pop();
}
/**
* Only use this in tests.
*
* @param Container $container
*/
public static function push(Container $container)
{
self::$containerStack[] = $container;
}
public static function pop()
{
array_pop(self::$containerStack);
}
public static function addDefinitions(array $definitions)
{
self::$definitions[] = $definitions;
}
/**
* Proxy to Container::get()
*
* @param string $name Container entry name.
* @return mixed
* @throws \DI\NotFoundException
*/
public static function get($name)
{
return self::getContainer()->get($name);
}
public static function getDefinitions()
{
return self::$definitions;
}
}

34
www/analytics/core/Cookie.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -143,7 +143,9 @@ class Cookie
// Remove port information.
$Port = strpos($Domain, ':');
if ($Port !== false) $Domain = substr($Domain, 0, $Port);
if ($Port !== false) {
$Domain = substr($Domain, 0, $Port);
}
}
$header = 'Set-Cookie: ' . rawurlencode($Name) . '=' . rawurlencode($Value)
@ -200,12 +202,14 @@ class Cookie
private function extractSignedContent($content)
{
$signature = substr($content, -40);
if (substr($content, -43, 3) == self::VALUE_SEPARATOR . '_=' &&
$signature == sha1(substr($content, 0, -40) . SettingsPiwik::getSalt())
) {
// strip trailing: VALUE_SEPARATOR '_=' signature"
return substr($content, 0, -43);
}
return false;
}
@ -218,6 +222,7 @@ class Cookie
protected function loadContentFromCookie()
{
$cookieStr = $this->extractSignedContent($_COOKIE[$this->name]);
if ($cookieStr === false) {
return;
}
@ -255,6 +260,7 @@ class Cookie
protected function generateContentString()
{
$cookieStr = '';
foreach ($this->value as $name => $value) {
if (!is_numeric($value)) {
$value = base64_encode(safe_serialize($value));
@ -335,6 +341,7 @@ class Cookie
$this->value[$name] = $value;
return;
}
$this->value[$this->keyStore][$name] = $value;
}
@ -347,14 +354,19 @@ class Cookie
public function get($name)
{
$name = self::escapeValue($name);
if ($this->keyStore === false) {
return isset($this->value[$name])
? self::escapeValue($this->value[$name])
: false;
if (false === $this->keyStore) {
if (isset($this->value[$name])) {
return self::escapeValue($this->value[$name]);
}
return false;
}
return isset($this->value[$this->keyStore][$name])
? self::escapeValue($this->value[$this->keyStore][$name])
: false;
if (isset($this->value[$this->keyStore][$name])) {
return self::escapeValue($this->value[$this->keyStore][$name]);
}
return false;
}
/**
@ -364,8 +376,10 @@ class Cookie
*/
public function __toString()
{
$str = 'COOKIE ' . $this->name . ', rows count: ' . count($this->value) . ', cookie size = ' . strlen($this->generateContentString()) . " bytes\n";
$str = 'COOKIE ' . $this->name . ', rows count: ' . count($this->value) . ', cookie size = ' . strlen($this->generateContentString()) . " bytes, ";
$str .= 'path: ' . $this->path. ', expire: ' . $this->expire . "\n";
$str .= var_export($this->value, $return = true);
return $str;
}

1740
www/analytics/core/CronArchive.php
View file

File diff suppressed because it is too large Load diff

5
www/analytics/core/CronArchive/FixedSiteIds.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -8,7 +8,6 @@
*/
namespace Piwik\CronArchive;
use Piwik\CronArchive;
class FixedSiteIds
{
@ -63,6 +62,4 @@ class FixedSiteIds
return null;
}
}

208
www/analytics/core/CronArchive/SegmentArchivingRequestUrlProvider.php Normal file
View file

@ -0,0 +1,208 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\CronArchive;
use Piwik\Cache\Cache;
use Piwik\Cache\Transient;
use Piwik\Container\StaticContainer;
use Piwik\Date;
use Piwik\Period\Factory as PeriodFactory;
use Piwik\Period\Range;
use Piwik\Plugins\SegmentEditor\Model;
use Psr\Log\LoggerInterface;
/**
* Provides URLs that initiate archiving during cron archiving for segments.
*
* Handles the `[General] process_new_segments_from` INI option.
*/
class SegmentArchivingRequestUrlProvider
{
const BEGINNING_OF_TIME = 'beginning_of_time';
const CREATION_TIME = 'segment_creation_time';
const LAST_EDIT_TIME = 'segment_last_edit_time';
/**
* @var Model
*/
private $segmentEditorModel;
/**
* @var Cache
*/
private $segmentListCache;
/**
* @var Date
*/
private $now;
private $processNewSegmentsFrom;
/**
* @var LoggerInterface
*/
private $logger;
public function __construct($processNewSegmentsFrom, Model $segmentEditorModel = null, Cache $segmentListCache = null,
Date $now = null, LoggerInterface $logger = null)
{
$this->processNewSegmentsFrom = $processNewSegmentsFrom;
$this->segmentEditorModel = $segmentEditorModel ?: new Model();
$this->segmentListCache = $segmentListCache ?: new Transient();
$this->now = $now ?: Date::factory('now');
$this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
}
public function getUrlParameterDateString($idSite, $period, $date, $segment)
{
$oldestDateToProcessForNewSegment = $this->getOldestDateToProcessForNewSegment($idSite, $segment);
if (empty($oldestDateToProcessForNewSegment)) {
return $date;
}
// if the start date for the archiving request is before the minimum date allowed for processing this segment,
// use the minimum allowed date as the start date
$periodObj = PeriodFactory::build($period, $date);
if ($periodObj->getDateStart()->getTimestamp() < $oldestDateToProcessForNewSegment->getTimestamp()) {
$this->logger->debug("Start date of archiving request period ({start}) is older than configured oldest date to process for the segment.", array(
'start' => $periodObj->getDateStart()
));
$endDate = $periodObj->getDateEnd();
// if the creation time of a segment is older than the end date of the archiving request range, we cannot
// blindly rewrite the date string, since the resulting range would be incorrect. instead we make the
// start date equal to the end date, so less archiving occurs, and no fatal error occurs.
if ($oldestDateToProcessForNewSegment->getTimestamp() > $endDate->getTimestamp()) {
$this->logger->debug("Oldest date to process is greater than end date of archiving request period ({end}), so setting oldest date to end date.", array(
'end' => $endDate
));
$oldestDateToProcessForNewSegment = $endDate;
}
$date = $oldestDateToProcessForNewSegment->toString().','.$endDate;
$this->logger->debug("Archiving request date range changed to {date} w/ period {period}.", array('date' => $date, 'period' => $period));
}
return $date;
}
private function getOldestDateToProcessForNewSegment($idSite, $segment)
{
/**
* @var Date $segmentCreatedTime
* @var Date $segmentLastEditedTime
*/
list($segmentCreatedTime, $segmentLastEditedTime) = $this->getCreatedTimeOfSegment($idSite, $segment);
if ($this->processNewSegmentsFrom == self::CREATION_TIME) {
$this->logger->debug("process_new_segments_from set to segment_creation_time, oldest date to process is {time}", array('time' => $segmentCreatedTime));
return $segmentCreatedTime;
} elseif ($this->processNewSegmentsFrom == self::LAST_EDIT_TIME) {
$this->logger->debug("process_new_segments_from set to segment_last_edit_time, segment last edit time is {time}",
array('time' => $segmentLastEditedTime));
if ($segmentLastEditedTime === null
|| $segmentLastEditedTime->getTimestamp() < $segmentCreatedTime->getTimestamp()
) {
$this->logger->debug("segment last edit time is older than created time, using created time instead");
$segmentLastEditedTime = $segmentCreatedTime;
}
return $segmentLastEditedTime;
} elseif (preg_match("/^last([0-9]+)$/", $this->processNewSegmentsFrom, $matches)) {
$lastN = $matches[1];
list($lastDate, $lastPeriod) = Range::getDateXPeriodsAgo($lastN, $segmentCreatedTime, 'day');
$result = Date::factory($lastDate);
$this->logger->debug("process_new_segments_from set to last{N}, oldest date to process is {time}", array('N' => $lastN, 'time' => $result));
return $result;
} else {
$this->logger->debug("process_new_segments_from set to beginning_of_time or cannot recognize value");
return null;
}
}
private function getCreatedTimeOfSegment($idSite, $segmentDefinition)
{
$segments = $this->getAllSegments();
/** @var Date $latestEditTime */
$latestEditTime = null;
$earliestCreatedTime = $this->now;
foreach ($segments as $segment) {
if (empty($segment['ts_created'])
|| empty($segment['definition'])
|| !isset($segment['enable_only_idsite'])
) {
continue;
}
if ($this->isSegmentForSite($segment, $idSite)
&& $segment['definition'] == $segmentDefinition
) {
// check for an earlier ts_created timestamp
$createdTime = Date::factory($segment['ts_created']);
if ($createdTime->getTimestamp() < $earliestCreatedTime->getTimestamp()) {
$earliestCreatedTime = $createdTime;
}
// if there is no ts_last_edit timestamp, initialize it to ts_created
if (empty($segment['ts_last_edit'])) {
$segment['ts_last_edit'] = $segment['ts_created'];
}
// check for a later ts_last_edit timestamp
$lastEditTime = Date::factory($segment['ts_last_edit']);
if ($latestEditTime === null
|| $latestEditTime->getTimestamp() < $lastEditTime->getTimestamp()
) {
$latestEditTime = $lastEditTime;
}
}
}
$this->logger->debug(
"Earliest created time of segment '{segment}' w/ idSite = {idSite} is found to be {createdTime}. Latest " .
"edit time is found to be {latestEditTime}.",
array(
'segment' => $segmentDefinition,
'idSite' => $idSite,
'createdTime' => $earliestCreatedTime,
'latestEditTime' => $latestEditTime,
)
);
return array($earliestCreatedTime, $latestEditTime);
}
private function getAllSegments()
{
if (!$this->segmentListCache->contains('all')) {
$segments = $this->segmentEditorModel->getAllSegmentsAndIgnoreVisibility();
$this->segmentListCache->save('all', $segments);
}
return $this->segmentListCache->fetch('all');
}
private function isSegmentForSite($segment, $idSite)
{
return $segment['enable_only_idsite'] == 0
|| $segment['enable_only_idsite'] == $idSite;
}
}

24
www/analytics/core/CronArchive/SharedSiteIds.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -18,12 +18,22 @@ use Piwik\Option;
*/
class SharedSiteIds
{
const OPTION_DEFAULT = 'SharedSiteIdsToArchive';
const OPTION_ALL_WEBSITES = 'SharedSiteIdsToArchive_AllWebsites';
/**
* @var string
*/
private $optionName;
private $siteIds = array();
private $currentSiteId;
private $done = false;
public function __construct($websiteIds)
public function __construct($websiteIds, $optionName = self::OPTION_DEFAULT)
{
$this->optionName = $optionName;
if (empty($websiteIds)) {
$websiteIds = array();
}
@ -86,16 +96,16 @@ class SharedSiteIds
public function setSiteIdsToArchive($siteIds)
{
if (!empty($siteIds)) {
Option::set('SharedSiteIdsToArchive', implode(',', $siteIds));
Option::set($this->optionName, implode(',', $siteIds));
} else {
Option::delete('SharedSiteIdsToArchive');
Option::delete($this->optionName);
}
}
public function getAllSiteIdsToArchive()
{
Option::clearCachedOption('SharedSiteIdsToArchive');
$siteIdsToArchive = Option::get('SharedSiteIdsToArchive');
Option::clearCachedOption($this->optionName);
$siteIdsToArchive = Option::get($this->optionName);
if (empty($siteIdsToArchive)) {
return array();
@ -171,6 +181,4 @@ class SharedSiteIds
{
return Process::isSupported();
}
}

40
www/analytics/core/CronArchive/SitesToReprocessDistributedList.php Normal file
View file

@ -0,0 +1,40 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\CronArchive;
use Piwik\Concurrency\DistributedList;
/**
* Distributed list that stores the list of IDs of sites whose archives should be reprocessed.
*
* CronArchive will read this list of sites when archiving is being run, and make sure the sites
* are re-archived.
*
* Any class/API method/command/etc. is allowed to add site IDs to this list.
*/
class SitesToReprocessDistributedList extends DistributedList
{
const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds';
public function __construct()
{
parent::__construct(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
}
/**
* @inheritdoc
*/
public function setAll($items)
{
$items = array_unique($items, SORT_REGULAR);
$items = array_values($items);
parent::setAll($items);
}
}

34
www/analytics/core/DataAccess/Actions.php Normal file
View file

@ -0,0 +1,34 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\DataAccess;
use Piwik\Db;
use Piwik\Common;
/**
* Data Access Object for operations dealing with the log_action table.
*/
class Actions
{
/**
* Removes a list of actions from the log_action table by ID.
*
* @param int[] $idActions
*/
public function delete($idActions)
{
foreach ($idActions as &$id) {
$id = (int)$id;
}
$table = Common::prefixTable('log_action');
$sql = "DELETE FROM $table WHERE idaction IN (" . implode(",", $idActions) . ")";
Db::query($sql);
}
}

281
www/analytics/core/DataAccess/ArchiveSelector.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,18 +9,16 @@
namespace Piwik\DataAccess;
use Exception;
use Piwik\ArchiveProcessor\Rules;
use Piwik\Archive;
use Piwik\Archive\Chunk;
use Piwik\ArchiveProcessor;
use Piwik\ArchiveProcessor\Rules;
use Piwik\Common;
use Piwik\Date;
use Piwik\Db;
use Piwik\Log;
use Piwik\Period;
use Piwik\Period\Range;
use Piwik\Piwik;
use Piwik\Segment;
use Piwik\Site;
/**
* Data Access object used to query archives
@ -42,52 +40,46 @@ class ArchiveSelector
const NB_VISITS_CONVERTED_RECORD_LOOKED_UP = "nb_visits_converted";
static public function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC)
private static function getModel()
{
$dateStart = $params->getPeriod()->getDateStart();
$bindSQL = array($params->getSite()->getId(),
$dateStart->toString('Y-m-d'),
$params->getPeriod()->getDateEnd()->toString('Y-m-d'),
$params->getPeriod()->getId(),
);
return new Model();
}
$timeStampWhere = '';
public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC)
{
$idSite = $params->getSite()->getId();
$period = $params->getPeriod()->getId();
$dateStart = $params->getPeriod()->getDateStart();
$dateStartIso = $dateStart->toString('Y-m-d');
$dateEndIso = $params->getPeriod()->getDateEnd()->toString('Y-m-d');
$numericTable = ArchiveTableCreator::getNumericTable($dateStart);
$minDatetimeIsoArchiveProcessedUTC = null;
if ($minDatetimeArchiveProcessedUTC) {
$timeStampWhere = " AND ts_archived >= ? ";
$bindSQL[] = Date::factory($minDatetimeArchiveProcessedUTC)->getDatetime();
$minDatetimeIsoArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC)->getDatetime();
}
$requestedPlugin = $params->getRequestedPlugin();
$segment = $params->getSegment();
$isSkipAggregationOfSubTables = $params->isSkipAggregationOfSubTables();
$segment = $params->getSegment();
$plugins = array("VisitsSummary", $requestedPlugin);
$sqlWhereArchiveName = self::getNameCondition($plugins, $segment, $isSkipAggregationOfSubTables);
$sqlQuery = " SELECT idarchive, value, name, date1 as startDate
FROM " . ArchiveTableCreator::getNumericTable($dateStart) . "``
WHERE idsite = ?
AND date1 = ?
AND date2 = ?
AND period = ?
AND ( ($sqlWhereArchiveName)
OR name = '" . self::NB_VISITS_RECORD_LOOKED_UP . "'
OR name = '" . self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP . "')
$timeStampWhere
ORDER BY idarchive DESC";
$results = Db::fetchAll($sqlQuery, $bindSQL);
$doneFlags = Rules::getDoneFlags($plugins, $segment);
$doneFlagValues = Rules::getSelectableDoneFlagValues();
$results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues);
if (empty($results)) {
return false;
}
$idArchive = self::getMostRecentIdArchiveFromResults($segment, $requestedPlugin, $isSkipAggregationOfSubTables, $results);
$idArchiveVisitsSummary = self::getMostRecentIdArchiveFromResults($segment, "VisitsSummary", $isSkipAggregationOfSubTables, $results);
$idArchive = self::getMostRecentIdArchiveFromResults($segment, $requestedPlugin, $results);
$idArchiveVisitsSummary = self::getMostRecentIdArchiveFromResults($segment, "VisitsSummary", $results);
list($visits, $visitsConverted) = self::getVisitsMetricsFromResults($idArchive, $idArchiveVisitsSummary, $results);
if ($visits === false
&& $idArchive === false
) {
if (false === $visits && false === $idArchive) {
return false;
}
@ -98,9 +90,11 @@ class ArchiveSelector
{
$visits = $visitsConverted = false;
$archiveWithVisitsMetricsWasFound = ($idArchiveVisitsSummary !== false);
if ($archiveWithVisitsMetricsWasFound) {
$visits = $visitsConverted = 0;
}
foreach ($results as $result) {
if (in_array($result['idarchive'], array($idArchive, $idArchiveVisitsSummary))) {
$value = (int)$result['value'];
@ -116,13 +110,15 @@ class ArchiveSelector
}
}
}
return array($visits, $visitsConverted);
}
protected static function getMostRecentIdArchiveFromResults(Segment $segment, $requestedPlugin, $isSkipAggregationOfSubTables, $results)
protected static function getMostRecentIdArchiveFromResults(Segment $segment, $requestedPlugin, $results)
{
$idArchive = false;
$namesRequestedPlugin = Rules::getDoneFlags(array($requestedPlugin), $segment, $isSkipAggregationOfSubTables);
$namesRequestedPlugin = Rules::getDoneFlags(array($requestedPlugin), $segment);
foreach ($results as $result) {
if ($idArchive === false
&& in_array($result['name'], $namesRequestedPlugin)
@ -131,6 +127,7 @@ class ArchiveSelector
break;
}
}
return $idArchive;
}
@ -141,21 +138,29 @@ class ArchiveSelector
* @param array $periods
* @param Segment $segment
* @param array $plugins List of plugin names for which data is being requested.
* @param bool $isSkipAggregationOfSubTables Whether we are selecting an archive that may be partial (no sub-tables)
* @return array Archive IDs are grouped by archive name and period range, ie,
* array(
* 'VisitsSummary.done' => array(
* '2010-01-01' => array(1,2,3)
* )
* )
* @throws
*/
static public function getArchiveIds($siteIds, $periods, $segment, $plugins, $isSkipAggregationOfSubTables = false)
public static function getArchiveIds($siteIds, $periods, $segment, $plugins)
{
if (empty($siteIds)) {
throw new \Exception("Website IDs could not be read from the request, ie. idSite=");
}
foreach ($siteIds as $index => $siteId) {
$siteIds[$index] = (int) $siteId;
}
$getArchiveIdsSql = "SELECT idsite, name, date1, date2, MAX(idarchive) as idarchive
FROM %s
WHERE %s
AND " . self::getNameCondition($plugins, $segment, $isSkipAggregationOfSubTables) . "
AND idsite IN (" . implode(',', $siteIds) . ")
WHERE idsite IN (" . implode(',', $siteIds) . ")
AND " . self::getNameCondition($plugins, $segment) . "
AND %s
GROUP BY idsite, date1, date2";
$monthToPeriods = array();
@ -197,14 +202,14 @@ class ArchiveSelector
$sql = sprintf($getArchiveIdsSql, $table, $dateCondition);
$archiveIds = Db::fetchAll($sql, $bind);
// get the archive IDs
foreach (Db::fetchAll($sql, $bind) as $row) {
$archiveName = $row['name'];
foreach ($archiveIds as $row) {
//FIXMEA duplicate with Archive.php
$dateStr = $row['date1'] . "," . $row['date2'];
$dateStr = $row['date1'] . ',' . $row['date2'];
$result[$archiveName][$dateStr][] = $row['idarchive'];
$result[$row['name']][$dateStr][] = $row['idarchive'];
}
}
@ -215,29 +220,52 @@ class ArchiveSelector
* Queries and returns archive data using a set of archive IDs.
*
* @param array $archiveIds The IDs of the archives to get data from.
* @param array $recordNames The names of the data to retrieve (ie, nb_visits, nb_actions, etc.)
* @param array $recordNames The names of the data to retrieve (ie, nb_visits, nb_actions, etc.).
* Note: You CANNOT pass multiple recordnames if $loadAllSubtables=true.
* @param string $archiveDataType The archive data type (either, 'blob' or 'numeric').
* @param bool $loadAllSubtables Whether to pre-load all subtables
* @param int|null|string $idSubtable null if the root blob should be loaded, an integer if a subtable should be
* loaded and 'all' if all subtables should be loaded.
* @throws Exception
* @return array
*/
static public function getArchiveData($archiveIds, $recordNames, $archiveDataType, $loadAllSubtables)
public static function getArchiveData($archiveIds, $recordNames, $archiveDataType, $idSubtable)
{
$chunk = new Chunk();
// create the SQL to select archive data
$inNames = Common::getSqlStringFieldsArray($recordNames);
$loadAllSubtables = $idSubtable == Archive::ID_SUBTABLE_LOAD_ALL_SUBTABLES;
if ($loadAllSubtables) {
$name = reset($recordNames);
// select blobs w/ name like "$name_[0-9]+" w/o using RLIKE
$nameEnd = strlen($name) + 2;
$whereNameIs = "(name = ?
OR (name LIKE ?
AND SUBSTRING(name, $nameEnd, 1) >= '0'
AND SUBSTRING(name, $nameEnd, 1) <= '9') )";
$nameEnd = strlen($name) + 1;
$nameEndAppendix = $nameEnd + 1;
$appendix = $chunk->getAppendix();
$lenAppendix = strlen($appendix);
$checkForChunkBlob = "SUBSTRING(name, $nameEnd, $lenAppendix) = '$appendix'";
$checkForSubtableId = "(SUBSTRING(name, $nameEndAppendix, 1) >= '0'
AND SUBSTRING(name, $nameEndAppendix, 1) <= '9')";
$whereNameIs = "(name = ? OR (name LIKE ? AND ( $checkForChunkBlob OR $checkForSubtableId ) ))";
$bind = array($name, $name . '%');
} else {
if ($idSubtable === null) {
// select root table or specific record names
$bind = array_values($recordNames);
} else {
// select a subtable id
$bind = array();
foreach ($recordNames as $recordName) {
// to be backwards compatibe we need to look for the exact idSubtable blob and for the chunk
// that stores the subtables (a chunk stores many blobs in one blob)
$bind[] = $chunk->getRecordNameForTableId($recordName, $idSubtable);
$bind[] = self::appendIdSubtable($recordName, $idSubtable);
}
}
$inNames = Common::getSqlStringFieldsArray($bind);
$whereNameIs = "name IN ($inNames)";
$bind = array_values($recordNames);
}
$getValuesSql = "SELECT value, name, idsite, date1, date2, ts_archived
@ -251,110 +279,91 @@ class ArchiveSelector
if (empty($ids)) {
throw new Exception("Unexpected: id archive not found for period '$period' '");
}
// $period = "2009-01-04,2009-01-04",
$date = Date::factory(substr($period, 0, 10));
if ($archiveDataType == 'numeric') {
$isNumeric = $archiveDataType == 'numeric';
if ($isNumeric) {
$table = ArchiveTableCreator::getNumericTable($date);
} else {
$table = ArchiveTableCreator::getBlobTable($date);
}
$sql = sprintf($getValuesSql, $table, implode(',', $ids));
$sql = sprintf($getValuesSql, $table, implode(',', $ids));
$dataRows = Db::fetchAll($sql, $bind);
foreach ($dataRows as $row) {
$rows[] = $row;
if ($isNumeric) {
$rows[] = $row;
} else {
$row['value'] = self::uncompress($row['value']);
if ($chunk->isRecordNameAChunk($row['name'])) {
self::moveChunkRowToRows($rows, $row, $chunk, $loadAllSubtables, $idSubtable);
} else {
$rows[] = $row;
}
}
}
}
return $rows;
}
private static function moveChunkRowToRows(&$rows, $row, Chunk $chunk, $loadAllSubtables, $idSubtable)
{
// $blobs = array([subtableID] = [blob of subtableId])
$blobs = unserialize($row['value']);
if (!is_array($blobs)) {
return;
}
// $rawName = eg 'PluginName_ArchiveName'
$rawName = $chunk->getRecordNameWithoutChunkAppendix($row['name']);
if ($loadAllSubtables) {
foreach ($blobs as $subtableId => $blob) {
$row['value'] = $blob;
$row['name'] = self::appendIdSubtable($rawName, $subtableId);
$rows[] = $row;
}
} elseif (array_key_exists($idSubtable, $blobs)) {
$row['value'] = $blobs[$idSubtable];
$row['name'] = self::appendIdSubtable($rawName, $idSubtable);
$rows[] = $row;
}
}
public static function appendIdSubtable($recordName, $id)
{
return $recordName . "_" . $id;
}
private static function uncompress($data)
{
return @gzuncompress($data);
}
/**
* Returns the SQL condition used to find successfully completed archives that
* this instance is querying for.
*
* @param array $plugins
* @param Segment $segment
* @param bool $isSkipAggregationOfSubTables
* @return string
*/
static private function getNameCondition(array $plugins, Segment $segment, $isSkipAggregationOfSubTables)
private static function getNameCondition(array $plugins, Segment $segment)
{
// the flags used to tell how the archiving process for a specific archive was completed,
// if it was completed
$doneFlags = Rules::getDoneFlags($plugins, $segment, $isSkipAggregationOfSubTables);
$doneFlags = Rules::getDoneFlags($plugins, $segment);
$allDoneFlags = "'" . implode("','", $doneFlags) . "'";
$possibleValues = Rules::getSelectableDoneFlagValues();
// create the SQL to find archives that are DONE
return "((name IN ($allDoneFlags)) AND " .
" (value = '" . ArchiveWriter::DONE_OK . "' OR " .
" value = '" . ArchiveWriter::DONE_OK_TEMPORARY . "'))";
}
static public function purgeOutdatedArchives(Date $dateStart)
{
$purgeArchivesOlderThan = Rules::shouldPurgeOutdatedArchives($dateStart);
if (!$purgeArchivesOlderThan) {
return;
}
$idArchivesToDelete = self::getTemporaryArchiveIdsOlderThan($dateStart, $purgeArchivesOlderThan);
if (!empty($idArchivesToDelete)) {
self::deleteArchiveIds($dateStart, $idArchivesToDelete);
}
self::deleteArchivesWithPeriodRange($dateStart);
Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]",
$purgeArchivesOlderThan, $dateStart->toString("Y-m"), implode(',', $idArchivesToDelete));
}
/*
* Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space
*/
protected static function deleteArchivesWithPeriodRange(Date $date)
{
$query = "DELETE FROM %s WHERE period = ? AND ts_archived < ?";
$yesterday = Date::factory('yesterday')->getDateTime();
$bind = array(Piwik::$idPeriods['range'], $yesterday);
$numericTable = ArchiveTableCreator::getNumericTable($date);
Db::query(sprintf($query, $numericTable), $bind);
Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]", $yesterday, $numericTable);
try {
Db::query(sprintf($query, ArchiveTableCreator::getBlobTable($date)), $bind);
} catch (Exception $e) {
// Individual blob tables could be missing
}
}
protected static function deleteArchiveIds(Date $date, $idArchivesToDelete)
{
$query = "DELETE FROM %s WHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")";
Db::query(sprintf($query, ArchiveTableCreator::getNumericTable($date)));
try {
Db::query(sprintf($query, ArchiveTableCreator::getBlobTable($date)));
} catch (Exception $e) {
// Individual blob tables could be missing
}
}
protected static function getTemporaryArchiveIdsOlderThan(Date $date, $purgeArchivesOlderThan)
{
$query = "SELECT idarchive
FROM " . ArchiveTableCreator::getNumericTable($date) . "
WHERE name LIKE 'done%'
AND (( value = " . ArchiveWriter::DONE_OK_TEMPORARY . "
AND ts_archived < ?)
OR value = " . ArchiveWriter::DONE_ERROR . ")";
$result = Db::fetchAll($query, array($purgeArchivesOlderThan));
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
}
return $idArchivesToDelete;
return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))";
}
}

72
www/analytics/core/DataAccess/ArchiveTableCreator.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,70 +9,61 @@
namespace Piwik\DataAccess;
use Exception;
use Piwik\Common;
use Piwik\Date;
use Piwik\Db;
use Piwik\DbHelper;
class ArchiveTableCreator
{
const NUMERIC_TABLE = "numeric";
const BLOB_TABLE = "blob";
const BLOB_TABLE = "blob";
public static $tablesAlreadyInstalled = null;
static public $tablesAlreadyInstalled = null;
static public function getNumericTable(Date $date)
public static function getNumericTable(Date $date)
{
return self::getTable($date, self::NUMERIC_TABLE);
}
static public function getBlobTable(Date $date)
public static function getBlobTable(Date $date)
{
return self::getTable($date, self::BLOB_TABLE);
}
static protected function getTable(Date $date, $type)
protected static function getTable(Date $date, $type)
{
$tableNamePrefix = "archive_" . $type;
$tableName = $tableNamePrefix . "_" . $date->toString('Y_m');
$tableName = $tableNamePrefix . "_" . self::getTableMonthFromDate($date);
$tableName = Common::prefixTable($tableName);
self::createArchiveTablesIfAbsent($tableName, $tableNamePrefix);
return $tableName;
}
static protected function createArchiveTablesIfAbsent($tableName, $tableNamePrefix)
protected static function createArchiveTablesIfAbsent($tableName, $tableNamePrefix)
{
if (is_null(self::$tablesAlreadyInstalled)) {
self::refreshTableList();
}
if (!in_array($tableName, self::$tablesAlreadyInstalled)) {
$db = Db::get();
$sql = DbHelper::getTableCreateSql($tableNamePrefix);
// replace table name template by real name
$tableNamePrefix = Common::prefixTable($tableNamePrefix);
$sql = str_replace($tableNamePrefix, $tableName, $sql);
try {
$db->query($sql);
} catch (Exception $e) {
// accept mysql error 1050: table already exists, throw otherwise
if (!$db->isErrNo($e, '1050')) {
throw $e;
}
}
self::getModel()->createArchiveTable($tableName, $tableNamePrefix);
self::$tablesAlreadyInstalled[] = $tableName;
}
}
static public function clear()
private static function getModel()
{
return new Model();
}
public static function clear()
{
self::$tablesAlreadyInstalled = null;
}
static public function refreshTableList($forceReload = false)
public static function refreshTableList($forceReload = false)
{
self::$tablesAlreadyInstalled = DbHelper::getTablesInstalled($forceReload);
}
@ -80,40 +71,53 @@ class ArchiveTableCreator
/**
* Returns all table names archive_*
*
* @param string $type The type of table to return. Either `self::NUMERIC_TABLE` or `self::BLOB_TABLE`.
* @return array
*/
static public function getTablesArchivesInstalled()
public static function getTablesArchivesInstalled($type = null)
{
if (is_null(self::$tablesAlreadyInstalled)) {
self::refreshTableList();
}
if (empty($type)) {
$tableMatchRegex = '/archive_(numeric|blob)_/';
} else {
$tableMatchRegex = '/archive_' . preg_quote($type) . '_/';
}
$archiveTables = array();
foreach (self::$tablesAlreadyInstalled as $table) {
if (strpos($table, 'archive_numeric_') !== false
|| strpos($table, 'archive_blob_') !== false
) {
if (preg_match($tableMatchRegex, $table)) {
$archiveTables[] = $table;
}
}
return $archiveTables;
}
static public function getDateFromTableName($tableName)
public static function getDateFromTableName($tableName)
{
$tableName = Common::unprefixTable($tableName);
$date = str_replace(array('archive_numeric_', 'archive_blob_'), '', $tableName);
$date = str_replace(array('archive_numeric_', 'archive_blob_'), '', $tableName);
return $date;
}
static public function getTypeFromTableName($tableName)
public static function getTableMonthFromDate(Date $date)
{
return $date->toString('Y_m');
}
public static function getTypeFromTableName($tableName)
{
if (strpos($tableName, 'archive_numeric_') !== false) {
return self::NUMERIC_TABLE;
}
if (strpos($tableName, 'archive_blob_') !== false) {
return self::BLOB_TABLE;
}
return false;
}
}

89
www/analytics/core/DataAccess/ArchiveTableDao.php Normal file
View file

@ -0,0 +1,89 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\DataAccess;
use Piwik\Common;
use Piwik\Config;
use Piwik\Db;
/**
* Data Access class for querying numeric & blob archive tables.
*/
class ArchiveTableDao
{
/**
* Analyzes numeric & blob tables for a single table date (ie, `'2015_01'`) and returns
* statistics including:
*
* - number of archives present
* - number of invalidated archives
* - number of temporary archives
* - number of error archives
* - number of segment archives
* - number of numeric rows
* - number of blob rows
*
* @param string $tableDate ie `'2015_01'`
* @return array
*/
public function getArchiveTableAnalysis($tableDate)
{
$numericQueryEmptyRow = array(
'count_archives' => '-',
'count_invalidated_archives' => '-',
'count_temporary_archives' => '-',
'count_error_archives' => '-',
'count_segment_archives' => '-',
'count_numeric_rows' => '-',
);
$tableDate = str_replace("`", "", $tableDate); // for sanity
$numericTable = Common::prefixTable("archive_numeric_$tableDate");
$blobTable = Common::prefixTable("archive_blob_$tableDate");
// query numeric table
$sql = "SELECT CONCAT_WS('.', idsite, date1, date2, period) AS label,
SUM(CASE WHEN name LIKE 'done%' THEN 1 ELSE 0 END) AS count_archives,
SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_invalidated_archives,
SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_temporary_archives,
SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_error_archives,
SUM(CASE WHEN name LIKE 'done%' AND CHAR_LENGTH(name) > 32 THEN 1 ELSE 0 END) AS count_segment_archives,
SUM(CASE WHEN name NOT LIKE 'done%' THEN 1 ELSE 0 END) AS count_numeric_rows,
0 AS count_blob_rows
FROM `$numericTable`
GROUP BY idsite, date1, date2, period";
$rows = Db::fetchAll($sql, array(ArchiveWriter::DONE_INVALIDATED, ArchiveWriter::DONE_OK_TEMPORARY,
ArchiveWriter::DONE_ERROR));
// index result
$result = array();
foreach ($rows as $row) {
$result[$row['label']] = $row;
}
// query blob table & manually merge results (no FULL OUTER JOIN in mysql)
$sql = "SELECT CONCAT_WS('.', idsite, date1, date2, period) AS label,
COUNT(*) AS count_blob_rows
FROM `$blobTable`
GROUP BY idsite, date1, date1, period";
foreach (Db::fetchAll($sql) as $blobStatsRow) {
$label = $blobStatsRow['label'];
if (isset($result[$label])) {
$result[$label] = array_merge($result[$label], $blobStatsRow);
} else {
$result[$label] = $blobStatsRow + $numericQueryEmptyRow;
}
}
return $result;
}
}

209
www/analytics/core/DataAccess/ArchiveWriter.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,17 +9,13 @@
namespace Piwik\DataAccess;
use Exception;
use Piwik\Archive;
use Piwik\Archive\Chunk;
use Piwik\ArchiveProcessor\Rules;
use Piwik\ArchiveProcessor;
use Piwik\Common;
use Piwik\Config;
use Piwik\Db;
use Piwik\Db\BatchInsert;
use Piwik\Log;
use Piwik\Period;
use Piwik\Segment;
use Piwik\SettingsPiwik;
/**
* This class is used to create a new Archive.
@ -28,7 +24,6 @@ use Piwik\SettingsPiwik;
*/
class ArchiveWriter
{
const PREFIX_SQL_LOCK = "locked_";
/**
* Flag stored at the end of the archiving
*
@ -50,23 +45,31 @@ class ArchiveWriter
*/
const DONE_OK_TEMPORARY = 3;
/**
* Flag indicated that archive is done but was marked as invalid later and needs to be re-processed during next archiving process
*
* @var int
*/
const DONE_INVALIDATED = 4;
protected $fields = array('idarchive',
'idsite',
'date1',
'date2',
'period',
'ts_archived',
'name',
'value');
'idsite',
'date1',
'date2',
'period',
'ts_archived',
'name',
'value');
public function __construct(ArchiveProcessor\Parameters $params, $isArchiveTemporary)
{
$this->idArchive = false;
$this->idSite = $params->getSite()->getId();
$this->segment = $params->getSegment();
$this->period = $params->getPeriod();
$this->idSite = $params->getSite()->getId();
$this->segment = $params->getSegment();
$this->period = $params->getPeriod();
$idSites = array($this->idSite);
$this->doneFlag = Rules::getDoneStringFlagFor($idSites, $this->segment, $this->period->getLabel(), $params->getRequestedPlugin(), $params->isSkipAggregationOfSubTables());
$this->doneFlag = Rules::getDoneStringFlagFor($idSites, $this->segment, $this->period->getLabel(), $params->getRequestedPlugin());
$this->isArchiveTemporary = $isArchiveTemporary;
$this->dateStart = $this->period->getDateStart();
@ -74,25 +77,32 @@ class ArchiveWriter
/**
* @param string $name
* @param string[] $values
* @param string|string[] $values A blob string or an array of blob strings. If an array
* is used, the first element in the array will be inserted
* with the `$name` name. The others will be splitted into chunks. All subtables
* within one chunk will be serialized as an array where the index is the
* subtableId.
*/
public function insertBlobRecord($name, $values)
{
if (is_array($values)) {
$clean = array();
foreach ($values as $id => $value) {
// for the parent Table we keep the name
// for example for the Table of searchEngines we keep the name 'referrer_search_engine'
// but for the child table of 'Google' which has the ID = 9 the name would be 'referrer_search_engine_9'
$newName = $name;
if ($id != 0) {
//FIXMEA: refactor
$newName = $name . '_' . $id;
}
$value = $this->compress($value);
$clean[] = array($newName, $value);
if (isset($values[0])) {
// we always store the root table in a single blob for fast access
$clean[] = array($name, $this->compress($values[0]));
unset($values[0]);
}
if (!empty($values)) {
// we move all subtables into chunks
$chunk = new Chunk();
$chunks = $chunk->moveArchiveBlobsIntoChunks($name, $values);
foreach ($chunks as $index => $subtables) {
$clean[] = array($index, $this->compress(serialize($subtables)));
}
}
$this->insertBulkRecords($clean);
return;
}
@ -106,6 +116,7 @@ class ArchiveWriter
if ($this->idArchive === false) {
throw new Exception("Must call allocateNewArchiveId() first");
}
return $this->idArchive;
}
@ -117,108 +128,49 @@ class ArchiveWriter
public function finalizeArchive()
{
$this->deletePreviousArchiveStatus();
$numericTable = $this->getTableNumeric();
$idArchive = $this->getIdArchive();
$this->getModel()->deletePreviousArchiveStatus($numericTable, $idArchive, $this->doneFlag);
$this->logArchiveStatusAsFinal();
}
static protected function compress($data)
protected function compress($data)
{
if (Db::get()->hasBlobDataType()) {
return gzcompress($data);
}
return $data;
}
protected function getArchiveLockName()
{
$numericTable = $this->getTableNumeric();
$dbLockName = "allocateNewArchiveId.$numericTable";
return $dbLockName;
}
protected function acquireArchiveTableLock()
{
$dbLockName = $this->getArchiveLockName();
if (Db::getDbLock($dbLockName, $maxRetries = 30) === false) {
throw new Exception("allocateNewArchiveId: Cannot get named lock $dbLockName.");
}
}
protected function releaseArchiveTableLock()
{
$dbLockName = $this->getArchiveLockName();
Db::releaseDbLock($dbLockName);
}
protected function allocateNewArchiveId()
{
$this->idArchive = $this->insertNewArchiveId();
$numericTable = $this->getTableNumeric();
$this->idArchive = $this->getModel()->allocateNewArchiveId($numericTable);
return $this->idArchive;
}
/**
* Locks the archive table to generate a new archive ID.
*
* We lock to make sure that
* if several archiving processes are running at the same time (for different websites and/or periods)
* then they will each use a unique archive ID.
*
* @return int
*/
protected function insertNewArchiveId()
private function getModel()
{
$numericTable = $this->getTableNumeric();
$idSite = $this->idSite;
$this->acquireArchiveTableLock();
$locked = self::PREFIX_SQL_LOCK . Common::generateUniqId();
$date = date("Y-m-d H:i:s");
$insertSql = "INSERT INTO $numericTable "
. " SELECT IFNULL( MAX(idarchive), 0 ) + 1,
'" . $locked . "',
" . (int)$idSite . ",
'" . $date . "',
'" . $date . "',
0,
'" . $date . "',
0 "
. " FROM $numericTable as tb1";
Db::get()->exec($insertSql);
$this->releaseArchiveTableLock();
$selectIdSql = "SELECT idarchive FROM $numericTable WHERE name = ? LIMIT 1";
$id = Db::get()->fetchOne($selectIdSql, $locked);
return $id;
return new Model();
}
protected function logArchiveStatusAsIncomplete()
{
$statusWhileProcessing = self::DONE_ERROR;
$this->insertRecord($this->doneFlag, $statusWhileProcessing);
}
protected function deletePreviousArchiveStatus()
{
// without advisory lock here, the DELETE would acquire Exclusive Lock
$this->acquireArchiveTableLock();
Db::query("DELETE FROM " . $this->getTableNumeric() . "
WHERE idarchive = ? AND (name = '" . $this->doneFlag
. "' OR name LIKE '" . self::PREFIX_SQL_LOCK . "%')",
array($this->getIdArchive())
);
$this->releaseArchiveTableLock();
$this->insertRecord($this->doneFlag, self::DONE_ERROR);
}
protected function logArchiveStatusAsFinal()
{
$status = self::DONE_OK;
if ($this->isArchiveTemporary) {
$status = self::DONE_OK_TEMPORARY;
}
$this->insertRecord($this->doneFlag, $status);
}
@ -231,27 +183,37 @@ class ArchiveWriter
foreach ($records as $record) {
$this->insertRecord($record[0], $record[1]);
}
return true;
}
$bindSql = $this->getInsertRecordBind();
$values = array();
$values = array();
$valueSeen = false;
foreach ($records as $record) {
// don't record zero
if (empty($record[1])) continue;
if (empty($record[1])) {
continue;
}
$bind = $bindSql;
$bind[] = $record[0]; // name
$bind[] = $record[1]; // value
$bind = $bindSql;
$bind[] = $record[0]; // name
$bind[] = $record[1]; // value
$values[] = $bind;
$valueSeen = $record[1];
}
if (empty($values)) return true;
if (empty($values)) {
return true;
}
$tableName = $this->getTableNameToInsert($valueSeen);
BatchInsert::tableInsertBatch($tableName, $this->getInsertFields(), $values);
$fields = $this->getInsertFields();
BatchInsert::tableInsertBatch($tableName, $fields, $values, $throwException = false, $charset = 'latin1');
return true;
}
@ -270,26 +232,22 @@ class ArchiveWriter
}
$tableName = $this->getTableNameToInsert($value);
$fields = $this->getInsertFields();
$record = $this->getInsertRecordBind();
$this->getModel()->insertRecord($tableName, $fields, $record, $name, $value);
// duplicate idarchives are Ignored, see http://dev.piwik.org/trac/ticket/987
$query = "INSERT IGNORE INTO " . $tableName . "
(" . implode(", ", $this->getInsertFields()) . ")
VALUES (?,?,?,?,?,?,?,?)";
$bindSql = $this->getInsertRecordBind();
$bindSql[] = $name;
$bindSql[] = $value;
Db::query($query, $bindSql);
return true;
}
protected function getInsertRecordBind()
{
return array($this->getIdArchive(),
$this->idSite,
$this->dateStart->toString('Y-m-d'),
$this->period->getDateEnd()->toString('Y-m-d'),
$this->period->getId(),
date("Y-m-d H:i:s"));
$this->idSite,
$this->dateStart->toString('Y-m-d'),
$this->period->getDateEnd()->toString('Y-m-d'),
$this->period->getId(),
date("Y-m-d H:i:s"));
}
protected function getTableNameToInsert($value)
@ -297,6 +255,7 @@ class ArchiveWriter
if (is_numeric($value)) {
return $this->getTableNumeric();
}
return ArchiveTableCreator::getBlobTable($this->dateStart);
}

221
www/analytics/core/DataAccess/LogAggregator.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -8,55 +8,53 @@
*/
namespace Piwik\DataAccess;
use PDOStatement;
use Piwik\ArchiveProcessor\Parameters;
use Piwik\Common;
use Piwik\DataArray;
use Piwik\Db;
use Piwik\Metrics;
use Piwik\Tracker\GoalManager;
/**
* Contains methods that calculate metrics by aggregating log data (visits, actions, conversions,
* ecommerce items).
*
*
* You can use the methods in this class within {@link Piwik\Plugin\Archiver Archiver} descendants
* to aggregate log data without having to write SQL queries.
*
*
* ### Aggregation Dimension
*
*
* All aggregation methods accept a **dimension** parameter. These parameters are important as
* they control how rows in a table are aggregated together.
*
*
* A **_dimension_** is just a table column. Rows that have the same values for these columns are
* aggregated together. The result of these aggregations is a set of metrics for every recorded value
* of a **dimension**.
*
*
* _Note: A dimension is essentially the same as a **GROUP BY** field._
*
*
* ### Examples
*
*
* **Aggregating visit data**
*
*
* $archiveProcessor = // ...
* $logAggregator = $archiveProcessor->getLogAggregator();
*
*
* // get metrics for every used browser language of all visits by returning visitors
* $query = $logAggregator->queryVisitsByDimension(
* $dimensions = array('log_visit.location_browser_lang'),
* $where = 'log_visit.visitor_returning = 1',
*
*
* // also count visits for each browser language that are not located in the US
* $additionalSelects = array('sum(case when log_visit.location_country <> 'us' then 1 else 0 end) as nonus'),
*
*
* // we're only interested in visits, unique visitors & actions, so don't waste time calculating anything else
* $metrics = array(Metrics::INDEX_NB_UNIQ_VISITORS, Metrics::INDEX_NB_VISITS, Metrics::INDEX_NB_ACTIONS),
* );
* if ($query === false) {
* return;
* }
*
*
* while ($row = $query->fetch()) {
* $uniqueVisitors = $row[Metrics::INDEX_NB_UNIQ_VISITORS];
* $visits = $row[Metrics::INDEX_NB_VISITS];
@ -89,8 +87,8 @@ use Piwik\Tracker\GoalManager;
* $country = $row['location_country'];
* $numEcommerceSales = $row[Metrics::INDEX_GOAL_NB_CONVERSIONS];
* $numVisitsWithEcommerceSales = $row[Metrics::INDEX_GOAL_NB_VISITS_CONVERTED];
* $avgTaxForCountry = $country['avg_tax'];
* $maxShippingForCountry = $country['max_shipping'];
* $avgTaxForCountry = $row['avg_tax'];
* $maxShippingForCountry = $row['max_shipping'];
*
* // ... do something with aggregated data ...
* }
@ -131,15 +129,20 @@ class LogAggregator
/** @var \Piwik\Date */
protected $dateEnd;
/** @var \Piwik\Site */
protected $site;
/** @var int[] */
protected $sites;
/** @var \Piwik\Segment */
protected $segment;
/**
* @var string
*/
private $queryOriginHint = '';
/**
* Constructor.
*
*
* @param \Piwik\ArchiveProcessor\Parameters $params
*/
public function __construct(Parameters $params)
@ -147,30 +150,44 @@ class LogAggregator
$this->dateStart = $params->getDateStart();
$this->dateEnd = $params->getDateEnd();
$this->segment = $params->getSegment();
$this->site = $params->getSite();
$this->sites = $params->getIdSites();
}
public function setQueryOriginHint($nameOfOrigiin)
{
$this->queryOriginHint = $nameOfOrigiin;
}
public function generateQuery($select, $from, $where, $groupBy, $orderBy)
{
$bind = $this->getBindDatetimeSite();
$bind = $this->getGeneralQueryBindParams();
$query = $this->segment->getSelectQuery($select, $from, $where, $bind, $orderBy, $groupBy);
$select = 'SELECT';
if ($this->queryOriginHint && is_array($query) && 0 === strpos(trim($query['sql']), $select)) {
$query['sql'] = trim($query['sql']);
$query['sql'] = 'SELECT /* ' . $this->queryOriginHint . ' */' . substr($query['sql'], strlen($select));
}
return $query;
}
protected function getVisitsMetricFields()
{
return array(
Metrics::INDEX_NB_UNIQ_VISITORS => "count(distinct " . self::LOG_VISIT_TABLE . ".idvisitor)",
Metrics::INDEX_NB_VISITS => "count(*)",
Metrics::INDEX_NB_ACTIONS => "sum(" . self::LOG_VISIT_TABLE . ".visit_total_actions)",
Metrics::INDEX_MAX_ACTIONS => "max(" . self::LOG_VISIT_TABLE . ".visit_total_actions)",
Metrics::INDEX_SUM_VISIT_LENGTH => "sum(" . self::LOG_VISIT_TABLE . ".visit_total_time)",
Metrics::INDEX_BOUNCE_COUNT => "sum(case " . self::LOG_VISIT_TABLE . ".visit_total_actions when 1 then 1 when 0 then 1 else 0 end)",
Metrics::INDEX_NB_VISITS_CONVERTED => "sum(case " . self::LOG_VISIT_TABLE . ".visit_goal_converted when 1 then 1 else 0 end)",
Metrics::INDEX_NB_UNIQ_VISITORS => "count(distinct " . self::LOG_VISIT_TABLE . ".idvisitor)",
Metrics::INDEX_NB_UNIQ_FINGERPRINTS => "count(distinct " . self::LOG_VISIT_TABLE . ".config_id)",
Metrics::INDEX_NB_VISITS => "count(*)",
Metrics::INDEX_NB_ACTIONS => "sum(" . self::LOG_VISIT_TABLE . ".visit_total_actions)",
Metrics::INDEX_MAX_ACTIONS => "max(" . self::LOG_VISIT_TABLE . ".visit_total_actions)",
Metrics::INDEX_SUM_VISIT_LENGTH => "sum(" . self::LOG_VISIT_TABLE . ".visit_total_time)",
Metrics::INDEX_BOUNCE_COUNT => "sum(case " . self::LOG_VISIT_TABLE . ".visit_total_actions when 1 then 1 when 0 then 1 else 0 end)",
Metrics::INDEX_NB_VISITS_CONVERTED => "sum(case " . self::LOG_VISIT_TABLE . ".visit_goal_converted when 1 then 1 else 0 end)",
Metrics::INDEX_NB_USERS => "count(distinct " . self::LOG_VISIT_TABLE . ".user_id)",
);
}
static public function getConversionsMetricFields()
public static function getConversionsMetricFields()
{
return array(
Metrics::INDEX_GOAL_NB_CONVERSIONS => "count(*)",
@ -184,12 +201,12 @@ class LogAggregator
);
}
static private function getSqlConversionRevenueSum($field)
private static function getSqlConversionRevenueSum($field)
{
return self::getSqlRevenue('SUM(' . self::LOG_CONVERSION_TABLE . '.' . $field . ')');
}
static public function getSqlRevenue($field)
public static function getSqlRevenue($field)
{
return "ROUND(" . $field . "," . GoalManager::REVENUE_PRECISION . ")";
}
@ -271,7 +288,7 @@ class LogAggregator
* clause. These can be aggregate expressions, eg, `SUM(somecol)`.
* @param bool|array $metrics The set of metrics to calculate and return. If false, the query will select
* all of them. The following values can be used:
*
*
* - {@link Piwik\Metrics::INDEX_NB_UNIQ_VISITORS}
* - {@link Piwik\Metrics::INDEX_NB_VISITS}
* - {@link Piwik\Metrics::INDEX_NB_ACTIONS}
@ -293,52 +310,61 @@ class LogAggregator
$tableName = self::LOG_VISIT_TABLE;
$availableMetrics = $this->getVisitsMetricFields();
$select = $this->getSelectStatement($dimensions, $tableName, $additionalSelects, $availableMetrics, $metrics);
$from = array($tableName);
$where = $this->getWhereStatement($tableName, self::VISIT_DATETIME_FIELD, $where);
$select = $this->getSelectStatement($dimensions, $tableName, $additionalSelects, $availableMetrics, $metrics);
$from = array($tableName);
$where = $this->getWhereStatement($tableName, self::VISIT_DATETIME_FIELD, $where);
$groupBy = $this->getGroupByStatement($dimensions, $tableName);
$orderBy = false;
if ($rankingQuery) {
$orderBy = '`' . Metrics::INDEX_NB_VISITS . '` DESC';
}
$query = $this->generateQuery($select, $from, $where, $groupBy, $orderBy);
if ($rankingQuery) {
unset($availableMetrics[Metrics::INDEX_MAX_ACTIONS]);
$sumColumns = array_keys($availableMetrics);
if ($metrics) {
$sumColumns = array_intersect($sumColumns, $metrics);
}
$rankingQuery->addColumn($sumColumns, 'sum');
if ($this->isMetricRequested(Metrics::INDEX_MAX_ACTIONS, $metrics)) {
$rankingQuery->addColumn(Metrics::INDEX_MAX_ACTIONS, 'max');
}
return $rankingQuery->execute($query['sql'], $query['bind']);
}
return $this->getDb()->query($query['sql'], $query['bind']);
}
protected function getSelectsMetrics($metricsAvailable, $metricsRequested = false)
{
$selects = array();
foreach ($metricsAvailable as $metricId => $statement) {
if ($this->isMetricRequested($metricId, $metricsRequested)) {
$aliasAs = $this->getSelectAliasAs($metricId);
$aliasAs = $this->getSelectAliasAs($metricId);
$selects[] = $statement . $aliasAs;
}
}
return $selects;
}
protected function getSelectStatement($dimensions, $tableName, $additionalSelects, array $availableMetrics, $requestedMetrics = false)
{
$dimensionsToSelect = $this->getDimensionsToSelect($dimensions, $additionalSelects);
$selects = array_merge(
$this->getSelectDimensions($dimensionsToSelect, $tableName),
$this->getSelectsMetrics($availableMetrics, $requestedMetrics),
!empty($additionalSelects) ? $additionalSelects : array()
);
$select = implode(self::FIELDS_SEPARATOR, $selects);
return $select;
}
@ -355,6 +381,7 @@ class LogAggregator
if (empty($additionalSelects)) {
return $dimensions;
}
$dimensionsToSelect = array();
foreach ($dimensions as $selectAs => $dimension) {
$asAlias = $this->getSelectAliasAs($dimension);
@ -364,6 +391,7 @@ class LogAggregator
}
}
}
$dimensionsToSelect = array_unique($dimensionsToSelect);
return $dimensionsToSelect;
}
@ -382,6 +410,7 @@ class LogAggregator
{
foreach ($dimensions as $selectAs => &$field) {
$selectAsString = $field;
if (!is_numeric($selectAs)) {
$selectAsString = $selectAs;
} else {
@ -390,16 +419,18 @@ class LogAggregator
$selectAsString = $appendSelectAs = false;
}
}
$isKnownField = !in_array($field, array('referrer_data'));
if ($selectAsString == $field
&& $isKnownField
) {
if ($selectAsString == $field && $isKnownField) {
$field = $this->prefixColumn($field, $tableName);
}
if ($appendSelectAs && $selectAsString) {
$field = $this->prefixColumn($field, $tableName) . $this->getSelectAliasAs($selectAsString);
}
}
return $dimensions;
}
@ -422,7 +453,7 @@ class LogAggregator
protected function isFieldFunctionOrComplexExpression($field)
{
return strpos($field, "(") !== false
|| strpos($field, "CASE") !== false;
|| strpos($field, "CASE") !== false;
}
protected function getSelectAliasAs($metricId)
@ -432,32 +463,50 @@ class LogAggregator
protected function isMetricRequested($metricId, $metricsRequested)
{
return $metricsRequested === false
|| in_array($metricId, $metricsRequested);
// do not process INDEX_NB_UNIQ_FINGERPRINTS unless specifically asked for
if ($metricsRequested === false) {
if ($metricId == Metrics::INDEX_NB_UNIQ_FINGERPRINTS) {
return false;
}
return true;
}
return in_array($metricId, $metricsRequested);
}
protected function getWhereStatement($tableName, $datetimeField, $extraWhere = false)
{
$where = "$tableName.$datetimeField >= ?
AND $tableName.$datetimeField <= ?
AND $tableName.idsite = ?";
AND $tableName.idsite IN (". Common::getSqlStringFieldsArray($this->sites) . ")";
if (!empty($extraWhere)) {
$extraWhere = sprintf($extraWhere, $tableName, $tableName);
$where .= ' AND ' . $extraWhere;
$where .= ' AND ' . $extraWhere;
}
return $where;
}
protected function getGroupByStatement($dimensions, $tableName)
{
$dimensions = $this->getSelectDimensions($dimensions, $tableName, $appendSelectAs = false);
$groupBy = implode(", ", $dimensions);
$groupBy = implode(", ", $dimensions);
return $groupBy;
}
protected function getBindDatetimeSite()
/**
* Returns general bind parameters for all log aggregation queries. This includes the datetime
* start of entities, datetime end of entities and IDs of all sites.
*
* @return array
*/
protected function getGeneralQueryBindParams()
{
return array($this->dateStart->getDateStartUTC(), $this->dateEnd->getDateEndUTC(), $this->site->getId());
$bind = array($this->dateStart->getDateStartUTC(), $this->dateEnd->getDateEndUTC());
$bind = array_merge($bind, $this->sites);
return $bind;
}
/**
@ -487,7 +536,7 @@ class LogAggregator
*
* @param string $dimension One or more **log\_conversion\_item** columns to group aggregated data by.
* Eg, `'idaction_sku'` or `'idaction_sku, idaction_category'`.
* @return Zend_Db_Statement A statement object that can be used to iterate through the query's
* @return \Zend_Db_Statement A statement object that can be used to iterate through the query's
* result set. See [above](#queryEcommerceItems-result-set) to learn more
* about what this query selects.
* @api
@ -547,7 +596,7 @@ class LogAggregator
array(
'log_conversion_item.server_time >= ?',
'log_conversion_item.server_time <= ?',
'log_conversion_item.idsite = ?',
'log_conversion_item.idsite IN (' . Common::getSqlStringFieldsArray($this->sites) . ')',
'log_conversion_item.deleted = 0'
)
),
@ -593,7 +642,7 @@ class LogAggregator
* clause. These can be aggregate expressions, eg, `SUM(somecol)`.
* @param bool|array $metrics The set of metrics to calculate and return. If `false`, the query will select
* all of them. The following values can be used:
*
*
* - {@link Piwik\Metrics::INDEX_NB_UNIQ_VISITORS}
* - {@link Piwik\Metrics::INDEX_NB_VISITS}
* - {@link Piwik\Metrics::INDEX_NB_ACTIONS}
@ -604,7 +653,7 @@ class LogAggregator
* log_action should be joined on. The table alias used for each join
* is `"log_action$i"` where `$i` is the index of the column in this
* array.
*
*
* If a string is used for this parameter, the table alias is not
* suffixed (since there is only one column).
* @return mixed A Zend_Db_Statement if `$rankingQuery` isn't supplied, otherwise the result of
@ -617,9 +666,9 @@ class LogAggregator
$tableName = self::LOG_ACTIONS_TABLE;
$availableMetrics = $this->getActionsMetricFields();
$select = $this->getSelectStatement($dimensions, $tableName, $additionalSelects, $availableMetrics, $metrics);
$from = array($tableName);
$where = $this->getWhereStatement($tableName, self::ACTION_DATETIME_FIELD, $where);
$select = $this->getSelectStatement($dimensions, $tableName, $additionalSelects, $availableMetrics, $metrics);
$from = array($tableName);
$where = $this->getWhereStatement($tableName, self::ACTION_DATETIME_FIELD, $where);
$groupBy = $this->getGroupByStatement($dimensions, $tableName);
$orderBy = false;
@ -631,12 +680,14 @@ class LogAggregator
foreach ($joinLogActionOnColumn as $i => $joinColumn) {
$tableAlias = 'log_action' . ($multiJoin ? $i + 1 : '');
if (strpos($joinColumn, ' ') === false) {
$joinOn = $tableAlias . '.idaction = ' . $tableName . '.' . $joinColumn;
} else {
// more complex join column like IF(...)
// more complex join column like if (...)
$joinOn = $tableAlias . '.idaction = ' . $joinColumn;
}
$from[] = array(
'table' => 'log_action',
'tableAlias' => $tableAlias,
@ -656,7 +707,9 @@ class LogAggregator
if ($metrics) {
$sumColumns = array_intersect($sumColumns, $metrics);
}
$rankingQuery->addColumn($sumColumns, 'sum');
return $rankingQuery->execute($query['sql'], $query['bind']);
}
@ -665,7 +718,7 @@ class LogAggregator
protected function getActionsMetricFields()
{
return $availableMetrics = array(
return array(
Metrics::INDEX_NB_VISITS => "count(distinct " . self::LOG_ACTIONS_TABLE . ".idvisit)",
Metrics::INDEX_NB_UNIQ_VISITORS => "count(distinct " . self::LOG_ACTIONS_TABLE . ".idvisitor)",
Metrics::INDEX_NB_ACTIONS => "count(*)",
@ -691,32 +744,32 @@ class LogAggregator
* - **{@link Piwik\Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_SUBTOTAL}**: The total cost of all ecommerce items sold
* within these conversions. This value does not
* include tax, shipping or any applied discount.
*
*
* _This metric is only applicable to the special
* **ecommerce** goal (where `idGoal == 'ecommerceOrder'`)._
* - **{@link Piwik\Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_TAX}**: The total tax applied to every transaction in these
* conversions.
*
*
* _This metric is only applicable to the special
* **ecommerce** goal (where `idGoal == 'ecommerceOrder'`)._
* - **{@link Piwik\Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_SHIPPING}**: The total shipping cost for every transaction
* in these conversions.
*
*
* _This metric is only applicable to the special
* **ecommerce** goal (where `idGoal == 'ecommerceOrder'`)._
* - **{@link Piwik\Metrics::INDEX_GOAL_ECOMMERCE_REVENUE_DISCOUNT}**: The total discount applied to every transaction
* in these conversions.
*
*
* _This metric is only applicable to the special
* **ecommerce** goal (where `idGoal == 'ecommerceOrder'`)._
* - **{@link Piwik\Metrics::INDEX_GOAL_ECOMMERCE_ITEMS}**: The total number of ecommerce items sold in each transaction
* in these conversions.
*
*
* _This metric is only applicable to the special
* **ecommerce** goal (where `idGoal == 'ecommerceOrder'`)._
*
*
* Additional data can be selected through the `$additionalSelects` parameter.
*
*
* _Note: This method will only query the **log_conversion** table. Other tables cannot be joined
* using this method._
*
@ -726,21 +779,22 @@ class LogAggregator
* @param bool|string $where An optional SQL expression used in the SQL's **WHERE** clause.
* @param array $additionalSelects Additional SELECT fields that are not included in the group by
* clause. These can be aggregate expressions, eg, `SUM(somecol)`.
* @return Zend_Db_Statement
* @return \Zend_Db_Statement
*/
public function queryConversionsByDimension($dimensions = array(), $where = false, $additionalSelects = array())
{
$dimensions = array_merge(array(self::IDGOAL_FIELD), $dimensions);
$tableName = self::LOG_CONVERSION_TABLE;
$availableMetrics = $this->getConversionsMetricFields();
$tableName = self::LOG_CONVERSION_TABLE;
$select = $this->getSelectStatement($dimensions, $tableName, $additionalSelects, $availableMetrics);
$from = array($tableName);
$where = $this->getWhereStatement($tableName, self::CONVERSION_DATETIME_FIELD, $where);
$from = array($tableName);
$where = $this->getWhereStatement($tableName, self::CONVERSION_DATETIME_FIELD, $where);
$groupBy = $this->getGroupByStatement($dimensions, $tableName);
$orderBy = false;
$query = $this->generateQuery($select, $from, $where, $groupBy, $orderBy);
$query = $this->generateQuery($select, $from, $where, $groupBy, $orderBy);
return $this->getDb()->query($query['sql'], $query['bind']);
}
@ -750,9 +804,9 @@ class LogAggregator
*
* **Note:** The result of this function is meant for use in the `$additionalSelects` parameter
* in one of the query... methods (for example {@link queryVisitsByDimension()}).
*
*
* **Example**
*
*
* // summarize one column
* $visitTotalActionsRanges = array(
* array(1, 1),
@ -760,7 +814,7 @@ class LogAggregator
* array(10)
* );
* $selects = LogAggregator::getSelectsFromRangedColumn('visit_total_actions', $visitTotalActionsRanges, 'log_visit', 'vta');
*
*
* // summarize another column in the same request
* $visitCountVisitsRanges = array(
* array(1, 1),
@ -771,17 +825,17 @@ class LogAggregator
* $selects,
* LogAggregator::getSelectsFromRangedColumn('visitor_count_visits', $visitCountVisitsRanges, 'log_visit', 'vcv')
* );
*
*
* // perform the query
* $logAggregator = // get the LogAggregator somehow
* $query = $logAggregator->queryVisitsByDimension($dimensions = array(), $where = false, $selects);
* $tableSummary = $query->fetch();
*
*
* $numberOfVisitsWithOneAction = $tableSummary['vta0'];
* $numberOfVisitsBetweenTwoAnd10 = $tableSummary['vta1'];
*
*
* $numberOfVisitsWithVisitCountOfOne = $tableSummary['vcv0'];
*
*
* @param string $column The name of a column in `$table` that will be summarized.
* @param array $ranges The array of ranges over which the data in the table
* will be summarized. For example,
@ -817,14 +871,16 @@ class LogAggregator
{
$selects = array();
$extraCondition = '';
if ($restrictToReturningVisitors) {
// extra condition for the SQL SELECT that makes sure only returning visits are counted
// when creating the 'days since last visit' report
$extraCondition = 'and log_visit.visitor_returning = 1';
$extraSelect = "sum(case when log_visit.visitor_returning = 0 then 1 else 0 end) "
. " as `" . $selectColumnPrefix . 'General_NewVisits' . "`";
$extraSelect = "sum(case when log_visit.visitor_returning = 0 then 1 else 0 end) "
. " as `" . $selectColumnPrefix . 'General_NewVisits' . "`";
$selects[] = $extraSelect;
}
foreach ($ranges as $gap) {
if (count($gap) == 2) {
$lowerBound = $gap[0];
@ -833,12 +889,11 @@ class LogAggregator
$selectAs = "$selectColumnPrefix$lowerBound-$upperBound";
$selects[] = "sum(case when $table.$column between $lowerBound and $upperBound $extraCondition" .
" then 1 else 0 end) as `$selectAs`";
" then 1 else 0 end) as `$selectAs`";
} else {
$lowerBound = $gap[0];
$selectAs = $selectColumnPrefix . ($lowerBound + 1) . urlencode('+');
$selectAs = $selectColumnPrefix . ($lowerBound + 1) . urlencode('+');
$selects[] = "sum(case when $table.$column > $lowerBound $extraCondition then 1 else 0 end) as `$selectAs`";
}
}
@ -859,9 +914,10 @@ class LogAggregator
* value is used.
* @return array
*/
static public function makeArrayOneColumn($row, $columnName, $lookForThisPrefix = false)
public static function makeArrayOneColumn($row, $columnName, $lookForThisPrefix = false)
{
$cleanRow = array();
foreach ($row as $label => $count) {
if (empty($lookForThisPrefix)
|| strpos($label, $lookForThisPrefix) === 0
@ -870,6 +926,7 @@ class LogAggregator
$cleanRow[$cleanLabel] = array($columnName => $count);
}
}
return $cleanRow;
}

391
www/analytics/core/DataAccess/LogQueryBuilder.php Normal file
View file

@ -0,0 +1,391 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\DataAccess;
use Exception;
use Piwik\Common;
use Piwik\Segment\SegmentExpression;
class LogQueryBuilder
{
public function getSelectQueryString(SegmentExpression $segmentExpression, $select, $from, $where, $bind, $groupBy,
$orderBy, $limit)
{
if (!is_array($from)) {
$from = array($from);
}
$fromInitially = $from;
if (!$segmentExpression->isEmpty()) {
$segmentExpression->parseSubExpressionsIntoSqlExpressions($from);
$segmentSql = $segmentExpression->getSql();
$where = $this->getWhereMatchBoth($where, $segmentSql['where']);
$bind = array_merge($bind, $segmentSql['bind']);
}
$joins = $this->generateJoinsString($from);
$joinWithSubSelect = $joins['joinWithSubSelect'];
$from = $joins['sql'];
// hack for https://github.com/piwik/piwik/issues/9194#issuecomment-164321612
$useSpecialConversionGroupBy = (!empty($segmentSql)
&& strpos($groupBy, 'log_conversion.idgoal') !== false
&& $fromInitially == array('log_conversion')
&& strpos($from, 'log_link_visit_action') !== false);
if ($useSpecialConversionGroupBy) {
$innerGroupBy = "CONCAT(log_conversion.idvisit, '_' , log_conversion.idgoal, '_', log_conversion.buster)";
$sql = $this->buildWrappedSelectQuery($select, $from, $where, $groupBy, $orderBy, $limit, $innerGroupBy);
} elseif ($joinWithSubSelect) {
$sql = $this->buildWrappedSelectQuery($select, $from, $where, $groupBy, $orderBy, $limit);
} else {
$sql = $this->buildSelectQuery($select, $from, $where, $groupBy, $orderBy, $limit);
}
return array(
'sql' => $sql,
'bind' => $bind
);
}
private function hasJoinedTableAlreadyManually($tableToFind, $joinToFind, $tables)
{
foreach ($tables as $index => $table) {
if (is_array($table)
&& !empty($table['table'])
&& $table['table'] === $tableToFind
&& (!isset($table['tableAlias']) || $table['tableAlias'] === $tableToFind)
&& isset($table['joinOn']) && $table['joinOn'] === $joinToFind) {
return true;
}
}
return false;
}
private function findIndexOfManuallyAddedTable($tableToFind, $tables)
{
foreach ($tables as $index => $table) {
if (is_array($table)
&& !empty($table['table'])
&& $table['table'] === $tableToFind
&& (!isset($table['tableAlias']) || $table['tableAlias'] === $tableToFind)) {
return $index;
}
}
}
private function hasTableAddedManually($tableToFind, $tables)
{
$table = $this->findIndexOfManuallyAddedTable($tableToFind, $tables);
return isset($table);
}
/**
* Generate the join sql based on the needed tables
* @param array $tables tables to join
* @throws Exception if tables can't be joined
* @return array
*/
private function generateJoinsString(&$tables)
{
$knownTables = array("log_action", "log_visit", "log_link_visit_action", "log_conversion", "log_conversion_item");
$visitsAvailable = $linkVisitActionsTableAvailable = $conversionsAvailable = $conversionItemAvailable = $actionsTableAvailable = false;
$defaultLogActionJoin = "log_link_visit_action.idaction_url = log_action.idaction";
$joinWithSubSelect = false;
$sql = '';
// make sure the tables are joined in the right order
// base table first, then action before conversion
// this way, conversions can be left joined on idvisit
$actionIndex = array_search("log_link_visit_action", $tables);
$conversionIndex = array_search("log_conversion", $tables);
if ($actionIndex > 0 && $conversionIndex > 0 && $actionIndex > $conversionIndex) {
$tables[$actionIndex] = "log_conversion";
$tables[$conversionIndex] = "log_link_visit_action";
}
// same as above: action before visit
$actionIndex = array_search("log_link_visit_action", $tables);
$visitIndex = array_search("log_visit", $tables);
if ($actionIndex > 0 && $visitIndex > 0 && $actionIndex > $visitIndex) {
$tables[$actionIndex] = "log_visit";
$tables[$visitIndex] = "log_link_visit_action";
}
// we need to add log_link_visit_action dynamically to join eg visit with action
$linkVisitAction = array_search("log_link_visit_action", $tables);
$actionIndex = array_search("log_action", $tables);
if ($linkVisitAction === false && $actionIndex > 0) {
$tables[] = "log_link_visit_action";
}
if ($actionIndex > 0
&& $this->hasTableAddedManually('log_action', $tables)
&& !$this->hasJoinedTableAlreadyManually('log_action', $defaultLogActionJoin, $tables)) {
// we cannot join the same table with same alias twice, therefore we need to combine the join via AND
$tableIndex = $this->findIndexOfManuallyAddedTable('log_action', $tables);
$defaultLogActionJoin = '(' . $tables[$tableIndex]['joinOn'] . ' AND ' . $defaultLogActionJoin . ')';
unset($tables[$tableIndex]);
}
$linkVisitAction = array_search("log_link_visit_action", $tables);
$actionIndex = array_search("log_action", $tables);
if ($linkVisitAction > 0 && $actionIndex > 0 && $linkVisitAction > $actionIndex) {
$tables[$actionIndex] = "log_link_visit_action";
$tables[$linkVisitAction] = "log_action";
}
foreach ($tables as $i => $table) {
if (is_array($table)) {
// join condition provided
$alias = isset($table['tableAlias']) ? $table['tableAlias'] : $table['table'];
$sql .= "
LEFT JOIN " . Common::prefixTable($table['table']) . " AS " . $alias
. " ON " . $table['joinOn'];
continue;
}
if (!in_array($table, $knownTables)) {
throw new Exception("Table '$table' can't be used for segmentation");
}
$tableSql = Common::prefixTable($table) . " AS $table";
if ($i == 0) {
// first table
$sql .= $tableSql;
} else {
if ($linkVisitActionsTableAvailable && $table === 'log_action') {
$join = $defaultLogActionJoin;
if ($this->hasJoinedTableAlreadyManually($table, $join, $tables)) {
$actionsTableAvailable = true;
continue;
}
} elseif ($linkVisitActionsTableAvailable && $table == "log_conversion") {
// have actions, need conversions => join on idvisit
$join = "log_conversion.idvisit = log_link_visit_action.idvisit";
} elseif ($linkVisitActionsTableAvailable && $table == "log_visit") {
// have actions, need visits => join on idvisit
$join = "log_visit.idvisit = log_link_visit_action.idvisit";
if ($this->hasJoinedTableAlreadyManually($table, $join, $tables)) {
$visitsAvailable = true;
continue;
}
} elseif ($visitsAvailable && $table == "log_link_visit_action") {
// have visits, need actions => we have to use a more complex join
// we don't hande this here, we just return joinWithSubSelect=true in this case
$joinWithSubSelect = true;
$join = "log_link_visit_action.idvisit = log_visit.idvisit";
if ($this->hasJoinedTableAlreadyManually($table, $join, $tables)) {
$linkVisitActionsTableAvailable = true;
continue;
}
} elseif ($conversionsAvailable && $table == "log_link_visit_action") {
// have conversions, need actions => join on idvisit
$join = "log_conversion.idvisit = log_link_visit_action.idvisit";
} elseif (($visitsAvailable && $table == "log_conversion")
|| ($conversionsAvailable && $table == "log_visit")
) {
// have visits, need conversion (or vice versa) => join on idvisit
// notice that joining conversions on visits has lower priority than joining it on actions
$join = "log_conversion.idvisit = log_visit.idvisit";
// if conversions are joined on visits, we need a complex join
if ($table == "log_conversion") {
$joinWithSubSelect = true;
}
} elseif ($conversionItemAvailable && $table === 'log_visit') {
$join = "log_conversion_item.idvisit = log_visit.idvisit";
} elseif ($conversionItemAvailable && $table === 'log_link_visit_action') {
$join = "log_conversion_item.idvisit = log_link_visit_action.idvisit";
} elseif ($conversionItemAvailable && $table === 'log_conversion') {
$join = "log_conversion_item.idvisit = log_conversion.idvisit";
} else {
throw new Exception("Table '$table' can't be joined for segmentation");
}
// the join sql the default way
$sql .= "
LEFT JOIN $tableSql ON $join";
}
// remember which tables are available
$visitsAvailable = ($visitsAvailable || $table == "log_visit");
$linkVisitActionsTableAvailable = ($linkVisitActionsTableAvailable || $table == "log_link_visit_action");
$actionsTableAvailable = ($actionsTableAvailable || $table == "log_action");
$conversionsAvailable = ($conversionsAvailable || $table == "log_conversion");
$conversionItemAvailable = ($conversionItemAvailable || $table == "log_conversion_item");
}
$return = array(
'sql' => $sql,
'joinWithSubSelect' => $joinWithSubSelect
);
return $return;
}
/**
* Build a select query where actions have to be joined on visits (or conversions)
* In this case, the query gets wrapped in another query so that grouping by visit is possible
* @param string $select
* @param string $from
* @param string $where
* @param string $groupBy
* @param string $orderBy
* @param string $limit
* @param null|string $innerGroupBy If given, this inner group by will be used. If not, we try to detect one
* @throws Exception
* @return string
*/
private function buildWrappedSelectQuery($select, $from, $where, $groupBy, $orderBy, $limit, $innerGroupBy = null)
{
$matchTables = "(log_visit|log_conversion_item|log_conversion|log_action)";
preg_match_all("/". $matchTables ."\.[a-z0-9_\*]+/", $select, $matches);
$neededFields = array_unique($matches[0]);
if (count($neededFields) == 0) {
throw new Exception("No needed fields found in select expression. "
. "Please use a table prefix.");
}
preg_match_all("/". $matchTables . "/", $from, $matchesFrom);
$innerSelect = implode(", \n", $neededFields);
$innerFrom = $from;
$innerWhere = $where;
$innerLimit = $limit;
if (!isset($innerGroupBy) && in_array('log_visit', $matchesFrom[1])) {
$innerGroupBy = "log_visit.idvisit";
} elseif (!isset($innerGroupBy)) {
throw new Exception('Cannot use subselect for join as no group by rule is specified');
}
$innerOrderBy = "NULL";
if ($innerLimit && $orderBy) {
// only When LIMITing we can apply to the inner query the same ORDER BY as the parent query
$innerOrderBy = $orderBy;
}
if ($innerLimit) {
// When LIMITing, no need to GROUP BY (GROUPing by is done before the LIMIT which is super slow when large amount of rows is matched)
$innerGroupBy = false;
}
$innerQuery = $this->buildSelectQuery($innerSelect, $innerFrom, $innerWhere, $innerGroupBy, $innerOrderBy, $innerLimit);
$select = preg_replace('/'.$matchTables.'\./', 'log_inner.', $select);
$from = "
(
$innerQuery
) AS log_inner";
$where = false;
$orderBy = preg_replace('/'.$matchTables.'\./', 'log_inner.', $orderBy);
$groupBy = preg_replace('/'.$matchTables.'\./', 'log_inner.', $groupBy);
$query = $this->buildSelectQuery($select, $from, $where, $groupBy, $orderBy, $limit);
return $query;
}
/**
* Build select query the normal way
*
* @param string $select fieldlist to be selected
* @param string $from tablelist to select from
* @param string $where where clause
* @param string $groupBy group by clause
* @param string $orderBy order by clause
* @param string|int $limit limit by clause eg '5' for Limit 5 Offset 0 or '10, 5' for Limit 5 Offset 10
* @return string
*/
private function buildSelectQuery($select, $from, $where, $groupBy, $orderBy, $limit)
{
$sql = "
SELECT
$select
FROM
$from";
if ($where) {
$sql .= "
WHERE
$where";
}
if ($groupBy) {
$sql .= "
GROUP BY
$groupBy";
}
if ($orderBy) {
$sql .= "
ORDER BY
$orderBy";
}
$sql = $this->appendLimitClauseToQuery($sql, $limit);
return $sql;
}
private function appendLimitClauseToQuery($sql, $limit)
{
$limitParts = explode(',', (string) $limit);
$isLimitWithOffset = 2 === count($limitParts);
if ($isLimitWithOffset) {
// $limit = "10, 5". We would not have to do this but we do to prevent possible injections.
$offset = trim($limitParts[0]);
$limit = trim($limitParts[1]);
$sql .= sprintf(' LIMIT %d, %d', $offset, $limit);
} else {
// $limit = "5"
$limit = (int)$limit;
if ($limit >= 1) {
$sql .= " LIMIT $limit";
}
}
return $sql;
}
/**
* @param $where
* @param $segmentWhere
* @return string
* @throws
*/
protected function getWhereMatchBoth($where, $segmentWhere)
{
if (empty($segmentWhere) && empty($where)) {
throw new \Exception("Segment where clause should be non empty.");
}
if (empty($segmentWhere)) {
return $where;
}
if (empty($where)) {
return $segmentWhere;
}
return "( $where )
AND
($segmentWhere)";
}
}

358
www/analytics/core/DataAccess/Model.php Normal file
View file

@ -0,0 +1,358 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\DataAccess;
use Exception;
use Piwik\ArchiveProcessor\Rules;
use Piwik\Common;
use Piwik\Container\StaticContainer;
use Piwik\Db;
use Piwik\DbHelper;
use Piwik\Period;
use Piwik\Segment;
use Piwik\Sequence;
use Psr\Log\LoggerInterface;
/**
* Cleans up outdated archives
*
* @package Piwik\DataAccess
*/
class Model
{
/**
* @var LoggerInterface
*/
private $logger;
public function __construct(LoggerInterface $logger = null)
{
$this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
}
/**
* Returns the archives IDs that have already been invalidated and have been since re-processed.
*
* These archives { archive name (includes segment hash) , idsite, date, period } will be deleted.
*
* @param string $archiveTable
* @param array $idSites
* @return array
* @throws Exception
*/
public function getInvalidatedArchiveIdsSafeToDelete($archiveTable, array $idSites)
{
try {
Db::get()->query('SET SESSION group_concat_max_len=' . (128 * 1024));
} catch (\Exception $ex) {
$this->logger->info("Could not set group_concat_max_len MySQL session variable.");
}
$idSites = array_map(function ($v) { return (int)$v; }, $idSites);
$sql = "SELECT idsite, date1, date2, period, name,
GROUP_CONCAT(idarchive, '.', value ORDER BY ts_archived DESC) as archives
FROM `$archiveTable`
WHERE name LIKE 'done%'
AND value IN (" . ArchiveWriter::DONE_INVALIDATED . ','
. ArchiveWriter::DONE_OK . ','
. ArchiveWriter::DONE_OK_TEMPORARY . ")
AND idsite IN (" . implode(',', $idSites) . ")
GROUP BY idsite, date1, date2, period, name";
$archiveIds = array();
$rows = Db::fetchAll($sql);
foreach ($rows as $row) {
$duplicateArchives = explode(',', $row['archives']);
$firstArchive = array_shift($duplicateArchives);
list($firstArchiveId, $firstArchiveValue) = explode('.', $firstArchive);
// if the first archive (ie, the newest) is an 'ok' or 'ok temporary' archive, then
// all invalidated archives after it can be deleted
if ($firstArchiveValue == ArchiveWriter::DONE_OK
|| $firstArchiveValue == ArchiveWriter::DONE_OK_TEMPORARY
) {
foreach ($duplicateArchives as $pair) {
if (strpos($pair, '.') === false) {
$this->logger->info("GROUP_CONCAT cut off the query result, you may have to purge archives again.");
break;
}
list($idarchive, $value) = explode('.', $pair);
if ($value == ArchiveWriter::DONE_INVALIDATED) {
$archiveIds[] = $idarchive;
}
}
}
}
return $archiveIds;
}
/**
* @param string $archiveTable Prefixed table name
* @param int[] $idSites
* @param string[][] $datesByPeriodType
* @param Segment $segment
* @return \Zend_Db_Statement
* @throws Exception
*/
public function updateArchiveAsInvalidated($archiveTable, $idSites, $datesByPeriodType, Segment $segment = null)
{
$idSites = array_map('intval', $idSites);
$bind = array();
$periodConditions = array();
foreach ($datesByPeriodType as $periodType => $dates) {
$dateConditions = array();
foreach ($dates as $date) {
$dateConditions[] = "(date1 <= ? AND ? <= date2)";
$bind[] = $date;
$bind[] = $date;
}
$dateConditionsSql = implode(" OR ", $dateConditions);
if (empty($periodType)
|| $periodType == Period\Day::PERIOD_ID
) {
// invalidate all periods if no period supplied or period is day
$periodConditions[] = "($dateConditionsSql)";
} else if ($periodType == Period\Range::PERIOD_ID) {
$periodConditions[] = "(period = " . Period\Range::PERIOD_ID . " AND ($dateConditionsSql))";
} else {
// for non-day periods, invalidate greater periods, but not range periods
$periodConditions[] = "(period >= " . (int)$periodType . " AND period < " . Period\Range::PERIOD_ID . " AND ($dateConditionsSql))";
}
}
if ($segment) {
$nameCondition = "name LIKE '" . Rules::getDoneFlagArchiveContainsAllPlugins($segment) . "%'";
} else {
$nameCondition = "name LIKE 'done%'";
}
$sql = "UPDATE $archiveTable SET value = " . ArchiveWriter::DONE_INVALIDATED
. " WHERE $nameCondition
AND idsite IN (" . implode(", ", $idSites) . ")
AND (" . implode(" OR ", $periodConditions) . ")";
return Db::query($sql, $bind);
}
public function getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan)
{
$query = "SELECT idarchive FROM " . $archiveTable . "
WHERE name LIKE 'done%'
AND (( value = " . ArchiveWriter::DONE_OK_TEMPORARY . "
AND ts_archived < ?)
OR value = " . ArchiveWriter::DONE_ERROR . ")";
return Db::fetchAll($query, array($purgeArchivesOlderThan));
}
public function deleteArchivesWithPeriod($numericTable, $blobTable, $period, $date)
{
$query = "DELETE FROM %s WHERE period = ? AND ts_archived < ?";
$bind = array($period, $date);
$queryObj = Db::query(sprintf($query, $numericTable), $bind);
$deletedRows = $queryObj->rowCount();
try {
$queryObj = Db::query(sprintf($query, $blobTable), $bind);
$deletedRows += $queryObj->rowCount();
} catch (Exception $e) {
// Individual blob tables could be missing
$this->logger->debug("Unable to delete archives by period from {blobTable}.", array(
'blobTable' => $blobTable,
'exception' => $e,
));
}
return $deletedRows;
}
public function deleteArchiveIds($numericTable, $blobTable, $idsToDelete)
{
$idsToDelete = array_values($idsToDelete);
$query = "DELETE FROM %s WHERE idarchive IN (" . Common::getSqlStringFieldsArray($idsToDelete) . ")";
$queryObj = Db::query(sprintf($query, $numericTable), $idsToDelete);
$deletedRows = $queryObj->rowCount();
try {
$queryObj = Db::query(sprintf($query, $blobTable), $idsToDelete);
$deletedRows += $queryObj->rowCount();
} catch (Exception $e) {
// Individual blob tables could be missing
$this->logger->debug("Unable to delete archive IDs from {blobTable}.", array(
'blobTable' => $blobTable,
'exception' => $e,
));
}
return $deletedRows;
}
public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues)
{
$bindSQL = array($idSite,
$dateStartIso,
$dateEndIso,
$period,
);
$timeStampWhere = '';
if ($minDatetimeIsoArchiveProcessedUTC) {
$timeStampWhere = " AND ts_archived >= ? ";
$bindSQL[] = $minDatetimeIsoArchiveProcessedUTC;
}
$sqlWhereArchiveName = self::getNameCondition($doneFlags, $doneFlagValues);
$sqlQuery = "SELECT idarchive, value, name, date1 as startDate FROM $numericTable
WHERE idsite = ?
AND date1 = ?
AND date2 = ?
AND period = ?
AND ( ($sqlWhereArchiveName)
OR name = '" . ArchiveSelector::NB_VISITS_RECORD_LOOKED_UP . "'
OR name = '" . ArchiveSelector::NB_VISITS_CONVERTED_RECORD_LOOKED_UP . "')
$timeStampWhere
ORDER BY idarchive DESC";
$results = Db::fetchAll($sqlQuery, $bindSQL);
return $results;
}
public function createArchiveTable($tableName, $tableNamePrefix)
{
$db = Db::get();
$sql = DbHelper::getTableCreateSql($tableNamePrefix);
// replace table name template by real name
$tableNamePrefix = Common::prefixTable($tableNamePrefix);
$sql = str_replace($tableNamePrefix, $tableName, $sql);
try {
$db->query($sql);
} catch (Exception $e) {
// accept mysql error 1050: table already exists, throw otherwise
if (!$db->isErrNo($e, '1050')) {
throw $e;
}
}
try {
if (ArchiveTableCreator::NUMERIC_TABLE === ArchiveTableCreator::getTypeFromTableName($tableName)) {
$sequence = new Sequence($tableName);
$sequence->create();
}
} catch (Exception $e) {
}
}
public function allocateNewArchiveId($numericTable)
{
$sequence = new Sequence($numericTable);
try {
$idarchive = $sequence->getNextId();
} catch (Exception $e) {
// edge case: sequence was not found, create it now
$sequence->create();
$idarchive = $sequence->getNextId();
}
return $idarchive;
}
public function deletePreviousArchiveStatus($numericTable, $archiveId, $doneFlag)
{
$tableWithoutLeadingPrefix = $numericTable;
$lenNumericTableWithoutPrefix = strlen('archive_numeric_MM_YYYY');
if (strlen($numericTable) >= $lenNumericTableWithoutPrefix) {
$tableWithoutLeadingPrefix = substr($numericTable, strlen($numericTable) - $lenNumericTableWithoutPrefix);
// we need to make sure lock name is less than 64 characters see https://github.com/piwik/piwik/issues/9131
}
$dbLockName = "rmPrevArchiveStatus.$tableWithoutLeadingPrefix.$archiveId";
// without advisory lock here, the DELETE would acquire Exclusive Lock
$this->acquireArchiveTableLock($dbLockName);
Db::query("DELETE FROM $numericTable WHERE idarchive = ? AND (name = '" . $doneFlag . "')",
array($archiveId)
);
$this->releaseArchiveTableLock($dbLockName);
}
public function insertRecord($tableName, $fields, $record, $name, $value)
{
// duplicate idarchives are Ignored, see https://github.com/piwik/piwik/issues/987
$query = "INSERT IGNORE INTO " . $tableName . " (" . implode(", ", $fields) . ")
VALUES (?,?,?,?,?,?,?,?)";
$bindSql = $record;
$bindSql[] = $name;
$bindSql[] = $value;
Db::query($query, $bindSql);
return true;
}
/**
* Returns the site IDs for invalidated archives in an archive table.
*
* @param string $numericTable The numeric table to search through.
* @return int[]
*/
public function getSitesWithInvalidatedArchive($numericTable)
{
$rows = Db::fetchAll("SELECT DISTINCT idsite FROM `$numericTable` WHERE name LIKE 'done%' AND value = " . ArchiveWriter::DONE_INVALIDATED);
$result = array();
foreach ($rows as $row) {
$result[] = $row['idsite'];
}
return $result;
}
/**
* Returns the SQL condition used to find successfully completed archives that
* this instance is querying for.
*/
private static function getNameCondition($doneFlags, $possibleValues)
{
$allDoneFlags = "'" . implode("','", $doneFlags) . "'";
// create the SQL to find archives that are DONE
return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))";
}
protected function acquireArchiveTableLock($dbLockName)
{
if (Db::getDbLock($dbLockName, $maxRetries = 30) === false) {
throw new Exception("Cannot get named lock $dbLockName.");
}
}
protected function releaseArchiveTableLock($dbLockName)
{
Db::releaseDbLock($dbLockName);
}
}

402
www/analytics/core/DataAccess/RawLogDao.php Normal file
View file

@ -0,0 +1,402 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\DataAccess;
use Piwik\Common;
use Piwik\Container\StaticContainer;
use Piwik\Db;
use Piwik\Plugin\Dimension\DimensionMetadataProvider;
/**
* DAO that queries log tables.
*/
class RawLogDao
{
const DELETE_UNUSED_ACTIONS_TEMP_TABLE_NAME = 'tmp_log_actions_to_keep';
/**
* @var DimensionMetadataProvider
*/
private $dimensionMetadataProvider;
public function __construct(DimensionMetadataProvider $provider = null)
{
$this->dimensionMetadataProvider = $provider ?: StaticContainer::get('Piwik\Plugin\Dimension\DimensionMetadataProvider');
}
/**
* @param array $values
* @param string $idVisit
*/
public function updateVisits(array $values, $idVisit)
{
$sql = "UPDATE " . Common::prefixTable('log_visit')
. " SET " . $this->getColumnSetExpressions(array_keys($values))
. " WHERE idvisit = ?";
$this->update($sql, $values, $idVisit);
}
/**
* @param array $values
* @param string $idVisit
*/
public function updateConversions(array $values, $idVisit)
{
$sql = "UPDATE " . Common::prefixTable('log_conversion')
. " SET " . $this->getColumnSetExpressions(array_keys($values))
. " WHERE idvisit = ?";
$this->update($sql, $values, $idVisit);
}
/**
* @param string $from
* @param string $to
* @return int
*/
public function countVisitsWithDatesLimit($from, $to)
{
$sql = "SELECT COUNT(*) AS num_rows"
. " FROM " . Common::prefixTable('log_visit')
. " WHERE visit_last_action_time >= ? AND visit_last_action_time < ?";
$bind = array($from, $to);
return (int) Db::fetchOne($sql, $bind);
}
/**
* Iterates over logs in a log table in chunks. Parameters to this function are as backend agnostic
* as possible w/o dramatically increasing code complexity.
*
* @param string $logTable The log table name. Unprefixed, eg, `log_visit`.
* @param array[] $conditions An array describing the conditions logs must match in the query. Translates to
* the WHERE part of a SELECT statement. Each element must contain three elements:
*
* * the column name
* * the operator (ie, '=', '<>', '<', etc.)
* * the operand (ie, a value)
*
* The elements are AND-ed together.
*
* Example:
*
* ```
* array(
* array('visit_first_action_time', '>=', ...),
* array('visit_first_action_time', '<', ...)
* )
* ```
* @param int $iterationStep The number of rows to query at a time.
* @param callable $callback The callback that processes each chunk of rows.
*/
public function forAllLogs($logTable, $fields, $conditions, $iterationStep, $callback)
{
$idField = $this->getIdFieldForLogTable($logTable);
list($query, $bind) = $this->createLogIterationQuery($logTable, $idField, $fields, $conditions, $iterationStep);
$lastId = 0;
do {
$rows = Db::fetchAll($query, array_merge(array($lastId), $bind));
if (!empty($rows)) {
$lastId = $rows[count($rows) - 1][$idField];
$callback($rows);
}
} while (count($rows) == $iterationStep);
}
/**
* Deletes visits with the supplied IDs from log_visit. This method does not cascade, so rows in other tables w/
* the same visit ID will still exist.
*
* @param int[] $idVisits
* @return int The number of deleted rows.
*/
public function deleteVisits($idVisits)
{
$sql = "DELETE FROM `" . Common::prefixTable('log_visit') . "` WHERE idvisit IN "
. $this->getInFieldExpressionWithInts($idVisits);
$statement = Db::query($sql);
return $statement->rowCount();
}
/**
* Deletes visit actions for the supplied visit IDs from log_link_visit_action.
*
* @param int[] $visitIds
* @return int The number of deleted rows.
*/
public function deleteVisitActionsForVisits($visitIds)
{
$sql = "DELETE FROM `" . Common::prefixTable('log_link_visit_action') . "` WHERE idvisit IN "
. $this->getInFieldExpressionWithInts($visitIds);
$statement = Db::query($sql);
return $statement->rowCount();
}
/**
* Deletes conversions for the supplied visit IDs from log_conversion. This method does not cascade, so
* conversion items will not be deleted.
*
* @param int[] $visitIds
* @return int The number of deleted rows.
*/
public function deleteConversions($visitIds)
{
$sql = "DELETE FROM `" . Common::prefixTable('log_conversion') . "` WHERE idvisit IN "
. $this->getInFieldExpressionWithInts($visitIds);
$statement = Db::query($sql);
return $statement->rowCount();
}
/**
* Deletes conversion items for the supplied visit IDs from log_conversion_item.
*
* @param int[] $visitIds
* @return int The number of deleted rows.
*/
public function deleteConversionItems($visitIds)
{
$sql = "DELETE FROM `" . Common::prefixTable('log_conversion_item') . "` WHERE idvisit IN "
. $this->getInFieldExpressionWithInts($visitIds);
$statement = Db::query($sql);
return $statement->rowCount();
}
/**
* Deletes all unused entries from the log_action table. This method uses a temporary table to store used
* actions, and then deletes rows from log_action that are not in this temporary table.
*
* Table locking is required to avoid concurrency issues.
*
* @throws \Exception If table locking permission is not granted to the current MySQL user.
*/
public function deleteUnusedLogActions()
{
if (!Db::isLockPrivilegeGranted()) {
throw new \Exception("RawLogDao.deleteUnusedLogActions() requires table locking permission in order to complete without error.");
}
// get current max ID in log tables w/ idaction references.
$maxIds = $this->getMaxIdsInLogTables();
$this->createTempTableForStoringUsedActions();
// do large insert (inserting everything before maxIds) w/o locking tables...
$this->insertActionsToKeep($maxIds, $deleteOlderThanMax = true);
// ... then do small insert w/ locked tables to minimize the amount of time tables are locked.
$this->lockLogTables();
$this->insertActionsToKeep($maxIds, $deleteOlderThanMax = false);
// delete before unlocking tables so there's no chance a new log row that references an
// unused action will be inserted.
$this->deleteUnusedActions();
Db::unlockAllTables();
}
/**
* Returns the list of the website IDs that received some visits between the specified timestamp.
*
* @param string $fromDateTime
* @param string $toDateTime
* @return bool true if there are visits for this site between the given timeframe, false if not
*/
public function hasSiteVisitsBetweenTimeframe($fromDateTime, $toDateTime, $idSite)
{
$sites = Db::fetchOne("SELECT 1
FROM " . Common::prefixTable('log_visit') . "
WHERE idsite = ?
AND visit_last_action_time > ?
AND visit_last_action_time < ?
LIMIT 1", array($idSite, $fromDateTime, $toDateTime));
return (bool) $sites;
}
/**
* @param array $columnsToSet
* @return string
*/
protected function getColumnSetExpressions(array $columnsToSet)
{
$columnsToSet = array_map(
function ($column) {
return $column . ' = ?';
},
$columnsToSet
);
return implode(', ', $columnsToSet);
}
/**
* @param array $values
* @param $idVisit
* @param $sql
* @return \Zend_Db_Statement
* @throws \Exception
*/
protected function update($sql, array $values, $idVisit)
{
return Db::query($sql, array_merge(array_values($values), array($idVisit)));
}
private function getIdFieldForLogTable($logTable)
{
switch ($logTable) {
case 'log_visit':
return 'idvisit';
case 'log_link_visit_action':
return 'idlink_va';
case 'log_conversion':
return 'idvisit';
case 'log_conversion_item':
return 'idvisit';
case 'log_action':
return 'idaction';
default:
throw new \InvalidArgumentException("Unknown log table '$logTable'.");
}
}
// TODO: instead of creating a log query like this, we should re-use segments. to do this, however, there must be a 1-1
// mapping for dimensions => segments, and each dimension should automatically have a segment.
private function createLogIterationQuery($logTable, $idField, $fields, $conditions, $iterationStep)
{
$bind = array();
$sql = "SELECT " . implode(', ', $fields) . " FROM `" . Common::prefixTable($logTable) . "` WHERE $idField > ?";
foreach ($conditions as $condition) {
list($column, $operator, $value) = $condition;
if (is_array($value)) {
$sql .= " AND $column IN (" . Common::getSqlStringFieldsArray($value) . ")";
$bind = array_merge($bind, $value);
} else {
$sql .= " AND $column $operator ?";
$bind[] = $value;
}
}
$sql .= " ORDER BY $idField ASC LIMIT " . (int)$iterationStep;
return array($sql, $bind);
}
private function getInFieldExpressionWithInts($idVisits)
{
$sql = "(";
$isFirst = true;
foreach ($idVisits as $idVisit) {
if ($isFirst) {
$isFirst = false;
} else {
$sql .= ', ';
}
$sql .= (int)$idVisit;
}
$sql .= ")";
return $sql;
}
private function getMaxIdsInLogTables()
{
$tables = array('log_conversion', 'log_link_visit_action', 'log_visit', 'log_conversion_item');
$idColumns = $this->getTableIdColumns();
$result = array();
foreach ($tables as $table) {
$idCol = $idColumns[$table];
$result[$table] = Db::fetchOne("SELECT MAX($idCol) FROM " . Common::prefixTable($table));
}
return $result;
}
private function createTempTableForStoringUsedActions()
{
$sql = "CREATE TEMPORARY TABLE " . Common::prefixTable(self::DELETE_UNUSED_ACTIONS_TEMP_TABLE_NAME) . " (
idaction INT(11),
PRIMARY KEY (idaction)
)";
Db::query($sql);
}
// protected for testing purposes
protected function insertActionsToKeep($maxIds, $olderThan = true, $insertIntoTempIterationStep = 100000)
{
$tempTableName = Common::prefixTable(self::DELETE_UNUSED_ACTIONS_TEMP_TABLE_NAME);
$idColumns = $this->getTableIdColumns();
foreach ($this->dimensionMetadataProvider->getActionReferenceColumnsByTable() as $table => $columns) {
$idCol = $idColumns[$table];
foreach ($columns as $col) {
$select = "SELECT $col FROM " . Common::prefixTable($table) . " WHERE $idCol >= ? AND $idCol < ?";
$sql = "INSERT IGNORE INTO $tempTableName $select";
if ($olderThan) {
$start = 0;
$finish = $maxIds[$table];
} else {
$start = $maxIds[$table];
$finish = Db::fetchOne("SELECT MAX($idCol) FROM " . Common::prefixTable($table));
}
Db::segmentedQuery($sql, $start, $finish, $insertIntoTempIterationStep);
}
}
}
private function lockLogTables()
{
Db::lockTables(
$readLocks = Common::prefixTables('log_conversion', 'log_link_visit_action', 'log_visit', 'log_conversion_item'),
$writeLocks = Common::prefixTables('log_action')
);
}
private function deleteUnusedActions()
{
list($logActionTable, $tempTableName) = Common::prefixTables("log_action", self::DELETE_UNUSED_ACTIONS_TEMP_TABLE_NAME);
$deleteSql = "DELETE LOW_PRIORITY QUICK IGNORE $logActionTable
FROM $logActionTable
LEFT JOIN $tempTableName tmp ON tmp.idaction = $logActionTable.idaction
WHERE tmp.idaction IS NULL";
Db::query($deleteSql);
}
private function getTableIdColumns()
{
return array(
'log_link_visit_action' => 'idlink_va',
'log_conversion' => 'idvisit',
'log_visit' => 'idvisit',
'log_conversion_item' => 'idvisit'
);
}
}

56
www/analytics/core/DataAccess/TableMetadata.php Normal file
View file

@ -0,0 +1,56 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\DataAccess;
use Piwik\Db;
/**
* Data Access Object that can be used to get metadata information about
* the MySQL tables Piwik uses.
*/
class TableMetadata
{
/**
* Returns the list of column names for a table.
*
* @param string $table Prefixed table name.
* @return string[] List of column names..
*/
public function getColumns($table)
{
$table = str_replace("`", "", $table);
$columns = Db::fetchAll("SHOW COLUMNS FROM `" . $table . "`");
$columnNames = array();
foreach ($columns as $column) {
$columnNames[] = $column['Field'];
}
return $columnNames;
}
/**
* Returns the list of idaction columns in a table. A column is
* assumed to be an idaction reference if it has `"idaction"` in its
* name (eg, `"idaction_url"` or `"idaction_content_name"`.
*
* @param string $table Prefixed table name.
* @return string[]
*/
public function getIdActionColumnNames($table)
{
$columns = $this->getColumns($table);
$columns = array_filter($columns, function ($columnName) {
return strpos($columnName, 'idaction') !== false;
});
return array_values($columns);
}
}

115
www/analytics/core/DataArray.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -47,7 +47,7 @@ class DataArray
public function sumMetricsVisits($label, $row)
{
if (!isset($this->data[$label])) {
$this->data[$label] = self::makeEmptyRow();
$this->data[$label] = static::makeEmptyRow();
}
$this->doSumVisitsMetrics($row, $this->data[$label]);
}
@ -57,11 +57,12 @@ class DataArray
*
* @return array
*/
static public function makeEmptyRow()
public static function makeEmptyRow()
{
return array(Metrics::INDEX_NB_UNIQ_VISITORS => 0,
Metrics::INDEX_NB_VISITS => 0,
Metrics::INDEX_NB_ACTIONS => 0,
Metrics::INDEX_NB_USERS => 0,
Metrics::INDEX_MAX_ACTIONS => 0,
Metrics::INDEX_SUM_VISIT_LENGTH => 0,
Metrics::INDEX_BOUNCE_COUNT => 0,
@ -79,7 +80,7 @@ class DataArray
*
* @return void
*/
protected function doSumVisitsMetrics($newRowToAdd, &$oldRowToUpdate, $onlyMetricsAvailableInActionsTable = false)
protected function doSumVisitsMetrics($newRowToAdd, &$oldRowToUpdate)
{
// Pre 1.2 format: string indexed rows are returned from the DB
// Left here for Backward compatibility with plugins doing custom SQL queries using these metrics as string
@ -87,9 +88,7 @@ class DataArray
$oldRowToUpdate[Metrics::INDEX_NB_VISITS] += $newRowToAdd['nb_visits'];
$oldRowToUpdate[Metrics::INDEX_NB_ACTIONS] += $newRowToAdd['nb_actions'];
$oldRowToUpdate[Metrics::INDEX_NB_UNIQ_VISITORS] += $newRowToAdd['nb_uniq_visitors'];
if ($onlyMetricsAvailableInActionsTable) {
return;
}
$oldRowToUpdate[Metrics::INDEX_NB_USERS] += $newRowToAdd['nb_users'];
$oldRowToUpdate[Metrics::INDEX_MAX_ACTIONS] = (float)max($newRowToAdd['max_actions'], $oldRowToUpdate[Metrics::INDEX_MAX_ACTIONS]);
$oldRowToUpdate[Metrics::INDEX_SUM_VISIT_LENGTH] += $newRowToAdd['sum_visit_length'];
$oldRowToUpdate[Metrics::INDEX_BOUNCE_COUNT] += $newRowToAdd['bounce_count'];
@ -97,36 +96,81 @@ class DataArray
return;
}
$oldRowToUpdate[Metrics::INDEX_NB_VISITS] += $newRowToAdd[Metrics::INDEX_NB_VISITS];
$oldRowToUpdate[Metrics::INDEX_NB_ACTIONS] += $newRowToAdd[Metrics::INDEX_NB_ACTIONS];
$oldRowToUpdate[Metrics::INDEX_NB_UNIQ_VISITORS] += $newRowToAdd[Metrics::INDEX_NB_UNIQ_VISITORS];
if ($onlyMetricsAvailableInActionsTable) {
// Edge case fail safe
if (!isset($oldRowToUpdate[Metrics::INDEX_NB_VISITS])) {
return;
}
$oldRowToUpdate[Metrics::INDEX_NB_VISITS] += $newRowToAdd[Metrics::INDEX_NB_VISITS];
$oldRowToUpdate[Metrics::INDEX_NB_ACTIONS] += $newRowToAdd[Metrics::INDEX_NB_ACTIONS];
$oldRowToUpdate[Metrics::INDEX_NB_UNIQ_VISITORS] += $newRowToAdd[Metrics::INDEX_NB_UNIQ_VISITORS];
// In case the existing Row had no action metrics (eg. Custom Variable XYZ with "visit" scope)
// but the new Row has action metrics (eg. same Custom Variable XYZ this time with a "page" scope)
if(!isset($oldRowToUpdate[Metrics::INDEX_MAX_ACTIONS])) {
$toZero = array(Metrics::INDEX_MAX_ACTIONS,
if (!isset($oldRowToUpdate[Metrics::INDEX_MAX_ACTIONS])) {
$toZero = array(
Metrics::INDEX_NB_USERS,
Metrics::INDEX_MAX_ACTIONS,
Metrics::INDEX_SUM_VISIT_LENGTH,
Metrics::INDEX_BOUNCE_COUNT,
Metrics::INDEX_NB_VISITS_CONVERTED);
foreach($toZero as $metric) {
Metrics::INDEX_NB_VISITS_CONVERTED
);
foreach ($toZero as $metric) {
$oldRowToUpdate[$metric] = 0;
}
}
$oldRowToUpdate[Metrics::INDEX_NB_USERS] += $newRowToAdd[Metrics::INDEX_NB_USERS];
$oldRowToUpdate[Metrics::INDEX_MAX_ACTIONS] = (float)max($newRowToAdd[Metrics::INDEX_MAX_ACTIONS], $oldRowToUpdate[Metrics::INDEX_MAX_ACTIONS]);
$oldRowToUpdate[Metrics::INDEX_SUM_VISIT_LENGTH] += $newRowToAdd[Metrics::INDEX_SUM_VISIT_LENGTH];
$oldRowToUpdate[Metrics::INDEX_BOUNCE_COUNT] += $newRowToAdd[Metrics::INDEX_BOUNCE_COUNT];
$oldRowToUpdate[Metrics::INDEX_NB_VISITS_CONVERTED] += $newRowToAdd[Metrics::INDEX_NB_VISITS_CONVERTED];
}
/**
* Adds the given row $newRowToAdd to the existing $oldRowToUpdate passed by reference
* The rows are php arrays Name => value
*
* @param array $newRowToAdd
* @param array $oldRowToUpdate
* @param bool $onlyMetricsAvailableInActionsTable
*
* @return void
*/
protected function doSumActionsMetrics($newRowToAdd, &$oldRowToUpdate)
{
// Pre 1.2 format: string indexed rows are returned from the DB
// Left here for Backward compatibility with plugins doing custom SQL queries using these metrics as string
if (!isset($newRowToAdd[Metrics::INDEX_NB_VISITS])) {
$oldRowToUpdate[Metrics::INDEX_NB_VISITS] += $newRowToAdd['nb_visits'];
$oldRowToUpdate[Metrics::INDEX_NB_ACTIONS] += $newRowToAdd['nb_actions'];
$oldRowToUpdate[Metrics::INDEX_NB_UNIQ_VISITORS] += $newRowToAdd['nb_uniq_visitors'];
return;
}
// Edge case fail safe
if (!isset($oldRowToUpdate[Metrics::INDEX_NB_VISITS])) {
return;
}
$oldRowToUpdate[Metrics::INDEX_NB_VISITS] += $newRowToAdd[Metrics::INDEX_NB_VISITS];
if (array_key_exists(Metrics::INDEX_NB_ACTIONS, $newRowToAdd)) {
$oldRowToUpdate[Metrics::INDEX_NB_ACTIONS] += $newRowToAdd[Metrics::INDEX_NB_ACTIONS];
}
if (array_key_exists(Metrics::INDEX_PAGE_NB_HITS, $newRowToAdd)) {
if (!array_key_exists(Metrics::INDEX_PAGE_NB_HITS, $oldRowToUpdate)) {
$oldRowToUpdate[Metrics::INDEX_PAGE_NB_HITS] = 0;
}
$oldRowToUpdate[Metrics::INDEX_PAGE_NB_HITS] += $newRowToAdd[Metrics::INDEX_PAGE_NB_HITS];
}
$oldRowToUpdate[Metrics::INDEX_NB_UNIQ_VISITORS] += $newRowToAdd[Metrics::INDEX_NB_UNIQ_VISITORS];
}
public function sumMetricsGoals($label, $row)
{
$idGoal = $row['idgoal'];
if (!isset($this->data[$label][Metrics::INDEX_GOALS][$idGoal])) {
$this->data[$label][Metrics::INDEX_GOALS][$idGoal] = self::makeEmptyGoalRow($idGoal);
$this->data[$label][Metrics::INDEX_GOALS][$idGoal] = static::makeEmptyGoalRow($idGoal);
}
$this->doSumGoalsMetrics($row, $this->data[$label][Metrics::INDEX_GOALS][$idGoal]);
}
@ -190,12 +234,13 @@ class DataArray
public function sumMetricsActions($label, $row)
{
if (!isset($this->data[$label])) {
$this->data[$label] = self::makeEmptyActionRow();
$this->data[$label] = static::makeEmptyActionRow();
}
$this->doSumVisitsMetrics($row, $this->data[$label], $onlyMetricsAvailableInActionsTable = true);
$this->doSumActionsMetrics($row, $this->data[$label]);
}
static protected function makeEmptyActionRow()
protected static function makeEmptyActionRow()
{
return array(
Metrics::INDEX_NB_UNIQ_VISITORS => 0,
@ -207,12 +252,12 @@ class DataArray
public function sumMetricsEvents($label, $row)
{
if (!isset($this->data[$label])) {
$this->data[$label] = self::makeEmptyEventRow();
$this->data[$label] = static::makeEmptyEventRow();
}
$this->doSumEventsMetrics($row, $this->data[$label], $onlyMetricsAvailableInActionsTable = true);
}
static protected function makeEmptyEventRow()
protected static function makeEmptyEventRow()
{
return array(
Metrics::INDEX_NB_UNIQ_VISITORS => 0,
@ -220,7 +265,7 @@ class DataArray
Metrics::INDEX_EVENT_NB_HITS => 0,
Metrics::INDEX_EVENT_NB_HITS_WITH_VALUE => 0,
Metrics::INDEX_EVENT_SUM_EVENT_VALUE => 0,
Metrics::INDEX_EVENT_MIN_EVENT_VALUE => 0,
Metrics::INDEX_EVENT_MIN_EVENT_VALUE => false,
Metrics::INDEX_EVENT_MAX_EVENT_VALUE => 0,
);
}
@ -239,16 +284,16 @@ class DataArray
$oldRowToUpdate[Metrics::INDEX_EVENT_NB_HITS] += $newRowToAdd[Metrics::INDEX_EVENT_NB_HITS];
$oldRowToUpdate[Metrics::INDEX_EVENT_NB_HITS_WITH_VALUE] += $newRowToAdd[Metrics::INDEX_EVENT_NB_HITS_WITH_VALUE];
$newRowToAdd[Metrics::INDEX_EVENT_SUM_EVENT_VALUE] = round($newRowToAdd[Metrics::INDEX_EVENT_SUM_EVENT_VALUE], self::EVENT_VALUE_PRECISION);
$newRowToAdd[Metrics::INDEX_EVENT_SUM_EVENT_VALUE] = round($newRowToAdd[Metrics::INDEX_EVENT_SUM_EVENT_VALUE], static::EVENT_VALUE_PRECISION);
$oldRowToUpdate[Metrics::INDEX_EVENT_SUM_EVENT_VALUE] += $newRowToAdd[Metrics::INDEX_EVENT_SUM_EVENT_VALUE];
$oldRowToUpdate[Metrics::INDEX_EVENT_MAX_EVENT_VALUE] = round(max($newRowToAdd[Metrics::INDEX_EVENT_MAX_EVENT_VALUE], $oldRowToUpdate[Metrics::INDEX_EVENT_MAX_EVENT_VALUE]), self::EVENT_VALUE_PRECISION);
$oldRowToUpdate[Metrics::INDEX_EVENT_MAX_EVENT_VALUE] = round(max($newRowToAdd[Metrics::INDEX_EVENT_MAX_EVENT_VALUE], $oldRowToUpdate[Metrics::INDEX_EVENT_MAX_EVENT_VALUE]), static::EVENT_VALUE_PRECISION);
// Update minimum only if it is set
if($newRowToAdd[Metrics::INDEX_EVENT_MIN_EVENT_VALUE] !== false) {
if($oldRowToUpdate[Metrics::INDEX_EVENT_MIN_EVENT_VALUE] === false) {
$oldRowToUpdate[Metrics::INDEX_EVENT_MIN_EVENT_VALUE] = round($newRowToAdd[Metrics::INDEX_EVENT_MIN_EVENT_VALUE], self::EVENT_VALUE_PRECISION);
if ($newRowToAdd[Metrics::INDEX_EVENT_MIN_EVENT_VALUE] !== false) {
if ($oldRowToUpdate[Metrics::INDEX_EVENT_MIN_EVENT_VALUE] === false) {
$oldRowToUpdate[Metrics::INDEX_EVENT_MIN_EVENT_VALUE] = round($newRowToAdd[Metrics::INDEX_EVENT_MIN_EVENT_VALUE], static::EVENT_VALUE_PRECISION);
} else {
$oldRowToUpdate[Metrics::INDEX_EVENT_MIN_EVENT_VALUE] = round(min($newRowToAdd[Metrics::INDEX_EVENT_MIN_EVENT_VALUE], $oldRowToUpdate[Metrics::INDEX_EVENT_MIN_EVENT_VALUE]), self::EVENT_VALUE_PRECISION);
$oldRowToUpdate[Metrics::INDEX_EVENT_MIN_EVENT_VALUE] = round(min($newRowToAdd[Metrics::INDEX_EVENT_MIN_EVENT_VALUE], $oldRowToUpdate[Metrics::INDEX_EVENT_MIN_EVENT_VALUE]), static::EVENT_VALUE_PRECISION);
}
}
}
@ -279,7 +324,7 @@ class DataArray
public function sumMetricsVisitsPivot($parentLabel, $label, $row)
{
if (!isset($this->dataTwoLevels[$parentLabel][$label])) {
$this->dataTwoLevels[$parentLabel][$label] = self::makeEmptyRow();
$this->dataTwoLevels[$parentLabel][$label] = static::makeEmptyRow();
}
$this->doSumVisitsMetrics($row, $this->dataTwoLevels[$parentLabel][$label]);
}
@ -288,7 +333,7 @@ class DataArray
{
$idGoal = $row['idgoal'];
if (!isset($this->dataTwoLevels[$parentLabel][$label][Metrics::INDEX_GOALS][$idGoal])) {
$this->dataTwoLevels[$parentLabel][$label][Metrics::INDEX_GOALS][$idGoal] = self::makeEmptyGoalRow($idGoal);
$this->dataTwoLevels[$parentLabel][$label][Metrics::INDEX_GOALS][$idGoal] = static::makeEmptyGoalRow($idGoal);
}
$this->doSumGoalsMetrics($row, $this->dataTwoLevels[$parentLabel][$label][Metrics::INDEX_GOALS][$idGoal]);
}
@ -298,7 +343,7 @@ class DataArray
if (!isset($this->dataTwoLevels[$parentLabel][$label])) {
$this->dataTwoLevels[$parentLabel][$label] = $this->makeEmptyActionRow();
}
$this->doSumVisitsMetrics($row, $this->dataTwoLevels[$parentLabel][$label], $onlyMetricsAvailableInActionsTable = true);
$this->doSumActionsMetrics($row, $this->dataTwoLevels[$parentLabel][$label]);
}
public function sumMetricsEventsPivot($parentLabel, $label, $row)
@ -332,7 +377,7 @@ class DataArray
*/
protected function enrichWithConversions(&$data)
{
foreach ($data as $label => &$values) {
foreach ($data as &$values) {
if (!isset($values[Metrics::INDEX_GOALS])) {
continue;
}
@ -357,7 +402,7 @@ class DataArray
// if there are no "visit" column, we force one to prevent future complications
// eg. This helps the setDefaultColumnsToDisplay() call
if(!isset($values[Metrics::INDEX_NB_VISITS])) {
if (!isset($values[Metrics::INDEX_NB_VISITS])) {
$values[Metrics::INDEX_NB_VISITS] = 0;
}
}
@ -369,9 +414,9 @@ class DataArray
* @param $row
* @return bool
*/
static public function isRowActions($row)
public static function isRowActions($row)
{
return (count($row) == count(self::makeEmptyActionRow())) && isset($row[Metrics::INDEX_NB_ACTIONS]);
return (count($row) == count(static::makeEmptyActionRow())) && isset($row[Metrics::INDEX_NB_ACTIONS]);
}
/**

326
www/analytics/core/DataFiles/Countries.php
View file

@ -1,326 +0,0 @@
<?php
/**
* Piwik - Open source web analytics
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
/**
* Country code and continent database.
*
* The mapping of countries to continents is from MaxMind with the exception
* of Central America. MaxMind groups Central American countries with
* North America. Piwik previously grouped Central American countries with
* South America. Given this conflict and the fact that most of Central
* America lies on its own continental plate (i.e., the Caribbean Plate), we
* currently use a separate continent code (amc).
*/
if (!isset($GLOBALS['Piwik_CountryList'])) {
// Primary reference: ISO 3166-1 alpha-2
$GLOBALS['Piwik_CountryList'] = array(
'ad' => 'eur',
'ae' => 'asi',
'af' => 'asi',
'ag' => 'amc',
'ai' => 'amc',
'al' => 'eur',
'am' => 'asi',
'ao' => 'afr',
'aq' => 'ant',
'ar' => 'ams',
'as' => 'oce',
'at' => 'eur',
'au' => 'oce',
'aw' => 'amc',
'ax' => 'eur',
'az' => 'asi',
'ba' => 'eur',
'bb' => 'amc',
'bd' => 'asi',
'be' => 'eur',
'bf' => 'afr',
'bg' => 'eur',
'bh' => 'asi',
'bi' => 'afr',
'bj' => 'afr',
'bl' => 'amc',
'bm' => 'amc',
'bn' => 'asi',
'bo' => 'ams',
'bq' => 'amc',
'br' => 'ams',
'bs' => 'amc',
'bt' => 'asi',
'bv' => 'ant',
'bw' => 'afr',
'by' => 'eur',
'bz' => 'amc',
'ca' => 'amn',
'cc' => 'asi',
'cd' => 'afr',
'cf' => 'afr',
'cg' => 'afr',
'ch' => 'eur',
'ci' => 'afr',
'ck' => 'oce',
'cl' => 'ams',
'cm' => 'afr',
'cn' => 'asi',
'co' => 'ams',
'cr' => 'amc',
'cu' => 'amc',
'cv' => 'afr',
'cw' => 'amc',
'cx' => 'asi',
'cy' => 'eur',
'cz' => 'eur',
'de' => 'eur',
'dj' => 'afr',
'dk' => 'eur',
'dm' => 'amc',
'do' => 'amc',
'dz' => 'afr',
'ec' => 'ams',
'ee' => 'eur',
'eg' => 'afr',
'eh' => 'afr',
'er' => 'afr',
'es' => 'eur',
'et' => 'afr',
'fi' => 'eur',
'fj' => 'oce',
'fk' => 'ams',
'fm' => 'oce',
'fo' => 'eur',
'fr' => 'eur',
'ga' => 'afr',
'gb' => 'eur',
'gd' => 'amc',
'ge' => 'asi',
'gf' => 'ams',
'gg' => 'eur',
'gh' => 'afr',
'gi' => 'eur',
'gl' => 'amn',
'gm' => 'afr',
'gn' => 'afr',
'gp' => 'amc',
'gq' => 'afr',
'gr' => 'eur',
'gs' => 'ant',
'gt' => 'amc',
'gu' => 'oce',
'gw' => 'afr',
'gy' => 'ams',
'hk' => 'asi',
'hm' => 'ant',
'hn' => 'amc',
'hr' => 'eur',
'ht' => 'amc',
'hu' => 'eur',
'id' => 'asi',
'ie' => 'eur',
'il' => 'asi',
'im' => 'eur',
'in' => 'asi',
'io' => 'asi',
'iq' => 'asi',
'ir' => 'asi',
'is' => 'eur',
'it' => 'eur',
'je' => 'eur',
'jm' => 'amc',
'jo' => 'asi',
'jp' => 'asi',
'ke' => 'afr',
'kg' => 'asi',
'kh' => 'asi',
'ki' => 'oce',
'km' => 'afr',
'kn' => 'amc',
'kp' => 'asi',
'kr' => 'asi',
'kw' => 'asi',
'ky' => 'amc',
'kz' => 'asi',
'la' => 'asi',
'lb' => 'asi',
'lc' => 'amc',
'li' => 'eur',
'lk' => 'asi',
'lr' => 'afr',
'ls' => 'afr',
'lt' => 'eur',
'lu' => 'eur',
'lv' => 'eur',
'ly' => 'afr',
'ma' => 'afr',
'mc' => 'eur',
'md' => 'eur',
'me' => 'eur',
'mf' => 'amc',
'mg' => 'afr',
'mh' => 'oce',
'mk' => 'eur',
'ml' => 'afr',
'mm' => 'asi',
'mn' => 'asi',
'mo' => 'asi',
'mp' => 'oce',
'mq' => 'amc',
'mr' => 'afr',
'ms' => 'amc',
'mt' => 'eur',
'mu' => 'afr',
'mv' => 'asi',
'mw' => 'afr',
'mx' => 'amn',
'my' => 'asi',
'mz' => 'afr',
'na' => 'afr',
'nc' => 'oce',
'ne' => 'afr',
'nf' => 'oce',
'ng' => 'afr',
'ni' => 'amc',
'nl' => 'eur',
'no' => 'eur',
'np' => 'asi',
'nr' => 'oce',
'nu' => 'oce',
'nz' => 'oce',
'om' => 'asi',
'pa' => 'amc',
'pe' => 'ams',
'pf' => 'oce',
'pg' => 'oce',
'ph' => 'asi',
'pk' => 'asi',
'pl' => 'eur',
'pm' => 'amn',
'pn' => 'oce',
'pr' => 'amc',
'ps' => 'asi',
'pt' => 'eur',
'pw' => 'oce',
'py' => 'ams',
'qa' => 'asi',
're' => 'afr',
'ro' => 'eur',
'rs' => 'eur',
'ru' => 'eur',
'rw' => 'afr',
'sa' => 'asi',
'sb' => 'oce',
'sc' => 'afr',
'sd' => 'afr',
'se' => 'eur',
'sg' => 'asi',
'sh' => 'afr',
'si' => 'eur',
'sj' => 'eur',
'sk' => 'eur',
'sl' => 'afr',
'sm' => 'eur',
'sn' => 'afr',
'so' => 'afr',
'sr' => 'ams',
'ss' => 'afr',
'st' => 'afr',
'sv' => 'amc',
'sx' => 'amc',
'sy' => 'asi',
'sz' => 'afr',
'tc' => 'amc',
'td' => 'afr',
'tf' => 'ant',
'tg' => 'afr',
'th' => 'asi',
'ti' => 'asi',
'tj' => 'asi',
'tk' => 'oce',
'tl' => 'asi',
'tm' => 'asi',
'tn' => 'afr',
'to' => 'oce',
'tr' => 'eur',
'tt' => 'amc',
'tv' => 'oce',
'tw' => 'asi',
'tz' => 'afr',
'ua' => 'eur',
'ug' => 'afr',
'um' => 'oce',
'us' => 'amn',
'uy' => 'ams',
'uz' => 'asi',
'va' => 'eur',
'vc' => 'amc',
've' => 'ams',
'vg' => 'amc',
'vi' => 'amc',
'vn' => 'asi',
'vu' => 'oce',
'wf' => 'oce',
'ws' => 'oce',
'ye' => 'asi',
'yt' => 'afr',
'za' => 'afr',
'zm' => 'afr',
'zw' => 'afr',
);
// codes for internal use
$GLOBALS['Piwik_CountryList_Extras'] = array(
// unknown
'xx' => 'unk',
// exceptionally reserved
'ac' => 'afr', // .ac TLD
'cp' => 'amc',
'dg' => 'asi',
'ea' => 'afr',
'eu' => 'eur', // .eu TLD
'fx' => 'eur',
'ic' => 'afr',
'su' => 'eur', // .su TLD
'ta' => 'afr',
'uk' => 'eur', // .uk TLD
// transitionally reserved
'an' => 'amc', // former Netherlands Antilles
'bu' => 'asi',
'cs' => 'eur', // former Serbia and Montenegro
'nt' => 'asi',
'sf' => 'eur',
'tp' => 'oce', // .tp TLD
'yu' => 'eur', // .yu TLD
'zr' => 'afr',
// MaxMind GeoIP specific
'a1' => 'unk',
'a2' => 'unk',
'ap' => 'asi',
'o1' => 'unk',
// Catalonia (Spain)
'cat' => 'eur',
);
}
if (!isset($GLOBALS['Piwik_ContinentList'])) {
// Primary reference: ISO 3166-1 alpha-2
$GLOBALS['Piwik_ContinentList'] = array(
'unk', // unknown
'amn', // North America
'amc', // Central America
'ams', // South America
'eur', // Europe
'afr', // Africa
'asi', // Asia
'oce', // Oceania
'ant', // Antarctica
);
}

186
www/analytics/core/DataFiles/Currencies.php
View file

@ -1,186 +0,0 @@
<?php
/**
* Piwik - Open source web analytics
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
/**
* International currencies in circulation.
*
* @see http://en.wikipedia.org/wiki/List_of_circulating_currencies
*/
if (!isset($GLOBALS['Piwik_CurrencyList'])) {
$GLOBALS['Piwik_CurrencyList'] = array(
// 'ISO-4217 CODE' => array('currency symbol', 'description'),
// Top 5 by global trading volume
'USD' => array('$', 'US dollar'),
'EUR' => array('€', 'Euro'),
'JPY' => array('¥', 'Japanese yen'),
'GBP' => array('£', 'British pound'),
'CHF' => array('Fr', 'Swiss franc'),
'AFN' => array('؋', 'Afghan afghani'),
'ALL' => array('L', 'Albanian lek'),
'DZD' => array('د.ج', 'Algerian dinar'),
'AOA' => array('Kz', 'Angolan kwanza'),
'ARS' => array('$', 'Argentine peso'),
'AMD' => array('դր.', 'Armenian dram'),
'AWG' => array('ƒ', 'Aruban florin'),
'AUD' => array('$', 'Australian dollar'),
'AZN' => array('m', 'Azerbaijani manat'),
'BSD' => array('$', 'Bahamian dollar'),
'BHD' => array('.د.ب', 'Bahraini dinar'),
'BDT' => array('৳', 'Bangladeshi taka'),
'BBD' => array('$', 'Barbadian dollar'),
'BYR' => array('Br', 'Belarusian ruble'),
'BZD' => array('$', 'Belize dollar'),
'BMD' => array('$', 'Bermudian dollar'),
'BTC' => array('BTC', 'Bitcoin'),
'BTN' => array('Nu.', 'Bhutanese ngultrum'),
'BOB' => array('Bs.', 'Bolivian boliviano'),
'BAM' => array('KM', 'Bosnia Herzegovina mark'),
'BWP' => array('P', 'Botswana pula'),
'BRL' => array('R$', 'Brazilian real'),
// 'GBP' => array('£', 'British pound'),
'BND' => array('$', 'Brunei dollar'),
'BGN' => array('лв', 'Bulgarian lev'),
'BIF' => array('Fr', 'Burundian franc'),
'KHR' => array('៛', 'Cambodian riel'),
'CAD' => array('$', 'Canadian dollar'),
'CVE' => array('$', 'Cape Verdean escudo'),
'KYD' => array('$', 'Cayman Islands dollar'),
'XAF' => array('Fr', 'Central African CFA franc'),
'CLP' => array('$', 'Chilean peso'),
'CNY' => array('元', 'Chinese yuan'),
'COP' => array('$', 'Colombian peso'),
'KMF' => array('Fr', 'Comorian franc'),
'CDF' => array('Fr', 'Congolese franc'),
'CRC' => array('₡', 'Costa Rican colón'),
'HRK' => array('kn', 'Croatian kuna'),
'XPF' => array('F', 'CFP franc'),
'CUC' => array('$', 'Cuban convertible peso'),
'CUP' => array('$', 'Cuban peso'),
'CMG' => array('ƒ', 'Curaçao and Sint Maarten guilder'),
'CZK' => array('Kč', 'Czech koruna'),
'DKK' => array('kr', 'Danish krone'),
'DJF' => array('Fr', 'Djiboutian franc'),
'DOP' => array('$', 'Dominican peso'),
'XCD' => array('$', 'East Caribbean dollar'),
'EGP' => array('ج.م', 'Egyptian pound'),
'ERN' => array('Nfk', 'Eritrean nakfa'),
'ETB' => array('Br', 'Ethiopian birr'),
// 'EUR' => array('€', 'Euro'),
'FKP' => array('£', 'Falkland Islands pound'),
'FJD' => array('$', 'Fijian dollar'),
'GMD' => array('D', 'Gambian dalasi'),
'GEL' => array('ლ', 'Georgian lari'),
'GHS' => array('₵', 'Ghanaian cedi'),
'GIP' => array('£', 'Gibraltar pound'),
'GTQ' => array('Q', 'Guatemalan quetzal'),
'GNF' => array('Fr', 'Guinean franc'),
'GYD' => array('$', 'Guyanese dollar'),
'HTG' => array('G', 'Haitian gourde'),
'HNL' => array('L', 'Honduran lempira'),
'HKD' => array('$', 'Hong Kong dollar'),
'HUF' => array('Ft', 'Hungarian forint'),
'ISK' => array('kr', 'Icelandic króna'),
'INR' => array('‎₹', 'Indian rupee'),
'IDR' => array('Rp', 'Indonesian rupiah'),
'IRR' => array('﷼', 'Iranian rial'),
'IQD' => array('ع.د', 'Iraqi dinar'),
'ILS' => array('₪', 'Israeli new shekel'),
'JMD' => array('$', 'Jamaican dollar'),
// 'JPY' => array('¥', 'Japanese yen'),
'JOD' => array('د.ا', 'Jordanian dinar'),
'KZT' => array('₸', 'Kazakhstani tenge'),
'KES' => array('Sh', 'Kenyan shilling'),
'KWD' => array('د.ك', 'Kuwaiti dinar'),
'KGS' => array('лв', 'Kyrgyzstani som'),
'LAK' => array('₭', 'Lao kip'),
'LBP' => array('ل.ل', 'Lebanese pound'),
'LSL' => array('L', 'Lesotho loti'),
'LRD' => array('$', 'Liberian dollar'),
'LYD' => array('ل.د', 'Libyan dinar'),
'LTL' => array('Lt', 'Lithuanian litas'),
'MOP' => array('P', 'Macanese pataca'),
'MKD' => array('ден', 'Macedonian denar'),
'MGA' => array('Ar', 'Malagasy ariary'),
'MWK' => array('MK', 'Malawian kwacha'),
'MYR' => array('RM', 'Malaysian ringgit'),
'MVR' => array('ރ.', 'Maldivian rufiyaa'),
'MRO' => array('UM', 'Mauritanian ouguiya'),
'MUR' => array('₨', 'Mauritian rupee'),
'MXN' => array('$', 'Mexican peso'),
'MDL' => array('L', 'Moldovan leu'),
'MNT' => array('₮', 'Mongolian tögrög'),
'MAD' => array('د.م.', 'Moroccan dirham'),
'MZN' => array('MTn', 'Mozambican metical'),
'MMK' => array('K', 'Myanma kyat'),
'NAD' => array('$', 'Namibian dollar'),
'NPR' => array('₨', 'Nepalese rupee'),
'ANG' => array('ƒ', 'Netherlands Antillean guilder'),
'TWD' => array('$', 'New Taiwan dollar'),
'NZD' => array('$', 'New Zealand dollar'),
'NIO' => array('C$', 'Nicaraguan córdoba'),
'NGN' => array('₦', 'Nigerian naira'),
'KPW' => array('₩', 'North Korean won'),
'NOK' => array('kr', 'Norwegian krone'),
'OMR' => array('ر.ع.', 'Omani rial'),
'PKR' => array('₨', 'Pakistani rupee'),
'PAB' => array('B/.', 'Panamanian balboa'),
'PGK' => array('K', 'Papua New Guinean kina'),
'PYG' => array('₲', 'Paraguayan guaraní'),
'PEN' => array('S/.', 'Peruvian nuevo sol'),
'PHP' => array('₱', 'Philippine peso'),
'PLN' => array('zł', 'Polish złoty'),
'QAR' => array('ر.ق', 'Qatari riyal'),
'RON' => array('L', 'Romanian leu'),
'RUB' => array('руб.', 'Russian ruble'),
'RWF' => array('Fr', 'Rwandan franc'),
'SHP' => array('£', 'Saint Helena pound'),
'SVC' => array('₡', 'Salvadoran colón'),
'WST' => array('T', 'Samoan tala'),
'STD' => array('Db', 'São Tomé and Príncipe dobra'),
'SAR' => array('ر.س', 'Saudi riyal'),
'RSD' => array('дин. or din.', 'Serbian dinar'),
'SCR' => array('₨', 'Seychellois rupee'),
'SLL' => array('Le', 'Sierra Leonean leone'),
'SGD' => array('$', 'Singapore dollar'),
'SBD' => array('$', 'Solomon Islands dollar'),
'SOS' => array('Sh', 'Somali shilling'),
'ZAR' => array('R', 'South African rand'),
'KRW' => array('₩', 'South Korean won'),
'LKR' => array('Rs', 'Sri Lankan rupee'),
'SDG' => array('جنيه سوداني', 'Sudanese pound'),
'SRD' => array('$', 'Surinamese dollar'),
'SZL' => array('L', 'Swazi lilangeni'),
'SEK' => array('kr', 'Swedish krona'),
// 'CHF' => array('Fr', 'Swiss franc'),
'SYP' => array('ل.س', 'Syrian pound'),
'TJS' => array('ЅМ', 'Tajikistani somoni'),
'TZS' => array('Sh', 'Tanzanian shilling'),
'THB' => array('฿', 'Thai baht'),
'TOP' => array('T$', 'Tongan paʻanga'),
'TTD' => array('$', 'Trinidad and Tobago dollar'),
'TND' => array('د.ت', 'Tunisian dinar'),
'TRY' => array('TL', 'Turkish lira'),
'TMM' => array('m', 'Turkmenistani manat'),
'UGX' => array('Sh', 'Ugandan shilling'),
'UAH' => array('₴', 'Ukrainian hryvnia'),
'AED' => array('د.إ', 'United Arab Emirates dirham'),
// 'USD' => array('$', 'United States dollar'),
'UYU' => array('$', 'Uruguayan peso'),
'UZS' => array('лв', 'Uzbekistani som'),
'VUV' => array('Vt', 'Vanuatu vatu'),
'VEF' => array('Bs F', 'Venezuelan bolívar'),
'VND' => array('₫', 'Vietnamese đồng'),
'XOF' => array('Fr', 'West African CFA franc'),
'YER' => array('﷼', 'Yemeni rial'),
'ZMW' => array('ZK', 'Zambian kwacha'),
'ZWL' => array('$', 'Zimbabwean dollar'),
);
}

63
www/analytics/core/DataFiles/LanguageToCountry.php
View file

@ -1,63 +0,0 @@
<?php
/**
* Piwik - Open source web analytics
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
/**
* Language to Country mapping
*
* This list is used to guess the visitor's country when the region is
* not specified in the first language tag. Inclusion/exclusion is
* based on Piwik.org visitor statistics and probability of disambiguation.
* (Notably, "en" and "zh" are excluded.)
*
* If you want to add a new entry, please email us at hello at piwik.org
*/
if (!isset($GLOBALS['Piwik_LanguageToCountry'])) {
$GLOBALS['Piwik_LanguageToCountry'] = array(
'bg' => 'bg', // Bulgarian => Bulgaria
'ca' => 'es', // Catalan => Spain
'cs' => 'cz', // Czech => Czech Republic
'da' => 'dk', // Danish => Denmark
'de' => 'de', // German => Germany
'el' => 'gr', // Greek => Greece
'es' => 'es', // Spanish => Spain
'et' => 'ee', // Estonian => Estonia
'fa' => 'ir', // Farsi => Iran
'fi' => 'fi', // Finnish => Finland
'fr' => 'fr', // French => France
'he' => 'il', // Hebrew => Israel
'hr' => 'hr', // Croatian => Croatia
'hu' => 'hu', // Hungarian => Hungary
'id' => 'id', // Indonesian => Indonesia
'is' => 'is', // Icelandic => Iceland
'it' => 'it', // Italian => Italy
'ja' => 'jp', // Japanese => Japan
'ko' => 'kr', // Korean => South Korea
'lt' => 'lt', // Lithuanian => Lithuania
'lv' => 'lv', // Latvian => Latvia
'mk' => 'mk', // Macedonian => Macedonia
'ms' => 'my', // Malay => Malaysia
'nb' => 'no', // Bokmål => Norway
'nl' => 'nl', // Dutch => Netherlands
'nn' => 'no', // Nynorsk => Norway
'no' => 'no', // Norwegian => Norway
'pl' => 'pl', // Polish => Poland
'pt' => 'pt', // Portugese => Portugal
'ro' => 'ro', // Romanian => Romania
'ru' => 'ru', // Russian => Russia
'sk' => 'sk', // Slovak => Slovakia
'sl' => 'si', // Slovene => Slovenia
'sq' => 'al', // Albanian => Albania
'sr' => 'rs', // Serbian => Serbia
'sv' => 'se', // Swedish => Sweden
'th' => 'th', // Thai => Thailand
'bo' => 'ti', // Tibetan => Tibet
'tr' => 'tr', // Turkish => Turkey
'uk' => 'ua', // Ukrainian => Ukraine
);
}

203
www/analytics/core/DataFiles/Languages.php
View file

@ -1,203 +0,0 @@
<?php
/**
* Piwik - Open source web analytics
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
/*
* Language database
*/
if (!isset($GLOBALS['Piwik_LanguageList'])) {
// Reference: ISO 639-1 alpha-2
$GLOBALS['Piwik_LanguageList'] = array(
'aa' => array('Afar'),
'ab' => array('Abkhazian'),
'ae' => array('Avestan'),
'af' => array('Afrikaans'),
'ak' => array('Akan'),
'am' => array('Amharic'),
'an' => array('Aragonese'),
'ar' => array('Arabic'),
'as' => array('Assamese'),
'av' => array('Avaric'),
'ay' => array('Aymara'),
'az' => array('Azerbaijani'),
'ba' => array('Bashkir'),
'be' => array('Belarusian'),
'bg' => array('Bulgarian'),
'bh' => array('Bihari'), // 'Bihari languages'
'bi' => array('Bislama'),
'bm' => array('Bambara'),
'bn' => array('Bengali'),
'bo' => array('Tibetan'),
'br' => array('Breton'),
'bs' => array('Bosnian'),
'ca' => array('Catalan', 'Valencian'),
'ce' => array('Chechen'),
'ch' => array('Chamorro'),
'co' => array('Corsican'),
'cr' => array('Cree'),
'cs' => array('Czech'),
'cu' => array('Church Slavic', 'Old Slavonic', 'Church Slavonic', 'Old Bulgarian', 'Old Church Slavonic'),
'cv' => array('Chuvash'),
'cy' => array('Welsh'),
'da' => array('Danish'),
'de' => array('German'),
'dv' => array('Divehi', 'Dhivehi', 'Maldivian'),
'dz' => array('Dzongkha'),
'ee' => array('Ewe'),
'el' => array('Greek', 'Modern Greek', 'Hellenic'), // Greek, Modern (1453-)
'en' => array('English'),
'eo' => array('Esperanto'),
'es' => array('Spanish', 'Castilian'),
'et' => array('Estonian'),
'eu' => array('Basque'),
'fa' => array('Persian'),
'ff' => array('Fulah'),
'fi' => array('Finnish'),
'fj' => array('Fijian'),
'fo' => array('Faroese'),
'fr' => array('French'),
'fy' => array('Western Frisian'),
'ga' => array('Irish'),
'gd' => array('Gaelic', 'Scottish Gaelic'),
'gl' => array('Galician'),
'gn' => array('Guarani'),
'gu' => array('Gujarati'),
'gv' => array('Manx'),
'ha' => array('Hausa'),
'he' => array('Hebrew'),
'hi' => array('Hindi'),
'ho' => array('Hiri Motu'),
'hr' => array('Croatian'),
'ht' => array('Haitian', 'Haitian Creole'),
'hu' => array('Hungarian'),
'hy' => array('Armenian'),
'hz' => array('Herero'),
'ia' => array('Interlingua'), // 'Interlingua (International Auxiliary Language Association)'
'id' => array('Indonesian'),
'ie' => array('Interlingue', 'Occidental'),
'ig' => array('Igbo'),
'ii' => array('Sichuan Yi', 'Nuosu'),
'ik' => array('Inupiaq'),
'io' => array('Ido'),
'is' => array('Icelandic'),
'it' => array('Italian'),
'iu' => array('Inuktitut'),
'ja' => array('Japanese'),
'jv' => array('Javanese'),
'ka' => array('Georgian'),
'kg' => array('Kongo'),
'ki' => array('Kikuyu', 'Gikuyu'),
'kj' => array('Kuanyama', 'Kwanyama'),
'kk' => array('Kazakh'),
'kl' => array('Kalaallisut', 'Greenlandic'),
'km' => array('Central Khmer'),
'kn' => array('Kannada'),
'ko' => array('Korean'),
'kr' => array('Kanuri'),
'ks' => array('Kashmiri'),
'ku' => array('Kurdish'),
'kv' => array('Komi'),
'kw' => array('Cornish'),
'ky' => array('Kirghiz', 'Kyrgyz'),
'la' => array('Latin'),
'lb' => array('Luxembourgish', 'Letzeburgesch'),
'lg' => array('Ganda'),
'li' => array('Limburgan', 'Limburger', 'Limburgish'),
'ln' => array('Lingala'),
'lo' => array('Lao'),
'lt' => array('Lithuanian'),
'lu' => array('Luba-Katanga'),
'lv' => array('Latvian'),
'mg' => array('Malagasy'),
'mh' => array('Marshallese'),
'mi' => array('Maori'),
'mk' => array('Macedonian'),
'ml' => array('Malayalam'),
'mn' => array('Mongolian'),
// 'mo' => array('Moldavian'), // deprecated
'mr' => array('Marathi'),
'ms' => array('Malay'),
'mt' => array('Maltese'),
'my' => array('Burmese'),
'na' => array('Nauru'),
'nb' => array('Norwegian Bokmål'),
'nd' => array('North Ndebele'),
'ne' => array('Nepali'),
'ng' => array('Ndonga'),
'nl' => array('Dutch', 'Flemish'),
'nn' => array('Norwegian Nynorsk'),
'no' => array('Norwegian'),
'nr' => array('South Ndebele'),
'nv' => array('Navajo', 'Navaho'),
'ny' => array('Chichewa', 'Chewa', 'Nyanja'),
'oc' => array('Occitan', 'Provençal'), // Occitan (post 1500)
'oj' => array('Ojibwa'),
'om' => array('Oromo'),
'or' => array('Oriya'),
'os' => array('Ossetian', 'Ossetic'),
'pa' => array('Panjabi', 'Punjabi'),
'pi' => array('Pali'),
'pl' => array('Polish'),
'ps' => array('Pushto', 'Pashto'),
'pt' => array('Portuguese'),
'qu' => array('Quechua'),
'rm' => array('Romansh'),
'rn' => array('Rundi'),
'ro' => array('Romanian', 'Moldavian', 'Moldovan'),
'ru' => array('Russian'),
'rw' => array('Kinyarwanda'),
'sa' => array('Sanskrit'),
'sc' => array('Sardinian'),
'sd' => array('Sindhi'),
'se' => array('Northern Sami'),
'sg' => array('Sango'),
// 'sh' => array('Serbo-Croatian'), // deprecated
'si' => array('Sinhala', 'Sinhalese'),
'sk' => array('Slovak'),
'sl' => array('Slovenian'),
'sm' => array('Samoan'),
'sn' => array('Shona'),
'so' => array('Somali'),
'sq' => array('Albanian'),
'sr' => array('Serbian'),
'ss' => array('Swati'),
'st' => array('Southern Soth'),
'su' => array('Sundanese'),
'sv' => array('Swedish'),
'sw' => array('Swahili'),
'ta' => array('Tamil'),
'te' => array('Telugu'),
'tg' => array('Tajik'),
'th' => array('Thai'),
'ti' => array('Tigrinya'),
'tk' => array('Turkmen'),
'tl' => array('Tagalog'),
'tn' => array('Tswana'),
'to' => array('Tonga'), // Tonga (Tonga Islands)
'tr' => array('Turkish'),
'ts' => array('Tsonga'),
'tt' => array('Tatar'),
'tw' => array('Twi'),
'ty' => array('Tahitian'),
'ug' => array('Uighur', 'Uyghur'),
'uk' => array('Ukrainian'),
'ur' => array('Urdu'),
'uz' => array('Uzbek'),
've' => array('Venda'),
'vi' => array('Vietnamese'),
'vo' => array('Volapük'),
'wa' => array('Walloon'),
'wo' => array('Wolof'),
'xh' => array('Xhosa'),
'yi' => array('Yiddish'),
'yo' => array('Yoruba'),
'za' => array('Zhuang', 'Chuang'),
'zh' => array('Chinese'),
'zu' => array('Zulu'),
);
}

2
www/analytics/core/DataFiles/Providers.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later

1033
www/analytics/core/DataFiles/SearchEngines.php
View file

File diff suppressed because it is too large Load diff

226
www/analytics/core/DataFiles/Socials.php
View file

@ -1,226 +0,0 @@
<?php
/**
* Piwik - Open source web analytics
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
if (!isset($GLOBALS['Piwik_socialUrl'])) {
// Note: the key of the array should have max 3 elements eg. sub.domain.ext
$GLOBALS['Piwik_socialUrl'] = array(
// Facebook
'facebook.com' => 'Facebook',
'fb.me' => 'Facebook',
// Ozone
'qzone.qq.com' => 'Qzone',
// Haboo
'habbo.com' => 'Haboo',
// Twitter
'twitter.com' => 'Twitter',
't.co' => 'Twitter',
// Renren
'renren.com' => 'Renren',
// Windows Live Spaces
'login.live.com' => 'Windows Live Spaces',
// LinkedIn
'linkedin.com' => 'LinkedIn',
// Bebo
'bebo.com' => 'Bebo',
// Vkontakte
'vk.com' => 'Vkontakte',
'vkontakte.ru' => 'Vkontakte',
// Tagged
'login.tagged.com' => 'Tagged',
// Orkut
'orkut.com' => 'Orkut',
// Myspace
'myspace.com' => 'Myspace',
// Frinedster
'friendster.com' => 'Friendster',
// Badoo
'badoo.com' => 'Badoo',
// hi5
'hi5.com' => 'hi5',
// Netlog
'netlog.com' => 'Netlog',
// Flixster
'flixster.com' => 'Flixster',
// MyLife
'mylife.ru' => 'MyLife',
// Classmates.com
'classmates.com' => 'Classmates.com',
// Github
'github.com' => 'Github',
// Google+
'url.google.com' => 'Google%2B',
// douban
'douban.com' => 'douban',
// Odnoklassniki
'odnoklassniki.ru' => 'Odnoklassniki',
// Viadeo
'viadeo.com' => 'Viadeo',
// Flickr
'flickr.com' => 'Flickr',
// WeeWorld
'weeworld.com' => 'WeeWorld',
// Last.fm
'last.fm' => 'Last.fm',
'lastfm.ru' => 'Last.fm',
'lastfm.de' => 'Last.fm',
'lastfm.es' => 'Last.fm',
'lastfm.fr' => 'Last.fm',
'lastfm.it' => 'Last.fm',
'lastfm.jp' => 'Last.fm',
'lastfm.pl' => 'Last.fm',
'lastfm.com.br' => 'Last.fm',
'lastfm.se' => 'Last.fm',
'lastfm.com.tr' => 'Last.fm',
// MyHeritage
'myheritage.com' => 'MyHeritage',
// Xanga
'xanga.com' => 'Xanga',
// Mixi
'mixi.jp' => 'Mixi',
// Cyworld
'global.cyworld.com' => 'Cyworld',
// Gaia Online
'gaiaonline.com' => 'Gaia Online',
// Skyrock
'skyrock.com' => 'Skyrock',
// BlackPlanet
'blackplanet.com' => 'BlackPlanet',
// myYearbook
'myyearbook.com' => 'myYearbook',
// Fotolog
'fotolog.com' => 'Fotolog',
// Friends Reunited
'friendsreunited.com' => 'Friends Reunited',
// LiveJournal
'livejournal.ru' => 'LiveJournal',
'livejournal.com' => 'LiveJournal',
// StudiVZ/MeinVZ
'studivz.net' => 'StudiVZ',
'meinvz.net' => 'MeinVZ',
// StackOverflow
'stackoverflow.com' => 'StackOverflow',
// Sonico.com
'sonico.com' => 'Sonico.com',
// Pinterest
'pinterest.com' => 'Pinterest',
// Plaxo
'plaxo.com' => 'Plaxo',
// Geni.com
'geni.com' => 'Geni.com',
// Tuenti
'tuenti.com' => 'Tuenti',
// XING
'xing.com' => 'XING',
// Taringa!
'taringa.net' => 'Taringa!',
// Nasza-klasa.pl
'nk.pl' => 'Nasza-klasa.pl',
// StumbleUpon
'stumbleupon.com' => 'StumbleUpon',
// Sourceforge
'sourceforge.net' => 'SourceForge',
// Hyves
'hyves.nl' => 'Hyves',
// WAYN
'wayn.com' => 'WAYN',
// Buzznet
'buzznet.com' => 'Buzznet',
// Multiply
'multiply.com' => 'Multiply',
// Foursquare
'foursquare.com' => 'Foursquare',
// vkrugudruzei.ru
'vkrugudruzei.ru' => 'vkrugudruzei.ru',
// my.mail.ru
'my.mail.ru' => 'my.mail.ru',
//MoiKrug.ru
'moikrug.ru' => 'moikrug.ru',
// Reddit
'reddit.com' => 'reddit',
// HackerNews
'news.ycombinator.com' => 'Hacker News',
// Identi.ca
'identi.ca' => 'identi.ca',
// Weibo
'weibo.com' => 'Weibo',
't.cn' => 'Weibo',
// YouTube
'youtube.com' => 'YouTube',
'youtu.be' => 'YouTube',
// Vimeo
'vimeo.com' => 'Vimeo',
//tumblr
'tumblr.com' => 'tumblr',
);
}

3981
www/analytics/core/DataFiles/cacert.pem Normal file
View file

File diff suppressed because it is too large Load diff

629
www/analytics/core/DataTable.php
View file

File diff suppressed because it is too large Load diff

17
www/analytics/core/DataTable/BaseFilter.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,12 +9,11 @@
namespace Piwik\DataTable;
use Piwik\DataTable;
use Piwik\DataTable\Row;
/**
* A filter is set of logic that manipulates a DataTable. Existing filters do things like,
*
* - remove rows
*
* - add/remove rows
* - change column values (change string to lowercase, truncate, etc.)
* - add/remove columns or metadata (compute percentage values, add an 'icon' metadata based on the label, etc.)
* - add/remove/edit subtable associated with rows
@ -22,10 +21,10 @@ use Piwik\DataTable\Row;
*
* Filters are called with a DataTable instance and extra parameters that are specified
* in {@link Piwik\DataTable::filter()} and {@link Piwik\DataTable::queueFilter()}.
*
*
* To see examples of Filters look at the existing ones in the Piwik\DataTable\BaseFilter
* namespace.
*
*
* @api
*/
abstract class BaseFilter
@ -37,7 +36,7 @@ abstract class BaseFilter
/**
* Constructor.
*
*
* @param DataTable $table
*/
public function __construct(DataTable $table)
@ -73,8 +72,8 @@ abstract class BaseFilter
if (!$this->enableRecursive) {
return;
}
if ($row->isSubtableLoaded()) {
$subTable = Manager::getInstance()->getTable($row->getIdSubDataTable());
$subTable = $row->getSubtable();
if ($subTable) {
$this->filter($subTable);
}
}

12
www/analytics/core/DataTable/Bridges.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -13,8 +13,8 @@
*/
namespace {
use Piwik\DataTable\Row\DataTableSummaryRow;
use Piwik\DataTable\Row;
use Piwik\DataTable\Row\DataTableSummaryRow;
class Piwik_DataTable_Row_DataTableSummary extends DataTableSummaryRow
{
@ -24,5 +24,11 @@ namespace {
{
}
}
// only used for BC to unserialize old archived Row instances. We cannot unserialize Row directly as it implements
// the Serializable interface and it would fail on PHP5.6+ when userializing the Row instance directly.
class Piwik_DataTable_SerializedRow
{
public $c;
}
}

Some files were not shown because too many files have changed in this diff Show more