coderkun/questlab
1
0
Fork 0
Code Issues 15 Pull requests Projects Releases Wiki Activity

update Piwik to version 2.16 (fixes #91)

Browse source
This commit is contained in:
oliver 2016-04-10 18:55:57 +02:00
commit d885a4baa9
5833 changed files with 418860 additions and 226988 deletions
Download patch file Download diff file Expand all files Collapse all files

260
www/analytics/core/ArchiveProcessor.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,82 +9,80 @@
namespace Piwik;
use Exception;
use Piwik\Archive\DataTableFactory;
use Piwik\ArchiveProcessor\Parameters;
use Piwik\ArchiveProcessor\Rules;
use Piwik\DataAccess\ArchiveWriter;
use Piwik\DataAccess\LogAggregator;
use Piwik\DataTable\Manager;
use Piwik\DataTable\Map;
use Piwik\DataTable\Row;
use Piwik\Db;
use Piwik\Period;
/**
* Used by {@link Piwik\Plugin\Archiver} instances to insert and aggregate archive data.
*
*
* ### See also
*
*
* - **{@link Piwik\Plugin\Archiver}** - to learn how plugins should implement their own analytics
* aggregation logic.
* - **{@link Piwik\DataAccess\LogAggregator}** - to learn how plugins can perform data aggregation
* across Piwik's log tables.
*
*
* ### Examples
*
*
* **Inserting numeric data**
*
*
* // function in an Archiver descendant
* public function aggregateDayReport()
* {
* $archiveProcessor = $this->getProcessor();
*
*
* $myFancyMetric = // ... calculate the metric value ...
* $archiveProcessor->insertNumericRecord('MyPlugin_myFancyMetric', $myFancyMetric);
* }
*
*
* **Inserting serialized DataTables**
*
*
* // function in an Archiver descendant
* public function aggregateDayReport()
* {
* $archiveProcessor = $this->getProcessor();
*
*
* $maxRowsInTable = Config::getInstance()->General['datatable_archiving_maximum_rows_standard'];j
*
*
* $dataTable = // ... build by aggregating visits ...
* $serializedData = $dataTable->getSerialized($maxRowsInTable, $maxRowsInSubtable = $maxRowsInTable,
* $columnToSortBy = Metrics::INDEX_NB_VISITS);
*
*
* $archiveProcessor->insertBlobRecords('MyPlugin_myFancyReport', $serializedData);
* }
*
*
* **Aggregating archive data**
*
*
* // function in Archiver descendant
* public function aggregateMultipleReports()
* {
* $archiveProcessor = $this->getProcessor();
*
*
* // aggregate a metric
* $archiveProcessor->aggregateNumericMetrics('MyPlugin_myFancyMetric');
* $archiveProcessor->aggregateNumericMetrics('MyPlugin_mySuperFancyMetric', 'max');
*
* // aggregate a report
*
* // aggregate a report
* $archiveProcessor->aggregateDataTableRecords('MyPlugin_myFancyReport');
* }
*
*
*/
class ArchiveProcessor
{
/**
* @var \Piwik\DataAccess\ArchiveWriter
*/
protected $archiveWriter;
private $archiveWriter;
/**
* @var \Piwik\DataAccess\LogAggregator
*/
protected $logAggregator;
private $logAggregator;
/**
* @var Archive
@ -94,28 +92,41 @@ class ArchiveProcessor
/**
* @var Parameters
*/
protected $params;
private $params;
/**
* @var int
*/
protected $numberOfVisits = false;
protected $numberOfVisitsConverted = false;
private $numberOfVisits = false;
public function __construct(Parameters $params, ArchiveWriter $archiveWriter)
private $numberOfVisitsConverted = false;
/**
* If true, unique visitors are not calculated when we are aggregating data for multiple sites.
* The `[General] enable_processing_unique_visitors_multiple_sites` INI config option controls
* the value of this variable.
*
* @var bool
*/
private $skipUniqueVisitorsCalculationForMultipleSites = true;
public function __construct(Parameters $params, ArchiveWriter $archiveWriter, LogAggregator $logAggregator)
{
$this->params = $params;
$this->logAggregator = new LogAggregator($params);
$this->logAggregator = $logAggregator;
$this->archiveWriter = $archiveWriter;
$this->skipUniqueVisitorsCalculationForMultipleSites = Rules::shouldSkipUniqueVisitorsCalculationForMultipleSites();
}
protected function getArchive()
{
if(empty($this->archive)) {
if (empty($this->archive)) {
$subPeriods = $this->params->getSubPeriods();
$idSites = $this->params->getIdSites();
$idSites = $this->params->getIdSites();
$this->archive = Archive::factory($this->params->getSegment(), $subPeriods, $idSites);
}
return $this->archive;
}
@ -155,7 +166,8 @@ class ArchiveProcessor
* @var array
*/
protected static $columnsToRenameAfterAggregation = array(
Metrics::INDEX_NB_UNIQ_VISITORS => Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS
Metrics::INDEX_NB_UNIQ_VISITORS => Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS,
Metrics::INDEX_NB_USERS => Metrics::INDEX_SUM_DAILY_NB_USERS,
);
/**
@ -172,8 +184,11 @@ class ArchiveProcessor
* @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names
* when summed because they cannot be summed, eg,
* `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`.
* @param bool|array $countRowsRecursive if set to true, will calculate the recursive rows count for all record names
* which makes it slower. If you only need it for some records pass an array of
* recordNames that defines for which ones you need a recursive row count.
* @return array Returns the row counts of each aggregated report before truncation, eg,
*
*
* array(
* 'report1' => array('level0' => $report1->getRowsCount,
* 'recursive' => $report1->getRowsCountRecursive()),
@ -188,25 +203,23 @@ class ArchiveProcessor
$maximumRowsInSubDataTable = null,
$columnToSortByBeforeTruncation = null,
&$columnsAggregationOperation = null,
$columnsToRenameAfterAggregation = null)
$columnsToRenameAfterAggregation = null,
$countRowsRecursive = true)
{
if (!is_array($recordNames)) {
$recordNames = array($recordNames);
}
$nameToCount = array();
foreach ($recordNames as $recordName) {
$latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
$table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation);
$rowsCount = $table->getRowsCount();
$nameToCount[$recordName]['level0'] = $rowsCount;
$rowsCountRecursive = $rowsCount;
if($this->isAggregateSubTables()) {
$rowsCountRecursive = $table->getRowsCountRecursive();
$nameToCount[$recordName]['level0'] = $table->getRowsCount();
if ($countRowsRecursive === true || (is_array($countRowsRecursive) && in_array($recordName, $countRowsRecursive))) {
$nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
}
$nameToCount[$recordName]['recursive'] = $rowsCountRecursive;
$blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
Common::destroy($table);
@ -228,12 +241,12 @@ class ArchiveProcessor
* @return array|int Returns the array of aggregate values. If only one metric was aggregated,
* the aggregate value will be returned as is, not in an array.
* For example, if `array('nb_visits', 'nb_hits')` is supplied for `$columns`,
*
*
* array(
* 'nb_visits' => 3040,
* 'nb_hits' => 405
* )
*
*
* could be returned. If `array('nb_visits')` or `'nb_visits'` is used for `$columns`,
* then `3040` would be returned.
* @api
@ -242,7 +255,8 @@ class ArchiveProcessor
{
$metrics = $this->getAggregatedNumericMetrics($columns, $operationToApply);
foreach($metrics as $column => $value) {
foreach ($metrics as $column => $value) {
$value = Common::forceDotAsSeparatorForDecimalPoint($value);
$this->archiveWriter->insertRecord($column, $value);
}
// if asked for only one field to sum
@ -256,7 +270,7 @@ class ArchiveProcessor
public function getNumberOfVisits()
{
if($this->numberOfVisits === false) {
if ($this->numberOfVisits === false) {
throw new Exception("visits should have been set here");
}
return $this->numberOfVisits;
@ -273,7 +287,7 @@ class ArchiveProcessor
*
* @param array $numericRecords A name-value mapping of numeric values that should be
* archived, eg,
*
*
* array('Referrers_distinctKeywords' => 23, 'Referrers_distinctCampaigns' => 234)
* @api
*/
@ -297,6 +311,8 @@ class ArchiveProcessor
public function insertNumericRecord($name, $value)
{
$value = round($value, 2);
$value = Common::forceDotAsSeparatorForDecimalPoint($value);
$this->archiveWriter->insertRecord($name, $value);
}
@ -328,20 +344,53 @@ class ArchiveProcessor
*/
protected function aggregateDataTableRecord($name, $columnsAggregationOperation = null, $columnsToRenameAfterAggregation = null)
{
if($this->isAggregateSubTables()) {
// By default we shall aggregate all sub-tables.
$dataTable = $this->getArchive()->getDataTableExpanded($name, $idSubTable = null, $depth = null, $addMetadataSubtableId = false);
} else {
// In some cases (eg. Actions plugin when period=range),
// for better performance we will only aggregate the parent table
$dataTable = $this->getArchive()->getDataTable($name, $idSubTable = null);
// By default we shall aggregate all sub-tables.
$dataTable = $this->getArchive()->getDataTableExpanded($name, $idSubTable = null, $depth = null, $addMetadataSubtableId = false);
$columnsRenamed = false;
if ($dataTable instanceof Map) {
$columnsRenamed = true;
// see https://github.com/piwik/piwik/issues/4377
$self = $this;
$dataTable->filter(function ($table) use ($self, $columnsToRenameAfterAggregation) {
if ($self->areColumnsNotAlreadyRenamed($table)) {
/**
* This makes archiving and range dates a lot faster. Imagine we archive a week, then we will
* rename all columns of each 7 day archives. Afterwards we know the columns will be replaced in a
* week archive. When generating month archives, which uses mostly week archives, we do not have
* to replace those columns for the week archives again since we can be sure they were already
* replaced. Same when aggregating year and range archives. This can save up 10% or more when
* aggregating Month, Year and Range archives.
*/
$self->renameColumnsAfterAggregation($table, $columnsToRenameAfterAggregation);
}
});
}
$dataTable = $this->getAggregatedDataTableMap($dataTable, $columnsAggregationOperation);
$this->renameColumnsAfterAggregation($dataTable, $columnsToRenameAfterAggregation);
if (!$columnsRenamed) {
$this->renameColumnsAfterAggregation($dataTable, $columnsToRenameAfterAggregation);
}
return $dataTable;
}
/**
* Note: public only for use in closure in PHP 5.3.
*
* @param $table
* @return \Piwik\Period
*/
public function areColumnsNotAlreadyRenamed($table)
{
$period = $table->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX);
return !$period || $period->getLabel() === 'day';
}
protected function getOperationForColumns($columns, $defaultOperation)
{
$operationForColumn = array();
@ -357,18 +406,54 @@ class ArchiveProcessor
protected function enrichWithUniqueVisitorsMetric(Row $row)
{
if(!$this->getParams()->isSingleSite() ) {
// we only compute unique visitors for a single site
// skip unique visitors metrics calculation if calculating for multiple sites is disabled
if (!$this->getParams()->isSingleSite()
&& $this->skipUniqueVisitorsCalculationForMultipleSites
) {
return;
}
if ( $row->getColumn('nb_uniq_visitors') !== false) {
if (SettingsPiwik::isUniqueVisitorsEnabled($this->getParams()->getPeriod()->getLabel())) {
$uniqueVisitors = (float)$this->computeNbUniqVisitors();
$row->setColumn('nb_uniq_visitors', $uniqueVisitors);
} else {
$row->deleteColumn('nb_uniq_visitors');
}
if ($row->getColumn('nb_uniq_visitors') === false
&& $row->getColumn('nb_users') === false
) {
return;
}
if (!SettingsPiwik::isUniqueVisitorsEnabled($this->getParams()->getPeriod()->getLabel())) {
$row->deleteColumn('nb_uniq_visitors');
$row->deleteColumn('nb_users');
return;
}
$metrics = array(
Metrics::INDEX_NB_USERS
);
if ($this->getParams()->isSingleSite()) {
$uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_VISITORS;
} else {
if (!SettingsPiwik::isSameFingerprintAcrossWebsites()) {
throw new Exception("Processing unique visitors across websites is enabled for this instance,
but to process this metric you must first set enable_fingerprinting_across_websites=1
in the config file, under the [Tracker] section.");
}
$uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_FINGERPRINTS;
}
$metrics[] = $uniqueVisitorsMetric;
$uniques = $this->computeNbUniques($metrics);
// see edge case as described in https://github.com/piwik/piwik/issues/9357 where uniq_visitors might be higher
// than visits because we archive / process it after nb_visits. Between archiving nb_visits and nb_uniq_visitors
// there could have been a new visit leading to a higher nb_unique_visitors than nb_visits which is not possible
// by definition. In this case we simply use the visits metric instead of unique visitors metric.
$visits = $row->getColumn('nb_visits');
if ($visits !== false && $uniques[$uniqueVisitorsMetric] !== false) {
$uniques[$uniqueVisitorsMetric] = min($uniques[$uniqueVisitorsMetric], $visits);
}
$row->setColumn('nb_uniq_visitors', $uniques[$uniqueVisitorsMetric]);
$row->setColumn('nb_users', $uniques[Metrics::INDEX_NB_USERS]);
}
protected function guessOperationForColumn($column)
@ -388,14 +473,15 @@ class ArchiveProcessor
* This is the only Period metric (ie. week/month/year/range) that we process from the logs directly,
* since unique visitors cannot be summed like other metrics.
*
* @return int
* @param array Metrics Ids for which to aggregates count of values
* @return array of metrics, where the key is metricid and the value is the metric value
*/
protected function computeNbUniqVisitors()
protected function computeNbUniques($metrics)
{
$logAggregator = $this->getLogAggregator();
$query = $logAggregator->queryVisitsByDimension(array(), false, array(), array(Metrics::INDEX_NB_UNIQ_VISITORS));
$query = $logAggregator->queryVisitsByDimension(array(), false, array(), $metrics);
$data = $query->fetch();
return $data[Metrics::INDEX_NB_UNIQ_VISITORS];
return $data;
}
/**
@ -409,15 +495,18 @@ class ArchiveProcessor
protected function getAggregatedDataTableMap($data, $columnsAggregationOperation)
{
$table = new DataTable();
if (!empty($columnsAggregationOperation)) {
$table->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation);
}
if ($data instanceof DataTable\Map) {
// as $date => $tableToSum
$this->aggregatedDataTableMapsAsOne($data, $table);
} else {
$table->addDataTable($data, $this->isAggregateSubTables());
$table->addDataTable($data);
}
return $table;
}
@ -429,22 +518,33 @@ class ArchiveProcessor
protected function aggregatedDataTableMapsAsOne(Map $map, DataTable $aggregated)
{
foreach ($map->getDataTables() as $tableToAggregate) {
if($tableToAggregate instanceof Map) {
if ($tableToAggregate instanceof Map) {
$this->aggregatedDataTableMapsAsOne($tableToAggregate, $aggregated);
} else {
$aggregated->addDataTable($tableToAggregate, $this->isAggregateSubTables());
$aggregated->addDataTable($tableToAggregate);
}
}
}
protected function renameColumnsAfterAggregation(DataTable $table, $columnsToRenameAfterAggregation = null)
/**
* Note: public only for use in closure in PHP 5.3.
*/
public function renameColumnsAfterAggregation(DataTable $table, $columnsToRenameAfterAggregation = null)
{
// Rename columns after aggregation
if (is_null($columnsToRenameAfterAggregation)) {
$columnsToRenameAfterAggregation = self::$columnsToRenameAfterAggregation;
}
foreach ($columnsToRenameAfterAggregation as $oldName => $newName) {
$table->renameColumn($oldName, $newName, $this->isAggregateSubTables());
foreach ($table->getRows() as $row) {
foreach ($columnsToRenameAfterAggregation as $oldName => $newName) {
$row->renameColumn($oldName, $newName);
}
$subTable = $row->getSubtable();
if ($subTable) {
$this->renameColumnsAfterAggregation($subTable, $columnsToRenameAfterAggregation);
}
}
}
@ -453,6 +553,7 @@ class ArchiveProcessor
if (!is_array($columns)) {
$columns = array($columns);
}
$operationForColumn = $this->getOperationForColumns($columns, $operationToApply);
$dataTable = $this->getArchive()->getDataTableFromNumeric($columns);
@ -463,11 +564,11 @@ class ArchiveProcessor
}
$rowMetrics = $results->getFirstRow();
if($rowMetrics === false) {
if ($rowMetrics === false) {
$rowMetrics = new Row;
}
$this->enrichWithUniqueVisitorsMetric($rowMetrics);
$this->renameColumnsAfterAggregation($results);
$this->renameColumnsAfterAggregation($results, self::$columnsToRenameAfterAggregation);
$metrics = $rowMetrics->getColumns();
@ -476,14 +577,7 @@ class ArchiveProcessor
$metrics[$name] = 0;
}
}
return $metrics;
}
/**
* @return bool
*/
protected function isAggregateSubTables()
{
return !$this->getParams()->isSkipAggregationOfSubTables();
}
}