update Piwik to version 2.16 (fixes #91)

This commit is contained in:
oliver 2016-04-10 18:55:57 +02:00
commit d885a4baa9
5833 changed files with 418860 additions and 226988 deletions

View file

@ -1,18 +1,20 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\ArchiveProcessor;
use Piwik\Archive;
use Piwik\ArchiveProcessor;
use Piwik\Cache;
use Piwik\Config;
use Piwik\DataAccess\ArchiveSelector;
use Piwik\Date;
use Piwik\Period;
use Piwik\Piwik;
/**
* This class uses PluginsArchiver class to trigger data aggregation and create archives.
@ -81,6 +83,7 @@ class Loader
* Prepares the core metrics if needed.
*
* @param $visits
* @return array
*/
protected function prepareCoreMetricsArchive($visits, $visitsConverted)
{
@ -101,12 +104,14 @@ class Loader
$visits = $metrics['nb_visits'];
$visitsConverted = $metrics['nb_visits_converted'];
}
return array($visits, $visitsConverted);
}
protected function prepareAllPluginsArchive($visits, $visitsConverted)
{
$pluginsArchiver = new PluginsArchiver($this->params, $this->isArchiveTemporary());
if ($this->mustProcessVisitCount($visits)
|| $this->doesRequestedPluginIncludeVisitsSummary()
) {
@ -114,14 +119,14 @@ class Loader
$visits = $metrics['nb_visits'];
$visitsConverted = $metrics['nb_visits_converted'];
}
if ($this->isThereSomeVisits($visits)) {
if ($this->isThereSomeVisits($visits)
|| $this->shouldArchiveForSiteEvenWhenNoVisits()
) {
$pluginsArchiver->callAggregateAllPlugins($visits, $visitsConverted);
}
$idArchive = $pluginsArchiver->finalizeArchive();
if (!$this->params->isSingleSiteDayArchive() && $visits) {
ArchiveSelector::purgeOutdatedArchives($this->params->getPeriod()->getDateStart());
}
$idArchive = $pluginsArchiver->finalizeArchive();
return array($idArchive, $visits);
}
@ -139,11 +144,13 @@ class Loader
{
$period = $this->params->getPeriod()->getLabel();
$debugSetting = 'always_archive_data_period'; // default
if ($period == 'day') {
$debugSetting = 'always_archive_data_day';
} elseif ($period == 'range') {
$debugSetting = 'always_archive_data_range';
}
return (bool) Config::getInstance()->Debug[$debugSetting];
}
@ -165,9 +172,11 @@ class Loader
}
$idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
if (!$idAndVisits) {
return $noArchiveFound;
}
return $idAndVisits;
}
@ -186,19 +195,27 @@ class Loader
// Permanent archive
return $endDateTimestamp;
}
$dateStart = $this->params->getDateStart();
$period = $this->params->getPeriod();
$segment = $this->params->getSegment();
$site = $this->params->getSite();
// Temporary archive
return Rules::getMinTimeProcessedForTemporaryArchive($this->params->getDateStart(), $this->params->getPeriod(), $this->params->getSegment(), $this->params->getSite());
return Rules::getMinTimeProcessedForTemporaryArchive($dateStart, $period, $segment, $site);
}
protected static function determineIfArchivePermanent(Date $dateEnd)
{
$now = time();
$endTimestampUTC = strtotime($dateEnd->getDateEndUTC());
if ($endTimestampUTC <= $now) {
// - if the period we are looking for is finished, we look for a ts_archived that
// is greater than the last day of the archive
return $endTimestampUTC;
}
return false;
}
@ -207,8 +224,30 @@ class Loader
if (is_null($this->temporaryArchive)) {
throw new \Exception("getMinTimeArchiveProcessed() should be called prior to isArchiveTemporary()");
}
return $this->temporaryArchive;
}
}
private function shouldArchiveForSiteEvenWhenNoVisits()
{
$idSitesToArchive = $this->getIdSitesToArchiveWhenNoVisits();
return in_array($this->params->getSite()->getId(), $idSitesToArchive);
}
private function getIdSitesToArchiveWhenNoVisits()
{
$cache = Cache::getTransientCache();
$cacheKey = 'Archiving.getIdSitesToArchiveWhenNoVisits';
if (!$cache->contains($cacheKey)) {
$idSites = array();
// leaving undocumented unless decided otherwise
Piwik::postEvent('Archiving.getIdSitesToArchiveWhenNoVisits', array(&$idSites));
$cache->save($cacheKey, $idSites);
}
return $cache->fetch($cacheKey);
}
}

View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -45,15 +45,14 @@ class Parameters
/**
* Constructor.
*
*
* @ignore
*/
public function __construct(Site $site, Period $period, Segment $segment, $skipAggregationOfSubTables = false)
public function __construct(Site $site, Period $period, Segment $segment)
{
$this->site = $site;
$this->period = $period;
$this->segment = $segment;
$this->skipAggregationOfSubTables = $skipAggregationOfSubTables;
}
/**
@ -91,7 +90,7 @@ class Parameters
*/
public function getSubPeriods()
{
if($this->getPeriod()->getLabel() == 'day') {
if ($this->getPeriod()->getLabel() == 'day') {
return array( $this->getPeriod() );
}
return $this->getPeriod()->getSubperiods();
@ -169,18 +168,13 @@ class Parameters
return count($this->getIdSites()) == 1;
}
public function isSkipAggregationOfSubTables()
{
return $this->skipAggregationOfSubTables;
}
public function logStatusDebug($isTemporary)
{
$temporary = 'definitive archive';
if ($isTemporary) {
$temporary = 'temporary archive';
}
Log::verbose(
Log::debug(
"%s archive, idSite = %d (%s), segment '%s', report = '%s', UTC datetime [%s -> %s]",
$this->getPeriod()->getLabel(),
$this->getSite()->getId(),

View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,13 +9,15 @@
namespace Piwik\ArchiveProcessor;
use Piwik\Archive;
use Piwik\ArchiveProcessor;
use Piwik\DataAccess\ArchiveSelector;
use Piwik\DataAccess\ArchiveWriter;
use Piwik\DataAccess\LogAggregator;
use Piwik\DataTable\Manager;
use Piwik\Metrics;
use Piwik\Plugin\Archiver;
use Piwik\Log;
use Piwik\Timer;
use Exception;
/**
* This class creates the Archiver objects found in plugins and will trigger aggregation,
@ -34,9 +36,16 @@ class PluginsArchiver
protected $params;
/**
* @var LogAggregator
*/
private $logAggregator;
/**
* Public only for tests. Won't be necessary after DI changes are complete.
*
* @var Archiver[] $archivers
*/
private static $archivers = array();
public static $archivers = array();
public function __construct(Parameters $params, $isTemporaryArchive)
{
@ -45,7 +54,9 @@ class PluginsArchiver
$this->archiveWriter = new ArchiveWriter($this->params, $isTemporaryArchive);
$this->archiveWriter->initNewArchive();
$this->archiveProcessor = new ArchiveProcessor($this->params, $this->archiveWriter);
$this->logAggregator = new LogAggregator($params);
$this->archiveProcessor = new ArchiveProcessor($this->params, $this->archiveWriter, $this->logAggregator);
$this->isSingleSiteDayArchive = $this->params->isSingleSiteDayArchive();
}
@ -57,7 +68,9 @@ class PluginsArchiver
*/
public function callAggregateCoreMetrics()
{
if($this->isSingleSiteDayArchive) {
$this->logAggregator->setQueryOriginHint('Core');
if ($this->isSingleSiteDayArchive) {
$metrics = $this->aggregateDayVisitsMetrics();
} else {
$metrics = $this->aggregateMultipleVisitsMetrics();
@ -81,27 +94,57 @@ class PluginsArchiver
*/
public function callAggregateAllPlugins($visits, $visitsConverted)
{
Log::debug("PluginsArchiver::%s: Initializing archiving process for all plugins [visits = %s, visits converted = %s]",
__FUNCTION__, $visits, $visitsConverted);
$this->archiveProcessor->setNumberOfVisits($visits, $visitsConverted);
$archivers = $this->getPluginArchivers();
foreach($archivers as $pluginName => $archiverClass) {
foreach ($archivers as $pluginName => $archiverClass) {
// We clean up below all tables created during this function call (and recursive calls)
$latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
/** @var Archiver $archiver */
$archiver = new $archiverClass($this->archiveProcessor);
if(!$archiver->isEnabled()) {
if (!$archiver->isEnabled()) {
Log::debug("PluginsArchiver::%s: Skipping archiving for plugin '%s'.", __FUNCTION__, $pluginName);
continue;
}
if($this->shouldProcessReportsForPlugin($pluginName)) {
if($this->isSingleSiteDayArchive) {
$archiver->aggregateDayReport();
} else {
$archiver->aggregateMultipleReports();
if ($this->shouldProcessReportsForPlugin($pluginName)) {
$this->logAggregator->setQueryOriginHint($pluginName);
try {
$timer = new Timer();
if ($this->isSingleSiteDayArchive) {
Log::debug("PluginsArchiver::%s: Archiving day reports for plugin '%s'.", __FUNCTION__, $pluginName);
$archiver->aggregateDayReport();
} else {
Log::debug("PluginsArchiver::%s: Archiving period reports for plugin '%s'.", __FUNCTION__, $pluginName);
$archiver->aggregateMultipleReports();
}
$this->logAggregator->setQueryOriginHint('');
Log::debug("PluginsArchiver::%s: %s while archiving %s reports for plugin '%s'.",
__FUNCTION__,
$timer->getMemoryLeak(),
$this->params->getPeriod()->getLabel(),
$pluginName
);
} catch (Exception $e) {
$className = get_class($e);
$exception = new $className($e->getMessage() . " - caused by plugin $pluginName", $e->getCode(), $e);
throw $exception;
}
} else {
Log::debug("PluginsArchiver::%s: Not archiving reports for plugin '%s'.", __FUNCTION__, $pluginName);
}
Manager::getInstance()->deleteAll($latestUsedTableId);
@ -111,7 +154,7 @@ class PluginsArchiver
public function finalizeArchive()
{
$this->params->logStatusDebug( $this->archiveWriter->isArchiveTemporary );
$this->params->logStatusDebug($this->archiveWriter->isArchiveTemporary);
$this->archiveWriter->finalizeArchive();
return $this->archiveWriter->getIdArchive();
}
@ -193,5 +236,4 @@ class PluginsArchiver
$metrics = $this->archiveProcessor->aggregateNumericMetrics($toSum);
return $metrics;
}
}

View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,14 +9,13 @@
namespace Piwik\ArchiveProcessor;
use Exception;
use Piwik\Common;
use Piwik\Config;
use Piwik\DataAccess\ArchiveWriter;
use Piwik\Date;
use Piwik\Log;
use Piwik\Option;
use Piwik\Piwik;
use Piwik\Plugins\CoreAdminHome\Controller;
use Piwik\Plugins\CoreAdminHome\CoreAdminHome;
use Piwik\Segment;
use Piwik\SettingsPiwik;
use Piwik\SettingsServer;
@ -25,7 +24,7 @@ use Piwik\Tracker\Cache;
/**
* This class contains Archiving rules/logic which are used when creating and processing Archives.
*
*
*/
class Rules
{
@ -35,9 +34,6 @@ class Rules
const FLAG_TABLE_PURGED = 'lastPurge_';
/** Old Archives purge can be disabled (used in tests only) */
static public $purgeDisabledByTests = false;
/** Flag that will forcefully disable the archiving process (used in tests only) */
public static $archivingDisabledByTests = false;
@ -45,15 +41,16 @@ class Rules
* Returns the name of the archive field used to tell the status of an archive, (ie,
* whether the archive was created successfully or not).
*
* @param array $idSites
* @param Segment $segment
* @param string $periodLabel
* @param string $plugin
* @return string
*/
public static function getDoneStringFlagFor(array $idSites, $segment, $periodLabel, $plugin, $isSkipAggregationOfSubTables)
public static function getDoneStringFlagFor(array $idSites, $segment, $periodLabel, $plugin)
{
if (!self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel)) {
return self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin, $isSkipAggregationOfSubTables);
return self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
}
return self::getDoneFlagArchiveContainsAllPlugins($segment);
}
@ -84,39 +81,24 @@ class Rules
return $segmentsToProcess;
}
public static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin, $isSkipAggregationOfSubTables = false)
public static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin)
{
$partial = self::isFlagArchivePartial($plugin, $isSkipAggregationOfSubTables);
return 'done' . $segment->getHash() . '.' . $plugin . $partial ;
return 'done' . $segment->getHash() . '.' . $plugin ;
}
private static function getDoneFlagArchiveContainsAllPlugins(Segment $segment)
public static function getDoneFlagArchiveContainsAllPlugins(Segment $segment)
{
return 'done' . $segment->getHash();
}
/**
* @param $plugin
* @param $isSkipAggregationOfSubTables
* @return string
*/
private static function isFlagArchivePartial($plugin, $isSkipAggregationOfSubTables)
{
$partialArchive = '';
if ($plugin != "VisitsSummary" // VisitsSummary is always called when segmenting and should not have its own .partial archive
&& $isSkipAggregationOfSubTables
) {
$partialArchive = '.partial';
}
return $partialArchive;
}
/**
* Return done flags used to tell how the archiving process for a specific archive was completed,
*
* @param array $plugins
* @param $segment
* @return array
*/
public static function getDoneFlags(array $plugins, Segment $segment, $isSkipAggregationOfSubTables)
public static function getDoneFlags(array $plugins, Segment $segment)
{
$doneFlags = array();
$doneAllPlugins = self::getDoneFlagArchiveContainsAllPlugins($segment);
@ -124,58 +106,12 @@ class Rules
$plugins = array_unique($plugins);
foreach ($plugins as $plugin) {
$doneOnePlugin = self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin, $isSkipAggregationOfSubTables);
$doneOnePlugin = self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
$doneFlags[$plugin] = $doneOnePlugin;
}
return $doneFlags;
}
/**
* Given a monthly archive table, will delete all reports that are now outdated,
* or reports that ended with an error
*
* @param \Piwik\Date $date
* @return int|bool False, or timestamp indicating which archives to delete
*/
public static function shouldPurgeOutdatedArchives(Date $date)
{
if (self::$purgeDisabledByTests) {
return false;
}
$key = self::FLAG_TABLE_PURGED . "blob_" . $date->toString('Y_m');
$timestamp = Option::get($key);
// we shall purge temporary archives after their timeout is finished, plus an extra 6 hours
// in case archiving is disabled or run once a day, we give it this extra time to run
// and re-process more recent records...
$temporaryArchivingTimeout = self::getTodayArchiveTimeToLive();
$hoursBetweenPurge = 6;
$purgeEveryNSeconds = max($temporaryArchivingTimeout, $hoursBetweenPurge * 3600);
// we only delete archives if we are able to process them, otherwise, the browser might process reports
// when &segment= is specified (or custom date range) and would below, delete temporary archives that the
// browser is not able to process until next cron run (which could be more than 1 hour away)
if (self::isRequestAuthorizedToArchive()
&& (!$timestamp
|| $timestamp < time() - $purgeEveryNSeconds)
) {
Option::set($key, time());
if (self::isBrowserTriggerEnabled()) {
// If Browser Archiving is enabled, it is likely there are many more temporary archives
// We delete more often which is safe, since reports are re-processed on demand
$purgeArchivesOlderThan = Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime();
} else {
// If archive.php via Cron is building the reports, we should keep all temporary reports from today
$purgeArchivesOlderThan = Date::factory('today')->getDateTime();
}
return $purgeArchivesOlderThan;
}
Log::info("Purging temporary archives: skipped.");
return false;
}
public static function getMinTimeProcessedForTemporaryArchive(
Date $dateStart, \Piwik\Period $period, Segment $segment, Site $site)
{
@ -213,30 +149,45 @@ class Rules
{
$uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
if($uiSettingIsEnabled) {
if ($uiSettingIsEnabled) {
$timeToLive = Option::get(self::OPTION_TODAY_ARCHIVE_TTL);
if ($timeToLive !== false) {
return $timeToLive;
}
}
return self::getTodayArchiveTimeToLiveDefault();
}
public static function getTodayArchiveTimeToLiveDefault()
{
return Config::getInstance()->General['time_before_today_archive_considered_outdated'];
}
public static function isArchivingDisabledFor(array $idSites, Segment $segment, $periodLabel)
{
$generalConfig = Config::getInstance()->General;
if ($periodLabel == 'range') {
return false;
if (!isset($generalConfig['archiving_range_force_on_browser_request'])
|| $generalConfig['archiving_range_force_on_browser_request'] != false
) {
return false;
} else {
Log::debug("Not forcing archiving for range period.");
}
}
$processOneReportOnly = !self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel);
$isArchivingDisabled = !self::isRequestAuthorizedToArchive() || self::$archivingDisabledByTests;
if ($processOneReportOnly) {
if ($processOneReportOnly
&& $periodLabel != 'range'
) {
// When there is a segment, we disable archiving when browser_archiving_disabled_enforce applies
if (!$segment->isEmpty()
&& $isArchivingDisabled
&& Config::getInstance()->General['browser_archiving_disabled_enforce']
&& !SettingsServer::isArchivePhpTriggered() // Only applies when we are not running archive.php
&& $generalConfig['browser_archiving_disabled_enforce']
&& !SettingsServer::isArchivePhpTriggered() // Only applies when we are not running core:archive command
) {
Log::debug("Archiving is disabled because of config setting browser_archiving_disabled_enforce=1");
return true;
@ -248,7 +199,7 @@ class Rules
return $isArchivingDisabled;
}
protected static function isRequestAuthorizedToArchive()
public static function isRequestAuthorizedToArchive()
{
return Rules::isBrowserTriggerEnabled() || SettingsServer::isArchivePhpTriggered();
}
@ -257,7 +208,7 @@ class Rules
{
$uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
if($uiSettingIsEnabled) {
if ($uiSettingIsEnabled) {
$browserArchivingEnabled = Option::get(self::OPTION_BROWSER_TRIGGER_ARCHIVING);
if ($browserArchivingEnabled !== false) {
return (bool)$browserArchivingEnabled;
@ -275,6 +226,18 @@ class Rules
Cache::clearCacheGeneral();
}
/**
* Returns true if the archiving process should skip the calculation of unique visitors
* across several sites. The `[General] enable_processing_unique_visitors_multiple_sites`
* INI config option controls the value of this variable.
*
* @return bool
*/
public static function shouldSkipUniqueVisitorsCalculationForMultipleSites()
{
return Config::getInstance()->General['enable_processing_unique_visitors_multiple_sites'] != 1;
}
/**
* @param array $idSites
* @param Segment $segment
@ -294,11 +257,24 @@ class Rules
// Turns out the getString() above returns the URL decoded segment string
$segmentsToProcessUrlDecoded = array_map('urldecode', $segmentsToProcess);
if (in_array($segment, $segmentsToProcess)
|| in_array($segment, $segmentsToProcessUrlDecoded)
) {
return true;
return in_array($segment, $segmentsToProcess)
|| in_array($segment, $segmentsToProcessUrlDecoded);
}
/**
* Returns done flag values allowed to be selected
*
* @return string
*/
public static function getSelectableDoneFlagValues()
{
$possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
if (!Rules::isRequestAuthorizedToArchive()) {
//If request is not authorized to archive then fetch also invalidated archives
$possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
}
return false;
return $possibleValues;
}
}