add piwik installation
This commit is contained in:
coderkun
parent
90aa4ef157
commit
8c5d4f0c31
3197 changed files with 563902 additions and 0 deletions
214
www/analytics/core/ArchiveProcessor/Loader.php
Normal file
214
www/analytics/core/ArchiveProcessor/Loader.php
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
<?php
|
||||
/**
|
||||
* Piwik - Open source web analytics
|
||||
*
|
||||
* @link http://piwik.org
|
||||
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
|
||||
*
|
||||
*/
|
||||
namespace Piwik\ArchiveProcessor;
|
||||
use Piwik\Archive;
|
||||
use Piwik\ArchiveProcessor;
|
||||
use Piwik\Config;
|
||||
use Piwik\DataAccess\ArchiveSelector;
|
||||
use Piwik\Date;
|
||||
use Piwik\Period;
|
||||
|
||||
/**
|
||||
* This class uses PluginsArchiver class to trigger data aggregation and create archives.
|
||||
*/
|
||||
class Loader
|
||||
{
|
||||
/**
|
||||
* Is the current archive temporary. ie.
|
||||
* - today
|
||||
* - current week / month / year
|
||||
*/
|
||||
protected $temporaryArchive;
|
||||
|
||||
/**
|
||||
* Idarchive in the DB for the requested archive
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
protected $idArchive;
|
||||
|
||||
/**
|
||||
* @var Parameters
|
||||
*/
|
||||
protected $params;
|
||||
|
||||
public function __construct(Parameters $params)
|
||||
{
|
||||
$this->params = $params;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
protected function isThereSomeVisits($visits)
|
||||
{
|
||||
return $visits > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
protected function mustProcessVisitCount($visits)
|
||||
{
|
||||
return $visits === false;
|
||||
}
|
||||
|
||||
public function prepareArchive($pluginName)
|
||||
{
|
||||
$this->params->setRequestedPlugin($pluginName);
|
||||
|
||||
list($idArchive, $visits, $visitsConverted) = $this->loadExistingArchiveIdFromDb();
|
||||
if (!empty($idArchive)) {
|
||||
return $idArchive;
|
||||
}
|
||||
|
||||
list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted);
|
||||
list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted);
|
||||
|
||||
if ($this->isThereSomeVisits($visits)) {
|
||||
return $idArchive;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares the core metrics if needed.
|
||||
*
|
||||
* @param $visits
|
||||
*/
|
||||
protected function prepareCoreMetricsArchive($visits, $visitsConverted)
|
||||
{
|
||||
$createSeparateArchiveForCoreMetrics = $this->mustProcessVisitCount($visits)
|
||||
&& !$this->doesRequestedPluginIncludeVisitsSummary();
|
||||
|
||||
if ($createSeparateArchiveForCoreMetrics) {
|
||||
$requestedPlugin = $this->params->getRequestedPlugin();
|
||||
|
||||
$this->params->setRequestedPlugin('VisitsSummary');
|
||||
|
||||
$pluginsArchiver = new PluginsArchiver($this->params, $this->isArchiveTemporary());
|
||||
$metrics = $pluginsArchiver->callAggregateCoreMetrics();
|
||||
$pluginsArchiver->finalizeArchive();
|
||||
|
||||
$this->params->setRequestedPlugin($requestedPlugin);
|
||||
|
||||
$visits = $metrics['nb_visits'];
|
||||
$visitsConverted = $metrics['nb_visits_converted'];
|
||||
}
|
||||
return array($visits, $visitsConverted);
|
||||
}
|
||||
|
||||
protected function prepareAllPluginsArchive($visits, $visitsConverted)
|
||||
{
|
||||
$pluginsArchiver = new PluginsArchiver($this->params, $this->isArchiveTemporary());
|
||||
if ($this->mustProcessVisitCount($visits)
|
||||
|| $this->doesRequestedPluginIncludeVisitsSummary()
|
||||
) {
|
||||
$metrics = $pluginsArchiver->callAggregateCoreMetrics();
|
||||
$visits = $metrics['nb_visits'];
|
||||
$visitsConverted = $metrics['nb_visits_converted'];
|
||||
}
|
||||
if ($this->isThereSomeVisits($visits)) {
|
||||
$pluginsArchiver->callAggregateAllPlugins($visits, $visitsConverted);
|
||||
}
|
||||
$idArchive = $pluginsArchiver->finalizeArchive();
|
||||
|
||||
if (!$this->params->isSingleSiteDayArchive() && $visits) {
|
||||
ArchiveSelector::purgeOutdatedArchives($this->params->getPeriod()->getDateStart());
|
||||
}
|
||||
|
||||
return array($idArchive, $visits);
|
||||
}
|
||||
|
||||
protected function doesRequestedPluginIncludeVisitsSummary()
|
||||
{
|
||||
$processAllReportsIncludingVisitsSummary =
|
||||
Rules::shouldProcessReportsAllPlugins($this->params->getIdSites(), $this->params->getSegment(), $this->params->getPeriod()->getLabel());
|
||||
$doesRequestedPluginIncludeVisitsSummary = $processAllReportsIncludingVisitsSummary
|
||||
|| $this->params->getRequestedPlugin() == 'VisitsSummary';
|
||||
return $doesRequestedPluginIncludeVisitsSummary;
|
||||
}
|
||||
|
||||
protected function isArchivingForcedToTrigger()
|
||||
{
|
||||
$period = $this->params->getPeriod()->getLabel();
|
||||
$debugSetting = 'always_archive_data_period'; // default
|
||||
if ($period == 'day') {
|
||||
$debugSetting = 'always_archive_data_day';
|
||||
} elseif ($period == 'range') {
|
||||
$debugSetting = 'always_archive_data_range';
|
||||
}
|
||||
return (bool) Config::getInstance()->Debug[$debugSetting];
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the idArchive if the archive is available in the database for the requested plugin.
|
||||
* Returns false if the archive needs to be processed.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
protected function loadExistingArchiveIdFromDb()
|
||||
{
|
||||
$noArchiveFound = array(false, false, false);
|
||||
|
||||
// see isArchiveTemporary()
|
||||
$minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();
|
||||
|
||||
if ($this->isArchivingForcedToTrigger()) {
|
||||
return $noArchiveFound;
|
||||
}
|
||||
|
||||
$idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
|
||||
if (!$idAndVisits) {
|
||||
return $noArchiveFound;
|
||||
}
|
||||
return $idAndVisits;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the minimum archive processed datetime to look at. Only public for tests.
|
||||
*
|
||||
* @return int|bool Datetime timestamp, or false if must look at any archive available
|
||||
*/
|
||||
protected function getMinTimeArchiveProcessed()
|
||||
{
|
||||
$endDateTimestamp = self::determineIfArchivePermanent($this->params->getDateEnd());
|
||||
$isArchiveTemporary = ($endDateTimestamp === false);
|
||||
$this->temporaryArchive = $isArchiveTemporary;
|
||||
|
||||
if ($endDateTimestamp) {
|
||||
// Permanent archive
|
||||
return $endDateTimestamp;
|
||||
}
|
||||
// Temporary archive
|
||||
return Rules::getMinTimeProcessedForTemporaryArchive($this->params->getDateStart(), $this->params->getPeriod(), $this->params->getSegment(), $this->params->getSite());
|
||||
}
|
||||
|
||||
protected static function determineIfArchivePermanent(Date $dateEnd)
|
||||
{
|
||||
$now = time();
|
||||
$endTimestampUTC = strtotime($dateEnd->getDateEndUTC());
|
||||
if ($endTimestampUTC <= $now) {
|
||||
// - if the period we are looking for is finished, we look for a ts_archived that
|
||||
// is greater than the last day of the archive
|
||||
return $endTimestampUTC;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
protected function isArchiveTemporary()
|
||||
{
|
||||
if (is_null($this->temporaryArchive)) {
|
||||
throw new \Exception("getMinTimeArchiveProcessed() should be called prior to isArchiveTemporary()");
|
||||
}
|
||||
return $this->temporaryArchive;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
194
www/analytics/core/ArchiveProcessor/Parameters.php
Normal file
194
www/analytics/core/ArchiveProcessor/Parameters.php
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
<?php
|
||||
/**
|
||||
* Piwik - Open source web analytics
|
||||
*
|
||||
* @link http://piwik.org
|
||||
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
|
||||
*
|
||||
*/
|
||||
|
||||
namespace Piwik\ArchiveProcessor;
|
||||
|
||||
use Piwik\Date;
|
||||
use Piwik\Log;
|
||||
use Piwik\Period;
|
||||
use Piwik\Piwik;
|
||||
use Piwik\Segment;
|
||||
use Piwik\Site;
|
||||
|
||||
/**
|
||||
* Contains the analytics parameters for the reports that are currently being archived. The analytics
|
||||
* parameters include the **website** the reports describe, the **period** of time the reports describe
|
||||
* and the **segment** used to limit the visit set.
|
||||
*/
|
||||
class Parameters
|
||||
{
|
||||
/**
|
||||
* @var Site
|
||||
*/
|
||||
private $site = null;
|
||||
|
||||
/**
|
||||
* @var Period
|
||||
*/
|
||||
private $period = null;
|
||||
|
||||
/**
|
||||
* @var Segment
|
||||
*/
|
||||
private $segment = null;
|
||||
|
||||
/**
|
||||
* @var string Plugin name which triggered this archive processor
|
||||
*/
|
||||
private $requestedPlugin = false;
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
*
|
||||
* @ignore
|
||||
*/
|
||||
public function __construct(Site $site, Period $period, Segment $segment, $skipAggregationOfSubTables = false)
|
||||
{
|
||||
$this->site = $site;
|
||||
$this->period = $period;
|
||||
$this->segment = $segment;
|
||||
$this->skipAggregationOfSubTables = $skipAggregationOfSubTables;
|
||||
}
|
||||
|
||||
/**
|
||||
* @ignore
|
||||
*/
|
||||
public function setRequestedPlugin($plugin)
|
||||
{
|
||||
$this->requestedPlugin = $plugin;
|
||||
}
|
||||
|
||||
/**
|
||||
* @ignore
|
||||
*/
|
||||
public function getRequestedPlugin()
|
||||
{
|
||||
return $this->requestedPlugin;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the period we are computing statistics for.
|
||||
*
|
||||
* @return Period
|
||||
* @api
|
||||
*/
|
||||
public function getPeriod()
|
||||
{
|
||||
return $this->period;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the array of Period which make up this archive.
|
||||
*
|
||||
* @return \Piwik\Period[]
|
||||
* @ignore
|
||||
*/
|
||||
public function getSubPeriods()
|
||||
{
|
||||
if($this->getPeriod()->getLabel() == 'day') {
|
||||
return array( $this->getPeriod() );
|
||||
}
|
||||
return $this->getPeriod()->getSubperiods();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array
|
||||
* @ignore
|
||||
*/
|
||||
public function getIdSites()
|
||||
{
|
||||
$idSite = $this->getSite()->getId();
|
||||
|
||||
$idSites = array($idSite);
|
||||
|
||||
Piwik::postEvent('ArchiveProcessor.Parameters.getIdSites', array(&$idSites, $this->getPeriod()));
|
||||
|
||||
return $idSites;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the site we are computing statistics for.
|
||||
*
|
||||
* @return Site
|
||||
* @api
|
||||
*/
|
||||
public function getSite()
|
||||
{
|
||||
return $this->site;
|
||||
}
|
||||
|
||||
/**
|
||||
* The Segment used to limit the set of visits that are being aggregated.
|
||||
*
|
||||
* @return Segment
|
||||
* @api
|
||||
*/
|
||||
public function getSegment()
|
||||
{
|
||||
return $this->segment;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the end day of the period in the site's timezone.
|
||||
*
|
||||
* @return Date
|
||||
*/
|
||||
public function getDateEnd()
|
||||
{
|
||||
return $this->getPeriod()->getDateEnd()->setTimezone($this->getSite()->getTimezone());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the start day of the period in the site's timezone.
|
||||
*
|
||||
* @return Date
|
||||
*/
|
||||
public function getDateStart()
|
||||
{
|
||||
return $this->getPeriod()->getDateStart()->setTimezone($this->getSite()->getTimezone());
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
public function isSingleSiteDayArchive()
|
||||
{
|
||||
$oneSite = $this->isSingleSite();
|
||||
$oneDay = $this->getPeriod()->getLabel() == 'day';
|
||||
return $oneDay && $oneSite;
|
||||
}
|
||||
|
||||
public function isSingleSite()
|
||||
{
|
||||
return count($this->getIdSites()) == 1;
|
||||
}
|
||||
|
||||
public function isSkipAggregationOfSubTables()
|
||||
{
|
||||
return $this->skipAggregationOfSubTables;
|
||||
}
|
||||
|
||||
public function logStatusDebug($isTemporary)
|
||||
{
|
||||
$temporary = 'definitive archive';
|
||||
if ($isTemporary) {
|
||||
$temporary = 'temporary archive';
|
||||
}
|
||||
Log::verbose(
|
||||
"%s archive, idSite = %d (%s), segment '%s', report = '%s', UTC datetime [%s -> %s]",
|
||||
$this->getPeriod()->getLabel(),
|
||||
$this->getSite()->getId(),
|
||||
$temporary,
|
||||
$this->getSegment()->getString(),
|
||||
$this->getRequestedPlugin(),
|
||||
$this->getDateStart()->getDateStartUTC(),
|
||||
$this->getDateEnd()->getDateEndUTC()
|
||||
);
|
||||
}
|
||||
}
|
||||
197
www/analytics/core/ArchiveProcessor/PluginsArchiver.php
Normal file
197
www/analytics/core/ArchiveProcessor/PluginsArchiver.php
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
<?php
|
||||
/**
|
||||
* Piwik - Open source web analytics
|
||||
*
|
||||
* @link http://piwik.org
|
||||
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
|
||||
*
|
||||
*/
|
||||
|
||||
namespace Piwik\ArchiveProcessor;
|
||||
|
||||
use Piwik\Archive;
|
||||
use Piwik\ArchiveProcessor;
|
||||
use Piwik\DataAccess\ArchiveSelector;
|
||||
use Piwik\DataAccess\ArchiveWriter;
|
||||
use Piwik\DataTable\Manager;
|
||||
use Piwik\Metrics;
|
||||
use Piwik\Plugin\Archiver;
|
||||
|
||||
/**
|
||||
* This class creates the Archiver objects found in plugins and will trigger aggregation,
|
||||
* so each plugin can process their reports.
|
||||
*/
|
||||
class PluginsArchiver
|
||||
{
|
||||
/**
|
||||
* @param ArchiveProcessor $archiveProcessor
|
||||
*/
|
||||
public $archiveProcessor;
|
||||
|
||||
/**
|
||||
* @var Parameters
|
||||
*/
|
||||
protected $params;
|
||||
|
||||
/**
|
||||
* @var Archiver[] $archivers
|
||||
*/
|
||||
private static $archivers = array();
|
||||
|
||||
public function __construct(Parameters $params, $isTemporaryArchive)
|
||||
{
|
||||
$this->params = $params;
|
||||
|
||||
$this->archiveWriter = new ArchiveWriter($this->params, $isTemporaryArchive);
|
||||
$this->archiveWriter->initNewArchive();
|
||||
|
||||
$this->archiveProcessor = new ArchiveProcessor($this->params, $this->archiveWriter);
|
||||
|
||||
$this->isSingleSiteDayArchive = $this->params->isSingleSiteDayArchive();
|
||||
}
|
||||
|
||||
/**
|
||||
* If period is day, will get the core metrics (including visits) from the logs.
|
||||
* If period is != day, will sum the core metrics from the existing archives.
|
||||
* @return array Core metrics
|
||||
*/
|
||||
public function callAggregateCoreMetrics()
|
||||
{
|
||||
if($this->isSingleSiteDayArchive) {
|
||||
$metrics = $this->aggregateDayVisitsMetrics();
|
||||
} else {
|
||||
$metrics = $this->aggregateMultipleVisitsMetrics();
|
||||
}
|
||||
|
||||
if (empty($metrics)) {
|
||||
return array(
|
||||
'nb_visits' => false,
|
||||
'nb_visits_converted' => false
|
||||
);
|
||||
}
|
||||
return array(
|
||||
'nb_visits' => $metrics['nb_visits'],
|
||||
'nb_visits_converted' => $metrics['nb_visits_converted']
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates the Archiver class in each plugin that defines it,
|
||||
* and triggers Aggregation processing on these plugins.
|
||||
*/
|
||||
public function callAggregateAllPlugins($visits, $visitsConverted)
|
||||
{
|
||||
$this->archiveProcessor->setNumberOfVisits($visits, $visitsConverted);
|
||||
|
||||
$archivers = $this->getPluginArchivers();
|
||||
|
||||
foreach($archivers as $pluginName => $archiverClass) {
|
||||
|
||||
// We clean up below all tables created during this function call (and recursive calls)
|
||||
$latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
|
||||
|
||||
/** @var Archiver $archiver */
|
||||
$archiver = new $archiverClass($this->archiveProcessor);
|
||||
|
||||
if(!$archiver->isEnabled()) {
|
||||
continue;
|
||||
}
|
||||
if($this->shouldProcessReportsForPlugin($pluginName)) {
|
||||
if($this->isSingleSiteDayArchive) {
|
||||
$archiver->aggregateDayReport();
|
||||
} else {
|
||||
$archiver->aggregateMultipleReports();
|
||||
}
|
||||
}
|
||||
|
||||
Manager::getInstance()->deleteAll($latestUsedTableId);
|
||||
unset($archiver);
|
||||
}
|
||||
}
|
||||
|
||||
public function finalizeArchive()
|
||||
{
|
||||
$this->params->logStatusDebug( $this->archiveWriter->isArchiveTemporary );
|
||||
$this->archiveWriter->finalizeArchive();
|
||||
return $this->archiveWriter->getIdArchive();
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads Archiver class from any plugin that defines one.
|
||||
*
|
||||
* @return \Piwik\Plugin\Archiver[]
|
||||
*/
|
||||
protected function getPluginArchivers()
|
||||
{
|
||||
if (empty(static::$archivers)) {
|
||||
$pluginNames = \Piwik\Plugin\Manager::getInstance()->getActivatedPlugins();
|
||||
$archivers = array();
|
||||
foreach ($pluginNames as $pluginName) {
|
||||
$archivers[$pluginName] = self::getPluginArchiverClass($pluginName);
|
||||
}
|
||||
static::$archivers = array_filter($archivers);
|
||||
}
|
||||
return static::$archivers;
|
||||
}
|
||||
|
||||
private static function getPluginArchiverClass($pluginName)
|
||||
{
|
||||
$klassName = 'Piwik\\Plugins\\' . $pluginName . '\\Archiver';
|
||||
if (class_exists($klassName)
|
||||
&& is_subclass_of($klassName, 'Piwik\\Plugin\\Archiver')) {
|
||||
return $klassName;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the specified plugin's reports should be archived
|
||||
* @param string $pluginName
|
||||
* @return bool
|
||||
*/
|
||||
protected function shouldProcessReportsForPlugin($pluginName)
|
||||
{
|
||||
if ($this->params->getRequestedPlugin() == $pluginName) {
|
||||
return true;
|
||||
}
|
||||
if (Rules::shouldProcessReportsAllPlugins(
|
||||
$this->params->getIdSites(),
|
||||
$this->params->getSegment(),
|
||||
$this->params->getPeriod()->getLabel())) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!\Piwik\Plugin\Manager::getInstance()->isPluginLoaded($this->params->getRequestedPlugin())) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
protected function aggregateDayVisitsMetrics()
|
||||
{
|
||||
$query = $this->archiveProcessor->getLogAggregator()->queryVisitsByDimension();
|
||||
$data = $query->fetch();
|
||||
|
||||
$metrics = $this->convertMetricsIdToName($data);
|
||||
$this->archiveProcessor->insertNumericRecords($metrics);
|
||||
return $metrics;
|
||||
}
|
||||
|
||||
protected function convertMetricsIdToName($data)
|
||||
{
|
||||
$metrics = array();
|
||||
foreach ($data as $metricId => $value) {
|
||||
$readableMetric = Metrics::$mappingFromIdToName[$metricId];
|
||||
$metrics[$readableMetric] = $value;
|
||||
}
|
||||
return $metrics;
|
||||
}
|
||||
|
||||
protected function aggregateMultipleVisitsMetrics()
|
||||
{
|
||||
$toSum = Metrics::getVisitsMetricNames();
|
||||
$metrics = $this->archiveProcessor->aggregateNumericMetrics($toSum);
|
||||
return $metrics;
|
||||
}
|
||||
|
||||
}
|
||||
304
www/analytics/core/ArchiveProcessor/Rules.php
Normal file
304
www/analytics/core/ArchiveProcessor/Rules.php
Normal file
|
|
@ -0,0 +1,304 @@
|
|||
<?php
|
||||
/**
|
||||
* Piwik - Open source web analytics
|
||||
*
|
||||
* @link http://piwik.org
|
||||
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
|
||||
*
|
||||
*/
|
||||
namespace Piwik\ArchiveProcessor;
|
||||
|
||||
use Exception;
|
||||
use Piwik\Common;
|
||||
use Piwik\Config;
|
||||
use Piwik\Date;
|
||||
use Piwik\Log;
|
||||
use Piwik\Option;
|
||||
use Piwik\Piwik;
|
||||
use Piwik\Plugins\CoreAdminHome\Controller;
|
||||
use Piwik\Plugins\CoreAdminHome\CoreAdminHome;
|
||||
use Piwik\Segment;
|
||||
use Piwik\SettingsPiwik;
|
||||
use Piwik\SettingsServer;
|
||||
use Piwik\Site;
|
||||
use Piwik\Tracker\Cache;
|
||||
|
||||
/**
|
||||
* This class contains Archiving rules/logic which are used when creating and processing Archives.
|
||||
*
|
||||
*/
|
||||
class Rules
|
||||
{
|
||||
const OPTION_TODAY_ARCHIVE_TTL = 'todayArchiveTimeToLive';
|
||||
|
||||
const OPTION_BROWSER_TRIGGER_ARCHIVING = 'enableBrowserTriggerArchiving';
|
||||
|
||||
const FLAG_TABLE_PURGED = 'lastPurge_';
|
||||
|
||||
/** Old Archives purge can be disabled (used in tests only) */
|
||||
static public $purgeDisabledByTests = false;
|
||||
|
||||
/** Flag that will forcefully disable the archiving process (used in tests only) */
|
||||
public static $archivingDisabledByTests = false;
|
||||
|
||||
/**
|
||||
* Returns the name of the archive field used to tell the status of an archive, (ie,
|
||||
* whether the archive was created successfully or not).
|
||||
*
|
||||
* @param Segment $segment
|
||||
* @param string $periodLabel
|
||||
* @param string $plugin
|
||||
* @return string
|
||||
*/
|
||||
public static function getDoneStringFlagFor(array $idSites, $segment, $periodLabel, $plugin, $isSkipAggregationOfSubTables)
|
||||
{
|
||||
if (!self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel)) {
|
||||
return self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin, $isSkipAggregationOfSubTables);
|
||||
}
|
||||
return self::getDoneFlagArchiveContainsAllPlugins($segment);
|
||||
}
|
||||
|
||||
public static function shouldProcessReportsAllPlugins(array $idSites, Segment $segment, $periodLabel)
|
||||
{
|
||||
if ($segment->isEmpty() && $periodLabel != 'range') {
|
||||
return true;
|
||||
}
|
||||
|
||||
return self::isSegmentPreProcessed($idSites, $segment);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $idSites
|
||||
* @return array
|
||||
*/
|
||||
private static function getSegmentsToProcess($idSites)
|
||||
{
|
||||
$knownSegmentsToArchiveAllSites = SettingsPiwik::getKnownSegmentsToArchive();
|
||||
|
||||
$segmentsToProcess = $knownSegmentsToArchiveAllSites;
|
||||
foreach ($idSites as $idSite) {
|
||||
$segmentForThisWebsite = SettingsPiwik::getKnownSegmentsToArchiveForSite($idSite);
|
||||
$segmentsToProcess = array_merge($segmentsToProcess, $segmentForThisWebsite);
|
||||
}
|
||||
$segmentsToProcess = array_unique($segmentsToProcess);
|
||||
return $segmentsToProcess;
|
||||
}
|
||||
|
||||
public static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin, $isSkipAggregationOfSubTables = false)
|
||||
{
|
||||
$partial = self::isFlagArchivePartial($plugin, $isSkipAggregationOfSubTables);
|
||||
return 'done' . $segment->getHash() . '.' . $plugin . $partial ;
|
||||
}
|
||||
|
||||
private static function getDoneFlagArchiveContainsAllPlugins(Segment $segment)
|
||||
{
|
||||
return 'done' . $segment->getHash();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $plugin
|
||||
* @param $isSkipAggregationOfSubTables
|
||||
* @return string
|
||||
*/
|
||||
private static function isFlagArchivePartial($plugin, $isSkipAggregationOfSubTables)
|
||||
{
|
||||
$partialArchive = '';
|
||||
if ($plugin != "VisitsSummary" // VisitsSummary is always called when segmenting and should not have its own .partial archive
|
||||
&& $isSkipAggregationOfSubTables
|
||||
) {
|
||||
$partialArchive = '.partial';
|
||||
}
|
||||
return $partialArchive;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $plugins
|
||||
* @param $segment
|
||||
* @return array
|
||||
*/
|
||||
public static function getDoneFlags(array $plugins, Segment $segment, $isSkipAggregationOfSubTables)
|
||||
{
|
||||
$doneFlags = array();
|
||||
$doneAllPlugins = self::getDoneFlagArchiveContainsAllPlugins($segment);
|
||||
$doneFlags[$doneAllPlugins] = $doneAllPlugins;
|
||||
|
||||
$plugins = array_unique($plugins);
|
||||
foreach ($plugins as $plugin) {
|
||||
$doneOnePlugin = self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin, $isSkipAggregationOfSubTables);
|
||||
$doneFlags[$plugin] = $doneOnePlugin;
|
||||
}
|
||||
return $doneFlags;
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a monthly archive table, will delete all reports that are now outdated,
|
||||
* or reports that ended with an error
|
||||
*
|
||||
* @param \Piwik\Date $date
|
||||
* @return int|bool False, or timestamp indicating which archives to delete
|
||||
*/
|
||||
public static function shouldPurgeOutdatedArchives(Date $date)
|
||||
{
|
||||
if (self::$purgeDisabledByTests) {
|
||||
return false;
|
||||
}
|
||||
$key = self::FLAG_TABLE_PURGED . "blob_" . $date->toString('Y_m');
|
||||
$timestamp = Option::get($key);
|
||||
|
||||
// we shall purge temporary archives after their timeout is finished, plus an extra 6 hours
|
||||
// in case archiving is disabled or run once a day, we give it this extra time to run
|
||||
// and re-process more recent records...
|
||||
$temporaryArchivingTimeout = self::getTodayArchiveTimeToLive();
|
||||
$hoursBetweenPurge = 6;
|
||||
$purgeEveryNSeconds = max($temporaryArchivingTimeout, $hoursBetweenPurge * 3600);
|
||||
|
||||
// we only delete archives if we are able to process them, otherwise, the browser might process reports
|
||||
// when &segment= is specified (or custom date range) and would below, delete temporary archives that the
|
||||
// browser is not able to process until next cron run (which could be more than 1 hour away)
|
||||
if (self::isRequestAuthorizedToArchive()
|
||||
&& (!$timestamp
|
||||
|| $timestamp < time() - $purgeEveryNSeconds)
|
||||
) {
|
||||
Option::set($key, time());
|
||||
|
||||
if (self::isBrowserTriggerEnabled()) {
|
||||
// If Browser Archiving is enabled, it is likely there are many more temporary archives
|
||||
// We delete more often which is safe, since reports are re-processed on demand
|
||||
$purgeArchivesOlderThan = Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime();
|
||||
} else {
|
||||
// If archive.php via Cron is building the reports, we should keep all temporary reports from today
|
||||
$purgeArchivesOlderThan = Date::factory('today')->getDateTime();
|
||||
}
|
||||
return $purgeArchivesOlderThan;
|
||||
}
|
||||
|
||||
Log::info("Purging temporary archives: skipped.");
|
||||
return false;
|
||||
}
|
||||
|
||||
public static function getMinTimeProcessedForTemporaryArchive(
|
||||
Date $dateStart, \Piwik\Period $period, Segment $segment, Site $site)
|
||||
{
|
||||
$now = time();
|
||||
$minimumArchiveTime = $now - Rules::getTodayArchiveTimeToLive();
|
||||
|
||||
$idSites = array($site->getId());
|
||||
$isArchivingDisabled = Rules::isArchivingDisabledFor($idSites, $segment, $period->getLabel());
|
||||
if ($isArchivingDisabled) {
|
||||
if ($period->getNumberOfSubperiods() == 0
|
||||
&& $dateStart->getTimestamp() <= $now
|
||||
) {
|
||||
// Today: accept any recent enough archive
|
||||
$minimumArchiveTime = false;
|
||||
} else {
|
||||
// This week, this month, this year:
|
||||
// accept any archive that was processed today after 00:00:01 this morning
|
||||
$timezone = $site->getTimezone();
|
||||
$minimumArchiveTime = Date::factory(Date::factory('now', $timezone)->getDateStartUTC())->setTimezone($timezone)->getTimestamp();
|
||||
}
|
||||
}
|
||||
return $minimumArchiveTime;
|
||||
}
|
||||
|
||||
public static function setTodayArchiveTimeToLive($timeToLiveSeconds)
|
||||
{
|
||||
$timeToLiveSeconds = (int)$timeToLiveSeconds;
|
||||
if ($timeToLiveSeconds <= 0) {
|
||||
throw new Exception(Piwik::translate('General_ExceptionInvalidArchiveTimeToLive'));
|
||||
}
|
||||
Option::set(self::OPTION_TODAY_ARCHIVE_TTL, $timeToLiveSeconds, $autoLoad = true);
|
||||
}
|
||||
|
||||
public static function getTodayArchiveTimeToLive()
|
||||
{
|
||||
$uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
|
||||
|
||||
if($uiSettingIsEnabled) {
|
||||
$timeToLive = Option::get(self::OPTION_TODAY_ARCHIVE_TTL);
|
||||
if ($timeToLive !== false) {
|
||||
return $timeToLive;
|
||||
}
|
||||
}
|
||||
return Config::getInstance()->General['time_before_today_archive_considered_outdated'];
|
||||
}
|
||||
|
||||
public static function isArchivingDisabledFor(array $idSites, Segment $segment, $periodLabel)
|
||||
{
|
||||
if ($periodLabel == 'range') {
|
||||
return false;
|
||||
}
|
||||
$processOneReportOnly = !self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel);
|
||||
$isArchivingDisabled = !self::isRequestAuthorizedToArchive() || self::$archivingDisabledByTests;
|
||||
|
||||
if ($processOneReportOnly) {
|
||||
|
||||
// When there is a segment, we disable archiving when browser_archiving_disabled_enforce applies
|
||||
if (!$segment->isEmpty()
|
||||
&& $isArchivingDisabled
|
||||
&& Config::getInstance()->General['browser_archiving_disabled_enforce']
|
||||
&& !SettingsServer::isArchivePhpTriggered() // Only applies when we are not running archive.php
|
||||
) {
|
||||
Log::debug("Archiving is disabled because of config setting browser_archiving_disabled_enforce=1");
|
||||
return true;
|
||||
}
|
||||
|
||||
// Always allow processing one report
|
||||
return false;
|
||||
}
|
||||
return $isArchivingDisabled;
|
||||
}
|
||||
|
||||
protected static function isRequestAuthorizedToArchive()
|
||||
{
|
||||
return Rules::isBrowserTriggerEnabled() || SettingsServer::isArchivePhpTriggered();
|
||||
}
|
||||
|
||||
public static function isBrowserTriggerEnabled()
|
||||
{
|
||||
$uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
|
||||
|
||||
if($uiSettingIsEnabled) {
|
||||
$browserArchivingEnabled = Option::get(self::OPTION_BROWSER_TRIGGER_ARCHIVING);
|
||||
if ($browserArchivingEnabled !== false) {
|
||||
return (bool)$browserArchivingEnabled;
|
||||
}
|
||||
}
|
||||
return (bool)Config::getInstance()->General['enable_browser_archiving_triggering'];
|
||||
}
|
||||
|
||||
public static function setBrowserTriggerArchiving($enabled)
|
||||
{
|
||||
if (!is_bool($enabled)) {
|
||||
throw new Exception('Browser trigger archiving must be set to true or false.');
|
||||
}
|
||||
Option::set(self::OPTION_BROWSER_TRIGGER_ARCHIVING, (int)$enabled, $autoLoad = true);
|
||||
Cache::clearCacheGeneral();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $idSites
|
||||
* @param Segment $segment
|
||||
* @return bool
|
||||
*/
|
||||
protected static function isSegmentPreProcessed(array $idSites, Segment $segment)
|
||||
{
|
||||
$segmentsToProcess = self::getSegmentsToProcess($idSites);
|
||||
|
||||
if (empty($segmentsToProcess)) {
|
||||
return false;
|
||||
}
|
||||
// If the requested segment is one of the segments to pre-process
|
||||
// we ensure that any call to the API will trigger archiving of all reports for this segment
|
||||
$segment = $segment->getString();
|
||||
|
||||
// Turns out the getString() above returns the URL decoded segment string
|
||||
$segmentsToProcessUrlDecoded = array_map('urldecode', $segmentsToProcess);
|
||||
|
||||
if (in_array($segment, $segmentsToProcess)
|
||||
|| in_array($segment, $segmentsToProcessUrlDecoded)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue