coderkun/questlab
1
0
Fork 0
Code Issues 15 Pull requests Projects Releases Wiki Activity

update Piwik to version 2.16 (fixes #91)

Browse source
This commit is contained in:
oliver 2016-04-10 18:55:57 +02:00
commit d885a4baa9
5833 changed files with 418860 additions and 226988 deletions
Download patch file Download diff file Expand all files Collapse all files

281
www/analytics/core/DataAccess/ArchiveSelector.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,18 +9,16 @@
namespace Piwik\DataAccess;
use Exception;
use Piwik\ArchiveProcessor\Rules;
use Piwik\Archive;
use Piwik\Archive\Chunk;
use Piwik\ArchiveProcessor;
use Piwik\ArchiveProcessor\Rules;
use Piwik\Common;
use Piwik\Date;
use Piwik\Db;
use Piwik\Log;
use Piwik\Period;
use Piwik\Period\Range;
use Piwik\Piwik;
use Piwik\Segment;
use Piwik\Site;
/**
* Data Access object used to query archives
@ -42,52 +40,46 @@ class ArchiveSelector
const NB_VISITS_CONVERTED_RECORD_LOOKED_UP = "nb_visits_converted";
static public function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC)
private static function getModel()
{
$dateStart = $params->getPeriod()->getDateStart();
$bindSQL = array($params->getSite()->getId(),
$dateStart->toString('Y-m-d'),
$params->getPeriod()->getDateEnd()->toString('Y-m-d'),
$params->getPeriod()->getId(),
);
return new Model();
}
$timeStampWhere = '';
public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC)
{
$idSite = $params->getSite()->getId();
$period = $params->getPeriod()->getId();
$dateStart = $params->getPeriod()->getDateStart();
$dateStartIso = $dateStart->toString('Y-m-d');
$dateEndIso = $params->getPeriod()->getDateEnd()->toString('Y-m-d');
$numericTable = ArchiveTableCreator::getNumericTable($dateStart);
$minDatetimeIsoArchiveProcessedUTC = null;
if ($minDatetimeArchiveProcessedUTC) {
$timeStampWhere = " AND ts_archived >= ? ";
$bindSQL[] = Date::factory($minDatetimeArchiveProcessedUTC)->getDatetime();
$minDatetimeIsoArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC)->getDatetime();
}
$requestedPlugin = $params->getRequestedPlugin();
$segment = $params->getSegment();
$isSkipAggregationOfSubTables = $params->isSkipAggregationOfSubTables();
$segment = $params->getSegment();
$plugins = array("VisitsSummary", $requestedPlugin);
$sqlWhereArchiveName = self::getNameCondition($plugins, $segment, $isSkipAggregationOfSubTables);
$sqlQuery = " SELECT idarchive, value, name, date1 as startDate
FROM " . ArchiveTableCreator::getNumericTable($dateStart) . "``
WHERE idsite = ?
AND date1 = ?
AND date2 = ?
AND period = ?
AND ( ($sqlWhereArchiveName)
OR name = '" . self::NB_VISITS_RECORD_LOOKED_UP . "'
OR name = '" . self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP . "')
$timeStampWhere
ORDER BY idarchive DESC";
$results = Db::fetchAll($sqlQuery, $bindSQL);
$doneFlags = Rules::getDoneFlags($plugins, $segment);
$doneFlagValues = Rules::getSelectableDoneFlagValues();
$results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues);
if (empty($results)) {
return false;
}
$idArchive = self::getMostRecentIdArchiveFromResults($segment, $requestedPlugin, $isSkipAggregationOfSubTables, $results);
$idArchiveVisitsSummary = self::getMostRecentIdArchiveFromResults($segment, "VisitsSummary", $isSkipAggregationOfSubTables, $results);
$idArchive = self::getMostRecentIdArchiveFromResults($segment, $requestedPlugin, $results);
$idArchiveVisitsSummary = self::getMostRecentIdArchiveFromResults($segment, "VisitsSummary", $results);
list($visits, $visitsConverted) = self::getVisitsMetricsFromResults($idArchive, $idArchiveVisitsSummary, $results);
if ($visits === false
&& $idArchive === false
) {
if (false === $visits && false === $idArchive) {
return false;
}
@ -98,9 +90,11 @@ class ArchiveSelector
{
$visits = $visitsConverted = false;
$archiveWithVisitsMetricsWasFound = ($idArchiveVisitsSummary !== false);
if ($archiveWithVisitsMetricsWasFound) {
$visits = $visitsConverted = 0;
}
foreach ($results as $result) {
if (in_array($result['idarchive'], array($idArchive, $idArchiveVisitsSummary))) {
$value = (int)$result['value'];
@ -116,13 +110,15 @@ class ArchiveSelector
}
}
}
return array($visits, $visitsConverted);
}
protected static function getMostRecentIdArchiveFromResults(Segment $segment, $requestedPlugin, $isSkipAggregationOfSubTables, $results)
protected static function getMostRecentIdArchiveFromResults(Segment $segment, $requestedPlugin, $results)
{
$idArchive = false;
$namesRequestedPlugin = Rules::getDoneFlags(array($requestedPlugin), $segment, $isSkipAggregationOfSubTables);
$namesRequestedPlugin = Rules::getDoneFlags(array($requestedPlugin), $segment);
foreach ($results as $result) {
if ($idArchive === false
&& in_array($result['name'], $namesRequestedPlugin)
@ -131,6 +127,7 @@ class ArchiveSelector
break;
}
}
return $idArchive;
}
@ -141,21 +138,29 @@ class ArchiveSelector
* @param array $periods
* @param Segment $segment
* @param array $plugins List of plugin names for which data is being requested.
* @param bool $isSkipAggregationOfSubTables Whether we are selecting an archive that may be partial (no sub-tables)
* @return array Archive IDs are grouped by archive name and period range, ie,
* array(
* 'VisitsSummary.done' => array(
* '2010-01-01' => array(1,2,3)
* )
* )
* @throws
*/
static public function getArchiveIds($siteIds, $periods, $segment, $plugins, $isSkipAggregationOfSubTables = false)
public static function getArchiveIds($siteIds, $periods, $segment, $plugins)
{
if (empty($siteIds)) {
throw new \Exception("Website IDs could not be read from the request, ie. idSite=");
}
foreach ($siteIds as $index => $siteId) {
$siteIds[$index] = (int) $siteId;
}
$getArchiveIdsSql = "SELECT idsite, name, date1, date2, MAX(idarchive) as idarchive
FROM %s
WHERE %s
AND " . self::getNameCondition($plugins, $segment, $isSkipAggregationOfSubTables) . "
AND idsite IN (" . implode(',', $siteIds) . ")
WHERE idsite IN (" . implode(',', $siteIds) . ")
AND " . self::getNameCondition($plugins, $segment) . "
AND %s
GROUP BY idsite, date1, date2";
$monthToPeriods = array();
@ -197,14 +202,14 @@ class ArchiveSelector
$sql = sprintf($getArchiveIdsSql, $table, $dateCondition);
$archiveIds = Db::fetchAll($sql, $bind);
// get the archive IDs
foreach (Db::fetchAll($sql, $bind) as $row) {
$archiveName = $row['name'];
foreach ($archiveIds as $row) {
//FIXMEA duplicate with Archive.php
$dateStr = $row['date1'] . "," . $row['date2'];
$dateStr = $row['date1'] . ',' . $row['date2'];
$result[$archiveName][$dateStr][] = $row['idarchive'];
$result[$row['name']][$dateStr][] = $row['idarchive'];
}
}
@ -215,29 +220,52 @@ class ArchiveSelector
* Queries and returns archive data using a set of archive IDs.
*
* @param array $archiveIds The IDs of the archives to get data from.
* @param array $recordNames The names of the data to retrieve (ie, nb_visits, nb_actions, etc.)
* @param array $recordNames The names of the data to retrieve (ie, nb_visits, nb_actions, etc.).
* Note: You CANNOT pass multiple recordnames if $loadAllSubtables=true.
* @param string $archiveDataType The archive data type (either, 'blob' or 'numeric').
* @param bool $loadAllSubtables Whether to pre-load all subtables
* @param int|null|string $idSubtable null if the root blob should be loaded, an integer if a subtable should be
* loaded and 'all' if all subtables should be loaded.
* @throws Exception
* @return array
*/
static public function getArchiveData($archiveIds, $recordNames, $archiveDataType, $loadAllSubtables)
public static function getArchiveData($archiveIds, $recordNames, $archiveDataType, $idSubtable)
{
$chunk = new Chunk();
// create the SQL to select archive data
$inNames = Common::getSqlStringFieldsArray($recordNames);
$loadAllSubtables = $idSubtable == Archive::ID_SUBTABLE_LOAD_ALL_SUBTABLES;
if ($loadAllSubtables) {
$name = reset($recordNames);
// select blobs w/ name like "$name_[0-9]+" w/o using RLIKE
$nameEnd = strlen($name) + 2;
$whereNameIs = "(name = ?
OR (name LIKE ?
AND SUBSTRING(name, $nameEnd, 1) >= '0'
AND SUBSTRING(name, $nameEnd, 1) <= '9') )";
$nameEnd = strlen($name) + 1;
$nameEndAppendix = $nameEnd + 1;
$appendix = $chunk->getAppendix();
$lenAppendix = strlen($appendix);
$checkForChunkBlob = "SUBSTRING(name, $nameEnd, $lenAppendix) = '$appendix'";
$checkForSubtableId = "(SUBSTRING(name, $nameEndAppendix, 1) >= '0'
AND SUBSTRING(name, $nameEndAppendix, 1) <= '9')";
$whereNameIs = "(name = ? OR (name LIKE ? AND ( $checkForChunkBlob OR $checkForSubtableId ) ))";
$bind = array($name, $name . '%');
} else {
if ($idSubtable === null) {
// select root table or specific record names
$bind = array_values($recordNames);
} else {
// select a subtable id
$bind = array();
foreach ($recordNames as $recordName) {
// to be backwards compatibe we need to look for the exact idSubtable blob and for the chunk
// that stores the subtables (a chunk stores many blobs in one blob)
$bind[] = $chunk->getRecordNameForTableId($recordName, $idSubtable);
$bind[] = self::appendIdSubtable($recordName, $idSubtable);
}
}
$inNames = Common::getSqlStringFieldsArray($bind);
$whereNameIs = "name IN ($inNames)";
$bind = array_values($recordNames);
}
$getValuesSql = "SELECT value, name, idsite, date1, date2, ts_archived
@ -251,110 +279,91 @@ class ArchiveSelector
if (empty($ids)) {
throw new Exception("Unexpected: id archive not found for period '$period' '");
}
// $period = "2009-01-04,2009-01-04",
$date = Date::factory(substr($period, 0, 10));
if ($archiveDataType == 'numeric') {
$isNumeric = $archiveDataType == 'numeric';
if ($isNumeric) {
$table = ArchiveTableCreator::getNumericTable($date);
} else {
$table = ArchiveTableCreator::getBlobTable($date);
}
$sql = sprintf($getValuesSql, $table, implode(',', $ids));
$sql = sprintf($getValuesSql, $table, implode(',', $ids));
$dataRows = Db::fetchAll($sql, $bind);
foreach ($dataRows as $row) {
$rows[] = $row;
if ($isNumeric) {
$rows[] = $row;
} else {
$row['value'] = self::uncompress($row['value']);
if ($chunk->isRecordNameAChunk($row['name'])) {
self::moveChunkRowToRows($rows, $row, $chunk, $loadAllSubtables, $idSubtable);
} else {
$rows[] = $row;
}
}
}
}
return $rows;
}
private static function moveChunkRowToRows(&$rows, $row, Chunk $chunk, $loadAllSubtables, $idSubtable)
{
// $blobs = array([subtableID] = [blob of subtableId])
$blobs = unserialize($row['value']);
if (!is_array($blobs)) {
return;
}
// $rawName = eg 'PluginName_ArchiveName'
$rawName = $chunk->getRecordNameWithoutChunkAppendix($row['name']);
if ($loadAllSubtables) {
foreach ($blobs as $subtableId => $blob) {
$row['value'] = $blob;
$row['name'] = self::appendIdSubtable($rawName, $subtableId);
$rows[] = $row;
}
} elseif (array_key_exists($idSubtable, $blobs)) {
$row['value'] = $blobs[$idSubtable];
$row['name'] = self::appendIdSubtable($rawName, $idSubtable);
$rows[] = $row;
}
}
public static function appendIdSubtable($recordName, $id)
{
return $recordName . "_" . $id;
}
private static function uncompress($data)
{
return @gzuncompress($data);
}
/**
* Returns the SQL condition used to find successfully completed archives that
* this instance is querying for.
*
* @param array $plugins
* @param Segment $segment
* @param bool $isSkipAggregationOfSubTables
* @return string
*/
static private function getNameCondition(array $plugins, Segment $segment, $isSkipAggregationOfSubTables)
private static function getNameCondition(array $plugins, Segment $segment)
{
// the flags used to tell how the archiving process for a specific archive was completed,
// if it was completed
$doneFlags = Rules::getDoneFlags($plugins, $segment, $isSkipAggregationOfSubTables);
$doneFlags = Rules::getDoneFlags($plugins, $segment);
$allDoneFlags = "'" . implode("','", $doneFlags) . "'";
$possibleValues = Rules::getSelectableDoneFlagValues();
// create the SQL to find archives that are DONE
return "((name IN ($allDoneFlags)) AND " .
" (value = '" . ArchiveWriter::DONE_OK . "' OR " .
" value = '" . ArchiveWriter::DONE_OK_TEMPORARY . "'))";
}
static public function purgeOutdatedArchives(Date $dateStart)
{
$purgeArchivesOlderThan = Rules::shouldPurgeOutdatedArchives($dateStart);
if (!$purgeArchivesOlderThan) {
return;
}
$idArchivesToDelete = self::getTemporaryArchiveIdsOlderThan($dateStart, $purgeArchivesOlderThan);
if (!empty($idArchivesToDelete)) {
self::deleteArchiveIds($dateStart, $idArchivesToDelete);
}
self::deleteArchivesWithPeriodRange($dateStart);
Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]",
$purgeArchivesOlderThan, $dateStart->toString("Y-m"), implode(',', $idArchivesToDelete));
}
/*
* Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space
*/
protected static function deleteArchivesWithPeriodRange(Date $date)
{
$query = "DELETE FROM %s WHERE period = ? AND ts_archived < ?";
$yesterday = Date::factory('yesterday')->getDateTime();
$bind = array(Piwik::$idPeriods['range'], $yesterday);
$numericTable = ArchiveTableCreator::getNumericTable($date);
Db::query(sprintf($query, $numericTable), $bind);
Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]", $yesterday, $numericTable);
try {
Db::query(sprintf($query, ArchiveTableCreator::getBlobTable($date)), $bind);
} catch (Exception $e) {
// Individual blob tables could be missing
}
}
protected static function deleteArchiveIds(Date $date, $idArchivesToDelete)
{
$query = "DELETE FROM %s WHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ")";
Db::query(sprintf($query, ArchiveTableCreator::getNumericTable($date)));
try {
Db::query(sprintf($query, ArchiveTableCreator::getBlobTable($date)));
} catch (Exception $e) {
// Individual blob tables could be missing
}
}
protected static function getTemporaryArchiveIdsOlderThan(Date $date, $purgeArchivesOlderThan)
{
$query = "SELECT idarchive
FROM " . ArchiveTableCreator::getNumericTable($date) . "
WHERE name LIKE 'done%'
AND (( value = " . ArchiveWriter::DONE_OK_TEMPORARY . "
AND ts_archived < ?)
OR value = " . ArchiveWriter::DONE_ERROR . ")";
$result = Db::fetchAll($query, array($purgeArchivesOlderThan));
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
}
return $idArchivesToDelete;
return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))";
}
}