coderkun/questlab
1
0
Fork 0
Code Issues 15 Pull requests Projects Releases Wiki Activity

update Piwik to version 2.16 (fixes #91)

Browse source
This commit is contained in:
oliver 2016-04-10 18:55:57 +02:00
commit d885a4baa9
5833 changed files with 418860 additions and 226988 deletions
Download patch file Download diff file Expand all files Collapse all files

209
www/analytics/core/DataAccess/ArchiveWriter.php
View file

@ -1,6 +1,6 @@
<?php
/**
* Piwik - Open source web analytics
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
@ -9,17 +9,13 @@
namespace Piwik\DataAccess;
use Exception;
use Piwik\Archive;
use Piwik\Archive\Chunk;
use Piwik\ArchiveProcessor\Rules;
use Piwik\ArchiveProcessor;
use Piwik\Common;
use Piwik\Config;
use Piwik\Db;
use Piwik\Db\BatchInsert;
use Piwik\Log;
use Piwik\Period;
use Piwik\Segment;
use Piwik\SettingsPiwik;
/**
* This class is used to create a new Archive.
@ -28,7 +24,6 @@ use Piwik\SettingsPiwik;
*/
class ArchiveWriter
{
const PREFIX_SQL_LOCK = "locked_";
/**
* Flag stored at the end of the archiving
*
@ -50,23 +45,31 @@ class ArchiveWriter
*/
const DONE_OK_TEMPORARY = 3;
/**
* Flag indicated that archive is done but was marked as invalid later and needs to be re-processed during next archiving process
*
* @var int
*/
const DONE_INVALIDATED = 4;
protected $fields = array('idarchive',
'idsite',
'date1',
'date2',
'period',
'ts_archived',
'name',
'value');
'idsite',
'date1',
'date2',
'period',
'ts_archived',
'name',
'value');
public function __construct(ArchiveProcessor\Parameters $params, $isArchiveTemporary)
{
$this->idArchive = false;
$this->idSite = $params->getSite()->getId();
$this->segment = $params->getSegment();
$this->period = $params->getPeriod();
$this->idSite = $params->getSite()->getId();
$this->segment = $params->getSegment();
$this->period = $params->getPeriod();
$idSites = array($this->idSite);
$this->doneFlag = Rules::getDoneStringFlagFor($idSites, $this->segment, $this->period->getLabel(), $params->getRequestedPlugin(), $params->isSkipAggregationOfSubTables());
$this->doneFlag = Rules::getDoneStringFlagFor($idSites, $this->segment, $this->period->getLabel(), $params->getRequestedPlugin());
$this->isArchiveTemporary = $isArchiveTemporary;
$this->dateStart = $this->period->getDateStart();
@ -74,25 +77,32 @@ class ArchiveWriter
/**
* @param string $name
* @param string[] $values
* @param string|string[] $values A blob string or an array of blob strings. If an array
* is used, the first element in the array will be inserted
* with the `$name` name. The others will be splitted into chunks. All subtables
* within one chunk will be serialized as an array where the index is the
* subtableId.
*/
public function insertBlobRecord($name, $values)
{
if (is_array($values)) {
$clean = array();
foreach ($values as $id => $value) {
// for the parent Table we keep the name
// for example for the Table of searchEngines we keep the name 'referrer_search_engine'
// but for the child table of 'Google' which has the ID = 9 the name would be 'referrer_search_engine_9'
$newName = $name;
if ($id != 0) {
//FIXMEA: refactor
$newName = $name . '_' . $id;
}
$value = $this->compress($value);
$clean[] = array($newName, $value);
if (isset($values[0])) {
// we always store the root table in a single blob for fast access
$clean[] = array($name, $this->compress($values[0]));
unset($values[0]);
}
if (!empty($values)) {
// we move all subtables into chunks
$chunk = new Chunk();
$chunks = $chunk->moveArchiveBlobsIntoChunks($name, $values);
foreach ($chunks as $index => $subtables) {
$clean[] = array($index, $this->compress(serialize($subtables)));
}
}
$this->insertBulkRecords($clean);
return;
}
@ -106,6 +116,7 @@ class ArchiveWriter
if ($this->idArchive === false) {
throw new Exception("Must call allocateNewArchiveId() first");
}
return $this->idArchive;
}
@ -117,108 +128,49 @@ class ArchiveWriter
public function finalizeArchive()
{
$this->deletePreviousArchiveStatus();
$numericTable = $this->getTableNumeric();
$idArchive = $this->getIdArchive();
$this->getModel()->deletePreviousArchiveStatus($numericTable, $idArchive, $this->doneFlag);
$this->logArchiveStatusAsFinal();
}
static protected function compress($data)
protected function compress($data)
{
if (Db::get()->hasBlobDataType()) {
return gzcompress($data);
}
return $data;
}
protected function getArchiveLockName()
{
$numericTable = $this->getTableNumeric();
$dbLockName = "allocateNewArchiveId.$numericTable";
return $dbLockName;
}
protected function acquireArchiveTableLock()
{
$dbLockName = $this->getArchiveLockName();
if (Db::getDbLock($dbLockName, $maxRetries = 30) === false) {
throw new Exception("allocateNewArchiveId: Cannot get named lock $dbLockName.");
}
}
protected function releaseArchiveTableLock()
{
$dbLockName = $this->getArchiveLockName();
Db::releaseDbLock($dbLockName);
}
protected function allocateNewArchiveId()
{
$this->idArchive = $this->insertNewArchiveId();
$numericTable = $this->getTableNumeric();
$this->idArchive = $this->getModel()->allocateNewArchiveId($numericTable);
return $this->idArchive;
}
/**
* Locks the archive table to generate a new archive ID.
*
* We lock to make sure that
* if several archiving processes are running at the same time (for different websites and/or periods)
* then they will each use a unique archive ID.
*
* @return int
*/
protected function insertNewArchiveId()
private function getModel()
{
$numericTable = $this->getTableNumeric();
$idSite = $this->idSite;
$this->acquireArchiveTableLock();
$locked = self::PREFIX_SQL_LOCK . Common::generateUniqId();
$date = date("Y-m-d H:i:s");
$insertSql = "INSERT INTO $numericTable "
. " SELECT IFNULL( MAX(idarchive), 0 ) + 1,
'" . $locked . "',
" . (int)$idSite . ",
'" . $date . "',
'" . $date . "',
0,
'" . $date . "',
0 "
. " FROM $numericTable as tb1";
Db::get()->exec($insertSql);
$this->releaseArchiveTableLock();
$selectIdSql = "SELECT idarchive FROM $numericTable WHERE name = ? LIMIT 1";
$id = Db::get()->fetchOne($selectIdSql, $locked);
return $id;
return new Model();
}
protected function logArchiveStatusAsIncomplete()
{
$statusWhileProcessing = self::DONE_ERROR;
$this->insertRecord($this->doneFlag, $statusWhileProcessing);
}
protected function deletePreviousArchiveStatus()
{
// without advisory lock here, the DELETE would acquire Exclusive Lock
$this->acquireArchiveTableLock();
Db::query("DELETE FROM " . $this->getTableNumeric() . "
WHERE idarchive = ? AND (name = '" . $this->doneFlag
. "' OR name LIKE '" . self::PREFIX_SQL_LOCK . "%')",
array($this->getIdArchive())
);
$this->releaseArchiveTableLock();
$this->insertRecord($this->doneFlag, self::DONE_ERROR);
}
protected function logArchiveStatusAsFinal()
{
$status = self::DONE_OK;
if ($this->isArchiveTemporary) {
$status = self::DONE_OK_TEMPORARY;
}
$this->insertRecord($this->doneFlag, $status);
}
@ -231,27 +183,37 @@ class ArchiveWriter
foreach ($records as $record) {
$this->insertRecord($record[0], $record[1]);
}
return true;
}
$bindSql = $this->getInsertRecordBind();
$values = array();
$values = array();
$valueSeen = false;
foreach ($records as $record) {
// don't record zero
if (empty($record[1])) continue;
if (empty($record[1])) {
continue;
}
$bind = $bindSql;
$bind[] = $record[0]; // name
$bind[] = $record[1]; // value
$bind = $bindSql;
$bind[] = $record[0]; // name
$bind[] = $record[1]; // value
$values[] = $bind;
$valueSeen = $record[1];
}
if (empty($values)) return true;
if (empty($values)) {
return true;
}
$tableName = $this->getTableNameToInsert($valueSeen);
BatchInsert::tableInsertBatch($tableName, $this->getInsertFields(), $values);
$fields = $this->getInsertFields();
BatchInsert::tableInsertBatch($tableName, $fields, $values, $throwException = false, $charset = 'latin1');
return true;
}
@ -270,26 +232,22 @@ class ArchiveWriter
}
$tableName = $this->getTableNameToInsert($value);
$fields = $this->getInsertFields();
$record = $this->getInsertRecordBind();
$this->getModel()->insertRecord($tableName, $fields, $record, $name, $value);
// duplicate idarchives are Ignored, see http://dev.piwik.org/trac/ticket/987
$query = "INSERT IGNORE INTO " . $tableName . "
(" . implode(", ", $this->getInsertFields()) . ")
VALUES (?,?,?,?,?,?,?,?)";
$bindSql = $this->getInsertRecordBind();
$bindSql[] = $name;
$bindSql[] = $value;
Db::query($query, $bindSql);
return true;
}
protected function getInsertRecordBind()
{
return array($this->getIdArchive(),
$this->idSite,
$this->dateStart->toString('Y-m-d'),
$this->period->getDateEnd()->toString('Y-m-d'),
$this->period->getId(),
date("Y-m-d H:i:s"));
$this->idSite,
$this->dateStart->toString('Y-m-d'),
$this->period->getDateEnd()->toString('Y-m-d'),
$this->period->getId(),
date("Y-m-d H:i:s"));
}
protected function getTableNameToInsert($value)
@ -297,6 +255,7 @@ class ArchiveWriter
if (is_numeric($value)) {
return $this->getTableNumeric();
}
return ArchiveTableCreator::getBlobTable($this->dateStart);
}