diff --git a/module/bi/config.php b/module/bi/config.php index 811abf82ff..34a330524f 100644 --- a/module/bi/config.php +++ b/module/bi/config.php @@ -14,6 +14,8 @@ $config->bi->conditionList['NOT IN'] = 'NOT IN'; $config->bi->conditionList['IS NOT NULL'] = 'IS NOT NULL'; $config->bi->conditionList['IS NULL'] = 'IS NULL'; +$config->bi->duckdbExcludeTables = array('metriclib', 'repofiles', 'repohistory', 'queue', 'duckdbqueue', ''); + $config->bi->builtin = new stdclass(); $config->bi->builtin->modules = new stdclass(); $config->bi->builtin->screens = array(1, 2, 3, 4, 5, 6, 7, 8,1001); diff --git a/module/bi/model.php b/module/bi/model.php index 008745ff6b..0e74021fe2 100644 --- a/module/bi/model.php +++ b/module/bi/model.php @@ -1127,6 +1127,80 @@ class biModel extends model return $duckdbTmpPath; } + /** + * Get sql by month. + * + * @param string $month + * @access public + * @return array + */ + public function getSqlByMonth($year = 'Y', $month = 'm') + { + $sqls = array(); + $prefix = $this->config->db->prefix; + $year = date($year); + $month = date($month); + + $begin = date("{$year}-{$month}-01 00:00:00"); + $end = date("{$year}-{$month}-t 23:59:59", strtotime("$year-$month-01")); + $sqls[$prefix . "action_{$year}_{$month}"] = "select * from zt_action where date >= TIMESTAMP '$begin' and date <= TIMESTAMP '$end'"; + + return $sqls; + } + + /** + * Get action sync sql. + * + * @param string $range + * @access public + * @return array + */ + public function getActionSyncSql($range = 'current') + { + if($range == 'current') return $this->getSqlByMonth(); + + $actionDate = $this->biTao->fetchActionDate(); + $begin = new DateTime($actionDate->minDate); + $end = new DateTime($actionDate->maxDate); + + $sqls = array(); + while($begin <= $end) + { + $year = $begin->format('Y'); + $month = $begin->format('m'); + $sqls += $this->getSqlByMonth($year, $month); + $begin->modify('+1 month'); + } + + return $sqls; + } + + /** + * Init parquet. + * + * @access public + * @return void + */ + public function initParquet() + { + $duckdb = $this->getDuckDBPath(); + if(!$duckdb) return $this->lang->bi->binNotExists; + + $duckdbTmpPath = $this->getDuckDBTmpDir(); + if(!$duckdbTmpPath) return sprintf($this->lang->bi->tmpPermissionDenied, $this->getDuckDBTmpDir(true), $this->getDuckDBTmpDir(true)); + + $tables = $this->biTao->fetchAllTables(); + $copySQLs = array(); + foreach($tables as $table) $copySQLs[] = "copy (select * from {$table}) to '{$duckdbTmpPath}{$table}.parquet'"; + + $copySQL = implode(';', $copySQLs); + if(empty($copySQL)) return true; + + $command = $this->prepareSyncCommand($duckdb->bin, $duckdb->extension, $copySQL); + $output = shell_exec($command); + $this->saveLogs("Sync command: $command"); + } + /** * 准备同步数据库所需的复制SQL。 * Prepare copy SQL for sync. @@ -1137,26 +1211,14 @@ class biModel extends model */ public function prepareCopySQL($duckdbTmpPath) { - $duckdbQueue = $this->dao->select('*')->from(TABLE_DUCKDBQUEUE) - ->where('updatedTime >= syncTime') - ->orWhere('syncTime IS NULL') - ->fetchAll('object'); + $tables = $this->biTao->fetchTableQueue(); - if(empty($duckdbQueue)) return ''; + if(empty($tables)) return ''; - $tables = array_keys($duckdbQueue); + $copySQLs = array(); + foreach($tables as $table) $copySQLs[] = "copy (select * from {$table}) to '{$duckdbTmpPath}{$table}.parquet'"; - $copySQLs = array(); - foreach($tables as $table) - { - $sql = "select * from {$table}"; - $copySQLs[] = "copy ($sql) to '{$duckdbTmpPath}{$table}.parquet'"; - } - - $this->dao->update(TABLE_DUCKDBQUEUE) - ->set('syncTime')->eq(helper::now()) - ->where('object')->in($tables) - ->exec(); + $this->biTao->updateSyncTime($tables); return implode(';', $copySQLs); } diff --git a/module/bi/tao.php b/module/bi/tao.php index c3d59a696b..c7a9346059 100644 --- a/module/bi/tao.php +++ b/module/bi/tao.php @@ -35,10 +35,10 @@ class biTao extends biModel foreach($excludes as $index => $table) $excludes[$index] = $prefix . $table; return $this->dao->select('object')->from(TABLE_DUCKDBQUEUE) - ->where('updatedTime >= syncTime', true) + ->where('object')->notin($excludes) + ->andWhere('updatedTime >= syncTime', true) ->orWhere('syncTime IS NULL') ->markRight(1) - ->andWhere('object')->notin($excludes) ->fetchPairs(); }