* Add function for duckdb gen parquet.
This commit is contained in:
宋辰轩
@@ -0,0 +1,118 @@
|
||||
<?php
|
||||
$config->bi = new stdclass();
|
||||
$config->bi->duckSQLTemp = <<<EOT
|
||||
INSTALL mysql;
|
||||
LOAD mysql;
|
||||
ATTACH 'host={HOST} user={USER} password={PASSWORD} port={PORT} database={DATABASE}' as mysqldb(TYPE MYSQL);
|
||||
USE mysqldb;
|
||||
{COPYSQL}
|
||||
EOT;
|
||||
|
||||
$config->bi->duckdb = new stdclass();
|
||||
$config->bi->duckdb->ignoreFields = array();
|
||||
$config->bi->duckdb->ignoreFields['account'] = array('extra');
|
||||
$config->bi->duckdb->ignoreFields['action'] = array('comment', 'extra');
|
||||
$config->bi->duckdb->ignoreFields['actionrecent'] = array('comment', 'extra');
|
||||
$config->bi->duckdb->ignoreFields['approvalrole'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['attend'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['auditplan'] = array('config');
|
||||
$config->bi->duckdb->ignoreFields['auditresult'] = array('comment');
|
||||
$config->bi->duckdb->ignoreFields['block'] = array('params');
|
||||
$config->bi->duckdb->ignoreFields['bug'] = array('entry');
|
||||
$config->bi->duckdb->ignoreFields['build'] = array('stories', 'bugs');
|
||||
$config->bi->duckdb->ignoreFields['case'] = array('precondition');
|
||||
$config->bi->duckdb->ignoreFields['casestep'] = array('desc', 'expect');
|
||||
$config->bi->duckdb->ignoreFields['chart'] = array('desc', 'langs', 'sql');
|
||||
$config->bi->duckdb->ignoreFields['charter'] = array('charterFiles');
|
||||
$config->bi->duckdb->ignoreFields['chart_back'] = array('desc', 'sql');
|
||||
$config->bi->duckdb->ignoreFields['cmcl'] = array('contents');
|
||||
$config->bi->duckdb->ignoreFields['compile'] = array('logs');
|
||||
$config->bi->duckdb->ignoreFields['cron'] = array('command');
|
||||
$config->bi->duckdb->ignoreFields['custom'] = array('value');
|
||||
$config->bi->duckdb->ignoreFields['dataset'] = array('sql');
|
||||
$config->bi->duckdb->ignoreFields['dataview'] = array('sql', 'langs');
|
||||
$config->bi->duckdb->ignoreFields['delivery'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['demandspec'] = array('files');
|
||||
$config->bi->duckdb->ignoreFields['deploy'] = array('members', 'notify', 'cases');
|
||||
$config->bi->duckdb->ignoreFields['deployscope'] = array('hosts', 'remove', 'add');
|
||||
$config->bi->duckdb->ignoreFields['deploystep'] = array('content');
|
||||
$config->bi->duckdb->ignoreFields['design'] = array('commit');
|
||||
$config->bi->duckdb->ignoreFields['dimension'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['doc'] = array('editingDate', 'users', 'editedList');
|
||||
$config->bi->duckdb->ignoreFields['doccontent'] = array('files');
|
||||
$config->bi->duckdb->ignoreFields['doclib'] = array('users', 'collector');
|
||||
$config->bi->duckdb->ignoreFields['effort'] = array('product', 'work', 'extra');
|
||||
$config->bi->duckdb->ignoreFields['expect'] = array('expect', 'progress');
|
||||
$config->bi->duckdb->ignoreFields['extension'] = array('license');
|
||||
$config->bi->duckdb->ignoreFields['faq'] = array('answer');
|
||||
$config->bi->duckdb->ignoreFields['feedback'] = array('desc', 'likes');
|
||||
$config->bi->duckdb->ignoreFields['flow_followrecord'] = array('followdesc');
|
||||
$config->bi->duckdb->ignoreFields['flow_investigation'] = array('objective', 'team', 'method', 'outline', 'report');
|
||||
$config->bi->duckdb->ignoreFields['flow_investigationreport'] = array('content');
|
||||
$config->bi->duckdb->ignoreFields['flow_keyproject'] = array('team', 'riskone', 'solutionone', 'situation', 'desc', 'description');
|
||||
$config->bi->duckdb->ignoreFields['flow_library'] = array('category', 'labels', 'summary');
|
||||
$config->bi->duckdb->ignoreFields['flow_metting'] = array('participant', 'log', 'notes');
|
||||
$config->bi->duckdb->ignoreFields['flow_question'] = array('risk', 'solution');
|
||||
$config->bi->duckdb->ignoreFields['group'] = array('acl');
|
||||
$config->bi->duckdb->ignoreFields['history'] = array('old', 'new');
|
||||
$config->bi->duckdb->ignoreFields['host'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['image'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['instance'] = array('desc', 'ldapSettings', 'dbSettings', 'env');
|
||||
$config->bi->duckdb->ignoreFields['intervention'] = array('partake');
|
||||
$config->bi->duckdb->ignoreFields['issue'] = array('resolutionComment');
|
||||
$config->bi->duckdb->ignoreFields['job'] = array('customParam');
|
||||
$config->bi->duckdb->ignoreFields['kanban'] = array('team', 'whitelist');
|
||||
$config->bi->duckdb->ignoreFields['kanbancard'] = array('assignedTo', 'whitelist');
|
||||
$config->bi->duckdb->ignoreFields['kanbancell'] = array('cards');
|
||||
$config->bi->duckdb->ignoreFields['kanbanspace'] = array('team', 'whitelist');
|
||||
$config->bi->duckdb->ignoreFields['lang'] = array('value');
|
||||
$config->bi->duckdb->ignoreFields['leave'] = array('desc', 'reviewers', 'backReviewers');
|
||||
$config->bi->duckdb->ignoreFields['lieu'] = array('desc', 'reviewers');
|
||||
$config->bi->duckdb->ignoreFields['log'] = array('data', 'result');
|
||||
$config->bi->duckdb->ignoreFields['mailqueue'] = array('body', 'failReason');
|
||||
$config->bi->duckdb->ignoreFields['meeting'] = array('participant', 'minutes');
|
||||
$config->bi->duckdb->ignoreFields['bi'] = array('collector', 'desc', 'definition');
|
||||
$config->bi->duckdb->ignoreFields['module'] = array('collector');
|
||||
$config->bi->duckdb->ignoreFields['mr'] = array('description');
|
||||
$config->bi->duckdb->ignoreFields['mrapproval'] = array('comment');
|
||||
$config->bi->duckdb->ignoreFields['notify'] = array('ccList', 'data', 'failReason', 'toList', 'subject');
|
||||
$config->bi->duckdb->ignoreFields['object'] = array('range');
|
||||
$config->bi->duckdb->ignoreFields['overtime'] = array('desc', 'reviewers');
|
||||
$config->bi->duckdb->ignoreFields['pivot'] = array('name', 'desc');
|
||||
$config->bi->duckdb->ignoreFields['practice'] = array('content');
|
||||
$config->bi->duckdb->ignoreFields['privlang'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['programactivity'] = array('content');
|
||||
$config->bi->duckdb->ignoreFields['programoutput'] = array('content');
|
||||
$config->bi->duckdb->ignoreFields['programprocess'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['programreport'] = array('params', 'content');
|
||||
$config->bi->duckdb->ignoreFields['project'] = array('output');
|
||||
$config->bi->duckdb->ignoreFields['prompt'] = array('desc', 'source', 'purpose', 'elaboration');
|
||||
$config->bi->duckdb->ignoreFields['promptrole'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['queue'] = array('command');
|
||||
$config->bi->duckdb->ignoreFields['reply'] = array('content');
|
||||
$config->bi->duckdb->ignoreFields['repo'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['repohistory'] = array('comment');
|
||||
$config->bi->duckdb->ignoreFields['report'] = array('name', 'sql', 'vars', 'langs', 'params', 'desc');
|
||||
$config->bi->duckdb->ignoreFields['reviewissue'] = array('opinion');
|
||||
$config->bi->duckdb->ignoreFields['reviewresult'] = array('opinion');
|
||||
$config->bi->duckdb->ignoreFields['searchindex'] = array('title', 'content');
|
||||
$config->bi->duckdb->ignoreFields['service'] = array('hosts');
|
||||
$config->bi->duckdb->ignoreFields['solution'] = array('desc', 'components');
|
||||
$config->bi->duckdb->ignoreFields['solutions'] = array('contents', 'support', 'measures');
|
||||
$config->bi->duckdb->ignoreFields['sqlite_queue'] = array('sql');
|
||||
$config->bi->duckdb->ignoreFields['sqlview'] = array('sql', 'desc');
|
||||
$config->bi->duckdb->ignoreFields['storyspec'] = array('files');
|
||||
$config->bi->duckdb->ignoreFields['testresult'] = array('stepResults', 'ZTFResult', 'xml');
|
||||
$config->bi->duckdb->ignoreFields['thread'] = array('content');
|
||||
$config->bi->duckdb->ignoreFields['ticket'] = array('desc', 'resolution');
|
||||
$config->bi->duckdb->ignoreFields['trainbook'] = array('skill', 'summary', 'category2', 'labels', 'location');
|
||||
$config->bi->duckdb->ignoreFields['traincontents'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['trainexamdata'] = array('answer');
|
||||
$config->bi->duckdb->ignoreFields['trainexamuser'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['trainplan'] = array('trainee');
|
||||
$config->bi->duckdb->ignoreFields['trip'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['user'] = array('avatar', 'nature', 'analysis', 'strategy');
|
||||
$config->bi->duckdb->ignoreFields['userquery'] = array('form', 'sql');
|
||||
$config->bi->duckdb->ignoreFields['usertpl'] = array('content');
|
||||
$config->bi->duckdb->ignoreFields['webapp'] = array('desc');
|
||||
$config->bi->duckdb->ignoreFields['webhook'] = array('desc');
|
||||
@@ -0,0 +1,74 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* The control file of branch of ZenTaoPMS.
|
||||
*
|
||||
* @copyright Copyright 2009-2024 禅道软件（青岛）有限公司(ZenTao Software (Qingdao) Co., Ltd. www.cnezsoft.com)
|
||||
* @license ZPL(http://zpl.pub/page/zplv12.html) or AGPL(https://www.gnu.org/licenses/agpl-3.0.en.html)
|
||||
* @author Chenxuan Song <songchenxuan@cnezsoft.com>
|
||||
* @package bi
|
||||
* @version $Id$
|
||||
* @link https://www.zentao.net
|
||||
*/
|
||||
class bi extends control
|
||||
{
|
||||
/**
|
||||
* 使用 duckDB 生成最新的 parquet 文件。
|
||||
* Use duckDB gen lastest parquet.
|
||||
*
|
||||
* @access public
|
||||
* @return void
|
||||
*/
|
||||
public function genParquetFile()
|
||||
{
|
||||
/* 检查 duckdb 可执行文件是否存在 */
|
||||
$duckFile = $this->app->getBasePath() . 'bin' . DS . 'duck' . DS . 'duckdb';
|
||||
if(!file_exists($duckFile)) return;
|
||||
if(is_executable($duckFile) && chmod($duckFile, 0777)) return;
|
||||
|
||||
/* 检查 duckdb 存放 parquet 文件的文件夹是否存在 */
|
||||
$duckTmpPath = $this->app->getTmpRoot() . 'duckdb';
|
||||
if(!is_dir($duckTmpPath) && !mkdir($duckTmpPath)) return;
|
||||
|
||||
$tables = $this->dao->query('SHOW TABLES')->fetchAll();
|
||||
if($tables) $tables = array_map(function($obj) { return $obj->Tables_in_maxmaster; }, $tables);
|
||||
|
||||
$copySQL = '';
|
||||
foreach($tables as $table)
|
||||
{
|
||||
$tablePath = $duckTmpPath . DS . $table;
|
||||
$copySQL .= "COPY $table TO '$tablePath.parquet';\n";
|
||||
}
|
||||
|
||||
$sqlContent = $this->config->bi->duckSQLTemp;
|
||||
$dbConfig = $this->config->db;
|
||||
$variables = array(
|
||||
'{DATABASE}' => $dbConfig->name,
|
||||
'{USER}' => $dbConfig->user,
|
||||
'{PASSWORD}' => $dbConfig->password,
|
||||
'{HOST}' => $dbConfig->host,
|
||||
'{PORT}' => $dbConfig->port,
|
||||
'{COPYSQL}' => $copySQL
|
||||
);
|
||||
|
||||
foreach($variables as $key => $value)
|
||||
{
|
||||
$sqlContent = str_replace($key, $value, $sqlContent);
|
||||
}
|
||||
|
||||
$runSQLPath = $this->app->getTmpRoot() . 'duckdb' . DS . 'run.sql';
|
||||
$result = file_put_contents($runSQLPath, $sqlContent);
|
||||
if($result === false) return;
|
||||
|
||||
$exec = "$duckFile :memory: < $runSQLPath 2>&1";
|
||||
$output = shell_exec($exec);
|
||||
|
||||
if(empty($output))
|
||||
{
|
||||
a('success');
|
||||
return;
|
||||
}
|
||||
|
||||
a($output);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user