Design/EasySoft-ZenTaoPMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 30 Wiki Activity

* Add explain and desc function for duckdb.

Browse Source
This commit is contained in:
宋辰轩
2024-05-14 14:56:29 +08:00
parent f761143481
commit fb45b947b7
8 changed files with 180 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
framework/router.class.php
+2 -1
View File
@@ -475,7 +475,8 @@ class router extends baseRouter
{
$this->loadClass('duckdb');
$duckdb = new duckdb();
return $dudkdb;
return $duckdb;
}
}
lib/duckdb/duckdb.class.php
+30 -2
View File
@@ -122,7 +122,7 @@ class duckdb
* @access public
* @return this.
*/
public function query($sql = '')
public function query($sql = '', $func = '')
{
/* $0全量匹配以 prefix 开头的表，替换为对应的 parquet 文件。 */
$pattern = "/{$this->prefix}\S+/";
@@ -130,9 +130,34 @@ class duckdb
$this->sql = preg_replace($pattern, $replacement, $sql);
if($func && method_exists($this, $func) && in_array($func, array('explain', 'desc'))) $this->$func();
return $this;
}
/**
* 设置前缀，获取一个查询的执行计划。
* Explain analysis sql.
*
* @access public
* @return this.
*/
private function explain()
{
$this->sql = 'PRAGMA enable_profiling=json; EXPLAIN ANALYZE ' . $this->sql;
}
/**
* 设置前缀，获取一个查询的执行计划。
* Explain analysis sql.
*
* @access public
* @return this.
*/
private function desc()
{
$this->sql = 'DESCRIBE ' . $this->sql;
}
/**
* 替换sql语句中的表为parquet文件路径。
* Replace sql table to parquet file path.
@@ -145,6 +170,7 @@ class duckdb
$exec = "$this->binPath :memory: \"$this->sql\" -json 2>&1";
$output = shell_exec($exec);
a($this->sql);die;
$rows = json_decode($output);
/* 有内容但是 json 解析失败，说明是报错。*/
if($output and !$rows)
@@ -157,6 +183,8 @@ class duckdb
}
}
//-------------------- Fetch相关方法(Fetch related methods) -------------------//
/**
* 获取一个记录。
* Fetch one record.
@@ -173,7 +201,7 @@ class duckdb
/**
* 获取所有记录。
* Fetch all records.
*
*7dbf3798764
* @access public
* @return array
*/
module/bi/config.php
+70
View File
@@ -8,6 +8,76 @@ USE mysqldb;
{COPYSQL}
EOT;
$config->bi->driverNames = array('mysql', 'duckdb');
$config->bi->columnTypes = new stdclass();
$config->bi->columnTypes->mysql['TINY'] = 'number';
$config->bi->columnTypes->mysql['SHORT'] = 'number';
$config->bi->columnTypes->mysql['LONG'] = 'number';
$config->bi->columnTypes->mysql['FLOAT'] = 'number';
$config->bi->columnTypes->mysql['DOUBLE'] = 'number';
$config->bi->columnTypes->mysql['TIMESTAMP'] = 'string';
$config->bi->columnTypes->mysql['LONGLONG'] = 'string';
$config->bi->columnTypes->mysql['INT24'] = 'number';
$config->bi->columnTypes->mysql['DATE'] = 'date';
$config->bi->columnTypes->mysql['TIME'] = 'string';
$config->bi->columnTypes->mysql['DATETIME'] = 'date';
$config->bi->columnTypes->mysql['YEAR'] = 'date';
$config->bi->columnTypes->mysql['ENUM'] = 'string';
$config->bi->columnTypes->mysql['SET'] = 'string';
$config->bi->columnTypes->mysql['TINYBLOB'] = 'string';
$config->bi->columnTypes->mysql['MEDIUMBLOB'] = 'string';
$config->bi->columnTypes->mysql['LONG_BLOB'] = 'string';
$config->bi->columnTypes->mysql['BLOB'] = 'string';
$config->bi->columnTypes->mysql['VAR_STRING'] = 'string';
$config->bi->columnTypes->mysql['STRING'] = 'string';
$config->bi->columnTypes->mysql['NULL'] = 'null';
$config->bi->columnTypes->mysql['NEWDATE'] = 'date';
$config->bi->columnTypes->mysql['INTERVAL'] = 'string';
$config->bi->columnTypes->mysql['GEOMETRY'] = 'string';
$config->bi->columnTypes->mysql['NEWDECIMAL'] = 'number';
/* Dameng native_type. */
$config->bi->columnTypes->mysql['int'] = 'number';
$config->bi->columnTypes->mysql['varchar'] = 'string';
$config->bi->columnTypes->mysql['text'] = 'string';
$config->bi->columnTypes->mysql['timestamp'] = 'string';
$config->bi->columnTypes->mysql['date'] = 'date';
$config->bi->columnTypes->mysql['time'] = 'string';
$config->bi->columnTypes->mysql['double'] = 'number';
$config->bi->columnTypes->mysql['number'] = 'number';
$config->bi->columnTypes->mysql['bigint'] = 'number';
/* DuckDB native_type. */
$config->bi->columnTypes->duckdb['BIGINT'] = 'number';
$config->bi->columnTypes->duckdb['UBIGINT'] = 'number';
$config->bi->columnTypes->duckdb['HUGEINT'] = 'number';
$config->bi->columnTypes->duckdb['UHUGEINT'] = 'number';
$config->bi->columnTypes->duckdb['INTEGER'] = 'number';
$config->bi->columnTypes->duckdb['UINTEGER'] = 'number';
$config->bi->columnTypes->duckdb['SMALLINT'] = 'number';
$config->bi->columnTypes->duckdb['USMALLINT'] = 'number';
$config->bi->columnTypes->duckdb['TINYINT'] = 'number';
$config->bi->columnTypes->duckdb['UTINYINT'] = 'number';
$config->bi->columnTypes->duckdb['REAL'] = 'number';
$config->bi->columnTypes->duckdb['DOUBLE'] = 'number';
$config->bi->columnTypes->duckdb['BOOLEAN'] = 'number';
$config->bi->columnTypes->duckdb['BLOB'] = 'string';
$config->bi->columnTypes->duckdb['VARCHAR'] = 'string';
$config->bi->columnTypes->duckdb['DATE'] = 'date';
$config->bi->columnTypes->duckdb['TIMESTAMP'] = 'date';
$config->bi->columnTypes->duckdb['TIME'] = 'date';
$config->bi->columnTypes->INTEGER = 'number';
$config->bi->columnTypes->UINTEGER = 'number';
$config->bi->columnTypes->UTINYINT = 'number';
$config->bi->columnTypes->SMALLINT = 'number';
$config->bi->columnTypes->FLOAT = 'number';
$config->bi->columnTypes->BOOLEAN = 'number';
$config->bi->columnTypes->VARCHAR = 'string';
$config->bi->columnTypes->TIMESTAMP = 'date';
$config->bi->columnTypes->DATE = 'date';
$config->bi->duckdb = new stdclass();
$config->bi->duckdb->tables = array();
$config->bi->duckdb->tables['action'] = <<<EOT
module/bi/model.php
+64
View File
@@ -48,6 +48,70 @@ class biModel extends model
return array('tables' => array_unique($tables), 'fields' => $fields);
}
/**
* Get sql result columns.
*
* @param string $sql
* @param string $driverName mysql|duckdb
* @access public
* @return array|false
*/
public function getColumns(string $sql, $driverName = 'mysql'): array|false
{
if(!in_array($driverName, $this->config->bi->driverNames)) return false;
if($driverName == 'mysql')
{
$columns = $this->dao->getColumns($sql);
}
else
{
$dbh = $this->app->loadDriver('duckdb');
$columns = $dbh->query("$sql", 'desc')->fetchAll();
}
$result = array();
foreach($columns as $column)
{
$column = (array)$column;
$name = $driverName == 'mysql' ? $column['name'] : $column['column_name'];
$nativeType = $driverName == 'mysql' ? $column['native_type'] : $column['column_type'];
$result[$name] = array('name' => $name, 'native_type' => $nativeType);
}
return $result;
}
/**
* 获取表的字段类型。
* Get table data.
*
* @param string $sql
* @param string $driverName mysql|duckdb
* @param array $columns
* @access public
* @return object
*/
public function getColumnsType(string $sql, string $driverName = 'mysql', array $columns = array()): object
{
if(empty($columns)) $columns = $this->getColumns($sql, $driverName);
$columnTypes = new stdclass();
foreach($columns as $column)
{
$field = $column['name'];
$nativeType = $column['native_type'];
$type = $this->config->bi->columnTypes->$driverName[$nativeType];
if(isset($columnTypes->$field)) $field = $column['table'] . $field;
$columnTypes->$field = $type;
}
return $columnTypes;
}
/**
* Get object options.
*
module/dataview/config.php
-38
View File
@@ -12,41 +12,3 @@ $config->dataview->groupChild['company'] = array('user' => 'user,usercontact,use
$config->dataview->multipleMappingFields = array();
$config->dataview->multipleMappingFields[] = 'testcase-stage';
$config->dataview->columnTypes = new stdclass();
$config->dataview->columnTypes->TINY = 'number';
$config->dataview->columnTypes->SHORT = 'number';
$config->dataview->columnTypes->LONG = 'number';
$config->dataview->columnTypes->FLOAT = 'number';
$config->dataview->columnTypes->DOUBLE = 'number';
$config->dataview->columnTypes->TIMESTAMP = 'string';
$config->dataview->columnTypes->LONGLONG = 'string';
$config->dataview->columnTypes->INT24 = 'number';
$config->dataview->columnTypes->DATE = 'date';
$config->dataview->columnTypes->TIME = 'string';
$config->dataview->columnTypes->DATETIME = 'date';
$config->dataview->columnTypes->YEAR = 'date';
$config->dataview->columnTypes->ENUM = 'string';
$config->dataview->columnTypes->SET = 'string';
$config->dataview->columnTypes->TINYBLOB = 'string';
$config->dataview->columnTypes->MEDIUMBLOB = 'string';
$config->dataview->columnTypes->LONG_BLOB = 'string';
$config->dataview->columnTypes->BLOB = 'string';
$config->dataview->columnTypes->VAR_STRING = 'string';
$config->dataview->columnTypes->STRING = 'string';
$config->dataview->columnTypes->NULL = 'null';
$config->dataview->columnTypes->NEWDATE = 'date';
$config->dataview->columnTypes->INTERVAL = 'string';
$config->dataview->columnTypes->GEOMETRY = 'string';
$config->dataview->columnTypes->NEWDECIMAL = 'number';
/* Dameng native_type. */
$config->dataview->columnTypes->int = 'number';
$config->dataview->columnTypes->varchar = 'string';
$config->dataview->columnTypes->text = 'string';
$config->dataview->columnTypes->timestamp = 'string';
$config->dataview->columnTypes->date = 'date';
$config->dataview->columnTypes->time = 'string';
$config->dataview->columnTypes->double = 'number';
$config->dataview->columnTypes->number = 'number';
$config->dataview->columnTypes->bigint = 'number';
module/dataview/model.php
+11 -38
View File
@@ -22,7 +22,7 @@ class dataviewModel extends model
public function __construct()
{
parent::__construct();
$this->loadBIDAO();
$this->loadModel('bi');
}
@@ -229,61 +229,34 @@ class dataviewModel extends model
return array($mergeFields, $relatedObject);
}
/**
* 获取表的字段类型。
* Get table data.
*
* @param string $sql
* @access public
* @return object
*/
public function getColumns(string $sql): object
{
$columns = $this->dao->getColumns($sql);
$columnTypes = new stdclass();
foreach($columns as $column)
{
$field = $column['name'];
$nativeType = $column['native_type'];
$type = $this->config->dataview->columnTypes->$nativeType;
if(isset($columnTypes->$field)) $field = $column['table'] . $field;
$columnTypes->$field = $type;
}
return $columnTypes;
}
/**
* 检查查询结果的唯一性。
* Check that the column of an sql query is unique.
*
* @param string $sql
* @param string $driverName mysql|duckdb
* @param bool $repeat
* @param array $columns
* @access public
* @return bool|array
*/
public function checkUniColumn(string $sql, bool $repeat = false): bool|array
public function checkUniColumn(string $sql, string $driverName = 'mysql', bool $repeat = false, $columns = array()): bool|array
{
$columns = $this->dao->getColumns($sql);
if(empty($columns)) $columns = $this->bi->getColumns($sql, $driverName);
$isUnique = true;
$columnList = array();
$repeatColumn = array();
foreach($columns as $column)
$repeatFields = array();
$fieldCounts = array_count_values(array_column($columns, 'name'));
foreach($fieldCounts as $field => $value)
{
$field = $column['name'];
if(isset($columnTypes[$field]))
if($value > 1)
{
$isUnique = false;
$repeatColumn[$field] = $field;
$repeatFields[$field] = $field;
}
$columnTypes[$field] = $field;
}
if($repeat) return array($isUnique, $repeatColumn);
if($repeat) return array($isUnique, $repeatFields);
return $isUnique;
}
module/pivot/model.php
+1 -1
View File
@@ -202,7 +202,7 @@ class pivotModel extends model
$stmt = $this->dbh->query($querySQL);
if(!$stmt) return;
$columns = $this->loadModel('dataview')->getColumns($querySQL);
$columns = $this->bi->getColumnsType($querySQL);
$columnFields = array();
foreach(array_keys(get_object_vars($columns)) as $type) $columnFields[$type] = $type;
module/upgrade/model.php
+2 -1
View File
@@ -7821,10 +7821,11 @@ class upgradeModel extends model
{
if(!$sql) return array();
$this->loadModel('bi');
$this->loadModel('dataview');
$this->loadModel('chart');
$columns = $this->dataview->getColumns($sql);
$columns = $this->bi->getColumnsType($sql);
$columnFields = array();
foreach($columns as $column => $type) $columnFields[$column] = $column;