From fb45b947b72efc33ab932cccfeb318eaa662bded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=8B=E8=BE=B0=E8=BD=A9?= Date: Tue, 14 May 2024 14:56:29 +0800 Subject: [PATCH] * Add explain and desc function for duckdb. --- framework/router.class.php | 3 +- lib/duckdb/duckdb.class.php | 32 +++++++++++++++-- module/bi/config.php | 70 +++++++++++++++++++++++++++++++++++++ module/bi/model.php | 64 +++++++++++++++++++++++++++++++++ module/dataview/config.php | 38 -------------------- module/dataview/model.php | 49 ++++++-------------------- module/pivot/model.php | 2 +- module/upgrade/model.php | 3 +- 8 files changed, 180 insertions(+), 81 deletions(-) diff --git a/framework/router.class.php b/framework/router.class.php index db5f6ce988..ce7de05d0d 100755 --- a/framework/router.class.php +++ b/framework/router.class.php @@ -475,7 +475,8 @@ class router extends baseRouter { $this->loadClass('duckdb'); $duckdb = new duckdb(); - return $dudkdb; + + return $duckdb; } } diff --git a/lib/duckdb/duckdb.class.php b/lib/duckdb/duckdb.class.php index 516bc72e48..6900520701 100644 --- a/lib/duckdb/duckdb.class.php +++ b/lib/duckdb/duckdb.class.php @@ -122,7 +122,7 @@ class duckdb * @access public * @return this. */ - public function query($sql = '') + public function query($sql = '', $func = '') { /* $0全量匹配以 prefix 开头的表，替换为对应的 parquet 文件。 */ $pattern = "/{$this->prefix}\S+/"; @@ -130,9 +130,34 @@ class duckdb $this->sql = preg_replace($pattern, $replacement, $sql); + if($func && method_exists($this, $func) && in_array($func, array('explain', 'desc'))) $this->$func(); return $this; } + /** + * 设置前缀，获取一个查询的执行计划。 + * Explain analysis sql. + * + * @access public + * @return this. + */ + private function explain() + { + $this->sql = 'PRAGMA enable_profiling=json; EXPLAIN ANALYZE ' . $this->sql; + } + + /** + * 设置前缀，获取一个查询的执行计划。 + * Explain analysis sql. + * + * @access public + * @return this. + */ + private function desc() + { + $this->sql = 'DESCRIBE ' . $this->sql; + } + /** * 替换sql语句中的表为parquet文件路径。 * Replace sql table to parquet file path. @@ -145,6 +170,7 @@ class duckdb $exec = "$this->binPath :memory: \"$this->sql\" -json 2>&1"; $output = shell_exec($exec); + a($this->sql);die; $rows = json_decode($output); /* 有内容但是 json 解析失败，说明是报错。*/ if($output and !$rows) @@ -157,6 +183,8 @@ class duckdb } } + //-------------------- Fetch相关方法(Fetch related methods) -------------------// + /** * 获取一个记录。 * Fetch one record. @@ -173,7 +201,7 @@ class duckdb /** * 获取所有记录。 * Fetch all records. - * + *7dbf3798764 * @access public * @return array */ diff --git a/module/bi/config.php b/module/bi/config.php index 5058ebce68..cc8116343f 100644 --- a/module/bi/config.php +++ b/module/bi/config.php @@ -8,6 +8,76 @@ USE mysqldb; {COPYSQL} EOT; +$config->bi->driverNames = array('mysql', 'duckdb'); + +$config->bi->columnTypes = new stdclass(); +$config->bi->columnTypes->mysql['TINY'] = 'number'; +$config->bi->columnTypes->mysql['SHORT'] = 'number'; +$config->bi->columnTypes->mysql['LONG'] = 'number'; +$config->bi->columnTypes->mysql['FLOAT'] = 'number'; +$config->bi->columnTypes->mysql['DOUBLE'] = 'number'; +$config->bi->columnTypes->mysql['TIMESTAMP'] = 'string'; +$config->bi->columnTypes->mysql['LONGLONG'] = 'string'; +$config->bi->columnTypes->mysql['INT24'] = 'number'; +$config->bi->columnTypes->mysql['DATE'] = 'date'; +$config->bi->columnTypes->mysql['TIME'] = 'string'; +$config->bi->columnTypes->mysql['DATETIME'] = 'date'; +$config->bi->columnTypes->mysql['YEAR'] = 'date'; +$config->bi->columnTypes->mysql['ENUM'] = 'string'; +$config->bi->columnTypes->mysql['SET'] = 'string'; +$config->bi->columnTypes->mysql['TINYBLOB'] = 'string'; +$config->bi->columnTypes->mysql['MEDIUMBLOB'] = 'string'; +$config->bi->columnTypes->mysql['LONG_BLOB'] = 'string'; +$config->bi->columnTypes->mysql['BLOB'] = 'string'; +$config->bi->columnTypes->mysql['VAR_STRING'] = 'string'; +$config->bi->columnTypes->mysql['STRING'] = 'string'; +$config->bi->columnTypes->mysql['NULL'] = 'null'; +$config->bi->columnTypes->mysql['NEWDATE'] = 'date'; +$config->bi->columnTypes->mysql['INTERVAL'] = 'string'; +$config->bi->columnTypes->mysql['GEOMETRY'] = 'string'; +$config->bi->columnTypes->mysql['NEWDECIMAL'] = 'number'; + +/* Dameng native_type. */ +$config->bi->columnTypes->mysql['int'] = 'number'; +$config->bi->columnTypes->mysql['varchar'] = 'string'; +$config->bi->columnTypes->mysql['text'] = 'string'; +$config->bi->columnTypes->mysql['timestamp'] = 'string'; +$config->bi->columnTypes->mysql['date'] = 'date'; +$config->bi->columnTypes->mysql['time'] = 'string'; +$config->bi->columnTypes->mysql['double'] = 'number'; +$config->bi->columnTypes->mysql['number'] = 'number'; +$config->bi->columnTypes->mysql['bigint'] = 'number'; + +/* DuckDB native_type. */ +$config->bi->columnTypes->duckdb['BIGINT'] = 'number'; +$config->bi->columnTypes->duckdb['UBIGINT'] = 'number'; +$config->bi->columnTypes->duckdb['HUGEINT'] = 'number'; +$config->bi->columnTypes->duckdb['UHUGEINT'] = 'number'; +$config->bi->columnTypes->duckdb['INTEGER'] = 'number'; +$config->bi->columnTypes->duckdb['UINTEGER'] = 'number'; +$config->bi->columnTypes->duckdb['SMALLINT'] = 'number'; +$config->bi->columnTypes->duckdb['USMALLINT'] = 'number'; +$config->bi->columnTypes->duckdb['TINYINT'] = 'number'; +$config->bi->columnTypes->duckdb['UTINYINT'] = 'number'; +$config->bi->columnTypes->duckdb['REAL'] = 'number'; +$config->bi->columnTypes->duckdb['DOUBLE'] = 'number'; +$config->bi->columnTypes->duckdb['BOOLEAN'] = 'number'; +$config->bi->columnTypes->duckdb['BLOB'] = 'string'; +$config->bi->columnTypes->duckdb['VARCHAR'] = 'string'; +$config->bi->columnTypes->duckdb['DATE'] = 'date'; +$config->bi->columnTypes->duckdb['TIMESTAMP'] = 'date'; +$config->bi->columnTypes->duckdb['TIME'] = 'date'; + +$config->bi->columnTypes->INTEGER = 'number'; +$config->bi->columnTypes->UINTEGER = 'number'; +$config->bi->columnTypes->UTINYINT = 'number'; +$config->bi->columnTypes->SMALLINT = 'number'; +$config->bi->columnTypes->FLOAT = 'number'; +$config->bi->columnTypes->BOOLEAN = 'number'; +$config->bi->columnTypes->VARCHAR = 'string'; +$config->bi->columnTypes->TIMESTAMP = 'date'; +$config->bi->columnTypes->DATE = 'date'; + $config->bi->duckdb = new stdclass(); $config->bi->duckdb->tables = array(); $config->bi->duckdb->tables['action'] = << array_unique($tables), 'fields' => $fields); } + /** + * Get sql result columns. + * + * @param string $sql + * @param string $driverName mysql|duckdb + * @access public + * @return array|false + */ + public function getColumns(string $sql, $driverName = 'mysql'): array|false + { + if(!in_array($driverName, $this->config->bi->driverNames)) return false; + + if($driverName == 'mysql') + { + $columns = $this->dao->getColumns($sql); + } + else + { + $dbh = $this->app->loadDriver('duckdb'); + $columns = $dbh->query("$sql", 'desc')->fetchAll(); + } + + $result = array(); + foreach($columns as $column) + { + $column = (array)$column; + + $name = $driverName == 'mysql' ? $column['name'] : $column['column_name']; + $nativeType = $driverName == 'mysql' ? $column['native_type'] : $column['column_type']; + + $result[$name] = array('name' => $name, 'native_type' => $nativeType); + } + + return $result; + } + + /** + * 获取表的字段类型。 + * Get table data. + * + * @param string $sql + * @param string $driverName mysql|duckdb + * @param array $columns + * @access public + * @return object + */ + public function getColumnsType(string $sql, string $driverName = 'mysql', array $columns = array()): object + { + if(empty($columns)) $columns = $this->getColumns($sql, $driverName); + + $columnTypes = new stdclass(); + foreach($columns as $column) + { + $field = $column['name']; + $nativeType = $column['native_type']; + $type = $this->config->bi->columnTypes->$driverName[$nativeType]; + + if(isset($columnTypes->$field)) $field = $column['table'] . $field; + $columnTypes->$field = $type; + } + + return $columnTypes; + } + /** * Get object options. * diff --git a/module/dataview/config.php b/module/dataview/config.php index 702c837065..7d2fa973a2 100644 --- a/module/dataview/config.php +++ b/module/dataview/config.php @@ -12,41 +12,3 @@ $config->dataview->groupChild['company'] = array('user' => 'user,usercontact,use $config->dataview->multipleMappingFields = array(); $config->dataview->multipleMappingFields[] = 'testcase-stage'; - -$config->dataview->columnTypes = new stdclass(); -$config->dataview->columnTypes->TINY = 'number'; -$config->dataview->columnTypes->SHORT = 'number'; -$config->dataview->columnTypes->LONG = 'number'; -$config->dataview->columnTypes->FLOAT = 'number'; -$config->dataview->columnTypes->DOUBLE = 'number'; -$config->dataview->columnTypes->TIMESTAMP = 'string'; -$config->dataview->columnTypes->LONGLONG = 'string'; -$config->dataview->columnTypes->INT24 = 'number'; -$config->dataview->columnTypes->DATE = 'date'; -$config->dataview->columnTypes->TIME = 'string'; -$config->dataview->columnTypes->DATETIME = 'date'; -$config->dataview->columnTypes->YEAR = 'date'; -$config->dataview->columnTypes->ENUM = 'string'; -$config->dataview->columnTypes->SET = 'string'; -$config->dataview->columnTypes->TINYBLOB = 'string'; -$config->dataview->columnTypes->MEDIUMBLOB = 'string'; -$config->dataview->columnTypes->LONG_BLOB = 'string'; -$config->dataview->columnTypes->BLOB = 'string'; -$config->dataview->columnTypes->VAR_STRING = 'string'; -$config->dataview->columnTypes->STRING = 'string'; -$config->dataview->columnTypes->NULL = 'null'; -$config->dataview->columnTypes->NEWDATE = 'date'; -$config->dataview->columnTypes->INTERVAL = 'string'; -$config->dataview->columnTypes->GEOMETRY = 'string'; -$config->dataview->columnTypes->NEWDECIMAL = 'number'; - -/* Dameng native_type. */ -$config->dataview->columnTypes->int = 'number'; -$config->dataview->columnTypes->varchar = 'string'; -$config->dataview->columnTypes->text = 'string'; -$config->dataview->columnTypes->timestamp = 'string'; -$config->dataview->columnTypes->date = 'date'; -$config->dataview->columnTypes->time = 'string'; -$config->dataview->columnTypes->double = 'number'; -$config->dataview->columnTypes->number = 'number'; -$config->dataview->columnTypes->bigint = 'number'; diff --git a/module/dataview/model.php b/module/dataview/model.php index 421666baa2..6558700b0e 100644 --- a/module/dataview/model.php +++ b/module/dataview/model.php @@ -22,7 +22,7 @@ class dataviewModel extends model public function __construct() { parent::__construct(); - $this->loadBIDAO(); + $this->loadModel('bi'); } @@ -229,61 +229,34 @@ class dataviewModel extends model return array($mergeFields, $relatedObject); } - /** - * 获取表的字段类型。 - * Get table data. - * - * @param string $sql - * @access public - * @return object - */ - public function getColumns(string $sql): object - { - $columns = $this->dao->getColumns($sql); - - $columnTypes = new stdclass(); - foreach($columns as $column) - { - $field = $column['name']; - $nativeType = $column['native_type']; - $type = $this->config->dataview->columnTypes->$nativeType; - - if(isset($columnTypes->$field)) $field = $column['table'] . $field; - $columnTypes->$field = $type; - } - - return $columnTypes; - } - /** * 检查查询结果的唯一性。 * Check that the column of an sql query is unique. * * @param string $sql + * @param string $driverName mysql|duckdb * @param bool $repeat + * @param array $columns * @access public * @return bool|array */ - public function checkUniColumn(string $sql, bool $repeat = false): bool|array + public function checkUniColumn(string $sql, string $driverName = 'mysql', bool $repeat = false, $columns = array()): bool|array { - $columns = $this->dao->getColumns($sql); + if(empty($columns)) $columns = $this->bi->getColumns($sql, $driverName); $isUnique = true; - $columnList = array(); - $repeatColumn = array(); - foreach($columns as $column) + $repeatFields = array(); + $fieldCounts = array_count_values(array_column($columns, 'name')); + foreach($fieldCounts as $field => $value) { - $field = $column['name']; - if(isset($columnTypes[$field])) + if($value > 1) { $isUnique = false; - $repeatColumn[$field] = $field; + $repeatFields[$field] = $field; } - - $columnTypes[$field] = $field; } - if($repeat) return array($isUnique, $repeatColumn); + if($repeat) return array($isUnique, $repeatFields); return $isUnique; } diff --git a/module/pivot/model.php b/module/pivot/model.php index 66ea682a17..85a5446596 100644 --- a/module/pivot/model.php +++ b/module/pivot/model.php @@ -202,7 +202,7 @@ class pivotModel extends model $stmt = $this->dbh->query($querySQL); if(!$stmt) return; - $columns = $this->loadModel('dataview')->getColumns($querySQL); + $columns = $this->bi->getColumnsType($querySQL); $columnFields = array(); foreach(array_keys(get_object_vars($columns)) as $type) $columnFields[$type] = $type; diff --git a/module/upgrade/model.php b/module/upgrade/model.php index f3dc8e273b..4d3d1f0572 100644 --- a/module/upgrade/model.php +++ b/module/upgrade/model.php @@ -7821,10 +7821,11 @@ class upgradeModel extends model { if(!$sql) return array(); + $this->loadModel('bi'); $this->loadModel('dataview'); $this->loadModel('chart'); - $columns = $this->dataview->getColumns($sql); + $columns = $this->bi->getColumnsType($sql); $columnFields = array(); foreach($columns as $column => $type) $columnFields[$column] = $column;