Skip to content

Commit ea706b7

Browse files
slvrtrnslvrtrn
authored
Exclude .inner MV tables, set Dictionary types for Maps (#150)
* Exclude inner MV tables * Set Map type to Dictionary * Add CHANGELOG * Add helper texts for the connection options
1 parent 91a3857 commit ea706b7

File tree

6 files changed

+112
-62
lines changed

6 files changed

+112
-62
lines changed

‎CHANGELOG.md

+5-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
# 1.1.4
1+
# 1.1.3
22

3-
### New features
3+
### New features
44

5-
* Database name can now contain multiple schemas in UI field, which tells the driver to scan selected databases. (separator can be set in `metabase.driver.clickhouse/SEPARATOR`)
5+
* Hide `.inner` tables of Materialized Views.
6+
* Resolve `Map` base type to `type/Dictionary`.
7+
* Database name can now contain multiple schemas in the UI field (space-separated by default), which tells the driver to scan selected databases. Separator can be set in `metabase.driver.clickhouse/SEPARATOR`. (@veschin)
68

79
# 1.1.2
810

‎resources/metabase-plugin.yaml

+5-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
info:
22
name: Metabase ClickHouse Driver
3-
version: 1.1.2
3+
version: 1.1.3
44
description: Allows Metabase to connect to ClickHouse databases.
55
contact-info:
66
name: ClickHouse
@@ -18,12 +18,15 @@ driver:
1818
- user
1919
- password
2020
- name: dbname
21-
display-name: Databases Names
21+
display-name: Databases
2222
placeholder: default
23+
helper-text: "To specify multiple databases, separate them by the space symbol. For example: default data logs"
2324
- name: scan-all-databases
2425
display-name: Scan all databases
2526
type: boolean
2627
default: false
28+
description: Scan all tables from all available ClickHouse databases except the system ones.
29+
2730
- ssl
2831
- ssh-tunnel
2932
- advanced-options-start

‎src/metabase/driver/clickhouse.clj

+29-21
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
[#"Int64" :type/BigInteger]
5454
[#"IPv4" :type/IPAddress]
5555
[#"IPv6" :type/IPAddress]
56+
[#"Map" :type/Dictionary]
5657
[#"String" :type/Text]
5758
[#"Tuple" :type/*]
5859
[#"UInt8" :type/Integer]
@@ -64,17 +65,22 @@
6465
(defmethod sql-jdbc.sync/database-type->base-type :clickhouse
6566
[_ database-type]
6667
(let [base-type (database-type->base-type
67-
(str/replace (name database-type)
68-
#"(?:Nullable|LowCardinality)\((\S+)\)"
69-
"$1"))]
68+
(let [normalized ;; extract the type from Nullable or LowCardinality first
69+
(str/replace (name database-type)
70+
#"(?:Nullable|LowCardinality)\((\S+)\)"
71+
"$1")]
72+
(cond
73+
(str/starts-with? normalized "Array(") "Array"
74+
(str/starts-with? normalized "Map(") "Map"
75+
:else normalized)))]
7076
base-type))
7177

7278
(def ^:private excluded-schemas #{"system" "information_schema" "INFORMATION_SCHEMA"})
7379
(defmethod sql-jdbc.sync/excluded-schemas :clickhouse [_] excluded-schemas)
7480

7581
(def ^:private default-connection-details
7682
{:user "default", :password "", :dbname "default", :host "localhost", :port "8123"})
77-
(def ^:private product-name "metabase/1.1.2")
83+
(def ^:private product-name "metabase/1.1.3")
7884

7985
(defmethod sql-jdbc.conn/connection-details->spec :clickhouse
8086
[_ details]
@@ -117,6 +123,10 @@
117123
"%" ; tablePattern "%" = match all tables
118124
allowed-table-types))
119125

126+
(defn ^:private not-inner-mv-table?
127+
[table]
128+
(not (str/starts-with? (:table_name table) ".inner")))
129+
120130
(defn- ->spec
121131
[db]
122132
(if (u/id db)
@@ -128,9 +138,9 @@
128138
(->> (get-tables-from-metadata metadata "%")
129139
(jdbc/metadata-result)
130140
(vec)
131-
(filter #(->> (get % :table_schem)
132-
(contains? excluded-schemas)
133-
(not)))
141+
(filter #(and
142+
(not (contains? excluded-schemas (:table_schem %)))
143+
(not-inner-mv-table? %)))
134144
(tables-set))))
135145

136146
;; Strangely enough, the tests only work with :db keyword,
@@ -140,29 +150,27 @@
140150
(or (get-in db [:details :dbname])
141151
(get-in db [:details :db])))
142152

143-
;; NOTE: option for collection tables from many schemas
144-
(def ^:private SEPARATOR #" ")
145-
(defn- get-multiple-tables [db]
146-
(->> (for [schema (as-> (or (get-db-name db) "default") schemas
147-
(str/split schemas SEPARATOR)
148-
(remove empty? schemas)
149-
(map (comp #(ddl.i/format-name :clickhouse %) str/trim) schemas))]
150-
(jdbc/with-db-metadata [metadata (->spec db)]
153+
(def ^:private db-names-separator #" ")
154+
(defn- get-tables-in-dbs [db-or-dbs]
155+
(->> (for [db (as-> (or (get-db-name db-or-dbs) "default") dbs
156+
(str/split dbs db-names-separator)
157+
(remove empty? dbs)
158+
(map (comp #(ddl.i/format-name :clickhouse %) str/trim) dbs))]
159+
(jdbc/with-db-metadata [metadata (->spec db-or-dbs)]
151160
(jdbc/metadata-result
152-
(get-tables-from-metadata metadata schema))))
161+
(get-tables-from-metadata metadata db))))
153162
(apply concat)
163+
(filter not-inner-mv-table?)
154164
(tables-set)))
155165

156166
(defmethod driver/describe-database :clickhouse
157167
[_ {{:keys [scan-all-databases]}
158168
:details :as db}]
159169
{:tables
160-
(cond
161-
scan-all-databases
170+
(if
171+
(boolean scan-all-databases)
162172
(get-all-tables db)
163-
164-
:selected-schemas
165-
(get-multiple-tables db))})
173+
(get-tables-in-dbs db))})
166174

167175
(defmethod driver/describe-table :clickhouse
168176
[_ database table]

‎test/metabase/driver/clickhouse_test.clj

+52-34
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
[cljc.java-time.local-date :as local-date]
66
[cljc.java-time.offset-date-time :as offset-date-time]
77
[cljc.java-time.temporal.chrono-unit :as chrono-unit]
8+
[clojure.string :as str]
89
[clojure.test :refer :all]
910
[metabase.driver :as driver]
1011
[metabase.driver.clickhouse-test-utils :as ctu]
@@ -438,6 +439,21 @@
438439
{:filter [:= $ipvfour "127.0.0.1"]
439440
:aggregation [:count]})))))))))
440441

442+
(defn- map-as-string [^java.util.LinkedHashMap m] (.toString m))
443+
(deftest clickhouse-simple-map-test
444+
(mt/test-driver
445+
:clickhouse
446+
(is (= [["{key1=1, key2=10}"] ["{key1=2, key2=20}"] ["{key1=3, key2=30}"]]
447+
(qp.test/formatted-rows
448+
[map-as-string]
449+
:format-nil-values
450+
(ctu/do-with-metabase-test-db
451+
(fn [db]
452+
(data/with-db db
453+
(data/run-mbql-query
454+
maps_test
455+
{})))))))))
456+
441457
(deftest clickhouse-connection-string
442458
(testing "connection with no additional options"
443459
(is (= default-connection-params
@@ -481,8 +497,9 @@
481497
(deftest clickhouse-boolean-tabledef-metadata
482498
(mt/test-driver
483499
:clickhouse
484-
(let [table_md (ctu/do-with-metabase-test-db
485-
(fn [db] (metabase.driver/describe-table :clickhouse db {:name "boolean_test"})))
500+
(let [table_md (ctu/do-with-metabase-test-db
501+
(fn [db]
502+
(metabase.driver/describe-table :clickhouse db {:name "boolean_test"})))
486503
colmap (->> (.get table_md :fields)
487504
(filter #(= (:name %) "b1")) first)
488505
database-type (.get colmap :database-type)
@@ -544,28 +561,36 @@
544561
{})))))))))))
545562

546563
(deftest clickhouse-describe-database
547-
(let [[agg-fn-table boolean-table enum-table ipaddress-table]
548-
[{:description nil,
549-
:name "aggregate_functions_filter_test",
550-
:schema "metabase_test"}
551-
{:description nil,
552-
:name "boolean_test",
553-
:schema "metabase_test"}
554-
{:description nil,
555-
:name "enums_test",
556-
:schema "metabase_test"}
557-
{:description nil,
558-
:name "ipaddress_test",
559-
:schema "metabase_test"}]]
564+
(let [test-tables
565+
#{{:description nil,
566+
:name "aggregate_functions_filter_test",
567+
:schema "metabase_test"}
568+
{:description nil,
569+
:name "boolean_test",
570+
:schema "metabase_test"}
571+
{:description nil,
572+
:name "enums_test",
573+
:schema "metabase_test"}
574+
{:description nil,
575+
:name "ipaddress_test",
576+
:schema "metabase_test"}
577+
{:description nil,
578+
:name "wikistat",
579+
:schema "metabase_test"}
580+
{:description nil,
581+
:name "wikistat_mv",
582+
:schema "metabase_test"}
583+
{:description nil,
584+
:name "maps_test",
585+
:schema "metabase_test"}}]
560586
(testing "scanning a single database"
561587
(mt/with-temp Database
562588
[db {:engine :clickhouse
563589
:details {:dbname "metabase_test"
564590
:scan-all-databases nil}}]
565591
(let [describe-result (driver/describe-database :clickhouse db)]
566592
(is (=
567-
{:tables
568-
#{agg-fn-table boolean-table enum-table ipaddress-table}}
593+
{:tables test-tables}
569594
describe-result)))))
570595
(testing "scanning all databases"
571596
(mt/with-temp Database
@@ -574,20 +599,15 @@
574599
:scan-all-databases true}}]
575600
(let [describe-result (driver/describe-database :clickhouse db)]
576601
;; check the existence of at least some test tables here
577-
(is (contains? (:tables describe-result)
578-
agg-fn-table))
579-
(is (contains? (:tables describe-result)
580-
boolean-table))
581-
(is (contains? (:tables describe-result)
582-
enum-table))
583-
(is (contains? (:tables describe-result)
584-
ipaddress-table))
602+
(doseq [table test-tables]
603+
(is (contains? (:tables describe-result)
604+
table)))
585605
;; should not contain any ClickHouse system tables
586-
(is (not (some #(= (get % :schema) "system")
606+
(is (not (some #(= (:schema %) "system")
587607
(:tables describe-result))))
588-
(is (not (some #(= (get % :schema) "information_schema")
608+
(is (not (some #(= (:schema %) "information_schema")
589609
(:tables describe-result))))
590-
(is (not (some #(= (get % :schema) "INFORMATION_SCHEMA")
610+
(is (not (some #(= (:schema %) "INFORMATION_SCHEMA")
591611
(:tables describe-result)))))))
592612
(testing "scanning multiple databases"
593613
(mt/with-temp Database
@@ -602,12 +622,10 @@
602622
:description nil
603623
:schema "information_schema"}]
604624

605-
;; NOTE: tables from `metabase_test`
606-
(is (contains? tables agg-fn-table))
607-
(is (contains? tables boolean-table))
608-
(is (contains? tables enum-table))
609-
(is (contains? tables ipaddress-table))
625+
;; tables from `metabase_test`
626+
(doseq [table test-tables]
627+
(is (contains? tables table)))
610628

611-
;; NOTE: tables from `information_schema`
629+
;; tables from `information_schema`
612630
(is (contains? tables tables-table))
613631
(is (contains? tables columns-table)))))))

‎test/metabase/driver/clickhouse_test_utils.clj

+20-1
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,31 @@
5656
" (1, true, true),"
5757
" (2, false, true),"
5858
" (3, true, false);")
59+
(str "CREATE TABLE `metabase_test`.`maps_test`"
60+
" (m Map(String, UInt64)) ENGINE = Memory;")
61+
(str "INSERT INTO `metabase_test`.`maps_test` VALUES"
62+
" ({'key1':1, 'key2':10}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});")
63+
;; Used for testing that AggregateFunction columns are excluded,
64+
;; while SimpleAggregateFunction columns are preserved
5965
(str "CREATE TABLE `metabase_test`.`aggregate_functions_filter_test` ("
6066
" idx UInt8, a AggregateFunction(uniq, String), lowest_value SimpleAggregateFunction(min, UInt8),"
6167
" count SimpleAggregateFunction(sum, Int64)"
6268
") ENGINE Memory;")
6369
(str "INSERT INTO `metabase_test`.`aggregate_functions_filter_test`"
64-
" (idx, lowest_value, count) VALUES (42, 144, 255255);")]]
70+
" (idx, lowest_value, count) VALUES (42, 144, 255255);")
71+
;; Materialized views (testing .inner tables exclusion)
72+
(str "CREATE TABLE `metabase_test`.`wikistat` ("
73+
" `date` Date,"
74+
" `project` LowCardinality(String),"
75+
" `hits` UInt32"
76+
") ENGINE = Memory;")
77+
(str "CREATE MATERIALIZED VIEW `metabase_test`.`wikistat_mv` ENGINE=Memory AS"
78+
" SELECT date, project, sum(hits) AS hits FROM `metabase_test`.`wikistat`"
79+
" GROUP BY date, project;")
80+
(str "INSERT INTO `metabase_test`.`wikistat` VALUES"
81+
" (now(), 'foo', 10),"
82+
" (now(), 'bar', 10),"
83+
" (now(), 'bar', 20);")]]
6584
(jdbc/execute! conn [sql]))))
6685

6786
(defn do-with-metabase-test-db

‎test/metabase/test/data/clickhouse.clj

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
:ssl false
2222
:use_no_proxy false
2323
:use_server_time_zone_for_dates true
24-
:product_name "metabase/1.1.2"})
24+
:product_name "metabase/1.1.3"})
2525

2626
(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Boolean] [_ _] "Boolean")
2727
(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/BigInteger] [_ _] "Int64")

0 commit comments

Comments
 (0)