From 321199ac5d2da1f20231d9663bef6fd8c37e6d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20H=C3=A4nsel?= Date: Tue, 21 Jan 2025 16:47:01 +0100 Subject: [PATCH] chore(outputs.sql): Make default create table template suitable for ClickHouse This changes the default TableTemplate of the SQL output plugin so that it can be used with ClickHouse. Previously it was invalid for ClickHouse because it didn't include a PRIMARY KEY or ORDER BY clause, which is required for ClickHouse. This then also allows to remove the custom TableTemplate from the ClickHouse unit test. This also adds another placeholder {TAG_COLUMNS} that allows to get a list of all tag columns in a TableTemplate. This can then be used to use the tag columns as a primary or sort key, especially in ClickHouse. --- plugins/outputs/sql/README.md | 1 + plugins/outputs/sql/sample.conf | 1 + plugins/outputs/sql/sql.go | 16 +++++++++++++++- plugins/outputs/sql/sql_test.go | 4 +++- plugins/outputs/sql/sqlite_test.go | 1 + 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/plugins/outputs/sql/README.md b/plugins/outputs/sql/README.md index 0fc1f7528ba46..78bf5309185f4 100644 --- a/plugins/outputs/sql/README.md +++ b/plugins/outputs/sql/README.md @@ -99,6 +99,7 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ## {TABLE} - table name as a quoted identifier ## {TABLELITERAL} - table name as a quoted string literal ## {COLUMNS} - column definitions (list of quoted identifiers and types) + ## {TAG_COLUMN_NAMES} - tag column definitions (list of quoted identifiers) # table_template = "CREATE TABLE {TABLE}({COLUMNS})" ## Table existence check template diff --git a/plugins/outputs/sql/sample.conf b/plugins/outputs/sql/sample.conf index f193d2b2e2dbf..c88bf8730baad 100644 --- a/plugins/outputs/sql/sample.conf +++ b/plugins/outputs/sql/sample.conf @@ -18,6 +18,7 @@ ## {TABLE} - table name as a quoted identifier ## {TABLELITERAL} - table name as a quoted string literal ## {COLUMNS} - column definitions (list of quoted identifiers and types) + ## {TAG_COLUMN_NAMES} - tag column definitions (list of quoted identifiers) # table_template = "CREATE TABLE {TABLE}({COLUMNS})" ## Table existence check template diff --git a/plugins/outputs/sql/sql.go b/plugins/outputs/sql/sql.go index d6b88641bc7dc..8d1fc65bb6fda 100644 --- a/plugins/outputs/sql/sql.go +++ b/plugins/outputs/sql/sql.go @@ -153,6 +153,7 @@ func (p *SQL) deriveDatatype(value interface{}) string { func (p *SQL) generateCreateTable(metric telegraf.Metric) string { columns := make([]string, 0, len(metric.TagList())+len(metric.FieldList())+1) + tagColumnNames := make([]string, 0, len(metric.TagList())) if p.TimestampColumn != "" { columns = append(columns, fmt.Sprintf("%s %s", quoteIdent(p.TimestampColumn), p.Convert.Timestamp)) @@ -160,6 +161,7 @@ func (p *SQL) generateCreateTable(metric telegraf.Metric) string { for _, tag := range metric.TagList() { columns = append(columns, fmt.Sprintf("%s %s", quoteIdent(tag.Key), p.Convert.Text)) + tagColumnNames = append(tagColumnNames, quoteIdent(tag.Key)) } var datatype string @@ -172,6 +174,7 @@ func (p *SQL) generateCreateTable(metric telegraf.Metric) string { query = strings.ReplaceAll(query, "{TABLE}", quoteIdent(metric.Name())) query = strings.ReplaceAll(query, "{TABLELITERAL}", quoteStr(metric.Name())) query = strings.ReplaceAll(query, "{COLUMNS}", strings.Join(columns, ",")) + query = strings.ReplaceAll(query, "{TAG_COLUMN_NAMES}", strings.Join(tagColumnNames, ",")) return query } @@ -274,13 +277,24 @@ func (p *SQL) Write(metrics []telegraf.Metric) error { return nil } +func (p *SQL) Init() error { + if p.TableTemplate == "" { + if p.Driver == "clickhouse" { + p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS}) ORDER BY ({TAG_COLUMN_NAMES}, " + p.TimestampColumn + ")" + } else { + p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS})" + } + } + + return nil +} + func init() { outputs.Add("sql", func() telegraf.Output { return newSQL() }) } func newSQL() *SQL { return &SQL{ - TableTemplate: "CREATE TABLE {TABLE}({COLUMNS})", TableExistsTemplate: "SELECT 1 FROM {TABLE} LIMIT 1", TimestampColumn: "timestamp", Convert: ConvertStruct{ diff --git a/plugins/outputs/sql/sql_test.go b/plugins/outputs/sql/sql_test.go index 89d4d68980882..4c6acc903dbef 100644 --- a/plugins/outputs/sql/sql_test.go +++ b/plugins/outputs/sql/sql_test.go @@ -201,6 +201,7 @@ func TestMysqlIntegration(t *testing.T) { p.Driver = "mysql" p.DataSourceName = address p.InitSQL = "SET sql_mode='ANSI_QUOTES';" + require.NoError(t, p.Init()) require.NoError(t, p.Connect()) require.NoError(t, p.Write( @@ -287,6 +288,7 @@ func TestPostgresIntegration(t *testing.T) { p.Convert.Real = "double precision" p.Convert.Unsigned = "bigint" p.Convert.ConversionStyle = "literal" + require.NoError(t, p.Init()) require.NoError(t, p.Connect()) defer p.Close() @@ -372,7 +374,6 @@ func TestClickHouseIntegration(t *testing.T) { p.Log = testutil.Logger{} p.Driver = "clickhouse" p.DataSourceName = address - p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS}) ENGINE MergeTree() ORDER by timestamp" p.Convert.Integer = "Int64" p.Convert.Text = "String" p.Convert.Timestamp = "DateTime" @@ -380,6 +381,7 @@ func TestClickHouseIntegration(t *testing.T) { p.Convert.Unsigned = "UInt64" p.Convert.Bool = "UInt8" p.Convert.ConversionStyle = "literal" + require.NoError(t, p.Init()) require.NoError(t, p.Connect()) require.NoError(t, p.Write(testMetrics)) diff --git a/plugins/outputs/sql/sqlite_test.go b/plugins/outputs/sql/sqlite_test.go index 08ae62c4ac976..051525ad6ec79 100644 --- a/plugins/outputs/sql/sqlite_test.go +++ b/plugins/outputs/sql/sqlite_test.go @@ -25,6 +25,7 @@ func TestSqlite(t *testing.T) { p.Log = testutil.Logger{} p.Driver = "sqlite" p.DataSourceName = address + require.NoError(t, p.Init()) require.NoError(t, p.Connect()) defer p.Close()