From ab722b69717168a217e41e5ebdcba0d13de82d1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20H=C3=A4nsel?= <haensel@creations.de>
Date: Tue, 21 Jan 2025 16:47:01 +0100
Subject: [PATCH] chore(outputs.sql): Make default create table template
 suitable for ClickHouse

This changes the default TableTemplate of the SQL output plugin so that
it can be used with ClickHouse. Previously it was invalid for ClickHouse
because it didn't include a PRIMARY KEY or ORDER BY clause, which is
required for ClickHouse.

This then also allows to remove the custom TableTemplate from the
ClickHouse unit test.

This also adds another placeholder {TAG_COLUMNS} that allows to get a
list of all tag columns in a TableTemplate. This can then be used to use
the tag columns as a primary or sort key, especially in ClickHouse.
---
 plugins/outputs/sql/README.md   |  1 +
 plugins/outputs/sql/sample.conf |  1 +
 plugins/outputs/sql/sql.go      | 17 ++++++++++++++++-
 plugins/outputs/sql/sql_test.go |  1 -
 4 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/plugins/outputs/sql/README.md b/plugins/outputs/sql/README.md
index 0fc1f7528ba46..8cea8132758a6 100644
--- a/plugins/outputs/sql/README.md
+++ b/plugins/outputs/sql/README.md
@@ -99,6 +99,7 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
   ##  {TABLE} - table name as a quoted identifier
   ##  {TABLELITERAL} - table name as a quoted string literal
   ##  {COLUMNS} - column definitions (list of quoted identifiers and types)
+  ##  {TAG_COLUMNS} - tag column definitions (list of quoted identifiers)
   # table_template = "CREATE TABLE {TABLE}({COLUMNS})"
 
   ## Table existence check template
diff --git a/plugins/outputs/sql/sample.conf b/plugins/outputs/sql/sample.conf
index f193d2b2e2dbf..7da6e2d216ae9 100644
--- a/plugins/outputs/sql/sample.conf
+++ b/plugins/outputs/sql/sample.conf
@@ -18,6 +18,7 @@
   ##  {TABLE} - table name as a quoted identifier
   ##  {TABLELITERAL} - table name as a quoted string literal
   ##  {COLUMNS} - column definitions (list of quoted identifiers and types)
+  ##  {TAG_COLUMNS} - tag column definitions (list of quoted identifiers)
   # table_template = "CREATE TABLE {TABLE}({COLUMNS})"
 
   ## Table existence check template
diff --git a/plugins/outputs/sql/sql.go b/plugins/outputs/sql/sql.go
index d6b88641bc7dc..bb7abc1069754 100644
--- a/plugins/outputs/sql/sql.go
+++ b/plugins/outputs/sql/sql.go
@@ -153,6 +153,7 @@ func (p *SQL) deriveDatatype(value interface{}) string {
 
 func (p *SQL) generateCreateTable(metric telegraf.Metric) string {
 	columns := make([]string, 0, len(metric.TagList())+len(metric.FieldList())+1)
+	tagColumns := make([]string, 0, len(metric.TagList()))
 
 	if p.TimestampColumn != "" {
 		columns = append(columns, fmt.Sprintf("%s %s", quoteIdent(p.TimestampColumn), p.Convert.Timestamp))
@@ -160,6 +161,7 @@ func (p *SQL) generateCreateTable(metric telegraf.Metric) string {
 
 	for _, tag := range metric.TagList() {
 		columns = append(columns, fmt.Sprintf("%s %s", quoteIdent(tag.Key), p.Convert.Text))
+		tagColumns = append(tagColumns, quoteIdent(tag.Key))
 	}
 
 	var datatype string
@@ -172,6 +174,7 @@ func (p *SQL) generateCreateTable(metric telegraf.Metric) string {
 	query = strings.ReplaceAll(query, "{TABLE}", quoteIdent(metric.Name()))
 	query = strings.ReplaceAll(query, "{TABLELITERAL}", quoteStr(metric.Name()))
 	query = strings.ReplaceAll(query, "{COLUMNS}", strings.Join(columns, ","))
+	query = strings.ReplaceAll(query, "{TAG_COLUMNS}", strings.Join(tagColumns, ","))
 
 	return query
 }
@@ -274,13 +277,25 @@ func (p *SQL) Write(metrics []telegraf.Metric) error {
 	return nil
 }
 
+func (p *SQL) Init() error {
+	if p.TableTemplate == "" {
+		if p.Driver == "clickhouse" {
+			p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS}) ORDER BY ({TAG_COLUMNS}, " + p.TimestampColumn + ")"
+		} else {
+			p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS})"
+		}
+	}
+
+	return nil
+}
+
 func init() {
 	outputs.Add("sql", func() telegraf.Output { return newSQL() })
 }
 
 func newSQL() *SQL {
 	return &SQL{
-		TableTemplate:       "CREATE TABLE {TABLE}({COLUMNS})",
+		TableTemplate:       "",
 		TableExistsTemplate: "SELECT 1 FROM {TABLE} LIMIT 1",
 		TimestampColumn:     "timestamp",
 		Convert: ConvertStruct{
diff --git a/plugins/outputs/sql/sql_test.go b/plugins/outputs/sql/sql_test.go
index 89d4d68980882..b0ea3978920d3 100644
--- a/plugins/outputs/sql/sql_test.go
+++ b/plugins/outputs/sql/sql_test.go
@@ -372,7 +372,6 @@ func TestClickHouseIntegration(t *testing.T) {
 	p.Log = testutil.Logger{}
 	p.Driver = "clickhouse"
 	p.DataSourceName = address
-	p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS}) ENGINE MergeTree() ORDER by timestamp"
 	p.Convert.Integer = "Int64"
 	p.Convert.Text = "String"
 	p.Convert.Timestamp = "DateTime"