From 58bd017f865fbbd61c27ba62fdfe323a26eda660 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20H=C3=A4nsel?= <haensel@creations.de>
Date: Tue, 21 Jan 2025 16:47:01 +0100
Subject: [PATCH] chore(outputs.sql): Make default create table template
 suitable for ClickHouse

This changes the default TableTemplate of the SQL output plugin so that
it can be used with ClickHouse. Previously it was invalid for ClickHouse
because it didn't include a PRIMARY KEY or ORDER BY clause, which is
required for ClickHouse.

This then also allows to remove the custom TableTemplate from the
ClickHouse unit test.

This also adds another placeholder {TAG_COLUMNS} that allows to get a
list of all tag columns in a TableTemplate. This can then be used to use
the tag columns as a primary or sort key, especially in ClickHouse.
---
 plugins/outputs/sql/README.md   |  1 +
 plugins/outputs/sql/sample.conf |  1 +
 plugins/outputs/sql/sql.go      | 11 ++++++++++-
 plugins/outputs/sql/sql_test.go |  1 -
 4 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/plugins/outputs/sql/README.md b/plugins/outputs/sql/README.md
index 74214c55a7832..ea04000c64f14 100644
--- a/plugins/outputs/sql/README.md
+++ b/plugins/outputs/sql/README.md
@@ -99,6 +99,7 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
   ##  {TABLE} - table name as a quoted identifier
   ##  {TABLELITERAL} - table name as a quoted string literal
   ##  {COLUMNS} - column definitions (list of quoted identifiers and types)
+  ##  {TAG_COLUMNS} - tag column definitions (list of quoted identifiers)
   # table_template = "CREATE TABLE {TABLE}({COLUMNS})"
 
   ## Table existence check template
diff --git a/plugins/outputs/sql/sample.conf b/plugins/outputs/sql/sample.conf
index f193d2b2e2dbf..7da6e2d216ae9 100644
--- a/plugins/outputs/sql/sample.conf
+++ b/plugins/outputs/sql/sample.conf
@@ -18,6 +18,7 @@
   ##  {TABLE} - table name as a quoted identifier
   ##  {TABLELITERAL} - table name as a quoted string literal
   ##  {COLUMNS} - column definitions (list of quoted identifiers and types)
+  ##  {TAG_COLUMNS} - tag column definitions (list of quoted identifiers)
   # table_template = "CREATE TABLE {TABLE}({COLUMNS})"
 
   ## Table existence check template
diff --git a/plugins/outputs/sql/sql.go b/plugins/outputs/sql/sql.go
index a216bd9483e7a..1f53885f6c1b2 100644
--- a/plugins/outputs/sql/sql.go
+++ b/plugins/outputs/sql/sql.go
@@ -154,6 +154,7 @@ func (p *SQL) deriveDatatype(value interface{}) string {
 
 func (p *SQL) generateCreateTable(metric telegraf.Metric) string {
 	columns := make([]string, 0, len(metric.TagList())+len(metric.FieldList())+1)
+	tagColumns := make([]string, 0, len(metric.TagList()))
 
 	if p.TimestampColumn != "" {
 		columns = append(columns, fmt.Sprintf("%s %s", quoteIdent(p.TimestampColumn), p.Convert.Timestamp))
@@ -161,6 +162,7 @@ func (p *SQL) generateCreateTable(metric telegraf.Metric) string {
 
 	for _, tag := range metric.TagList() {
 		columns = append(columns, fmt.Sprintf("%s %s", quoteIdent(tag.Key), p.Convert.Text))
+		tagColumns = append(tagColumns, quoteIdent(tag.Key))
 	}
 
 	var datatype string
@@ -173,6 +175,13 @@ func (p *SQL) generateCreateTable(metric telegraf.Metric) string {
 	query = strings.ReplaceAll(query, "{TABLE}", quoteIdent(metric.Name()))
 	query = strings.ReplaceAll(query, "{TABLELITERAL}", quoteStr(metric.Name()))
 	query = strings.ReplaceAll(query, "{COLUMNS}", strings.Join(columns, ","))
+	query = strings.ReplaceAll(query, "{TAG_COLUMNS}", strings.Join(tagColumns, ","))
+
+	if p.Driver == "clickhouse" {
+		query = strings.ReplaceAll(query, "{SORT_KEY_CLAUSE}", fmt.Sprintf("ORDER BY (%s, %s)", strings.Join(tagColumns, ","), p.TimestampColumn))
+	} else {
+		query = strings.ReplaceAll(query, "{SORT_KEY_CLAUSE}", "")
+	}
 
 	return query
 }
@@ -281,7 +290,7 @@ func init() {
 
 func newSQL() *SQL {
 	return &SQL{
-		TableTemplate:       "CREATE TABLE {TABLE}({COLUMNS})",
+		TableTemplate:       "CREATE TABLE {TABLE}({COLUMNS}) {SORT_KEY_CLAUSE}",
 		TableExistsTemplate: "SELECT 1 FROM {TABLE} LIMIT 1",
 		TimestampColumn:     "timestamp",
 		Convert: ConvertStruct{
diff --git a/plugins/outputs/sql/sql_test.go b/plugins/outputs/sql/sql_test.go
index f57bc5592cc18..083dff17f5ae9 100644
--- a/plugins/outputs/sql/sql_test.go
+++ b/plugins/outputs/sql/sql_test.go
@@ -367,7 +367,6 @@ func TestClickHouseIntegration(t *testing.T) {
 	p.Log = testutil.Logger{}
 	p.Driver = "clickhouse"
 	p.DataSourceName = address
-	p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS}) ENGINE MergeTree() ORDER by timestamp"
 	p.Convert.Integer = "Int64"
 	p.Convert.Text = "String"
 	p.Convert.Timestamp = "DateTime"