From e510ef506fe82306dde2aba2dbaad929fcf246e9 Mon Sep 17 00:00:00 2001 From: loomt Date: Wed, 24 Dec 2025 10:59:04 +0800 Subject: [PATCH 1/2] feat: add TLS support for Keeper connections --- pkg/backup/create.go | 4 ++-- pkg/backup/restore.go | 2 +- pkg/clickhouse/clickhouse.go | 31 ++++------------------------ pkg/keeper/keeper.go | 39 ++++++++++++++++++++++++++++++++---- pkg/utils/utils.go | 31 +++++++++++++++++++++++++++- 5 files changed, 72 insertions(+), 35 deletions(-) diff --git a/pkg/backup/create.go b/pkg/backup/create.go index e068e0b0e..bee38a125 100644 --- a/pkg/backup/create.go +++ b/pkg/backup/create.go @@ -794,7 +794,7 @@ func (b *Backuper) createBackupNamedCollections(ctx context.Context, backupPath } k := keeper.Keeper{} - if err = k.Connect(ctx, b.ch); err != nil { + if err = k.Connect(ctx, b.ch, b.cfg); err != nil { return 0, err } defer k.Close() @@ -844,7 +844,7 @@ func (b *Backuper) createBackupRBACReplicated(ctx context.Context, rbacBackup st rbacDataSize := uint64(0) if err = b.ch.SelectContext(ctx, &replicatedRBAC, "SELECT name FROM system.user_directories WHERE type='replicated'"); err == nil && len(replicatedRBAC) > 0 { k := keeper.Keeper{} - if err = k.Connect(ctx, b.ch); err != nil { + if err = k.Connect(ctx, b.ch, b.cfg); err != nil { return 0, err } defer k.Close() diff --git a/pkg/backup/restore.go b/pkg/backup/restore.go index 7e0ca0814..d2cd48242 100644 --- a/pkg/backup/restore.go +++ b/pkg/backup/restore.go @@ -500,7 +500,7 @@ func (b *Backuper) restoreRBAC(ctx context.Context, backupName string, disks []c replicatedUserDirectories := make([]clickhouse.UserDirectory, 0) if err = b.ch.SelectContext(ctx, &replicatedUserDirectories, "SELECT name FROM system.user_directories WHERE type='replicated'"); err == nil && len(replicatedUserDirectories) > 0 { k = &keeper.Keeper{} - if connErr := k.Connect(ctx, b.ch); connErr != nil { + if connErr := k.Connect(ctx, b.ch, b.cfg); connErr != nil { return errors.Wrap(connErr, "but can't connect to keeper") } defer k.Close() diff --git a/pkg/clickhouse/clickhouse.go b/pkg/clickhouse/clickhouse.go index 8e8748fc6..ee8e9b673 100644 --- a/pkg/clickhouse/clickhouse.go +++ b/pkg/clickhouse/clickhouse.go @@ -2,8 +2,6 @@ package clickhouse import ( "context" - "crypto/tls" - "crypto/x509" "database/sql" "fmt" "os" @@ -18,6 +16,7 @@ import ( "github.com/Altinity/clickhouse-backup/v2/pkg/common" "github.com/Altinity/clickhouse-backup/v2/pkg/config" "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "github.com/antchfx/xmlquery" @@ -87,31 +86,9 @@ func (ch *ClickHouse) Connect() error { } if ch.Config.Secure { - tlsConfig := &tls.Config{ - InsecureSkipVerify: ch.Config.SkipVerify, - } - if ch.Config.TLSKey != "" || ch.Config.TLSCert != "" || ch.Config.TLSCa != "" { - if ch.Config.TLSCert != "" || ch.Config.TLSKey != "" { - cert, err := tls.LoadX509KeyPair(ch.Config.TLSCert, ch.Config.TLSKey) - if err != nil { - log.Error().Msgf("tls.LoadX509KeyPair error: %v", err) - return err - } - tlsConfig.Certificates = []tls.Certificate{cert} - } - if ch.Config.TLSCa != "" { - caCert, err := os.ReadFile(ch.Config.TLSCa) - if err != nil { - log.Error().Msgf("read `tls_ca` file %s return error: %v ", ch.Config.TLSCa, err) - return err - } - caCertPool := x509.NewCertPool() - if caCertPool.AppendCertsFromPEM(caCert) != true { - log.Error().Msgf("AppendCertsFromPEM %s return false", ch.Config.TLSCa) - return fmt.Errorf("AppendCertsFromPEM %s return false", ch.Config.TLSCa) - } - tlsConfig.RootCAs = caCertPool - } + tlsConfig, err := utils.NewTLSConfig(ch.Config.TLSCa, ch.Config.TLSCert, ch.Config.TLSKey, ch.Config.SkipVerify) + if err != nil { + return err } opt.TLS = tlsConfig } diff --git a/pkg/keeper/keeper.go b/pkg/keeper/keeper.go index 7e20e9c2f..8b634dbac 100644 --- a/pkg/keeper/keeper.go +++ b/pkg/keeper/keeper.go @@ -3,9 +3,11 @@ package keeper import ( "bufio" "context" + "crypto/tls" "encoding/json" "fmt" "io" + "net" "os" "path" "strconv" @@ -18,6 +20,8 @@ import ( "github.com/rs/zerolog/log" "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" "github.com/go-zookeeper/zk" ) @@ -59,7 +63,7 @@ type Keeper struct { } // Connect - connect to any zookeeper server from /var/lib/clickhouse/preprocessed_configs/config.xml -func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse) error { +func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config) error { configFile, doc, err := ch.ParseXML(ctx, "config.xml") if err != nil { return errors.Wrapf(err, "can't parse config.xml from %s, error", configFile) @@ -83,6 +87,7 @@ func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse) error { return errors.WithStack(fmt.Errorf("/zookeeper/node not exists in %s", configFile)) } keeperHosts := make([]string, len(nodeList)) + isSecure := false for i, node := range nodeList { hostNode := node.SelectElement("host") if hostNode == nil { @@ -93,11 +98,37 @@ func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse) error { if portNode != nil { port = portNode.InnerText() } + secureNode := node.SelectElement("secure") + if secureNode != nil && (secureNode.InnerText() == "1" || secureNode.InnerText() == "true") { + isSecure = true + } keeperHosts[i] = fmt.Sprintf("%s:%s", hostNode.InnerText(), port) } - conn, _, err := zk.Connect(keeperHosts, sessionTimeout, zk.WithLogger(newKeeperLogger())) - if err != nil { - return err + var conn *zk.Conn + if isSecure { + log.Info().Msgf("isSecure=%v, keeperHosts=%v, caPath=%v, certPath=%v, keyPath=%v, skipVerify=%v, use TLS for keeper connection", isSecure, keeperHosts, cfg.ClickHouse.TLSCa, cfg.ClickHouse.TLSCert, cfg.ClickHouse.TLSKey, cfg.ClickHouse.SkipVerify) + tlsConfig, err := utils.NewTLSConfig(cfg.ClickHouse.TLSCa, cfg.ClickHouse.TLSCert, cfg.ClickHouse.TLSKey, cfg.ClickHouse.SkipVerify) + if err != nil { + return errors.Wrap(err, "can't create TLS config") + } + conn, _, err = zk.Connect(keeperHosts, sessionTimeout, zk.WithLogger(newKeeperLogger()), zk.WithDialer(func(network, address string, timeout time.Duration) (net.Conn, error) { + tlsConn, dialErr := tls.DialWithDialer(&net.Dialer{Timeout: timeout}, network, address, tlsConfig) + if dialErr != nil { + log.Error().Msgf("TLS dial to %s failed: %v", address, dialErr) + return nil, dialErr + } + return tlsConn, nil + })) + if err != nil { + log.Error().Msgf("zk.Connect with TLS failed: %v", err) + return err + } + } else { + log.Info().Msgf("isSecure=%v, keeperHosts=%v", isSecure, keeperHosts) + conn, _, err = zk.Connect(keeperHosts, sessionTimeout, zk.WithLogger(newKeeperLogger())) + if err != nil { + return err + } } if digestNode := zookeeperNode.SelectElement("digest"); digestNode != nil { if err = conn.AddAuth("digest", []byte(digestNode.InnerText())); err != nil { diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 9e09afcd8..18e92b2b2 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -2,12 +2,16 @@ package utils import ( "context" + "crypto/tls" + "crypto/x509" "fmt" - "github.com/rs/zerolog/log" + "os" "os/exec" "regexp" "strings" "time" + + "github.com/rs/zerolog/log" ) const ( @@ -72,3 +76,28 @@ func ExecCmdOut(ctx context.Context, timeout time.Duration, cmd string, args ... cancel() return string(out), err } + +func NewTLSConfig(caPath, certPath, keyPath string, skipVerify bool) (*tls.Config, error) { + tlsConfig := &tls.Config{ + InsecureSkipVerify: skipVerify, + } + if certPath != "" || keyPath != "" { + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, fmt.Errorf("tls.LoadX509KeyPair error: %v", err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + } + if caPath != "" { + caCert, err := os.ReadFile(caPath) + if err != nil { + return nil, fmt.Errorf("read ca file %s return error: %v", caPath, err) + } + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("AppendCertsFromPEM %s return false", caPath) + } + tlsConfig.RootCAs = caCertPool + } + return tlsConfig, nil +} From 0b2608a2ea0e4d057f50fd1c454478d88c63a26f Mon Sep 17 00:00:00 2001 From: loomt Date: Mon, 29 Dec 2025 17:26:46 +0800 Subject: [PATCH 2/2] fix: parse openSSL/client config and add test --- pkg/backup/create.go | 4 +- pkg/backup/restore.go | 2 +- pkg/clickhouse/clickhouse.go | 2 +- pkg/keeper/keeper.go | 52 ++++++++++++++++-- pkg/utils/utils.go | 5 +- test/integration/clickhouse-keeper.xml | 23 ++++++++ test/integration/docker-compose_advanced.yml | 5 ++ test/integration/dynamic_settings.sh | 30 ++++++++++ test/integration/integration_test.go | 58 ++++++++++++++++++++ 9 files changed, 171 insertions(+), 10 deletions(-) diff --git a/pkg/backup/create.go b/pkg/backup/create.go index bee38a125..e068e0b0e 100644 --- a/pkg/backup/create.go +++ b/pkg/backup/create.go @@ -794,7 +794,7 @@ func (b *Backuper) createBackupNamedCollections(ctx context.Context, backupPath } k := keeper.Keeper{} - if err = k.Connect(ctx, b.ch, b.cfg); err != nil { + if err = k.Connect(ctx, b.ch); err != nil { return 0, err } defer k.Close() @@ -844,7 +844,7 @@ func (b *Backuper) createBackupRBACReplicated(ctx context.Context, rbacBackup st rbacDataSize := uint64(0) if err = b.ch.SelectContext(ctx, &replicatedRBAC, "SELECT name FROM system.user_directories WHERE type='replicated'"); err == nil && len(replicatedRBAC) > 0 { k := keeper.Keeper{} - if err = k.Connect(ctx, b.ch, b.cfg); err != nil { + if err = k.Connect(ctx, b.ch); err != nil { return 0, err } defer k.Close() diff --git a/pkg/backup/restore.go b/pkg/backup/restore.go index d2cd48242..7e0ca0814 100644 --- a/pkg/backup/restore.go +++ b/pkg/backup/restore.go @@ -500,7 +500,7 @@ func (b *Backuper) restoreRBAC(ctx context.Context, backupName string, disks []c replicatedUserDirectories := make([]clickhouse.UserDirectory, 0) if err = b.ch.SelectContext(ctx, &replicatedUserDirectories, "SELECT name FROM system.user_directories WHERE type='replicated'"); err == nil && len(replicatedUserDirectories) > 0 { k = &keeper.Keeper{} - if connErr := k.Connect(ctx, b.ch, b.cfg); connErr != nil { + if connErr := k.Connect(ctx, b.ch); connErr != nil { return errors.Wrap(connErr, "but can't connect to keeper") } defer k.Close() diff --git a/pkg/clickhouse/clickhouse.go b/pkg/clickhouse/clickhouse.go index ee8e9b673..3e02912a9 100644 --- a/pkg/clickhouse/clickhouse.go +++ b/pkg/clickhouse/clickhouse.go @@ -86,7 +86,7 @@ func (ch *ClickHouse) Connect() error { } if ch.Config.Secure { - tlsConfig, err := utils.NewTLSConfig(ch.Config.TLSCa, ch.Config.TLSCert, ch.Config.TLSKey, ch.Config.SkipVerify) + tlsConfig, err := utils.NewTLSConfig(ch.Config.TLSCa, ch.Config.TLSCert, ch.Config.TLSKey, ch.Config.SkipVerify, true) if err != nil { return err } diff --git a/pkg/keeper/keeper.go b/pkg/keeper/keeper.go index 8b634dbac..29b882b5b 100644 --- a/pkg/keeper/keeper.go +++ b/pkg/keeper/keeper.go @@ -20,7 +20,6 @@ import ( "github.com/rs/zerolog/log" "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/v2/pkg/config" "github.com/Altinity/clickhouse-backup/v2/pkg/utils" "github.com/go-zookeeper/zk" ) @@ -62,8 +61,48 @@ type Keeper struct { xmlConfigFile string } +// https://clickhouse.com/docs/operations/ssl-zookeeper - parse TLS configuration from ClickHouse's config.xml */openSSL/client/* section +func parseClientTLSConfig(doc *xmlquery.Node, configFile string) (caPath, certPath, keyPath string, skipVerify, loadDefaultCAFile bool) { + clientNode := xmlquery.FindOne(doc, "//openSSL/client") + if clientNode == nil { + log.Warn().Msgf("no /openSSL/client in %s, using empty TLS config", configFile) + return "", "", "", true, true + } + + getText := func(selector string) string { + if n := clientNode.SelectElement(selector); n != nil { + return n.InnerText() + } + return "" + } + + certPath = getText("certificateFile") + keyPath = getText("privateKeyFile") + caPath = getText("caConfig") + loadDefaultCAFile = true + if val := getText("loadDefaultCAFile"); val != "" { + if useDefault, err := strconv.ParseBool(val); err == nil { + loadDefaultCAFile = useDefault + } + } + if !loadDefaultCAFile && caPath == "" { + log.Warn().Msgf("//openSSL/client/loadDefaultCAFile=false provided in %s but caConfig is empty; system CAs will NOT be loaded", configFile) + } + + if mode := getText("verificationMode"); mode == "none" { + skipVerify = true + } + // invalidCertificateHandler is alternative way to disable verification + if handler := getText("invalidCertificateHandler/name"); handler == "AcceptCertificateHandler" { + skipVerify = true + } + + log.Debug().Msgf("parsed TLS config from %s: certPath=%s, keyPath=%s, caPath=%s, skipVerify=%v, loadDefaultCAFile=%v", configFile, certPath, keyPath, caPath, skipVerify, loadDefaultCAFile) + return caPath, certPath, keyPath, skipVerify, loadDefaultCAFile +} + // Connect - connect to any zookeeper server from /var/lib/clickhouse/preprocessed_configs/config.xml -func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config) error { +func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse) error { configFile, doc, err := ch.ParseXML(ctx, "config.xml") if err != nil { return errors.Wrapf(err, "can't parse config.xml from %s, error", configFile) @@ -106,10 +145,13 @@ func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse, cfg *co } var conn *zk.Conn if isSecure { - log.Info().Msgf("isSecure=%v, keeperHosts=%v, caPath=%v, certPath=%v, keyPath=%v, skipVerify=%v, use TLS for keeper connection", isSecure, keeperHosts, cfg.ClickHouse.TLSCa, cfg.ClickHouse.TLSCert, cfg.ClickHouse.TLSKey, cfg.ClickHouse.SkipVerify) - tlsConfig, err := utils.NewTLSConfig(cfg.ClickHouse.TLSCa, cfg.ClickHouse.TLSCert, cfg.ClickHouse.TLSKey, cfg.ClickHouse.SkipVerify) + // Parse TLS config from ClickHouse's config.xml /openSSL/client/* section + // according to https://clickhouse.com/docs/operations/ssl-zookeeper + caPath, certPath, keyPath, skipVerify, loadDefaultCAFile := parseClientTLSConfig(doc, configFile) + log.Info().Msgf("isSecure=%v, keeperHosts=%v, caPath=%v, certPath=%v, keyPath=%v, skipVerify=%v, loadDefaultCAFile=%v, use TLS for keeper connection (from config.xml /openSSL/client/*)", isSecure, keeperHosts, caPath, certPath, keyPath, skipVerify, loadDefaultCAFile) + tlsConfig, err := utils.NewTLSConfig(caPath, certPath, keyPath, skipVerify, loadDefaultCAFile) if err != nil { - return errors.Wrap(err, "can't create TLS config") + return errors.Wrap(err, "can't create TLS config from config.xml /openSSL/client/* settings") } conn, _, err = zk.Connect(keeperHosts, sessionTimeout, zk.WithLogger(newKeeperLogger()), zk.WithDialer(func(network, address string, timeout time.Duration) (net.Conn, error) { tlsConn, dialErr := tls.DialWithDialer(&net.Dialer{Timeout: timeout}, network, address, tlsConfig) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 18e92b2b2..51504d43c 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -77,7 +77,7 @@ func ExecCmdOut(ctx context.Context, timeout time.Duration, cmd string, args ... return string(out), err } -func NewTLSConfig(caPath, certPath, keyPath string, skipVerify bool) (*tls.Config, error) { +func NewTLSConfig(caPath, certPath, keyPath string, skipVerify, loadSystemCAs bool) (*tls.Config, error) { tlsConfig := &tls.Config{ InsecureSkipVerify: skipVerify, } @@ -98,6 +98,9 @@ func NewTLSConfig(caPath, certPath, keyPath string, skipVerify bool) (*tls.Confi return nil, fmt.Errorf("AppendCertsFromPEM %s return false", caPath) } tlsConfig.RootCAs = caCertPool + } else if !loadSystemCAs { + // If RootCAs is nil, TLS uses the host's root CA set + tlsConfig.RootCAs = x509.NewCertPool() } return tlsConfig, nil } diff --git a/test/integration/clickhouse-keeper.xml b/test/integration/clickhouse-keeper.xml index 9d8199562..9dd0bcb22 100644 --- a/test/integration/clickhouse-keeper.xml +++ b/test/integration/clickhouse-keeper.xml @@ -3,7 +3,30 @@ 1 + + + /etc/clickhouse-keeper/server.crt + /etc/clickhouse-keeper/server.key + none + true + true + sslv2,sslv3 + true + + + /etc/clickhouse-keeper/rootCA.crt + true + true + sslv2,sslv3 + true + strict + + RejectCertificateHandler + + + 2181 + 2281 \ No newline at end of file diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml index 979f67e85..51cff9d1a 100644 --- a/test/integration/docker-compose_advanced.yml +++ b/test/integration/docker-compose_advanced.yml @@ -134,6 +134,9 @@ services: hostname: zookeeper volumes: - ./clickhouse-keeper.xml:/etc/clickhouse-keeper/conf.d/clickhouse-keeper.xml + - ./server.crt:/etc/clickhouse-keeper/server.crt + - ./server.key:/etc/clickhouse-keeper/server.key + - ./server.crt:/etc/clickhouse-keeper/rootCA.crt - /var/lib/clickhouse - /var/lib/clickhouse-keeper environment: @@ -207,6 +210,7 @@ services: user: root environment: CLICKHOUSE_VERSION: ${CLICKHOUSE_VERSION:-19.17} + KEEPER_TLS_ENABLED: "${KEEPER_TLS_ENABLED:-false}" CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS: "true" CLICKHOUSE_SKIP_USER_SETUP: "1" TZ: UTC @@ -287,6 +291,7 @@ services: - ./dhparam.pem:/etc/clickhouse-server/dhparam.pem - ./ssl.xml:/etc/clickhouse-server/config.d/ssl.xml - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/clickhouse-config.xml + - ./server.crt:/etc/clickhouse-server/rootCA.crt - /var/lib/clickhouse - /hdd1_data - /hdd2_data diff --git a/test/integration/dynamic_settings.sh b/test/integration/dynamic_settings.sh index 584ae3680..1207ec473 100755 --- a/test/integration/dynamic_settings.sh +++ b/test/integration/dynamic_settings.sh @@ -476,6 +476,36 @@ EOT fi +# Keeper TLS configuration - available from 21.9, when KEEPER_TLS_ENABLED is set +if [[ "${KEEPER_TLS_ENABLED:-}" == "1" ]] && [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.9 || "${CLICKHOUSE_VERSION}" =~ ^21\.1[0-9] || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[1-9] || "${CLICKHOUSE_VERSION}" =~ ^[3-9] ]]; then + +cat < /etc/clickhouse-server/config.d/keeper_tls.xml + + + + /etc/clickhouse-server/rootCA.crt + true + true + sslv2,sslv3 + true + strict + + RejectCertificateHandler + + + + + + zookeeper + 2281 + 1 + + + +EOT + +fi + # @todo LIVE VIEW deprecated, available 21.3+ if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.[3-9] || "${CLICKHOUSE_VERSION}" =~ ^21\.1[0-9] || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[1-9] || "${CLICKHOUSE_VERSION}" =~ ^[3-9] ]]; then diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index d568ea3a6..1fecfaa46 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -3854,6 +3854,64 @@ func TestPostgreSQLMaterialized(t *testing.T) { env.Cleanup(t, r) } +func TestKeeperTLS(t *testing.T) { + if isTestShouldSkip("KEEPER_TLS_ENABLED") { + t.Skip("KEEPER_TLS_ENABLED is not set or false, skipping TestKeeperTLS") + } + env, r := NewTestEnvironment(t) + defer env.Cleanup(t, r) + env.connectWithWait(t, r, 500*time.Millisecond, 1*time.Second, 2*time.Minute) + + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.9") >= 0 { + // create table using ZooKeeper + dbName := "test_keeper_tls" + env.queryWithNoError(r, "CREATE DATABASE IF NOT EXISTS "+dbName) + tableName := "test_table_tls" + zkPath := "/clickhouse/tables/{shard}/{database}/{table}" + onCluster := "" + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.3") >= 0 { + zkPath = "/clickhouse/tables/{shard}/{database}/{table}/{uuid}" + onCluster = "ON CLUSTER '{cluster}'" + } + createSQL := fmt.Sprintf("CREATE TABLE %s.%s %s (id UInt64) ENGINE=ReplicatedMergeTree('%s', '{replica}') ORDER BY id", + dbName, tableName, onCluster, zkPath) + env.queryWithNoError(r, createSQL) + env.queryWithNoError(r, fmt.Sprintf("INSERT INTO %s.%s SELECT number FROM numbers(100)", dbName, tableName)) + + // create RBAC User + rbacUser := "test_keeper_tls_user" + env.queryWithNoError(r, fmt.Sprintf("DROP USER IF EXISTS %s %s", rbacUser, onCluster)) + env.queryWithNoError(r, fmt.Sprintf("CREATE USER %s %s IDENTIFIED WITH sha256_password BY '123'", rbacUser, onCluster)) + defer env.queryWithNoError(r, fmt.Sprintf("DROP USER IF EXISTS %s %s", rbacUser, onCluster)) + + // backup with RBAC + backupName := "test_keeper_tls_backup" + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "--rbac", backupName) + out, err := env.DockerExecOut("clickhouse-backup", "ls", "-laR", fmt.Sprintf("/var/lib/clickhouse/backup/%s/", backupName)) + r.NoError(err) + log.Info().Msgf("Backup directory content:\n%s", out) + + // clean and restore + r.NoError(env.dropDatabase(dbName, false)) + env.queryWithNoError(r, fmt.Sprintf("DROP USER IF EXISTS %s %s", rbacUser, onCluster)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "--rbac", backupName) + + // wait for restart after RBAC restore and check + env.connectWithWait(t, r, 500*time.Millisecond, 1*time.Second, 2*time.Minute) + var rowCount, userCount uint64 + r.NoError(env.ch.SelectSingleRowNoCtx(&rowCount, fmt.Sprintf("SELECT count() FROM %s.%s", dbName, tableName))) + r.Equal(uint64(100), rowCount) + r.NoError(env.ch.SelectSingleRowNoCtx(&userCount, fmt.Sprintf("SELECT count() FROM system.users WHERE name='%s'", rbacUser))) + r.Equal(uint64(1), userCount) + + // cleanup + r.NoError(env.dropDatabase(dbName, false)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", backupName) + } else { + t.Skipf("ClickHouse version %s is too old for KEEPER_TLS_ENABLED tests", os.Getenv("CLICKHOUSE_VERSION")) + } +} + func (env *TestEnvironment) uploadSSHKeys(r *require.Assertions, container string) { r.NoError(env.DockerCP("sftp/clickhouse-backup_rsa", container+":/id_rsa")) env.DockerExecNoError(r, container, "cp", "-vf", "/id_rsa", "/tmp/id_rsa")