diff --git a/pkg/clickhouse/clickhouse.go b/pkg/clickhouse/clickhouse.go
index 8e8748fc6..3e02912a9 100644
--- a/pkg/clickhouse/clickhouse.go
+++ b/pkg/clickhouse/clickhouse.go
@@ -2,8 +2,6 @@ package clickhouse
import (
"context"
- "crypto/tls"
- "crypto/x509"
"database/sql"
"fmt"
"os"
@@ -18,6 +16,7 @@ import (
"github.com/Altinity/clickhouse-backup/v2/pkg/common"
"github.com/Altinity/clickhouse-backup/v2/pkg/config"
"github.com/Altinity/clickhouse-backup/v2/pkg/metadata"
+ "github.com/Altinity/clickhouse-backup/v2/pkg/utils"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/antchfx/xmlquery"
@@ -87,31 +86,9 @@ func (ch *ClickHouse) Connect() error {
}
if ch.Config.Secure {
- tlsConfig := &tls.Config{
- InsecureSkipVerify: ch.Config.SkipVerify,
- }
- if ch.Config.TLSKey != "" || ch.Config.TLSCert != "" || ch.Config.TLSCa != "" {
- if ch.Config.TLSCert != "" || ch.Config.TLSKey != "" {
- cert, err := tls.LoadX509KeyPair(ch.Config.TLSCert, ch.Config.TLSKey)
- if err != nil {
- log.Error().Msgf("tls.LoadX509KeyPair error: %v", err)
- return err
- }
- tlsConfig.Certificates = []tls.Certificate{cert}
- }
- if ch.Config.TLSCa != "" {
- caCert, err := os.ReadFile(ch.Config.TLSCa)
- if err != nil {
- log.Error().Msgf("read `tls_ca` file %s return error: %v ", ch.Config.TLSCa, err)
- return err
- }
- caCertPool := x509.NewCertPool()
- if caCertPool.AppendCertsFromPEM(caCert) != true {
- log.Error().Msgf("AppendCertsFromPEM %s return false", ch.Config.TLSCa)
- return fmt.Errorf("AppendCertsFromPEM %s return false", ch.Config.TLSCa)
- }
- tlsConfig.RootCAs = caCertPool
- }
+ tlsConfig, err := utils.NewTLSConfig(ch.Config.TLSCa, ch.Config.TLSCert, ch.Config.TLSKey, ch.Config.SkipVerify, true)
+ if err != nil {
+ return err
}
opt.TLS = tlsConfig
}
diff --git a/pkg/keeper/keeper.go b/pkg/keeper/keeper.go
index 7e20e9c2f..29b882b5b 100644
--- a/pkg/keeper/keeper.go
+++ b/pkg/keeper/keeper.go
@@ -3,9 +3,11 @@ package keeper
import (
"bufio"
"context"
+ "crypto/tls"
"encoding/json"
"fmt"
"io"
+ "net"
"os"
"path"
"strconv"
@@ -18,6 +20,7 @@ import (
"github.com/rs/zerolog/log"
"github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse"
+ "github.com/Altinity/clickhouse-backup/v2/pkg/utils"
"github.com/go-zookeeper/zk"
)
@@ -58,6 +61,46 @@ type Keeper struct {
xmlConfigFile string
}
+// https://clickhouse.com/docs/operations/ssl-zookeeper - parse TLS configuration from ClickHouse's config.xml */openSSL/client/* section
+func parseClientTLSConfig(doc *xmlquery.Node, configFile string) (caPath, certPath, keyPath string, skipVerify, loadDefaultCAFile bool) {
+ clientNode := xmlquery.FindOne(doc, "//openSSL/client")
+ if clientNode == nil {
+ log.Warn().Msgf("no /openSSL/client in %s, using empty TLS config", configFile)
+ return "", "", "", true, true
+ }
+
+ getText := func(selector string) string {
+ if n := clientNode.SelectElement(selector); n != nil {
+ return n.InnerText()
+ }
+ return ""
+ }
+
+ certPath = getText("certificateFile")
+ keyPath = getText("privateKeyFile")
+ caPath = getText("caConfig")
+ loadDefaultCAFile = true
+ if val := getText("loadDefaultCAFile"); val != "" {
+ if useDefault, err := strconv.ParseBool(val); err == nil {
+ loadDefaultCAFile = useDefault
+ }
+ }
+ if !loadDefaultCAFile && caPath == "" {
+ log.Warn().Msgf("//openSSL/client/loadDefaultCAFile=false provided in %s but caConfig is empty; system CAs will NOT be loaded", configFile)
+ }
+
+ if mode := getText("verificationMode"); mode == "none" {
+ skipVerify = true
+ }
+ // invalidCertificateHandler is alternative way to disable verification
+ if handler := getText("invalidCertificateHandler/name"); handler == "AcceptCertificateHandler" {
+ skipVerify = true
+ }
+
+ log.Debug().Msgf("parsed TLS config from %s: certPath=%s, keyPath=%s, caPath=%s, skipVerify=%v, loadDefaultCAFile=%v", configFile, certPath, keyPath, caPath, skipVerify, loadDefaultCAFile)
+ return caPath, certPath, keyPath, skipVerify, loadDefaultCAFile
+}
+
// Connect - connect to any zookeeper server from /var/lib/clickhouse/preprocessed_configs/config.xml
func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse) error {
configFile, doc, err := ch.ParseXML(ctx, "config.xml")
@@ -83,6 +126,7 @@ func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse) error {
return errors.WithStack(fmt.Errorf("/zookeeper/node not exists in %s", configFile))
}
keeperHosts := make([]string, len(nodeList))
+ isSecure := false
for i, node := range nodeList {
hostNode := node.SelectElement("host")
if hostNode == nil {
@@ -93,11 +137,40 @@ func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse) error {
if portNode != nil {
port = portNode.InnerText()
}
+ secureNode := node.SelectElement("secure")
+ if secureNode != nil && (secureNode.InnerText() == "1" || secureNode.InnerText() == "true") {
+ isSecure = true
+ }
keeperHosts[i] = fmt.Sprintf("%s:%s", hostNode.InnerText(), port)
}
- conn, _, err := zk.Connect(keeperHosts, sessionTimeout, zk.WithLogger(newKeeperLogger()))
- if err != nil {
- return err
+ var conn *zk.Conn
+ if isSecure {
+ // Parse TLS config from ClickHouse's config.xml /openSSL/client/* section
+ // according to https://clickhouse.com/docs/operations/ssl-zookeeper
+ caPath, certPath, keyPath, skipVerify, loadDefaultCAFile := parseClientTLSConfig(doc, configFile)
+ log.Info().Msgf("isSecure=%v, keeperHosts=%v, caPath=%v, certPath=%v, keyPath=%v, skipVerify=%v, loadDefaultCAFile=%v, use TLS for keeper connection (from config.xml /openSSL/client/*)", isSecure, keeperHosts, caPath, certPath, keyPath, skipVerify, loadDefaultCAFile)
+ tlsConfig, err := utils.NewTLSConfig(caPath, certPath, keyPath, skipVerify, loadDefaultCAFile)
+ if err != nil {
+ return errors.Wrap(err, "can't create TLS config from config.xml /openSSL/client/* settings")
+ }
+ conn, _, err = zk.Connect(keeperHosts, sessionTimeout, zk.WithLogger(newKeeperLogger()), zk.WithDialer(func(network, address string, timeout time.Duration) (net.Conn, error) {
+ tlsConn, dialErr := tls.DialWithDialer(&net.Dialer{Timeout: timeout}, network, address, tlsConfig)
+ if dialErr != nil {
+ log.Error().Msgf("TLS dial to %s failed: %v", address, dialErr)
+ return nil, dialErr
+ }
+ return tlsConn, nil
+ }))
+ if err != nil {
+ log.Error().Msgf("zk.Connect with TLS failed: %v", err)
+ return err
+ }
+ } else {
+ log.Info().Msgf("isSecure=%v, keeperHosts=%v", isSecure, keeperHosts)
+ conn, _, err = zk.Connect(keeperHosts, sessionTimeout, zk.WithLogger(newKeeperLogger()))
+ if err != nil {
+ return err
+ }
}
if digestNode := zookeeperNode.SelectElement("digest"); digestNode != nil {
if err = conn.AddAuth("digest", []byte(digestNode.InnerText())); err != nil {
diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
index 9e09afcd8..51504d43c 100644
--- a/pkg/utils/utils.go
+++ b/pkg/utils/utils.go
@@ -2,12 +2,16 @@ package utils
import (
"context"
+ "crypto/tls"
+ "crypto/x509"
"fmt"
- "github.com/rs/zerolog/log"
+ "os"
"os/exec"
"regexp"
"strings"
"time"
+
+ "github.com/rs/zerolog/log"
)
const (
@@ -72,3 +76,31 @@ func ExecCmdOut(ctx context.Context, timeout time.Duration, cmd string, args ...
cancel()
return string(out), err
}
+
+func NewTLSConfig(caPath, certPath, keyPath string, skipVerify, loadSystemCAs bool) (*tls.Config, error) {
+ tlsConfig := &tls.Config{
+ InsecureSkipVerify: skipVerify,
+ }
+ if certPath != "" || keyPath != "" {
+ cert, err := tls.LoadX509KeyPair(certPath, keyPath)
+ if err != nil {
+ return nil, fmt.Errorf("tls.LoadX509KeyPair error: %v", err)
+ }
+ tlsConfig.Certificates = []tls.Certificate{cert}
+ }
+ if caPath != "" {
+ caCert, err := os.ReadFile(caPath)
+ if err != nil {
+ return nil, fmt.Errorf("read ca file %s return error: %v", caPath, err)
+ }
+ caCertPool := x509.NewCertPool()
+ if !caCertPool.AppendCertsFromPEM(caCert) {
+ return nil, fmt.Errorf("AppendCertsFromPEM %s return false", caPath)
+ }
+ tlsConfig.RootCAs = caCertPool
+ } else if !loadSystemCAs {
+ // If RootCAs is nil, TLS uses the host's root CA set
+ tlsConfig.RootCAs = x509.NewCertPool()
+ }
+ return tlsConfig, nil
+}
diff --git a/test/integration/clickhouse-keeper.xml b/test/integration/clickhouse-keeper.xml
index 9d8199562..9dd0bcb22 100644
--- a/test/integration/clickhouse-keeper.xml
+++ b/test/integration/clickhouse-keeper.xml
@@ -3,7 +3,30 @@
1
+
+
+ /etc/clickhouse-keeper/server.crt
+ /etc/clickhouse-keeper/server.key
+ none
+ true
+ true
+ sslv2,sslv3
+ true
+
+
+ /etc/clickhouse-keeper/rootCA.crt
+ true
+ true
+ sslv2,sslv3
+ true
+ strict
+
+ RejectCertificateHandler
+
+
+
2181
+ 2281
\ No newline at end of file
diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml
index 979f67e85..51cff9d1a 100644
--- a/test/integration/docker-compose_advanced.yml
+++ b/test/integration/docker-compose_advanced.yml
@@ -134,6 +134,9 @@ services:
hostname: zookeeper
volumes:
- ./clickhouse-keeper.xml:/etc/clickhouse-keeper/conf.d/clickhouse-keeper.xml
+ - ./server.crt:/etc/clickhouse-keeper/server.crt
+ - ./server.key:/etc/clickhouse-keeper/server.key
+ - ./server.crt:/etc/clickhouse-keeper/rootCA.crt
- /var/lib/clickhouse
- /var/lib/clickhouse-keeper
environment:
@@ -207,6 +210,7 @@ services:
user: root
environment:
CLICKHOUSE_VERSION: ${CLICKHOUSE_VERSION:-19.17}
+ KEEPER_TLS_ENABLED: "${KEEPER_TLS_ENABLED:-false}"
CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS: "true"
CLICKHOUSE_SKIP_USER_SETUP: "1"
TZ: UTC
@@ -287,6 +291,7 @@ services:
- ./dhparam.pem:/etc/clickhouse-server/dhparam.pem
- ./ssl.xml:/etc/clickhouse-server/config.d/ssl.xml
- ./clickhouse-config.xml:/etc/clickhouse-server/config.d/clickhouse-config.xml
+ - ./server.crt:/etc/clickhouse-server/rootCA.crt
- /var/lib/clickhouse
- /hdd1_data
- /hdd2_data
diff --git a/test/integration/dynamic_settings.sh b/test/integration/dynamic_settings.sh
index 584ae3680..1207ec473 100755
--- a/test/integration/dynamic_settings.sh
+++ b/test/integration/dynamic_settings.sh
@@ -476,6 +476,36 @@ EOT
fi
+# Keeper TLS configuration - available from 21.9, when KEEPER_TLS_ENABLED is set
+if [[ "${KEEPER_TLS_ENABLED:-}" == "1" ]] && [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.9 || "${CLICKHOUSE_VERSION}" =~ ^21\.1[0-9] || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[1-9] || "${CLICKHOUSE_VERSION}" =~ ^[3-9] ]]; then
+
+cat < /etc/clickhouse-server/config.d/keeper_tls.xml
+
+
+
+ /etc/clickhouse-server/rootCA.crt
+ true
+ true
+ sslv2,sslv3
+ true
+ strict
+
+ RejectCertificateHandler
+
+
+
+
+
+ zookeeper
+ 2281
+ 1
+
+
+
+EOT
+
+fi
+
# @todo LIVE VIEW deprecated, available 21.3+
if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.[3-9] || "${CLICKHOUSE_VERSION}" =~ ^21\.1[0-9] || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[1-9] || "${CLICKHOUSE_VERSION}" =~ ^[3-9] ]]; then
diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go
index d568ea3a6..1fecfaa46 100644
--- a/test/integration/integration_test.go
+++ b/test/integration/integration_test.go
@@ -3854,6 +3854,64 @@ func TestPostgreSQLMaterialized(t *testing.T) {
env.Cleanup(t, r)
}
+func TestKeeperTLS(t *testing.T) {
+ if isTestShouldSkip("KEEPER_TLS_ENABLED") {
+ t.Skip("KEEPER_TLS_ENABLED is not set or false, skipping TestKeeperTLS")
+ }
+ env, r := NewTestEnvironment(t)
+ defer env.Cleanup(t, r)
+ env.connectWithWait(t, r, 500*time.Millisecond, 1*time.Second, 2*time.Minute)
+
+ if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.9") >= 0 {
+ // create table using ZooKeeper
+ dbName := "test_keeper_tls"
+ env.queryWithNoError(r, "CREATE DATABASE IF NOT EXISTS "+dbName)
+ tableName := "test_table_tls"
+ zkPath := "/clickhouse/tables/{shard}/{database}/{table}"
+ onCluster := ""
+ if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.3") >= 0 {
+ zkPath = "/clickhouse/tables/{shard}/{database}/{table}/{uuid}"
+ onCluster = "ON CLUSTER '{cluster}'"
+ }
+ createSQL := fmt.Sprintf("CREATE TABLE %s.%s %s (id UInt64) ENGINE=ReplicatedMergeTree('%s', '{replica}') ORDER BY id",
+ dbName, tableName, onCluster, zkPath)
+ env.queryWithNoError(r, createSQL)
+ env.queryWithNoError(r, fmt.Sprintf("INSERT INTO %s.%s SELECT number FROM numbers(100)", dbName, tableName))
+
+ // create RBAC User
+ rbacUser := "test_keeper_tls_user"
+ env.queryWithNoError(r, fmt.Sprintf("DROP USER IF EXISTS %s %s", rbacUser, onCluster))
+ env.queryWithNoError(r, fmt.Sprintf("CREATE USER %s %s IDENTIFIED WITH sha256_password BY '123'", rbacUser, onCluster))
+ defer env.queryWithNoError(r, fmt.Sprintf("DROP USER IF EXISTS %s %s", rbacUser, onCluster))
+
+ // backup with RBAC
+ backupName := "test_keeper_tls_backup"
+ env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "--rbac", backupName)
+ out, err := env.DockerExecOut("clickhouse-backup", "ls", "-laR", fmt.Sprintf("/var/lib/clickhouse/backup/%s/", backupName))
+ r.NoError(err)
+ log.Info().Msgf("Backup directory content:\n%s", out)
+
+ // clean and restore
+ r.NoError(env.dropDatabase(dbName, false))
+ env.queryWithNoError(r, fmt.Sprintf("DROP USER IF EXISTS %s %s", rbacUser, onCluster))
+ env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "--rbac", backupName)
+
+ // wait for restart after RBAC restore and check
+ env.connectWithWait(t, r, 500*time.Millisecond, 1*time.Second, 2*time.Minute)
+ var rowCount, userCount uint64
+ r.NoError(env.ch.SelectSingleRowNoCtx(&rowCount, fmt.Sprintf("SELECT count() FROM %s.%s", dbName, tableName)))
+ r.Equal(uint64(100), rowCount)
+ r.NoError(env.ch.SelectSingleRowNoCtx(&userCount, fmt.Sprintf("SELECT count() FROM system.users WHERE name='%s'", rbacUser)))
+ r.Equal(uint64(1), userCount)
+
+ // cleanup
+ r.NoError(env.dropDatabase(dbName, false))
+ env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", backupName)
+ } else {
+ t.Skipf("ClickHouse version %s is too old for KEEPER_TLS_ENABLED tests", os.Getenv("CLICKHOUSE_VERSION"))
+ }
+}
+
func (env *TestEnvironment) uploadSSHKeys(r *require.Assertions, container string) {
r.NoError(env.DockerCP("sftp/clickhouse-backup_rsa", container+":/id_rsa"))
env.DockerExecNoError(r, container, "cp", "-vf", "/id_rsa", "/tmp/id_rsa")