Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 4 additions & 27 deletions pkg/clickhouse/clickhouse.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package clickhouse

import (
"context"
"crypto/tls"
"crypto/x509"
"database/sql"
"fmt"
"os"
Expand All @@ -18,6 +16,7 @@ import (
"github.com/Altinity/clickhouse-backup/v2/pkg/common"
"github.com/Altinity/clickhouse-backup/v2/pkg/config"
"github.com/Altinity/clickhouse-backup/v2/pkg/metadata"
"github.com/Altinity/clickhouse-backup/v2/pkg/utils"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/antchfx/xmlquery"
Expand Down Expand Up @@ -87,31 +86,9 @@ func (ch *ClickHouse) Connect() error {
}

if ch.Config.Secure {
tlsConfig := &tls.Config{
InsecureSkipVerify: ch.Config.SkipVerify,
}
if ch.Config.TLSKey != "" || ch.Config.TLSCert != "" || ch.Config.TLSCa != "" {
if ch.Config.TLSCert != "" || ch.Config.TLSKey != "" {
cert, err := tls.LoadX509KeyPair(ch.Config.TLSCert, ch.Config.TLSKey)
if err != nil {
log.Error().Msgf("tls.LoadX509KeyPair error: %v", err)
return err
}
tlsConfig.Certificates = []tls.Certificate{cert}
}
if ch.Config.TLSCa != "" {
caCert, err := os.ReadFile(ch.Config.TLSCa)
if err != nil {
log.Error().Msgf("read `tls_ca` file %s return error: %v ", ch.Config.TLSCa, err)
return err
}
caCertPool := x509.NewCertPool()
if caCertPool.AppendCertsFromPEM(caCert) != true {
log.Error().Msgf("AppendCertsFromPEM %s return false", ch.Config.TLSCa)
return fmt.Errorf("AppendCertsFromPEM %s return false", ch.Config.TLSCa)
}
tlsConfig.RootCAs = caCertPool
}
tlsConfig, err := utils.NewTLSConfig(ch.Config.TLSCa, ch.Config.TLSCert, ch.Config.TLSKey, ch.Config.SkipVerify, true)
if err != nil {
return err
}
opt.TLS = tlsConfig
}
Expand Down
79 changes: 76 additions & 3 deletions pkg/keeper/keeper.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ package keeper
import (
"bufio"
"context"
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net"
"os"
"path"
"strconv"
Expand All @@ -18,6 +20,7 @@ import (
"github.com/rs/zerolog/log"

"github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse"
"github.com/Altinity/clickhouse-backup/v2/pkg/utils"
"github.com/go-zookeeper/zk"
)

Expand Down Expand Up @@ -58,6 +61,46 @@ type Keeper struct {
xmlConfigFile string
}

// https://clickhouse.com/docs/operations/ssl-zookeeper - parse TLS configuration from ClickHouse's config.xml */openSSL/client/* section
func parseClientTLSConfig(doc *xmlquery.Node, configFile string) (caPath, certPath, keyPath string, skipVerify, loadDefaultCAFile bool) {
clientNode := xmlquery.FindOne(doc, "//openSSL/client")
if clientNode == nil {
log.Warn().Msgf("no /openSSL/client in %s, using empty TLS config", configFile)
return "", "", "", true, true
}

getText := func(selector string) string {
if n := clientNode.SelectElement(selector); n != nil {
return n.InnerText()
}
return ""
}

certPath = getText("certificateFile")
keyPath = getText("privateKeyFile")
caPath = getText("caConfig")
loadDefaultCAFile = true
if val := getText("loadDefaultCAFile"); val != "" {
if useDefault, err := strconv.ParseBool(val); err == nil {
loadDefaultCAFile = useDefault
}
}
if !loadDefaultCAFile && caPath == "" {
log.Warn().Msgf("//openSSL/client/loadDefaultCAFile=false provided in %s but caConfig is empty; system CAs will NOT be loaded", configFile)
}

if mode := getText("verificationMode"); mode == "none" {
skipVerify = true
}
// invalidCertificateHandler is alternative way to disable verification
if handler := getText("invalidCertificateHandler/name"); handler == "AcceptCertificateHandler" {
skipVerify = true
}

log.Debug().Msgf("parsed TLS config from %s: certPath=%s, keyPath=%s, caPath=%s, skipVerify=%v, loadDefaultCAFile=%v", configFile, certPath, keyPath, caPath, skipVerify, loadDefaultCAFile)
return caPath, certPath, keyPath, skipVerify, loadDefaultCAFile
}

// Connect - connect to any zookeeper server from /var/lib/clickhouse/preprocessed_configs/config.xml
func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse) error {
configFile, doc, err := ch.ParseXML(ctx, "config.xml")
Expand All @@ -83,6 +126,7 @@ func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse) error {
return errors.WithStack(fmt.Errorf("/zookeeper/node not exists in %s", configFile))
}
keeperHosts := make([]string, len(nodeList))
isSecure := false
for i, node := range nodeList {
hostNode := node.SelectElement("host")
if hostNode == nil {
Expand All @@ -93,11 +137,40 @@ func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse) error {
if portNode != nil {
port = portNode.InnerText()
}
secureNode := node.SelectElement("secure")
if secureNode != nil && (secureNode.InnerText() == "1" || secureNode.InnerText() == "true") {
isSecure = true
}
keeperHosts[i] = fmt.Sprintf("%s:%s", hostNode.InnerText(), port)
}
conn, _, err := zk.Connect(keeperHosts, sessionTimeout, zk.WithLogger(newKeeperLogger()))
if err != nil {
return err
var conn *zk.Conn
if isSecure {
// Parse TLS config from ClickHouse's config.xml /openSSL/client/* section
// according to https://clickhouse.com/docs/operations/ssl-zookeeper
caPath, certPath, keyPath, skipVerify, loadDefaultCAFile := parseClientTLSConfig(doc, configFile)
log.Info().Msgf("isSecure=%v, keeperHosts=%v, caPath=%v, certPath=%v, keyPath=%v, skipVerify=%v, loadDefaultCAFile=%v, use TLS for keeper connection (from config.xml /openSSL/client/*)", isSecure, keeperHosts, caPath, certPath, keyPath, skipVerify, loadDefaultCAFile)
tlsConfig, err := utils.NewTLSConfig(caPath, certPath, keyPath, skipVerify, loadDefaultCAFile)
if err != nil {
return errors.Wrap(err, "can't create TLS config from config.xml /openSSL/client/* settings")
}
conn, _, err = zk.Connect(keeperHosts, sessionTimeout, zk.WithLogger(newKeeperLogger()), zk.WithDialer(func(network, address string, timeout time.Duration) (net.Conn, error) {
tlsConn, dialErr := tls.DialWithDialer(&net.Dialer{Timeout: timeout}, network, address, tlsConfig)
if dialErr != nil {
log.Error().Msgf("TLS dial to %s failed: %v", address, dialErr)
return nil, dialErr
}
return tlsConn, nil
}))
if err != nil {
log.Error().Msgf("zk.Connect with TLS failed: %v", err)
return err
}
} else {
log.Info().Msgf("isSecure=%v, keeperHosts=%v", isSecure, keeperHosts)
conn, _, err = zk.Connect(keeperHosts, sessionTimeout, zk.WithLogger(newKeeperLogger()))
if err != nil {
return err
}
}
if digestNode := zookeeperNode.SelectElement("digest"); digestNode != nil {
if err = conn.AddAuth("digest", []byte(digestNode.InnerText())); err != nil {
Expand Down
34 changes: 33 additions & 1 deletion pkg/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,16 @@ package utils

import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"github.com/rs/zerolog/log"
"os"
"os/exec"
"regexp"
"strings"
"time"

"github.com/rs/zerolog/log"
)

const (
Expand Down Expand Up @@ -72,3 +76,31 @@ func ExecCmdOut(ctx context.Context, timeout time.Duration, cmd string, args ...
cancel()
return string(out), err
}

func NewTLSConfig(caPath, certPath, keyPath string, skipVerify, loadSystemCAs bool) (*tls.Config, error) {
tlsConfig := &tls.Config{
InsecureSkipVerify: skipVerify,
}
if certPath != "" || keyPath != "" {
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, fmt.Errorf("tls.LoadX509KeyPair error: %v", err)
}
tlsConfig.Certificates = []tls.Certificate{cert}
}
if caPath != "" {
caCert, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("read ca file %s return error: %v", caPath, err)
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM(caCert) {
return nil, fmt.Errorf("AppendCertsFromPEM %s return false", caPath)
}
tlsConfig.RootCAs = caCertPool
} else if !loadSystemCAs {
// If RootCAs is nil, TLS uses the host's root CA set
tlsConfig.RootCAs = x509.NewCertPool()
}
return tlsConfig, nil
}
23 changes: 23 additions & 0 deletions test/integration/clickhouse-keeper.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,30 @@
<logger>
<console>1</console>
</logger>
<openSSL>
<server>
<certificateFile>/etc/clickhouse-keeper/server.crt</certificateFile>
<privateKeyFile>/etc/clickhouse-keeper/server.key</privateKeyFile>
<verificationMode>none</verificationMode>
<loadDefaultCAFile>true</loadDefaultCAFile>
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
</server>
<client>
<caConfig>/etc/clickhouse-keeper/rootCA.crt</caConfig>
<loadDefaultCAFile>true</loadDefaultCAFile>
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
<verificationMode>strict</verificationMode>
<invalidCertificateHandler>
<name>RejectCertificateHandler</name>
</invalidCertificateHandler>
</client>
</openSSL>
<keeper_server>
<tcp_port>2181</tcp_port>
<tcp_port_secure>2281</tcp_port_secure>
</keeper_server>
</clickhouse>
5 changes: 5 additions & 0 deletions test/integration/docker-compose_advanced.yml
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,9 @@ services:
hostname: zookeeper
volumes:
- ./clickhouse-keeper.xml:/etc/clickhouse-keeper/conf.d/clickhouse-keeper.xml
- ./server.crt:/etc/clickhouse-keeper/server.crt
- ./server.key:/etc/clickhouse-keeper/server.key
- ./server.crt:/etc/clickhouse-keeper/rootCA.crt
- /var/lib/clickhouse
- /var/lib/clickhouse-keeper
environment:
Expand Down Expand Up @@ -207,6 +210,7 @@ services:
user: root
environment:
CLICKHOUSE_VERSION: ${CLICKHOUSE_VERSION:-19.17}
KEEPER_TLS_ENABLED: "${KEEPER_TLS_ENABLED:-false}"
CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS: "true"
CLICKHOUSE_SKIP_USER_SETUP: "1"
TZ: UTC
Expand Down Expand Up @@ -287,6 +291,7 @@ services:
- ./dhparam.pem:/etc/clickhouse-server/dhparam.pem
- ./ssl.xml:/etc/clickhouse-server/config.d/ssl.xml
- ./clickhouse-config.xml:/etc/clickhouse-server/config.d/clickhouse-config.xml
- ./server.crt:/etc/clickhouse-server/rootCA.crt
- /var/lib/clickhouse
- /hdd1_data
- /hdd2_data
Expand Down
30 changes: 30 additions & 0 deletions test/integration/dynamic_settings.sh
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,36 @@ EOT

fi

# Keeper TLS configuration - available from 21.9, when KEEPER_TLS_ENABLED is set
if [[ "${KEEPER_TLS_ENABLED:-}" == "1" ]] && [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.9 || "${CLICKHOUSE_VERSION}" =~ ^21\.1[0-9] || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[1-9] || "${CLICKHOUSE_VERSION}" =~ ^[3-9] ]]; then

cat <<EOT > /etc/clickhouse-server/config.d/keeper_tls.xml
<yandex>
<openSSL>
<client>
<caConfig>/etc/clickhouse-server/rootCA.crt</caConfig>
<loadDefaultCAFile>true</loadDefaultCAFile>
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
<verificationMode>strict</verificationMode>
<invalidCertificateHandler>
<name>RejectCertificateHandler</name>
</invalidCertificateHandler>
</client>
</openSSL>
<zookeeper>
<node>
<host>zookeeper</host>
<port>2281</port>
<secure>1</secure>
</node>
</zookeeper>
</yandex>
EOT

fi

# @todo LIVE VIEW deprecated, available 21.3+
if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.[3-9] || "${CLICKHOUSE_VERSION}" =~ ^21\.1[0-9] || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[1-9] || "${CLICKHOUSE_VERSION}" =~ ^[3-9] ]]; then

Expand Down
58 changes: 58 additions & 0 deletions test/integration/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3854,6 +3854,64 @@ func TestPostgreSQLMaterialized(t *testing.T) {
env.Cleanup(t, r)
}

func TestKeeperTLS(t *testing.T) {
if isTestShouldSkip("KEEPER_TLS_ENABLED") {
t.Skip("KEEPER_TLS_ENABLED is not set or false, skipping TestKeeperTLS")
}
env, r := NewTestEnvironment(t)
defer env.Cleanup(t, r)
env.connectWithWait(t, r, 500*time.Millisecond, 1*time.Second, 2*time.Minute)

if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.9") >= 0 {
// create table using ZooKeeper
dbName := "test_keeper_tls"
env.queryWithNoError(r, "CREATE DATABASE IF NOT EXISTS "+dbName)
tableName := "test_table_tls"
zkPath := "/clickhouse/tables/{shard}/{database}/{table}"
onCluster := ""
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.3") >= 0 {
zkPath = "/clickhouse/tables/{shard}/{database}/{table}/{uuid}"
onCluster = "ON CLUSTER '{cluster}'"
}
createSQL := fmt.Sprintf("CREATE TABLE %s.%s %s (id UInt64) ENGINE=ReplicatedMergeTree('%s', '{replica}') ORDER BY id",
dbName, tableName, onCluster, zkPath)
env.queryWithNoError(r, createSQL)
env.queryWithNoError(r, fmt.Sprintf("INSERT INTO %s.%s SELECT number FROM numbers(100)", dbName, tableName))

// create RBAC User
rbacUser := "test_keeper_tls_user"
env.queryWithNoError(r, fmt.Sprintf("DROP USER IF EXISTS %s %s", rbacUser, onCluster))
env.queryWithNoError(r, fmt.Sprintf("CREATE USER %s %s IDENTIFIED WITH sha256_password BY '123'", rbacUser, onCluster))
defer env.queryWithNoError(r, fmt.Sprintf("DROP USER IF EXISTS %s %s", rbacUser, onCluster))

// backup with RBAC
backupName := "test_keeper_tls_backup"
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "--rbac", backupName)
out, err := env.DockerExecOut("clickhouse-backup", "ls", "-laR", fmt.Sprintf("/var/lib/clickhouse/backup/%s/", backupName))
r.NoError(err)
log.Info().Msgf("Backup directory content:\n%s", out)

// clean and restore
r.NoError(env.dropDatabase(dbName, false))
env.queryWithNoError(r, fmt.Sprintf("DROP USER IF EXISTS %s %s", rbacUser, onCluster))
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "--rbac", backupName)

// wait for restart after RBAC restore and check
env.connectWithWait(t, r, 500*time.Millisecond, 1*time.Second, 2*time.Minute)
var rowCount, userCount uint64
r.NoError(env.ch.SelectSingleRowNoCtx(&rowCount, fmt.Sprintf("SELECT count() FROM %s.%s", dbName, tableName)))
r.Equal(uint64(100), rowCount)
r.NoError(env.ch.SelectSingleRowNoCtx(&userCount, fmt.Sprintf("SELECT count() FROM system.users WHERE name='%s'", rbacUser)))
r.Equal(uint64(1), userCount)

// cleanup
r.NoError(env.dropDatabase(dbName, false))
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", backupName)
} else {
t.Skipf("ClickHouse version %s is too old for KEEPER_TLS_ENABLED tests", os.Getenv("CLICKHOUSE_VERSION"))
}
}

func (env *TestEnvironment) uploadSSHKeys(r *require.Assertions, container string) {
r.NoError(env.DockerCP("sftp/clickhouse-backup_rsa", container+":/id_rsa"))
env.DockerExecNoError(r, container, "cp", "-vf", "/id_rsa", "/tmp/id_rsa")
Expand Down