Skip to content

Commit 40b6066

Browse files
committed
fix: TestFIPS broken by PR #1366 global openSSL.client strict override
PR #1366 wrote zz_keeper_tls.xml with verificationMode=strict and a CA bundle that doesn't include the test-generated ca-cert.pem used by the clickhouse-backup-fips API server, so ClickHouse failed to verify the cert when inserting into system.backup_actions (CERTIFICATE_VERIFY_FAILED on fips_test.go:112). Revert the global override to verificationMode=none / AcceptCertificateHandler (matching ssl.xml defaults), and move the strict client config into TestKeeperTLS itself where the per-test xml is already written/cleaned up. Also bump CI matrix back to clickhouse 25.8 (away from the 25.8.21 workaround), and small cleanups in serverAPI_test.go / s3_test.go.
1 parent d10b579 commit 40b6066

9 files changed

Lines changed: 66 additions & 59 deletions

File tree

‎.github/workflows/build.yaml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,7 @@ jobs:
110110
- '24.3'
111111
- '24.8'
112112
- '25.3'
113-
# todo wait when resolve https://github.com/ClickHouse/ClickHouse/issues/103232
114-
- '25.8.21'
113+
- '25.8'
115114
- '26.3'
116115
steps:
117116
- name: Checkout project
@@ -214,8 +213,7 @@ jobs:
214213
- '24.3'
215214
- '24.8'
216215
- '25.3'
217-
# todo wait when resolve https://github.com/ClickHouse/ClickHouse/issues/103232
218-
- '25.8.21'
216+
- '25.8'
219217
- '26.3'
220218
steps:
221219
- name: Checkout project

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ GEMINI.md
1818
.claude/
1919
.crush/
2020
.omc/
21+
.omx/
2122
.mcp.json
2223
# TestFlows tests artifacts that shall not be stored in Git
2324
_instances/

‎test/integration/configs/dynamic_settings.sh‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -477,28 +477,25 @@ EOT
477477

478478
fi
479479

480-
# Keeper TLS configuration - available from 21.9, when KEEPER_TLS_ENABLED is set
480+
# Keeper TLS configuration - available from 21.9, when KEEPER_TLS_ENABLED is set.
481+
# Configures the <zookeeper> node to use port 2281 with TLS and explicitly resets
482+
# <openSSL><client> to verificationMode=none / AcceptCertificateHandler so any
483+
# earlier strict override (e.g. left over from a previous test run) cannot leak
484+
# into TestFIPS or other tests. The strict-verification path is applied locally
485+
# inside TestKeeperTLS via a per-test zz_keeper_tls.xml file (see keeperTLS_test.go).
481486
if [[ "${KEEPER_TLS_ENABLED:-}" == "1" ]] && [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.9 || "${CLICKHOUSE_VERSION}" =~ ^21\.1[0-9] || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[1-9] || "${CLICKHOUSE_VERSION}" =~ ^[3-9] ]]; then
482487

483-
cat /etc/clickhouse-server/keeper.crt /etc/clickhouse-server/minio.crt > /etc/clickhouse-server/keeper_tls_ca_bundle.crt
484-
chown clickhouse:clickhouse /etc/clickhouse-server/keeper_tls_ca_bundle.crt
485-
486-
# Keep this file after ssl.xml so the Keeper client settings override its insecure defaults.
487-
cat <<EOT > /etc/clickhouse-server/config.d/zz_keeper_tls.xml
488+
cat <<EOT > /etc/clickhouse-server/config.d/keeper_tls.xml
488489
<yandex>
489490
<openSSL>
490491
<client replace="replace">
491-
<caConfig>/etc/clickhouse-server/keeper_tls_ca_bundle.crt</caConfig>
492-
<certificateFile>/etc/clickhouse-server/keeper.crt</certificateFile>
493-
<privateKeyFile>/etc/clickhouse-server/keeper.key</privateKeyFile>
494492
<loadDefaultCAFile>true</loadDefaultCAFile>
495493
<cacheSessions>true</cacheSessions>
496494
<disableProtocols>sslv2,sslv3</disableProtocols>
497495
<preferServerCiphers>true</preferServerCiphers>
498-
<verificationMode replace="replace">relaxed</verificationMode>
499-
<extendedVerification replace="replace">false</extendedVerification>
496+
<verificationMode replace="replace">none</verificationMode>
500497
<invalidCertificateHandler replace="replace">
501-
<name replace="replace">RejectCertificateHandler</name>
498+
<name replace="replace">AcceptCertificateHandler</name>
502499
</invalidCertificateHandler>
503500
</client>
504501
</openSSL>

‎test/integration/containers.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -877,7 +877,6 @@ func (tc *TestContainers) startClickHouseBackup(ctx context.Context, curDir, con
877877

878878
// Also mount backup binary and configs (same as clickhouse)
879879
binds = append(binds, tc.clickHouseBinds(curDir, configsDir)...)
880-
binds = append(binds, filepath.Join(configsDir, "keeper.crt")+":/etc/clickhouse-server/keeper_tls_ca_bundle.crt")
881880

882881
cfg := &container.Config{
883882
Image: chImage,

‎test/integration/fips_test.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,9 +158,8 @@ func TestFIPS(t *testing.T) {
158158
// P1: Test create_remote doesn't work with clickhouse-server which not compatible with FIPS
159159
fipsOnlyBackupName := fmt.Sprintf("fips_only_backup_%d", rand.Int())
160160
out, err := env.DockerExecOut("clickhouse", "bash", "-ce", "GODEBUG=fips140=only clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml create_remote --tables="+t.Name()+".fips_table "+fipsOnlyBackupName)
161-
r.Error(err)
161+
r.Error(err, "FIPS-compatible clickhouse-backup -> FIPS-incompatible clickhouse-server connection shall return error: %s", out)
162162
r.Contains(out, "is not allowed in FIPS 140-only mode")
163-
log.Debug().Msg(out)
164163

165164
// https://www.perplexity.ai/search/0920f1e8-59ec-4e14-b779-ba7b2e037196
166165
testTLSCerts("rsa", "4096", "", "ECDHE-RSA-AES128-GCM-SHA256", "ECDHE-RSA-AES256-GCM-SHA384", "AES_128_GCM_SHA256", "AES_256_GCM_SHA384")

‎test/integration/keeperTLS_test.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,26 @@ import (
99
"time"
1010
)
1111

12+
const strictKeeperTLSXML = `<yandex>
13+
<openSSL>
14+
<client replace="replace">
15+
<caConfig>/etc/clickhouse-server/keeper.crt</caConfig>
16+
<certificateFile>/etc/clickhouse-server/keeper.crt</certificateFile>
17+
<privateKeyFile>/etc/clickhouse-server/keeper.key</privateKeyFile>
18+
<loadDefaultCAFile>true</loadDefaultCAFile>
19+
<cacheSessions>true</cacheSessions>
20+
<disableProtocols>sslv2,sslv3</disableProtocols>
21+
<preferServerCiphers>true</preferServerCiphers>
22+
<verificationMode replace="replace">relaxed</verificationMode>
23+
<extendedVerification replace="replace">false</extendedVerification>
24+
<invalidCertificateHandler replace="replace">
25+
<name replace="replace">RejectCertificateHandler</name>
26+
</invalidCertificateHandler>
27+
</client>
28+
</openSSL>
29+
</yandex>
30+
`
31+
1232
func TestKeeperTLS(t *testing.T) {
1333
if isTestShouldSkip("KEEPER_TLS_ENABLED") {
1434
t.Skip("KEEPER_TLS_ENABLED is not set or false, skipping TestKeeperTLS")
@@ -20,6 +40,15 @@ func TestKeeperTLS(t *testing.T) {
2040
defer env.Cleanup(t, r)
2141
env.connectWithWait(t, r, 500*time.Millisecond, 1*time.Second, 2*time.Minute)
2242

43+
// Install strict <openSSL><client> so ClickHouse verifies Keeper TLS certs
44+
// (ssl.xml uses verificationMode=none — fine for most tests, but this test
45+
// specifically validates the strict path). Restart so Poco reloads SSL context.
46+
env.DockerExecNoError(r, "clickhouse", "bash", "-c",
47+
fmt.Sprintf("cat > /etc/clickhouse-server/config.d/zz_keeper_tls.xml <<'XML'\n%s\nXML", strictKeeperTLSXML))
48+
env.ch.Close()
49+
r.NoError(env.tc.RestartContainer(t.Context(), "clickhouse"))
50+
env.connectWithWait(t, r, 3*time.Second, 1500*time.Millisecond, 3*time.Minute)
51+
2352
// create table using ZooKeeper
2453
dbName := "test_keeper_tls"
2554
env.queryWithNoError(r, "CREATE DATABASE IF NOT EXISTS "+dbName)
@@ -64,4 +93,9 @@ func TestKeeperTLS(t *testing.T) {
6493
// cleanup
6594
r.NoError(env.dropDatabase(dbName, false))
6695
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", backupName)
96+
97+
env.DockerExecNoError(r, "clickhouse", "rm", "-f", "/etc/clickhouse-server/config.d/zz_keeper_tls.xml")
98+
env.ch.Close()
99+
r.NoError(env.tc.RestartContainer(t.Context(), "clickhouse"))
100+
env.connectWithWait(t, r, 3*time.Second, 1500*time.Millisecond, 3*time.Minute)
67101
}

‎test/integration/s3_test.go‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,34 +3,12 @@
33
package main
44

55
import (
6-
"strings"
76
"testing"
87
)
98

109
func TestS3(t *testing.T) {
1110
env, r := NewTestEnvironment(t)
1211
defer env.Cleanup(t, r)
13-
if shouldProbeKeeperTLS() {
14-
verifyOut, verifyErr := env.DockerExecOut("clickhouse-backup", "openssl", "verify", "-CAfile", "/etc/clickhouse-server/keeper.crt", "-verify_hostname", "zookeeper", "/etc/clickhouse-server/keeper.crt")
15-
r.NoError(verifyErr, verifyOut)
16-
verifyIPOut, verifyIPErr := env.DockerExecOut("clickhouse-backup", "openssl", "verify", "-CAfile", "/etc/clickhouse-server/keeper.crt", "-verify_ip", "127.0.0.1", "/etc/clickhouse-server/keeper.crt")
17-
r.Error(verifyIPErr, verifyIPOut)
18-
r.Contains(strings.ToLower(verifyIPOut), "ip address mismatch")
19-
probeOut, probeErr := env.DockerExecOut(
20-
"clickhouse-backup",
21-
"bash", "-ce",
22-
"LOG_LEVEL=debug ALLOW_EMPTY_BACKUPS=1 clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml create --rbac-only test_s3_keeper_tls_probe; status=$?; clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete local test_s3_keeper_tls_probe || true; exit $status",
23-
)
24-
r.NoError(probeErr, probeOut)
25-
r.Contains(probeOut, "parsed TLS config")
26-
r.Contains(probeOut, "caPath=/etc/clickhouse-server/keeper_tls_ca_bundle.crt")
27-
r.Contains(probeOut, "skipChainVerify=false")
28-
r.Contains(probeOut, "skipHostnameVerify=true")
29-
}
3012
env.checkObjectStorageIsEmpty(t, r, "S3")
3113
env.runMainIntegrationScenario(t, "S3", "config-s3.yml")
3214
}
33-
34-
func shouldProbeKeeperTLS() bool {
35-
return !isTestShouldSkip("KEEPER_TLS_ENABLED") && compareVersion(getEnvDefault("CLICKHOUSE_VERSION", "26.3"), "21.9") >= 0
36-
}

‎test/integration/serverAPI_test.go‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -90,24 +90,6 @@ func testAPIRestart(r *require.Assertions, env *TestEnvironment) {
9090
r.Equal(uint64(0), inProgressActions)
9191
}
9292

93-
func runClickHouseClientInsertSystemBackupActions(r *require.Assertions, env *TestEnvironment, commands []string, needWait bool) {
94-
sql := "INSERT INTO system.backup_actions(command) " + "VALUES ('" + strings.Join(commands, "'),('") + "')"
95-
out, err := env.DockerExecOut("clickhouse", "bash", "-ce", fmt.Sprintf("clickhouse client --echo -mn -q \"%s\"", sql))
96-
r.NoError(err, "%s -> %s unexpected error: %v", sql, out, err)
97-
if needWait {
98-
for _, command := range commands {
99-
for {
100-
time.Sleep(500 * time.Millisecond)
101-
var commandStatus string
102-
r.NoError(env.ch.SelectSingleRowNoCtx(&commandStatus, "SELECT status FROM system.backup_actions WHERE command=?", command))
103-
if commandStatus != status.InProgressStatus {
104-
break
105-
}
106-
}
107-
}
108-
}
109-
}
110-
11193
func testAPIBackupStatus(r *require.Assertions, env *TestEnvironment) {
11294
log.Debug().Msg("Check system.backup_actions with /backup/actions call")
11395
env.queryWithNoError(r, "SELECT count() FROM system.backup_actions")

‎test/integration/utils.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818

1919
"github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse"
2020
"github.com/Altinity/clickhouse-backup/v2/pkg/config"
21+
"github.com/Altinity/clickhouse-backup/v2/pkg/status"
2122

2223
stdlog "log"
2324

@@ -1504,6 +1505,24 @@ func testBackupSpecifiedPartitions(t *testing.T, r *require.Assertions, env *Tes
15041505
log.Debug().Msg("testBackupSpecifiedPartitions finish")
15051506
}
15061507

1508+
func runClickHouseClientInsertSystemBackupActions(r *require.Assertions, env *TestEnvironment, commands []string, needWait bool) {
1509+
sql := "INSERT INTO system.backup_actions(command) " + "VALUES ('" + strings.Join(commands, "'),('") + "')"
1510+
out, err := env.DockerExecOut("clickhouse", "bash", "-ce", fmt.Sprintf("clickhouse client --echo -mn -q \"%s\"", sql))
1511+
r.NoError(err, "%s -> %s unexpected error: %v", sql, out, err)
1512+
if needWait {
1513+
for _, command := range commands {
1514+
for {
1515+
time.Sleep(500 * time.Millisecond)
1516+
var commandStatus string
1517+
r.NoError(env.ch.SelectSingleRowNoCtx(&commandStatus, "SELECT status FROM system.backup_actions WHERE command=?", command))
1518+
if commandStatus != status.InProgressStatus {
1519+
break
1520+
}
1521+
}
1522+
}
1523+
}
1524+
}
1525+
15071526
// Unused import placeholders to ensure compilation
15081527
var _ = bufio.Scanner{}
15091528
var _ = rand.Int

0 commit comments

Comments
 (0)