Skip to content

Commit 6903aa2

Browse files
authored
Merge pull request #1578 from assimelha/fix/close-failed-clickhouse-pools
Close failed ClickHouse connection pools before retrying
2 parents 156d289 + 86b573b commit 6903aa2

6 files changed

Lines changed: 116 additions & 5 deletions

File tree

‎pkg/clickhouse/clickhouse.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,12 @@ func (ch *ClickHouse) Connect() error {
172172
ch.IsOpen = true
173173
break
174174
}
175+
// Open creates a pool even when Ping fails. Close it before retrying or
176+
// returning so its drain goroutine and other resources are released.
177+
if closeErr := ch.conn.Close(); closeErr != nil {
178+
log.Warn().Msgf("can't close failed clickhouse connection: %v", closeErr)
179+
}
180+
ch.conn = nil
175181
if ch.BreakConnectOnError || ch.shutdownRequested() || strings.Contains(err.Error(), "FIPS 140-only") {
176182
return errors.WithStack(err)
177183
}

‎pkg/clickhouse/connect_test.go‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package clickhouse
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"net"
7+
"runtime/pprof"
8+
"strings"
9+
"sync/atomic"
10+
"testing"
11+
"time"
12+
13+
"github.com/Altinity/clickhouse-backup/v2/pkg/config"
14+
"github.com/stretchr/testify/require"
15+
)
16+
17+
func TestConnectClosesFailedPools(t *testing.T) {
18+
for _, stop := range []string{"break on error", "shutdown", "retry then shutdown"} {
19+
t.Run(stop, func(t *testing.T) {
20+
ctx, cancel := context.WithCancel(context.Background())
21+
defer cancel()
22+
var attempts atomic.Int32
23+
listener, err := net.Listen("tcp", "127.0.0.1:0")
24+
require.NoError(t, err)
25+
defer listener.Close()
26+
serverDone := make(chan struct{})
27+
go func() {
28+
defer close(serverDone)
29+
for {
30+
conn, err := listener.Accept()
31+
if err != nil {
32+
return
33+
}
34+
// Reject the native handshake so Open succeeds but Ping fails.
35+
if attempts.Add(1) == 2 && stop == "retry then shutdown" {
36+
cancel()
37+
}
38+
_ = conn.Close()
39+
}
40+
}()
41+
t.Cleanup(func() {
42+
_ = listener.Close()
43+
<-serverDone
44+
})
45+
46+
ch := NewClickHouse(&config.ClickHouseConfig{
47+
Host: "127.0.0.1",
48+
Port: uint(listener.Addr().(*net.TCPAddr).Port),
49+
Timeout: "1s",
50+
MaxConnections: 1,
51+
})
52+
ch.ShutdownCtx = ctx
53+
ch.BreakConnectOnError = stop == "break on error"
54+
if stop == "shutdown" {
55+
cancel()
56+
}
57+
58+
before := countDrainPoolGoroutines(t)
59+
// Bound the retry test even if it regresses and stops observing shutdown.
60+
connectDone := make(chan error, 1)
61+
go func() { connectDone <- ch.Connect() }()
62+
select {
63+
case err := <-connectDone:
64+
require.Error(t, err)
65+
case <-time.After(10 * time.Second):
66+
t.Fatal("Connect did not stop after a failed Ping")
67+
}
68+
// Clean up the last pool on failure, without hiding the leak assertion.
69+
if ch.conn != nil {
70+
t.Cleanup(func() { _ = ch.conn.Close() })
71+
}
72+
wantAttempts := int32(1)
73+
if stop == "retry then shutdown" {
74+
wantAttempts = 2
75+
}
76+
require.Equal(t, wantAttempts, attempts.Load())
77+
require.False(t, ch.IsOpen)
78+
require.Eventually(t, func() bool {
79+
return countDrainPoolGoroutines(t) == before
80+
}, time.Second, 10*time.Millisecond, "failed Ping left a connection pool running")
81+
require.Nil(t, ch.conn)
82+
})
83+
}
84+
}
85+
86+
// Each native driver pool owns a drain goroutine, including pools with no open
87+
// sockets. Checking these catches a leak that connection statistics cannot see.
88+
func countDrainPoolGoroutines(t *testing.T) int {
89+
t.Helper()
90+
var stacks bytes.Buffer
91+
require.NoError(t, pprof.Lookup("goroutine").WriteTo(&stacks, 2))
92+
return strings.Count(stacks.String(), "github.com/ClickHouse/clickhouse-go/v2.(*connPool).runDrainPool(")
93+
}

‎test/integration/containers.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -680,6 +680,11 @@ func (tc *TestContainers) pullImageIfNeeded(ctx context.Context, imageName strin
680680
log.Debug().Msgf("image %s already exists locally, skipping pull", imageName)
681681
return nil
682682
}
683+
return tc.pullImage(ctx, imageName)
684+
}
685+
686+
// pullImage always pulls, so floating tags like 26.3 or latest-alpine are refreshed to the same image CI gets
687+
func (tc *TestContainers) pullImage(ctx context.Context, imageName string) error {
683688
reader, err := tc.client.ImagePull(ctx, imageName, dockerClient.ImagePullOptions{})
684689
if err != nil {
685690
return fmt.Errorf("pull %s: %w", imageName, err)
@@ -695,7 +700,7 @@ func (tc *TestContainers) pullImageIfNeeded(ctx context.Context, imageName strin
695700
return fmt.Errorf("pull %s: read progress: %w", imageName, err)
696701
}
697702
}
698-
if _, inspectErr = tc.client.ImageInspect(ctx, imageName); inspectErr != nil {
703+
if _, inspectErr := tc.client.ImageInspect(ctx, imageName); inspectErr != nil {
699704
return fmt.Errorf("pull %s: image is still missing after pull: %w", imageName, inspectErr)
700705
}
701706
return nil

‎test/integration/main_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ func prePullImages() {
112112
"mcr.microsoft.com/azure-storage/azurite:latest",
113113
"mcr.microsoft.com/azure-cli:latest",
114114
"gcr.io/google.com/cloudsdktool/google-cloud-cli:slim",
115+
"amazon/aws-cli:latest",
115116
chImage,
116117
}
117118

@@ -139,7 +140,7 @@ func prePullImages() {
139140

140141
for _, img := range images {
141142
log.Info().Msgf("pre-pulling image %s", img)
142-
if err = tc.pullImageIfNeeded(ctx, img); err != nil {
143+
if err = tc.pullImage(ctx, img); err != nil {
143144
// not fatal here, startContainer will fail with the same error when the image is really needed
144145
log.Warn().Err(err).Msgf("prePullImages: can't pull %s", img)
145146
}

‎test/integration/utils.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -860,9 +860,9 @@ func (env *TestEnvironment) createTestSchema(t *testing.T, data TestDataStruct,
860860
createSQL = strings.Replace(createSQL, "{database}", data.Database, -1)
861861
}
862862
}
863-
// since 26.8 a {database}/{table} macro whose value contains '/', '{' or '}' is rejected in replicated ZooKeeper paths
864-
// (https://github.com/ClickHouse/ClickHouse/commit/617b6626fcf), explicit literal paths are still allowed
865-
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "26.8") >= 0 {
863+
// since 26.8 (backported to 26.3.34, 26.7) a {database}/{table} macro whose value contains '/', '{' or '}' is rejected in replicated ZooKeeper paths
864+
// (https://github.com/ClickHouse/ClickHouse/pull/114006), explicit literal paths are still allowed
865+
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "26.3") >= 0 {
866866
if strings.Contains(createSQL, "{database}") && strings.ContainsAny(data.Database, "/{}") {
867867
createSQL = strings.Replace(createSQL, "{database}", data.Database, -1)
868868
}

‎test/testflows/clickhouse_backup/regression.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,16 @@
55
# of Altinity LTD. Any dissemination of this information or
66
# reproduction of this material is strictly forbidden unless
77
# prior written permission is obtained from Altinity LTD.
8+
import faulthandler
89
import os
910
import shutil
11+
import signal
1012
import sys
1113
import yaml
14+
15+
# run.sh kills a hung suite via `timeout` (SIGTERM), dump all thread stacks to stderr first to see where it hung,
16+
# chain=True keeps the default SIGTERM termination, so `timeout` still exits with 124
17+
faulthandler.register(signal.SIGTERM, all_threads=True, chain=True)
1218
import testflows.settings as testflows_settings
1319
from testflows.core import *
1420

0 commit comments

Comments
 (0)