Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions pkg/clickhouse/clickhouse.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,12 @@ func (ch *ClickHouse) Connect() error {
ch.IsOpen = true
break
}
// Open creates a pool even when Ping fails. Close it before retrying or
// returning so its drain goroutine and other resources are released.
if closeErr := ch.conn.Close(); closeErr != nil {
log.Warn().Msgf("can't close failed clickhouse connection: %v", closeErr)
}
ch.conn = nil
if ch.BreakConnectOnError || ch.shutdownRequested() || strings.Contains(err.Error(), "FIPS 140-only") {
return errors.WithStack(err)
}
Expand Down
93 changes: 93 additions & 0 deletions pkg/clickhouse/connect_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package clickhouse

import (
"bytes"
"context"
"net"
"runtime/pprof"
"strings"
"sync/atomic"
"testing"
"time"

"github.com/Altinity/clickhouse-backup/v2/pkg/config"
"github.com/stretchr/testify/require"
)

func TestConnectClosesFailedPools(t *testing.T) {
for _, stop := range []string{"break on error", "shutdown", "retry then shutdown"} {
t.Run(stop, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var attempts atomic.Int32
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer listener.Close()
serverDone := make(chan struct{})
go func() {
defer close(serverDone)
for {
conn, err := listener.Accept()
if err != nil {
return
}
// Reject the native handshake so Open succeeds but Ping fails.
if attempts.Add(1) == 2 && stop == "retry then shutdown" {
cancel()
}
_ = conn.Close()
}
}()
t.Cleanup(func() {
_ = listener.Close()
<-serverDone
})

ch := NewClickHouse(&config.ClickHouseConfig{
Host: "127.0.0.1",
Port: uint(listener.Addr().(*net.TCPAddr).Port),
Timeout: "1s",
MaxConnections: 1,
})
ch.ShutdownCtx = ctx
ch.BreakConnectOnError = stop == "break on error"
if stop == "shutdown" {
cancel()
}

before := countDrainPoolGoroutines(t)
// Bound the retry test even if it regresses and stops observing shutdown.
connectDone := make(chan error, 1)
go func() { connectDone <- ch.Connect() }()
select {
case err := <-connectDone:
require.Error(t, err)
case <-time.After(10 * time.Second):
t.Fatal("Connect did not stop after a failed Ping")
}
// Clean up the last pool on failure, without hiding the leak assertion.
if ch.conn != nil {
t.Cleanup(func() { _ = ch.conn.Close() })
}
wantAttempts := int32(1)
if stop == "retry then shutdown" {
wantAttempts = 2
}
require.Equal(t, wantAttempts, attempts.Load())
require.False(t, ch.IsOpen)
require.Eventually(t, func() bool {
return countDrainPoolGoroutines(t) == before
}, time.Second, 10*time.Millisecond, "failed Ping left a connection pool running")
require.Nil(t, ch.conn)
})
}
}

// Each native driver pool owns a drain goroutine, including pools with no open
// sockets. Checking these catches a leak that connection statistics cannot see.
func countDrainPoolGoroutines(t *testing.T) int {
t.Helper()
var stacks bytes.Buffer
require.NoError(t, pprof.Lookup("goroutine").WriteTo(&stacks, 2))
return strings.Count(stacks.String(), "github.com/ClickHouse/clickhouse-go/v2.(*connPool).runDrainPool(")
}
7 changes: 6 additions & 1 deletion test/integration/containers.go
Original file line number Diff line number Diff line change
Expand Up @@ -680,6 +680,11 @@ func (tc *TestContainers) pullImageIfNeeded(ctx context.Context, imageName strin
log.Debug().Msgf("image %s already exists locally, skipping pull", imageName)
return nil
}
return tc.pullImage(ctx, imageName)
}

// pullImage always pulls, so floating tags like 26.3 or latest-alpine are refreshed to the same image CI gets
func (tc *TestContainers) pullImage(ctx context.Context, imageName string) error {
reader, err := tc.client.ImagePull(ctx, imageName, dockerClient.ImagePullOptions{})
if err != nil {
return fmt.Errorf("pull %s: %w", imageName, err)
Expand All @@ -695,7 +700,7 @@ func (tc *TestContainers) pullImageIfNeeded(ctx context.Context, imageName strin
return fmt.Errorf("pull %s: read progress: %w", imageName, err)
}
}
if _, inspectErr = tc.client.ImageInspect(ctx, imageName); inspectErr != nil {
if _, inspectErr := tc.client.ImageInspect(ctx, imageName); inspectErr != nil {
return fmt.Errorf("pull %s: image is still missing after pull: %w", imageName, inspectErr)
}
return nil
Expand Down
3 changes: 2 additions & 1 deletion test/integration/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ func prePullImages() {
"mcr.microsoft.com/azure-storage/azurite:latest",
"mcr.microsoft.com/azure-cli:latest",
"gcr.io/google.com/cloudsdktool/google-cloud-cli:slim",
"amazon/aws-cli:latest",
chImage,
}

Expand Down Expand Up @@ -139,7 +140,7 @@ func prePullImages() {

for _, img := range images {
log.Info().Msgf("pre-pulling image %s", img)
if err = tc.pullImageIfNeeded(ctx, img); err != nil {
if err = tc.pullImage(ctx, img); err != nil {
// not fatal here, startContainer will fail with the same error when the image is really needed
log.Warn().Err(err).Msgf("prePullImages: can't pull %s", img)
}
Expand Down
6 changes: 3 additions & 3 deletions test/integration/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -860,9 +860,9 @@ func (env *TestEnvironment) createTestSchema(t *testing.T, data TestDataStruct,
createSQL = strings.Replace(createSQL, "{database}", data.Database, -1)
}
}
// since 26.8 a {database}/{table} macro whose value contains '/', '{' or '}' is rejected in replicated ZooKeeper paths
// (https://github.com/ClickHouse/ClickHouse/commit/617b6626fcf), explicit literal paths are still allowed
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "26.8") >= 0 {
// since 26.8 (backported to 26.3.34, 26.7) a {database}/{table} macro whose value contains '/', '{' or '}' is rejected in replicated ZooKeeper paths
// (https://github.com/ClickHouse/ClickHouse/pull/114006), explicit literal paths are still allowed
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "26.3") >= 0 {
if strings.Contains(createSQL, "{database}") && strings.ContainsAny(data.Database, "/{}") {
createSQL = strings.Replace(createSQL, "{database}", data.Database, -1)
}
Expand Down
6 changes: 6 additions & 0 deletions test/testflows/clickhouse_backup/regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,16 @@
# of Altinity LTD. Any dissemination of this information or
# reproduction of this material is strictly forbidden unless
# prior written permission is obtained from Altinity LTD.
import faulthandler
import os
import shutil
import signal
import sys
import yaml

# run.sh kills a hung suite via `timeout` (SIGTERM), dump all thread stacks to stderr first to see where it hung,
# chain=True keeps the default SIGTERM termination, so `timeout` still exits with 124
faulthandler.register(signal.SIGTERM, all_threads=True, chain=True)
import testflows.settings as testflows_settings
from testflows.core import *

Expand Down