From 659e630e70f91513a1279ccf8b7cb41bc41145f8 Mon Sep 17 00:00:00 2001 From: loomt Date: Tue, 28 Apr 2026 11:59:16 +0800 Subject: [PATCH 1/8] fix: update ClickHouse versions and enhance TLS support in backup/restore --- addons/clickhouse/README.md | 4 +- addons/clickhouse/dataprotection/common.sh | 3 + .../dataprotection/incremental-restore.sh | 5 - addons/clickhouse/dataprotection/restore.sh | 5 - addons/clickhouse/scripts/bootstrap-keeper.sh | 25 +++++ addons/clickhouse/scripts/common.sh | 50 ++++++---- .../clickhouse/scripts/keeper-switchover.sh | 94 ++++++++++++------- addons/clickhouse/values.yaml | 2 +- examples/clickhouse/README.md | 4 +- 9 files changed, 124 insertions(+), 68 deletions(-) diff --git a/addons/clickhouse/README.md b/addons/clickhouse/README.md index be3ded2eee..b7cc15565a 100644 --- a/addons/clickhouse/README.md +++ b/addons/clickhouse/README.md @@ -1479,9 +1479,9 @@ The incremental backup's `parentBackupName` must refer to an existing backup cre #### Restore Settings > [!NOTE] -> Restoring a TLS-enabled cluster directly from backup is NOT supported. You should restore the cluster with TLS disabled first, and then enable TLS manually after the restore process is complete. +> When the restore target has TLS enabled, the restore job mounts the ClickHouse TLS files and connects through the secure TCP port. -Restore process will restore schema and rbac first, then restore data. You can tune schema-ready waiting behavior for restore jobs via Helm values: +Restore process will restore schema first, then restore data. Non-TLS restores also restore RBAC metadata; TLS restores focus on schema and data because replicated RBAC backup requires a direct Keeper TLS connection. You can tune schema-ready waiting behavior for restore jobs via Helm values: ```yaml restore: diff --git a/addons/clickhouse/dataprotection/common.sh b/addons/clickhouse/dataprotection/common.sh index 6a9ea54a18..21ef70f993 100644 --- a/addons/clickhouse/dataprotection/common.sh +++ b/addons/clickhouse/dataprotection/common.sh @@ -468,6 +468,9 @@ function ch_query() { local query="$1" local ch_port="${CLICKHOUSE_PORT:-9000}" local ch_args=(--user "${CLICKHOUSE_USERNAME}" --password "${CLICKHOUSE_PASSWORD}" --host "${CLICKHOUSE_HOST}" --port "$ch_port" --connect_timeout=5) + if [[ "${CLICKHOUSE_SECURE:-false}" == "true" ]]; then + ch_args+=(--secure --accept-invalid-certificate) + fi clickhouse-client "${ch_args[@]}" --query "$query" } diff --git a/addons/clickhouse/dataprotection/incremental-restore.sh b/addons/clickhouse/dataprotection/incremental-restore.sh index 30a7957df9..a5078600f3 100644 --- a/addons/clickhouse/dataprotection/incremental-restore.sh +++ b/addons/clickhouse/dataprotection/incremental-restore.sh @@ -8,11 +8,6 @@ trap handle_exit EXIT generate_backup_config set_clickhouse_backup_config_env -if [[ "${CLICKHOUSE_SECURE}" = "true" ]]; then - DP_error_log "ClickHouse restore does not support TLS" - exit 1 -fi - # 1. Download full (base) backup export S3_PATH="$DP_BACKUP_ROOT_PATH/$DP_BASE_BACKUP_NAME/$DP_TARGET_RELATIVE_PATH" fetch_backup "$DP_BASE_BACKUP_NAME" diff --git a/addons/clickhouse/dataprotection/restore.sh b/addons/clickhouse/dataprotection/restore.sh index a71a82f0c5..9f27d71b7a 100644 --- a/addons/clickhouse/dataprotection/restore.sh +++ b/addons/clickhouse/dataprotection/restore.sh @@ -8,11 +8,6 @@ trap handle_exit EXIT generate_backup_config set_clickhouse_backup_config_env -if [[ "${CLICKHOUSE_SECURE}" = "true" ]]; then - DP_error_log "ClickHouse restore does not support TLS" - exit 1 -fi - # 1. Detect topology mode: standalone (no ':' in FQDN) or cluster first_entry="${ALL_COMBINED_SHARDS_POD_FQDN_LIST%%,*}" first_component="${first_entry%%:*}" diff --git a/addons/clickhouse/scripts/bootstrap-keeper.sh b/addons/clickhouse/scripts/bootstrap-keeper.sh index b3f7a2f598..8eceb2dce5 100644 --- a/addons/clickhouse/scripts/bootstrap-keeper.sh +++ b/addons/clickhouse/scripts/bootstrap-keeper.sh @@ -1,4 +1,6 @@ #!/bin/bash +set -euo pipefail + function chown_data_dir() { data_dir=/bitnami/clickhouse/data if [ ! -d "${data_dir}" ]; then @@ -10,7 +12,30 @@ function chown_data_dir() { chown -R root:root ${data_dir} fi } + +function wait_peer_dns_ready() { + local peers="${CH_KEEPER_POD_FQDN_LIST:-}" + local attempts=30 + local sleep_seconds=2 + + [[ -z "$peers" ]] && return 0 + + for peer in ${peers//,/ }; do + for attempt in $(seq 1 $attempts); do + if getent hosts "$peer" >/dev/null 2>&1; then + break + fi + if [[ $attempt -eq $attempts ]]; then + echo "$(date) ERROR: peer DNS $peer is still unresolved after $((attempts * sleep_seconds))s" >&2 + return 1 + fi + sleep "$sleep_seconds" + done + done +} + chown_data_dir +wait_peer_dns_ready HOSTNAME="$(hostname -s)" if grep -q "0" /opt/bitnami/clickhouse/etc/conf.d/ch-keeper_00_default_overrides.xml; then # compatible old version diff --git a/addons/clickhouse/scripts/common.sh b/addons/clickhouse/scripts/common.sh index dc7114d7c5..6a6d80f869 100644 --- a/addons/clickhouse/scripts/common.sh +++ b/addons/clickhouse/scripts/common.sh @@ -20,6 +20,28 @@ function ch_query() { clickhouse-client "${ch_args[@]}" --query "$query" } +function send_4lw() { + local host="$1" + local cmd="$2" + + if [[ "${TLS_ENABLED:-false}" == "true" ]]; then + timeout 2 openssl s_client \ + -connect "$host:$CLICKHOUSE_KEEPER_PORT" \ + -CAfile "$CLICKHOUSE_TLS_CA" \ + -cert "$CLICKHOUSE_TLS_CERT" \ + -key "$CLICKHOUSE_TLS_KEY" \ + -quiet \ + -ign_eof 2>/dev/null <<<"$cmd" + else + printf "%s" "$cmd" | /shared-tools/nc -w 2 "$host" "$CLICKHOUSE_KEEPER_PORT" + fi +} + +function get_srvr() { + local host="$1" + send_4lw "$host" "srvr" +} + # Low-level keeper client execution with connection retry # Handles connection issues, timeouts, and transient network problems function keeper_run() { @@ -45,7 +67,7 @@ function keeper_run() { if [[ "${TLS_ENABLED:-false}" == "true" ]]; then keeper_args+=(--tls-ca-file "$CLICKHOUSE_TLS_CA" --tls-cert-file "$CLICKHOUSE_TLS_CERT" --tls-key-file "$CLICKHOUSE_TLS_KEY") fi - if output=$(clickhouse-keeper-client "${keeper_args[@]}" 2>&1); then + if output=$(/opt/bitnami/clickhouse/bin/clickhouse-keeper-client "${keeper_args[@]}" 2>&1); then if [[ "$output" != *"Coordination error"* ]] && [[ "$output" != *"Connection refused"* ]] && [[ "$output" != *"Timeout"* ]]; then @@ -103,24 +125,16 @@ function get_config() { # Get keeper node mode (leader/follower/observer) function get_mode() { local host="$1" + local mode + mode=$(get_srvr "$host" 2>/dev/null | awk '/^Mode:/ {print $2; exit}') + echo "$mode" +} - # Use TLS port if TLS is enabled - if [[ "$TLS_ENABLED" == "true" ]]; then - # For TLS connections, use openssl s_client for 4LW commands - local mode - mode=$(printf "srvr\n" | timeout 2 openssl s_client \ - -connect "$host:$CLICKHOUSE_KEEPER_PORT" \ - -CAfile "$CLICKHOUSE_TLS_CA" \ - -cert "$CLICKHOUSE_TLS_CERT" \ - -key "$CLICKHOUSE_TLS_KEY" \ - -quiet \ - -ign_eof 2>/dev/null | grep Mode) - echo "$mode" | awk '{print $2}' - else - local mode - mode=$(echo srvr | /shared-tools/nc -w 1 "$host" "$CLICKHOUSE_KEEPER_PORT" | grep Mode) - echo "$mode" | awk '{print $2}' - fi +function get_zxid() { + local host="$1" + local zxid + zxid=$(get_srvr "$host" 2>/dev/null | awk '/^Zxid:/ {print $2; exit}') + echo "$zxid" } # Get keeper node mode by keeper diff --git a/addons/clickhouse/scripts/keeper-switchover.sh b/addons/clickhouse/scripts/keeper-switchover.sh index ad24b2a625..d8b7683d3d 100644 --- a/addons/clickhouse/scripts/keeper-switchover.sh +++ b/addons/clickhouse/scripts/keeper-switchover.sh @@ -5,57 +5,81 @@ source /scripts/common.sh leader_fqdn="$KB_SWITCHOVER_CURRENT_FQDN" candidate_fqdn="${KB_SWITCHOVER_CANDIDATE_FQDN:-}" -if [ $COMPONENT_REPLICAS -lt 2 ]; then - exit 0 +[[ $COMPONENT_REPLICAS -lt 2 ]] && exit 0 +[[ -z "$candidate_fqdn" ]] && { echo "No candidate specified, exit."; exit 0; } +[[ "$KB_SWITCHOVER_ROLE" != "leader" ]] && { echo "switchover not triggered for primary, nothing to do, exit 0."; exit 0; } + +# TODO(future): prefer use ClickHouse Keeper `rcfg` after 26.1.1.912+. + +function update_priority() { + local host="$1" + local line="$2" + local priority="$3" + local base_config="${line%;*}" + + retry_keeper_operation \ + "keeper_run '$host' 'reconfig add \"$base_config;$priority\"'" \ + "echo \"\$(get_config '$host')\" | grep -Fqx \"$base_config;$priority\"" +} + +candidate_mode=$(get_mode "$candidate_fqdn" 2>/dev/null || true) +if [[ "$candidate_mode" != "follower" ]]; then + echo "ERROR: candidate $candidate_fqdn is not a stable follower, current mode: ${candidate_mode:-unknown}" >&2 + exit 1 fi -if [ -z $candidate_fqdn ]; then - echo "No candidate specified, exit." - exit 0 +leader_zxid=$(get_zxid "$leader_fqdn" 2>/dev/null || true) +candidate_zxid=$(get_zxid "$candidate_fqdn" 2>/dev/null || true) +if [[ -z "$leader_zxid" || -z "$candidate_zxid" || "$leader_zxid" != "$candidate_zxid" ]]; then + echo "ERROR: candidate $candidate_fqdn is not caught up with leader $leader_fqdn (leader_zxid=${leader_zxid:-unknown}, candidate_zxid=${candidate_zxid:-unknown})" >&2 + exit 1 fi -if [ "$KB_SWITCHOVER_ROLE" != "leader" ]; then - echo "switchover not triggered for primary, nothing to do, exit 0." - exit 0 +current_leader_fqdn=$(find_leader "${CH_KEEPER_POD_FQDN_LIST}" 2>/dev/null || true) +if [[ -n "$current_leader_fqdn" ]] && [[ "$current_leader_fqdn" != "$leader_fqdn" ]]; then + if [[ "$current_leader_fqdn" == "$candidate_fqdn" ]]; then + echo "Switchover already completed successfully" + exit 0 + fi + echo "ERROR: Expected new leader $candidate_fqdn, but got $current_leader_fqdn" >&2 + exit 1 fi -# 1. Get current config config=$(get_config "$leader_fqdn") +original_config=$(printf "%s\n" "$config" | grep '^server\..*;participant;') -# 2. Change the priority of the candidate to 8, and the others to 1 -pre_leader=$(echo "$config" | grep "$leader_fqdn") -pre_leader=${pre_leader%;*}";1" -pre_leader_config_name=$(echo "$pre_leader" | cut -d'=' -f1) -pre_leader_config_id=$(echo "$pre_leader_config_name" | cut -d'.' -f2) -# server.1=ch-cluster-ch-keeper-0.ch-cluster-ch-keeper-headless.default.svc.cluster.local:9234;participant;1 while IFS= read -r line; do if [[ "$line" == server.*";participant;"* ]]; then line_fqdn=$(echo "$line" | cut -d'=' -f2 | cut -d':' -f1) original_priority="${line##*;}" - base_config="${line%;*}" - new_priority="" - if echo "$line_fqdn" | grep -q "$candidate_fqdn"; then - [[ "$original_priority" -ne 8 ]] && new_priority=8 - else - [[ "$original_priority" -ne 1 ]] && new_priority=1 - fi - [[ -n "$new_priority" ]] && retry_keeper_operation \ - "keeper_run '$leader_fqdn' 'reconfig add \"$base_config;$new_priority\"'" \ - "echo \"\$(get_config '$leader_fqdn')\" | grep -q \"$base_config;$new_priority\"" + target_priority=1 + [[ "$line_fqdn" == "$candidate_fqdn" ]] && target_priority=8 + [[ "$original_priority" != "$target_priority" ]] && update_priority "$leader_fqdn" "$line" "$target_priority" fi -done <<<"$config" +done <<<"$original_config" -# 3. Remove the leader from the config, remove once, because only the leader can remove itself -keeper_run "$leader_fqdn" "reconfig remove '$pre_leader_config_id'" +send_4lw "$candidate_fqdn" "rqld" | grep -q "Sent leadership request to leader." || { + echo "ERROR: failed to request leadership from candidate $candidate_fqdn" >&2 + exit 1 +} -# 4. Re-add after pre leader reboot +new_leader_fqdn="$leader_fqdn" retry_keeper_operation \ - "keeper_run '$candidate_fqdn' 'reconfig add \"$pre_leader\"'" \ - "get_config '$candidate_fqdn' | grep -q '$pre_leader'" + "new_leader_fqdn=\$(find_leader '${CH_KEEPER_POD_FQDN_LIST}')" \ + "[[ \"\$new_leader_fqdn\" == \"$candidate_fqdn\" ]]" -# 5. Check if the candidate is the leader -retry_keeper_operation \ - "mode=\$(get_mode_by_keeper '$candidate_fqdn')" \ - "[[ \"\$mode\" == \"leader\" ]]" +if [[ "$new_leader_fqdn" != "$candidate_fqdn" ]]; then + echo "ERROR: Expected new leader $candidate_fqdn, but got $new_leader_fqdn" >&2 + exit 1 +fi + +while IFS= read -r line; do + if [[ "$line" == server.*";participant;"* ]]; then + original_priority="${line##*;}" + current_line=$(get_config "$new_leader_fqdn" | grep -F "${line%;*};" || true) + current_priority="${current_line##*;}" + [[ -n "$current_line" && "$current_priority" != "$original_priority" ]] && update_priority "$new_leader_fqdn" "$line" "$original_priority" + fi +done <<<"$original_config" echo "Switchover completed successfully" diff --git a/addons/clickhouse/values.yaml b/addons/clickhouse/values.yaml index 5f2a0fc67d..8a419ddb86 100644 --- a/addons/clickhouse/values.yaml +++ b/addons/clickhouse/values.yaml @@ -44,7 +44,7 @@ roleProbe: initialDelaySeconds: 15 failureThreshold: 3 periodSeconds: 3 - timeoutSeconds: 3 + timeoutSeconds: 10 busyboxImage: registry: "" diff --git a/examples/clickhouse/README.md b/examples/clickhouse/README.md index 1480c22e50..2155c914bf 100644 --- a/examples/clickhouse/README.md +++ b/examples/clickhouse/README.md @@ -656,9 +656,9 @@ The incremental backup's `parentBackupName` must refer to an existing backup cre #### Restore Settings > [!NOTE] -> Restoring a TLS-enabled cluster directly from backup is NOT supported. You should restore the cluster with TLS disabled first, and then enable TLS manually after the restore process is complete. +> When the restore target has TLS enabled, the restore job mounts the ClickHouse TLS files and connects through the secure TCP port. -Restore process will restore schema and rbac first, then restore data. You can tune schema-ready waiting behavior for restore jobs via Helm values: +Restore process will restore schema first, then restore data. Non-TLS restores also restore RBAC metadata; TLS restores focus on schema and data because replicated RBAC backup requires a direct Keeper TLS connection. You can tune schema-ready waiting behavior for restore jobs via Helm values: ```yaml restore: From f0e5db96662e6f3332790f41d7590246ac2ea1ce Mon Sep 17 00:00:00 2001 From: loomt Date: Tue, 28 Apr 2026 15:52:43 +0800 Subject: [PATCH 2/8] revert versions changes --- README.md | 2 +- addons/clickhouse/README.md | 2 +- addons/clickhouse/values.yaml | 4 ---- examples/clickhouse/README.md | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ee89555766..697cd6f81d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ KubeBlocks add-ons. | NAME | VERSIOINS | DESCRIPTION | MAINTAINERS | | ---- | ---- | ----------- | ----------- | | apecloud-mysql | apecloud-mysql-8.0.30
wescale-0.2.7 | ApeCloud MySQL is a database that is compatible with MySQL syntax and achieves high availability through the utilization of the RAFT consensus protocol. | xuriwuyun | -| clickhouse | clickhouse-22.3.18
clickhouse-22.3.20
clickhouse-22.6.1
clickhouse-22.8.21
clickhouse-22.9.4
clickhouse-24.12.2
clickhouse-24.7.2
clickhouse-24.8.3
clickhouse-25.4.4
clickhouse-25.9.7 | ClickHouse is an open-source column-oriented OLAP database management system. Use it to boost your database performance while providing linear scalability and hardware efficiency. | loomt | +| clickhouse | clickhouse-22.3.18
clickhouse-22.3.20
clickhouse-22.6.1
clickhouse-22.8.21
clickhouse-24.8.3
clickhouse-25.4.4
clickhouse-25.9.7 | ClickHouse is an open-source column-oriented OLAP database management system. Use it to boost your database performance while providing linear scalability and hardware efficiency. | loomt | | elasticsearch | elasticsearch-6.8.23
elasticsearch-7.10.1
elasticsearch-7.10.2
elasticsearch-7.7.1
elasticsearch-7.8.1
elasticsearch-8.1.3
elasticsearch-8.15.5
elasticsearch-8.8.2
kibana-6.8.23
kibana-7.10.1
kibana-7.10.2
kibana-7.7.1
kibana-7.8.1
kibana-8.1.3
kibana-8.15.5
kibana-8.8.2
kibana-8.9.1 | Elasticsearch is a distributed, RESTful search engine optimized for speed and relevance on production-scale workloads. | kizuna-lek vipshop | | etcd | etcd-3.5.15
etcd-3.5.6
etcd-3.6.1 | Etcd is a strongly consistent, distributed key-value store that provides a reliable way to store data that needs to be accessed by a distributed system or cluster of machines. | loomt | | falkordb | falkordb-4.12.5
falkordb-cluster-4.12.5
falkordb-sent-4.12.5 | FalkorDB is an in-memory graph database based on Redis. | ApeCloud dudizimber | diff --git a/addons/clickhouse/README.md b/addons/clickhouse/README.md index b7cc15565a..e619695bd3 100644 --- a/addons/clickhouse/README.md +++ b/addons/clickhouse/README.md @@ -36,7 +36,7 @@ There are two key components in the ClickHouse cluster: | Major Versions | Description | | -------------- | ------------------------- | | 22 | 22.3.18, 22.3.20, 22.8.21 | -| 24 | 24.7.2, 24.8.3, 24.12.2 | +| 24 | 24.8.3 | | 25 | 25.4.4, 25.9.7 | ## Prerequisites diff --git a/addons/clickhouse/values.yaml b/addons/clickhouse/values.yaml index 8a419ddb86..257b524847 100644 --- a/addons/clickhouse/values.yaml +++ b/addons/clickhouse/values.yaml @@ -23,12 +23,8 @@ clickhouseVersions: imageTag: 25.9.7 - version: 25.4.4 imageTag: 25.4.4-debian-12-r1 - - version: 24.12.2 - imageTag: 24.12.2.29 - version: 24.8.3 imageTag: 24.8.3-debian-12-r1 - - version: 24.7.2 - imageTag: 24.7.2.13 - version: 22.8.21 imageTag: 22.8.21-debian-11-r33 - version: 22.3.20 diff --git a/examples/clickhouse/README.md b/examples/clickhouse/README.md index 2155c914bf..57afff0b1c 100644 --- a/examples/clickhouse/README.md +++ b/examples/clickhouse/README.md @@ -36,7 +36,7 @@ There are two key components in the ClickHouse cluster: | Major Versions | Description | | -------------- | ------------------------- | | 22 | 22.3.18, 22.3.20, 22.8.21 | -| 24 | 24.7.2, 24.8.3, 24.12.2 | +| 24 | 24.8.3 | | 25 | 25.4.4, 25.9.7 | ## Prerequisites From df39b774147901ce7856e791a762a140fa79f504 Mon Sep 17 00:00:00 2001 From: loomt Date: Wed, 6 May 2026 00:21:34 +0800 Subject: [PATCH 3/8] fix tls backups --- .../configs/00_default_overrides.xml.tpl | 7 ++- .../ch_keeper_00_default_overrides.xml.tpl | 7 ++- addons/clickhouse/dataprotection/common.sh | 60 +++++++++++++++++-- .../dataprotection/incremental-restore.sh | 20 +++---- addons/clickhouse/dataprotection/restore.sh | 15 +---- 5 files changed, 73 insertions(+), 36 deletions(-) diff --git a/addons/clickhouse/configs/00_default_overrides.xml.tpl b/addons/clickhouse/configs/00_default_overrides.xml.tpl index d17eed346f..5fba4fffab 100644 --- a/addons/clickhouse/configs/00_default_overrides.xml.tpl +++ b/addons/clickhouse/configs/00_default_overrides.xml.tpl @@ -106,10 +106,9 @@ {{$CERT_FILE}} {{$KEY_FILE}} - relaxed + + false {{$CA_FILE}} true sslv2,sslv3 @@ -124,6 +123,8 @@ sslv2,sslv3 true strict + + false RejectCertificateHandler diff --git a/addons/clickhouse/configs/ch_keeper_00_default_overrides.xml.tpl b/addons/clickhouse/configs/ch_keeper_00_default_overrides.xml.tpl index f855056075..748938033a 100644 --- a/addons/clickhouse/configs/ch_keeper_00_default_overrides.xml.tpl +++ b/addons/clickhouse/configs/ch_keeper_00_default_overrides.xml.tpl @@ -85,10 +85,9 @@ {{$CERT_FILE}} {{$KEY_FILE}} - relaxed + + false {{$CA_FILE}} true sslv2,sslv3 @@ -103,6 +102,8 @@ sslv2,sslv3 true strict + + false RejectCertificateHandler diff --git a/addons/clickhouse/dataprotection/common.sh b/addons/clickhouse/dataprotection/common.sh index 21ef70f993..087a1f7159 100644 --- a/addons/clickhouse/dataprotection/common.sh +++ b/addons/clickhouse/dataprotection/common.sh @@ -502,6 +502,37 @@ function fetch_backup() { DP_log "Backup '$backup_name' is available locally." } +function stage_required_backup_metadata() { + local target_base_path="$1" + shift + local backup_name + local metadata_file + + # clickhouse-backup expects incremental parents under the same S3_PATH. + # KubeBlocks stores each Backup under its own path, so stage parent metadata only. + export PATH="$PATH:$DP_DATASAFED_BIN_PATH" + for backup_name in "$@"; do + [[ -z "$backup_name" ]] && continue + metadata_file=$(mktemp) || { + DP_error_log "Failed to create temporary metadata file" + return 1 + } + export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_ROOT_PATH/$backup_name/$DP_TARGET_RELATIVE_PATH" + if ! datasafed pull "$backup_name/metadata.json" "$metadata_file"; then + rm -f "$metadata_file" + DP_error_log "Failed to pull metadata for backup '$backup_name'" + return 1 + fi + export DATASAFED_BACKEND_BASE_PATH="$target_base_path" + if ! datasafed push "$metadata_file" "$backup_name/metadata.json"; then + rm -f "$metadata_file" + DP_error_log "Failed to stage metadata for backup '$backup_name'" + return 1 + fi + rm -f "$metadata_file" + done +} + function delete_backups_except() { local latest_backup=$1 DP_log "delete backup except $latest_backup" @@ -526,6 +557,25 @@ function save_backup_size() { DP_save_backup_status_info "$backup_size" } +function detect_restore_mode() { + local first_entry="${ALL_COMBINED_SHARDS_POD_FQDN_LIST%%,*}" + local first_component="${first_entry%%:*}" + if [[ -z "$first_component" ]]; then + DP_error_log "Invalid ALL_COMBINED_SHARDS_POD_FQDN_LIST" >&2 + return 1 + fi + + if [[ -z "${CH_KEEPER_POD_FQDN_LIST:-}" ]]; then + echo "standalone" + return 0 + fi + + if [[ "$first_component" == "$first_entry" ]]; then + first_component="${CURRENT_SHARD_COMPONENT_SHORT_NAME}" + fi + echo "cluster:$first_component" +} + # Restore schema and wait for sync across shards function restore_schema_and_sync() { local backup_name="$1" @@ -547,14 +597,14 @@ function restore_schema_and_sync() { if [[ "$should_restore_schema" == "true" ]]; then # No-keeper restore: unset ON CLUSTER because distributed DDL requires Keeper/ZooKeeper. [[ "$mode_info" == "standalone" ]] && unset RESTORE_SCHEMA_ON_CLUSTER - local restore_args=(restore_remote "$backup_name" --schema) + local restore_args=(restore "$backup_name" --schema) if [[ -n "${CH_KEEPER_POD_FQDN_LIST:-}" ]]; then restore_args+=(--rbac) else DP_log "Skip RBAC restore because ClickHouse keeper is not configured" fi clickhouse-backup "${restore_args[@]}" || { - DP_error_log "Clickhouse-backup restore_remote backup $backup_name FAILED" + DP_error_log "Clickhouse-backup restore backup $backup_name FAILED" return 1 } # Cluster mode: create marker table for cross-shard coordination @@ -592,12 +642,14 @@ function do_restore() { local schema_db="kubeblocks" local schema_table="__restore_ready__" + fetch_backup "$backup_name" || return 1 + # Restore schema (first shard in cluster uses ON CLUSTER DDL) restore_schema_and_sync "$backup_name" "$mode_info" || return 1 # Restore data - clickhouse-backup restore_remote "$backup_name" --data || { - DP_error_log "Clickhouse-backup restore_remote --data FAILED" + clickhouse-backup restore "$backup_name" --data || { + DP_error_log "Clickhouse-backup restore --data FAILED" return 1 } diff --git a/addons/clickhouse/dataprotection/incremental-restore.sh b/addons/clickhouse/dataprotection/incremental-restore.sh index a5078600f3..a67f68b249 100644 --- a/addons/clickhouse/dataprotection/incremental-restore.sh +++ b/addons/clickhouse/dataprotection/incremental-restore.sh @@ -11,28 +11,22 @@ set_clickhouse_backup_config_env # 1. Download full (base) backup export S3_PATH="$DP_BACKUP_ROOT_PATH/$DP_BASE_BACKUP_NAME/$DP_TARGET_RELATIVE_PATH" fetch_backup "$DP_BASE_BACKUP_NAME" +downloaded_backups=("$DP_BASE_BACKUP_NAME") # 2. Download ancestor incremental backups IFS=',' read -r -a ancestors <<<"${DP_ANCESTOR_INCREMENTAL_BACKUP_NAMES}" for parent_name in "${ancestors[@]}"; do + [[ -z "$parent_name" ]] && continue export S3_PATH="$DP_BACKUP_ROOT_PATH/$parent_name/$DP_TARGET_RELATIVE_PATH" + stage_required_backup_metadata "$S3_PATH" "${downloaded_backups[@]}" || exit 1 fetch_backup "$parent_name" + downloaded_backups+=("$parent_name") done -# 3. Detect topology mode: standalone (no ':' in FQDN) or cluster +# 3. Detect topology mode: standalone or cluster export S3_PATH="${DP_BACKUP_BASE_PATH}" -first_entry="${ALL_COMBINED_SHARDS_POD_FQDN_LIST%%,*}" -first_component="${first_entry%%:*}" -if [[ -z "$first_component" ]]; then - DP_error_log "Invalid ALL_COMBINED_SHARDS_POD_FQDN_LIST" - exit 1 -fi -if [[ "$first_component" == "$first_entry" ]]; then - mode_info="standalone" - DP_log "Standalone mode detected" -else - mode_info="cluster:$first_component" -fi +stage_required_backup_metadata "$S3_PATH" "${downloaded_backups[@]}" || exit 1 +mode_info=$(detect_restore_mode) || exit 1 # 4. Restore schema + data + marker do_restore "${DP_BACKUP_NAME}" "$mode_info" || exit 1 diff --git a/addons/clickhouse/dataprotection/restore.sh b/addons/clickhouse/dataprotection/restore.sh index 9f27d71b7a..4314e60355 100644 --- a/addons/clickhouse/dataprotection/restore.sh +++ b/addons/clickhouse/dataprotection/restore.sh @@ -8,19 +8,8 @@ trap handle_exit EXIT generate_backup_config set_clickhouse_backup_config_env -# 1. Detect topology mode: standalone (no ':' in FQDN) or cluster -first_entry="${ALL_COMBINED_SHARDS_POD_FQDN_LIST%%,*}" -first_component="${first_entry%%:*}" -if [[ -z "$first_component" ]]; then - DP_error_log "Invalid ALL_COMBINED_SHARDS_POD_FQDN_LIST" - exit 1 -fi -if [[ "$first_component" == "$first_entry" ]]; then - mode_info="standalone" - DP_log "Standalone mode detected" -else - mode_info="cluster:$first_component" -fi +# 1. Detect topology mode: standalone or cluster +mode_info=$(detect_restore_mode) || exit 1 # 2. Restore schema + data + marker do_restore "${DP_BACKUP_NAME}" "$mode_info" || exit 1 From 6d801a87fafa0b54ab5817f9a271dad58212c5d8 Mon Sep 17 00:00:00 2001 From: loomt Date: Thu, 7 May 2026 11:06:32 +0800 Subject: [PATCH 4/8] chore: update ClickHouse switchover documentation and configurations --- addons/clickhouse/README.md | 90 ++----------------- .../clickhouse/templates/opsdefinition.yaml | 4 +- examples/clickhouse/README.md | 39 ++------ .../cluster-with-config-templates.yaml | 2 +- .../clickhouse/cluster-with-nodeport.yaml | 6 +- .../keeper-switchover-specified-instance.yaml | 6 +- examples/clickhouse/keeper-switchover.yaml | 16 ---- examples/clickhouse/restore.yaml | 2 +- 8 files changed, 24 insertions(+), 141 deletions(-) delete mode 100644 examples/clickhouse/keeper-switchover.yaml diff --git a/addons/clickhouse/README.md b/addons/clickhouse/README.md index e619695bd3..8e19e41a0b 100644 --- a/addons/clickhouse/README.md +++ b/addons/clickhouse/README.md @@ -802,58 +802,9 @@ kubectl apply -f examples/clickhouse/verticalscale.yaml ### Switchover for Clickhouse Keeper -#### Switchover without preferred candidates +#### Switchover with specified candidate -Switchover a specified instance as the new primary or leader of the cluster - -```yaml -# cat examples/clickhouse/keeper-switchover.yaml -apiVersion: operations.kubeblocks.io/v1alpha1 -kind: OpsRequest -metadata: - name: keeper-switchover - namespace: demo -spec: - # Specifies the name of the Cluster resource that this operation is targeting. - clusterName: clickhouse-cluster - type: Switchover - # Lists Switchover objects, each specifying a Component to perform the switchover operation. - switchover: - # Specifies the name of the Component. - - componentName: ch-keeper - # Specifies the instance whose role will be transferred. - # A typical usage is to transfer the leader role in a consensus system. - instanceName: "clickhouse-cluster-ch-keeper-0" - -``` - -```yaml -# cat examples/clickhouse/keeper-switchover.yaml -apiVersion: operations.kubeblocks.io/v1alpha1 -kind: OpsRequest -metadata: - name: keeper-switchover - namespace: demo -spec: - # Specifies the name of the Cluster resource that this operation is targeting. - clusterName: clickhouse-cluster - type: Switchover - # Lists Switchover objects, each specifying a Component to perform the switchover operation. - switchover: - # Specifies the name of the Component. - - componentName: ch-keeper - # Specifies the instance whose role will be transferred. - # A typical usage is to transfer the leader role in a consensus system. - instanceName: "clickhouse-cluster-ch-keeper-0" -``` - -```bash -kubectl apply -f examples/clickhouse/keeper-switchover.yaml -``` - -#### Switchover-specified-instance - -Switchover a specified instance as the new primary or leader of the cluster +Switchover the current leader to a selected candidate instance. ClickHouse Keeper switchover currently requires `candidateName`. ```yaml # cat examples/clickhouse/keeper-switchover-specified-instance.yaml @@ -873,40 +824,17 @@ spec: # Specifies the instance whose role will be transferred. # A typical usage is to transfer the leader role in a consensus system. instanceName: "clickhouse-cluster-ch-keeper-0" - # Specifies the instance that will become the new leader, if not specify, the first non leader instance will become candidate. - # Need to ensure the candidate instance is catch up logs of the quorum, otherwise the switchover will transfer the leader to other instance. + # ClickHouse Keeper switchover currently requires candidateName. + # Need to ensure the candidate instance has caught up with quorum logs. candidateName: "clickhouse-cluster-ch-keeper-1" ``` -```yaml -# cat examples/clickhouse/keeper-switchover-specified-instance.yaml -apiVersion: operations.kubeblocks.io/v1alpha1 -kind: OpsRequest -metadata: - name: keeper-switchover - namespace: demo -spec: - # Specifies the name of the Cluster resource that this operation is targeting. - clusterName: clickhouse-cluster - type: Switchover - # Lists Switchover objects, each specifying a Component to perform the switchover operation. - switchover: - # Specifies the name of the Component. - - componentName: ch-keeper - # Specifies the instance whose role will be transferred. - # A typical usage is to transfer the leader role in a consensus system. - instanceName: "clickhouse-cluster-ch-keeper-0" - # Specifies the instance that will become the new leader, if not specify, the first non leader instance will become candidate. - # Need to ensure the candidate instance is catch up logs of the quorum, otherwise the switchover will transfer the leader to other instance. - candidateName: "clickhouse-cluster-ch-keeper-1" -``` - ```bash kubectl apply -f examples/clickhouse/keeper-switchover-specified-instance.yaml ``` -You may need to update the `opsrequest.spec.switchover.instanceName` field to your desired instance name. +You may need to update the `opsrequest.spec.switchover.instanceName` and `opsrequest.spec.switchover.candidateName` fields to your desired instance names. ### Expand volume @@ -1101,7 +1029,7 @@ spec: configMap: name: custom-user-configuration-tpl # refers to the configmap with your customized configuration info. variables: - max_threads: "16" + udf_max_threads: "16" systemAccounts: - name: admin secretRef: @@ -1531,7 +1459,7 @@ spec: storage: 10Gi shardings: - name: clickhouse - shards: 3 + shards: 2 template: name: clickhouse componentDef: clickhouse-1 @@ -1833,10 +1761,10 @@ spec: services: # Exposes ClickHouse service using NodePort - name: clickhouse-nodeport + # Sharding associated with this service + componentSelector: clickhouse # Type of the exposed service. Valid options are 'ClusterIP', 'NodePort', and 'LoadBalancer'. serviceType: NodePort - # Sharding associated with this service - shardingSelector: clickhouse # Role selector for the component service. # In a ClickHouse cluster, we only expose nodes without specifying roles # since all nodes can handle read/write requests diff --git a/addons/clickhouse/templates/opsdefinition.yaml b/addons/clickhouse/templates/opsdefinition.yaml index 1d9f8212f4..5a193e974c 100644 --- a/addons/clickhouse/templates/opsdefinition.yaml +++ b/addons/clickhouse/templates/opsdefinition.yaml @@ -43,7 +43,7 @@ spec: podSpec: containers: - name: post-rebuild - image: {{ include "clickhouse.repository" . }}:24.8.3-debian-12-r1 + image: {{ include "clickhouse.repository" . }}:{{ .Chart.AppVersion }} imagePullPolicy: IfNotPresent command: - bash @@ -92,7 +92,7 @@ spec: podSpec: containers: - name: post-rebuild - image: {{ include "clickhouse.repository" . }}:24.8.3-debian-12-r1 + image: {{ include "clickhouse.repository" . }}:{{ .Chart.AppVersion }} imagePullPolicy: IfNotPresent command: - bash diff --git a/examples/clickhouse/README.md b/examples/clickhouse/README.md index 57afff0b1c..4aea72f085 100644 --- a/examples/clickhouse/README.md +++ b/examples/clickhouse/README.md @@ -288,38 +288,9 @@ kubectl apply -f examples/clickhouse/verticalscale.yaml ### Switchover for Clickhouse Keeper -#### Switchover without preferred candidates +#### Switchover with specified candidate -Switchover a specified instance as the new primary or leader of the cluster - -```yaml -# cat examples/clickhouse/keeper-switchover.yaml -apiVersion: operations.kubeblocks.io/v1alpha1 -kind: OpsRequest -metadata: - name: keeper-switchover - namespace: demo -spec: - # Specifies the name of the Cluster resource that this operation is targeting. - clusterName: clickhouse-cluster - type: Switchover - # Lists Switchover objects, each specifying a Component to perform the switchover operation. - switchover: - # Specifies the name of the Component. - - componentName: ch-keeper - # Specifies the instance whose role will be transferred. - # A typical usage is to transfer the leader role in a consensus system. - instanceName: "clickhouse-cluster-ch-keeper-0" - -``` - -```bash -kubectl apply -f examples/clickhouse/keeper-switchover.yaml -``` - -#### Switchover-specified-instance - -Switchover a specified instance as the new primary or leader of the cluster +Switchover the current leader to a selected candidate instance. ClickHouse Keeper switchover currently requires `candidateName`. ```yaml # cat examples/clickhouse/keeper-switchover-specified-instance.yaml @@ -339,8 +310,8 @@ spec: # Specifies the instance whose role will be transferred. # A typical usage is to transfer the leader role in a consensus system. instanceName: "clickhouse-cluster-ch-keeper-0" - # Specifies the instance that will become the new leader, if not specify, the first non leader instance will become candidate. - # Need to ensure the candidate instance is catch up logs of the quorum, otherwise the switchover will transfer the leader to other instance. + # ClickHouse Keeper switchover currently requires candidateName. + # Need to ensure the candidate instance has caught up with quorum logs. candidateName: "clickhouse-cluster-ch-keeper-1" ``` @@ -349,7 +320,7 @@ spec: kubectl apply -f examples/clickhouse/keeper-switchover-specified-instance.yaml ``` -You may need to update the `opsrequest.spec.switchover.instanceName` field to your desired instance name. +You may need to update the `opsrequest.spec.switchover.instanceName` and `opsrequest.spec.switchover.candidateName` fields to your desired instance names. ### [Expand volume](volumeexpand.yaml) diff --git a/examples/clickhouse/cluster-with-config-templates.yaml b/examples/clickhouse/cluster-with-config-templates.yaml index b1f80bae26..891819ae00 100644 --- a/examples/clickhouse/cluster-with-config-templates.yaml +++ b/examples/clickhouse/cluster-with-config-templates.yaml @@ -43,7 +43,7 @@ spec: configMap: name: custom-user-configuration-tpl # refers to the configmap with your customized configuration info. variables: - max_threads: "16" + udf_max_threads: "16" systemAccounts: - name: admin secretRef: diff --git a/examples/clickhouse/cluster-with-nodeport.yaml b/examples/clickhouse/cluster-with-nodeport.yaml index db819ec7d5..b8f734befc 100644 --- a/examples/clickhouse/cluster-with-nodeport.yaml +++ b/examples/clickhouse/cluster-with-nodeport.yaml @@ -9,10 +9,10 @@ spec: services: # Exposes ClickHouse service using NodePort - name: clickhouse-nodeport + # Sharding associated with this service + componentSelector: clickhouse # Type of the exposed service. Valid options are 'ClusterIP', 'NodePort', and 'LoadBalancer'. serviceType: NodePort - # Sharding associated with this service - shardingSelector: clickhouse # Role selector for the component service. # In a ClickHouse cluster, we only expose nodes without specifying roles # since all nodes can handle read/write requests @@ -84,4 +84,4 @@ metadata: namespace: demo type: Opaque data: - password: cGFzc3dvcmQxMjM= # 'password123' in base64 \ No newline at end of file + password: cGFzc3dvcmQxMjM= # 'password123' in base64 diff --git a/examples/clickhouse/keeper-switchover-specified-instance.yaml b/examples/clickhouse/keeper-switchover-specified-instance.yaml index db20429c6c..e105c2b28d 100644 --- a/examples/clickhouse/keeper-switchover-specified-instance.yaml +++ b/examples/clickhouse/keeper-switchover-specified-instance.yaml @@ -14,6 +14,6 @@ spec: # Specifies the instance whose role will be transferred. # A typical usage is to transfer the leader role in a consensus system. instanceName: "clickhouse-cluster-ch-keeper-0" - # Specifies the instance that will become the new leader, if not specify, the first non leader instance will become candidate. - # Need to ensure the candidate instance is catch up logs of the quorum, otherwise the switchover will transfer the leader to other instance. - candidateName: "clickhouse-cluster-ch-keeper-1" \ No newline at end of file + # ClickHouse Keeper switchover currently requires candidateName. + # Need to ensure the candidate instance has caught up with quorum logs. + candidateName: "clickhouse-cluster-ch-keeper-1" diff --git a/examples/clickhouse/keeper-switchover.yaml b/examples/clickhouse/keeper-switchover.yaml deleted file mode 100644 index 7b1b4d8b5d..0000000000 --- a/examples/clickhouse/keeper-switchover.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: operations.kubeblocks.io/v1alpha1 -kind: OpsRequest -metadata: - name: keeper-switchover - namespace: demo -spec: - # Specifies the name of the Cluster resource that this operation is targeting. - clusterName: clickhouse-cluster - type: Switchover - # Lists Switchover objects, each specifying a Component to perform the switchover operation. - switchover: - # Specifies the name of the Component. - - componentName: ch-keeper - # Specifies the instance whose role will be transferred. - # A typical usage is to transfer the leader role in a consensus system. - instanceName: "clickhouse-cluster-ch-keeper-0" \ No newline at end of file diff --git a/examples/clickhouse/restore.yaml b/examples/clickhouse/restore.yaml index 3b7b0fd64f..3cf080b314 100644 --- a/examples/clickhouse/restore.yaml +++ b/examples/clickhouse/restore.yaml @@ -36,7 +36,7 @@ spec: storage: 10Gi shardings: - name: clickhouse - shards: 3 + shards: 2 template: name: clickhouse componentDef: clickhouse-1 From 28a8a960370928ad92c990bdb1a7e3026ee7163b Mon Sep 17 00:00:00 2001 From: loomt Date: Fri, 8 May 2026 15:07:30 +0800 Subject: [PATCH 5/8] revert some changes --- README.md | 2 +- addons/clickhouse/README.md | 2 +- addons/clickhouse/dataprotection/common.sh | 10 ++++------ addons/clickhouse/values.yaml | 4 ++++ examples/clickhouse/README.md | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 697cd6f81d..0aa35bd715 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ KubeBlocks add-ons. | NAME | VERSIOINS | DESCRIPTION | MAINTAINERS | | ---- | ---- | ----------- | ----------- | | apecloud-mysql | apecloud-mysql-8.0.30
wescale-0.2.7 | ApeCloud MySQL is a database that is compatible with MySQL syntax and achieves high availability through the utilization of the RAFT consensus protocol. | xuriwuyun | -| clickhouse | clickhouse-22.3.18
clickhouse-22.3.20
clickhouse-22.6.1
clickhouse-22.8.21
clickhouse-24.8.3
clickhouse-25.4.4
clickhouse-25.9.7 | ClickHouse is an open-source column-oriented OLAP database management system. Use it to boost your database performance while providing linear scalability and hardware efficiency. | loomt | +| clickhouse | clickhouse-22.3.18
clickhouse-22.3.20
clickhouse-22.6.1
clickhouse-22.8.21
clickhouse-24.12.2
clickhouse-24.7.2
clickhouse-24.8.3
clickhouse-25.4.4
clickhouse-25.9.7 | ClickHouse is an open-source column-oriented OLAP database management system. Use it to boost your database performance while providing linear scalability and hardware efficiency. | loomt | | elasticsearch | elasticsearch-6.8.23
elasticsearch-7.10.1
elasticsearch-7.10.2
elasticsearch-7.7.1
elasticsearch-7.8.1
elasticsearch-8.1.3
elasticsearch-8.15.5
elasticsearch-8.8.2
kibana-6.8.23
kibana-7.10.1
kibana-7.10.2
kibana-7.7.1
kibana-7.8.1
kibana-8.1.3
kibana-8.15.5
kibana-8.8.2
kibana-8.9.1 | Elasticsearch is a distributed, RESTful search engine optimized for speed and relevance on production-scale workloads. | kizuna-lek vipshop | | etcd | etcd-3.5.15
etcd-3.5.6
etcd-3.6.1 | Etcd is a strongly consistent, distributed key-value store that provides a reliable way to store data that needs to be accessed by a distributed system or cluster of machines. | loomt | | falkordb | falkordb-4.12.5
falkordb-cluster-4.12.5
falkordb-sent-4.12.5 | FalkorDB is an in-memory graph database based on Redis. | ApeCloud dudizimber | diff --git a/addons/clickhouse/README.md b/addons/clickhouse/README.md index 8e19e41a0b..496b8bbc61 100644 --- a/addons/clickhouse/README.md +++ b/addons/clickhouse/README.md @@ -36,7 +36,7 @@ There are two key components in the ClickHouse cluster: | Major Versions | Description | | -------------- | ------------------------- | | 22 | 22.3.18, 22.3.20, 22.8.21 | -| 24 | 24.8.3 | +| 24 | 24.7.2, 24.8.3, 24.12.2 | | 25 | 25.4.4, 25.9.7 | ## Prerequisites diff --git a/addons/clickhouse/dataprotection/common.sh b/addons/clickhouse/dataprotection/common.sh index 087a1f7159..527ba03317 100644 --- a/addons/clickhouse/dataprotection/common.sh +++ b/addons/clickhouse/dataprotection/common.sh @@ -597,14 +597,14 @@ function restore_schema_and_sync() { if [[ "$should_restore_schema" == "true" ]]; then # No-keeper restore: unset ON CLUSTER because distributed DDL requires Keeper/ZooKeeper. [[ "$mode_info" == "standalone" ]] && unset RESTORE_SCHEMA_ON_CLUSTER - local restore_args=(restore "$backup_name" --schema) + local restore_args=(restore_remote "$backup_name" --schema) if [[ -n "${CH_KEEPER_POD_FQDN_LIST:-}" ]]; then restore_args+=(--rbac) else DP_log "Skip RBAC restore because ClickHouse keeper is not configured" fi clickhouse-backup "${restore_args[@]}" || { - DP_error_log "Clickhouse-backup restore backup $backup_name FAILED" + DP_error_log "Clickhouse-backup restore_remote backup $backup_name FAILED" return 1 } # Cluster mode: create marker table for cross-shard coordination @@ -642,14 +642,12 @@ function do_restore() { local schema_db="kubeblocks" local schema_table="__restore_ready__" - fetch_backup "$backup_name" || return 1 - # Restore schema (first shard in cluster uses ON CLUSTER DDL) restore_schema_and_sync "$backup_name" "$mode_info" || return 1 # Restore data - clickhouse-backup restore "$backup_name" --data || { - DP_error_log "Clickhouse-backup restore --data FAILED" + clickhouse-backup restore_remote "$backup_name" --data || { + DP_error_log "Clickhouse-backup restore_remote --data FAILED" return 1 } diff --git a/addons/clickhouse/values.yaml b/addons/clickhouse/values.yaml index 257b524847..8a419ddb86 100644 --- a/addons/clickhouse/values.yaml +++ b/addons/clickhouse/values.yaml @@ -23,8 +23,12 @@ clickhouseVersions: imageTag: 25.9.7 - version: 25.4.4 imageTag: 25.4.4-debian-12-r1 + - version: 24.12.2 + imageTag: 24.12.2.29 - version: 24.8.3 imageTag: 24.8.3-debian-12-r1 + - version: 24.7.2 + imageTag: 24.7.2.13 - version: 22.8.21 imageTag: 22.8.21-debian-11-r33 - version: 22.3.20 diff --git a/examples/clickhouse/README.md b/examples/clickhouse/README.md index 4aea72f085..18798376d6 100644 --- a/examples/clickhouse/README.md +++ b/examples/clickhouse/README.md @@ -36,7 +36,7 @@ There are two key components in the ClickHouse cluster: | Major Versions | Description | | -------------- | ------------------------- | | 22 | 22.3.18, 22.3.20, 22.8.21 | -| 24 | 24.8.3 | +| 24 | 24.7.2, 24.8.3, 24.12.2 | | 25 | 25.4.4, 25.9.7 | ## Prerequisites From 38597957dc89617890eb52abaf9d1b2ad8c3b252 Mon Sep 17 00:00:00 2001 From: loomt Date: Mon, 1 Jun 2026 16:15:29 +0800 Subject: [PATCH 6/8] chore: bump up clickhouse-backup --- addons/clickhouse/README.md | 2 +- addons/clickhouse/values.yaml | 2 +- examples/clickhouse/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/addons/clickhouse/README.md b/addons/clickhouse/README.md index 496b8bbc61..f6ea235181 100644 --- a/addons/clickhouse/README.md +++ b/addons/clickhouse/README.md @@ -1409,7 +1409,7 @@ The incremental backup's `parentBackupName` must refer to an existing backup cre > [!NOTE] > When the restore target has TLS enabled, the restore job mounts the ClickHouse TLS files and connects through the secure TCP port. -Restore process will restore schema first, then restore data. Non-TLS restores also restore RBAC metadata; TLS restores focus on schema and data because replicated RBAC backup requires a direct Keeper TLS connection. You can tune schema-ready waiting behavior for restore jobs via Helm values: +Restore process will restore schema and RBAC metadata first, then restore data. When the restore target has TLS enabled, the restore job uses the secure ClickHouse TCP port and Keeper TLS settings for RBAC metadata. You can tune schema-ready waiting behavior for restore jobs via Helm values: ```yaml restore: diff --git a/addons/clickhouse/values.yaml b/addons/clickhouse/values.yaml index 8a419ddb86..d0854a1cd8 100644 --- a/addons/clickhouse/values.yaml +++ b/addons/clickhouse/values.yaml @@ -57,7 +57,7 @@ backupImage: registry: "" repository: apecloud/clickhouse-backup-full pullPolicy: IfNotPresent - tag: 2.6.42 + tag: 2.7.0 restore: schemaReadyTimeoutSeconds: 1800 diff --git a/examples/clickhouse/README.md b/examples/clickhouse/README.md index 18798376d6..ce367c9acf 100644 --- a/examples/clickhouse/README.md +++ b/examples/clickhouse/README.md @@ -629,7 +629,7 @@ The incremental backup's `parentBackupName` must refer to an existing backup cre > [!NOTE] > When the restore target has TLS enabled, the restore job mounts the ClickHouse TLS files and connects through the secure TCP port. -Restore process will restore schema first, then restore data. Non-TLS restores also restore RBAC metadata; TLS restores focus on schema and data because replicated RBAC backup requires a direct Keeper TLS connection. You can tune schema-ready waiting behavior for restore jobs via Helm values: +Restore process will restore schema and RBAC metadata first, then restore data. When the restore target has TLS enabled, the restore job uses the secure ClickHouse TCP port and Keeper TLS settings for RBAC metadata. You can tune schema-ready waiting behavior for restore jobs via Helm values: ```yaml restore: From 5ee8c3f6284a8e7010a953d5ed4495869e179a8a Mon Sep 17 00:00:00 2001 From: loomt Date: Mon, 1 Jun 2026 16:48:51 +0800 Subject: [PATCH 7/8] chore: revert readme changed --- README.md | 2 +- addons/clickhouse/README.md | 2 +- examples/clickhouse/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0aa35bd715..ee89555766 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ KubeBlocks add-ons. | NAME | VERSIOINS | DESCRIPTION | MAINTAINERS | | ---- | ---- | ----------- | ----------- | | apecloud-mysql | apecloud-mysql-8.0.30
wescale-0.2.7 | ApeCloud MySQL is a database that is compatible with MySQL syntax and achieves high availability through the utilization of the RAFT consensus protocol. | xuriwuyun | -| clickhouse | clickhouse-22.3.18
clickhouse-22.3.20
clickhouse-22.6.1
clickhouse-22.8.21
clickhouse-24.12.2
clickhouse-24.7.2
clickhouse-24.8.3
clickhouse-25.4.4
clickhouse-25.9.7 | ClickHouse is an open-source column-oriented OLAP database management system. Use it to boost your database performance while providing linear scalability and hardware efficiency. | loomt | +| clickhouse | clickhouse-22.3.18
clickhouse-22.3.20
clickhouse-22.6.1
clickhouse-22.8.21
clickhouse-22.9.4
clickhouse-24.12.2
clickhouse-24.7.2
clickhouse-24.8.3
clickhouse-25.4.4
clickhouse-25.9.7 | ClickHouse is an open-source column-oriented OLAP database management system. Use it to boost your database performance while providing linear scalability and hardware efficiency. | loomt | | elasticsearch | elasticsearch-6.8.23
elasticsearch-7.10.1
elasticsearch-7.10.2
elasticsearch-7.7.1
elasticsearch-7.8.1
elasticsearch-8.1.3
elasticsearch-8.15.5
elasticsearch-8.8.2
kibana-6.8.23
kibana-7.10.1
kibana-7.10.2
kibana-7.7.1
kibana-7.8.1
kibana-8.1.3
kibana-8.15.5
kibana-8.8.2
kibana-8.9.1 | Elasticsearch is a distributed, RESTful search engine optimized for speed and relevance on production-scale workloads. | kizuna-lek vipshop | | etcd | etcd-3.5.15
etcd-3.5.6
etcd-3.6.1 | Etcd is a strongly consistent, distributed key-value store that provides a reliable way to store data that needs to be accessed by a distributed system or cluster of machines. | loomt | | falkordb | falkordb-4.12.5
falkordb-cluster-4.12.5
falkordb-sent-4.12.5 | FalkorDB is an in-memory graph database based on Redis. | ApeCloud dudizimber | diff --git a/addons/clickhouse/README.md b/addons/clickhouse/README.md index f6ea235181..6d6ba271e1 100644 --- a/addons/clickhouse/README.md +++ b/addons/clickhouse/README.md @@ -35,7 +35,7 @@ There are two key components in the ClickHouse cluster: | Major Versions | Description | | -------------- | ------------------------- | -| 22 | 22.3.18, 22.3.20, 22.8.21 | +| 22 | 22.3.18, 22.3.20, 22.6.1, 22.8.21, 22.9.4 | | 24 | 24.7.2, 24.8.3, 24.12.2 | | 25 | 25.4.4, 25.9.7 | diff --git a/examples/clickhouse/README.md b/examples/clickhouse/README.md index ce367c9acf..b92e770d64 100644 --- a/examples/clickhouse/README.md +++ b/examples/clickhouse/README.md @@ -35,7 +35,7 @@ There are two key components in the ClickHouse cluster: | Major Versions | Description | | -------------- | ------------------------- | -| 22 | 22.3.18, 22.3.20, 22.8.21 | +| 22 | 22.3.18, 22.3.20, 22.6.1, 22.8.21, 22.9.4 | | 24 | 24.7.2, 24.8.3, 24.12.2 | | 25 | 25.4.4, 25.9.7 | From 9f7cab5bce858eb35fc1ee034276d04f3d204353 Mon Sep 17 00:00:00 2001 From: loomt Date: Tue, 2 Jun 2026 11:25:12 +0800 Subject: [PATCH 8/8] improve: enhance ClickHouse switchover using rqld instead of update priority --- addons/clickhouse/dataprotection/common.sh | 3 + .../clickhouse/scripts/keeper-switchover.sh | 97 +++++++------------ 2 files changed, 37 insertions(+), 63 deletions(-) diff --git a/addons/clickhouse/dataprotection/common.sh b/addons/clickhouse/dataprotection/common.sh index 527ba03317..77d1c71bb3 100644 --- a/addons/clickhouse/dataprotection/common.sh +++ b/addons/clickhouse/dataprotection/common.sh @@ -260,6 +260,9 @@ function init_datasafed() { function clickhouse_query() { local ch_port="${CLICKHOUSE_PORT:-9000}" local ch_args=(--user "${CLICKHOUSE_USERNAME}" --password "${CLICKHOUSE_PASSWORD}" --host "${CLICKHOUSE_HOST}" --port "$ch_port" --connect_timeout=5) + if [[ "${CLICKHOUSE_SECURE:-false}" == "true" ]]; then + ch_args+=(--secure --accept-invalid-certificate) + fi clickhouse-client "${ch_args[@]}" --query "$1" } diff --git a/addons/clickhouse/scripts/keeper-switchover.sh b/addons/clickhouse/scripts/keeper-switchover.sh index d8b7683d3d..8d4a97a848 100644 --- a/addons/clickhouse/scripts/keeper-switchover.sh +++ b/addons/clickhouse/scripts/keeper-switchover.sh @@ -5,81 +5,52 @@ source /scripts/common.sh leader_fqdn="$KB_SWITCHOVER_CURRENT_FQDN" candidate_fqdn="${KB_SWITCHOVER_CANDIDATE_FQDN:-}" -[[ $COMPONENT_REPLICAS -lt 2 ]] && exit 0 -[[ -z "$candidate_fqdn" ]] && { echo "No candidate specified, exit."; exit 0; } -[[ "$KB_SWITCHOVER_ROLE" != "leader" ]] && { echo "switchover not triggered for primary, nothing to do, exit 0."; exit 0; } - -# TODO(future): prefer use ClickHouse Keeper `rcfg` after 26.1.1.912+. - -function update_priority() { - local host="$1" - local line="$2" - local priority="$3" - local base_config="${line%;*}" - - retry_keeper_operation \ - "keeper_run '$host' 'reconfig add \"$base_config;$priority\"'" \ - "echo \"\$(get_config '$host')\" | grep -Fqx \"$base_config;$priority\"" +[[ $COMPONENT_REPLICAS -lt 2 ]] && { + echo "Skip: Keeper switchover requires at least 2 replicas." + exit 0 } -candidate_mode=$(get_mode "$candidate_fqdn" 2>/dev/null || true) -if [[ "$candidate_mode" != "follower" ]]; then - echo "ERROR: candidate $candidate_fqdn is not a stable follower, current mode: ${candidate_mode:-unknown}" >&2 - exit 1 -fi - -leader_zxid=$(get_zxid "$leader_fqdn" 2>/dev/null || true) -candidate_zxid=$(get_zxid "$candidate_fqdn" 2>/dev/null || true) -if [[ -z "$leader_zxid" || -z "$candidate_zxid" || "$leader_zxid" != "$candidate_zxid" ]]; then - echo "ERROR: candidate $candidate_fqdn is not caught up with leader $leader_fqdn (leader_zxid=${leader_zxid:-unknown}, candidate_zxid=${candidate_zxid:-unknown})" >&2 +[[ -n "$candidate_fqdn" ]] || { + echo "ERROR: ClickHouse Keeper switchover requires KB_SWITCHOVER_CANDIDATE_FQDN." >&2 exit 1 -fi +} -current_leader_fqdn=$(find_leader "${CH_KEEPER_POD_FQDN_LIST}" 2>/dev/null || true) -if [[ -n "$current_leader_fqdn" ]] && [[ "$current_leader_fqdn" != "$leader_fqdn" ]]; then - if [[ "$current_leader_fqdn" == "$candidate_fqdn" ]]; then - echo "Switchover already completed successfully" - exit 0 - fi - echo "ERROR: Expected new leader $candidate_fqdn, but got $current_leader_fqdn" >&2 +function die() { + echo "ERROR: $*" >&2 exit 1 -fi +} -config=$(get_config "$leader_fqdn") -original_config=$(printf "%s\n" "$config" | grep '^server\..*;participant;') +bin_path="/opt/bitnami/clickhouse/bin" +version=$("$bin_path/clickhouse-keeper" --version 2>/dev/null || "$bin_path/clickhouse" --version 2>/dev/null) || die "failed to determine ClickHouse Keeper version." +major=$(awk 'match($0, /[0-9]+\./) { print substr($0, RSTART, RLENGTH - 1); exit }' <<<"$version") +[[ "${major:-0}" -ge 24 ]] || die "Keeper switchover requires ClickHouse >= 24 (rqld support)." -while IFS= read -r line; do - if [[ "$line" == server.*";participant;"* ]]; then - line_fqdn=$(echo "$line" | cut -d'=' -f2 | cut -d':' -f1) - original_priority="${line##*;}" - target_priority=1 - [[ "$line_fqdn" == "$candidate_fqdn" ]] && target_priority=8 - [[ "$original_priority" != "$target_priority" ]] && update_priority "$leader_fqdn" "$line" "$target_priority" - fi -done <<<"$original_config" +[[ "$KB_SWITCHOVER_ROLE" == "leader" ]] || { + echo "Switchover already completed: action is not running on the leader, current role: ${KB_SWITCHOVER_ROLE:-unknown}." + exit 0 +} -send_4lw "$candidate_fqdn" "rqld" | grep -q "Sent leadership request to leader." || { - echo "ERROR: failed to request leadership from candidate $candidate_fqdn" >&2 - exit 1 +current_leader_fqdn=$(find_leader "${CH_KEEPER_POD_FQDN_LIST}") || die "failed to find current Keeper leader." +[[ "$current_leader_fqdn" == "$candidate_fqdn" ]] && { + echo "Switchover already completed: candidate $candidate_fqdn is leader." + exit 0 } +[[ "$current_leader_fqdn" == "$leader_fqdn" ]] || die "Expected current leader $leader_fqdn, but got $current_leader_fqdn" -new_leader_fqdn="$leader_fqdn" retry_keeper_operation \ - "new_leader_fqdn=\$(find_leader '${CH_KEEPER_POD_FQDN_LIST}')" \ - "[[ \"\$new_leader_fqdn\" == \"$candidate_fqdn\" ]]" + "candidate_mode=\$(get_mode '$candidate_fqdn')" \ + "[[ \"\$candidate_mode\" == \"follower\" ]]" || die "candidate $candidate_fqdn is not a stable follower, current mode: ${candidate_mode:-unknown}" +[[ "$candidate_mode" == "follower" ]] || die "candidate $candidate_fqdn is not a stable follower, current mode: ${candidate_mode:-unknown}" -if [[ "$new_leader_fqdn" != "$candidate_fqdn" ]]; then - echo "ERROR: Expected new leader $candidate_fqdn, but got $new_leader_fqdn" >&2 - exit 1 -fi +retry_keeper_operation \ + "leader_zxid=\$(get_zxid '$leader_fqdn'); candidate_zxid=\$(get_zxid '$candidate_fqdn')" \ + "[[ -n \"\$leader_zxid\" && \"\$leader_zxid\" == \"\$candidate_zxid\" ]]" || die "candidate $candidate_fqdn is not caught up with leader $leader_fqdn (leader_zxid=${leader_zxid:-unknown}, candidate_zxid=${candidate_zxid:-unknown})" +[[ -n "$leader_zxid" && "$leader_zxid" == "$candidate_zxid" ]] || die "candidate $candidate_fqdn is not caught up with leader $leader_fqdn (leader_zxid=${leader_zxid:-unknown}, candidate_zxid=${candidate_zxid:-unknown})" -while IFS= read -r line; do - if [[ "$line" == server.*";participant;"* ]]; then - original_priority="${line##*;}" - current_line=$(get_config "$new_leader_fqdn" | grep -F "${line%;*};" || true) - current_priority="${current_line##*;}" - [[ -n "$current_line" && "$current_priority" != "$original_priority" ]] && update_priority "$new_leader_fqdn" "$line" "$original_priority" - fi -done <<<"$original_config" +send_4lw "$candidate_fqdn" "rqld" | grep -q "Sent leadership request" || die "failed to request leadership from candidate $candidate_fqdn" + +retry_keeper_operation \ + "new_leader_fqdn=\$(find_leader '${CH_KEEPER_POD_FQDN_LIST}')" \ + "[[ \"\$new_leader_fqdn\" == \"$candidate_fqdn\" ]]" || die "Switchover timeout." echo "Switchover completed successfully"