diff --git a/addons/clickhouse/README.md b/addons/clickhouse/README.md index be3ded2eee..6d6ba271e1 100644 --- a/addons/clickhouse/README.md +++ b/addons/clickhouse/README.md @@ -35,7 +35,7 @@ There are two key components in the ClickHouse cluster: | Major Versions | Description | | -------------- | ------------------------- | -| 22 | 22.3.18, 22.3.20, 22.8.21 | +| 22 | 22.3.18, 22.3.20, 22.6.1, 22.8.21, 22.9.4 | | 24 | 24.7.2, 24.8.3, 24.12.2 | | 25 | 25.4.4, 25.9.7 | @@ -802,58 +802,9 @@ kubectl apply -f examples/clickhouse/verticalscale.yaml ### Switchover for Clickhouse Keeper -#### Switchover without preferred candidates +#### Switchover with specified candidate -Switchover a specified instance as the new primary or leader of the cluster - -```yaml -# cat examples/clickhouse/keeper-switchover.yaml -apiVersion: operations.kubeblocks.io/v1alpha1 -kind: OpsRequest -metadata: - name: keeper-switchover - namespace: demo -spec: - # Specifies the name of the Cluster resource that this operation is targeting. - clusterName: clickhouse-cluster - type: Switchover - # Lists Switchover objects, each specifying a Component to perform the switchover operation. - switchover: - # Specifies the name of the Component. - - componentName: ch-keeper - # Specifies the instance whose role will be transferred. - # A typical usage is to transfer the leader role in a consensus system. - instanceName: "clickhouse-cluster-ch-keeper-0" - -``` - -```yaml -# cat examples/clickhouse/keeper-switchover.yaml -apiVersion: operations.kubeblocks.io/v1alpha1 -kind: OpsRequest -metadata: - name: keeper-switchover - namespace: demo -spec: - # Specifies the name of the Cluster resource that this operation is targeting. - clusterName: clickhouse-cluster - type: Switchover - # Lists Switchover objects, each specifying a Component to perform the switchover operation. - switchover: - # Specifies the name of the Component. - - componentName: ch-keeper - # Specifies the instance whose role will be transferred. - # A typical usage is to transfer the leader role in a consensus system. - instanceName: "clickhouse-cluster-ch-keeper-0" -``` - -```bash -kubectl apply -f examples/clickhouse/keeper-switchover.yaml -``` - -#### Switchover-specified-instance - -Switchover a specified instance as the new primary or leader of the cluster +Switchover the current leader to a selected candidate instance. ClickHouse Keeper switchover currently requires `candidateName`. ```yaml # cat examples/clickhouse/keeper-switchover-specified-instance.yaml @@ -873,40 +824,17 @@ spec: # Specifies the instance whose role will be transferred. # A typical usage is to transfer the leader role in a consensus system. instanceName: "clickhouse-cluster-ch-keeper-0" - # Specifies the instance that will become the new leader, if not specify, the first non leader instance will become candidate. - # Need to ensure the candidate instance is catch up logs of the quorum, otherwise the switchover will transfer the leader to other instance. + # ClickHouse Keeper switchover currently requires candidateName. + # Need to ensure the candidate instance has caught up with quorum logs. candidateName: "clickhouse-cluster-ch-keeper-1" ``` -```yaml -# cat examples/clickhouse/keeper-switchover-specified-instance.yaml -apiVersion: operations.kubeblocks.io/v1alpha1 -kind: OpsRequest -metadata: - name: keeper-switchover - namespace: demo -spec: - # Specifies the name of the Cluster resource that this operation is targeting. - clusterName: clickhouse-cluster - type: Switchover - # Lists Switchover objects, each specifying a Component to perform the switchover operation. - switchover: - # Specifies the name of the Component. - - componentName: ch-keeper - # Specifies the instance whose role will be transferred. - # A typical usage is to transfer the leader role in a consensus system. - instanceName: "clickhouse-cluster-ch-keeper-0" - # Specifies the instance that will become the new leader, if not specify, the first non leader instance will become candidate. - # Need to ensure the candidate instance is catch up logs of the quorum, otherwise the switchover will transfer the leader to other instance. - candidateName: "clickhouse-cluster-ch-keeper-1" -``` - ```bash kubectl apply -f examples/clickhouse/keeper-switchover-specified-instance.yaml ``` -You may need to update the `opsrequest.spec.switchover.instanceName` field to your desired instance name. +You may need to update the `opsrequest.spec.switchover.instanceName` and `opsrequest.spec.switchover.candidateName` fields to your desired instance names. ### Expand volume @@ -1101,7 +1029,7 @@ spec: configMap: name: custom-user-configuration-tpl # refers to the configmap with your customized configuration info. variables: - max_threads: "16" + udf_max_threads: "16" systemAccounts: - name: admin secretRef: @@ -1479,9 +1407,9 @@ The incremental backup's `parentBackupName` must refer to an existing backup cre #### Restore Settings > [!NOTE] -> Restoring a TLS-enabled cluster directly from backup is NOT supported. You should restore the cluster with TLS disabled first, and then enable TLS manually after the restore process is complete. +> When the restore target has TLS enabled, the restore job mounts the ClickHouse TLS files and connects through the secure TCP port. -Restore process will restore schema and rbac first, then restore data. You can tune schema-ready waiting behavior for restore jobs via Helm values: +Restore process will restore schema and RBAC metadata first, then restore data. When the restore target has TLS enabled, the restore job uses the secure ClickHouse TCP port and Keeper TLS settings for RBAC metadata. You can tune schema-ready waiting behavior for restore jobs via Helm values: ```yaml restore: @@ -1531,7 +1459,7 @@ spec: storage: 10Gi shardings: - name: clickhouse - shards: 3 + shards: 2 template: name: clickhouse componentDef: clickhouse-1 @@ -1833,10 +1761,10 @@ spec: services: # Exposes ClickHouse service using NodePort - name: clickhouse-nodeport + # Sharding associated with this service + componentSelector: clickhouse # Type of the exposed service. Valid options are 'ClusterIP', 'NodePort', and 'LoadBalancer'. serviceType: NodePort - # Sharding associated with this service - shardingSelector: clickhouse # Role selector for the component service. # In a ClickHouse cluster, we only expose nodes without specifying roles # since all nodes can handle read/write requests diff --git a/addons/clickhouse/configs/00_default_overrides.xml.tpl b/addons/clickhouse/configs/00_default_overrides.xml.tpl index d17eed346f..5fba4fffab 100644 --- a/addons/clickhouse/configs/00_default_overrides.xml.tpl +++ b/addons/clickhouse/configs/00_default_overrides.xml.tpl @@ -106,10 +106,9 @@ {{$CERT_FILE}} {{$KEY_FILE}} - relaxed + + false {{$CA_FILE}} true sslv2,sslv3 @@ -124,6 +123,8 @@ sslv2,sslv3 true strict + + false RejectCertificateHandler diff --git a/addons/clickhouse/configs/ch_keeper_00_default_overrides.xml.tpl b/addons/clickhouse/configs/ch_keeper_00_default_overrides.xml.tpl index f855056075..748938033a 100644 --- a/addons/clickhouse/configs/ch_keeper_00_default_overrides.xml.tpl +++ b/addons/clickhouse/configs/ch_keeper_00_default_overrides.xml.tpl @@ -85,10 +85,9 @@ {{$CERT_FILE}} {{$KEY_FILE}} - relaxed + + false {{$CA_FILE}} true sslv2,sslv3 @@ -103,6 +102,8 @@ sslv2,sslv3 true strict + + false RejectCertificateHandler diff --git a/addons/clickhouse/dataprotection/common.sh b/addons/clickhouse/dataprotection/common.sh index 6a9ea54a18..77d1c71bb3 100644 --- a/addons/clickhouse/dataprotection/common.sh +++ b/addons/clickhouse/dataprotection/common.sh @@ -260,6 +260,9 @@ function init_datasafed() { function clickhouse_query() { local ch_port="${CLICKHOUSE_PORT:-9000}" local ch_args=(--user "${CLICKHOUSE_USERNAME}" --password "${CLICKHOUSE_PASSWORD}" --host "${CLICKHOUSE_HOST}" --port "$ch_port" --connect_timeout=5) + if [[ "${CLICKHOUSE_SECURE:-false}" == "true" ]]; then + ch_args+=(--secure --accept-invalid-certificate) + fi clickhouse-client "${ch_args[@]}" --query "$1" } @@ -468,6 +471,9 @@ function ch_query() { local query="$1" local ch_port="${CLICKHOUSE_PORT:-9000}" local ch_args=(--user "${CLICKHOUSE_USERNAME}" --password "${CLICKHOUSE_PASSWORD}" --host "${CLICKHOUSE_HOST}" --port "$ch_port" --connect_timeout=5) + if [[ "${CLICKHOUSE_SECURE:-false}" == "true" ]]; then + ch_args+=(--secure --accept-invalid-certificate) + fi clickhouse-client "${ch_args[@]}" --query "$query" } @@ -499,6 +505,37 @@ function fetch_backup() { DP_log "Backup '$backup_name' is available locally." } +function stage_required_backup_metadata() { + local target_base_path="$1" + shift + local backup_name + local metadata_file + + # clickhouse-backup expects incremental parents under the same S3_PATH. + # KubeBlocks stores each Backup under its own path, so stage parent metadata only. + export PATH="$PATH:$DP_DATASAFED_BIN_PATH" + for backup_name in "$@"; do + [[ -z "$backup_name" ]] && continue + metadata_file=$(mktemp) || { + DP_error_log "Failed to create temporary metadata file" + return 1 + } + export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_ROOT_PATH/$backup_name/$DP_TARGET_RELATIVE_PATH" + if ! datasafed pull "$backup_name/metadata.json" "$metadata_file"; then + rm -f "$metadata_file" + DP_error_log "Failed to pull metadata for backup '$backup_name'" + return 1 + fi + export DATASAFED_BACKEND_BASE_PATH="$target_base_path" + if ! datasafed push "$metadata_file" "$backup_name/metadata.json"; then + rm -f "$metadata_file" + DP_error_log "Failed to stage metadata for backup '$backup_name'" + return 1 + fi + rm -f "$metadata_file" + done +} + function delete_backups_except() { local latest_backup=$1 DP_log "delete backup except $latest_backup" @@ -523,6 +560,25 @@ function save_backup_size() { DP_save_backup_status_info "$backup_size" } +function detect_restore_mode() { + local first_entry="${ALL_COMBINED_SHARDS_POD_FQDN_LIST%%,*}" + local first_component="${first_entry%%:*}" + if [[ -z "$first_component" ]]; then + DP_error_log "Invalid ALL_COMBINED_SHARDS_POD_FQDN_LIST" >&2 + return 1 + fi + + if [[ -z "${CH_KEEPER_POD_FQDN_LIST:-}" ]]; then + echo "standalone" + return 0 + fi + + if [[ "$first_component" == "$first_entry" ]]; then + first_component="${CURRENT_SHARD_COMPONENT_SHORT_NAME}" + fi + echo "cluster:$first_component" +} + # Restore schema and wait for sync across shards function restore_schema_and_sync() { local backup_name="$1" diff --git a/addons/clickhouse/dataprotection/incremental-restore.sh b/addons/clickhouse/dataprotection/incremental-restore.sh index 30a7957df9..a67f68b249 100644 --- a/addons/clickhouse/dataprotection/incremental-restore.sh +++ b/addons/clickhouse/dataprotection/incremental-restore.sh @@ -8,36 +8,25 @@ trap handle_exit EXIT generate_backup_config set_clickhouse_backup_config_env -if [[ "${CLICKHOUSE_SECURE}" = "true" ]]; then - DP_error_log "ClickHouse restore does not support TLS" - exit 1 -fi - # 1. Download full (base) backup export S3_PATH="$DP_BACKUP_ROOT_PATH/$DP_BASE_BACKUP_NAME/$DP_TARGET_RELATIVE_PATH" fetch_backup "$DP_BASE_BACKUP_NAME" +downloaded_backups=("$DP_BASE_BACKUP_NAME") # 2. Download ancestor incremental backups IFS=',' read -r -a ancestors <<<"${DP_ANCESTOR_INCREMENTAL_BACKUP_NAMES}" for parent_name in "${ancestors[@]}"; do + [[ -z "$parent_name" ]] && continue export S3_PATH="$DP_BACKUP_ROOT_PATH/$parent_name/$DP_TARGET_RELATIVE_PATH" + stage_required_backup_metadata "$S3_PATH" "${downloaded_backups[@]}" || exit 1 fetch_backup "$parent_name" + downloaded_backups+=("$parent_name") done -# 3. Detect topology mode: standalone (no ':' in FQDN) or cluster +# 3. Detect topology mode: standalone or cluster export S3_PATH="${DP_BACKUP_BASE_PATH}" -first_entry="${ALL_COMBINED_SHARDS_POD_FQDN_LIST%%,*}" -first_component="${first_entry%%:*}" -if [[ -z "$first_component" ]]; then - DP_error_log "Invalid ALL_COMBINED_SHARDS_POD_FQDN_LIST" - exit 1 -fi -if [[ "$first_component" == "$first_entry" ]]; then - mode_info="standalone" - DP_log "Standalone mode detected" -else - mode_info="cluster:$first_component" -fi +stage_required_backup_metadata "$S3_PATH" "${downloaded_backups[@]}" || exit 1 +mode_info=$(detect_restore_mode) || exit 1 # 4. Restore schema + data + marker do_restore "${DP_BACKUP_NAME}" "$mode_info" || exit 1 diff --git a/addons/clickhouse/dataprotection/restore.sh b/addons/clickhouse/dataprotection/restore.sh index a71a82f0c5..4314e60355 100644 --- a/addons/clickhouse/dataprotection/restore.sh +++ b/addons/clickhouse/dataprotection/restore.sh @@ -8,24 +8,8 @@ trap handle_exit EXIT generate_backup_config set_clickhouse_backup_config_env -if [[ "${CLICKHOUSE_SECURE}" = "true" ]]; then - DP_error_log "ClickHouse restore does not support TLS" - exit 1 -fi - -# 1. Detect topology mode: standalone (no ':' in FQDN) or cluster -first_entry="${ALL_COMBINED_SHARDS_POD_FQDN_LIST%%,*}" -first_component="${first_entry%%:*}" -if [[ -z "$first_component" ]]; then - DP_error_log "Invalid ALL_COMBINED_SHARDS_POD_FQDN_LIST" - exit 1 -fi -if [[ "$first_component" == "$first_entry" ]]; then - mode_info="standalone" - DP_log "Standalone mode detected" -else - mode_info="cluster:$first_component" -fi +# 1. Detect topology mode: standalone or cluster +mode_info=$(detect_restore_mode) || exit 1 # 2. Restore schema + data + marker do_restore "${DP_BACKUP_NAME}" "$mode_info" || exit 1 diff --git a/addons/clickhouse/scripts/bootstrap-keeper.sh b/addons/clickhouse/scripts/bootstrap-keeper.sh index b3f7a2f598..8eceb2dce5 100644 --- a/addons/clickhouse/scripts/bootstrap-keeper.sh +++ b/addons/clickhouse/scripts/bootstrap-keeper.sh @@ -1,4 +1,6 @@ #!/bin/bash +set -euo pipefail + function chown_data_dir() { data_dir=/bitnami/clickhouse/data if [ ! -d "${data_dir}" ]; then @@ -10,7 +12,30 @@ function chown_data_dir() { chown -R root:root ${data_dir} fi } + +function wait_peer_dns_ready() { + local peers="${CH_KEEPER_POD_FQDN_LIST:-}" + local attempts=30 + local sleep_seconds=2 + + [[ -z "$peers" ]] && return 0 + + for peer in ${peers//,/ }; do + for attempt in $(seq 1 $attempts); do + if getent hosts "$peer" >/dev/null 2>&1; then + break + fi + if [[ $attempt -eq $attempts ]]; then + echo "$(date) ERROR: peer DNS $peer is still unresolved after $((attempts * sleep_seconds))s" >&2 + return 1 + fi + sleep "$sleep_seconds" + done + done +} + chown_data_dir +wait_peer_dns_ready HOSTNAME="$(hostname -s)" if grep -q "0" /opt/bitnami/clickhouse/etc/conf.d/ch-keeper_00_default_overrides.xml; then # compatible old version diff --git a/addons/clickhouse/scripts/common.sh b/addons/clickhouse/scripts/common.sh index dc7114d7c5..6a6d80f869 100644 --- a/addons/clickhouse/scripts/common.sh +++ b/addons/clickhouse/scripts/common.sh @@ -20,6 +20,28 @@ function ch_query() { clickhouse-client "${ch_args[@]}" --query "$query" } +function send_4lw() { + local host="$1" + local cmd="$2" + + if [[ "${TLS_ENABLED:-false}" == "true" ]]; then + timeout 2 openssl s_client \ + -connect "$host:$CLICKHOUSE_KEEPER_PORT" \ + -CAfile "$CLICKHOUSE_TLS_CA" \ + -cert "$CLICKHOUSE_TLS_CERT" \ + -key "$CLICKHOUSE_TLS_KEY" \ + -quiet \ + -ign_eof 2>/dev/null <<<"$cmd" + else + printf "%s" "$cmd" | /shared-tools/nc -w 2 "$host" "$CLICKHOUSE_KEEPER_PORT" + fi +} + +function get_srvr() { + local host="$1" + send_4lw "$host" "srvr" +} + # Low-level keeper client execution with connection retry # Handles connection issues, timeouts, and transient network problems function keeper_run() { @@ -45,7 +67,7 @@ function keeper_run() { if [[ "${TLS_ENABLED:-false}" == "true" ]]; then keeper_args+=(--tls-ca-file "$CLICKHOUSE_TLS_CA" --tls-cert-file "$CLICKHOUSE_TLS_CERT" --tls-key-file "$CLICKHOUSE_TLS_KEY") fi - if output=$(clickhouse-keeper-client "${keeper_args[@]}" 2>&1); then + if output=$(/opt/bitnami/clickhouse/bin/clickhouse-keeper-client "${keeper_args[@]}" 2>&1); then if [[ "$output" != *"Coordination error"* ]] && [[ "$output" != *"Connection refused"* ]] && [[ "$output" != *"Timeout"* ]]; then @@ -103,24 +125,16 @@ function get_config() { # Get keeper node mode (leader/follower/observer) function get_mode() { local host="$1" + local mode + mode=$(get_srvr "$host" 2>/dev/null | awk '/^Mode:/ {print $2; exit}') + echo "$mode" +} - # Use TLS port if TLS is enabled - if [[ "$TLS_ENABLED" == "true" ]]; then - # For TLS connections, use openssl s_client for 4LW commands - local mode - mode=$(printf "srvr\n" | timeout 2 openssl s_client \ - -connect "$host:$CLICKHOUSE_KEEPER_PORT" \ - -CAfile "$CLICKHOUSE_TLS_CA" \ - -cert "$CLICKHOUSE_TLS_CERT" \ - -key "$CLICKHOUSE_TLS_KEY" \ - -quiet \ - -ign_eof 2>/dev/null | grep Mode) - echo "$mode" | awk '{print $2}' - else - local mode - mode=$(echo srvr | /shared-tools/nc -w 1 "$host" "$CLICKHOUSE_KEEPER_PORT" | grep Mode) - echo "$mode" | awk '{print $2}' - fi +function get_zxid() { + local host="$1" + local zxid + zxid=$(get_srvr "$host" 2>/dev/null | awk '/^Zxid:/ {print $2; exit}') + echo "$zxid" } # Get keeper node mode by keeper diff --git a/addons/clickhouse/scripts/keeper-switchover.sh b/addons/clickhouse/scripts/keeper-switchover.sh index ad24b2a625..8d4a97a848 100644 --- a/addons/clickhouse/scripts/keeper-switchover.sh +++ b/addons/clickhouse/scripts/keeper-switchover.sh @@ -5,57 +5,52 @@ source /scripts/common.sh leader_fqdn="$KB_SWITCHOVER_CURRENT_FQDN" candidate_fqdn="${KB_SWITCHOVER_CANDIDATE_FQDN:-}" -if [ $COMPONENT_REPLICAS -lt 2 ]; then +[[ $COMPONENT_REPLICAS -lt 2 ]] && { + echo "Skip: Keeper switchover requires at least 2 replicas." exit 0 -fi +} -if [ -z $candidate_fqdn ]; then - echo "No candidate specified, exit." +[[ -n "$candidate_fqdn" ]] || { + echo "ERROR: ClickHouse Keeper switchover requires KB_SWITCHOVER_CANDIDATE_FQDN." >&2 + exit 1 +} + +function die() { + echo "ERROR: $*" >&2 + exit 1 +} + +bin_path="/opt/bitnami/clickhouse/bin" +version=$("$bin_path/clickhouse-keeper" --version 2>/dev/null || "$bin_path/clickhouse" --version 2>/dev/null) || die "failed to determine ClickHouse Keeper version." +major=$(awk 'match($0, /[0-9]+\./) { print substr($0, RSTART, RLENGTH - 1); exit }' <<<"$version") +[[ "${major:-0}" -ge 24 ]] || die "Keeper switchover requires ClickHouse >= 24 (rqld support)." + +[[ "$KB_SWITCHOVER_ROLE" == "leader" ]] || { + echo "Switchover already completed: action is not running on the leader, current role: ${KB_SWITCHOVER_ROLE:-unknown}." exit 0 -fi +} -if [ "$KB_SWITCHOVER_ROLE" != "leader" ]; then - echo "switchover not triggered for primary, nothing to do, exit 0." +current_leader_fqdn=$(find_leader "${CH_KEEPER_POD_FQDN_LIST}") || die "failed to find current Keeper leader." +[[ "$current_leader_fqdn" == "$candidate_fqdn" ]] && { + echo "Switchover already completed: candidate $candidate_fqdn is leader." exit 0 -fi - -# 1. Get current config -config=$(get_config "$leader_fqdn") - -# 2. Change the priority of the candidate to 8, and the others to 1 -pre_leader=$(echo "$config" | grep "$leader_fqdn") -pre_leader=${pre_leader%;*}";1" -pre_leader_config_name=$(echo "$pre_leader" | cut -d'=' -f1) -pre_leader_config_id=$(echo "$pre_leader_config_name" | cut -d'.' -f2) -# server.1=ch-cluster-ch-keeper-0.ch-cluster-ch-keeper-headless.default.svc.cluster.local:9234;participant;1 -while IFS= read -r line; do - if [[ "$line" == server.*";participant;"* ]]; then - line_fqdn=$(echo "$line" | cut -d'=' -f2 | cut -d':' -f1) - original_priority="${line##*;}" - base_config="${line%;*}" - new_priority="" - if echo "$line_fqdn" | grep -q "$candidate_fqdn"; then - [[ "$original_priority" -ne 8 ]] && new_priority=8 - else - [[ "$original_priority" -ne 1 ]] && new_priority=1 - fi - [[ -n "$new_priority" ]] && retry_keeper_operation \ - "keeper_run '$leader_fqdn' 'reconfig add \"$base_config;$new_priority\"'" \ - "echo \"\$(get_config '$leader_fqdn')\" | grep -q \"$base_config;$new_priority\"" - fi -done <<<"$config" - -# 3. Remove the leader from the config, remove once, because only the leader can remove itself -keeper_run "$leader_fqdn" "reconfig remove '$pre_leader_config_id'" - -# 4. Re-add after pre leader reboot +} +[[ "$current_leader_fqdn" == "$leader_fqdn" ]] || die "Expected current leader $leader_fqdn, but got $current_leader_fqdn" + retry_keeper_operation \ - "keeper_run '$candidate_fqdn' 'reconfig add \"$pre_leader\"'" \ - "get_config '$candidate_fqdn' | grep -q '$pre_leader'" + "candidate_mode=\$(get_mode '$candidate_fqdn')" \ + "[[ \"\$candidate_mode\" == \"follower\" ]]" || die "candidate $candidate_fqdn is not a stable follower, current mode: ${candidate_mode:-unknown}" +[[ "$candidate_mode" == "follower" ]] || die "candidate $candidate_fqdn is not a stable follower, current mode: ${candidate_mode:-unknown}" + +retry_keeper_operation \ + "leader_zxid=\$(get_zxid '$leader_fqdn'); candidate_zxid=\$(get_zxid '$candidate_fqdn')" \ + "[[ -n \"\$leader_zxid\" && \"\$leader_zxid\" == \"\$candidate_zxid\" ]]" || die "candidate $candidate_fqdn is not caught up with leader $leader_fqdn (leader_zxid=${leader_zxid:-unknown}, candidate_zxid=${candidate_zxid:-unknown})" +[[ -n "$leader_zxid" && "$leader_zxid" == "$candidate_zxid" ]] || die "candidate $candidate_fqdn is not caught up with leader $leader_fqdn (leader_zxid=${leader_zxid:-unknown}, candidate_zxid=${candidate_zxid:-unknown})" + +send_4lw "$candidate_fqdn" "rqld" | grep -q "Sent leadership request" || die "failed to request leadership from candidate $candidate_fqdn" -# 5. Check if the candidate is the leader retry_keeper_operation \ - "mode=\$(get_mode_by_keeper '$candidate_fqdn')" \ - "[[ \"\$mode\" == \"leader\" ]]" + "new_leader_fqdn=\$(find_leader '${CH_KEEPER_POD_FQDN_LIST}')" \ + "[[ \"\$new_leader_fqdn\" == \"$candidate_fqdn\" ]]" || die "Switchover timeout." echo "Switchover completed successfully" diff --git a/addons/clickhouse/templates/opsdefinition.yaml b/addons/clickhouse/templates/opsdefinition.yaml index 1d9f8212f4..5a193e974c 100644 --- a/addons/clickhouse/templates/opsdefinition.yaml +++ b/addons/clickhouse/templates/opsdefinition.yaml @@ -43,7 +43,7 @@ spec: podSpec: containers: - name: post-rebuild - image: {{ include "clickhouse.repository" . }}:24.8.3-debian-12-r1 + image: {{ include "clickhouse.repository" . }}:{{ .Chart.AppVersion }} imagePullPolicy: IfNotPresent command: - bash @@ -92,7 +92,7 @@ spec: podSpec: containers: - name: post-rebuild - image: {{ include "clickhouse.repository" . }}:24.8.3-debian-12-r1 + image: {{ include "clickhouse.repository" . }}:{{ .Chart.AppVersion }} imagePullPolicy: IfNotPresent command: - bash diff --git a/addons/clickhouse/values.yaml b/addons/clickhouse/values.yaml index 5f2a0fc67d..d0854a1cd8 100644 --- a/addons/clickhouse/values.yaml +++ b/addons/clickhouse/values.yaml @@ -44,7 +44,7 @@ roleProbe: initialDelaySeconds: 15 failureThreshold: 3 periodSeconds: 3 - timeoutSeconds: 3 + timeoutSeconds: 10 busyboxImage: registry: "" @@ -57,7 +57,7 @@ backupImage: registry: "" repository: apecloud/clickhouse-backup-full pullPolicy: IfNotPresent - tag: 2.6.42 + tag: 2.7.0 restore: schemaReadyTimeoutSeconds: 1800 diff --git a/examples/clickhouse/README.md b/examples/clickhouse/README.md index 1480c22e50..b92e770d64 100644 --- a/examples/clickhouse/README.md +++ b/examples/clickhouse/README.md @@ -35,7 +35,7 @@ There are two key components in the ClickHouse cluster: | Major Versions | Description | | -------------- | ------------------------- | -| 22 | 22.3.18, 22.3.20, 22.8.21 | +| 22 | 22.3.18, 22.3.20, 22.6.1, 22.8.21, 22.9.4 | | 24 | 24.7.2, 24.8.3, 24.12.2 | | 25 | 25.4.4, 25.9.7 | @@ -288,38 +288,9 @@ kubectl apply -f examples/clickhouse/verticalscale.yaml ### Switchover for Clickhouse Keeper -#### Switchover without preferred candidates +#### Switchover with specified candidate -Switchover a specified instance as the new primary or leader of the cluster - -```yaml -# cat examples/clickhouse/keeper-switchover.yaml -apiVersion: operations.kubeblocks.io/v1alpha1 -kind: OpsRequest -metadata: - name: keeper-switchover - namespace: demo -spec: - # Specifies the name of the Cluster resource that this operation is targeting. - clusterName: clickhouse-cluster - type: Switchover - # Lists Switchover objects, each specifying a Component to perform the switchover operation. - switchover: - # Specifies the name of the Component. - - componentName: ch-keeper - # Specifies the instance whose role will be transferred. - # A typical usage is to transfer the leader role in a consensus system. - instanceName: "clickhouse-cluster-ch-keeper-0" - -``` - -```bash -kubectl apply -f examples/clickhouse/keeper-switchover.yaml -``` - -#### Switchover-specified-instance - -Switchover a specified instance as the new primary or leader of the cluster +Switchover the current leader to a selected candidate instance. ClickHouse Keeper switchover currently requires `candidateName`. ```yaml # cat examples/clickhouse/keeper-switchover-specified-instance.yaml @@ -339,8 +310,8 @@ spec: # Specifies the instance whose role will be transferred. # A typical usage is to transfer the leader role in a consensus system. instanceName: "clickhouse-cluster-ch-keeper-0" - # Specifies the instance that will become the new leader, if not specify, the first non leader instance will become candidate. - # Need to ensure the candidate instance is catch up logs of the quorum, otherwise the switchover will transfer the leader to other instance. + # ClickHouse Keeper switchover currently requires candidateName. + # Need to ensure the candidate instance has caught up with quorum logs. candidateName: "clickhouse-cluster-ch-keeper-1" ``` @@ -349,7 +320,7 @@ spec: kubectl apply -f examples/clickhouse/keeper-switchover-specified-instance.yaml ``` -You may need to update the `opsrequest.spec.switchover.instanceName` field to your desired instance name. +You may need to update the `opsrequest.spec.switchover.instanceName` and `opsrequest.spec.switchover.candidateName` fields to your desired instance names. ### [Expand volume](volumeexpand.yaml) @@ -656,9 +627,9 @@ The incremental backup's `parentBackupName` must refer to an existing backup cre #### Restore Settings > [!NOTE] -> Restoring a TLS-enabled cluster directly from backup is NOT supported. You should restore the cluster with TLS disabled first, and then enable TLS manually after the restore process is complete. +> When the restore target has TLS enabled, the restore job mounts the ClickHouse TLS files and connects through the secure TCP port. -Restore process will restore schema and rbac first, then restore data. You can tune schema-ready waiting behavior for restore jobs via Helm values: +Restore process will restore schema and RBAC metadata first, then restore data. When the restore target has TLS enabled, the restore job uses the secure ClickHouse TCP port and Keeper TLS settings for RBAC metadata. You can tune schema-ready waiting behavior for restore jobs via Helm values: ```yaml restore: diff --git a/examples/clickhouse/cluster-with-config-templates.yaml b/examples/clickhouse/cluster-with-config-templates.yaml index b1f80bae26..891819ae00 100644 --- a/examples/clickhouse/cluster-with-config-templates.yaml +++ b/examples/clickhouse/cluster-with-config-templates.yaml @@ -43,7 +43,7 @@ spec: configMap: name: custom-user-configuration-tpl # refers to the configmap with your customized configuration info. variables: - max_threads: "16" + udf_max_threads: "16" systemAccounts: - name: admin secretRef: diff --git a/examples/clickhouse/cluster-with-nodeport.yaml b/examples/clickhouse/cluster-with-nodeport.yaml index db819ec7d5..b8f734befc 100644 --- a/examples/clickhouse/cluster-with-nodeport.yaml +++ b/examples/clickhouse/cluster-with-nodeport.yaml @@ -9,10 +9,10 @@ spec: services: # Exposes ClickHouse service using NodePort - name: clickhouse-nodeport + # Sharding associated with this service + componentSelector: clickhouse # Type of the exposed service. Valid options are 'ClusterIP', 'NodePort', and 'LoadBalancer'. serviceType: NodePort - # Sharding associated with this service - shardingSelector: clickhouse # Role selector for the component service. # In a ClickHouse cluster, we only expose nodes without specifying roles # since all nodes can handle read/write requests @@ -84,4 +84,4 @@ metadata: namespace: demo type: Opaque data: - password: cGFzc3dvcmQxMjM= # 'password123' in base64 \ No newline at end of file + password: cGFzc3dvcmQxMjM= # 'password123' in base64 diff --git a/examples/clickhouse/keeper-switchover-specified-instance.yaml b/examples/clickhouse/keeper-switchover-specified-instance.yaml index db20429c6c..e105c2b28d 100644 --- a/examples/clickhouse/keeper-switchover-specified-instance.yaml +++ b/examples/clickhouse/keeper-switchover-specified-instance.yaml @@ -14,6 +14,6 @@ spec: # Specifies the instance whose role will be transferred. # A typical usage is to transfer the leader role in a consensus system. instanceName: "clickhouse-cluster-ch-keeper-0" - # Specifies the instance that will become the new leader, if not specify, the first non leader instance will become candidate. - # Need to ensure the candidate instance is catch up logs of the quorum, otherwise the switchover will transfer the leader to other instance. - candidateName: "clickhouse-cluster-ch-keeper-1" \ No newline at end of file + # ClickHouse Keeper switchover currently requires candidateName. + # Need to ensure the candidate instance has caught up with quorum logs. + candidateName: "clickhouse-cluster-ch-keeper-1" diff --git a/examples/clickhouse/keeper-switchover.yaml b/examples/clickhouse/keeper-switchover.yaml deleted file mode 100644 index 7b1b4d8b5d..0000000000 --- a/examples/clickhouse/keeper-switchover.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: operations.kubeblocks.io/v1alpha1 -kind: OpsRequest -metadata: - name: keeper-switchover - namespace: demo -spec: - # Specifies the name of the Cluster resource that this operation is targeting. - clusterName: clickhouse-cluster - type: Switchover - # Lists Switchover objects, each specifying a Component to perform the switchover operation. - switchover: - # Specifies the name of the Component. - - componentName: ch-keeper - # Specifies the instance whose role will be transferred. - # A typical usage is to transfer the leader role in a consensus system. - instanceName: "clickhouse-cluster-ch-keeper-0" \ No newline at end of file diff --git a/examples/clickhouse/restore.yaml b/examples/clickhouse/restore.yaml index 3b7b0fd64f..3cf080b314 100644 --- a/examples/clickhouse/restore.yaml +++ b/examples/clickhouse/restore.yaml @@ -36,7 +36,7 @@ spec: storage: 10Gi shardings: - name: clickhouse - shards: 3 + shards: 2 template: name: clickhouse componentDef: clickhouse-1