feat(clm): add active-standby HA with leader election and reconciler (#1211)

cube-lifecycle-manager/README.md

@@ -42,3 +42,52 @@ Overrides:
 
 All configuration is via environment variables (prefix `CUBE_LCM_`); see
 `internal/config/config.go` for the authoritative list.
+
+## Active-standby HA (issue #1211)
+
+By default (`CUBE_LCM_HA_ENABLED` unset) the process runs every loop
+unconditionally, which is the right mode for the single-replica one-click
+deployment. When `CUBE_LCM_HA_ENABLED=1`, multiple replicas can run against
+the same Redis in active-standby mode:
+
+- Replicas elect a leader through a Redis lease
+  (`cube:v1:shared:lock:lifecycle_manager:leader`, registered in
+  `docs/zh/dev/redis-key-spec.md`). Only the leader runs the stateful loops:
+  stream consumer, idle sweeper, last-active poller, and the periodic
+  reconciler.
+- Standbys keep serving HTTP: `/readyz` returns 503 so the Kubernetes
+  Service routes `/internal/resume` only to the leader, but a standby can
+  still answer a resume that reaches it by looking the sandbox up directly
+  in the authoritative meta hash (registry-miss fallback).
+- On failover (lease expiry, at most one `CUBE_LCM_LEADER_TTL`) the new
+  leader re-bootstraps from the meta hash, replays the snapshot to every
+  CubeProxy, claims the dead consumer's pending stream entries via
+  `XAUTOCLAIM` (idle ≥ `CUBE_LCM_RECONCILE_INTERVAL`), and lets the
+  reconciler converge any remaining drift between the meta hash, the
+  in-memory registry, and the proxy meta dicts.
+
+HA-specific variables:
+
+| Variable | Default | Meaning |
+| --- | --- | --- |
+| `CUBE_LCM_HA_ENABLED` | `false` | Enable active-standby leader election |
+| `CUBE_LCM_INSTANCE_ID` | hostname | Unique replica identity written into the lease |
+| `CUBE_LCM_LEADER_KEY` | `cube:v1:shared:lock:lifecycle_manager:leader` | Lease key |
+| `CUBE_LCM_LEADER_TTL` | `15s` | Lease expiry; upper bound on failover time |
+| `CUBE_LCM_LEADER_RENEW_INTERVAL` | `5s` | Lease renewal / acquisition retry cadence |
+| `CUBE_LCM_RECONCILE_INTERVAL` | `60s` | Reconciler cadence; also the min idle time for `XAUTOCLAIM` takeover (must be ≥ `CUBE_LCM_LEADER_TTL`) |
+
+Two failure-handling notes:
+
+- If the leader loops fail fast three times in a row (each stint shorter
+  than `CUBE_LCM_LEADER_TTL`, e.g. a bootstrap dependency is down while
+  Redis itself is fine), the process exits so the pod supervisor restarts
+  it — instead of hot-looping elect → fail → step down forever. A stint
+  that survives at least one TTL counts as healthy and resets the counter.
+- `CUBE_LCM_RECONCILE_INTERVAL` must be ≥ `CUBE_LCM_LEADER_TTL` (enforced
+  at startup): it doubles as the `XAUTOCLAIM` min-idle, and a smaller
+  value could steal pending stream entries from a merely partitioned —
+  still alive — old leader.
+
+The Helm chart enables this by default (`lifecycleManager.ha.enabled: true`,
+`replicas: 2`) and derives `CUBE_LCM_INSTANCE_ID` from the pod name.

cube-lifecycle-manager/cmd/cube-lifecycle-manager/leadersupervisor.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2026 Tencent Inc.
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+)
+
+// maxLeaderStintFails is how many consecutive fast-failing leader stints are
+// tolerated before the process exits (see leaderSupervisor).
+const maxLeaderStintFails = 3
+
+// leaderSupervisor decides what happens when a leader stint ends. It exists
+// to keep a *permanently* failing leader loop from hot-looping: without it a
+// replica whose runLeaderLoops always fails fast (e.g. a bootstrap dependency
+// is down while Redis itself is fine) would cycle elect → fail → step down →
+// re-elect every renew interval forever — the process never exits, so the pod
+// supervisor never restarts it and the only signal is log spam.
+//
+// Policy:
+//   - a clean finish (nil / context.Canceled: leadership lost or shutdown)
+//     resets the counter;
+//   - a stint that survived at least stableAfter before failing counts as
+//     healthy — a transient mid-run failure (Redis blip killing the stream
+//     consumer) resets the counter too and just triggers a step-down;
+//   - maxFails consecutive fast-failed stints → Record reports exit=true so
+//     the process dies and Kubernetes (or any process supervisor) restarts it.
+type leaderSupervisor struct {
+	maxFails    int
+	stableAfter time.Duration
+
+	mu    sync.Mutex // a demoted stint may still be draining when the next one starts
+	fails int
+}
+
+// newLeaderSupervisor builds a supervisor; stableAfter is typically the
+// leader lease TTL.
+func newLeaderSupervisor(maxFails int, stableAfter time.Duration) *leaderSupervisor {
+	return &leaderSupervisor{maxFails: maxFails, stableAfter: stableAfter}
+}
+
+// Record notes the outcome of a leader stint and reports whether the process
+// should exit instead of stepping down and re-electing.
+func (s *leaderSupervisor) Record(err error, stint time.Duration) (exit bool) {
+	if err == nil || errors.Is(err, context.Canceled) {
+		s.reset()
+		return false
+	}
+	if stint >= s.stableAfter {
+		s.reset()
+		return false
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.fails++
+	return s.fails >= s.maxFails
+}
+
+// Fails is the current consecutive fast-failure count (for logging).
+func (s *leaderSupervisor) Fails() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.fails
+}
+
+func (s *leaderSupervisor) reset() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.fails = 0
+}

cube-lifecycle-manager/cmd/cube-lifecycle-manager/leadersupervisor_test.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2026 Tencent Inc.
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLeaderSupervisorFastFailuresExitAtThreshold(t *testing.T) {
+	s := newLeaderSupervisor(3, 15*time.Second)
+	boom := errors.New("boom")
+
+	assert.False(t, s.Record(boom, time.Second), "first fast failure should step down, not exit")
+	assert.False(t, s.Record(boom, time.Second), "second fast failure should step down, not exit")
+	assert.True(t, s.Record(boom, time.Second), "third consecutive fast failure should exit")
+	assert.Equal(t, 3, s.Fails())
+}
+
+func TestLeaderSupervisorCleanStintResets(t *testing.T) {
+	s := newLeaderSupervisor(3, 15*time.Second)
+	boom := errors.New("boom")
+
+	assert.False(t, s.Record(boom, time.Second))
+	assert.False(t, s.Record(boom, time.Second))
+
+	// A clean finish (leadership lost normally, or shutdown) resets the count.
+	assert.False(t, s.Record(nil, time.Minute))
+	assert.False(t, s.Record(context.Canceled, time.Minute))
+	assert.Equal(t, 0, s.Fails())
+
+	// So the next failure run starts from scratch.
+	assert.False(t, s.Record(boom, time.Second))
+	assert.False(t, s.Record(boom, time.Second))
+	assert.True(t, s.Record(boom, time.Second))
+}
+
+func TestLeaderSupervisorStableStintFailureResets(t *testing.T) {
+	s := newLeaderSupervisor(3, 15*time.Second)
+	boom := errors.New("boom")
+
+	assert.False(t, s.Record(boom, time.Second))
+	assert.False(t, s.Record(boom, time.Second))
+
+	// A stint that survived past stableAfter before failing counts as
+	// healthy: transient mid-run failures must not march the process
+	// towards exit.
+	assert.False(t, s.Record(boom, 16*time.Second))
+	assert.Equal(t, 0, s.Fails())
+
+	assert.False(t, s.Record(boom, time.Second))
+	assert.False(t, s.Record(boom, time.Second))
+	assert.True(t, s.Record(boom, time.Second))
+}
+
+func TestLeaderSupervisorContextWrappedCancelIsClean(t *testing.T) {
+	s := newLeaderSupervisor(3, 15*time.Second)
+	wrapped := errors.Join(errors.New("loop done"), context.Canceled)
+	assert.False(t, s.Record(wrapped, time.Second))
+	assert.Equal(t, 0, s.Fails())
+}