diff --git a/.github/workflows/kubernetes-chart-check.yml b/.github/workflows/kubernetes-chart-check.yml index da892d53d..73202ce08 100644 --- a/.github/workflows/kubernetes-chart-check.yml +++ b/.github/workflows/kubernetes-chart-check.yml @@ -53,6 +53,7 @@ jobs: sh -n deploy/kubernetes/images/scripts/pvm-startup-gate-reconcile.sh sh -n deploy/kubernetes/images/scripts/pvm-host-bootstrap.sh sh -n deploy/kubernetes/images/scripts/wait-node-prep.sh + sh -n deploy/kubernetes/chart/files/cube-proxy/cube-proxy-entrypoint.sh # Cloud-free chart/image guard suite (deploy/kubernetes/*/scripts/ # test-*.sh). Helm is installed above; every script is self-contained. # Loop so new test-*.sh are picked up automatically instead of drifting diff --git a/deploy/kubernetes/chart/README.md b/deploy/kubernetes/chart/README.md index d15cb775e..dac4e6286 100644 --- a/deploy/kubernetes/chart/README.md +++ b/deploy/kubernetes/chart/README.md @@ -231,6 +231,8 @@ postgres: The chart installs `cube-redis` StatefulSet only when `redis.enabled=true` and `redis.host` is empty. Set `redis.host` to use an existing Redis service; the chart will not install `cube-redis`. +CubeProxy registry heartbeats resolve Redis by hostname, so a redis Pod IP change self-heals. +Use an FQDN or IP literal: nginx's `resolver` does not apply search domains. ## CubeMaster configuration @@ -464,14 +466,16 @@ connections, the chart renders an nginx `resolver` into discovers resolver addresses from the Pod `/etc/resolv.conf`, which resolves the chart-managed `cube-redis..svc.cluster.local` Service name and third-party Redis DNS names. Override only when the cluster requires explicit -DNS servers: +DNS servers. Keep `cubeProxy.resolver.valid` well below +`lifecycleManager.heartbeatTTL` (defaults `5s` / `15s`) so a redis restart +does not look like a dead proxy. ```yaml cubeProxy: resolver: addresses: - 172.18.0.10 - valid: 30s + valid: 5s timeout: 5s ipv6: false ``` diff --git a/deploy/kubernetes/chart/files/cube-proxy/cube-proxy-entrypoint.sh b/deploy/kubernetes/chart/files/cube-proxy/cube-proxy-entrypoint.sh index f85b230ed..00979799e 100755 --- a/deploy/kubernetes/chart/files/cube-proxy/cube-proxy-entrypoint.sh +++ b/deploy/kubernetes/chart/files/cube-proxy/cube-proxy-entrypoint.sh @@ -96,21 +96,6 @@ sidecar_addr="${CUBE_SIDECAR_LISTEN_ADDR:-}" exit 1 } -# proxy_registry.lua publishes from ngx.timer, which has no nginx resolver. -# Resolve a hostname REDIS target to an IP up-front when possible. -# Sentinel mode does not need a fixed registry host. -if [ -z "${CUBE_PROXY_REGISTRY_REDIS_MASTER_NAME:-}" ] && [ -n "${CUBE_PROXY_REGISTRY_REDIS_HOST:-}" ]; then - case "${CUBE_PROXY_REGISTRY_REDIS_HOST}" in - *[!0-9.]* ) - resolved="$(getent ahostsv4 "${CUBE_PROXY_REGISTRY_REDIS_HOST}" 2>/dev/null | awk 'NR==1 {print $1}')" - if [ -n "${resolved}" ]; then - CUBE_PROXY_REGISTRY_REDIS_HOST="${resolved}" - export CUBE_PROXY_REGISTRY_REDIS_HOST - fi - ;; - esac -fi - cat > /usr/local/openresty/nginx/conf/global/global.conf < "$output" +} + +extract_registry_host() { + input="$1" + python3 - "$input" <<'PY' +import pathlib +import re +import sys + +text = pathlib.Path(sys.argv[1]).read_text() +m = re.search(r"name: CUBE_PROXY_REGISTRY_REDIS_HOST\s*\n\s*value:\s*\"([^\"]*)\"", text) +if not m: + raise SystemExit("CUBE_PROXY_REGISTRY_REDIS_HOST not found in rendered output") +sys.stdout.write(m.group(1) + "\n") +PY +} + +extract_checksum() { + input="$1" + python3 - "$input" <<'PY' +import pathlib +import re +import sys + +def main(): + text = pathlib.Path(sys.argv[1]).read_text() + for doc in text.split("\n---\n"): + if re.search(r"^kind: Deployment$", doc, re.M) and "app.kubernetes.io/component: cube-proxy" in doc: + m = re.search(r"checksum/entrypoint:\s*\"([0-9a-f]{64})\"", doc) + if not m: + raise SystemExit("checksum/entrypoint annotation not found on proxy Deployment") + sys.stdout.write(m.group(1) + "\n") + return + raise SystemExit("cube-proxy Deployment not found") + +main() +PY +} + +render "$TMP_DIR/builtin.yaml" +host_builtin="$(extract_registry_host "$TMP_DIR/builtin.yaml")" + +expected="registry-host-cube-redis.default.svc.cluster.local" +if [ "$host_builtin" != "$expected" ]; then + echo "expected builtin registry host '$expected', got '$host_builtin'" >&2 + exit 1 +fi +echo "builtin registry host OK: $host_builtin" + +render "$TMP_DIR/external.yaml" --set-string redis.host=redis.ext.example.com +host_external="$(extract_registry_host "$TMP_DIR/external.yaml")" +if [ "$host_external" != "redis.ext.example.com" ]; then + echo "expected external registry host passthrough, got '$host_external'" >&2 + exit 1 +fi +echo "external registry host passthrough OK: $host_external" + +render "$TMP_DIR/external-ip.yaml" --set-string redis.host=10.20.30.40 +host_ip="$(extract_registry_host "$TMP_DIR/external-ip.yaml")" +if [ "$host_ip" != "10.20.30.40" ]; then + echo "expected IP literal passthrough, got '$host_ip'" >&2 + exit 1 +fi +echo "external registry IP passthrough OK: $host_ip" + +render "$TMP_DIR/sentinel.yaml" \ + --set-string redis.masterName=mymaster \ + --set-string redis.sentinelNodes=10.0.0.11:26379 +host_sentinel="$(extract_registry_host "$TMP_DIR/sentinel.yaml")" +if [ -n "$host_sentinel" ]; then + echo "expected empty registry host in sentinel mode, got '$host_sentinel'" >&2 + exit 1 +fi +echo "sentinel registry host empty OK" + +checksum_base="$(extract_checksum "$TMP_DIR/builtin.yaml")" +[ -n "$checksum_base" ] || { echo "empty checksum/entrypoint" >&2; exit 1; } + +CHART_COPY="$TMP_DIR/chart-copy" +cp -r "$CHART_DIR" "$CHART_COPY" +printf '\n# guard: change to force a different checksum\n' >> "$CHART_COPY/files/cube-proxy/cube-proxy-entrypoint.sh" +helm template registry-host "$CHART_COPY" \ + --set-string mysql.password=test \ + --set-string mysql.rootPassword=test \ + --set-string redis.password=test \ + > "$TMP_DIR/tampered.yaml" +checksum_tampered="$(extract_checksum "$TMP_DIR/tampered.yaml")" +if [ "$checksum_base" = "$checksum_tampered" ]; then + echo "checksum/entrypoint did not change when entrypoint content changed" >&2 + exit 1 +fi +echo "checksum/entrypoint OK (tracks entrypoint content)" + +echo "Proxy registry host guard passed" diff --git a/deploy/kubernetes/chart/templates/proxy.yaml b/deploy/kubernetes/chart/templates/proxy.yaml index c03247a67..13c877f91 100644 --- a/deploy/kubernetes/chart/templates/proxy.yaml +++ b/deploy/kubernetes/chart/templates/proxy.yaml @@ -1,28 +1,14 @@ {{- if eq (include "cube.proxyEnabled" .) "true" }} +{{- /* Hash the entrypoint body so ConfigMap edits force a pod roll (directory mount, + no subPath). */ -}} +{{- $proxyEntrypointBody := .Files.Get "files/cube-proxy/cube-proxy-entrypoint.sh" -}} +{{- $proxyEntrypointChecksum := $proxyEntrypointBody | sha256sum -}} {{- $redisSentinel := eq (include "cube.redisSentinelEnabled" .) "true" -}} {{- $redisHost := "" -}} {{- if not $redisSentinel -}} {{- $redisHost = default (include "cube.redisHost" .) .Values.cubeProxy.redis.host -}} {{- end -}} {{- $redisPort := default .Values.redis.port .Values.cubeProxy.redis.port -}} -{{- /* proxy_registry.lua runs in ngx.timer and cannot use nginx location resolvers; - prefer a concrete IP (ClusterIP, or headless Endpoints address) for cosocket connect(). - Sentinel mode resolves the master via SENTINEL, so skip ClusterIP baking. */ -}} -{{- $redisRegistryHost := $redisHost -}} -{{- if and (not $redisSentinel) (eq (include "cube.redisBuiltinEnabled" .) "true") }} -{{- $redisSvc := lookup "v1" "Service" .Release.Namespace (include "cube.redisName" .) -}} -{{- if and $redisSvc $redisSvc.spec.clusterIP (ne $redisSvc.spec.clusterIP "None") -}} -{{- $redisRegistryHost = $redisSvc.spec.clusterIP -}} -{{- else -}} -{{- $redisEps := lookup "v1" "Endpoints" .Release.Namespace (include "cube.redisName" .) -}} -{{- if and $redisEps $redisEps.subsets (gt (len $redisEps.subsets) 0) -}} -{{- $subset := index $redisEps.subsets 0 -}} -{{- if and $subset.addresses (gt (len $subset.addresses) 0) -}} -{{- $redisRegistryHost = (index $subset.addresses 0).ip -}} -{{- end -}} -{{- end -}} -{{- end -}} -{{- end }} {{- $proxyCertSecretName := include "cube.proxyCertSecretName" . -}} {{- if eq .Values.cubeProxy.tls.mode "inline" }} apiVersion: v1 @@ -138,8 +124,9 @@ spec: labels: {{- include "cube.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: cube-proxy - {{- with .Values.cubeProxy.podAnnotations }} annotations: + checksum/entrypoint: {{ $proxyEntrypointChecksum | quote }} + {{- with .Values.cubeProxy.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} spec: @@ -239,7 +226,7 @@ spec: - name: CUBE_PROXY_HEARTBEAT_INTERVAL_MS value: {{ .Values.cubeProxy.registry.heartbeatIntervalMs | quote }} - name: CUBE_PROXY_REGISTRY_REDIS_HOST - value: {{ $redisRegistryHost | quote }} + value: {{ $redisHost | quote }} - name: CUBE_PROXY_REGISTRY_REDIS_PORT value: {{ $redisPort | quote }} - name: CUBE_PROXY_REGISTRY_REDIS_PASSWORD diff --git a/deploy/kubernetes/chart/values.yaml b/deploy/kubernetes/chart/values.yaml index 9c6159f3c..251b72639 100644 --- a/deploy/kubernetes/chart/values.yaml +++ b/deploy/kubernetes/chart/values.yaml @@ -474,6 +474,7 @@ redis: # Empty host installs and uses the chart-managed cube-redis service. # Non-empty host uses the configured third-party Redis service and skips cube-redis. # Sentinel mode (non-empty masterName) skips cube-redis and must leave host empty. + # CubeProxy resolves this via nginx (no search domains): use an FQDN or IP literal. host: "" port: 6379 # Redis Sentinel (external). Non-empty masterName enables Sentinel for @@ -713,6 +714,7 @@ cubeProxy: containerPort: 9090 redis: # Empty host uses redis.host or chart-managed cube-redis. + # Overrides must be an FQDN or IP literal (nginx resolver has no search domains). host: "" port: "" db: 0 @@ -776,7 +778,8 @@ cubeProxy: # configured by DNS name (the default Kubernetes Service DNS name). # Empty addresses auto-discovers nameservers from the Pod /etc/resolv.conf. addresses: [] - valid: 30s + # Keep well below lifecycleManager.heartbeatTTL (15s) so heartbeats recover after redis IP changes. + valid: 5s timeout: 5s ipv6: false probes: