Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/kubernetes-chart-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ jobs:
sh -n deploy/kubernetes/images/scripts/pvm-startup-gate-reconcile.sh
sh -n deploy/kubernetes/images/scripts/pvm-host-bootstrap.sh
sh -n deploy/kubernetes/images/scripts/wait-node-prep.sh
sh -n deploy/kubernetes/chart/files/cube-proxy/cube-proxy-entrypoint.sh
# Cloud-free chart/image guard suite (deploy/kubernetes/*/scripts/
# test-*.sh). Helm is installed above; every script is self-contained.
# Loop so new test-*.sh are picked up automatically instead of drifting
Expand Down
8 changes: 6 additions & 2 deletions deploy/kubernetes/chart/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,8 @@ postgres:

The chart installs `cube-redis` StatefulSet only when `redis.enabled=true` and `redis.host` is empty.
Set `redis.host` to use an existing Redis service; the chart will not install `cube-redis`.
CubeProxy registry heartbeats resolve Redis by hostname, so a redis Pod IP change self-heals.
Use an FQDN or IP literal: nginx's `resolver` does not apply search domains.

## CubeMaster configuration

Expand Down Expand Up @@ -464,14 +466,16 @@ connections, the chart renders an nginx `resolver` into
discovers resolver addresses from the Pod `/etc/resolv.conf`, which resolves the
chart-managed `cube-redis.<namespace>.svc.cluster.local` Service name and
third-party Redis DNS names. Override only when the cluster requires explicit
DNS servers:
DNS servers. Keep `cubeProxy.resolver.valid` well below
`lifecycleManager.heartbeatTTL` (defaults `5s` / `15s`) so a redis restart
does not look like a dead proxy.

```yaml
cubeProxy:
resolver:
addresses:
- 172.18.0.10
valid: 30s
valid: 5s
timeout: 5s
ipv6: false
```
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,21 +96,6 @@ sidecar_addr="${CUBE_SIDECAR_LISTEN_ADDR:-}"
exit 1
}

# proxy_registry.lua publishes from ngx.timer, which has no nginx resolver.
# Resolve a hostname REDIS target to an IP up-front when possible.
# Sentinel mode does not need a fixed registry host.
if [ -z "${CUBE_PROXY_REGISTRY_REDIS_MASTER_NAME:-}" ] && [ -n "${CUBE_PROXY_REGISTRY_REDIS_HOST:-}" ]; then
case "${CUBE_PROXY_REGISTRY_REDIS_HOST}" in
*[!0-9.]* )
resolved="$(getent ahostsv4 "${CUBE_PROXY_REGISTRY_REDIS_HOST}" 2>/dev/null | awk 'NR==1 {print $1}')"
if [ -n "${resolved}" ]; then
CUBE_PROXY_REGISTRY_REDIS_HOST="${resolved}"
export CUBE_PROXY_REGISTRY_REDIS_HOST
fi
;;
esac
fi

cat > /usr/local/openresty/nginx/conf/global/global.conf <<EOF
resolver ${resolver_addrs} valid=${CUBE_PROXY_RESOLVER_VALID} ipv6=${CUBE_PROXY_RESOLVER_IPV6};
resolver_timeout ${CUBE_PROXY_RESOLVER_TIMEOUT};
Expand Down
110 changes: 110 additions & 0 deletions deploy/kubernetes/chart/scripts/test-proxy-registry-host.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
#!/bin/sh
set -eu

SCRIPT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
CHART_DIR="$(dirname "$SCRIPT_DIR")"
TMP_DIR="$(mktemp -d)"
trap 'rm -rf "$TMP_DIR"' EXIT

render() {
output="$1"
shift
helm template registry-host "$CHART_DIR" \
--set-string mysql.password=test \
--set-string mysql.rootPassword=test \
--set-string redis.password=test \
"$@" > "$output"
}

extract_registry_host() {
input="$1"
python3 - "$input" <<'PY'
import pathlib
import re
import sys

text = pathlib.Path(sys.argv[1]).read_text()
m = re.search(r"name: CUBE_PROXY_REGISTRY_REDIS_HOST\s*\n\s*value:\s*\"([^\"]*)\"", text)
if not m:
raise SystemExit("CUBE_PROXY_REGISTRY_REDIS_HOST not found in rendered output")
sys.stdout.write(m.group(1) + "\n")
PY
}

extract_checksum() {
input="$1"
python3 - "$input" <<'PY'
import pathlib
import re
import sys

def main():
text = pathlib.Path(sys.argv[1]).read_text()
for doc in text.split("\n---\n"):
if re.search(r"^kind: Deployment$", doc, re.M) and "app.kubernetes.io/component: cube-proxy" in doc:
m = re.search(r"checksum/entrypoint:\s*\"([0-9a-f]{64})\"", doc)
if not m:
raise SystemExit("checksum/entrypoint annotation not found on proxy Deployment")
sys.stdout.write(m.group(1) + "\n")
return
raise SystemExit("cube-proxy Deployment not found")

main()
PY
}

render "$TMP_DIR/builtin.yaml"
host_builtin="$(extract_registry_host "$TMP_DIR/builtin.yaml")"

expected="registry-host-cube-redis.default.svc.cluster.local"
if [ "$host_builtin" != "$expected" ]; then
echo "expected builtin registry host '$expected', got '$host_builtin'" >&2
exit 1
fi
echo "builtin registry host OK: $host_builtin"

render "$TMP_DIR/external.yaml" --set-string redis.host=redis.ext.example.com
host_external="$(extract_registry_host "$TMP_DIR/external.yaml")"
if [ "$host_external" != "redis.ext.example.com" ]; then
echo "expected external registry host passthrough, got '$host_external'" >&2
exit 1
fi
echo "external registry host passthrough OK: $host_external"

render "$TMP_DIR/external-ip.yaml" --set-string redis.host=10.20.30.40
host_ip="$(extract_registry_host "$TMP_DIR/external-ip.yaml")"
if [ "$host_ip" != "10.20.30.40" ]; then
echo "expected IP literal passthrough, got '$host_ip'" >&2
exit 1
fi
echo "external registry IP passthrough OK: $host_ip"

render "$TMP_DIR/sentinel.yaml" \
--set-string redis.masterName=mymaster \
--set-string redis.sentinelNodes=10.0.0.11:26379
host_sentinel="$(extract_registry_host "$TMP_DIR/sentinel.yaml")"
if [ -n "$host_sentinel" ]; then
echo "expected empty registry host in sentinel mode, got '$host_sentinel'" >&2
exit 1
fi
echo "sentinel registry host empty OK"

checksum_base="$(extract_checksum "$TMP_DIR/builtin.yaml")"
[ -n "$checksum_base" ] || { echo "empty checksum/entrypoint" >&2; exit 1; }

CHART_COPY="$TMP_DIR/chart-copy"
cp -r "$CHART_DIR" "$CHART_COPY"
printf '\n# guard: change to force a different checksum\n' >> "$CHART_COPY/files/cube-proxy/cube-proxy-entrypoint.sh"
helm template registry-host "$CHART_COPY" \
--set-string mysql.password=test \
--set-string mysql.rootPassword=test \
--set-string redis.password=test \
> "$TMP_DIR/tampered.yaml"
checksum_tampered="$(extract_checksum "$TMP_DIR/tampered.yaml")"
if [ "$checksum_base" = "$checksum_tampered" ]; then
echo "checksum/entrypoint did not change when entrypoint content changed" >&2
exit 1
fi
echo "checksum/entrypoint OK (tracks entrypoint content)"

echo "Proxy registry host guard passed"
27 changes: 7 additions & 20 deletions deploy/kubernetes/chart/templates/proxy.yaml
Original file line number Diff line number Diff line change
@@ -1,28 +1,14 @@
{{- if eq (include "cube.proxyEnabled" .) "true" }}
{{- /* Hash the entrypoint body so ConfigMap edits force a pod roll (directory mount,
no subPath). */ -}}
{{- $proxyEntrypointBody := .Files.Get "files/cube-proxy/cube-proxy-entrypoint.sh" -}}
{{- $proxyEntrypointChecksum := $proxyEntrypointBody | sha256sum -}}
{{- $redisSentinel := eq (include "cube.redisSentinelEnabled" .) "true" -}}
{{- $redisHost := "" -}}
{{- if not $redisSentinel -}}
{{- $redisHost = default (include "cube.redisHost" .) .Values.cubeProxy.redis.host -}}
{{- end -}}
{{- $redisPort := default .Values.redis.port .Values.cubeProxy.redis.port -}}
{{- /* proxy_registry.lua runs in ngx.timer and cannot use nginx location resolvers;
prefer a concrete IP (ClusterIP, or headless Endpoints address) for cosocket connect().
Sentinel mode resolves the master via SENTINEL, so skip ClusterIP baking. */ -}}
{{- $redisRegistryHost := $redisHost -}}
{{- if and (not $redisSentinel) (eq (include "cube.redisBuiltinEnabled" .) "true") }}
{{- $redisSvc := lookup "v1" "Service" .Release.Namespace (include "cube.redisName" .) -}}
{{- if and $redisSvc $redisSvc.spec.clusterIP (ne $redisSvc.spec.clusterIP "None") -}}
{{- $redisRegistryHost = $redisSvc.spec.clusterIP -}}
{{- else -}}
{{- $redisEps := lookup "v1" "Endpoints" .Release.Namespace (include "cube.redisName" .) -}}
{{- if and $redisEps $redisEps.subsets (gt (len $redisEps.subsets) 0) -}}
{{- $subset := index $redisEps.subsets 0 -}}
{{- if and $subset.addresses (gt (len $subset.addresses) 0) -}}
{{- $redisRegistryHost = (index $subset.addresses 0).ip -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- end }}
{{- $proxyCertSecretName := include "cube.proxyCertSecretName" . -}}
{{- if eq .Values.cubeProxy.tls.mode "inline" }}
apiVersion: v1
Expand Down Expand Up @@ -138,8 +124,9 @@ spec:
labels:
{{- include "cube.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: cube-proxy
{{- with .Values.cubeProxy.podAnnotations }}
annotations:
checksum/entrypoint: {{ $proxyEntrypointChecksum | quote }}
{{- with .Values.cubeProxy.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
Expand Down Expand Up @@ -239,7 +226,7 @@ spec:
- name: CUBE_PROXY_HEARTBEAT_INTERVAL_MS
value: {{ .Values.cubeProxy.registry.heartbeatIntervalMs | quote }}
- name: CUBE_PROXY_REGISTRY_REDIS_HOST
value: {{ $redisRegistryHost | quote }}
value: {{ $redisHost | quote }}
- name: CUBE_PROXY_REGISTRY_REDIS_PORT
value: {{ $redisPort | quote }}
- name: CUBE_PROXY_REGISTRY_REDIS_PASSWORD
Expand Down
5 changes: 4 additions & 1 deletion deploy/kubernetes/chart/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,7 @@ redis:
# Empty host installs and uses the chart-managed cube-redis service.
# Non-empty host uses the configured third-party Redis service and skips cube-redis.
# Sentinel mode (non-empty masterName) skips cube-redis and must leave host empty.
# CubeProxy resolves this via nginx (no search domains): use an FQDN or IP literal.
host: ""
port: 6379
# Redis Sentinel (external). Non-empty masterName enables Sentinel for
Expand Down Expand Up @@ -713,6 +714,7 @@ cubeProxy:
containerPort: 9090
redis:
# Empty host uses redis.host or chart-managed cube-redis.
# Overrides must be an FQDN or IP literal (nginx resolver has no search domains).
host: ""
port: ""
db: 0
Expand Down Expand Up @@ -776,7 +778,8 @@ cubeProxy:
# configured by DNS name (the default Kubernetes Service DNS name).
# Empty addresses auto-discovers nameservers from the Pod /etc/resolv.conf.
addresses: []
valid: 30s
# Keep well below lifecycleManager.heartbeatTTL (15s) so heartbeats recover after redis IP changes.
valid: 5s
timeout: 5s
ipv6: false
probes:
Expand Down
Loading