From 6b1272cd469a2450d71f1a788be0cd0fee3b9a35 Mon Sep 17 00:00:00 2001 From: jinlong Date: Wed, 19 Aug 2026 16:32:16 +0800 Subject: [PATCH] fix(k8s): make helm test pods schedulable and probe CubeProxy healthz Test pods that only talk to Services/the API now share cube.testPlacement (both plane taint tolerations, no nodeSelector) so they stay schedulable on control-only, compute-only, and mixed topologies. node-runtime-test keeps computePlacement and pins runAsUser 0 for hostPath sockets. proxy-control-test probes /admin/healthz; dns-test uses getent ahostsv4. Signed-off-by: jinlong --- deploy/kubernetes/chart/README.md | 18 +- .../chart/scripts/test-helm-test-guards.sh | 288 ++++++++++++++++++ .../kubernetes/chart/templates/_helpers.tpl | 9 + .../chart/templates/tests/node-health.yaml | 110 ++++--- 4 files changed, 383 insertions(+), 42 deletions(-) create mode 100755 deploy/kubernetes/chart/scripts/test-helm-test-guards.sh diff --git a/deploy/kubernetes/chart/README.md b/deploy/kubernetes/chart/README.md index d15cb775e..c5045ee99 100644 --- a/deploy/kubernetes/chart/README.md +++ b/deploy/kubernetes/chart/README.md @@ -455,7 +455,7 @@ Without an Ingress / cloud LB, set `cubeProxy.service.type` / `controlPlane.api. When the sandbox owner is on a compute node, CubeProxy still uses Redis routing metadata to connect to the owner `HostIP:hostPort`. The chart patches the image's default nginx listeners to the configured `cubeProxy.ports.*.containerPort` values (default `80` / `443`). -CubeProxy admin is reachable in-cluster at each Pod IP:`adminPort` (default `8082`) for cube-lifecycle-manager discovery; probes use the admin token header. +CubeProxy admin is at Pod IP:`adminPort` (default `8082`) for CLM; helm test uses the Service admin port. Probes send the admin token header. CubeProxy reads sandbox routing metadata from Redis in nginx Lua. Because nginx does not automatically inherit Kubernetes DNS resolution for Lua cosocket @@ -479,8 +479,8 @@ cubeProxy: ## Cluster DNS for sandbox domain When CubeProxy is enabled, the chart patches **cluster CoreDNS** so -`cubeProxy.domain` / `*.domain` rewrite to the CubeProxy ClusterIP Service -(Pod IP). Users only set the domain: +`cubeProxy.domain` / `*.domain` rewrite to the CubeProxy Service FQDN +(ClusterIP). Users only set the domain: ```yaml cubeProxy: @@ -572,6 +572,18 @@ kubectl exec -n cube-system deploy/cube-cubemastercli -- \ helm test cube -n cube-system --timeout 20m ``` +`helm test` pods except `node-runtime-test` use `cube.testPlacement` (both +plane taints, no nodeSelector). `node-runtime-test` uses +`cube.computePlacement` and is skipped when `cubeNode.enabled=false`. + +`proxy-control-test` GETs `/admin/healthz` on the proxy Service admin port with +`X-Cube-Admin-Token` (Secret `cube-admin-token`) and requires HTTP 200. +Dataplane `/` returns 400. + +Health / proxy / node-image use `curl -4`; dns-test uses `getent ahostsv4`. +Override `helmTest.image` with curl+sh+awk+getent. `helmTest.dnsImage` is +busybox for node-runtime-test only. + ## Upgrade policy `cube-node` is a native `apps/v1` DaemonSet. Bumping Big Pod runtime images diff --git a/deploy/kubernetes/chart/scripts/test-helm-test-guards.sh b/deploy/kubernetes/chart/scripts/test-helm-test-guards.sh new file mode 100755 index 000000000..f1c8e0d48 --- /dev/null +++ b/deploy/kubernetes/chart/scripts/test-helm-test-guards.sh @@ -0,0 +1,288 @@ +#!/bin/sh +# Guard: helm test pods schedule across topologies, proxy probes admin +# healthz, DNS uses getent ahostsv4, and node-runtime-test runs as root +# against read-only hostPaths. +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)" +CHART_DIR="$(dirname "$SCRIPT_DIR")" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +COMMON_SETS="--set-string mysql.password=test --set-string mysql.rootPassword=test --set-string redis.password=test" + +render() { + output="$1" + shift + helm template helm-guard "$CHART_DIR" $COMMON_SETS "$@" > "$output" +} + +python_check() { + python3 - "$1" <<'PY' +import pathlib +import re +import sys + +text = pathlib.Path(sys.argv[1]).read_text() +docs = text.split("\n---\n") + +TEST_PLACEMENT_PODS = ( + "helm-guard-cube-health-test", + "helm-guard-cube-cubemastercli-test", + "helm-guard-cube-mysql-test", + "helm-guard-cube-redis-test", + "helm-guard-cube-proxy-control-test", + "helm-guard-cube-dns-test", + "helm-guard-cube-node-image-test", +) + +ALL_TEST_PODS = TEST_PLACEMENT_PODS + ("helm-guard-cube-node-runtime-test",) + + +def pod_doc(name, required=True): + for d in docs: + if re.search(r"^kind: Pod$", d, re.M) and re.search( + rf"^ name: {re.escape(name)}$", d, re.M + ): + return d + if required: + raise SystemExit(f"pod {name} not found") + return None + + +def spec_top_level_key(doc, key): + return bool(re.search(rf"(?m)^ {re.escape(key)}:", doc)) + + +def spec_toleration_keys(doc): + m = re.search(r"(?ms)^ tolerations:\n((?: .+\n)*)", doc) + if not m: + return set() + keys = set() + for line in m.group(1).splitlines(): + mm = re.match(r"^\s+-?\s*key:\s*[\"']?([^\"'\s]+)[\"']?\s*$", line) + if mm: + keys.add(mm.group(1)) + return keys + + +def has_node_selector_label(doc, label): + return bool( + re.search(rf"(?m)^ nodeSelector:\n(?: .+\n)* {re.escape(label)}:", doc) + ) + + +def uncommented_has(doc, needle): + return any( + needle in line and not line.lstrip().startswith("#") + for line in doc.splitlines() + ) + + +def check_test_placement(name): + d = pod_doc(name) + if spec_top_level_key(d, "nodeSelector"): + raise SystemExit(f"{name}: testPlacement must not set nodeSelector") + if spec_top_level_key(d, "affinity"): + raise SystemExit(f"{name}: testPlacement must not set affinity") + keys = spec_toleration_keys(d) + for want in ("cube.tencent.com/control", "cube.tencent.com/compute"): + if want not in keys: + raise SystemExit(f"{name}: missing toleration key {want} (have {sorted(keys)})") + print(f"OK {name} (testPlacement)") + + +def check_compute_placement(name, require_control_taint=False): + d = pod_doc(name) + if not has_node_selector_label(d, "cube.tencent.com/cube-node"): + raise SystemExit(f"{name}: expected computePlacement nodeSelector") + if has_node_selector_label(d, "cube.tencent.com/cube-control"): + raise SystemExit(f"{name}: computePlacement must not pin cube-control") + keys = spec_toleration_keys(d) + if "cube.tencent.com/compute" not in keys: + raise SystemExit( + f"{name}: missing compute taint toleration (have {sorted(keys)})" + ) + if require_control_taint and "cube.tencent.com/control" not in keys: + raise SystemExit( + f"{name}: single-node computePlacement missing control taint " + f"(have {sorted(keys)})" + ) + print(f"OK {name} (computePlacement)") + + +def check_image(name, container, needle): + d = pod_doc(name) + m = re.search(rf"- name: {container}\s*\n\s+image: (\S+)", d) + if not m: + raise SystemExit(f"{name}: container {container} missing") + if needle not in m.group(1): + raise SystemExit( + f"{name}: container {container} image {m.group(1)} " + f"does not contain {needle}" + ) + + +def check_readonly_mounts(name): + d = pod_doc(name) + m = re.search(r"(?ms)^ volumeMounts:\n((?: .+\n)*)", d) + if not m: + raise SystemExit(f"{name}: no volumeMounts") + entries = [e for e in re.split(r"(?m)^ - name:", m.group(1)) if e.strip()] + if not entries: + raise SystemExit(f"{name}: empty volumeMounts") + for entry in entries: + if "readOnly: true" not in entry: + raise SystemExit(f"{name}: volumeMount missing readOnly: true:\n{entry}") + + +def assert_absent(name): + if pod_doc(name, required=False) is not None: + raise SystemExit(f"{name} must be omitted") + + +def assert_no_test_pods(): + for name in ALL_TEST_PODS: + if pod_doc(name, required=False) is not None: + raise SystemExit(f"{name} must be omitted when helmTest.enabled=false") + + +def check_concat_custom_taints(name): + d = pod_doc(name) + if spec_top_level_key(d, "nodeSelector"): + raise SystemExit(f"{name}: testPlacement must not set nodeSelector") + if spec_top_level_key(d, "affinity"): + raise SystemExit(f"{name}: testPlacement must not set affinity") + keys = spec_toleration_keys(d) + for want in ("custom/control", "custom/compute"): + if want not in keys: + raise SystemExit( + f"{name}: concat missing {want} (have {sorted(keys)})" + ) + print(f"OK {name} (custom taint concat)") + + +mode = pathlib.Path(sys.argv[1]).name + +if mode == "default.yaml": + for name in TEST_PLACEMENT_PODS: + check_test_placement(name) + check_compute_placement("helm-guard-cube-node-runtime-test") + + check_image("helm-guard-cube-health-test", "curl", "curlimages/curl") + check_image("helm-guard-cube-dns-test", "dns", "curlimages/curl") + check_image("helm-guard-cube-node-runtime-test", "node-runtime", "busybox") + check_image("helm-guard-cube-proxy-control-test", "proxy", "curlimages/curl") + + d = pod_doc("helm-guard-cube-proxy-control-test") + if "/admin/healthz" not in d: + raise SystemExit("proxy-control-test does not probe admin healthz") + if "CUBE_PROXY_ADMIN_TOKEN" not in d or "secretKeyRef" not in d or "cube-admin-token" not in d: + raise SystemExit("proxy-control-test does not source the admin token from the release Secret") + if not uncommented_has(d, "X-Cube-Admin-Token"): + raise SystemExit("proxy-control-test missing X-Cube-Admin-Token header") + if not uncommented_has(d, "curl -4"): + raise SystemExit("proxy-control-test must pass curl -4") + if not uncommented_has(d, "--retry-connrefused"): + raise SystemExit("proxy-control-test must pass --retry-connrefused") + if uncommented_has(d, "--retry-all-errors"): + raise SystemExit("proxy-control-test must not pass --retry-all-errors (retries HTTP 4xx)") + if 'test "$status" = "200"' not in d: + raise SystemExit("proxy-control-test must assert HTTP 200") + + d = pod_doc("helm-guard-cube-dns-test") + if "getent ahostsv4" not in d: + raise SystemExit("dns-test does not use getent ahostsv4") + if uncommented_has(d, "nslookup"): + raise SystemExit("dns-test still uses nslookup") + if not uncommented_has(d, "tries=20"): + raise SystemExit("dns-test must retry with tries=20") + if "printf 'could not resolve %s" not in d: + raise SystemExit("dns-test diagnostics must printf quoted domain names") + + d = pod_doc("helm-guard-cube-health-test") + if "command curl -4" not in d: + raise SystemExit("health-test must wrap curl with -4") + + d = pod_doc("helm-guard-cube-node-image-test") + if "command curl -4" not in d: + raise SystemExit("node-image-test must wrap curl with -4") + + check_readonly_mounts("helm-guard-cube-node-runtime-test") + d = pod_doc("helm-guard-cube-node-runtime-test") + if "runAsUser: 0" not in d or "runAsGroup: 0" not in d: + raise SystemExit("node-runtime-test must pin runAsUser/runAsGroup 0") + if "allowPrivilegeEscalation: false" not in d or 'drop: ["ALL"]' not in d: + raise SystemExit("node-runtime-test securityContext missing privilege hardening") + + print("helm test default placement/image/probe guard passed") + +elif mode == "control-only.yaml": + for name in ( + "helm-guard-cube-health-test", + "helm-guard-cube-cubemastercli-test", + "helm-guard-cube-mysql-test", + "helm-guard-cube-redis-test", + "helm-guard-cube-proxy-control-test", + "helm-guard-cube-dns-test", + ): + check_test_placement(name) + assert_absent("helm-guard-cube-node-runtime-test") + assert_absent("helm-guard-cube-node-image-test") + print("helm test control-only placement guard passed") + +elif mode == "compute-only.yaml": + check_test_placement("helm-guard-cube-health-test") + check_test_placement("helm-guard-cube-cubemastercli-test") + check_test_placement("helm-guard-cube-node-image-test") + check_compute_placement("helm-guard-cube-node-runtime-test") + assert_absent("helm-guard-cube-mysql-test") + assert_absent("helm-guard-cube-redis-test") + assert_absent("helm-guard-cube-proxy-control-test") + assert_absent("helm-guard-cube-dns-test") + print("helm test compute-only placement guard passed") + +elif mode == "single-node.yaml": + check_test_placement("helm-guard-cube-health-test") + check_compute_placement( + "helm-guard-cube-node-runtime-test", require_control_taint=True + ) + print("helm test single-node placement guard passed") + +elif mode == "custom-taint.yaml": + check_concat_custom_taints("helm-guard-cube-health-test") + print("helm test custom taint concat guard passed") + +elif mode == "disabled.yaml": + assert_no_test_pods() + print("helm test disabled omit guard passed") + +else: + raise SystemExit(f"unknown render {mode}") +PY +} + +render "$TMP_DIR/default.yaml" +python_check "$TMP_DIR/default.yaml" + +render "$TMP_DIR/control-only.yaml" --set cubeNode.enabled=false +python_check "$TMP_DIR/control-only.yaml" + +render "$TMP_DIR/compute-only.yaml" \ + --set controlPlane.enabled=false \ + --set externalControlPlane.enabled=true \ + --set-string externalControlPlane.masterEndpoint=http://10.0.0.1:8080 \ + --set mysql.enabled=false \ + --set redis.enabled=false +python_check "$TMP_DIR/compute-only.yaml" + +render "$TMP_DIR/single-node.yaml" -f "$CHART_DIR/values-single-node.yaml" +python_check "$TMP_DIR/single-node.yaml" + +render "$TMP_DIR/custom-taint.yaml" \ + --set-json 'placement.controlPlane.tolerations=[{"key":"custom/control","operator":"Exists","effect":"NoSchedule"}]' \ + --set-json 'placement.compute.tolerations=[{"key":"custom/compute","operator":"Exists","effect":"NoSchedule"}]' +python_check "$TMP_DIR/custom-taint.yaml" + +render "$TMP_DIR/disabled.yaml" --set helmTest.enabled=false +python_check "$TMP_DIR/disabled.yaml" diff --git a/deploy/kubernetes/chart/templates/_helpers.tpl b/deploy/kubernetes/chart/templates/_helpers.tpl index fe552ad0b..0a34149f1 100644 --- a/deploy/kubernetes/chart/templates/_helpers.tpl +++ b/deploy/kubernetes/chart/templates/_helpers.tpl @@ -98,6 +98,15 @@ tolerations: {{- end }} {{- end -}} +{{- /* Helm tests except node-runtime-test: both plane taints, no nodeSelector. */ -}} +{{- define "cube.testPlacement" -}} +{{- $tolerations := concat (.Values.placement.controlPlane.tolerations | default list) (.Values.placement.compute.tolerations | default list) -}} +{{- with $tolerations }} +tolerations: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end -}} + {{- define "cube.pvmPlacement" -}} {{- $root := . -}} {{- $gateEnabled := eq (include "cube.startupGateEnabled" .) "true" -}} diff --git a/deploy/kubernetes/chart/templates/tests/node-health.yaml b/deploy/kubernetes/chart/templates/tests/node-health.yaml index 8dae19086..cf644e4b1 100644 --- a/deploy/kubernetes/chart/templates/tests/node-health.yaml +++ b/deploy/kubernetes/chart/templates/tests/node-health.yaml @@ -20,6 +20,7 @@ spec: imagePullSecrets: {{- toYaml . | nindent 4 }} {{- end }} + {{- include "cube.testPlacement" . | nindent 2 }} serviceAccountName: {{ include "cube.fullname" . }}-test containers: - name: curl @@ -33,6 +34,8 @@ spec: ca_path=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt ns={{ .Release.Namespace | quote }} kube_api="https://kubernetes.default.svc" + # IPv4 only (AAAA probes stall); no global --retry. + curl() { command curl -4 "$@"; } kget() { curl --connect-timeout 5 --max-time 20 --cacert "${ca_path}" \ -H "Authorization: Bearer $(cat "${token_path}")" \ @@ -139,6 +142,7 @@ spec: imagePullSecrets: {{- toYaml . | nindent 4 }} {{- end }} + {{- include "cube.testPlacement" . | nindent 2 }} containers: - name: cubemastercli image: {{ include "cube.cubeImage" (dict "image" .Values.images.cubemastercli "context" $) | quote }} @@ -178,6 +182,7 @@ spec: imagePullSecrets: {{- toYaml . | nindent 4 }} {{- end }} + {{- include "cube.testPlacement" . | nindent 2 }} containers: - name: mysql image: {{ include "cube.image" .Values.mysql.image | quote }} @@ -214,6 +219,7 @@ spec: imagePullSecrets: {{- toYaml . | nindent 4 }} {{- end }} + {{- include "cube.testPlacement" . | nindent 2 }} containers: - name: redis image: {{ include "cube.image" .Values.redis.image | quote }} @@ -250,23 +256,30 @@ spec: imagePullSecrets: {{- toYaml . | nindent 4 }} {{- end }} - {{- include "cube.controlPlanePlacement" . | nindent 2 }} + {{- include "cube.testPlacement" . | nindent 2 }} containers: - name: proxy image: {{ include "cube.image" .Values.helmTest.image | quote }} imagePullPolicy: {{ .Values.helmTest.image.pullPolicy }} env: {{- include "cube.timezoneEnv" . | nindent 8 }} + - name: CUBE_PROXY_ADMIN_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "cube.secretName" . }} + key: cube-admin-token command: - sh - -ec - | - echo '[health-test] check CubeProxy via ClusterIP Service' - curl --connect-timeout 5 --max-time 15 -sS -o /dev/null \ - "http://{{ include "cube.proxyServiceFQDN" . }}:{{ .Values.cubeProxy.ports.http.containerPort }}/" \ - || curl --connect-timeout 5 --max-time 15 -k -sS -o /dev/null \ - "https://{{ include "cube.proxyServiceFQDN" . }}:{{ .Values.cubeProxy.ports.https.containerPort }}/" \ - || { echo "cube-proxy Service unreachable"; exit 1; } + echo '[health-test] check CubeProxy admin healthz' + # "/" is 400. Retry refused/timeout/5xx; never --retry-all-errors. + status="$(curl -4 --retry 3 --retry-connrefused --retry-delay 1 --connect-timeout 5 --max-time 15 -sS \ + -H "X-Cube-Admin-Token: ${CUBE_PROXY_ADMIN_TOKEN}" \ + -o /dev/null -w "%{http_code}" \ + "http://{{ include "cube.proxyServiceFQDN" . }}:{{ .Values.cubeProxy.adminPort }}/admin/healthz")" \ + || { echo "cube-proxy admin healthz unreachable"; exit 1; } + test "$status" = "200" || { echo "cube-proxy admin healthz status=$status"; exit 1; } {{- end }} {{ if eq (include "cube.configureClusterDNS" .) "true" }} --- @@ -287,37 +300,48 @@ spec: imagePullSecrets: {{- toYaml . | nindent 4 }} {{- end }} - {{- include "cube.controlPlanePlacement" . | nindent 2 }} + {{- include "cube.testPlacement" . | nindent 2 }} containers: - name: dns - image: {{ include "cube.image" .Values.helmTest.dnsImage | quote }} - imagePullPolicy: {{ .Values.helmTest.dnsImage.pullPolicy }} + image: {{ include "cube.image" .Values.helmTest.image | quote }} + imagePullPolicy: {{ .Values.helmTest.image.pullPolicy }} command: - sh - -ec - | echo '[health-test] check cluster DNS for sandbox domain' target={{ include "cube.proxyServiceFQDN" . | quote }} - # busybox nslookup prints the DNS server Address first; take the last A record. - a_record() { nslookup "$1" | sed -n 's/^Address: \([0-9][0-9.]*\)$/\1/p' | tail -n 1; } - domain_ip="$(a_record {{ $domain | quote }})" - target_ip="$(a_record "$target")" - test -n "$domain_ip" - test -n "$target_ip" - test "$domain_ip" = "$target_ip" - test -n "$(a_record {{ printf "wildcard-check.%s" $domain | quote }})" + domain={{ $domain | quote }} + wildcard={{ printf "wildcard-check.%s" $domain | quote }} + # IPv4 A records; busybox nslookup times out on rewrite zones. + tries=20 + a_record() { + n=0 + while [ "$n" -lt "$tries" ]; do + ip="$(getent ahostsv4 "$1" 2>/dev/null | awk 'NR==1 {print $1}')" + if [ -n "$ip" ]; then + printf '%s\n' "$ip" + return 0 + fi + n=$((n + 1)) + sleep 1 + done + return 1 + } + domain_ip="$(a_record "$domain")" \ + || { printf 'could not resolve %s (A)\n' "$domain"; exit 1; } + target_ip="$(a_record "$target")" \ + || { printf 'could not resolve %s (A)\n' "$target"; exit 1; } + test "$domain_ip" = "$target_ip" \ + || { printf 'DNS mismatch: %s -> %s, %s -> %s\n' "$domain" "$domain_ip" "$target" "$target_ip"; exit 1; } + wildcard_ip="$(a_record "$wildcard")" \ + || { printf 'could not resolve %s (A)\n' "$wildcard"; exit 1; } + test "$domain_ip" = "$wildcard_ip" \ + || { printf 'DNS mismatch: %s -> %s, %s -> %s\n' "$wildcard" "$wildcard_ip" "$domain" "$domain_ip"; exit 1; } {{- end }} {{ if .Values.cubeNode.enabled }} --- -{{- /* - node-image-test used to pull the full multi-GB cube-node image just to run - `test -f` on baked-in files. That inflated helm test bandwidth and startup - time on every run. Instead reuse the already-running cube-node DaemonSet - Pod: a lightweight curl-based test pod calls the Kubernetes exec subresource - through the test ServiceAccount and runs the same assertions inside the - live cube-node container. `helm.test.nodeImage.enabled=false` disables the - assertion when a minimal cluster does not permit exec. -*/ -}} +{{- /* Ready cube-node Pod only; no exec. helmTest.nodeImage.enabled=false skips. */ -}} {{- $nodeImageEnabled := true -}} {{- if hasKey .Values.helmTest "nodeImage" -}} {{- $nodeImageEnabled = default true .Values.helmTest.nodeImage.enabled -}} @@ -339,6 +363,7 @@ spec: imagePullSecrets: {{- toYaml . | nindent 4 }} {{- end }} + {{- include "cube.testPlacement" . | nindent 2 }} serviceAccountName: {{ include "cube.fullname" . }}-test containers: - name: node-image-assets @@ -349,26 +374,23 @@ spec: - -ec - | set -eu - TOOLBOX_ROOT="/usr/local/services/cubetoolbox" ns={{ .Release.Namespace | quote }} token_path=/var/run/secrets/kubernetes.io/serviceaccount/token ca_path=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt kube_api="https://kubernetes.default.svc" - pod="$(curl --connect-timeout 5 --max-time 15 --cacert "${ca_path}" \ - -H "Authorization: Bearer $(cat "${token_path}")" -fsS \ - "${kube_api}/api/v1/namespaces/${ns}/pods?labelSelector=app.kubernetes.io%2Fcomponent%3Dcube-node&limit=1" \ + curl() { command curl -4 "$@"; } + kget() { + curl --connect-timeout 5 --max-time 15 --cacert "${ca_path}" \ + -H "Authorization: Bearer $(cat "${token_path}")" -fsS \ + "${kube_api}$1" + } + pod="$(kget "/api/v1/namespaces/${ns}/pods?labelSelector=app.kubernetes.io%2Fcomponent%3Dcube-node&limit=1" \ | grep -oE '"name"[[:space:]]*:[[:space:]]*"[^"]+"' \ | head -1 | awk -F'"' '{print $4}')" test -n "${pod}" || { echo "no cube-node pod found"; exit 1; } echo "[health-test] node-image assets on pod ${pod}" - # Fall through: rely on cube-node-init preflight and cube-node - # readiness to prove these assets exist inside the running Pod. - # Explicit re-check via `kubectl exec` requires kubectl in the test - # image; skipping keeps the test image tiny. - curl --connect-timeout 5 --max-time 15 --cacert "${ca_path}" \ - -H "Authorization: Bearer $(cat "${token_path}")" -fsS \ - "${kube_api}/api/v1/namespaces/${ns}/pods/${pod}" \ - | grep -oE '"ready"[[:space:]]*:[[:space:]]*true' | grep -q true + kget "/api/v1/namespaces/${ns}/pods/${pod}" \ + | grep -oE '"ready"[[:space:]]*:[[:space:]]*true' | grep -q true {{- end }} --- apiVersion: v1 @@ -392,6 +414,14 @@ spec: - name: node-runtime image: {{ include "cube.image" .Values.helmTest.dnsImage | quote }} imagePullPolicy: {{ .Values.helmTest.dnsImage.pullPolicy }} + # hostPath sockets are root-owned; pin uid 0. + securityContext: + runAsUser: 0 + runAsGroup: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] command: - sh - -ec @@ -406,8 +436,10 @@ spec: readOnly: true - name: data-cubelet mountPath: {{ .Values.hostPaths.dataCubelet }} + readOnly: true - name: tmp-cube mountPath: {{ .Values.hostPaths.tmpCube }} + readOnly: true volumes: - name: dev hostPath: