fix(deploy): publish dependency images consistently

.github/workflows/ci.yml

@@ -14,6 +14,55 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  etcd-image-smoke:
+    name: Official etcd image smoke test
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Render etcd StatefulSet
+        run: |
+          helm template akernel deploy/akernel \
+            --show-only charts/core/templates/etcd/etcd.yaml \
+            > /tmp/etcd-statefulset.yaml
+          grep -F 'image: "busybox:1.37.0-musl"' /tmp/etcd-statefulset.yaml
+          grep -F 'image: "gcr.io/etcd-development/etcd:v3.6.8"' /tmp/etcd-statefulset.yaml
+          grep -F -- '- /usr/local/bin/etcd' /tmp/etcd-statefulset.yaml
+          grep -F -- '- --listen-client-urls=http://$(MY_POD_IP):2379' /tmp/etcd-statefulset.yaml
+          test "$(grep -cF -- '- /bin/sh' /tmp/etcd-statefulset.yaml)" = 1
+
+      - name: Verify official image runtime contract
+        run: |
+          volume="akernel-etcd-smoke-${GITHUB_RUN_ID}"
+          trap 'docker volume rm -f "${volume}" >/dev/null 2>&1 || true' EXIT
+          docker volume create "${volume}" >/dev/null
+          docker run --rm --user 0:0 \
+            -v "${volume}:/etcd" busybox:1.37.0-musl sh -ec '
+              mkdir -p /etcd
+              chown -R 1001:1001 /etcd
+              chmod 0700 /etcd
+            '
+          set +e
+          timeout 5 docker run --rm --user 1001:1001 \
+            -v "${volume}:/etcd" \
+            gcr.io/etcd-development/etcd:v3.6.8 \
+            /usr/local/bin/etcd \
+            --name=etcd0 \
+            --data-dir=/etcd \
+            --listen-client-urls=http://127.0.0.1:2379 \
+            --advertise-client-urls=http://127.0.0.1:2379 \
+            --listen-peer-urls=http://127.0.0.1:2378 \
+            --initial-advertise-peer-urls=http://127.0.0.1:2378 \
+            --initial-cluster=etcd0=http://127.0.0.1:2378
+          status=$?
+          set -e
+          test "${status}" = 124
+          docker run --rm -v "${volume}:/etcd" busybox:1.37.0-musl \
+            test -s /etcd/member/snap/db
+
   sdk-unit-tests:
     name: Python SDK unit tests (${{ matrix.python-version }})
     runs-on: ubuntu-latest
@@ -68,6 +117,28 @@ jobs:
           unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY no_proxy NO_PROXY
           make deploy-script-check
 
+      - name: Reject stale image push profiles
+        run: |
+          profile=".akernel/ci-stale-image-profile"
+          mkdir -p "${profile}"
+          printf '%s\n' \
+            'IMAGE_REPOSITORY=example.invalid/akernel/all-in-one' \
+            'IMAGE_TAG=test' \
+            > "${profile}/config.env"
+
+          set +e
+          output="$(./deploy/scripts/push-image.sh \
+            --vendor aliyun \
+            --env ci-stale-image-profile 2>&1)"
+          status=$?
+          set -e
+
+          test "${status}" -ne 0
+          case "${output}" in
+            *"missing dependency image settings"*) ;;
+            *) printf '%s\n' "${output}" >&2; exit 1 ;;
+          esac
+
   standalone-e2e:
     name: Standalone E2E
     runs-on: ubuntu-22.04

AGENTS.md

@@ -111,8 +111,16 @@ make build RUNTIME_PROFILE=python
 For a build that will be pushed and deployed, set `IMAGE_REPOSITORY` and
 `IMAGE_TAG` when creating the deployment profile. A one-off `IMAGE_TAG`
 override on `make build` does not update the profile consumed by `make push`.
-The build creates only the selected image reference; it does not add a second
-`akernel-all-in-one` alias. `make push` pushes that selected reference directly.
+The build creates only the selected all-in-one image reference; it does not add
+a second `akernel-all-in-one` alias. `make push` also mirrors the official etcd
+and BusyBox images to the repositories recorded in the deployment profile. It
+does not mirror Traefik itself, monitoring, or Dragonfly.
+
+The bundled Helm chart runs the official etcd image directly as UID 1001. A
+separate BusyBox init container prepares `/etcd` on the mounted volume because
+the minimal official etcd image intentionally has no shell. Keep the etcd main
+container free of shell commands so the official image remains usable without
+an AKernel-specific rebuild.
 
 The build helper performs two Docker builds. `builder/runtime.Dockerfile`
 creates `yr-runtime-rootfs.img`; the default `rrt` profile contains the

Makefile

@@ -52,7 +52,7 @@ help:
 	@echo "  make build AKERNEL_ENABLE_KATA=false Exclude the optional Kata payload"
 	@echo "  make build GVISOR_RELEASE=<tag>     Override the pinned official gVisor tag"
 	@echo "  make versions                       Show locally selected component versions"
-	@echo "  make push                          Push the configured all-in-one image"
+	@echo "  make push                          Push all-in-one and mirror etcd/BusyBox"
 	@echo "  make plan                          Terraform plan"
 	@echo "  make deploy                        Terraform apply"
 	@echo "  make token TTL=24h                 Generate a local JWT token"

deploy/README.md

@@ -177,6 +177,11 @@ fsync enabled and uses persistent storage by default. Production environments
 that require etcd high availability should point AKernel at an externally
 managed multi-member etcd cluster instead of increasing `etcd.replicas`.
 
+The chart runs `gcr.io/etcd-development/etcd:v3.6.8` directly as UID 1001. The
+official image intentionally has no shell, so a separate BusyBox init container
+prepares ownership and permissions on the mounted `/etcd` volume before the
+etcd process starts.
+
 The core chart defaults master, frontend, and node to the same all-in-one image:
 
 ```yaml
@@ -294,10 +299,25 @@ through its own LoadBalancer when `install_monitor=true`. Set
 `install_dragonfly=true` to install the pinned official Dragonfly chart and
 inject its seed-client proxy into the node runtime configuration.
 
-Only the AKernel all-in-one image is pushed to the registry selected by
-`make config`. etcd, Traefik, Grafana, Prometheus, Loki, Tempo, and BusyBox use
-their pinned official public images by default. Set the per-component image
-overrides when a private cluster requires mirrored third-party images.
+`make push` pushes the all-in-one image and mirrors the official etcd and
+BusyBox images to the image namespace selected by `make config`. For example,
+`registry.example.com/akernel/all-in-one:<tag>` produces sibling repositories
+`registry.example.com/akernel/etcd:v3.6.8` and
+`registry.example.com/akernel/busybox:1.37.0-musl`. Monitoring and Dragonfly
+images remain separate; set their registry overrides when private mirrors are
+required.
+
+> **Release note.** When `IMAGE_REPOSITORY` points at the public
+> `akerneldev/all-in-one` namespace (the default for users who consume AKernel's
+> published images instead of mirroring), `make config` writes
+> `akerneldev/etcd:v3.6.8` and `akerneldev/busybox:1.37.0-musl` into the profile.
+> These are mirrors of the pinned upstream images. A release that uses this
+> profile must run `make push` so the sibling repositories exist, otherwise
+> etcd, its permissions init container, or `/internal-stats` will fail with
+> `ImagePullBackOff`. `make push` pushes the already-built all-in-one first,
+> then prepares and pushes the dependency images, so a build-host failure to
+> reach `gcr.io`/Docker Hub for the upstream etcd/busybox sources no longer
+> blocks the primary image.
 
 ## Directory Layout
 

deploy/akernel/charts/core/templates/etcd/etcd.yaml

@@ -40,8 +40,8 @@ spec:
       {{- if .Values.etcd.volumePermissions.enabled }}
       initContainers:
         - name: init-etcd-permissions
-          image: "{{ .Values.etcd.image.repository }}:{{ .Values.etcd.image.tag }}"
-          imagePullPolicy: {{ .Values.etcd.image.pullPolicy }}
+          image: "{{ .Values.etcd.volumePermissions.image }}"
+          imagePullPolicy: {{ .Values.etcd.volumePermissions.pullPolicy }}
           command:
             - /bin/sh
             - -ec
@@ -60,21 +60,23 @@ spec:
           image: "{{ .Values.etcd.image.repository }}:{{ .Values.etcd.image.tag }}"
           imagePullPolicy: {{ .Values.etcd.image.pullPolicy }}
           command:
-            - /bin/sh
-            - -ec
-            - |
-              mkdir -p {{ .Values.etcd.dataDir }}
-              exec etcd --name=etcd0 \
-                --data-dir={{ .Values.etcd.dataDir }} \
-                --auto-compaction-mode=revision \
-                --auto-compaction-retention={{ .Values.etcd.args.autoCompactionRetention }} \
-                --quota-backend-bytes={{ .Values.etcd.args.quotaBackendBytes }} \
-                --listen-client-urls=http://${MY_POD_IP}:2379 \
-                --advertise-client-urls=http://${MY_POD_IP}:2379 \
-                --listen-peer-urls=http://${MY_POD_IP}:2378 \
-                --initial-advertise-peer-urls=http://${MY_POD_IP}:2378 \
-                --initial-cluster=etcd0=http://${MY_POD_IP}:2378 \
-                --initial-cluster-state=new
+            - /usr/local/bin/etcd
+          args:
+            - --name=etcd0
+            - --data-dir={{ .Values.etcd.dataDir }}
+            - --auto-compaction-mode=revision
+            - --auto-compaction-retention={{ .Values.etcd.args.autoCompactionRetention }}
+            - --quota-backend-bytes={{ .Values.etcd.args.quotaBackendBytes }}
+            - --listen-client-urls=http://$(MY_POD_IP):2379
+            - --advertise-client-urls=http://$(MY_POD_IP):2379
+            - --listen-peer-urls=http://$(MY_POD_IP):2378
+            - --initial-advertise-peer-urls=http://$(MY_POD_IP):2378
+            - --initial-cluster=etcd0=http://$(MY_POD_IP):2378
+            - --initial-cluster-state=new
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: {{ .Values.etcd.volumePermissions.dataUser }}
+            runAsGroup: {{ .Values.etcd.volumePermissions.dataUser }}
           env:
             - name: MY_POD_IP
               valueFrom:

deploy/akernel/charts/core/values.yaml

@@ -57,9 +57,11 @@ etcd:
     enabled: true
     runAsUser: 0
     dataUser: 1001
+    image: busybox:1.37.0-musl
+    pullPolicy: IfNotPresent
   image:
-    repository: public.ecr.aws/bitnami/etcd
-    tag: 3.6.8
+    repository: gcr.io/etcd-development/etcd
+    tag: v3.6.8
     pullPolicy: IfNotPresent
   resources:
     limits:

deploy/scripts/common.sh

@@ -17,6 +17,10 @@ info() {
   echo "==> $*"
 }
 
+warn() {
+  echo "WARNING: $*" >&2
+}
+
 require_cmd() {
   local missing=0
   for cmd in "$@"; do

deploy/scripts/configure.sh

@@ -213,6 +213,14 @@ case "${vendor}" in
     ;;
 esac
 set_or_prompt image_tag "All-in-one image tag" "${default_tag}" "${image_tag_override}"
+if [[ "${image_repository}" != */* ]]; then
+  die "all-in-one image repository must include a namespace: ${image_repository}"
+fi
+image_namespace="${image_repository%/*}"
+etcd_image_repository="${image_namespace}/etcd"
+etcd_image_tag="v3.6.8"
+traefik_internal_stats_image="${image_namespace}/busybox:1.37.0-musl"
+etcd_volume_permissions_image="${traefik_internal_stats_image}"
 set_or_prompt install_monitor "Install monitor chart (true/false)" "true" "${install_monitor_override}"
 set_or_prompt install_dragonfly "Install Dragonfly and dedicated node pools (true/false)" "false" "${install_dragonfly_override}"
 set_or_prompt grafana_public_access "Expose Grafana LoadBalancer (true/false)" "true" "${grafana_public_access_override}"
@@ -313,6 +321,9 @@ master_image_repository = "${image_repository}"
 master_image_tag        = "${image_tag}"
 node_image_repository   = "${image_repository}"
 node_image_tag          = "${image_tag}"
+etcd_image_repository   = "${etcd_image_repository}"
+etcd_image_tag          = "${etcd_image_tag}"
+etcd_volume_permissions_image = "${etcd_volume_permissions_image}"
 iam_litebus_data_key    = "${iam_seed}"
 
 frontend_enabled  = true
@@ -329,6 +340,7 @@ traefik_web_port              = 80
 traefik_tls_enabled           = false
 traefik_tls_create_secret     = false
 traefik_internal_stats_enabled = true
+traefik_internal_stats_image   = "${traefik_internal_stats_image}"
 
 install_prereqs = false
 
@@ -367,6 +379,9 @@ master_image_repository = "${image_repository}"
 master_image_tag        = "${image_tag}"
 node_image_repository   = "${image_repository}"
 node_image_tag          = "${image_tag}"
+etcd_image_repository   = "${etcd_image_repository}"
+etcd_image_tag          = "${etcd_image_tag}"
+etcd_volume_permissions_image = "${etcd_volume_permissions_image}"
 iam_litebus_data_key    = "${iam_seed}"
 
 frontend_enabled  = true
@@ -382,6 +397,7 @@ traefik_web_port                 = 80
 traefik_tls_enabled              = false
 traefik_tls_create_secret        = false
 traefik_internal_stats_enabled   = true
+traefik_internal_stats_image     = "${traefik_internal_stats_image}"
 
 install_prereqs = false
 
@@ -409,6 +425,9 @@ GRAFANA_PASSWORD_FILE=${grafana_password_file}
 KUBECONFIG_PATH=${kubeconfig_file}
 IMAGE_REPOSITORY=${image_repository}
 IMAGE_TAG=${image_tag}
+ETCD_IMAGE_REPOSITORY=${etcd_image_repository}
+ETCD_IMAGE_TAG=${etcd_image_tag}
+TRAEFIK_INTERNAL_STATS_IMAGE=${traefik_internal_stats_image}
 CORE_NAMESPACE=akernel
 MONITOR_NAMESPACE=akernel-monitor
 INSTALL_DRAGONFLY=${install_dragonfly}

deploy/scripts/push-image.sh

@@ -33,10 +33,56 @@ vendor_dir "${vendor}" >/dev/null
 load_env_config "${env_name}"
 require_cmd docker
 
-image="${IMAGE_REPOSITORY}:${IMAGE_TAG}"
+missing_profile_vars=()
+for var_name in ETCD_IMAGE_REPOSITORY ETCD_IMAGE_TAG TRAEFIK_INTERNAL_STATS_IMAGE; do
+  if [[ -z "${!var_name:-}" ]]; then
+    missing_profile_vars+=("${var_name}")
+  fi
+done
+if [[ "${#missing_profile_vars[@]}" -gt 0 ]]; then
+  die "deployment profile ${env_name} is missing dependency image settings (${missing_profile_vars[*]}); rerun make config ENV=${env_name} and review the generated Terraform plan"
+fi
+
+# Pinned dependency image versions. Keep these in sync with the chart defaults.
+readonly etcd_version="3.6.8"
+readonly busybox_tag="1.37.0-musl"
+
+all_in_one_image="${IMAGE_REPOSITORY}:${IMAGE_TAG}"
+etcd_source_image="gcr.io/etcd-development/etcd:v${etcd_version}"
+busybox_source_image="busybox:${busybox_tag}"
+etcd_image="${ETCD_IMAGE_REPOSITORY}:${ETCD_IMAGE_TAG}"
+busybox_image="${TRAEFIK_INTERNAL_STATS_IMAGE}"
+
+# Warn if an override disagrees with the chart default, so a stale pin does not
+# silently push a tag the bundled chart will not request.
+[[ "${ETCD_IMAGE_TAG}" == "v${etcd_version}" ]] || \
+  warn "ETCD_IMAGE_TAG=${ETCD_IMAGE_TAG} overrides the chart default v${etcd_version}; ensure the chart's etcd.image.tag matches"
+[[ "${TRAEFIK_INTERNAL_STATS_IMAGE}" == *":${busybox_tag}" ]] || \
+  warn "TRAEFIK_INTERNAL_STATS_IMAGE=${TRAEFIK_INTERNAL_STATS_IMAGE} does not use the chart default tag ${busybox_tag}; ensure the chart's traefik.internalStats.image matches"
 
-docker image inspect "${image}" >/dev/null 2>&1 || \
-  die "missing local image ${image}; run make build ENV=${env_name} first"
+docker image inspect "${all_in_one_image}" >/dev/null 2>&1 || \
+  die "missing local image ${all_in_one_image}; run make build ENV=${env_name} first"
 
-info "pushing ${image}"
-docker push "${image}"
+# Push the primary artifact first. It is already built locally, so pushing it
+# must not depend on pulling etcd/busybox from upstream registries that may be
+# unreachable on the build host (e.g. cross-border network restrictions).
+info "pushing ${all_in_one_image}"
+docker push "${all_in_one_image}"
+
+info "preparing deployment dependency images (etcd, BusyBox)"
+docker pull "${etcd_source_image}" || \
+  die "failed to pull ${etcd_source_image}"
+docker pull "${busybox_source_image}" || \
+  die "failed to pull ${busybox_source_image}"
+
+if [[ "${etcd_source_image}" != "${etcd_image}" ]]; then
+  docker tag "${etcd_source_image}" "${etcd_image}"
+fi
+if [[ "${busybox_source_image}" != "${busybox_image}" ]]; then
+  docker tag "${busybox_source_image}" "${busybox_image}"
+fi
+
+for image in "${etcd_image}" "${busybox_image}"; do
+  info "pushing ${image}"
+  docker push "${image}"
+done

deploy/terraform/aliyun/README.md

@@ -144,6 +144,18 @@ export AKERNEL_SERVER_ADDRESS=<traefik-load-balancer-ip>
 not required for the `websecure` router on port 443; Traefik serves its default
 certificate when the variable is `false`.
 
+The module uses the official `gcr.io/etcd-development/etcd:v3.6.8` image by
+default. A BusyBox init container prepares the etcd volume because the official
+image has no shell; it uses the same pinned BusyBox image as Traefik's
+`/internal-stats` sidecar. Override the component image variables when using a
+private mirror.
+
+The guided `make config` flow configures that private mirror: it derives sibling
+`etcd` and `busybox` repositories from `IMAGE_REPOSITORY`, and `make push`
+mirrors the official etcd and BusyBox images alongside the all-in-one image.
+It does not mirror the Traefik, monitoring, or Dragonfly images. Direct
+Terraform users retain the public defaults shown above.
+
 To use the legacy single-entrypoint mode, set
 `traefik_enable_web_entrypoint=false` and configure `traefik_tcp_port`. In that
 mode SDK clients must include the port explicitly:

deploy/terraform/aliyun/main.tf

@@ -195,11 +195,11 @@ locals {
     auths = { for host, cred in var.registry_auths : host => { auth = base64encode("${cred.username}:${cred.password}") } }
   }
 
-  etcd_image_repo              = length(var.etcd_image_repository) > 0 ? var.etcd_image_repository : "public.ecr.aws/bitnami/etcd"
+  etcd_image_repo              = length(var.etcd_image_repository) > 0 ? var.etcd_image_repository : "gcr.io/etcd-development/etcd"
   master_image_repo            = length(var.master_image_repository) > 0 ? var.master_image_repository : "${local.acr_registry}/all-in-one"
   node_image_repo              = length(var.node_image_repository) > 0 ? var.node_image_repository : "${local.acr_registry}/all-in-one"
   traefik_image_repo           = length(var.traefik_image_repository) > 0 ? var.traefik_image_repository : "traefik"
-  traefik_internal_stats_image = length(var.traefik_internal_stats_image) > 0 ? var.traefik_internal_stats_image : "${local.acr_registry}/busybox:1.37.0-musl"
+  traefik_internal_stats_image = length(var.traefik_internal_stats_image) > 0 ? var.traefik_internal_stats_image : "busybox:1.37.0-musl"
 
   core_values = templatefile("${path.module}/values-akernel.yaml.tmpl", {
     acr_registry                      = local.acr_registry
@@ -209,6 +209,7 @@ locals {
     acr_password                      = var.acr_password
     etcd_image_repository             = local.etcd_image_repo
     etcd_image_tag                    = var.etcd_image_tag
+    etcd_volume_permissions_image     = var.etcd_volume_permissions_image
     master_image_repository           = local.master_image_repo
     master_image_tag                  = var.master_image_tag
     node_image_repository             = local.node_image_repo