From f8b05df206ce0317e1f03336cff8000990858154 Mon Sep 17 00:00:00 2001 From: ChamberlainJI Date: Wed, 2 Sep 2026 21:37:06 +0800 Subject: [PATCH 1/3] feat(snapshot): wire S3-compatible storage Signed-off-by: ChamberlainJI --- builder/node.Dockerfile | 4 + ...ect-openyuanrong-s3-snapshot-capability.sh | 150 ++++++++++++++++++ builder/scripts/yr_node_bootstrap.sh | 14 +- builder/scripts/yr_pause_resume_args.sh | 104 ++++++++++++ builder/systemd_services/yuanrong.service | 4 +- .../charts/core/templates/node/daemonset.yaml | 61 +++++++ deploy/akernel/charts/core/values.yaml | 18 +++ deploy/akernel/values.yaml | 15 ++ deploy/standalone/start.sh | 66 +++++++- src/yuanrong | 2 +- 10 files changed, 429 insertions(+), 9 deletions(-) create mode 100755 builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh create mode 100755 builder/scripts/yr_pause_resume_args.sh diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 82171c0..65fde21 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -395,6 +395,10 @@ COPY ./builder/scripts/yr_node_bootstrap.sh ${YR_INSTALLATION_DIR}/yr_node_boots COPY ./builder/scripts/master_entrypoint.sh ${YR_INSTALLATION_DIR}/entrypoint.sh COPY ./builder/scripts/*.sh /root/ COPY ./builder/systemd_services/*.service /etc/systemd/system/ +RUN chmod 0755 \ + /root/yr_pause_resume_args.sh \ + /root/detect-openyuanrong-s3-snapshot-capability.sh && \ + /root/detect-openyuanrong-s3-snapshot-capability.sh ${YR_INSTALLATION_DIR} RUN curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ "${OTELCOL_CONTRIB_URL}" \ diff --git a/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh b/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh new file mode 100755 index 0000000..107f007 --- /dev/null +++ b/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +yr_root="${1:?usage: $0 YR_ROOT}" +marker="${yr_root}/.akernel-s3-snapshot-capable" +config="${yr_root}/deploy/process/config.sh" +install="${yr_root}/functionsystem/deploy/install.sh" +agent="${yr_root}/functionsystem/bin/function_agent" + +rm -f "${marker}" + +contains_all_text() { + local file=$1 + shift + local evidence + for evidence in "$@"; do + grep -Fq -- "${evidence}" "${file}" || return 1 + done +} + +contains_all_binary() { + local file=$1 + shift + local evidence + for evidence in "$@"; do + grep -aFq -- "${evidence}" "${file}" || return 1 + done +} + +extract_install_function_body() { + local file=$1 + local function_header=$2 + awk -v function_header="${function_header}" ' + $0 == function_header { + in_function = 1 + next + } + in_function && $0 == "}" { + found_close = 1 + exit + } + in_function { + print + } + END { + if (!in_function || !found_close) { + exit 1 + } + } + ' "${file}" +} + +normalize_install_lines() { + awk ' + { + line = $0 + sub(/^[[:space:]]*/, "", line) + if (line ~ /^#/) { + next + } + sub(/[[:space:]]*\\[[:space:]]*$/, "", line) + sub(/[[:space:]]*$/, "", line) + if (line != "") { + print line + } + } + ' +} + +has_exact_normalized_occurrences() { + local normalized_text=$1 + local expected_count=$2 + local evidence=$3 + local actual_count + actual_count=$(printf '%s\n' "${normalized_text}" | grep -Fxc -- "${evidence}" || true) + [[ "${actual_count}" -eq "${expected_count}" ]] +} + +[[ -f "${config}" && -f "${install}" && -x "${agent}" ]] || exit 0 + +contains_all_text "${config}" \ + '--snapshot_s3_provider) SNAPSHOT_S3_PROVIDER=$2 && shift 2 ;;' \ + '--snapshot_s3_endpoint) SNAPSHOT_S3_ENDPOINT=$2 && shift 2 ;;' \ + '--snapshot_s3_region) SNAPSHOT_S3_REGION=$2 && shift 2 ;;' \ + '--snapshot_s3_bucket) SNAPSHOT_S3_BUCKET=$2 && shift 2 ;;' \ + '--snapshot_s3_use_https) SNAPSHOT_S3_USE_HTTPS=$2 && shift 2 ;;' \ + '--snapshot_s3_path_style) SNAPSHOT_S3_PATH_STYLE=$2 && shift 2 ;;' \ + 'if [ "X${SNAPSHOT_STORAGE_MODE}" != "Xlocal_only" ]; then' \ + 'case "${SNAPSHOT_STORAGE_BACKEND}" in' \ + ' s3)' \ + 'case "${SNAPSHOT_S3_PROVIDER}" in' \ + 'generic|obs|oss) ;;' \ + 'log_error "snapshot_s3_provider must be generic, obs, or oss"' \ + 'if [ -z "${SNAPSHOT_S3_ENDPOINT}" ] || [ -z "${SNAPSHOT_S3_REGION}" ] || [ -z "${SNAPSHOT_S3_BUCKET}" ]; then' \ + 'log_error "snapshot_s3_endpoint, snapshot_s3_region, and snapshot_s3_bucket are required for S3"' \ + 'case "${SNAPSHOT_S3_USE_HTTPS}" in' \ + 'log_error "snapshot_s3_use_https must be true or false"' \ + 'case "${SNAPSHOT_S3_PATH_STYLE}" in' \ + 'log_error "snapshot_s3_path_style must be true or false"' \ + 'if [ "X${SNAPSHOT_S3_PROVIDER}" = "Xoss" ] && [ "X${SNAPSHOT_S3_PATH_STYLE}" = "Xtrue" ]; then' \ + 'log_error "snapshot S3 OSS provider requires virtual-host addressing"' \ + 'if [ -z "${SNAPSHOT_S3_ACCESS_KEY:-}" ] || [ -z "${SNAPSHOT_S3_SECRET_KEY:-}" ]; then' \ + 'log_error "SNAPSHOT_S3_ACCESS_KEY and SNAPSHOT_S3_SECRET_KEY are required for S3"' \ + 'log_error "distributed snapshot storage backend must be datasystem, obs, or s3"' \ + 'export SNAPSHOT_S3_PROVIDER SNAPSHOT_S3_ENDPOINT SNAPSHOT_S3_REGION SNAPSHOT_S3_BUCKET' \ + 'export SNAPSHOT_S3_USE_HTTPS SNAPSHOT_S3_PATH_STYLE' \ + 'export SNAPSHOT_S3_ACCESS_KEY SNAPSHOT_S3_SECRET_KEY SNAPSHOT_S3_SECURITY_TOKEN' || exit 0 + +normalized_install=$(normalize_install_lines <"${install}") +proxy_install_body=$(extract_install_function_body "${install}" \ + 'function install_function_proxy() {' | normalize_install_lines) || exit 0 +agent_install_body=$(extract_install_function_body "${install}" \ + 'function install_function_agent_and_runtime_manager_in_the_same_process() {' \ + | normalize_install_lines) || exit 0 + +for install_argument in \ + '--snapshot_s3_provider="${SNAPSHOT_S3_PROVIDER:-generic}"' \ + '--snapshot_s3_endpoint="${SNAPSHOT_S3_ENDPOINT:-}"' \ + '--snapshot_s3_region="${SNAPSHOT_S3_REGION:-}"' \ + '--snapshot_s3_bucket="${SNAPSHOT_S3_BUCKET:-}"' \ + '--snapshot_s3_use_https="${SNAPSHOT_S3_USE_HTTPS:-true}"' \ + '--snapshot_s3_path_style="${SNAPSHOT_S3_PATH_STYLE:-false}"'; do + has_exact_normalized_occurrences "${proxy_install_body}" 1 "${install_argument}" || exit 0 + has_exact_normalized_occurrences "${agent_install_body}" 1 "${install_argument}" || exit 0 + has_exact_normalized_occurrences "${normalized_install}" 2 "${install_argument}" || exit 0 +done + +if printf '%s\n' "${normalized_install}" \ + | grep -Eq -- '--snapshot_s3_(access_key|secret_key|security_token)(=|[[:space:]])'; then + exit 0 +fi + +contains_all_binary "${agent}" \ + snapshot_s3_provider \ + snapshot_s3_endpoint \ + snapshot_s3_region \ + snapshot_s3_bucket \ + snapshot_s3_use_https \ + snapshot_s3_path_style \ + SNAPSHOT_S3_ACCESS_KEY \ + SNAPSHOT_S3_SECRET_KEY \ + SNAPSHOT_S3_SECURITY_TOKEN \ + 'snapshot S3 object exceeds the 5 GiB limit' \ + 'published object snapshot failed postcondition verification' || exit 0 + +touch "${marker}" diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index 7c68326..31b5b74 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -5,6 +5,7 @@ # SPDX-License-Identifier: Apache-2.0 ulimit -n 32768 export YR_RUNTIME_BACKEND=sandboxd +source /root/yr_pause_resume_args.sh resolve_node_ip() { local default_device @@ -43,6 +44,11 @@ YR_NODE_IP="$(resolve_node_ip)" echo "Using ${YR_NODE_IP} as the YuanRong node address" CHECKPOINT_DIR="/home/akernel/checkpoints" mkdir -p "${CHECKPOINT_DIR}" +configure_snapshot_args \ + /home/yuanrong/.akernel-rrt-capable \ + "${CHECKPOINT_DIR}" \ + "${AKS_LOCAL_MODE:-false}" \ + /home/yuanrong/.akernel-s3-snapshot-capable || exit 1 # Select the legacy etcd registry or the FunctionMaster HTTP provider. if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then @@ -109,10 +115,9 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --iam_local_ip 127.0.0.1 \ --frontend_lease_bypass true \ --force_low_reliability_instance true \ - --snapshot_storage_mode local_only \ - --checkpoint_dir "${CHECKPOINT_DIR}" \ --enable_sandbox_router true \ - --enable_direct_routing false + --enable_direct_routing false \ + "${standalone_snapshot_args[@]}" else /usr/bin/yr start \ --ip_address "${YR_NODE_IP}" \ @@ -149,7 +154,6 @@ else --function_proxy_merge_process_enable true \ --enable_direct_routing false \ --force_low_reliability_instance true \ - --snapshot_storage_mode local_only \ - --checkpoint_dir "${CHECKPOINT_DIR}" \ + "${snapshot_args[@]}" \ --block true fi diff --git a/builder/scripts/yr_pause_resume_args.sh b/builder/scripts/yr_pause_resume_args.sh new file mode 100755 index 0000000..d4c581a --- /dev/null +++ b/builder/scripts/yr_pause_resume_args.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +configure_snapshot_args() { + local rrt_capability_file="${1:?RRT capability file is required}" + local checkpoint_dir="${2:?checkpoint directory is required}" + local standalone="${3:?standalone mode value is required}" + local s3_capability_file="${4:-/home/yuanrong/.akernel-s3-snapshot-capable}" + local backend="${AKERNEL_SNAPSHOT_STORAGE_BACKEND:-datasystem}" + + snapshot_args=() + standalone_snapshot_args=() + unset SNAPSHOT_S3_ACCESS_KEY SNAPSHOT_S3_SECRET_KEY SNAPSHOT_S3_SECURITY_TOKEN + if [ ! -f "${rrt_capability_file}" ] \ + && [ ! -f /home/yuanrong/yr-runtime-rootfs.img ]; then + echo "snapshot requires an image built with the RRT runtime" >&2 + return 1 + fi + mkdir -p "${checkpoint_dir}" + if [ ! -w "${checkpoint_dir}" ]; then + echo "checkpoint directory is not writable: ${checkpoint_dir}" >&2 + return 1 + fi + local configured_snapshot_args=( + --snapshot_storage_backend "${backend}" + --checkpoint_dir "${checkpoint_dir}" + ) + local configured_standalone_snapshot_args=() + local s3_access_key="" + local s3_secret_key="" + local s3_security_token="" + case "${backend}" in + datasystem) ;; + s3) + if [ ! -f "${s3_capability_file}" ]; then + echo "S3 snapshot storage requires an S3-capable openYuanRong core" >&2 + return 1 + fi + local provider="${AKERNEL_SNAPSHOT_S3_PROVIDER:-}" + local storage_mode="${AKERNEL_SNAPSHOT_STORAGE_MODE:-distributed_cache}" + local endpoint="${AKERNEL_SNAPSHOT_S3_ENDPOINT:-}" + local region="${AKERNEL_SNAPSHOT_S3_REGION:-}" + local bucket="${AKERNEL_SNAPSHOT_S3_BUCKET:-}" + s3_access_key="${AKERNEL_SNAPSHOT_S3_ACCESS_KEY:-}" + s3_secret_key="${AKERNEL_SNAPSHOT_S3_SECRET_KEY:-}" + s3_security_token="${AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN:-}" + local use_https="${AKERNEL_SNAPSHOT_S3_USE_HTTPS:-}" + local path_style="${AKERNEL_SNAPSHOT_S3_PATH_STYLE:-}" + case "${provider}" in generic|obs|oss) ;; *) + echo "AKERNEL_SNAPSHOT_S3_PROVIDER must be generic, obs, or oss" >&2 + return 1 + esac + case "${storage_mode}" in distributed_cache|distributed_only) ;; *) + echo "AKERNEL_SNAPSHOT_STORAGE_MODE must be distributed_cache or distributed_only" >&2 + return 1 + esac + if [ -z "${endpoint}" ] || [ -z "${region}" ] || [ -z "${bucket}" ] || \ + [ -z "${s3_access_key}" ] || [ -z "${s3_secret_key}" ]; then + echo "S3 snapshot storage requires endpoint, region, bucket, access key, and secret key" >&2 + return 1 + fi + case "${use_https}" in true|false) ;; *) + echo "AKERNEL_SNAPSHOT_S3_USE_HTTPS must be true or false" >&2 + return 1 + esac + case "${path_style}" in true|false) ;; *) + echo "AKERNEL_SNAPSHOT_S3_PATH_STYLE must be true or false" >&2 + return 1 + esac + if [ "${provider}" = "oss" ] && [ "${path_style}" = "true" ]; then + echo "OSS S3-compatible snapshot storage requires virtual-hosted addressing" >&2 + return 1 + fi + configured_snapshot_args+=( + --snapshot_storage_mode "${storage_mode}" + --snapshot_s3_provider "${provider}" + --snapshot_s3_endpoint "${endpoint}" + --snapshot_s3_region "${region}" + --snapshot_s3_bucket "${bucket}" + --snapshot_s3_use_https "${use_https}" + --snapshot_s3_path_style "${path_style}" + ) + ;; + *) + echo "AKERNEL_SNAPSHOT_STORAGE_BACKEND must be datasystem or s3" >&2 + return 1 + ;; + esac + configured_standalone_snapshot_args=("${configured_snapshot_args[@]}") + case "${standalone}" in + true) configured_standalone_snapshot_args+=(--data_system_enable true) ;; + false) ;; + *) echo "AKS_LOCAL_MODE must be true or false" >&2; return 1 ;; + esac + snapshot_args=("${configured_snapshot_args[@]}") + standalone_snapshot_args=("${configured_standalone_snapshot_args[@]}") + if [ "${backend}" = "s3" ]; then + export SNAPSHOT_S3_ACCESS_KEY="${s3_access_key}" + export SNAPSHOT_S3_SECRET_KEY="${s3_secret_key}" + export SNAPSHOT_S3_SECURITY_TOKEN="${s3_security_token}" + fi +} diff --git a/builder/systemd_services/yuanrong.service b/builder/systemd_services/yuanrong.service index 13a5486..eb16764 100644 --- a/builder/systemd_services/yuanrong.service +++ b/builder/systemd_services/yuanrong.service @@ -4,7 +4,9 @@ Description=yuanrong.service [Service] #Type=simple PIDFile=/run/yuanrong.pid -PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT +PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME NODE_NAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT YR_RRT_CONTROL_SOCKET_PATH +PassEnvironment=AKERNEL_SNAPSHOT_STORAGE_BACKEND AKERNEL_SNAPSHOT_STORAGE_MODE +PassEnvironment=AKERNEL_SNAPSHOT_S3_PROVIDER AKERNEL_SNAPSHOT_S3_ENDPOINT AKERNEL_SNAPSHOT_S3_REGION AKERNEL_SNAPSHOT_S3_BUCKET AKERNEL_SNAPSHOT_S3_ACCESS_KEY AKERNEL_SNAPSHOT_S3_SECRET_KEY AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN AKERNEL_SNAPSHOT_S3_USE_HTTPS AKERNEL_SNAPSHOT_S3_PATH_STYLE Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock" Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime" Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1" diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index ba6a974..62c7e9b 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -1,3 +1,30 @@ +{{- $snapshotBackend := .Values.snapshot.storage.backend -}} +{{- if not (has $snapshotBackend (list "datasystem" "s3")) -}} +{{- fail "snapshot.storage.backend must be datasystem or s3" -}} +{{- end -}} +{{- if eq $snapshotBackend "s3" -}} +{{- $snapshotMode := .Values.snapshot.storage.mode -}} +{{- if not (has $snapshotMode (list "distributed_cache" "distributed_only")) -}} +{{- fail "snapshot.storage.mode must be distributed_cache or distributed_only for S3" -}} +{{- end -}} +{{- $snapshotProvider := .Values.snapshot.storage.s3.provider -}} +{{- if not (has $snapshotProvider (list "generic" "obs" "oss")) -}} +{{- fail "snapshot.storage.s3.provider must be generic, obs, or oss" -}} +{{- end -}} +{{- if not (kindIs "bool" .Values.snapshot.storage.s3.useHttps) -}} +{{- fail "snapshot.storage.s3.useHttps must be a boolean" -}} +{{- end -}} +{{- if not (kindIs "bool" .Values.snapshot.storage.s3.pathStyle) -}} +{{- fail "snapshot.storage.s3.pathStyle must be a boolean" -}} +{{- end -}} +{{- if and (eq $snapshotProvider "oss") .Values.snapshot.storage.s3.pathStyle -}} +{{- fail "OSS S3-compatible storage requires virtual-hosted addressing" -}} +{{- end -}} +{{- $snapshotEndpoint := required "snapshot.storage.s3.endpoint is required" .Values.snapshot.storage.s3.endpoint -}} +{{- $snapshotRegion := required "snapshot.storage.s3.region is required" .Values.snapshot.storage.s3.region -}} +{{- $snapshotBucket := required "snapshot.storage.s3.bucket is required" .Values.snapshot.storage.s3.bucket -}} +{{- $snapshotSecret := required "snapshot.storage.s3.existingSecret is required" .Values.snapshot.storage.s3.existingSecret -}} +{{- end -}} {{- if .Values.kruise.enabled }} apiVersion: apps.kruise.io/v1alpha1 {{- else }} @@ -82,6 +109,40 @@ spec: value: "1" - name: YR_RRT_CONTROL_SOCKET_PATH value: "/run/akernel" + - name: AKERNEL_SNAPSHOT_STORAGE_BACKEND + value: {{ .Values.snapshot.storage.backend | quote }} + {{- if eq .Values.snapshot.storage.backend "s3" }} + - name: AKERNEL_SNAPSHOT_STORAGE_MODE + value: {{ .Values.snapshot.storage.mode | quote }} + - name: AKERNEL_SNAPSHOT_S3_PROVIDER + value: {{ .Values.snapshot.storage.s3.provider | quote }} + - name: AKERNEL_SNAPSHOT_S3_ENDPOINT + value: {{ .Values.snapshot.storage.s3.endpoint | quote }} + - name: AKERNEL_SNAPSHOT_S3_REGION + value: {{ .Values.snapshot.storage.s3.region | quote }} + - name: AKERNEL_SNAPSHOT_S3_BUCKET + value: {{ .Values.snapshot.storage.s3.bucket | quote }} + - name: AKERNEL_SNAPSHOT_S3_USE_HTTPS + value: {{ .Values.snapshot.storage.s3.useHttps | quote }} + - name: AKERNEL_SNAPSHOT_S3_PATH_STYLE + value: {{ .Values.snapshot.storage.s3.pathStyle | quote }} + - name: AKERNEL_SNAPSHOT_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.snapshot.storage.s3.existingSecret | quote }} + key: {{ required "snapshot.storage.s3.accessKeyKey is required" .Values.snapshot.storage.s3.accessKeyKey | quote }} + - name: AKERNEL_SNAPSHOT_S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.snapshot.storage.s3.existingSecret | quote }} + key: {{ required "snapshot.storage.s3.secretKeyKey is required" .Values.snapshot.storage.s3.secretKeyKey | quote }} + - name: AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.snapshot.storage.s3.existingSecret | quote }} + key: {{ required "snapshot.storage.s3.securityTokenKey is required" .Values.snapshot.storage.s3.securityTokenKey | quote }} + optional: true + {{- end }} - name: ETCD_ADDRESS value: {{ get $nodeEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) | quote }} - name: ETCD_PORT diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index ac28507..fae6cbb 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -23,6 +23,24 @@ image: tag: "latest" pullPolicy: IfNotPresent +# Snapshot-backed sandbox lifecycle data plane. +snapshot: + storage: + backend: datasystem + mode: distributed_cache + s3: + # Supported provider profiles: generic, obs, oss. + provider: generic + endpoint: "" + region: us-east-1 + bucket: "" + existingSecret: "" + accessKeyKey: access-key + secretKeyKey: secret-key + securityTokenKey: security-token + useHttps: true + pathStyle: false + auth: # Existing Secret that contains the JWT signing seed. For # `helm template | kubectl apply`, prefer pre-creating this Secret with diff --git a/deploy/akernel/values.yaml b/deploy/akernel/values.yaml index 77bbe26..dde5ec4 100644 --- a/deploy/akernel/values.yaml +++ b/deploy/akernel/values.yaml @@ -1,5 +1,20 @@ core: createNamespace: false + snapshot: + storage: + backend: datasystem + mode: distributed_cache + s3: + provider: generic + endpoint: "" + region: us-east-1 + bucket: "" + existingSecret: "" + accessKeyKey: access-key + secretKeyKey: secret-key + securityTokenKey: security-token + useHttps: true + pathStyle: false monitor: enabled: true diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index 1fff5d2..ed93c9f 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -350,11 +350,72 @@ prepare_host_network_modules() { # Start the AKernel all-in-one container. Traefik runs separately so traffic # from the gateway enters this network namespace through PREROUTING. -start_node_container() { +start_node_container() ( log_info "Starting container: ${NODE_CONTAINER_NAME}" # FunctionMaster's HTTP provider publishes the per-sandbox routes required # by reverse tunnels; the legacy etcd mode cannot publish those routes. + local snapshot_backend="${AKERNEL_SNAPSHOT_STORAGE_BACKEND:-datasystem}" + local snapshot_env_file="" + cleanup_snapshot_env_file() { + if [[ -n "${snapshot_env_file}" ]]; then + rm -f -- "${snapshot_env_file}" + fi + } + trap cleanup_snapshot_env_file EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + + local snapshot_docker_env=( + -e AKERNEL_SNAPSHOT_STORAGE_BACKEND="${snapshot_backend}" + ) + if [ "${snapshot_backend}" = "s3" ]; then + local snapshot_env_names=( + AKERNEL_SNAPSHOT_STORAGE_BACKEND + AKERNEL_SNAPSHOT_STORAGE_MODE + AKERNEL_SNAPSHOT_S3_PROVIDER + AKERNEL_SNAPSHOT_S3_ENDPOINT + AKERNEL_SNAPSHOT_S3_REGION + AKERNEL_SNAPSHOT_S3_BUCKET + AKERNEL_SNAPSHOT_S3_ACCESS_KEY + AKERNEL_SNAPSHOT_S3_SECRET_KEY + AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN + AKERNEL_SNAPSHOT_S3_USE_HTTPS + AKERNEL_SNAPSHOT_S3_PATH_STYLE + ) + local snapshot_env_values=( + s3 + "${AKERNEL_SNAPSHOT_STORAGE_MODE:-distributed_cache}" + "${AKERNEL_SNAPSHOT_S3_PROVIDER:-}" + "${AKERNEL_SNAPSHOT_S3_ENDPOINT:-}" + "${AKERNEL_SNAPSHOT_S3_REGION:-}" + "${AKERNEL_SNAPSHOT_S3_BUCKET:-}" + "${AKERNEL_SNAPSHOT_S3_ACCESS_KEY:-}" + "${AKERNEL_SNAPSHOT_S3_SECRET_KEY:-}" + "${AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN:-}" + "${AKERNEL_SNAPSHOT_S3_USE_HTTPS:-true}" + "${AKERNEL_SNAPSHOT_S3_PATH_STYLE:-false}" + ) + local index + for ((index = 0; index < ${#snapshot_env_names[@]}; index++)); do + case "${snapshot_env_values[index]}" in + *$'\r'*|*$'\n'*) + log_error "${snapshot_env_names[index]} must not contain CR or LF" + return 1 + ;; + esac + done + umask 077 + snapshot_env_file=$(mktemp "${TMPDIR:-/tmp}/akernel-snapshot-s3.XXXXXX") + chmod 0600 "${snapshot_env_file}" + for ((index = 0; index < ${#snapshot_env_names[@]}; index++)); do + printf '%s=%s\n' "${snapshot_env_names[index]}" \ + "${snapshot_env_values[index]}" >>"${snapshot_env_file}" + done + snapshot_docker_env=(--env-file "${snapshot_env_file}") + fi + "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} run -d \ --name "${NODE_CONTAINER_NAME}" \ --privileged \ @@ -373,6 +434,7 @@ start_node_container() { -e TZ=Asia/Shanghai \ -e ENABLE_TRACE="${ENABLE_TRACE:-false}" \ -e ENABLE_METRICS="${ENABLE_METRICS:-false}" \ + "${snapshot_docker_env[@]}" \ "${PROXY_RUN_ARGS[@]}" \ "${GPU_RUN_ARGS[@]}" \ --entrypoint=/usr/local/bin/akernel-entrypoint \ @@ -384,7 +446,7 @@ start_node_container() { -v "${CONFIG_DIR}/config.json:/home/akernel/images/config.json:ro" \ -v "${SANDBOXD_CONFIG_FILE}:/home/akernel/sandboxd/config.toml:ro" \ "${IMAGE}" -} +) # Wait for container to be ready wait_for_ready() { diff --git a/src/yuanrong b/src/yuanrong index 353e955..e0b8cac 160000 --- a/src/yuanrong +++ b/src/yuanrong @@ -1 +1 @@ -Subproject commit 353e955ff6aea6c175a453e1b38c895269758f62 +Subproject commit e0b8cac6bd19a5c47ff528b628a41e8b0701e65b From 16ba10bcd2e388bded874ae86b7262bdd8809ea2 Mon Sep 17 00:00:00 2001 From: ChamberlainJI Date: Wed, 2 Sep 2026 21:37:06 +0800 Subject: [PATCH 2/3] test(snapshot): cover S3 deployment wiring Signed-off-by: ChamberlainJI --- .../tests/test-openyuanrong-s3-capability.sh | 352 ++++++++++++++++ .../akernel/tests/test-pause-resume-wiring.sh | 151 +++++++ .../tests/test-pause-resume-wiring.sh | 384 ++++++++++++++++++ 3 files changed, 887 insertions(+) create mode 100755 builder/scripts/tests/test-openyuanrong-s3-capability.sh create mode 100755 deploy/akernel/tests/test-pause-resume-wiring.sh create mode 100755 deploy/standalone/tests/test-pause-resume-wiring.sh diff --git a/builder/scripts/tests/test-openyuanrong-s3-capability.sh b/builder/scripts/tests/test-openyuanrong-s3-capability.sh new file mode 100755 index 0000000..b33371c --- /dev/null +++ b/builder/scripts/tests/test-openyuanrong-s3-capability.sh @@ -0,0 +1,352 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +detector="${root}/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh" +task7_source_root="${YR_S3_TASK7_SOURCE_ROOT:-}" +tmp=$(mktemp -d) +trap 'rm -rf "${tmp}"' EXIT +marker="${tmp}/.akernel-s3-snapshot-capable" +sentinel="${tmp}/detector-must-not-remove" + +config_parser_evidence=( + '--snapshot_s3_provider) SNAPSHOT_S3_PROVIDER=$2 && shift 2 ;;' + '--snapshot_s3_endpoint) SNAPSHOT_S3_ENDPOINT=$2 && shift 2 ;;' + '--snapshot_s3_region) SNAPSHOT_S3_REGION=$2 && shift 2 ;;' + '--snapshot_s3_bucket) SNAPSHOT_S3_BUCKET=$2 && shift 2 ;;' + '--snapshot_s3_use_https) SNAPSHOT_S3_USE_HTTPS=$2 && shift 2 ;;' + '--snapshot_s3_path_style) SNAPSHOT_S3_PATH_STYLE=$2 && shift 2 ;;' +) +config_backend_evidence=( + 'if [ "X${SNAPSHOT_STORAGE_MODE}" != "Xlocal_only" ]; then' + 'case "${SNAPSHOT_STORAGE_BACKEND}" in' + ' s3)' + 'case "${SNAPSHOT_S3_PROVIDER}" in' + 'generic|obs|oss) ;;' + 'log_error "snapshot_s3_provider must be generic, obs, or oss"' + 'if [ -z "${SNAPSHOT_S3_ENDPOINT}" ] || [ -z "${SNAPSHOT_S3_REGION}" ] || [ -z "${SNAPSHOT_S3_BUCKET}" ]; then' + 'log_error "snapshot_s3_endpoint, snapshot_s3_region, and snapshot_s3_bucket are required for S3"' + 'case "${SNAPSHOT_S3_USE_HTTPS}" in' + 'log_error "snapshot_s3_use_https must be true or false"' + 'case "${SNAPSHOT_S3_PATH_STYLE}" in' + 'log_error "snapshot_s3_path_style must be true or false"' + 'if [ "X${SNAPSHOT_S3_PROVIDER}" = "Xoss" ] && [ "X${SNAPSHOT_S3_PATH_STYLE}" = "Xtrue" ]; then' + 'log_error "snapshot S3 OSS provider requires virtual-host addressing"' + 'if [ -z "${SNAPSHOT_S3_ACCESS_KEY:-}" ] || [ -z "${SNAPSHOT_S3_SECRET_KEY:-}" ]; then' + 'log_error "SNAPSHOT_S3_ACCESS_KEY and SNAPSHOT_S3_SECRET_KEY are required for S3"' + 'log_error "distributed snapshot storage backend must be datasystem, obs, or s3"' +) +config_export_evidence=( + 'export SNAPSHOT_S3_PROVIDER SNAPSHOT_S3_ENDPOINT SNAPSHOT_S3_REGION SNAPSHOT_S3_BUCKET' + 'export SNAPSHOT_S3_USE_HTTPS SNAPSHOT_S3_PATH_STYLE' + 'export SNAPSHOT_S3_ACCESS_KEY SNAPSHOT_S3_SECRET_KEY SNAPSHOT_S3_SECURITY_TOKEN' +) +install_evidence=( + '--snapshot_s3_provider="${SNAPSHOT_S3_PROVIDER:-generic}"' + '--snapshot_s3_endpoint="${SNAPSHOT_S3_ENDPOINT:-}"' + '--snapshot_s3_region="${SNAPSHOT_S3_REGION:-}"' + '--snapshot_s3_bucket="${SNAPSHOT_S3_BUCKET:-}"' + '--snapshot_s3_use_https="${SNAPSHOT_S3_USE_HTTPS:-true}"' + '--snapshot_s3_path_style="${SNAPSHOT_S3_PATH_STYLE:-false}"' +) +install_path_evidence=( + 'function install_function_proxy() {' + 'function install_function_agent_and_runtime_manager_in_the_same_process() {' +) +agent_evidence=( + snapshot_s3_provider + snapshot_s3_endpoint + snapshot_s3_region + snapshot_s3_bucket + snapshot_s3_use_https + snapshot_s3_path_style + SNAPSHOT_S3_ACCESS_KEY + SNAPSHOT_S3_SECRET_KEY + SNAPSHOT_S3_SECURITY_TOKEN + 'snapshot S3 object exceeds the 5 GiB limit' + 'published object snapshot failed postcondition verification' +) + +copy_task7_sources() { + local config_source="${task7_source_root}/deploy/process/config.sh" + local install_source="${task7_source_root}/functionsystem/scripts/deploy/function_system/install.sh" + if [[ ! -f "${install_source}" ]]; then + install_source="${task7_source_root}/functionsystem/deploy/install.sh" + fi + if [[ ! -f "${config_source}" || ! -f "${install_source}" ]]; then + echo "YR_S3_TASK7_SOURCE_ROOT does not contain reviewed Task 7 config/install sources" >&2 + exit 1 + fi + cp "${config_source}" "${tmp}/deploy/process/config.sh" + cp "${install_source}" "${tmp}/functionsystem/deploy/install.sh" +} + +write_synthetic_task7_sources() { + { + printf '%s\n' \ + 'parse_snapshot_args() {' \ + ' while true; do' \ + ' case "$1" in' + printf ' %s\n' "${config_parser_evidence[@]}" + printf '%s\n' \ + ' --) shift && break ;;' \ + ' esac' \ + ' done' \ + ' if [ "X${SNAPSHOT_STORAGE_MODE}" != "Xlocal_only" ]; then' \ + ' case "${SNAPSHOT_STORAGE_BACKEND}" in' \ + ' datasystem|obs) ;;' \ + ' s3)' \ + ' case "${SNAPSHOT_S3_PROVIDER}" in' \ + ' generic|obs|oss) ;;' \ + ' *) log_error "snapshot_s3_provider must be generic, obs, or oss"; return 1 ;;' \ + ' esac' \ + ' if [ -z "${SNAPSHOT_S3_ENDPOINT}" ] || [ -z "${SNAPSHOT_S3_REGION}" ] || [ -z "${SNAPSHOT_S3_BUCKET}" ]; then' \ + ' log_error "snapshot_s3_endpoint, snapshot_s3_region, and snapshot_s3_bucket are required for S3"' \ + ' return 1' \ + ' fi' \ + ' case "${SNAPSHOT_S3_USE_HTTPS}" in' \ + ' true|false) ;;' \ + ' *) log_error "snapshot_s3_use_https must be true or false"; return 1 ;;' \ + ' esac' \ + ' case "${SNAPSHOT_S3_PATH_STYLE}" in' \ + ' true|false) ;;' \ + ' *) log_error "snapshot_s3_path_style must be true or false"; return 1 ;;' \ + ' esac' \ + ' if [ "X${SNAPSHOT_S3_PROVIDER}" = "Xoss" ] && [ "X${SNAPSHOT_S3_PATH_STYLE}" = "Xtrue" ]; then' \ + ' log_error "snapshot S3 OSS provider requires virtual-host addressing"' \ + ' return 1' \ + ' fi' \ + ' if [ -z "${SNAPSHOT_S3_ACCESS_KEY:-}" ] || [ -z "${SNAPSHOT_S3_SECRET_KEY:-}" ]; then' \ + ' log_error "SNAPSHOT_S3_ACCESS_KEY and SNAPSHOT_S3_SECRET_KEY are required for S3"' \ + ' return 1' \ + ' fi' \ + ' ;;' \ + ' *)' \ + ' log_error "distributed snapshot storage backend must be datasystem, obs, or s3"' \ + ' return 1' \ + ' ;;' \ + ' esac' \ + ' fi' \ + '}' \ + 'export_config() {' \ + ' export SNAPSHOT_S3_PROVIDER SNAPSHOT_S3_ENDPOINT SNAPSHOT_S3_REGION SNAPSHOT_S3_BUCKET' \ + ' export SNAPSHOT_S3_USE_HTTPS SNAPSHOT_S3_PATH_STYLE' \ + ' export SNAPSHOT_S3_ACCESS_KEY SNAPSHOT_S3_SECRET_KEY SNAPSHOT_S3_SECURITY_TOKEN' \ + '}' + } >"${tmp}/deploy/process/config.sh" + + { + printf '%s\n' 'function install_function_proxy() {' ' merge_process_args="' + printf ' %s \\\n' "${install_evidence[@]}" + printf '%s\n' ' "' '}' \ + 'function install_function_agent_and_runtime_manager_in_the_same_process() {' \ + ' agent_args=(' + printf ' %s\n' "${install_evidence[@]}" + printf '%s\n' ' )' '}' + } >"${tmp}/functionsystem/deploy/install.sh" +} + +write_complete_package() { + rm -rf "${tmp}/deploy" "${tmp}/functionsystem" + mkdir -p "${tmp}/deploy/process" "${tmp}/functionsystem/deploy" \ + "${tmp}/functionsystem/bin" + if [[ -n "${task7_source_root}" ]]; then + copy_task7_sources + else + write_synthetic_task7_sources + fi + printf '%s\n' "${agent_evidence[@]}" \ + >"${tmp}/functionsystem/bin/function_agent" + chmod +x "${tmp}/functionsystem/bin/function_agent" +} + +remove_first_occurrence() { + local file=$1 + local evidence=$2 + python3 - "${file}" "${evidence}" <<'PY' +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +evidence = sys.argv[2] +text = path.read_text() +if evidence not in text: + raise SystemExit(f"fixture does not contain evidence: {evidence}") +path.write_text(text.replace(evidence, "", 1)) +PY +} + +remove_from_function_body() { + local file=$1 + local function_header=$2 + local evidence=$3 + python3 - "${file}" "${function_header}" "${evidence}" <<'PY' +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +header = sys.argv[2] +evidence = sys.argv[3] +lines = path.read_text().splitlines(keepends=True) +try: + start = next(index for index, line in enumerate(lines) if line.rstrip("\r\n") == header) + end = next(index for index in range(start + 1, len(lines)) if lines[index].rstrip("\r\n") == "}") + target = next(index for index in range(start + 1, end) if evidence in lines[index]) +except StopIteration as error: + raise SystemExit(f"function fixture does not contain evidence: {header}: {evidence}") from error +lines[target] = lines[target].replace(evidence, "", 1) +path.write_text("".join(lines)) +PY +} + +insert_before_function_close() { + local file=$1 + local function_header=$2 + local line=$3 + python3 - "${file}" "${function_header}" "${line}" <<'PY' +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +header = sys.argv[2] +inserted = sys.argv[3] +lines = path.read_text().splitlines(keepends=True) +try: + start = next(index for index, line in enumerate(lines) if line.rstrip("\r\n") == header) + end = next(index for index in range(start + 1, len(lines)) if lines[index].rstrip("\r\n") == "}") +except StopIteration as error: + raise SystemExit(f"function fixture is malformed: {header}") from error +lines.insert(end, f" {inserted}\n") +path.write_text("".join(lines)) +PY +} + +assert_fixture_occurrences() { + local file=$1 + local expected_count=$2 + local evidence=$3 + local actual_count + actual_count=$(grep -Fc -- "${evidence}" "${file}" || true) + if [[ "${actual_count}" -ne "${expected_count}" ]]; then + echo "fixture expected ${expected_count} occurrences of ${evidence}, got ${actual_count}" >&2 + exit 1 + fi +} + +assert_capable() { + local description=$1 + "${detector}" "${tmp}" + if [[ ! -f "${marker}" ]]; then + echo "${description}: capable package did not create marker" >&2 + exit 1 + fi + [[ -f "${sentinel}" ]] +} + +assert_not_capable() { + local description=$1 + touch "${marker}" + "${detector}" "${tmp}" + if [[ -e "${marker}" ]]; then + echo "${description}: incomplete package retained capability marker" >&2 + exit 1 + fi + [[ -f "${sentinel}" ]] +} + +touch "${sentinel}" +write_complete_package +assert_capable "complete Task 7 evidence" + +write_complete_package +rm "${tmp}/deploy/process/config.sh" +assert_not_capable "missing process config" + +write_complete_package +rm "${tmp}/functionsystem/deploy/install.sh" +assert_not_capable "missing install wiring" + +write_complete_package +rm "${tmp}/functionsystem/bin/function_agent" +assert_not_capable "missing FunctionAgent" + +for item in "${config_parser_evidence[@]}" "${config_backend_evidence[@]}" \ + "${config_export_evidence[@]}"; do + write_complete_package + remove_first_occurrence "${tmp}/deploy/process/config.sh" "${item}" + assert_not_capable "partial process config without ${item}" +done + +for item in "${install_evidence[@]}"; do + write_complete_package + remove_first_occurrence "${tmp}/functionsystem/deploy/install.sh" "${item}" + assert_not_capable "install wiring has only one ${item} occurrence" +done + +for item in "${install_evidence[@]}"; do + write_complete_package + remove_from_function_body "${tmp}/functionsystem/deploy/install.sh" \ + "${install_path_evidence[0]}" "${item}" + insert_before_function_close "${tmp}/functionsystem/deploy/install.sh" \ + "${install_path_evidence[1]}" "${item}" + assert_fixture_occurrences "${tmp}/functionsystem/deploy/install.sh" 2 "${item}" + assert_not_capable "global count two but merged launch lacks ${item}" +done + +for item in "${install_evidence[@]}"; do + write_complete_package + remove_from_function_body "${tmp}/functionsystem/deploy/install.sh" \ + "${install_path_evidence[0]}" "${item}" + insert_before_function_close "${tmp}/functionsystem/deploy/install.sh" \ + "${install_path_evidence[0]}" "# ${item} \\" + assert_fixture_occurrences "${tmp}/functionsystem/deploy/install.sh" 2 "${item}" + assert_not_capable "comment must not replace merged launch argument ${item}" +done + +for item in "${install_path_evidence[@]}"; do + write_complete_package + remove_first_occurrence "${tmp}/functionsystem/deploy/install.sh" "${item}" + assert_not_capable "install wiring lacks startup path ${item}" +done + +write_complete_package +printf '%s\n' "${install_evidence[0]}" >>"${tmp}/functionsystem/deploy/install.sh" +assert_not_capable "install wiring has an unexpected third S3 argument occurrence" + +write_complete_package +printf '%s\n' '--snapshot_s3_access_key="${SNAPSHOT_S3_ACCESS_KEY:-}"' \ + >>"${tmp}/functionsystem/deploy/install.sh" +assert_not_capable "credential argv contamination" + +for item in "${agent_evidence[@]}"; do + write_complete_package + remove_first_occurrence "${tmp}/functionsystem/bin/function_agent" "${item}" + chmod +x "${tmp}/functionsystem/bin/function_agent" + assert_not_capable "partial FunctionAgent evidence without ${item}" +done + +write_complete_package +remove_first_occurrence "${tmp}/functionsystem/bin/function_agent" \ + 'snapshot S3 object exceeds the 5 GiB limit' +printf '%s\n' 'remote S3 snapshot exceeds the 5 GiB capability limit' \ + >>"${tmp}/functionsystem/bin/function_agent" +assert_not_capable "stale 5 GiB prototype string" + +write_complete_package +chmod -x "${tmp}/functionsystem/bin/function_agent" +assert_not_capable "non-executable FunctionAgent" + +write_complete_package +printf '%s\n' 'legacy function agent' >"${tmp}/functionsystem/bin/function_agent" +chmod +x "${tmp}/functionsystem/bin/function_agent" +assert_not_capable "legacy package" + +if [[ -n "${task7_source_root}" ]]; then + echo "openYuanRong S3 capability detection checks passed with Task 7 source fixtures" +else + echo "openYuanRong S3 capability detection checks passed with representative fixtures" +fi diff --git a/deploy/akernel/tests/test-pause-resume-wiring.sh b/deploy/akernel/tests/test-pause-resume-wiring.sh new file mode 100755 index 0000000..e0d02ff --- /dev/null +++ b/deploy/akernel/tests/test-pause-resume-wiring.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +chart="${repo_root}/deploy/akernel" +tmp_dir=$(mktemp -d) +trap 'rm -rf "${tmp_dir}"' EXIT + +default_render="${tmp_dir}/default.yaml" +s3_render="${tmp_dir}/s3.yaml" +s3_default_addressing_render="${tmp_dir}/s3-default-addressing.yaml" +s3_distributed_only_render="${tmp_dir}/s3-distributed-only.yaml" +helm template akernel-snapshot "${chart}" --set monitor.enabled=false >"${default_render}" +if grep -q 'name: AKERNEL_ENABLE_SNAPSHOT' "${default_render}"; then + echo "Helm still emits the removed snapshot enable switch" >&2 + exit 1 +fi +grep -A1 'name: AKERNEL_SNAPSHOT_STORAGE_BACKEND' "${default_render}" | grep -q 'value: "datasystem"' +if grep -Eq 'name: AKERNEL_SNAPSHOT_STORAGE_MODE|name: AKERNEL_SNAPSHOT_S3_' "${default_render}"; then + echo "default DataSystem Helm render contains S3-only environment" >&2 + exit 1 +fi + +helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=generic \ + --set core.snapshot.storage.s3.endpoint=s3.private.example \ + --set core.snapshot.storage.s3.region=us-east-1 \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 \ + --set core.snapshot.storage.s3.pathStyle=true >"${s3_render}" +grep -A1 'name: AKERNEL_SNAPSHOT_S3_PROVIDER' "${s3_render}" | grep -q 'value: "generic"' +grep -A1 'name: AKERNEL_SNAPSHOT_STORAGE_MODE' "${s3_render}" \ + | grep -q 'value: "distributed_cache"' +grep -A1 'name: AKERNEL_SNAPSHOT_S3_ENDPOINT' "${s3_render}" | grep -q 'value: "s3.private.example"' +grep -A1 'name: AKERNEL_SNAPSHOT_S3_REGION' "${s3_render}" | grep -q 'value: "us-east-1"' +grep -A1 'name: AKERNEL_SNAPSHOT_S3_BUCKET' "${s3_render}" | grep -q 'value: "akernel-test"' +grep -A1 'name: AKERNEL_SNAPSHOT_S3_USE_HTTPS' "${s3_render}" | grep -q 'value: "true"' +grep -A1 'name: AKERNEL_SNAPSHOT_S3_PATH_STYLE' "${s3_render}" | grep -q 'value: "true"' +for credential in ACCESS_KEY SECRET_KEY SECURITY_TOKEN; do + grep -A5 "name: AKERNEL_SNAPSHOT_S3_${credential}" "${s3_render}" \ + | grep -q 'name: "akernel-snapshot-s3"' +done + +helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=generic \ + --set core.snapshot.storage.s3.endpoint=s3.private.example \ + --set core.snapshot.storage.s3.region=us-east-1 \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 \ + >"${s3_default_addressing_render}" +grep -A1 'name: AKERNEL_SNAPSHOT_S3_PATH_STYLE' "${s3_default_addressing_render}" \ + | grep -q 'value: "false"' + +helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.mode=distributed_only \ + --set core.snapshot.storage.s3.provider=generic \ + --set core.snapshot.storage.s3.endpoint=s3.private.example \ + --set core.snapshot.storage.s3.region=us-east-1 \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 \ + >"${s3_distributed_only_render}" +grep -A1 'name: AKERNEL_SNAPSHOT_STORAGE_MODE' "${s3_distributed_only_render}" \ + | grep -q 'value: "distributed_only"' + +if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=obs >/dev/null 2>&1; then + echo "Helm accepted the removed snapshot OBS backend" >&2 + exit 1 +fi + +for boolean_field in useHttps pathStyle; do + if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=generic \ + --set core.snapshot.storage.s3.endpoint=s3.example \ + --set core.snapshot.storage.s3.region=us-east-1 \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 \ + --set-string "core.snapshot.storage.s3.${boolean_field}=not-a-boolean" >/dev/null 2>&1; then + echo "Helm accepted non-boolean S3 ${boolean_field}" >&2 + exit 1 + fi +done + +for storage_mode in local_only invalid; do + if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.mode="${storage_mode}" \ + --set core.snapshot.storage.s3.provider=generic \ + --set core.snapshot.storage.s3.endpoint=s3.example \ + --set core.snapshot.storage.s3.region=us-east-1 \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 >/dev/null 2>&1; then + echo "Helm accepted S3 storage mode ${storage_mode}" >&2 + exit 1 + fi +done + +for secret_key_field in accessKeyKey secretKeyKey securityTokenKey; do + if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=generic \ + --set core.snapshot.storage.s3.endpoint=s3.example \ + --set core.snapshot.storage.s3.region=us-east-1 \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 \ + --set-string "core.snapshot.storage.s3.${secret_key_field}=" >/dev/null 2>&1; then + echo "Helm accepted empty S3 ${secret_key_field}" >&2 + exit 1 + fi +done + +if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=unknown \ + --set core.snapshot.storage.s3.endpoint=s3.example \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 >/dev/null 2>&1; then + echo "Helm accepted an unknown S3 provider" >&2 + exit 1 +fi + +if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=oss \ + --set core.snapshot.storage.s3.endpoint=oss-cname.example \ + --set core.snapshot.storage.s3.region=cn-hangzhou \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 \ + --set core.snapshot.storage.s3.pathStyle=true >/dev/null 2>&1; then + echo "Helm accepted path-style addressing for OSS" >&2 + exit 1 +fi + +echo "Kubernetes snapshot wiring contract passed" diff --git a/deploy/standalone/tests/test-pause-resume-wiring.sh b/deploy/standalone/tests/test-pause-resume-wiring.sh new file mode 100755 index 0000000..5e742fa --- /dev/null +++ b/deploy/standalone/tests/test-pause-resume-wiring.sh @@ -0,0 +1,384 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +helper="${repo_root}/builder/scripts/yr_pause_resume_args.sh" +standalone_start="${repo_root}/deploy/standalone/start.sh" +systemd_unit="${repo_root}/builder/systemd_services/yuanrong.service" +tmp_dir=$(mktemp -d) +trap 'rm -rf "${tmp_dir}"' EXIT +rrt_capability="${tmp_dir}/rrt-capable" +s3_capability="${tmp_dir}/s3-capable" +checkpoint_dir="${tmp_dir}/checkpoints" +output="${tmp_dir}/output" +touch "${rrt_capability}" "${s3_capability}" + +source "${helper}" + +fail() { + echo "$1" >&2 + exit 1 +} + +assert_output_credentials_unset() { + [[ -z "${SNAPSHOT_S3_ACCESS_KEY+x}" ]] || fail "stale S3 access key remained exported" + [[ -z "${SNAPSHOT_S3_SECRET_KEY+x}" ]] || fail "stale S3 secret key remained exported" + [[ -z "${SNAPSHOT_S3_SECURITY_TOKEN+x}" ]] || fail "stale S3 security token remained exported" +} + +assert_no_credential_canary() { + local candidate=$1 + local canary + for canary in AK_CANARY_MUST_NOT_REACH_ARGV SK_CANARY_MUST_NOT_REACH_ARGV \ + TOKEN_CANARY_MUST_NOT_REACH_ARGV STALE_AK_CANARY STALE_SK_CANARY STALE_TOKEN_CANARY; do + if [[ "${candidate}" == *"${canary}"* ]]; then + fail "S3 credential value reached argv or command output" + fi + done +} + +set_s3_environment() { + export AKERNEL_SNAPSHOT_STORAGE_BACKEND=s3 + export AKERNEL_SNAPSHOT_STORAGE_MODE="${3:-distributed_cache}" + export AKERNEL_SNAPSHOT_S3_PROVIDER=$1 + export AKERNEL_SNAPSHOT_S3_ENDPOINT="${1}.private.example:9000" + export AKERNEL_SNAPSHOT_S3_REGION=test-region-1 + export AKERNEL_SNAPSHOT_S3_BUCKET=akernel-test + export AKERNEL_SNAPSHOT_S3_ACCESS_KEY=AK_CANARY_MUST_NOT_REACH_ARGV + export AKERNEL_SNAPSHOT_S3_SECRET_KEY=SK_CANARY_MUST_NOT_REACH_ARGV + export AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN=TOKEN_CANARY_MUST_NOT_REACH_ARGV + export AKERNEL_SNAPSHOT_S3_USE_HTTPS=true + export AKERNEL_SNAPSHOT_S3_PATH_STYLE=$2 +} + +assert_s3_success() { + local provider=$1 + local path_style=$2 + local storage_mode=${3:-distributed_cache} + set_s3_environment "${provider}" "${path_style}" "${storage_mode}" + : >"${output}" + configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false \ + "${s3_capability}" >"${output}" 2>&1 + local expected="--snapshot_storage_backend s3 --checkpoint_dir ${checkpoint_dir}" + expected+=" --snapshot_storage_mode ${storage_mode}" + expected+=" --snapshot_s3_provider ${provider}" + expected+=" --snapshot_s3_endpoint ${provider}.private.example:9000" + expected+=" --snapshot_s3_region test-region-1 --snapshot_s3_bucket akernel-test" + expected+=" --snapshot_s3_use_https true --snapshot_s3_path_style ${path_style}" + [[ "${snapshot_args[*]}" == "${expected}" ]] || fail "${provider} S3 argv is not exact" + [[ "${standalone_snapshot_args[*]}" == "${expected}" ]] \ + || fail "${provider} standalone S3 argv is not exact" + [[ "${SNAPSHOT_S3_ACCESS_KEY}" == "${AKERNEL_SNAPSHOT_S3_ACCESS_KEY}" ]] + [[ "${SNAPSHOT_S3_SECRET_KEY}" == "${AKERNEL_SNAPSHOT_S3_SECRET_KEY}" ]] + [[ "${SNAPSHOT_S3_SECURITY_TOKEN}" == "${AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN}" ]] + assert_no_credential_canary "${snapshot_args[*]} ${standalone_snapshot_args[*]} $(<"${output}")" +} + +assert_s3_failure() { + local description=$1 + : >"${output}" + export SNAPSHOT_S3_ACCESS_KEY=STALE_AK_CANARY + export SNAPSHOT_S3_SECRET_KEY=STALE_SK_CANARY + export SNAPSHOT_S3_SECURITY_TOKEN=STALE_TOKEN_CANARY + if configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false \ + "${s3_capability}" >"${output}" 2>&1; then + fail "${description}: invalid S3 configuration was accepted" + fi + [[ ${#snapshot_args[@]} -eq 0 ]] || fail "${description}: failed call returned snapshot argv" + [[ ${#standalone_snapshot_args[@]} -eq 0 ]] \ + || fail "${description}: failed call returned standalone argv" + assert_output_credentials_unset + assert_no_credential_canary "$(<"${output}")" +} + +# Default DataSystem behavior remains exact and removes stale S3 output. +unset AKERNEL_SNAPSHOT_STORAGE_BACKEND +export SNAPSHOT_S3_ACCESS_KEY=STALE_AK_CANARY +export SNAPSHOT_S3_SECRET_KEY=STALE_SK_CANARY +export SNAPSHOT_S3_SECURITY_TOKEN=STALE_TOKEN_CANARY +configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" true "${s3_capability}" +expected="--snapshot_storage_backend datasystem --checkpoint_dir ${checkpoint_dir} --data_system_enable true" +[[ "${standalone_snapshot_args[*]}" == "${expected}" ]] || fail "DataSystem argv changed" +assert_output_credentials_unset + +configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false "${s3_capability}" +expected="--snapshot_storage_backend datasystem --checkpoint_dir ${checkpoint_dir}" +[[ "${snapshot_args[*]}" == "${expected}" ]] || fail "cluster DataSystem argv changed" +[[ "${standalone_snapshot_args[*]}" == "${expected}" ]] || fail "standalone=false DataSystem argv changed" +assert_output_credentials_unset + +export AKERNEL_SNAPSHOT_STORAGE_MODE=invalid +configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false "${s3_capability}" +[[ "${snapshot_args[*]}" == "${expected}" ]] \ + || fail "DataSystem unexpectedly consumed the S3-only storage mode" +unset AKERNEL_SNAPSHOT_STORAGE_MODE + +assert_s3_success generic true +assert_s3_success obs true +assert_s3_success oss false +assert_s3_success generic false distributed_only + +set_s3_environment generic false distributed_cache +configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" true "${s3_capability}" +[[ "${standalone_snapshot_args[*]}" == "${snapshot_args[*]} --data_system_enable true" ]] \ + || fail "standalone S3 argv did not preserve mode before DataSystem enablement" + +set_s3_environment generic false +unset AKERNEL_SNAPSHOT_STORAGE_MODE +configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false "${s3_capability}" +[[ " ${snapshot_args[*]} " == *" --snapshot_storage_mode distributed_cache "* ]] \ + || fail "S3 storage mode did not default to distributed_cache" + +set_s3_environment generic false +unset AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN +configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false "${s3_capability}" +[[ -n "${SNAPSHOT_S3_SECURITY_TOKEN+x}" && -z "${SNAPSHOT_S3_SECURITY_TOKEN}" ]] \ + || fail "optional S3 security token was not exported as empty" +assert_no_credential_canary "${snapshot_args[*]} ${standalone_snapshot_args[*]}" + +# A missing capability marker fails before any argv or credential output is returned. +set_s3_environment generic true +rm "${s3_capability}" +assert_s3_failure "missing capability marker" +touch "${s3_capability}" + +set_s3_environment generic true +export SNAPSHOT_S3_ACCESS_KEY=STALE_AK_CANARY +export SNAPSHOT_S3_SECRET_KEY=STALE_SK_CANARY +export SNAPSHOT_S3_SECURITY_TOKEN=STALE_TOKEN_CANARY +if configure_snapshot_args "${tmp_dir}/missing-rrt-capability" "${checkpoint_dir}" false \ + "${s3_capability}" >"${output}" 2>&1; then + fail "missing RRT capability was accepted" +fi +[[ ${#snapshot_args[@]} -eq 0 && ${#standalone_snapshot_args[@]} -eq 0 ]] \ + || fail "missing RRT capability returned argv" +assert_output_credentials_unset +assert_no_credential_canary "$(<"${output}")" + +for missing in \ + AKERNEL_SNAPSHOT_S3_PROVIDER AKERNEL_SNAPSHOT_S3_ENDPOINT \ + AKERNEL_SNAPSHOT_S3_REGION AKERNEL_SNAPSHOT_S3_BUCKET \ + AKERNEL_SNAPSHOT_S3_ACCESS_KEY AKERNEL_SNAPSHOT_S3_SECRET_KEY \ + AKERNEL_SNAPSHOT_S3_USE_HTTPS AKERNEL_SNAPSHOT_S3_PATH_STYLE; do + set_s3_environment generic true + unset "${missing}" + assert_s3_failure "missing ${missing}" +done + +set_s3_environment unknown false +assert_s3_failure "unknown provider" + +set_s3_environment generic true +export AKERNEL_SNAPSHOT_S3_USE_HTTPS=yes +assert_s3_failure "non-boolean HTTPS" + +set_s3_environment generic true +export AKERNEL_SNAPSHOT_S3_PATH_STYLE=1 +assert_s3_failure "non-boolean path style" + +set_s3_environment oss true +assert_s3_failure "OSS path-style addressing" + +for invalid_mode in local_only invalid; do + set_s3_environment generic false "${invalid_mode}" + assert_s3_failure "invalid S3 storage mode ${invalid_mode}" +done + +# Exercise the only validation that follows S3 credential collection explicitly. +set_s3_environment generic true +export SNAPSHOT_S3_ACCESS_KEY=STALE_AK_CANARY +export SNAPSHOT_S3_SECRET_KEY=STALE_SK_CANARY +export SNAPSHOT_S3_SECURITY_TOKEN=STALE_TOKEN_CANARY +if configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" invalid \ + "${s3_capability}" >"${output}" 2>&1; then + fail "invalid standalone selector was accepted" +fi +[[ ${#snapshot_args[@]} -eq 0 && ${#standalone_snapshot_args[@]} -eq 0 ]] \ + || fail "invalid standalone selector returned argv" +assert_output_credentials_unset +assert_no_credential_canary "$(<"${output}")" + +export AKERNEL_SNAPSHOT_STORAGE_BACKEND=obs +assert_s3_failure "legacy OBS backend" + +# Standalone S3 configuration crosses sudo through a protected env-file; no +# snapshot value or variable name is exposed in Docker argv. +# Source the real function body without executing start.sh's main program. +standalone_functions="${tmp_dir}/standalone-functions.sh" +sed '/^# Main$/,$d' "${standalone_start}" >"${standalone_functions}" +source "${standalone_functions}" +docker_argv_capture="${tmp_dir}/docker-argv" +docker_env_capture="${tmp_dir}/docker-env" +docker_env_file_capture="${tmp_dir}/docker-env-file" +docker_env_mode_capture="${tmp_dir}/docker-env-mode" +docker_capture_status=0 +docker_capture_signal="" + +clear_docker_capture() { + : >"${docker_argv_capture}" + : >"${docker_env_capture}" + : >"${docker_env_file_capture}" + : >"${docker_env_mode_capture}" +} + +sudo_sanitized_docker_capture() { + printf '%s\n' "$@" >"${docker_argv_capture}" + local env_file="" + while [[ $# -gt 0 ]]; do + if [[ "$1" == "--env-file" ]]; then + shift + env_file="${1:-}" + break + fi + shift + done + if [[ -n "${env_file}" ]]; then + printf '%s\n' "${env_file}" >"${docker_env_file_capture}" + local env_mode + if env_mode=$(stat -f '%Lp' "${env_file}" 2>/dev/null); then + : + else + env_mode=$(stat -c '%a' "${env_file}") + fi + printf '%s\n' "${env_mode}" >"${docker_env_mode_capture}" + /usr/bin/env -i ENV_FILE="${env_file}" CAPTURE_FILE="${docker_env_capture}" \ + /bin/bash -c ' + while IFS= read -r assignment || [[ -n "${assignment}" ]]; do + export "${assignment}" + done <"${ENV_FILE}" + /usr/bin/env | LC_ALL=C sort >"${CAPTURE_FILE}" + ' + fi + if [[ -n "${docker_capture_signal}" ]]; then + local current_shell_pid + current_shell_pid=$(/bin/sh -c 'printf "%s\n" "$PPID"') + kill -s "${docker_capture_signal}" "${current_shell_pid}" + fi + return "${docker_capture_status}" +} +# Bash 3.2 preserves one empty word for an empty quoted array in command +# position. Put the capture function in the prefix so this test exercises the +# same command construction on both macOS and Linux. +DOCKER_PREFIX=(sudo_sanitized_docker_capture) +DOCKER_CMD="" +PROXY_RUN_ARGS=(--label task8-proxy=unused) +GPU_RUN_ARGS=(--label task8-gpu=unused) +IMAGE=container-image:task8 +set_s3_environment generic true distributed_only +clear_docker_capture +start_node_container >"${output}" 2>&1 +grep -Fxq -- '--env-file' "${docker_argv_capture}" \ + || fail "standalone S3 launch did not use an env-file" +[[ "$(grep -Fxc -- '--env-file' "${docker_argv_capture}")" -eq 1 ]] \ + || fail "standalone S3 launch emitted multiple env-file options" +if grep -Fq 'AKERNEL_SNAPSHOT_' "${docker_argv_capture}"; then + fail "standalone S3 Docker argv contains snapshot environment names" +fi +for forbidden_value in distributed_only generic.private.example:9000 test-region-1 akernel-test; do + if grep -Fq -- "${forbidden_value}" "${docker_argv_capture}"; then + fail "standalone S3 Docker argv contains snapshot configuration values" + fi +done +assert_no_credential_canary "$(<"${docker_argv_capture}") $(<"${output}")" +[[ "$(<"${docker_env_mode_capture}")" == "600" ]] \ + || fail "standalone S3 env-file is not mode 0600" +snapshot_env_file=$(<"${docker_env_file_capture}") +[[ -n "${snapshot_env_file}" && ! -e "${snapshot_env_file}" ]] \ + || fail "standalone S3 env-file survived successful Docker launch" +for expected_env in \ + 'AKERNEL_SNAPSHOT_STORAGE_BACKEND=s3' \ + 'AKERNEL_SNAPSHOT_STORAGE_MODE=distributed_only' \ + 'AKERNEL_SNAPSHOT_S3_PROVIDER=generic' \ + 'AKERNEL_SNAPSHOT_S3_ENDPOINT=generic.private.example:9000' \ + 'AKERNEL_SNAPSHOT_S3_REGION=test-region-1' \ + 'AKERNEL_SNAPSHOT_S3_BUCKET=akernel-test' \ + 'AKERNEL_SNAPSHOT_S3_ACCESS_KEY=AK_CANARY_MUST_NOT_REACH_ARGV' \ + 'AKERNEL_SNAPSHOT_S3_SECRET_KEY=SK_CANARY_MUST_NOT_REACH_ARGV' \ + 'AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN=TOKEN_CANARY_MUST_NOT_REACH_ARGV' \ + 'AKERNEL_SNAPSHOT_S3_USE_HTTPS=true' \ + 'AKERNEL_SNAPSHOT_S3_PATH_STYLE=true'; do + grep -Fxq -- "${expected_env}" "${docker_env_capture}" \ + || fail "sudo-sanitized Docker environment omitted ${expected_env%%=*}" +done +[[ "$(grep -c '^AKERNEL_SNAPSHOT_' "${docker_env_capture}")" -eq 11 ]] \ + || fail "sudo-sanitized Docker environment contains an unexpected snapshot field set" + +set_s3_environment generic false +unset AKERNEL_SNAPSHOT_STORAGE_MODE +clear_docker_capture +start_node_container >"${output}" 2>&1 +grep -Fxq -- 'AKERNEL_SNAPSHOT_STORAGE_MODE=distributed_cache' "${docker_env_capture}" \ + || fail "standalone env-file did not default S3 mode to distributed_cache" +unset AKERNEL_SNAPSHOT_S3_PATH_STYLE +clear_docker_capture +start_node_container >"${output}" 2>&1 +grep -Fxq -- 'AKERNEL_SNAPSHOT_S3_PATH_STYLE=false' "${docker_env_capture}" \ + || fail "standalone path-style default does not match Task 7" + +# The env-file is deleted when Docker fails or the launch subshell is signaled. +set_s3_environment generic false +docker_capture_status=23 +clear_docker_capture +if start_node_container >"${output}" 2>&1; then + fail "standalone ignored a Docker launch failure" +fi +snapshot_env_file=$(<"${docker_env_file_capture}") +[[ -n "${snapshot_env_file}" && ! -e "${snapshot_env_file}" ]] \ + || fail "standalone S3 env-file survived Docker failure" +assert_no_credential_canary "$(<"${docker_argv_capture}") $(<"${output}")" +docker_capture_status=0 + +docker_capture_signal=TERM +clear_docker_capture +if start_node_container >"${output}" 2>&1; then + fail "standalone ignored a terminated Docker launch" +fi +snapshot_env_file=$(<"${docker_env_file_capture}") +[[ -n "${snapshot_env_file}" && ! -e "${snapshot_env_file}" ]] \ + || fail "standalone S3 env-file survived launch termination" +assert_no_credential_canary "$(<"${docker_argv_capture}") $(<"${output}")" +docker_capture_signal="" + +for line_break_value in $'endpoint\nINJECTED_VALUE' $'endpoint\rINJECTED_VALUE'; do + set_s3_environment generic false + export AKERNEL_SNAPSHOT_S3_ENDPOINT="${line_break_value}" + clear_docker_capture + if start_node_container >"${output}" 2>&1; then + fail "standalone accepted CR/LF in an S3 env-file value" + fi + [[ ! -s "${docker_env_file_capture}" ]] \ + || fail "standalone created an env-file before CR/LF validation" + if grep -Fq 'INJECTED_VALUE' "${output}"; then + fail "standalone logged an invalid S3 env-file value" + fi +done + +export AKERNEL_SNAPSHOT_STORAGE_BACKEND=datasystem +clear_docker_capture +start_node_container >"${output}" 2>&1 +[[ ! -s "${docker_env_file_capture}" ]] \ + || fail "DataSystem standalone launch created an S3 env-file" +if grep -Eq 'AKERNEL_SNAPSHOT_S3_|AKERNEL_SNAPSHOT_STORAGE_MODE|--env-file' \ + "${docker_argv_capture}"; then + fail "DataSystem standalone launch inherited S3-specific Docker arguments" +fi +grep -Fxq -- 'AKERNEL_SNAPSHOT_STORAGE_BACKEND=datasystem' "${docker_argv_capture}" \ + || fail "DataSystem standalone backend argument changed" + +# systemd must carry all host-facing inputs into the bootstrap shell. +pass_environment=$(sed -n 's/^PassEnvironment=//p' "${systemd_unit}" | tr '\n' ' ') +for variable in AKERNEL_SNAPSHOT_STORAGE_BACKEND AKERNEL_SNAPSHOT_STORAGE_MODE \ + AKERNEL_SNAPSHOT_S3_PROVIDER \ + AKERNEL_SNAPSHOT_S3_ENDPOINT AKERNEL_SNAPSHOT_S3_REGION AKERNEL_SNAPSHOT_S3_BUCKET \ + AKERNEL_SNAPSHOT_S3_ACCESS_KEY AKERNEL_SNAPSHOT_S3_SECRET_KEY \ + AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN AKERNEL_SNAPSHOT_S3_USE_HTTPS \ + AKERNEL_SNAPSHOT_S3_PATH_STYLE; do + [[ " ${pass_environment} " == *" ${variable} "* ]] \ + || fail "systemd omitted ${variable}" +done + +echo "standalone snapshot wiring contract passed" From 15380a770a6de3dad2d5fa19beca2d0f633ad88c Mon Sep 17 00:00:00 2001 From: ChamberlainJI Date: Wed, 2 Sep 2026 21:37:06 +0800 Subject: [PATCH 3/3] docs(snapshot): explain S3 storage rollout Signed-off-by: ChamberlainJI --- deploy/standalone/README.md | 98 ++++++++++++++++++++++++++++++++++--- 1 file changed, 91 insertions(+), 7 deletions(-) diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index b24912b..50b82e5 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -57,13 +57,97 @@ rather than tmpfs. Without `storage_mb`, runsc retains its configured memory-backed overlay while Firecracker uses its configured sparse ext4 default. -Sandbox checkpoints for runsc and Firecracker use YuanRong's local-only -snapshot mode. Checkpoint state is kept under the persistent -`/home/akernel/checkpoints` data mount. Workloads trigger an anonymous recovery -point through `POST /checkpoint` on `/run/akernel/rrt.sock`, and the SDK can -reload the same logical sandbox from the latest usable point. Recovery points -follow the source sandbox lifecycle; they are not exposed as reusable SDK -objects. +### Snapshot storage + +Sandbox checkpoints for runsc and Firecracker use the embedded YuanRong +DataSystem by default. With no snapshot environment overrides, `start.sh` +passes `AKERNEL_SNAPSHOT_STORAGE_BACKEND=datasystem`; the S3-only storage mode, +provider, endpoint and credentials are not emitted. + +Workloads trigger an anonymous recovery point through `POST /checkpoint` on +`/run/akernel/rrt.sock`, and the SDK can reload the same logical sandbox from +the latest usable point. Recovery points follow the source sandbox lifecycle; +they are not exposed as reusable SDK objects. + +Select the unified S3-compatible backend when snapshots must live in object +storage. S3 is always an explicit distributed mode: + +```bash +AKERNEL_SNAPSHOT_STORAGE_BACKEND=s3 \ +AKERNEL_SNAPSHOT_STORAGE_MODE=distributed_cache \ +AKERNEL_SNAPSHOT_S3_PROVIDER=generic \ +AKERNEL_SNAPSHOT_S3_ENDPOINT=minio.example.internal:9000 \ +AKERNEL_SNAPSHOT_S3_REGION=us-east-1 \ +AKERNEL_SNAPSHOT_S3_BUCKET=akernel-snapshots \ +AKERNEL_SNAPSHOT_S3_ACCESS_KEY='' \ +AKERNEL_SNAPSHOT_S3_SECRET_KEY='' \ +AKERNEL_SNAPSHOT_S3_USE_HTTPS=false \ +AKERNEL_SNAPSHOT_S3_PATH_STYLE=true \ +./start.sh +``` + +`AKERNEL_SNAPSHOT_STORAGE_MODE` accepts only `distributed_cache` or +`distributed_only` for S3 and defaults to `distributed_cache`. In cache mode, +the object is authoritative after publication while the local checkpoint is a +bounded restore cache. In distributed-only mode, the local directory exists +only while capturing, publishing, materializing or pinned for restore. +AKernel deliberately rejects `local_only` with `backend=s3`; omit all S3 +settings to keep the default embedded DataSystem behavior. + +The provider is `generic`, `obs`, or `oss`; it selects validation and +addressing defaults while every provider uses the same AWS Signature V4 S3 +protocol client. Private endpoints and CNAMEs are allowed. OSS requires +virtual-hosted addressing (`PATH_STYLE=false`). The optional +`AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN` carries an encrypted temporary token. +`provider=obs` is not the removed OBS-native backend and does not migrate +`AKERNEL_SNAPSHOT_OBS_*` configuration or existing objects. Those removed +variables are not accepted. + +The all-in-one image contains `/home/yuanrong/.akernel-s3-snapshot-capable` +only when its real openYuanRong package passes the build-time detector. The +detector requires all of the following evidence from the same package: + +- process config parses, validates and exports the six non-secret S3 options + plus credential environment; +- both FunctionSystem launch paths pass the six non-secret options without + credential argv; +- the executable FunctionAgent contains the S3 flags, credential environment, + 5 GiB guard and publication postcondition contract. + +The detector removes a stale marker before checking. If the marker is absent, +`AKERNEL_SNAPSHOT_STORAGE_BACKEND=s3` fails before producing YuanRong argv or +exporting credentials. Do not create the marker manually: it is package +capability evidence, not a user feature flag. Upgrade the all-in-one image as a +unit before enabling S3, and test rollback by confirming an old or incomplete +package removes the marker. + +For standalone S3, `start.sh` validates every value for CR/LF, writes all +snapshot environment to a mode-`0600` temporary env-file under +`${TMPDIR:-/tmp}`, and supplies Docker/Pouch with `--env-file`. The file is +removed by an EXIT/signal trap after container creation or on failure. This +keeps AK/SK/token out of the host process list and generated command output; +they remain visible to the container engine, host root and the FunctionAgent +process, which are therefore part of the credential trust boundary. + +`start.sh` uses the container engine directly when the caller has access. If +that probe fails, it uses only passwordless `sudo -n`; the same root-owned +engine reads the `0600` env-file by path. Interactive sudo is not attempted. +Failure to read the file, start the container or validate the S3 capability +causes startup to fail, and the cleanup trap still removes the file. + +Remote snapshots larger than 5 GiB are rejected before upload because this +version does not implement multipart CopyObject. `/home/akernel/checkpoints` +is the node's local staging directory; SDK checkpoint records have no automatic +TTL and remain until `Sandbox.delete_checkpoint()` is called. A restored +sandbox is a new sandbox and receives fresh network routes. + +One node configures one remote snapshot backend. Snapshot records freeze their +backend, and restore rejects a record whose backend does not match the current +FunctionAgent. Before switching from DataSystem to S3 or rolling back, stop new +checkpoint creation, wait for in-flight publish/restore/delete operations, and +restore or delete records that belong to the old backend. Do not remove the S3 +Secret or bucket while S3-backed records remain. There is no implicit dual-read, +cross-provider copy or destination atomic CAS. `start.sh` loads the host `tun` module and verifies `/dev/net/tun` before starting the pooled-TAP runtimes. Runc retains its separate veth network path.