feat(snapshot): add same-node recovery and S3 storage

.github/workflows/ci.yml

@@ -121,7 +121,11 @@ jobs:
           sudo modprobe loop
           sudo modprobe erofs
           sudo modprobe br_netfilter
+          sudo modprobe kvm
+          sudo modprobe kvm_intel || sudo modprobe kvm_amd || true
           test -c /dev/net/tun
+          test -c /dev/kvm
+          sudo chmod 0666 /dev/kvm
           sudo sysctl -w net.bridge.bridge-nf-call-iptables=1
 
       - name: Build all-in-one image
@@ -166,6 +170,25 @@ jobs:
             timeout 120s python "sdk/python/examples/${example}"
           done
 
+      - name: Run runsc and Firecracker checkpoint restore E2E
+        run: |
+          gateway_ip="$(docker inspect \
+            --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \
+            akernel-traefik)"
+          test -n "${gateway_ip}"
+          token="$(cat deploy/standalone/data/token)"
+          export AKERNEL_TOKEN="${token}"
+          export AKERNEL_SERVER_ADDRESS="${gateway_ip}"
+          export AKERNEL_RUN_INTEGRATION=1
+          export PYTHONPATH="${GITHUB_WORKSPACE}/sdk/python"
+
+          for runtime in runsc firecracker; do
+            echo "=== Checkpoint/restore runtime=${runtime} ==="
+            AKERNEL_TEST_RUNTIME="${runtime}" timeout 300s python \
+              sdk/python/tests/integration/test_sandbox.py \
+              SandboxCheckpointIntegrationTest -v
+          done
+
       - name: Show standalone diagnostics
         if: failure()
         run: |

AGENTS.md

@@ -368,6 +368,25 @@ quotas for runsc and Firecracker use this local-disk filestore. Without an
 explicit quota, runsc retains its configured memory-backed overlay while
 Firecracker creates its configured sparse ext4 default.
 
+The bundled node always enables YuanRong's sandbox snapshot data plane; there
+is no separate global pause/resume/snapshot switch. It uses the DataSystem
+backend by default and `/home/akernel/checkpoints` on the
+existing node home disk as same-node persistent staging. Remote storage uses
+only `snapshot_storage_backend=s3` with the shared SigV4 client and provider
+profiles `generic`, `obs`, or `oss`; the old snapshot OBS backend and
+`snapshot_obs_*` settings are unsupported. Provider profiles must allow legal
+private endpoints and CNAMEs, while OSS requires virtual-hosted addressing.
+Remote snapshots over 5 GiB are rejected before upload until multipart copy is
+implemented. Do not remove the independent DataSystem backend or the unrelated
+object-storage code-package downloader.
+Kubernetes node roles use the downward-API `NODE_NAME` as the stable YuanRong
+node identity, so a DaemonSet Pod replacement on the same physical node can
+rebuild its local runtime and snapshot views. Standalone falls back to the
+container hostname.
+The public SDK does not expose snapshot TTLs: reusable checkpoints remain
+until explicitly deleted. Keep the selected remote backend and checkpoint
+staging configuration together when changing node startup arguments.
+
 Keep detailed SDK reference material with the SDK. The root README should
 contain only the project-level entry points and representative examples:
 

README.md

@@ -207,7 +207,7 @@ See the complete [basic usage example](./sdk/python/examples/basic_usage.py), th
 - [x] Optional native Linux runc runtime
 - [x] Sandbox network ACL
 - [ ] Fork-based sandbox launch based on gVisor
-- [ ] Sandbox checkpoint and restore
+- [x] Sandbox checkpoint and restore for runsc and Firecracker
 - [ ] Support for GKE and AWS
 - [x] Cgroup v2 node support
 

builder/node.Dockerfile

@@ -394,6 +394,10 @@ COPY ./builder/scripts/yr_node_bootstrap.sh ${YR_INSTALLATION_DIR}/yr_node_boots
 COPY ./builder/scripts/master_entrypoint.sh ${YR_INSTALLATION_DIR}/entrypoint.sh
 COPY ./builder/scripts/*.sh /root/
 COPY ./builder/systemd_services/*.service /etc/systemd/system/
+RUN chmod 0755 \
+      /root/yr_pause_resume_args.sh \
+      /root/detect-openyuanrong-s3-snapshot-capability.sh && \
+    /root/detect-openyuanrong-s3-snapshot-capability.sh ${YR_INSTALLATION_DIR}
 
 RUN curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \
         "${OTELCOL_CONTRIB_URL}" \

builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2026 Ant Group Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+yr_root="${1:?usage: $0 YR_ROOT}"
+marker="${yr_root}/.akernel-s3-snapshot-capable"
+config="${yr_root}/deploy/process/config.sh"
+install="${yr_root}/functionsystem/deploy/install.sh"
+agent="${yr_root}/functionsystem/bin/function_agent"
+
+rm -f "${marker}"
+if [[ -f "${config}" && -f "${install}" && -x "${agent}" ]] \
+  && grep -Fq 'snapshot_s3_provider:' "${config}" \
+  && grep -Fq -- '--snapshot_s3_provider="${SNAPSHOT_S3_PROVIDER:-}"' "${install}" \
+  && grep -aFq 'snapshot_s3_provider' "${agent}" \
+  && grep -aFq 'remote S3 snapshot exceeds the 5 GiB capability limit' "${agent}"; then
+  touch "${marker}"
+fi

builder/scripts/tests/test-openyuanrong-s3-capability.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2026 Ant Group Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)
+detector="${root}/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh"
+tmp=$(mktemp -d)
+trap 'rm -rf "${tmp}"' EXIT
+mkdir -p "${tmp}/deploy/process" "${tmp}/functionsystem/deploy" "${tmp}/functionsystem/bin"
+printf '%s\n' 'snapshot_s3_provider:' >"${tmp}/deploy/process/config.sh"
+printf '%s\n' '--snapshot_s3_provider="${SNAPSHOT_S3_PROVIDER:-}"' \
+  >"${tmp}/functionsystem/deploy/install.sh"
+printf '%s\n' 'snapshot_s3_provider' 'remote S3 snapshot exceeds the 5 GiB capability limit' \
+  >"${tmp}/functionsystem/bin/function_agent"
+chmod +x "${tmp}/functionsystem/bin/function_agent"
+
+"${detector}" "${tmp}"
+[[ -f "${tmp}/.akernel-s3-snapshot-capable" ]]
+
+printf '%s\n' 'legacy function agent' >"${tmp}/functionsystem/bin/function_agent"
+"${detector}" "${tmp}"
+[[ ! -e "${tmp}/.akernel-s3-snapshot-capable" ]]
+
+echo "openYuanRong S3 capability detection checks passed"

builder/scripts/yr_node_bootstrap.sh

@@ -5,6 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0
 ulimit -n 32768
 export YR_RUNTIME_BACKEND=sandboxd
+source /root/yr_pause_resume_args.sh
 
 resolve_node_ip() {
     local default_device
@@ -41,6 +42,13 @@ resolve_node_ip() {
 
 YR_NODE_IP="$(resolve_node_ip)"
 echo "Using ${YR_NODE_IP} as the YuanRong node address"
+CHECKPOINT_DIR="/home/akernel/checkpoints"
+mkdir -p "${CHECKPOINT_DIR}"
+configure_snapshot_args \
+    /home/yuanrong/.akernel-rrt-capable \
+    "${CHECKPOINT_DIR}" \
+    "${AKS_LOCAL_MODE:-false}" \
+    /home/yuanrong/.akernel-s3-snapshot-capable || exit 1
 
 # Select the legacy etcd registry or the FunctionMaster HTTP provider.
 if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then
@@ -108,7 +116,8 @@ if [  "x${AKS_LOCAL_MODE}" == "xtrue" ]; then
         --frontend_lease_bypass true \
         --force_low_reliability_instance true \
         --enable_sandbox_router true \
-        --enable_direct_routing false
+        --enable_direct_routing false \
+        "${standalone_snapshot_args[@]}"
 else
     /usr/bin/yr start \
         --ip_address "${YR_NODE_IP}" \
@@ -130,7 +139,7 @@ else
         --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \
         --enable_trace ${ENABLE_TRACE} \
         --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \
-        -n ${HOSTNAME} \
+        -n "${NODE_NAME:-${HOSTNAME}}" \
         --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \
         --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \
         --traefik_etcd_prefix=traefik \
@@ -145,5 +154,6 @@ else
         --function_proxy_merge_process_enable true \
         --enable_direct_routing false \
         --force_low_reliability_instance true \
+        "${snapshot_args[@]}" \
         --block true
 fi

builder/scripts/yr_pause_resume_args.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Copyright (c) 2026 Ant Group Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+configure_snapshot_args() {
+    local rrt_capability_file="${1:?RRT capability file is required}"
+    local checkpoint_dir="${2:?checkpoint directory is required}"
+    local standalone="${3:?standalone mode value is required}"
+    local s3_capability_file="${4:-/home/yuanrong/.akernel-s3-snapshot-capable}"
+    local backend="${AKERNEL_SNAPSHOT_STORAGE_BACKEND:-datasystem}"
+
+    snapshot_args=()
+    standalone_snapshot_args=()
+    unset SNAPSHOT_S3_ACCESS_KEY SNAPSHOT_S3_SECRET_KEY SNAPSHOT_S3_SECURITY_TOKEN
+    if [ ! -f "${rrt_capability_file}" ] \
+        && [ ! -f /home/yuanrong/yr-runtime-rootfs.img ]; then
+        echo "snapshot requires an image built with the RRT runtime" >&2
+        return 1
+    fi
+    mkdir -p "${checkpoint_dir}"
+    if [ ! -w "${checkpoint_dir}" ]; then
+        echo "checkpoint directory is not writable: ${checkpoint_dir}" >&2
+        return 1
+    fi
+    snapshot_args=(
+        --snapshot_storage_backend "${backend}"
+        --checkpoint_dir "${checkpoint_dir}"
+    )
+    case "${backend}" in
+        datasystem) ;;
+        s3)
+            if [ ! -f "${s3_capability_file}" ]; then
+                echo "S3 snapshot storage requires an S3-capable openYuanRong core" >&2
+                return 1
+            fi
+            local provider="${AKERNEL_SNAPSHOT_S3_PROVIDER:-}"
+            local endpoint="${AKERNEL_SNAPSHOT_S3_ENDPOINT:-}"
+            local region="${AKERNEL_SNAPSHOT_S3_REGION:-}"
+            local bucket="${AKERNEL_SNAPSHOT_S3_BUCKET:-}"
+            local access_key="${AKERNEL_SNAPSHOT_S3_ACCESS_KEY:-}"
+            local secret_key="${AKERNEL_SNAPSHOT_S3_SECRET_KEY:-}"
+            local security_token="${AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN:-}"
+            local use_https="${AKERNEL_SNAPSHOT_S3_USE_HTTPS:-true}"
+            local path_style="${AKERNEL_SNAPSHOT_S3_PATH_STYLE:-true}"
+            case "${provider}" in generic|obs|oss) ;; *)
+                echo "AKERNEL_SNAPSHOT_S3_PROVIDER must be generic, obs, or oss" >&2
+                return 1
+            esac
+            if [ -z "${endpoint}" ] || [ -z "${region}" ] || [ -z "${bucket}" ] || \
+               [ -z "${access_key}" ] || [ -z "${secret_key}" ]; then
+                echo "S3 snapshot storage requires endpoint, region, bucket, access key, and secret key" >&2
+                return 1
+            fi
+            case "${use_https}" in true|false) ;; *)
+                echo "AKERNEL_SNAPSHOT_S3_USE_HTTPS must be true or false" >&2
+                return 1
+            esac
+            case "${path_style}" in true|false) ;; *)
+                echo "AKERNEL_SNAPSHOT_S3_PATH_STYLE must be true or false" >&2
+                return 1
+            esac
+            if [ "${provider}" = "oss" ] && [ "${path_style}" = "true" ]; then
+                echo "OSS S3-compatible snapshot storage requires virtual-hosted addressing" >&2
+                return 1
+            fi
+            snapshot_args+=(
+                --snapshot_s3_provider "${provider}"
+                --snapshot_s3_endpoint "${endpoint}"
+                --snapshot_s3_region "${region}"
+                --snapshot_s3_bucket "${bucket}"
+                --snapshot_s3_use_https "${use_https}"
+                --snapshot_s3_path_style "${path_style}"
+            )
+            export SNAPSHOT_S3_ACCESS_KEY="${access_key}"
+            export SNAPSHOT_S3_SECRET_KEY="${secret_key}"
+            export SNAPSHOT_S3_SECURITY_TOKEN="${security_token}"
+            ;;
+        *)
+            echo "AKERNEL_SNAPSHOT_STORAGE_BACKEND must be datasystem or s3" >&2
+            return 1
+            ;;
+    esac
+    standalone_snapshot_args=("${snapshot_args[@]}")
+    case "${standalone}" in
+        true) standalone_snapshot_args+=(--data_system_enable true) ;;
+        false) ;;
+        *) echo "AKS_LOCAL_MODE must be true or false" >&2; return 1 ;;
+    esac
+}

builder/systemd_services/yuanrong.service

@@ -4,7 +4,9 @@ Description=yuanrong.service
 [Service]
 #Type=simple
 PIDFile=/run/yuanrong.pid
-PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT
+PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME NODE_NAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT YR_RRT_CONTROL_SOCKET_PATH
+PassEnvironment=AKERNEL_SNAPSHOT_STORAGE_BACKEND
+PassEnvironment=AKERNEL_SNAPSHOT_S3_PROVIDER AKERNEL_SNAPSHOT_S3_ENDPOINT AKERNEL_SNAPSHOT_S3_REGION AKERNEL_SNAPSHOT_S3_BUCKET AKERNEL_SNAPSHOT_S3_ACCESS_KEY AKERNEL_SNAPSHOT_S3_SECRET_KEY AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN AKERNEL_SNAPSHOT_S3_USE_HTTPS AKERNEL_SNAPSHOT_S3_PATH_STYLE
 Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock"
 Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime"
 Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1"

deploy/README.md

@@ -207,6 +207,34 @@ image:
 Each component can still override `master.image`, `frontend.image`, or
 `node.image` when a split-image deployment is required.
 
+### Snapshot object storage
+
+The core chart keeps the existing DataSystem snapshot backend by default. To
+use S3-compatible object storage, create a Secret containing encrypted
+credentials and configure the single S3 backend:
+
+```yaml
+core:
+  snapshot:
+    storage:
+      backend: s3
+      s3:
+        provider: generic # generic, obs, or oss
+        endpoint: minio.storage.svc:9000
+        region: us-east-1
+        bucket: akernel-snapshots
+        existingSecret: akernel-snapshot-s3
+        useHttps: false
+        pathStyle: true
+```
+
+The Secret keys default to `access-key`, `secret-key`, and the optional
+`security-token`. All providers share the same AWS Signature V4 client;
+profiles do not select provider SDKs or restrict valid private endpoints and
+CNAMEs. OSS requires virtual-hosted addressing. The removed `backend=obs` and
+`snapshot_obs_*` configuration are not accepted. Remote snapshots larger than
+5 GiB fail before upload because multipart CopyObject is not supported yet.
+
 ### Public Traefik entrypoints
 
 For cloud deployments, use Traefik with two public entrypoints:

deploy/akernel/charts/core/templates/node/daemonset.yaml

@@ -1,3 +1,20 @@
+{{- $snapshotBackend := .Values.snapshot.storage.backend -}}
+{{- if not (has $snapshotBackend (list "datasystem" "s3")) -}}
+{{- fail "snapshot.storage.backend must be datasystem or s3" -}}
+{{- end -}}
+{{- if eq $snapshotBackend "s3" -}}
+{{- $snapshotProvider := .Values.snapshot.storage.s3.provider -}}
+{{- if not (has $snapshotProvider (list "generic" "obs" "oss")) -}}
+{{- fail "snapshot.storage.s3.provider must be generic, obs, or oss" -}}
+{{- end -}}
+{{- if and (eq $snapshotProvider "oss") .Values.snapshot.storage.s3.pathStyle -}}
+{{- fail "OSS S3-compatible storage requires virtual-hosted addressing" -}}
+{{- end -}}
+{{- $snapshotEndpoint := required "snapshot.storage.s3.endpoint is required" .Values.snapshot.storage.s3.endpoint -}}
+{{- $snapshotRegion := required "snapshot.storage.s3.region is required" .Values.snapshot.storage.s3.region -}}
+{{- $snapshotBucket := required "snapshot.storage.s3.bucket is required" .Values.snapshot.storage.s3.bucket -}}
+{{- $snapshotSecret := required "snapshot.storage.s3.existingSecret is required" .Values.snapshot.storage.s3.existingSecret -}}
+{{- end -}}
 {{- if .Values.kruise.enabled }}
 apiVersion: apps.kruise.io/v1alpha1
 {{- else }}
@@ -80,6 +97,42 @@ spec:
           value: "node"
         - name: RUNSC_AKERNEL
           value: "1"
+        {{- if .Values.rrtControlSocketPath }}
+        - name: YR_RRT_CONTROL_SOCKET_PATH
+          value: {{ .Values.rrtControlSocketPath | quote }}
+        {{- end }}
+        - name: AKERNEL_SNAPSHOT_STORAGE_BACKEND
+          value: {{ .Values.snapshot.storage.backend | quote }}
+        {{- if eq .Values.snapshot.storage.backend "s3" }}
+        - name: AKERNEL_SNAPSHOT_S3_PROVIDER
+          value: {{ .Values.snapshot.storage.s3.provider | quote }}
+        - name: AKERNEL_SNAPSHOT_S3_ENDPOINT
+          value: {{ .Values.snapshot.storage.s3.endpoint | quote }}
+        - name: AKERNEL_SNAPSHOT_S3_REGION
+          value: {{ .Values.snapshot.storage.s3.region | quote }}
+        - name: AKERNEL_SNAPSHOT_S3_BUCKET
+          value: {{ .Values.snapshot.storage.s3.bucket | quote }}
+        - name: AKERNEL_SNAPSHOT_S3_USE_HTTPS
+          value: {{ .Values.snapshot.storage.s3.useHttps | quote }}
+        - name: AKERNEL_SNAPSHOT_S3_PATH_STYLE
+          value: {{ .Values.snapshot.storage.s3.pathStyle | quote }}
+        - name: AKERNEL_SNAPSHOT_S3_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.snapshot.storage.s3.existingSecret | quote }}
+              key: {{ .Values.snapshot.storage.s3.accessKeyKey | quote }}
+        - name: AKERNEL_SNAPSHOT_S3_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.snapshot.storage.s3.existingSecret | quote }}
+              key: {{ .Values.snapshot.storage.s3.secretKeyKey | quote }}
+        - name: AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.snapshot.storage.s3.existingSecret | quote }}
+              key: {{ .Values.snapshot.storage.s3.securityTokenKey | quote }}
+              optional: true
+        {{- end }}
         - name: ETCD_ADDRESS
           value: {{ get $nodeEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) | quote }}
         - name: ETCD_PORT