From 81b40a95dad77e9096f64a9b512144b9719b61be Mon Sep 17 00:00:00 2001 From: ChamberlainJI Date: Fri, 28 Aug 2026 14:30:25 +0800 Subject: [PATCH] feat(snapshot): add checkpoint and same-node recovery Expose checkpoint, restore, reload, pause, resume, and failover through the SDK and deployment surfaces. Package the synchronized YuanRong and sandboxd runtime stack, enable local checkpoint control paths, add S3 snapshot configuration, and preserve snapshot resource inheritance. Signed-off-by: ChamberlainJI --- .github/workflows/ci.yml | 23 ++ AGENTS.md | 19 ++ README.md | 2 +- builder/node.Dockerfile | 4 + ...ect-openyuanrong-s3-snapshot-capability.sh | 21 ++ .../tests/test-openyuanrong-s3-capability.sh | 27 +++ builder/scripts/yr_node_bootstrap.sh | 14 +- builder/scripts/yr_pause_resume_args.sh | 90 +++++++ builder/systemd_services/yuanrong.service | 4 +- deploy/README.md | 28 +++ .../charts/core/templates/node/daemonset.yaml | 53 ++++ deploy/akernel/charts/core/values.yaml | 30 ++- .../akernel/tests/test-pause-resume-wiring.sh | 66 +++++ deploy/akernel/values.yaml | 14 ++ deploy/standalone/README.md | 37 +++ deploy/standalone/config/config.json | 9 - deploy/standalone/start.sh | 21 ++ .../tests/test-pause-resume-wiring.sh | 57 +++++ sdk/python/README.md | 82 +++++++ sdk/python/akernel_sdk/__init__.py | 2 + sdk/python/akernel_sdk/_backends/base.py | 17 ++ .../_backends/openyuanrong_sandbox.py | 133 +++++++++- .../akernel_sdk/_backends/openyuanrong_sdk.py | 57 ++++- .../_backends/openyuanrong_sdk_impl.py | 9 + sdk/python/akernel_sdk/sandbox.py | 147 +++++++++++- sdk/python/akernel_sdk/types.py | 15 ++ sdk/python/examples/checkpoint_restore.py | 39 +++ sdk/python/pyproject.toml | 2 +- sdk/python/tests/integration/test_sandbox.py | 89 +++++++ sdk/python/tests/unit/test_backends.py | 227 ++++++++++++++++++ .../tests/unit/test_openyuanrong_sdk_impl.py | 6 + sdk/python/tests/unit/test_sandbox.py | 116 +++++++++ sdk/python/tests/unit/test_types.py | 17 +- src/sandboxd | 2 +- src/yuanrong | 2 +- 35 files changed, 1450 insertions(+), 31 deletions(-) create mode 100755 builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh create mode 100755 builder/scripts/tests/test-openyuanrong-s3-capability.sh create mode 100755 builder/scripts/yr_pause_resume_args.sh create mode 100755 deploy/akernel/tests/test-pause-resume-wiring.sh create mode 100755 deploy/standalone/tests/test-pause-resume-wiring.sh create mode 100644 sdk/python/examples/checkpoint_restore.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e6c682..b24c993 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,11 @@ jobs: sudo modprobe loop sudo modprobe erofs sudo modprobe br_netfilter + sudo modprobe kvm + sudo modprobe kvm_intel || sudo modprobe kvm_amd || true test -c /dev/net/tun + test -c /dev/kvm + sudo chmod 0666 /dev/kvm sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 - name: Build all-in-one image @@ -166,6 +170,25 @@ jobs: timeout 120s python "sdk/python/examples/${example}" done + - name: Run runsc and Firecracker checkpoint restore E2E + run: | + gateway_ip="$(docker inspect \ + --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ + akernel-traefik)" + test -n "${gateway_ip}" + token="$(cat deploy/standalone/data/token)" + export AKERNEL_TOKEN="${token}" + export AKERNEL_SERVER_ADDRESS="${gateway_ip}" + export AKERNEL_RUN_INTEGRATION=1 + export PYTHONPATH="${GITHUB_WORKSPACE}/sdk/python" + + for runtime in runsc firecracker; do + echo "=== Checkpoint/restore runtime=${runtime} ===" + AKERNEL_TEST_RUNTIME="${runtime}" timeout 300s python \ + sdk/python/tests/integration/test_sandbox.py \ + SandboxCheckpointIntegrationTest -v + done + - name: Show standalone diagnostics if: failure() run: | diff --git a/AGENTS.md b/AGENTS.md index 7fdbfed..f1db140 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -368,6 +368,25 @@ quotas for runsc and Firecracker use this local-disk filestore. Without an explicit quota, runsc retains its configured memory-backed overlay while Firecracker creates its configured sparse ext4 default. +The bundled node always enables YuanRong's sandbox snapshot data plane; there +is no separate global pause/resume/snapshot switch. It uses the DataSystem +backend by default and `/home/akernel/checkpoints` on the +existing node home disk as same-node persistent staging. Remote storage uses +only `snapshot_storage_backend=s3` with the shared SigV4 client and provider +profiles `generic`, `obs`, or `oss`; the old snapshot OBS backend and +`snapshot_obs_*` settings are unsupported. Provider profiles must allow legal +private endpoints and CNAMEs, while OSS requires virtual-hosted addressing. +Remote snapshots over 5 GiB are rejected before upload until multipart copy is +implemented. Do not remove the independent DataSystem backend or the unrelated +object-storage code-package downloader. +Kubernetes node roles use the downward-API `NODE_NAME` as the stable YuanRong +node identity, so a DaemonSet Pod replacement on the same physical node can +rebuild its local runtime and snapshot views. Standalone falls back to the +container hostname. +The public SDK does not expose snapshot TTLs: reusable checkpoints remain +until explicitly deleted. Keep the selected remote backend and checkpoint +staging configuration together when changing node startup arguments. + Keep detailed SDK reference material with the SDK. The root README should contain only the project-level entry points and representative examples: diff --git a/README.md b/README.md index aa78185..9ed91ce 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ See the complete [basic usage example](./sdk/python/examples/basic_usage.py), th - [x] Optional native Linux runc runtime - [x] Sandbox network ACL - [ ] Fork-based sandbox launch based on gVisor -- [ ] Sandbox checkpoint and restore +- [x] Sandbox checkpoint and restore for runsc and Firecracker - [ ] Support for GKE and AWS - [x] Cgroup v2 node support diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 4b8404a..a1d6d61 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -394,6 +394,10 @@ COPY ./builder/scripts/yr_node_bootstrap.sh ${YR_INSTALLATION_DIR}/yr_node_boots COPY ./builder/scripts/master_entrypoint.sh ${YR_INSTALLATION_DIR}/entrypoint.sh COPY ./builder/scripts/*.sh /root/ COPY ./builder/systemd_services/*.service /etc/systemd/system/ +RUN chmod 0755 \ + /root/yr_pause_resume_args.sh \ + /root/detect-openyuanrong-s3-snapshot-capability.sh && \ + /root/detect-openyuanrong-s3-snapshot-capability.sh ${YR_INSTALLATION_DIR} RUN curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ "${OTELCOL_CONTRIB_URL}" \ diff --git a/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh b/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh new file mode 100755 index 0000000..1b64ee8 --- /dev/null +++ b/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +yr_root="${1:?usage: $0 YR_ROOT}" +marker="${yr_root}/.akernel-s3-snapshot-capable" +config="${yr_root}/deploy/process/config.sh" +install="${yr_root}/functionsystem/deploy/install.sh" +agent="${yr_root}/functionsystem/bin/function_agent" + +rm -f "${marker}" +if [[ -f "${config}" && -f "${install}" && -x "${agent}" ]] \ + && grep -Fq 'snapshot_s3_provider:' "${config}" \ + && grep -Fq -- '--snapshot_s3_provider="${SNAPSHOT_S3_PROVIDER:-}"' "${install}" \ + && grep -aFq 'snapshot_s3_provider' "${agent}" \ + && grep -aFq 'remote S3 snapshot exceeds the 5 GiB capability limit' "${agent}"; then + touch "${marker}" +fi diff --git a/builder/scripts/tests/test-openyuanrong-s3-capability.sh b/builder/scripts/tests/test-openyuanrong-s3-capability.sh new file mode 100755 index 0000000..6801df9 --- /dev/null +++ b/builder/scripts/tests/test-openyuanrong-s3-capability.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +detector="${root}/builder/scripts/detect-openyuanrong-s3-snapshot-capability.sh" +tmp=$(mktemp -d) +trap 'rm -rf "${tmp}"' EXIT +mkdir -p "${tmp}/deploy/process" "${tmp}/functionsystem/deploy" "${tmp}/functionsystem/bin" +printf '%s\n' 'snapshot_s3_provider:' >"${tmp}/deploy/process/config.sh" +printf '%s\n' '--snapshot_s3_provider="${SNAPSHOT_S3_PROVIDER:-}"' \ + >"${tmp}/functionsystem/deploy/install.sh" +printf '%s\n' 'snapshot_s3_provider' 'remote S3 snapshot exceeds the 5 GiB capability limit' \ + >"${tmp}/functionsystem/bin/function_agent" +chmod +x "${tmp}/functionsystem/bin/function_agent" + +"${detector}" "${tmp}" +[[ -f "${tmp}/.akernel-s3-snapshot-capable" ]] + +printf '%s\n' 'legacy function agent' >"${tmp}/functionsystem/bin/function_agent" +"${detector}" "${tmp}" +[[ ! -e "${tmp}/.akernel-s3-snapshot-capable" ]] + +echo "openYuanRong S3 capability detection checks passed" diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index 52e5e21..6f25e59 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -5,6 +5,7 @@ # SPDX-License-Identifier: Apache-2.0 ulimit -n 32768 export YR_RUNTIME_BACKEND=sandboxd +source /root/yr_pause_resume_args.sh resolve_node_ip() { local default_device @@ -41,6 +42,13 @@ resolve_node_ip() { YR_NODE_IP="$(resolve_node_ip)" echo "Using ${YR_NODE_IP} as the YuanRong node address" +CHECKPOINT_DIR="/home/akernel/checkpoints" +mkdir -p "${CHECKPOINT_DIR}" +configure_snapshot_args \ + /home/yuanrong/.akernel-rrt-capable \ + "${CHECKPOINT_DIR}" \ + "${AKS_LOCAL_MODE:-false}" \ + /home/yuanrong/.akernel-s3-snapshot-capable || exit 1 # Select the legacy etcd registry or the FunctionMaster HTTP provider. if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then @@ -108,7 +116,8 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --frontend_lease_bypass true \ --force_low_reliability_instance true \ --enable_sandbox_router true \ - --enable_direct_routing false + --enable_direct_routing false \ + "${standalone_snapshot_args[@]}" else /usr/bin/yr start \ --ip_address "${YR_NODE_IP}" \ @@ -130,7 +139,7 @@ else --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ --enable_trace ${ENABLE_TRACE} \ --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ - -n ${HOSTNAME} \ + -n "${NODE_NAME:-${HOSTNAME}}" \ --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ --traefik_etcd_prefix=traefik \ @@ -145,5 +154,6 @@ else --function_proxy_merge_process_enable true \ --enable_direct_routing false \ --force_low_reliability_instance true \ + "${snapshot_args[@]}" \ --block true fi diff --git a/builder/scripts/yr_pause_resume_args.sh b/builder/scripts/yr_pause_resume_args.sh new file mode 100755 index 0000000..67c78c0 --- /dev/null +++ b/builder/scripts/yr_pause_resume_args.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +configure_snapshot_args() { + local rrt_capability_file="${1:?RRT capability file is required}" + local checkpoint_dir="${2:?checkpoint directory is required}" + local standalone="${3:?standalone mode value is required}" + local s3_capability_file="${4:-/home/yuanrong/.akernel-s3-snapshot-capable}" + local backend="${AKERNEL_SNAPSHOT_STORAGE_BACKEND:-datasystem}" + + snapshot_args=() + standalone_snapshot_args=() + unset SNAPSHOT_S3_ACCESS_KEY SNAPSHOT_S3_SECRET_KEY SNAPSHOT_S3_SECURITY_TOKEN + if [ ! -f "${rrt_capability_file}" ] \ + && [ ! -f /home/yuanrong/yr-runtime-rootfs.img ]; then + echo "snapshot requires an image built with the RRT runtime" >&2 + return 1 + fi + mkdir -p "${checkpoint_dir}" + if [ ! -w "${checkpoint_dir}" ]; then + echo "checkpoint directory is not writable: ${checkpoint_dir}" >&2 + return 1 + fi + snapshot_args=( + --snapshot_storage_backend "${backend}" + --checkpoint_dir "${checkpoint_dir}" + ) + case "${backend}" in + datasystem) ;; + s3) + if [ ! -f "${s3_capability_file}" ]; then + echo "S3 snapshot storage requires an S3-capable openYuanRong core" >&2 + return 1 + fi + local provider="${AKERNEL_SNAPSHOT_S3_PROVIDER:-}" + local endpoint="${AKERNEL_SNAPSHOT_S3_ENDPOINT:-}" + local region="${AKERNEL_SNAPSHOT_S3_REGION:-}" + local bucket="${AKERNEL_SNAPSHOT_S3_BUCKET:-}" + local access_key="${AKERNEL_SNAPSHOT_S3_ACCESS_KEY:-}" + local secret_key="${AKERNEL_SNAPSHOT_S3_SECRET_KEY:-}" + local security_token="${AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN:-}" + local use_https="${AKERNEL_SNAPSHOT_S3_USE_HTTPS:-true}" + local path_style="${AKERNEL_SNAPSHOT_S3_PATH_STYLE:-true}" + case "${provider}" in generic|obs|oss) ;; *) + echo "AKERNEL_SNAPSHOT_S3_PROVIDER must be generic, obs, or oss" >&2 + return 1 + esac + if [ -z "${endpoint}" ] || [ -z "${region}" ] || [ -z "${bucket}" ] || \ + [ -z "${access_key}" ] || [ -z "${secret_key}" ]; then + echo "S3 snapshot storage requires endpoint, region, bucket, access key, and secret key" >&2 + return 1 + fi + case "${use_https}" in true|false) ;; *) + echo "AKERNEL_SNAPSHOT_S3_USE_HTTPS must be true or false" >&2 + return 1 + esac + case "${path_style}" in true|false) ;; *) + echo "AKERNEL_SNAPSHOT_S3_PATH_STYLE must be true or false" >&2 + return 1 + esac + if [ "${provider}" = "oss" ] && [ "${path_style}" = "true" ]; then + echo "OSS S3-compatible snapshot storage requires virtual-hosted addressing" >&2 + return 1 + fi + snapshot_args+=( + --snapshot_s3_provider "${provider}" + --snapshot_s3_endpoint "${endpoint}" + --snapshot_s3_region "${region}" + --snapshot_s3_bucket "${bucket}" + --snapshot_s3_use_https "${use_https}" + --snapshot_s3_path_style "${path_style}" + ) + export SNAPSHOT_S3_ACCESS_KEY="${access_key}" + export SNAPSHOT_S3_SECRET_KEY="${secret_key}" + export SNAPSHOT_S3_SECURITY_TOKEN="${security_token}" + ;; + *) + echo "AKERNEL_SNAPSHOT_STORAGE_BACKEND must be datasystem or s3" >&2 + return 1 + ;; + esac + standalone_snapshot_args=("${snapshot_args[@]}") + case "${standalone}" in + true) standalone_snapshot_args+=(--data_system_enable true) ;; + false) ;; + *) echo "AKS_LOCAL_MODE must be true or false" >&2; return 1 ;; + esac +} diff --git a/builder/systemd_services/yuanrong.service b/builder/systemd_services/yuanrong.service index 3960526..38070cd 100644 --- a/builder/systemd_services/yuanrong.service +++ b/builder/systemd_services/yuanrong.service @@ -4,7 +4,9 @@ Description=yuanrong.service [Service] #Type=simple PIDFile=/run/yuanrong.pid -PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT +PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME NODE_NAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT YR_RRT_CONTROL_SOCKET_PATH +PassEnvironment=AKERNEL_SNAPSHOT_STORAGE_BACKEND +PassEnvironment=AKERNEL_SNAPSHOT_S3_PROVIDER AKERNEL_SNAPSHOT_S3_ENDPOINT AKERNEL_SNAPSHOT_S3_REGION AKERNEL_SNAPSHOT_S3_BUCKET AKERNEL_SNAPSHOT_S3_ACCESS_KEY AKERNEL_SNAPSHOT_S3_SECRET_KEY AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN AKERNEL_SNAPSHOT_S3_USE_HTTPS AKERNEL_SNAPSHOT_S3_PATH_STYLE Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock" Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime" Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1" diff --git a/deploy/README.md b/deploy/README.md index afaedf4..c14ef6a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -207,6 +207,34 @@ image: Each component can still override `master.image`, `frontend.image`, or `node.image` when a split-image deployment is required. +### Snapshot object storage + +The core chart keeps the existing DataSystem snapshot backend by default. To +use S3-compatible object storage, create a Secret containing encrypted +credentials and configure the single S3 backend: + +```yaml +core: + snapshot: + storage: + backend: s3 + s3: + provider: generic # generic, obs, or oss + endpoint: minio.storage.svc:9000 + region: us-east-1 + bucket: akernel-snapshots + existingSecret: akernel-snapshot-s3 + useHttps: false + pathStyle: true +``` + +The Secret keys default to `access-key`, `secret-key`, and the optional +`security-token`. All providers share the same AWS Signature V4 client; +profiles do not select provider SDKs or restrict valid private endpoints and +CNAMEs. OSS requires virtual-hosted addressing. The removed `backend=obs` and +`snapshot_obs_*` configuration are not accepted. Remote snapshots larger than +5 GiB fail before upload because multipart CopyObject is not supported yet. + ### Public Traefik entrypoints For cloud deployments, use Traefik with two public entrypoints: diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index 36b8443..40f3bba 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -1,3 +1,20 @@ +{{- $snapshotBackend := .Values.snapshot.storage.backend -}} +{{- if not (has $snapshotBackend (list "datasystem" "s3")) -}} +{{- fail "snapshot.storage.backend must be datasystem or s3" -}} +{{- end -}} +{{- if eq $snapshotBackend "s3" -}} +{{- $snapshotProvider := .Values.snapshot.storage.s3.provider -}} +{{- if not (has $snapshotProvider (list "generic" "obs" "oss")) -}} +{{- fail "snapshot.storage.s3.provider must be generic, obs, or oss" -}} +{{- end -}} +{{- if and (eq $snapshotProvider "oss") .Values.snapshot.storage.s3.pathStyle -}} +{{- fail "OSS S3-compatible storage requires virtual-hosted addressing" -}} +{{- end -}} +{{- $snapshotEndpoint := required "snapshot.storage.s3.endpoint is required" .Values.snapshot.storage.s3.endpoint -}} +{{- $snapshotRegion := required "snapshot.storage.s3.region is required" .Values.snapshot.storage.s3.region -}} +{{- $snapshotBucket := required "snapshot.storage.s3.bucket is required" .Values.snapshot.storage.s3.bucket -}} +{{- $snapshotSecret := required "snapshot.storage.s3.existingSecret is required" .Values.snapshot.storage.s3.existingSecret -}} +{{- end -}} {{- if .Values.kruise.enabled }} apiVersion: apps.kruise.io/v1alpha1 {{- else }} @@ -80,6 +97,42 @@ spec: value: "node" - name: RUNSC_AKERNEL value: "1" + {{- if .Values.rrtControlSocketPath }} + - name: YR_RRT_CONTROL_SOCKET_PATH + value: {{ .Values.rrtControlSocketPath | quote }} + {{- end }} + - name: AKERNEL_SNAPSHOT_STORAGE_BACKEND + value: {{ .Values.snapshot.storage.backend | quote }} + {{- if eq .Values.snapshot.storage.backend "s3" }} + - name: AKERNEL_SNAPSHOT_S3_PROVIDER + value: {{ .Values.snapshot.storage.s3.provider | quote }} + - name: AKERNEL_SNAPSHOT_S3_ENDPOINT + value: {{ .Values.snapshot.storage.s3.endpoint | quote }} + - name: AKERNEL_SNAPSHOT_S3_REGION + value: {{ .Values.snapshot.storage.s3.region | quote }} + - name: AKERNEL_SNAPSHOT_S3_BUCKET + value: {{ .Values.snapshot.storage.s3.bucket | quote }} + - name: AKERNEL_SNAPSHOT_S3_USE_HTTPS + value: {{ .Values.snapshot.storage.s3.useHttps | quote }} + - name: AKERNEL_SNAPSHOT_S3_PATH_STYLE + value: {{ .Values.snapshot.storage.s3.pathStyle | quote }} + - name: AKERNEL_SNAPSHOT_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.snapshot.storage.s3.existingSecret | quote }} + key: {{ .Values.snapshot.storage.s3.accessKeyKey | quote }} + - name: AKERNEL_SNAPSHOT_S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.snapshot.storage.s3.existingSecret | quote }} + key: {{ .Values.snapshot.storage.s3.secretKeyKey | quote }} + - name: AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.snapshot.storage.s3.existingSecret | quote }} + key: {{ .Values.snapshot.storage.s3.securityTokenKey | quote }} + optional: true + {{- end }} - name: ETCD_ADDRESS value: {{ get $nodeEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) | quote }} - name: ETCD_PORT diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index 6dfd158..6d81bab 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -23,6 +23,27 @@ image: tag: "latest" pullPolicy: IfNotPresent +# Directory exposed inside RRT sandboxes for the optional control socket. +# Empty disables the socket and the in-sandbox POST /checkpoint endpoint. +rrtControlSocketPath: "/run/openyuanrong" + +# Snapshot-backed sandbox lifecycle data plane. +snapshot: + storage: + backend: datasystem + s3: + # Supported provider profiles: generic, obs, oss. + provider: generic + endpoint: "" + region: us-east-1 + bucket: "" + existingSecret: "" + accessKeyKey: access-key + secretKeyKey: secret-key + securityTokenKey: security-token + useHttps: true + pathStyle: true + auth: # Existing Secret that contains the JWT signing seed. For # `helm template | kubectl apply`, prefer pre-creating this Secret with @@ -379,15 +400,6 @@ node: "noexec", "nodev" ] - }, - { - "destination": "/etc/resolv.conf", - "type": "bind", - "source": "/etc/resolv_akernel.conf", - "options": [ - "bind", - "ro" - ] } ], "linux": { diff --git a/deploy/akernel/tests/test-pause-resume-wiring.sh b/deploy/akernel/tests/test-pause-resume-wiring.sh new file mode 100755 index 0000000..2f3fee8 --- /dev/null +++ b/deploy/akernel/tests/test-pause-resume-wiring.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +chart="${repo_root}/deploy/akernel" +tmp_dir=$(mktemp -d) +trap 'rm -rf "${tmp_dir}"' EXIT + +default_render="${tmp_dir}/default.yaml" +s3_render="${tmp_dir}/s3.yaml" +helm template akernel-snapshot "${chart}" --set monitor.enabled=false >"${default_render}" +if grep -q 'name: AKERNEL_ENABLE_SNAPSHOT' "${default_render}"; then + echo "Helm still emits the removed snapshot enable switch" >&2 + exit 1 +fi +grep -A1 'name: AKERNEL_SNAPSHOT_STORAGE_BACKEND' "${default_render}" | grep -q 'value: "datasystem"' + +helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=generic \ + --set core.snapshot.storage.s3.endpoint=s3.private.example \ + --set core.snapshot.storage.s3.region=us-east-1 \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 \ + --set core.snapshot.storage.s3.pathStyle=true >"${s3_render}" +grep -A1 'name: AKERNEL_SNAPSHOT_S3_PROVIDER' "${s3_render}" | grep -q 'value: "generic"' +grep -A1 'name: AKERNEL_SNAPSHOT_S3_ENDPOINT' "${s3_render}" | grep -q 'value: "s3.private.example"' +grep -A5 'name: AKERNEL_SNAPSHOT_S3_ACCESS_KEY' "${s3_render}" | grep -q 'name: "akernel-snapshot-s3"' + +if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=obs >/dev/null 2>&1; then + echo "Helm accepted the removed snapshot OBS backend" >&2 + exit 1 +fi + +if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=unknown \ + --set core.snapshot.storage.s3.endpoint=s3.example \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 >/dev/null 2>&1; then + echo "Helm accepted an unknown S3 provider" >&2 + exit 1 +fi + +if helm template akernel-snapshot "${chart}" \ + --set monitor.enabled=false \ + --set core.snapshot.storage.backend=s3 \ + --set core.snapshot.storage.s3.provider=oss \ + --set core.snapshot.storage.s3.endpoint=oss-cname.example \ + --set core.snapshot.storage.s3.region=cn-hangzhou \ + --set core.snapshot.storage.s3.bucket=akernel-test \ + --set core.snapshot.storage.s3.existingSecret=akernel-snapshot-s3 \ + --set core.snapshot.storage.s3.pathStyle=true >/dev/null 2>&1; then + echo "Helm accepted path-style addressing for OSS" >&2 + exit 1 +fi + +echo "Kubernetes snapshot wiring contract passed" diff --git a/deploy/akernel/values.yaml b/deploy/akernel/values.yaml index 77bbe26..0ef06e6 100644 --- a/deploy/akernel/values.yaml +++ b/deploy/akernel/values.yaml @@ -1,5 +1,19 @@ core: createNamespace: false + snapshot: + storage: + backend: datasystem + s3: + provider: generic + endpoint: "" + region: us-east-1 + bucket: "" + existingSecret: "" + accessKeyKey: access-key + secretKeyKey: secret-key + securityTokenKey: security-token + useHttps: true + pathStyle: true monitor: enabled: true diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index 28f359a..9bfac5e 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -57,9 +57,46 @@ rather than tmpfs. Without `storage_mb`, runsc retains its configured memory-backed overlay while Firecracker uses its configured sparse ext4 default. +Sandbox checkpoints for runsc and Firecracker use the embedded YuanRong +DataSystem by default. Select the unified S3-compatible backend when snapshots +must live in object storage: + +```bash +AKERNEL_SNAPSHOT_STORAGE_BACKEND=s3 \ +AKERNEL_SNAPSHOT_S3_PROVIDER=generic \ +AKERNEL_SNAPSHOT_S3_ENDPOINT=minio.example.internal:9000 \ +AKERNEL_SNAPSHOT_S3_REGION=us-east-1 \ +AKERNEL_SNAPSHOT_S3_BUCKET=akernel-snapshots \ +AKERNEL_SNAPSHOT_S3_ACCESS_KEY='' \ +AKERNEL_SNAPSHOT_S3_SECRET_KEY='' \ +AKERNEL_SNAPSHOT_S3_USE_HTTPS=false \ +AKERNEL_SNAPSHOT_S3_PATH_STYLE=true \ +./start.sh +``` + +The provider is `generic`, `obs`, or `oss`; it selects validation and +addressing defaults while every provider uses the same AWS Signature V4 S3 +protocol client. Private endpoints and CNAMEs are allowed. OSS requires +virtual-hosted addressing (`PATH_STYLE=false`). The optional +`AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN` carries an encrypted temporary token. +The removed OBS-native backend and `AKERNEL_SNAPSHOT_OBS_*` variables are not +accepted. + +Remote snapshots larger than 5 GiB are rejected before upload because this +version does not implement multipart CopyObject. `/home/akernel/checkpoints` +is the node's local staging directory; SDK checkpoint records have no automatic +TTL and remain until `Sandbox.delete_checkpoint()` is called. A restored +sandbox is a new sandbox and receives fresh network routes. + `start.sh` loads the host `tun` module and verifies `/dev/net/tun` before starting the pooled-TAP runtimes. Runc retains its separate veth network path. +The in-sandbox checkpoint endpoint uses +`YR_RRT_CONTROL_SOCKET_PATH=/run/openyuanrong` by default. Set +`YR_RRT_CONTROL_SOCKET_PATH=` explicitly when starting standalone to disable +the RRT control socket; an empty value is preserved rather than replaced by +the default. + ### Network backend Standalone uses the iptables NAT backend by default. Nodes without the diff --git a/deploy/standalone/config/config.json b/deploy/standalone/config/config.json index 5387596..25458c9 100644 --- a/deploy/standalone/config/config.json +++ b/deploy/standalone/config/config.json @@ -232,15 +232,6 @@ "noexec", "nodev" ] - }, - { - "destination": "/etc/resolv.conf", - "type": "bind", - "source": "/etc/resolv.conf", - "options": [ - "bind", - "ro" - ] } ], "linux": { diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index 25a17b8..81a9a6c 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -23,6 +23,7 @@ TOKEN_FILE="${DATA_DIR}/token" SANDBOXD_CONFIG_FILE="${DATA_DIR}/sandboxd/config.toml" AKERNEL_NAT_BACKEND="${AKERNEL_NAT_BACKEND:-iptables}" AKERNEL_ENABLE_RUNC="${AKERNEL_ENABLE_RUNC:-false}" +YR_RRT_CONTROL_SOCKET_PATH="${YR_RRT_CONTROL_SOCKET_PATH-/run/openyuanrong}" LITEBUS_DATA_KEY="" # Container runtime command (docker or pouch) @@ -330,12 +331,31 @@ start_node_container() { # FunctionMaster's HTTP provider publishes the per-sandbox routes required # by reverse tunnels; the legacy etcd mode cannot publish those routes. + local snapshot_backend="${AKERNEL_SNAPSHOT_STORAGE_BACKEND:-datasystem}" + local snapshot_docker_env=( + -e AKERNEL_SNAPSHOT_STORAGE_BACKEND="${snapshot_backend}" + ) + if [ "${snapshot_backend}" = "s3" ]; then + snapshot_docker_env+=( + -e AKERNEL_SNAPSHOT_S3_PROVIDER="${AKERNEL_SNAPSHOT_S3_PROVIDER:-}" + -e AKERNEL_SNAPSHOT_S3_ENDPOINT="${AKERNEL_SNAPSHOT_S3_ENDPOINT:-}" + -e AKERNEL_SNAPSHOT_S3_REGION="${AKERNEL_SNAPSHOT_S3_REGION:-}" + -e AKERNEL_SNAPSHOT_S3_BUCKET="${AKERNEL_SNAPSHOT_S3_BUCKET:-}" + -e AKERNEL_SNAPSHOT_S3_ACCESS_KEY + -e AKERNEL_SNAPSHOT_S3_SECRET_KEY + -e AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN + -e AKERNEL_SNAPSHOT_S3_USE_HTTPS="${AKERNEL_SNAPSHOT_S3_USE_HTTPS:-true}" + -e AKERNEL_SNAPSHOT_S3_PATH_STYLE="${AKERNEL_SNAPSHOT_S3_PATH_STYLE:-true}" + ) + fi + "${DOCKER_PREFIX[@]}" ${DOCKER_CMD} run -d \ --name "${NODE_CONTAINER_NAME}" \ --privileged \ --net bridge \ --restart always \ -e AKS_LOCAL_MODE="true" \ + "${snapshot_docker_env[@]}" \ -e TRAEFIK_MODE="http" \ -e TRAEFIK_HTTP_ENTRYPOINT="web" \ -e TRAEFIK_ENABLE_TLS="false" \ @@ -347,6 +367,7 @@ start_node_container() { -e TZ=Asia/Shanghai \ -e ENABLE_TRACE="${ENABLE_TRACE:-false}" \ -e ENABLE_METRICS="${ENABLE_METRICS:-false}" \ + -e YR_RRT_CONTROL_SOCKET_PATH="${YR_RRT_CONTROL_SOCKET_PATH}" \ "${PROXY_RUN_ARGS[@]}" \ "${GPU_RUN_ARGS[@]}" \ --entrypoint=/usr/local/bin/akernel-entrypoint \ diff --git a/deploy/standalone/tests/test-pause-resume-wiring.sh b/deploy/standalone/tests/test-pause-resume-wiring.sh new file mode 100755 index 0000000..bf0d307 --- /dev/null +++ b/deploy/standalone/tests/test-pause-resume-wiring.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +helper="${repo_root}/builder/scripts/yr_pause_resume_args.sh" +tmp_dir=$(mktemp -d) +trap 'rm -rf "${tmp_dir}"' EXIT +rrt_capability="${tmp_dir}/rrt-capable" +s3_capability="${tmp_dir}/s3-capable" +checkpoint_dir="${tmp_dir}/checkpoints" +touch "${rrt_capability}" "${s3_capability}" + +source "${helper}" +configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" true "${s3_capability}" +expected="--snapshot_storage_backend datasystem --checkpoint_dir ${checkpoint_dir} --data_system_enable true" +[[ "${standalone_snapshot_args[*]}" == "${expected}" ]] + +export AKERNEL_SNAPSHOT_STORAGE_BACKEND=s3 +export AKERNEL_SNAPSHOT_S3_PROVIDER=generic +export AKERNEL_SNAPSHOT_S3_ENDPOINT=s3.private.example:9000 +export AKERNEL_SNAPSHOT_S3_REGION=us-east-1 +export AKERNEL_SNAPSHOT_S3_BUCKET=akernel-test +export AKERNEL_SNAPSHOT_S3_ACCESS_KEY=encrypted-access +export AKERNEL_SNAPSHOT_S3_SECRET_KEY=encrypted-secret +export AKERNEL_SNAPSHOT_S3_SECURITY_TOKEN=encrypted-token +export AKERNEL_SNAPSHOT_S3_USE_HTTPS=false +export AKERNEL_SNAPSHOT_S3_PATH_STYLE=true +configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false "${s3_capability}" +[[ " ${snapshot_args[*]} " == *" --snapshot_s3_provider generic "* ]] +[[ " ${snapshot_args[*]} " == *" --snapshot_s3_endpoint s3.private.example:9000 "* ]] +[[ " ${snapshot_args[*]} " != *" --snapshot_s3_access_key "* ]] +[[ " ${snapshot_args[*]} " != *" --snapshot_s3_secret_key "* ]] +[[ " ${snapshot_args[*]} " != *" --snapshot_s3_security_token "* ]] +[[ "${SNAPSHOT_S3_ACCESS_KEY}" == "encrypted-access" ]] +[[ "${SNAPSHOT_S3_SECRET_KEY}" == "encrypted-secret" ]] +[[ "${SNAPSHOT_S3_SECURITY_TOKEN}" == "encrypted-token" ]] + +export AKERNEL_SNAPSHOT_S3_PROVIDER=oss +export AKERNEL_SNAPSHOT_S3_PATH_STYLE=true +if configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false "${s3_capability}" \ + >/dev/null 2>&1; then + echo "OSS snapshot storage accepted path-style addressing" >&2 + exit 1 +fi + +export AKERNEL_SNAPSHOT_STORAGE_BACKEND=obs +if configure_snapshot_args "${rrt_capability}" "${checkpoint_dir}" false "${s3_capability}" \ + >/dev/null 2>&1; then + echo "removed snapshot OBS backend was accepted" >&2 + exit 1 +fi + +echo "standalone snapshot wiring contract passed" diff --git a/sdk/python/README.md b/sdk/python/README.md index 700f93f..435fa20 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -22,6 +22,7 @@ It supports two backends: - [Filesystem](#filesystem) - [Interactive PTYs](#interactive-ptys) - [Port forwarding](#port-forwarding) + - [Checkpoint and restore](#checkpoint-and-restore) - [Reverse tunnels](#reverse-tunnels) - [Rootfs and mounts](#rootfs-and-mounts) - [Launch from a Dockerfile](#launch-from-a-dockerfile) @@ -99,6 +100,7 @@ Sandbox( detached: bool = False, node_id: str | None = None, *, + failover: bool = False, xpu: str | None = None, storage_mb: int | None = None, network_policy: NetworkPolicy | None = None, @@ -107,6 +109,23 @@ Sandbox( ) ``` +Set `failover=True` to opt into same-node recovery. If the sandbox has a valid +local checkpoint, recovery restores it. If the node explicitly reports that no +anonymous checkpoint exists, AKernel cold-starts the same logical sandbox on +the same physical node with its original creation parameters. Snapshot lookup, +metadata, validation, and restore errors do not silently fall back to a cold +start. The default is `False`; enabling the policy does not create a checkpoint +by itself. + +`sandbox.reload()` follows the same selection rule: it restores the latest +valid local checkpoint, or cold-starts on the original node when no checkpoint +exists, and returns only `True` or `False`. Existing `commands`, `files`, and +PTY factories remain owned by the original `Sandbox` object; no replacement +handle is returned. Completed `CommandResult` values remain readable. A cold +start does not preserve sandbox memory, writable-file changes, or the RRT +process table, so an incomplete command handle from the old runtime fails +instead of being associated with a PID in the new runtime. + ### Experimental GPU and writable storage Request a whole NVIDIA GPU by type, exact product model, and count: @@ -352,6 +371,67 @@ with Sandbox(port_forwardings=[8080]) as sandbox: deployment operator explicitly wants the direct Traefik address instead of the public gateway. +## Checkpoint and restore + +Create an immutable checkpoint of a running sandbox and restore it as a new, +independent sandbox: + +```python +from akernel_sdk import Sandbox + +checkpoint = None +try: + with Sandbox(runtime="runsc") as source: + source.commands.run("printf before > /tmp/state && sync") + checkpoint = source.checkpoint(timeout=180) + source.commands.run("printf after > /tmp/state") + + with Sandbox.restore(checkpoint) as restored: + assert restored.id != source.id + assert restored.commands.run("cat /tmp/state").stdout == "before" +finally: + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) +``` + +`checkpoint()` keeps the source running by default. Set +`leave_running=False` to terminate it after the checkpoint commits. Checkpoints +do not expire and must be removed explicitly with `delete_checkpoint()`; +`list_checkpoints()` returns all checkpoint identities visible to the current +tenant. + +Each restore gets a new sandbox ID, placement, network attachment, and routes. +The runtime, root filesystem, resources, mounts, environment, network policy, +and filesystem/process state come from the checkpoint. v1 does not support +in-place rollback or restore-time resource and configuration overrides. + +The bundled backend supports checkpoints for runsc and Firecracker. A restore +must use compatible runtime binaries, architecture, kernel, and runtime +configuration. The cluster prefers the source node when it is available and +may fall back to another compatible node through the configured snapshot +storage. + +When the deployment uses the S3-compatible snapshot backend, remote checkpoint +objects are limited to 5 GiB. Larger checkpoints fail before upload because +this version does not support multipart CopyObject; the independent DataSystem +backend is unaffected. This is an operator-selected storage capability rather +than an SDK-side size override. + +For a checkpoint created from a sandbox with a reverse tunnel, pass an +explicit `reverse_tunnel` to `restore()` using the same `reverse_port` and +`listen_port`. The target and connection timeout may change. A checkpoint made +without a tunnel rejects adding one during restore. The source tunnel is +briefly disconnected during checkpoint creation and then reconnected. + +Checkpoint/restore is available through the default `openyuanrong-sandbox` +backend. The legacy `openyuanrong-sdk` actor backend reports it as unsupported. +The current official backend package supports the default 180-second +checkpoint timeout. Custom checkpoint timeouts and checkpointing a sandbox +with an active reverse tunnel require a backend release containing the +corresponding YuanRong changes. +See [`examples/checkpoint_restore.py`](./examples/checkpoint_restore.py) for a +runnable example. + ## Reverse tunnels A reverse tunnel lets sandbox code call an HTTP or HTTPS service reachable @@ -519,6 +599,7 @@ Maintained examples are under [`examples/`](./examples): - `basic_usage.py` - `command_stdin.py` +- `checkpoint_restore.py` - `custom_image.py` - `dockerfile_launch.py` - `gpu_sandbox.py` @@ -556,6 +637,7 @@ not part of the default test suite. | `CommandInfo` | `pid`, `command`, `running` | | `EntryInfo` | `name`, `path`, `type`, `size`, `permissions`, `modified_time` | | `SandboxInfo` | `id`, `state`, `cpu`, `memory`, `image`, `xpu`, `storage_mb` | +| `CheckpointInfo` | `id` | | `NodeInfo` | `id`, `status`, `capacity`, `allocatable`, `labels` | | `S3Config` | `endpoint`, `bucket`, `object`, optional credentials | | `Mount` | `target`, one source, and `type` | diff --git a/sdk/python/akernel_sdk/__init__.py b/sdk/python/akernel_sdk/__init__.py index 5e77dd9..78fff0c 100644 --- a/sdk/python/akernel_sdk/__init__.py +++ b/sdk/python/akernel_sdk/__init__.py @@ -24,6 +24,7 @@ ) from ._backends.registry import selected_backend from .types import ( + CheckpointInfo, CommandInfo, CommandResult, EntryInfo, @@ -37,6 +38,7 @@ __all__ = [ "Sandbox", + "CheckpointInfo", "S3Config", "Mount", "NetworkPolicy", diff --git a/sdk/python/akernel_sdk/_backends/base.py b/sdk/python/akernel_sdk/_backends/base.py index e0ec3b7..c1a1f2b 100644 --- a/sdk/python/akernel_sdk/_backends/base.py +++ b/sdk/python/akernel_sdk/_backends/base.py @@ -41,6 +41,7 @@ class Capability(Enum): NODE_PLACEMENT = auto() CUSTOM_REVERSE_TUNNEL_PORTS = auto() REVERSE_WEBSOCKET = auto() + CHECKPOINT_RESTORE = auto() @dataclass(frozen=True) @@ -76,6 +77,7 @@ class SandboxSpec: xpu: str | None storage_mb: int | None network_policy: NetworkPolicy | None + failover: bool extra_config: Mapping[str, object] @@ -149,6 +151,10 @@ def is_running(self) -> bool: ... def get_info(self) -> SandboxInfo: ... + def reload(self) -> bool: ... + + def checkpoint(self, *, timeout: int) -> str: ... + def terminate(self) -> None: ... def close(self) -> None: ... @@ -163,6 +169,17 @@ class Backend(Protocol): def create(self, spec: SandboxSpec) -> BackendSession: ... + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: ... + + def list_checkpoints(self) -> list[str]: ... + + def delete_checkpoint(self, checkpoint_id: str) -> None: ... + def delete_named(self, name: str) -> None: ... def close(self) -> None: ... diff --git a/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py b/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py index 56773b5..57a7571 100644 --- a/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py +++ b/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py @@ -16,13 +16,20 @@ from __future__ import annotations +import inspect import os from collections.abc import Mapping from typing import Any import yr_sandbox -from ..types import CommandInfo, CommandResult, EntryInfo, SandboxInfo +from ..types import ( + CommandInfo, + CommandResult, + EntryInfo, + HttpReverseTunnel, + SandboxInfo, +) from .base import ( Backend, BackendConfig, @@ -246,6 +253,48 @@ def get_info(self) -> SandboxInfo: storage_mb=self._spec.storage_mb, ) + def reload(self) -> bool: + if self._terminated or self._closed: + return False + try: + return bool(self._sandbox.reload()) + except Exception: + return False + + def checkpoint(self, *, timeout: int) -> str: + if self._terminated or self._closed: + raise BackendOperationError("checkpoint requires a running sandbox") + create_snapshot = self._sandbox.create_snapshot + try: + parameters = inspect.signature(create_snapshot).parameters.values() + supports_timeout = any( + parameter.name == "timeout" + or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + except (TypeError, ValueError): + # Unknown callable signatures are treated as current backends. If + # they reject the keyword, the normal backend error includes the + # native failure instead of silently changing timeout semantics. + supports_timeout = True + if not supports_timeout and timeout != 180: + raise UnsupportedBackendFeatureError( + "The installed openyuanrong-sandbox backend only supports " + "the default 180-second checkpoint timeout. Upgrade the " + "backend to use a custom timeout." + ) + try: + if supports_timeout: + value = create_snapshot(timeout=timeout) + else: + value = create_snapshot() + except Exception as error: + raise _convert_error("checkpoint sandbox", error) from error + checkpoint_id = str(value.snapshot_id).strip() + if not checkpoint_id: + raise BackendOperationError("checkpoint returned an empty identity") + return checkpoint_id + def terminate(self) -> None: if self._terminated: return @@ -280,6 +329,7 @@ class OpenYuanRongSandboxBackend: { Capability.S3_ROOTFS, Capability.NODE_PLACEMENT, + Capability.CHECKPOINT_RESTORE, } ) @@ -373,6 +423,7 @@ def create(self, spec: SandboxSpec) -> BackendSession: xpu=spec.xpu, storage_mb=spec.storage_mb, network=network, + failover=spec.failover, extra_config=dict(spec.extra_config), create_timeout=create_timeout, ) @@ -380,6 +431,86 @@ def create(self, spec: SandboxSpec) -> BackendSession: raise _convert_error("create sandbox", error) from error return _Session(sandbox, spec) + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: + if reverse_tunnel is not None and ( + reverse_tunnel.reverse_port != reverse_tunnel.listen_port - 1 + ): + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sandbox' requires reverse_port to equal " + "listen_port - 1." + ) + try: + sandbox = yr_sandbox.Sandbox.create( + checkpoint_id, + upstream=( + reverse_tunnel.target if reverse_tunnel is not None else None + ), + tunnel_connect_timeout=( + reverse_tunnel.connect_timeout + if reverse_tunnel is not None + else None + ), + proxy_port=( + reverse_tunnel.listen_port + if reverse_tunnel is not None + else _DEFAULT_LISTEN_PORT + ), + ) + except Exception as error: + raise _convert_error("restore checkpoint", error) from error + restored_spec = SandboxSpec( + image=None, + rootfs=None, + runtime="runsc", + cpu=1000, + memory=4096, + cpu_limit=0, + mem_limit=0, + idle_timeout=300, + schedule_timeout=30, + env={}, + name=None, + command_cwd=None, + port_forwardings=(), + mounts=(), + reverse_tunnel=reverse_tunnel, + detached=False, + node_id=None, + xpu=None, + storage_mb=None, + network_policy=None, + failover=False, + extra_config={}, + ) + return _Session(sandbox, restored_spec) + + def list_checkpoints(self) -> list[str]: + checkpoint_ids: list[str] = [] + page_token: str | None = None + try: + while True: + items, next_page_token = yr_sandbox.Sandbox.list_snapshots( + page_token=page_token, + page_size=100, + ) + checkpoint_ids.extend(str(item.snapshot_id) for item in items) + if not next_page_token: + return checkpoint_ids + page_token = next_page_token + except Exception as error: + raise _convert_error("list checkpoints", error) from error + + def delete_checkpoint(self, checkpoint_id: str) -> None: + try: + yr_sandbox.Sandbox.delete_snapshot(checkpoint_id) + except Exception as error: + raise _convert_error("delete checkpoint", error) from error + def delete_named(self, name: str) -> None: sandbox_id = f"{self.namespace}-{name}" try: diff --git a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py index 51cc488..5b646fb 100644 --- a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py +++ b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py @@ -20,7 +20,13 @@ from collections.abc import Mapping from typing import Any -from ..types import CommandInfo, CommandResult, EntryInfo, SandboxInfo +from ..types import ( + CommandInfo, + CommandResult, + EntryInfo, + HttpReverseTunnel, + SandboxInfo, +) from . import openyuanrong_sdk_impl as _impl from .base import ( Backend, @@ -29,7 +35,7 @@ Capability, SandboxSpec, ) -from .errors import BackendOperationError +from .errors import BackendOperationError, UnsupportedBackendFeatureError from .openyuanrong_sdk_commands import ( CommandHandle as NativeCommandHandle, ) @@ -238,6 +244,21 @@ def get_info(self) -> SandboxInfo: storage_mb=self._spec.storage_mb, ) + def reload(self) -> bool: + if self._terminated or self._closed: + return False + try: + return bool(_impl.reload_instance(self._instance)) + except Exception: + return False + + def checkpoint(self, *, timeout: int) -> str: + del timeout + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + def terminate(self) -> None: if self._terminated: return @@ -265,7 +286,11 @@ class OpenYuanRongSdkBackend: name = "openyuanrong-sdk" namespace = _NAMESPACE - capabilities = frozenset(Capability) + capabilities: frozenset[Capability] = frozenset( + capability + for capability in Capability + if capability is not Capability.CHECKPOINT_RESTORE + ) def __init__(self, _config: BackendConfig) -> None: _impl.ensure_initialized() @@ -291,6 +316,7 @@ def create(self, spec: SandboxSpec) -> BackendSession: xpu=spec.xpu, storage_mb=spec.storage_mb, network_policy=spec.network_policy, + failover=spec.failover, extra_config=spec.extra_config, ) try: @@ -333,6 +359,31 @@ def create(self, spec: SandboxSpec) -> BackendSession: _rollback_instance(instance, "session initialization") raise _convert_error("initialize sandbox session", error) from error + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: + del checkpoint_id, reverse_tunnel + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + + def list_checkpoints(self) -> list[str]: + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + + def delete_checkpoint(self, checkpoint_id: str) -> None: + del checkpoint_id + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + def delete_named(self, name: str) -> None: try: _impl.delete_named_instance(name) diff --git a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk_impl.py b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk_impl.py index a7cbd8a..6087d4d 100644 --- a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk_impl.py +++ b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk_impl.py @@ -156,6 +156,7 @@ def build_options( xpu: str | None, storage_mb: int | None, network_policy: NetworkPolicy | None, + failover: bool, extra_config: Mapping[str, object], ) -> Any: """Translate the stable SDK configuration to openYuanrong options.""" @@ -174,6 +175,7 @@ def build_options( validate_storage_mb(storage_mb) options = yr.InvokeOptions() + options.failover = failover # A Sandbox is driven by one sequential SDK client. Disabling ordered RPC # execution prevents a missing sequence number from stalling later calls. options.need_order = False @@ -264,6 +266,13 @@ def terminate_instance(handle: Any) -> None: handle.terminate() +def reload_instance(handle: Any) -> bool: + instance_id = global_runtime.get_runtime().get_real_instance_id(handle.instance_id) + if not global_runtime.get_runtime().reload_instance(instance_id): + return False + return ping_instance(handle) + + def delete_named_instance(name: str) -> None: ensure_initialized() handle = yr.get_instance(name, namespace=_NAMESPACE) diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index f48851a..f0a0d27 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -32,7 +32,14 @@ from .commands import CommandHandle, Commands from .filesystem import Filesystem from .pty import Pty -from .types import HttpReverseTunnel, Mount, NetworkPolicy, S3Config, SandboxInfo +from .types import ( + CheckpointInfo, + HttpReverseTunnel, + Mount, + NetworkPolicy, + S3Config, + SandboxInfo, +) _traefik_internal_ip_cache: str | None = None logger = logging.getLogger(__name__) @@ -135,6 +142,17 @@ def _validate_integer( raise ValueError(f"{name} must be greater than or equal to {minimum}") +def _checkpoint_id(checkpoint: CheckpointInfo | str) -> str: + if isinstance(checkpoint, CheckpointInfo): + return checkpoint.id + if not isinstance(checkpoint, str): + raise TypeError("checkpoint must be a CheckpointInfo or string") + value = checkpoint.strip() + if not value: + raise ValueError("checkpoint must be a non-empty string") + return value + + def _get_traefik_internal_ip(gateway: Endpoint) -> tuple[str, int]: """Resolve Traefik's direct address for ``internal=True`` URLs.""" @@ -186,6 +204,7 @@ def __init__( detached: bool = False, node_id: str | None = None, *, + failover: bool = False, xpu: str | None = None, storage_mb: int | None = None, network_policy: NetworkPolicy | None = None, @@ -213,6 +232,8 @@ def __init__( reverse_tunnel: SDK-side HTTP service exposed inside the sandbox. detached: Keep the sandbox alive when this client closes. node_id: Require placement on a specific AKernel node. + failover: Restore this sandbox on the same node from its latest + local anonymous checkpoint after a sandbox failure. xpu: Experimental whole-device accelerator request in ``type:model:count`` format. Currently only exact-model NVIDIA GPU requests are supported. The backend validates runtime @@ -291,6 +312,8 @@ def __init__( raise ValueError("cwd must be an absolute POSIX path") if not isinstance(detached, bool): raise TypeError("detached must be a boolean") + if not isinstance(failover, bool): + raise TypeError("failover must be a boolean") if node_id is not None: if not isinstance(node_id, str): raise TypeError("node_id must be a string") @@ -362,6 +385,7 @@ def __init__( if network_policy is None or network_policy.is_empty else network_policy ), + failover=failover, extra_config=normalized_extra_config, ) self._session = load_backend().create(spec) @@ -443,6 +467,120 @@ def reverse_tunnel(self) -> HttpReverseTunnel | None: return self._reverse_tunnel + def checkpoint( + self, + *, + timeout: int = 180, + leave_running: bool = True, + ) -> CheckpointInfo: + """Create a reusable checkpoint of this sandbox. + + The checkpoint has no TTL and remains available until explicitly + deleted with :meth:`delete_checkpoint`. A successful checkpoint is an + immutable template; each restore creates a new sandbox identity with + fresh placement and routes. + + Args: + timeout: Positive checkpoint timeout in seconds. + leave_running: Keep this source sandbox running after success. + + Returns: + The stable checkpoint identity. + """ + + _validate_integer("timeout", timeout, minimum=1) + if not isinstance(leave_running, bool): + raise TypeError("leave_running must be a boolean") + if self._closed or self._session is None or not self.is_running(): + raise RuntimeError("checkpoint requires a running sandbox") + checkpoint = CheckpointInfo(self._session.checkpoint(timeout=timeout)) + if not leave_running: + self.kill() + return checkpoint + + @classmethod + def restore( + cls, + checkpoint: CheckpointInfo | str, + *, + reverse_tunnel: HttpReverseTunnel | None = None, + ) -> Sandbox: + """Restore an independent sandbox from a reusable checkpoint. + + Runtime, root filesystem, resources, mounts, environment, network + policy, and exposed-port shape are inherited from the checkpoint. + Resource overrides and in-place rollback are intentionally not part of + v1. A source created with a reverse tunnel requires an explicit tunnel + with the same ports here; its target and connect timeout may differ. + """ + + checkpoint_id = _checkpoint_id(checkpoint) + if reverse_tunnel is not None and not isinstance( + reverse_tunnel, HttpReverseTunnel + ): + raise TypeError("reverse_tunnel must be an HttpReverseTunnel") + session = load_backend().restore( + checkpoint_id, + reverse_tunnel=reverse_tunnel, + ) + restored = cls.__new__(cls) + restored._session = session + restored._startup_command = None + restored._pty = None + restored._closed = False + restored._terminated = False + restored._reverse_tunnel = reverse_tunnel + restored._forwarded_ports = set() + restored._image = None + restored._cpu = 0 + restored._memory = 0 + restored._xpu = None + restored._storage_mb = None + restored._id = "" + try: + restored._id = session.id + restored._files = Filesystem(session.files) + restored._commands = Commands(session.commands) + restored._pty = Pty(restored._id) + info = session.get_info() + restored._image = info.image + restored._cpu = info.cpu if info.cpu is not None else 0 + restored._memory = info.memory if info.memory is not None else 0 + restored._xpu = info.xpu + restored._storage_mb = info.storage_mb + except Exception: + restored._closed = True + try: + session.terminate() + except Exception: + logger.warning( + "failed to roll back a partially initialized restore", + exc_info=True, + ) + try: + session.close() + except Exception: + logger.warning( + "failed to close a partially initialized restore session", + exc_info=True, + ) + raise + return restored + + @classmethod + def list_checkpoints(cls) -> list[CheckpointInfo]: + """List reusable checkpoints visible to the current tenant.""" + + del cls + return [CheckpointInfo(value) for value in load_backend().list_checkpoints()] + + @classmethod + def delete_checkpoint(cls, checkpoint: CheckpointInfo | str) -> None: + """Permanently delete one reusable checkpoint.""" + + del cls + load_backend().delete_checkpoint(_checkpoint_id(checkpoint)) + def get_port_url(self, port: int, *, internal: bool = False) -> str: """Return the gateway URL for a declared sandbox port. @@ -481,6 +619,13 @@ def is_running(self) -> bool: return False return self._session.is_running() + def reload(self) -> bool: + """Restore this sandbox from its latest local anonymous checkpoint.""" + + if self._closed or self._session is None: + return False + return bool(self._session.reload()) + def get_info(self) -> SandboxInfo: """Return current sandbox state and requested resources.""" diff --git a/sdk/python/akernel_sdk/types.py b/sdk/python/akernel_sdk/types.py index 67a43cd..db137da 100644 --- a/sdk/python/akernel_sdk/types.py +++ b/sdk/python/akernel_sdk/types.py @@ -160,6 +160,21 @@ class SandboxInfo: storage_mb: int | None = None +@dataclass(frozen=True) +class CheckpointInfo: + """Stable identity of a reusable sandbox checkpoint.""" + + id: str + + def __post_init__(self) -> None: + if not isinstance(self.id, str): + raise TypeError("id must be a string") + normalized = self.id.strip() + if not normalized: + raise ValueError("id must be a non-empty string") + object.__setattr__(self, "id", normalized) + + @dataclass(frozen=True) class NodeInfo: """Capacity, allocation, and labels advertised by an AKernel node.""" diff --git a/sdk/python/examples/checkpoint_restore.py b/sdk/python/examples/checkpoint_restore.py new file mode 100644 index 0000000..b9359b7 --- /dev/null +++ b/sdk/python/examples/checkpoint_restore.py @@ -0,0 +1,39 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create, restore, and explicitly delete a reusable checkpoint.""" + +from akernel_sdk import CheckpointInfo, Sandbox + + +def main() -> None: + checkpoint: CheckpointInfo | None = None + try: + with Sandbox(runtime="runsc") as source: + source.commands.run("printf before > /tmp/checkpoint-state && sync") + checkpoint = source.checkpoint() + source.commands.run("printf after > /tmp/checkpoint-state") + print("source:", source.id, "checkpoint:", checkpoint.id) + + with Sandbox.restore(checkpoint) as restored: + value = restored.commands.run("cat /tmp/checkpoint-state") + print("restored:", restored.id, "state:", value.stdout) + assert value.stdout == "before" + finally: + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) + + +if __name__ == "__main__": + main() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 166f6d6..f3619e7 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ "Topic :: System :: Distributed Computing", ] dependencies = [ - "openyuanrong-sandbox==0.9.9", + "openyuanrong-sandbox==0.9.10", "websockets>=10.0", "dockerfile-parse>=2.0.1", ] diff --git a/sdk/python/tests/integration/test_sandbox.py b/sdk/python/tests/integration/test_sandbox.py index 08586e3..1303ac9 100644 --- a/sdk/python/tests/integration/test_sandbox.py +++ b/sdk/python/tests/integration/test_sandbox.py @@ -17,6 +17,7 @@ import unittest from akernel_sdk import Sandbox +from akernel_sdk._backends.errors import BackendOperationError _ENABLED = ( os.environ.get("AKERNEL_RUN_INTEGRATION") == "1" @@ -24,6 +25,9 @@ and bool(os.environ.get("AKERNEL_TOKEN")) ) _RUNTIME = os.environ.get("AKERNEL_TEST_RUNTIME", "runsc") +_RECOVERY_ENABLED = ( + _ENABLED and os.environ.get("AKERNEL_RUN_RECOVERY_INTEGRATION") == "1" +) @unittest.skipUnless( @@ -99,5 +103,90 @@ def test_pty_interrupts_foreground_process(self): self.assertIn(b"PTY_AFTER_INTERRUPT", output) +@unittest.skipUnless( + _ENABLED, + "set AKERNEL_RUN_INTEGRATION=1 and the AKernel SDK environment", +) +class SandboxCheckpointIntegrationTest(unittest.TestCase): + def test_checkpoint_restore_and_delete(self): + source = Sandbox(cpu=1000, memory=2048, runtime=_RUNTIME) + restored = None + checkpoint = None + try: + source_id = source.id + created = source.commands.run( + "printf checkpoint-before > /tmp/akernel-checkpoint-state && sync" + ) + self.assertEqual(created.exit_code, 0) + + checkpoint = source.checkpoint(timeout=180) + self.assertTrue(source.is_running()) + checkpoint_ids = {item.id for item in Sandbox.list_checkpoints()} + self.assertIn(checkpoint.id, checkpoint_ids) + + changed = source.commands.run( + "printf source-after > /tmp/akernel-checkpoint-state && sync" + ) + self.assertEqual(changed.exit_code, 0) + + restored = Sandbox.restore(checkpoint) + self.assertNotEqual(restored.id, source_id) + restored_value = restored.commands.run("cat /tmp/akernel-checkpoint-state") + self.assertEqual(restored_value.exit_code, 0) + self.assertEqual(restored_value.stdout, "checkpoint-before") + self.assertEqual( + source.commands.run("cat /tmp/akernel-checkpoint-state").stdout, + "source-after", + ) + finally: + if restored is not None: + restored.kill() + source.kill() + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) + + +@unittest.skipUnless( + _RECOVERY_ENABLED, + "set AKERNEL_RUN_RECOVERY_INTEGRATION=1 with the SDK environment", +) +class SandboxColdRecoveryIntegrationTest(unittest.TestCase): + def test_reload_without_snapshot_cold_starts_same_logical_sandbox(self): + sandbox = Sandbox( + cpu=1000, + memory=2048, + runtime=_RUNTIME, + failover=True, + ) + try: + logical_id = sandbox.id + facades = (sandbox.commands, sandbox.files, sandbox.pty) + completed = sandbox.commands.run("printf completed-before-cold-start") + sandbox.files.write("/tmp/cold-start-only", "old-runtime") + pending = sandbox.commands.run("sleep 60", background=True) + + self.assertIs(sandbox.reload(), True) + + self.assertEqual(sandbox.id, logical_id) + self.assertEqual((sandbox.commands, sandbox.files, sandbox.pty), facades) + self.assertEqual(completed.stdout, "completed-before-cold-start") + self.assertEqual( + sandbox.commands.run("test ! -e /tmp/cold-start-only").exit_code, + 0, + ) + self.assertEqual( + sandbox.commands.run("printf command-after-cold-start").stdout, + "command-after-cold-start", + ) + try: + old_result = pending.wait(timeout=10) + except BackendOperationError: + pass + else: + self.assertNotEqual(old_result.exit_code, 0) + finally: + sandbox.kill() + + if __name__ == "__main__": unittest.main() diff --git a/sdk/python/tests/unit/test_backends.py b/sdk/python/tests/unit/test_backends.py index c65924c..1dd8e3b 100644 --- a/sdk/python/tests/unit/test_backends.py +++ b/sdk/python/tests/unit/test_backends.py @@ -63,6 +63,7 @@ def _spec(**overrides): "xpu": None, "storage_mb": None, "network_policy": None, + "failover": False, "extra_config": MappingProxyType({}), } values.update(overrides) @@ -197,6 +198,59 @@ def test_extra_config_is_forwarded_to_native_sdk(self): {"featureFlag": True}, ) + def test_failover_is_forwarded_to_native_sandbox_sdk(self): + native = MagicMock() + native.id = "default-failover" + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ) as sandbox_type: + self.backend.create(_spec(failover=True)) + + self.assertIs(sandbox_type.call_args.kwargs["failover"], True) + + def test_reload_cold_start_success_keeps_native_session_and_facades(self): + native = MagicMock() + native.id = "default-reload" + native.reload.return_value = True + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + self.assertIs(session.reload(), True) + native.reload.assert_called_once_with() + + def test_incomplete_old_command_handle_fails_after_cold_start(self): + native = MagicMock() + native.id = "default-cold-start-handle" + native.reload.return_value = True + old_handle = MagicMock() + old_handle.pid = 321 + old_handle.wait.side_effect = RuntimeError( + "process handle is unavailable after cold start" + ) + native.commands.run.return_value = old_handle + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + pid = session.commands.start( + "sleep 60", envs=None, cwd=None, stdin=False + ) + self.assertIs(session.reload(), True) + + with self.assertRaisesRegex( + BackendOperationError, "unavailable after cold start" + ): + session.commands.wait(pid, 30) + def test_explicit_kata_image_is_forwarded_to_native_sdk(self): native = MagicMock() native.id = "default-kata-image" @@ -464,6 +518,118 @@ def test_named_delete_uses_deterministic_sid(self): self.backend.delete_named("worker") delete.assert_called_once_with("default-worker") + def test_checkpoint_delegates_to_reusable_snapshot_api(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + native.create_snapshot.return_value = SimpleNamespace( + snapshot_id="checkpoint-1" + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + self.assertEqual(session.checkpoint(timeout=240), "checkpoint-1") + + native.create_snapshot.assert_called_once_with(timeout=240) + + def test_checkpoint_uses_official_backend_default_timeout(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + calls = [] + + def create_snapshot(): + calls.append(True) + return SimpleNamespace(snapshot_id="checkpoint-1") + + native.create_snapshot = create_snapshot + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + self.assertEqual(session.checkpoint(timeout=180), "checkpoint-1") + + self.assertEqual(calls, [True]) + + def test_checkpoint_rejects_custom_timeout_on_official_backend(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + native.create_snapshot = lambda: SimpleNamespace( + snapshot_id="checkpoint-1" + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + with self.assertRaisesRegex( + UnsupportedBackendFeatureError, + "default 180-second", + ): + session.checkpoint(timeout=240) + + def test_restore_uses_snapshot_template_and_explicit_tunnel(self): + native = MagicMock() + native.id = "default-restored" + native.commands = MagicMock() + native.files = MagicMock() + tunnel = HttpReverseTunnel( + "https://new-target.example", + reverse_port=9000, + listen_port=9001, + connect_timeout=12, + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + ) as sandbox_type: + sandbox_type.create.return_value = native + session = self.backend.restore( + "checkpoint-1", + reverse_tunnel=tunnel, + ) + + self.assertEqual(session.id, "default-restored") + sandbox_type.create.assert_called_once_with( + "checkpoint-1", + upstream="https://new-target.example", + tunnel_connect_timeout=12, + proxy_port=9001, + ) + + def test_checkpoint_catalog_pages_and_deletes(self): + first = ([SimpleNamespace(snapshot_id="checkpoint-1")], "next") + second = ([SimpleNamespace(snapshot_id="checkpoint-2")], "") + with patch.object( + openyuanrong_sandbox.yr_sandbox.Sandbox, + "list_snapshots", + side_effect=[first, second], + ) as list_snapshots, patch.object( + openyuanrong_sandbox.yr_sandbox.Sandbox, + "delete_snapshot", + ) as delete_snapshot: + self.assertEqual( + self.backend.list_checkpoints(), + ["checkpoint-1", "checkpoint-2"], + ) + self.backend.delete_checkpoint("checkpoint-1") + + self.assertEqual(list_snapshots.call_count, 2) + delete_snapshot.assert_called_once_with("checkpoint-1") + class OpenYuanRongSdkBackendTest(unittest.TestCase): def setUp(self): @@ -477,6 +643,59 @@ def setUp(self): self.addCleanup(initialized.stop) self.backend = openyuanrong_sdk.OpenYuanRongSdkBackend(self.config) + def test_failover_is_forwarded_to_native_invoke_options(self): + instance = MagicMock() + with ( + patch.object( + openyuanrong_sdk._impl, + "build_options", + return_value=MagicMock(), + ) as build_options, + patch.object( + openyuanrong_sdk._impl, + "create_instance", + return_value=instance, + ), + patch.object( + openyuanrong_sdk._impl, + "real_instance_id", + return_value="physical-id", + ), + ): + self.backend.create(_spec(failover=True)) + + self.assertIs(build_options.call_args.kwargs["failover"], True) + + def test_reload_forwards_to_actor_runtime(self): + instance = MagicMock() + with ( + patch.object( + openyuanrong_sdk._impl, + "build_options", + return_value=MagicMock(), + ), + patch.object( + openyuanrong_sdk._impl, + "create_instance", + return_value=instance, + ), + patch.object( + openyuanrong_sdk._impl, + "real_instance_id", + return_value="physical-id", + ), + patch.object( + openyuanrong_sdk._impl, + "reload_instance", + return_value=True, + ) as reload_instance, + ): + session = self.backend.create(_spec()) + result = session.reload() + + self.assertIs(result, True) + reload_instance.assert_called_once_with(instance) + def test_physical_id_failure_rolls_back_created_actor(self): instance = MagicMock() physical_id_error = RuntimeError("physical ID unavailable") @@ -598,6 +817,14 @@ def test_close_finalizes_actor_sdk(self): finalize.assert_called_once_with() + def test_reusable_checkpoint_operations_are_explicitly_unsupported(self): + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.restore("checkpoint-1", reverse_tunnel=None) + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.list_checkpoints() + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.delete_checkpoint("checkpoint-1") + if __name__ == "__main__": unittest.main() diff --git a/sdk/python/tests/unit/test_openyuanrong_sdk_impl.py b/sdk/python/tests/unit/test_openyuanrong_sdk_impl.py index a766a68..5cc3127 100644 --- a/sdk/python/tests/unit/test_openyuanrong_sdk_impl.py +++ b/sdk/python/tests/unit/test_openyuanrong_sdk_impl.py @@ -42,6 +42,7 @@ def build_options(self, **overrides): "xpu": None, "storage_mb": None, "network_policy": None, + "failover": False, "extra_config": {}, } values.update(overrides) @@ -150,6 +151,11 @@ def test_extra_config_uses_custom_extension_wire_format(self): {"featureFlag": True, "labels": ["one", "two"]}, ) + def test_failover_uses_typed_native_option(self): + options = self.build_options(failover=True) + + self.assertIs(options.failover, True) + def test_node_info_conversion(self): node = _impl._to_node_info( { diff --git a/sdk/python/tests/unit/test_sandbox.py b/sdk/python/tests/unit/test_sandbox.py index 14a1386..50f6f9f 100644 --- a/sdk/python/tests/unit/test_sandbox.py +++ b/sdk/python/tests/unit/test_sandbox.py @@ -18,6 +18,7 @@ from unittest.mock import MagicMock, patch from akernel_sdk import ( + CheckpointInfo, DockerfileLaunch, HttpReverseTunnel, NetworkPolicy, @@ -73,11 +74,43 @@ def test_default_constructor_and_info(self): self.assertIsNone(spec.xpu) self.assertIsNone(spec.storage_mb) self.assertIsNone(spec.network_policy) + self.assertFalse(spec.failover) self.assertEqual(dict(spec.extra_config), {}) sandbox.kill() self.session.terminate.assert_called_once_with() self.session.close.assert_called_once_with() + def test_failover_is_typed_and_forwarded(self): + sandbox = Sandbox(failover=True) + spec = self.backend.create.call_args.args[0] + + self.assertTrue(spec.failover) + sandbox.kill() + + def test_failover_rejects_non_boolean_values(self): + with self.assertRaisesRegex(TypeError, "failover"): + Sandbox(failover=1) + self.backend.create.assert_not_called() + + def test_reload_cold_start_success_returns_true_without_replacing_facades(self): + self.session.reload.return_value = True + sandbox = Sandbox() + before = (sandbox.commands, sandbox.files, sandbox.pty, sandbox._session) + + self.assertIs(sandbox.reload(), True) + self.assertEqual( + (sandbox.commands, sandbox.files, sandbox.pty, sandbox._session), + before, + ) + self.session.reload.assert_called_once_with() + + def test_reload_returns_false_after_close(self): + sandbox = Sandbox() + sandbox.kill() + + self.assertIs(sandbox.reload(), False) + self.session.reload.assert_not_called() + def test_extra_config_is_validated_and_defensively_copied(self): labels = ["worker"] requested = {"featureFlag": True, "nested": {"labels": labels}} @@ -169,6 +202,89 @@ def test_named_delete_hides_backend_namespace(self): Sandbox.delete("worker") self.backend.delete_named.assert_called_once_with("worker") + def test_checkpoint_returns_public_identity_and_keeps_source_running(self): + self.session.checkpoint.return_value = "checkpoint-1" + sandbox = Sandbox() + + checkpoint = sandbox.checkpoint(timeout=240) + + self.assertEqual(checkpoint, CheckpointInfo("checkpoint-1")) + self.session.checkpoint.assert_called_once_with(timeout=240) + self.session.terminate.assert_not_called() + sandbox.kill() + + def test_checkpoint_can_terminate_source_after_success(self): + self.session.checkpoint.return_value = "checkpoint-1" + sandbox = Sandbox() + + checkpoint = sandbox.checkpoint(leave_running=False) + + self.assertEqual(checkpoint.id, "checkpoint-1") + self.session.terminate.assert_called_once_with() + self.session.close.assert_called_once_with() + + def test_checkpoint_validates_arguments_and_running_state(self): + sandbox = Sandbox() + for timeout in (True, 0, -1, 1.5): + with self.subTest(timeout=timeout), self.assertRaises( + (TypeError, ValueError) + ): + sandbox.checkpoint(timeout=timeout) + with self.assertRaisesRegex(TypeError, "leave_running"): + sandbox.checkpoint(leave_running=1) + self.session.is_running.return_value = False + with self.assertRaisesRegex(RuntimeError, "running sandbox"): + sandbox.checkpoint() + sandbox.kill() + + def test_restore_builds_facades_around_new_backend_session(self): + restored_session = MagicMock() + restored_session.id = "restored-physical-id" + restored_session.commands = MagicMock() + restored_session.files = MagicMock() + restored_session.get_info.return_value = SandboxInfo( + id="restored-physical-id", + state="running", + cpu=2000, + memory=8192, + image="base-image", + ) + self.backend.restore.return_value = restored_session + tunnel = HttpReverseTunnel("http://127.0.0.1:9000") + + restored = Sandbox.restore( + CheckpointInfo("checkpoint-1"), reverse_tunnel=tunnel + ) + + self.backend.restore.assert_called_once_with( + "checkpoint-1", reverse_tunnel=tunnel + ) + self.assertEqual(restored.id, "restored-physical-id") + self.assertIs(restored.reverse_tunnel, tunnel) + self.assertEqual(restored.get_info().cpu, 2000) + restored.kill() + restored_session.terminate.assert_called_once_with() + restored_session.close.assert_called_once_with() + + def test_list_and_delete_checkpoints_hide_backend_details(self): + self.backend.list_checkpoints.return_value = ["checkpoint-1", "checkpoint-2"] + + self.assertEqual( + Sandbox.list_checkpoints(), + [CheckpointInfo("checkpoint-1"), CheckpointInfo("checkpoint-2")], + ) + Sandbox.delete_checkpoint(CheckpointInfo("checkpoint-1")) + Sandbox.delete_checkpoint(" checkpoint-2 ") + + self.assertEqual( + self.backend.delete_checkpoint.call_args_list, + [unittest.mock.call("checkpoint-1"), unittest.mock.call("checkpoint-2")], + ) + with self.assertRaises(ValueError): + Sandbox.delete_checkpoint(" ") + with self.assertRaises(TypeError): + Sandbox.restore(object()) # type: ignore[arg-type] + def test_rootfs_requires_s3_config(self): with self.assertRaisesRegex(TypeError, "S3Config"): Sandbox(rootfs={"type": "s3"}) diff --git a/sdk/python/tests/unit/test_types.py b/sdk/python/tests/unit/test_types.py index e9766c4..2a8f4a8 100644 --- a/sdk/python/tests/unit/test_types.py +++ b/sdk/python/tests/unit/test_types.py @@ -19,7 +19,13 @@ from pathlib import Path import akernel_sdk -from akernel_sdk import DockerContextEntry, HttpReverseTunnel, Mount, S3Config +from akernel_sdk import ( + CheckpointInfo, + DockerContextEntry, + HttpReverseTunnel, + Mount, + S3Config, +) class PublicTypesTest(unittest.TestCase): @@ -69,6 +75,7 @@ def test_public_exports_are_minimal(self): set(akernel_sdk.__all__), { "Sandbox", + "CheckpointInfo", "S3Config", "Mount", "NetworkPolicy", @@ -120,6 +127,14 @@ def test_docker_context_entry_is_public_and_immutable(self): with self.assertRaisesRegex(AttributeError, "cannot assign"): entry.mode = 0o700 # type: ignore[misc] + def test_checkpoint_info_is_public_normalized_and_immutable(self): + checkpoint = CheckpointInfo(" checkpoint-1 ") + self.assertEqual(checkpoint.id, "checkpoint-1") + with self.assertRaisesRegex(AttributeError, "cannot assign"): + checkpoint.id = "changed" # type: ignore[misc] + with self.assertRaises(ValueError): + CheckpointInfo(" ") + def test_s3_config_serialization(self): config = S3Config( endpoint="https://s3.example.com", diff --git a/src/sandboxd b/src/sandboxd index 1918fad..446b3ab 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit 1918fadb03b59bc6f540196e14b90a91bdf31b7d +Subproject commit 446b3aba74332e84c6260c5551994157e1d9a518 diff --git a/src/yuanrong b/src/yuanrong index 177592c..abb0051 160000 --- a/src/yuanrong +++ b/src/yuanrong @@ -1 +1 @@ -Subproject commit 177592c7eb208698bd9b048a818b9cfb02060545 +Subproject commit abb0051c79c7ce1e032b53f1261f52b9102a2b78