From 931110a54f9b0ceee4ae0e1a009fc1c710ce4ca2 Mon Sep 17 00:00:00 2001 From: zyl1121 <62997582+zyl1121@users.noreply.github.com> Date: Wed, 1 Jul 2026 14:26:08 +0800 Subject: [PATCH 1/2] fix(CubeAPI): propagate create-time env vars to envd-backed commands (#566) * fix(sandbox): propagate create-time env vars to envd Create-time env vars were dropped during sandbox creation, so envd-backed command execution (e.g. commands.run) could not see them. Forward env vars from CubeAPI into CubeMaster, serialize them into an internal annotation, and have cubelet initialize envd via its /init data-plane endpoint after sandbox startup. Gate on the propagated envd capability signal (cube.master.components.envd.version) and fail fast when injection is requested on a template without it. Scope is limited to the envd data-plane init path; container-level startup env refresh is intentionally not included. Add focused tests across CubeAPI/CubeMaster/cubelet and a quickstart example. Signed-off-by: zhengyilei Co-authored-by: jinlong * fix(cubelet): fall back when envd support annotation is missing Keep create-time env var injection backward-compatible for templates built before envd capability propagation. When create_time_env_vars are requested and the template lacks the envd support annotation, probe the default envd init endpoint with bounded retry instead of rejecting upfront. If init still fails, return an explicit error instead of silently dropping the env vars. Also surface missing-annotation context in failure messages and add focused tests for the fallback path. Signed-off-by: zhengyilei --------- Signed-off-by: zhengyilei Co-authored-by: jinlong --- CubeAPI/src/cubemaster/mod.rs | 14 +- CubeAPI/src/models/mod.rs | 29 +- CubeAPI/src/services/sandboxes.rs | 379 +++++++++++++++++- CubeMaster/pkg/base/constants/constants.go | 3 + .../service/httpservice/cube/cubeboxutil.go | 1 - CubeMaster/pkg/service/sandbox/types/types.go | 8 +- CubeMaster/pkg/service/sandbox/util.go | 29 ++ .../pkg/service/sandbox/util_env_test.go | 51 +++ Cubelet/pkg/constants/const.go | 2 + .../services/cubebox/cube_container_create.go | 142 ++++--- .../cubebox/cube_container_create_test.go | 53 +++ Cubelet/services/cubebox/local.go | 26 +- Cubelet/services/cubebox/probe.go | 172 ++++++++ Cubelet/services/cubebox/probe_test.go | 223 +++++++++++ examples/code-sandbox-quickstart/README.md | 32 +- examples/code-sandbox-quickstart/README_zh.md | 31 +- .../create_with_envs.py | 20 + 17 files changed, 1126 insertions(+), 89 deletions(-) create mode 100644 CubeMaster/pkg/service/sandbox/util_env_test.go create mode 100644 examples/code-sandbox-quickstart/create_with_envs.py diff --git a/CubeAPI/src/cubemaster/mod.rs b/CubeAPI/src/cubemaster/mod.rs index bff23c375..02702fc75 100644 --- a/CubeAPI/src/cubemaster/mod.rs +++ b/CubeAPI/src/cubemaster/mod.rs @@ -564,7 +564,10 @@ impl CubeMasterError { /// as a potential source of routing ambiguity; /// * `.` and `..` are reserved for relative path resolution and easily slip /// through naive equality checks. -fn validate_path_segment(name: &'static str, value: &str) -> Result<(), CubeMasterError> { +pub(crate) fn validate_path_segment( + name: &'static str, + value: &str, +) -> Result<(), CubeMasterError> { let is_valid = !value.is_empty() && value .bytes() @@ -637,6 +640,15 @@ pub struct CreateSandboxRequest { #[serde(skip_serializing_if = "Option::is_none")] pub labels: Option>, + #[serde( + rename = "create_time_env_vars", + skip_serializing_if = "Option::is_none" + )] + /// Sandbox-level env vars requested at create time. CubeMaster forwards + /// them to cubelet via an internal annotation, and cubelet uses them to + /// initialize envd after sandbox startup. + pub create_time_env_vars: Option>, + #[serde(rename = "distribution_scope", skip_serializing_if = "Option::is_none")] pub distribution_scope: Option>, diff --git a/CubeAPI/src/models/mod.rs b/CubeAPI/src/models/mod.rs index 9061215e3..b71338b18 100644 --- a/CubeAPI/src/models/mod.rs +++ b/CubeAPI/src/models/mod.rs @@ -163,6 +163,8 @@ pub struct SandboxVolumeMount { /// Rule: ID abbreviations → uppercase (templateID, sandboxID, envVars); /// allow_internet_access is a known SDK snake_case quirk; /// lifecycle is a nested object — see SandboxLifecycleConfig. +/// `envVars` is the canonical field name; `envs` is accepted as a compatibility +/// alias for E2B SDK callers. #[derive(Debug, Deserialize, Validate, ToSchema)] #[allow(dead_code)] pub struct NewSandbox { @@ -199,7 +201,11 @@ pub struct NewSandbox { )] pub distribution_scope: Option>, - #[serde(rename = "envVars", skip_serializing_if = "Option::is_none")] + #[serde( + alias = "envs", + rename = "envVars", + skip_serializing_if = "Option::is_none" + )] pub env_vars: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -510,7 +516,7 @@ fn default_page_limit() -> i32 { #[cfg(test)] mod tests { - use super::SandboxNetworkConfig; + use super::{NewSandbox, SandboxNetworkConfig}; #[test] fn sandbox_network_config_accepts_snake_case_policy_fields() { @@ -526,6 +532,25 @@ mod tests { ); assert_eq!(cfg.deny_out, Some(vec!["0.0.0.0/0".to_string()])); } + + #[test] + fn new_sandbox_accepts_e2b_envs_alias() { + let req: NewSandbox = serde_json::from_value(serde_json::json!({ + "templateID": "tpl-1", + "envs": { + "CUBE_TEST_ENV": "value" + } + })) + .expect("new sandbox request should deserialize"); + + assert_eq!( + req.env_vars + .as_ref() + .and_then(|envs| envs.get("CUBE_TEST_ENV")) + .map(String::as_str), + Some("value") + ); + } } // ─── Templates ───────────────────────────────────────────────────────────── diff --git a/CubeAPI/src/services/sandboxes.rs b/CubeAPI/src/services/sandboxes.rs index f590088aa..9b631155c 100644 --- a/CubeAPI/src/services/sandboxes.rs +++ b/CubeAPI/src/services/sandboxes.rs @@ -3,7 +3,6 @@ // use std::collections::HashMap; - use uuid::Uuid; use super::validate_allow_out_domains_require_deny_all; @@ -28,6 +27,34 @@ const RET_CODE_HTTP_OK: i32 = 200; const RET_CODE_NOT_FOUND: i32 = 130404; const RET_CODE_CONFLICT: i32 = 130409; const HOSTDIR_MOUNT_KEY: &str = "host-mount"; +const ENV_VAR_NAME_MAX_LEN: usize = 256; +const ENV_VAR_VALUE_MAX_LEN: usize = 4096; + +/// Environment variable names that may compromise sandbox isolation if injected +/// at the runtime level (loader overrides, language runtime paths). +const FORBIDDEN_ENV_NAMES: &[&str] = &[ + "BASH_ENV", + "ENV", + "LD_PRELOAD", + "LD_AUDIT", + "LD_LIBRARY_PATH", + "LD_ORIGIN_PATH", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + "GCONV_PATH", + "PATH", + "PYTHONPATH", + "NODE_PATH", + "JAVA_TOOL_OPTIONS", + "_JAVA_OPTIONS", + "GEM_PATH", + "RUBYOPT", + "RUBYLIB", + "PERL5LIB", + "PERLLIB", + "CLASSPATH", + "IFS", +]; #[derive(Clone)] pub struct SandboxService { @@ -117,7 +144,20 @@ impl SandboxService { } pub async fn create_sandbox(&self, body: NewSandbox) -> AppResult { - let template_id = body.template_id.clone(); + let NewSandbox { + template_id, + timeout, + lifecycle, + allow_internet_access, + network, + metadata, + distribution_scope, + env_vars, + .. + } = body; + if let Some(env_vars) = env_vars.as_ref() { + validate_env_vars(env_vars)?; + } let mut annotations = HashMap::from([ ( "cube.master.appsnapshot.template.id".to_string(), @@ -129,7 +169,7 @@ impl SandboxService { ), ]); - let labels = body.metadata.map(|mut meta| { + let labels = metadata.map(|mut meta| { if let Some(value) = meta.remove(HOSTDIR_MOUNT_KEY) { annotations.insert(HOSTDIR_MOUNT_KEY.to_string(), value); } @@ -137,13 +177,12 @@ impl SandboxService { }); let cube_network_config = - build_cube_network_config(body.allow_internet_access, body.network.as_ref())?; + build_cube_network_config(allow_internet_access, network.as_ref())?; // Derive the two CubeMaster-side bools from the e2b-shaped lifecycle // object. Absent lifecycle keeps today's behaviour: idle sandboxes // are killed (auto_pause = false), and auto_resume defaults off. - let (auto_pause, auto_resume) = body - .lifecycle + let (auto_pause, auto_resume) = lifecycle .as_ref() .map(|lc| { use crate::models::SandboxOnTimeout; @@ -157,10 +196,11 @@ impl SandboxService { let req = CreateSandboxRequest { request_id: new_request_id(), instance_type: self.instance_type.clone(), - timeout: Some(body.timeout), + timeout: Some(timeout), annotations, labels, - distribution_scope: body.distribution_scope, + create_time_env_vars: env_vars, + distribution_scope, volumes: None, containers: vec![], exposed_ports: vec![], @@ -519,6 +559,56 @@ impl SandboxService { } } +/// Validate environment variable names against the POSIX name convention +/// and a deny-list of runtime-loader / path-override names that could +/// compromise sandbox isolation. +fn validate_env_vars(env_vars: &HashMap) -> AppResult<()> { + for (name, value) in env_vars { + if name.is_empty() || name.len() > ENV_VAR_NAME_MAX_LEN { + return Err(AppError::BadRequest(format!( + "invalid env var name length: {name:?}" + ))); + } + let bytes = name.as_bytes(); + if !bytes + .first() + .map_or(false, |b| b.is_ascii_alphabetic() || *b == b'_') + || !bytes + .iter() + .all(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + return Err(AppError::BadRequest(format!( + "env var name must match [a-zA-Z_][a-zA-Z0-9_]*: {name:?}" + ))); + } + if FORBIDDEN_ENV_NAMES + .iter() + .any(|forbidden| name.eq_ignore_ascii_case(forbidden)) + { + return Err(AppError::BadRequest(format!( + "env var name not allowed: {name}" + ))); + } + if value.len() > ENV_VAR_VALUE_MAX_LEN { + return Err(AppError::BadRequest(format!( + "env var value too large for {name:?}: {} bytes", + value.len() + ))); + } + if value.contains('\0') { + return Err(AppError::BadRequest(format!( + "env var value contains NUL byte: {name:?}" + ))); + } + if value.chars().any(|ch| ch != '\t' && ch.is_control()) { + return Err(AppError::BadRequest(format!( + "env var value contains control character: {name:?}" + ))); + } + } + Ok(()) +} + fn internal_error(error: impl std::fmt::Display) -> AppError { AppError::Internal(anyhow::anyhow!(error.to_string())) } @@ -780,13 +870,21 @@ fn map_egress_rule(rule: &EgressRule) -> CubeEgressRule { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Arc; - use super::{build_cube_network_config, filter_by_metadata, from_cubemaster_info}; - use crate::cubemaster::{CreateSandboxRequest, ListSandboxResponse, SandboxInfo}; + use super::{ + build_cube_network_config, filter_by_metadata, from_cubemaster_info, SandboxService, + }; + use crate::cubemaster::{ + CreateSandboxRequest, CubeMasterClient, ListSandboxResponse, SandboxInfo, + }; use crate::models::{ - EgressRule, EgressRuleAction, EgressRuleInject, EgressRuleMatch, SandboxNetworkConfig, - SandboxState, + EgressRule, EgressRuleAction, EgressRuleInject, EgressRuleMatch, NewSandbox, + SandboxNetworkConfig, SandboxState, }; + use axum::{extract::State, routing::post, Json, Router}; + use serde_json::Value; + use tokio::sync::Mutex; #[test] fn metadata_filter_matches_all_pairs() { @@ -1030,12 +1128,13 @@ mod tests { timeout: Some(60), annotations: HashMap::new(), labels: None, + create_time_env_vars: None, + distribution_scope: None, volumes: None, containers: vec![], exposed_ports: vec![], network_type: None, cube_network_config: None, - distribution_scope: None, auto_pause: false, auto_resume: false, }; @@ -1126,4 +1225,258 @@ mod tests { .unwrap(); assert_eq!(translate(&empty), (false, false)); } + + #[tokio::test] + async fn create_sandbox_forwards_create_time_env_vars_to_cubemaster() { + #[derive(Clone, Default)] + struct Capture { + create_body: Arc>>, + } + + async fn create_handler( + State(capture): State, + Json(body): Json, + ) -> Json { + *capture.create_body.lock().await = Some(body); + Json(serde_json::json!({ + "requestID": "req-1", + "sandbox_id": "sb-123", + "ret": { "ret_code": 0, "ret_msg": "ok" } + })) + } + + async fn spawn_server(app: Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("server should run"); + }); + format!("http://{}", addr) + } + + let capture = Capture::default(); + let cubemaster_url = spawn_server( + Router::new() + .route("/cube/sandbox", post(create_handler)) + .with_state(capture.clone()), + ) + .await; + + let service = SandboxService::new( + CubeMasterClient::new(cubemaster_url, reqwest::Client::new()), + "cubebox".to_string(), + "cube.app".to_string(), + ); + + let env_vars = HashMap::from([( + "CUBE_TEST_CREATE_ENV".to_string(), + "from-create".to_string(), + )]); + let sandbox = service + .create_sandbox(NewSandbox { + template_id: "tpl-1".to_string(), + timeout: 15, + lifecycle: None, + secure: None, + allow_internet_access: None, + network: None, + metadata: None, + distribution_scope: None, + env_vars: Some(env_vars), + mcp: None, + volume_mounts: None, + }) + .await + .expect("sandbox create should succeed"); + + assert_eq!(sandbox.sandbox_id, "sb-123"); + let create_body = capture + .create_body + .lock() + .await + .clone() + .expect("create body"); + assert_eq!( + create_body["create_time_env_vars"]["CUBE_TEST_CREATE_ENV"], + serde_json::json!("from-create") + ); + assert!(create_body.get("envVars").is_none()); + } + + #[tokio::test] + async fn create_sandbox_omits_create_time_env_vars_when_absent() { + #[derive(Clone, Default)] + struct Capture { + create_body: Arc>>, + } + + async fn create_handler( + State(capture): State, + Json(body): Json, + ) -> Json { + *capture.create_body.lock().await = Some(body); + Json(serde_json::json!({ + "requestID": "req-1", + "sandbox_id": "sb-no-envs", + "ret": { "ret_code": 0, "ret_msg": "ok" } + })) + } + + async fn spawn_server(app: Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("server should run"); + }); + format!("http://{}", addr) + } + + let capture = Capture::default(); + let cubemaster_url = spawn_server( + Router::new() + .route("/cube/sandbox", post(create_handler)) + .with_state(capture.clone()), + ) + .await; + + let service = SandboxService::new( + CubeMasterClient::new(cubemaster_url, reqwest::Client::new()), + "cubebox".to_string(), + "cube.app".to_string(), + ); + + let sandbox = service + .create_sandbox(NewSandbox { + template_id: "tpl-1".to_string(), + timeout: 15, + lifecycle: None, + secure: None, + allow_internet_access: None, + network: None, + metadata: None, + distribution_scope: None, + env_vars: None, + mcp: None, + volume_mounts: None, + }) + .await + .expect("sandbox create should succeed"); + + assert_eq!(sandbox.sandbox_id, "sb-no-envs"); + let create_body = capture + .create_body + .lock() + .await + .clone() + .expect("create body"); + assert!( + create_body.get("create_time_env_vars").is_none(), + "create_time_env_vars should be omitted when caller did not provide envs" + ); + } + + #[test] + fn create_sandbox_rejects_dangerous_env_var_names() { + for name in super::FORBIDDEN_ENV_NAMES { + let err = super::validate_env_vars(&HashMap::from([( + (*name).to_string(), + "val".to_string(), + )])) + .expect_err("dangerous env var name should be rejected"); + assert!( + err.to_string().contains("not allowed"), + "error for {name} should say 'not allowed': {err}" + ); + } + } + + #[test] + fn create_sandbox_rejects_dangerous_env_var_names_case_insensitive() { + for name in ["ld_preload", "Ld_Preload", "LD_PRELOAD"] { + let err = + super::validate_env_vars(&HashMap::from([(name.to_string(), "val".to_string())])) + .expect_err(&format!( + "dangerous env var name {name} should be rejected case-insensitively" + )); + assert!( + err.to_string().contains("not allowed"), + "error for {name} should say 'not allowed': {err}" + ); + } + } + + #[test] + fn create_sandbox_rejects_invalid_env_var_name_format() { + for (name, desc) in [ + ("", "empty"), + ("9VAR", "starts with digit"), + ("MY-VAR", "contains hyphen"), + ("MY.VAR", "contains dot"), + ] { + let err = + super::validate_env_vars(&HashMap::from([(name.to_string(), "v".to_string())])) + .expect_err(&format!("{desc} should be rejected: {name}")); + let msg = err.to_string(); + assert!( + msg.contains("must match") || msg.contains("invalid env var name"), + "error for {desc} ({name}) should mention name validation: {err}" + ); + } + } + + #[test] + fn create_sandbox_rejects_invalid_env_var_value() { + let too_large = "x".repeat(super::ENV_VAR_VALUE_MAX_LEN + 1); + let err = super::validate_env_vars(&HashMap::from([("TOO_LARGE".to_string(), too_large)])) + .expect_err("oversized env var value should be rejected"); + assert!( + err.to_string().contains("value too large"), + "oversized env var value error should mention size: {err}" + ); + + let err = super::validate_env_vars(&HashMap::from([( + "HAS_NUL".to_string(), + "abc\0def".to_string(), + )])) + .expect_err("env var value with NUL should be rejected"); + assert!( + err.to_string().contains("contains NUL"), + "NUL-containing env var value error should mention NUL: {err}" + ); + + let err = super::validate_env_vars(&HashMap::from([( + "HAS_ESC".to_string(), + "line\x1b[31mred".to_string(), + )])) + .expect_err("env var value with control character should be rejected"); + assert!( + err.to_string().contains("control character"), + "control-character env var value error should mention control character: {err}" + ); + + let err = super::validate_env_vars(&HashMap::from([( + "HAS_NEWLINE".to_string(), + "line1\nline2".to_string(), + )])) + .expect_err("env var value with newline should be rejected"); + assert!( + err.to_string().contains("control character"), + "newline env var value error should mention control character: {err}" + ); + } + + #[test] + fn create_sandbox_accepts_valid_env_var_names() { + super::validate_env_vars(&HashMap::from([ + ("MY_VAR".to_string(), "val".to_string()), + ("_underscore_prefix".to_string(), "val".to_string()), + ("CUBE_TEST_ENV".to_string(), "val".to_string()), + ("TAB_OK".to_string(), "hello\tworld".to_string()), + ])) + .expect("valid env var names should be accepted"); + } } diff --git a/CubeMaster/pkg/base/constants/constants.go b/CubeMaster/pkg/base/constants/constants.go index 7386edbe0..ec03736ee 100644 --- a/CubeMaster/pkg/base/constants/constants.go +++ b/CubeMaster/pkg/base/constants/constants.go @@ -79,6 +79,9 @@ const ( CubeAnnotationRootfsArtifactSizeBytes = "cube.master.rootfs.artifact.size_bytes" CubeAnnotationWritableLayerSize = "cube.master.rootfs.writable_layer_size" CubeAnnotationTemplateSpecFingerprint = "cube.master.template.spec_fingerprint" + // CubeAnnotationCreateTimeEnvVars stores the serialized create-time env map + // that CubeMaster passes to cubelet for envd initialization. + CubeAnnotationCreateTimeEnvVars = "cube.master.internal.create_time_env_vars" CubeAnnotationsVirtiofsCache = "cube.master.virtiofs.cache" diff --git a/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go b/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go index fe734606a..51e2a7b6e 100644 --- a/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go +++ b/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go @@ -530,7 +530,6 @@ func dealCubeboxCreateReqWithTemplateCenter(ctx context.Context, templateID stri return err } } - if templateReq.NetworkType != "" { reqInOut.NetworkType = templateReq.NetworkType } diff --git a/CubeMaster/pkg/service/sandbox/types/types.go b/CubeMaster/pkg/service/sandbox/types/types.go index 0068750d5..9c0c51c9f 100644 --- a/CubeMaster/pkg/service/sandbox/types/types.go +++ b/CubeMaster/pkg/service/sandbox/types/types.go @@ -44,8 +44,12 @@ type CreateCubeSandboxReq struct { Containers []*Container `json:"containers,omitempty"` - Annotations map[string]string `json:"annotations,omitempty" ` - Labels map[string]string `json:"labels,omitempty" ` + Annotations map[string]string `json:"annotations,omitempty" ` + Labels map[string]string `json:"labels,omitempty" ` + // CreateTimeEnvVars carries sandbox-level env vars requested at create + // time. CubeMaster serializes them into an internal annotation so cubelet + // can initialize envd after sandbox startup. + CreateTimeEnvVars map[string]string `json:"create_time_env_vars,omitempty"` DistributionScope []string `json:"distribution_scope,omitempty"` InstanceType string `json:"instance_type,omitempty"` NetworkType string `json:"network_type,omitempty"` diff --git a/CubeMaster/pkg/service/sandbox/util.go b/CubeMaster/pkg/service/sandbox/util.go index 01e2f8a00..0a4946d31 100644 --- a/CubeMaster/pkg/service/sandbox/util.go +++ b/CubeMaster/pkg/service/sandbox/util.go @@ -26,6 +26,8 @@ import ( "github.com/tencentcloud/CubeSandbox/CubeMaster/pkg/base/log" ) +const maxCreateTimeEnvVarsAnnotationBytes = 16 * 1024 + func checkAndGetReqResource(req *types.CreateCubeSandboxReq) (*selctx.RequestResource, error) { res := &selctx.RequestResource{ Cpu: resource.MustParse("0"), @@ -683,6 +685,33 @@ func checkAndGetAnnotation(req *types.CreateCubeSandboxReq, out *cubebox.RunCube if v, ok := req.Annotations[constants.CubeAnnotationsInsRegion]; !ok || v == "" { out.Annotations[constants.CubeAnnotationsInsRegion] = config.GetConfig().Log.Region } + if err := setCreateTimeEnvVarsAnnotation(out.Annotations, req.CreateTimeEnvVars); err != nil { + return err + } + return nil +} + +func setCreateTimeEnvVarsAnnotation(out map[string]string, envVars map[string]string) error { + if len(envVars) == 0 { + return nil + } + if out == nil { + return errors.New("annotation output map is nil") + } + // Carry the create-time env map to cubelet so the sandbox runtime can + // initialize envd after startup for envd-backed command execution. + payload, err := utils.JSONTool.Marshal(envVars) + if err != nil { + return fmt.Errorf("marshal create_time_env_vars failed: %w", err) + } + if len(payload) > maxCreateTimeEnvVarsAnnotationBytes { + return fmt.Errorf( + "create_time_env_vars annotation payload too large: %d bytes exceeds limit %d", + len(payload), + maxCreateTimeEnvVarsAnnotationBytes, + ) + } + out[constants.CubeAnnotationCreateTimeEnvVars] = string(payload) return nil } diff --git a/CubeMaster/pkg/service/sandbox/util_env_test.go b/CubeMaster/pkg/service/sandbox/util_env_test.go new file mode 100644 index 000000000..9fdd5f8d3 --- /dev/null +++ b/CubeMaster/pkg/service/sandbox/util_env_test.go @@ -0,0 +1,51 @@ +package sandbox + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/tencentcloud/CubeSandbox/CubeMaster/pkg/base/constants" +) + +func TestSetCreateTimeEnvVarsAnnotation(t *testing.T) { + out := map[string]string{} + envVars := map[string]string{ + "SESSION_ID": "user-session-test", + "USER_ID": "42", + } + + if err := setCreateTimeEnvVarsAnnotation(out, envVars); err != nil { + t.Fatalf("setCreateTimeEnvVarsAnnotation err=%v", err) + } + + raw := out[constants.CubeAnnotationCreateTimeEnvVars] + if raw == "" { + t.Fatalf("missing %s annotation", constants.CubeAnnotationCreateTimeEnvVars) + } + var decoded map[string]string + if err := json.Unmarshal([]byte(raw), &decoded); err != nil { + t.Fatalf("unmarshal create time env vars annotation: %v", err) + } + if decoded["SESSION_ID"] != "user-session-test" { + t.Fatalf("SESSION_ID=%q, want user-session-test", decoded["SESSION_ID"]) + } + if decoded["USER_ID"] != "42" { + t.Fatalf("USER_ID=%q, want 42", decoded["USER_ID"]) + } +} + +func TestSetCreateTimeEnvVarsAnnotationRejectsOversizedPayload(t *testing.T) { + out := map[string]string{} + envVars := map[string]string{ + "OVERSIZED": strings.Repeat("x", maxCreateTimeEnvVarsAnnotationBytes), + } + + err := setCreateTimeEnvVarsAnnotation(out, envVars) + if err == nil { + t.Fatal("expected oversized payload error") + } + if !strings.Contains(err.Error(), "annotation payload too large") { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/Cubelet/pkg/constants/const.go b/Cubelet/pkg/constants/const.go index 414614d07..29e2e53c3 100644 --- a/Cubelet/pkg/constants/const.go +++ b/Cubelet/pkg/constants/const.go @@ -206,6 +206,8 @@ const ( MasterAnnotationRootfsArtifactSizeBytes = "cube.master.rootfs.artifact.size_bytes" MasterAnnotationWritableLayerSize = "cube.master.rootfs.writable_layer_size" MasterAnnotationTemplateSpecFingerprint = "cube.master.template.spec_fingerprint" + MasterAnnotationComponentEnvdVersion = "cube.master.components.envd.version" + MasterAnnotationCreateTimeEnvVars = "cube.master.internal.create_time_env_vars" MasterAnnotationInstanceType = "cube.master.instance.type" MasterAnnotationNetworkPolicyBlockAll = "cube.master.network.policy.block_all" MasterAnnotationNetworkPolicyAllowPublicServices = "cube.master.network.policy.allow_public_services" diff --git a/Cubelet/services/cubebox/cube_container_create.go b/Cubelet/services/cubebox/cube_container_create.go index a4cbcd00c..858b85663 100644 --- a/Cubelet/services/cubebox/cube_container_create.go +++ b/Cubelet/services/cubebox/cube_container_create.go @@ -75,7 +75,8 @@ import ( const ( cubeSharedBindRootPath = "/run/cube-bind-share" - K8sEmptyDirPath = "kubernetes.io~empty-dir" + K8sEmptyDirPath = "kubernetes.io~empty-dir" + envdInitCleanupTimeout = 10 * time.Second ) func init() { @@ -283,58 +284,75 @@ func (l *local) createContainers(ctx context.Context, flowOpts *workflow.CreateC }) } - sandBox.Lock() - defer func() { - if err := l.cubeboxManger.Save(ctx, sandBox); err != nil { - log.G(ctx).Warnf("saveSandBoxInfo failed.%s", err.Error()) - } - sandBox.Unlock() - }() + if err := func() error { + sandBox.Lock() + defer func() { + if err := l.cubeboxManger.Save(ctx, sandBox); err != nil { + log.G(ctx).Warnf("saveSandBoxInfo failed.%s", err.Error()) + } + sandBox.Unlock() + }() - for _, param := range params { - ci := param.ci - containerLog := sanboxlog.WithFields(CubeLog.Fields{ - "containerID": ci.ID, - "isPod": ci.IsPod, - }) - err = func() (retE error) { - containerLog := log.G(ctx).WithField("container-id", ci.ID) - retE = l.runContainer(param.ctxTmp, sandBox, param.ci, param.cOpts, ociRuntime) - withOciSpec := log.IsDebug() || retE != nil - if ci.Container != nil && withOciSpec { - info, err := ci.Container.Info(ctx, containerd.WithoutRefreshedMetadata) - if err == nil { - v, err := typeurl.UnmarshalAny(info.Spec) - if err != nil { - return fmt.Errorf("failed to unmarshal container spec with url %s: %w", info.Spec.GetTypeUrl(), err) + for _, param := range params { + ci := param.ci + containerLog := sanboxlog.WithFields(CubeLog.Fields{ + "containerID": ci.ID, + "isPod": ci.IsPod, + }) + err = func() (retE error) { + containerLog := log.G(ctx).WithField("container-id", ci.ID) + retE = l.runContainer(param.ctxTmp, sandBox, param.ci, param.cOpts, ociRuntime) + withOciSpec := log.IsDebug() || retE != nil + if ci.Container != nil && withOciSpec { + info, err := ci.Container.Info(ctx, containerd.WithoutRefreshedMetadata) + if err == nil { + v, err := typeurl.UnmarshalAny(info.Spec) + if err != nil { + return fmt.Errorf("failed to unmarshal container spec with url %s: %w", info.Spec.GetTypeUrl(), err) + } + jsonstr := log.WithJsonValue(struct { + containers.Container + Spec interface{} `json:"Spec,omitempty"` + }{ + Container: info, + Spec: v, + }) + containerLog.Debugf("container-oci-spec: %s", jsonstr) } - jsonstr := log.WithJsonValue(struct { - containers.Container - Spec interface{} `json:"Spec,omitempty"` - }{ - Container: info, - Spec: v, - }) - containerLog.Debugf("container-oci-spec: %s", jsonstr) } + if retE != nil { + containerLog.Errorf("run container failed.%s", retE.Error()) + } else { + containerLog.Debug("run container success") + } + return retE + }() + if err != nil { + return fmt.Errorf("failed to run container %s: %w", param.ci.ID, err) } - if retE != nil { - containerLog.Errorf("run container failed.%s", retE.Error()) - } else { - containerLog.Debug("run container success") + if err := l.doProbe(param.ctxTmp, param.cntrReq, param.ci); err != nil { + return err + } + err = l.cbriManager.PostCreateContainer(ctx, sandBox, param.ci) + if err != nil { + containerLog.Errorf("post create container failed, err: %v", err) } - return retE - }() - if err != nil { - return fmt.Errorf("failed to run container %s: %w", param.ci.ID, err) - } - if err := l.doProbe(param.ctxTmp, param.cntrReq, param.ci); err != nil { - return err } - err = l.cbriManager.PostCreateContainer(ctx, sandBox, param.ci) - if err != nil { - containerLog.Errorf("post create container failed, err: %v", err) + return nil + }(); err != nil { + return err + } + + if err := l.doCreateTimeEnvdInit(ctx, realReq, sandBox); err != nil { + cleanupErr := l.cleanupAfterEnvdInitFailure(flowOpts, realReq, sandBox) + if cleanupErr == nil { + // The sandbox has already been torn down synchronously, so the outer + // workflow failover does not need to repeat the same destroy path. + flowOpts.Failover = false + } else { + sanboxlog.Errorf("cleanup sandbox after envd init failure failed: %v", cleanupErr) } + return err } pid := sandBox.Endpoint.Pid @@ -357,6 +375,38 @@ func (l *local) createContainers(ctx context.Context, flowOpts *workflow.CreateC return nil } +func (l *local) cleanupAfterEnvdInitFailure(flowOpts *workflow.CreateContext, + realReq *cubebox.RunCubeSandboxRequest, sandBox *cubeboxstore.CubeBox) error { + // CubeMaster already compensates create failures on the main path, but + // cubelet workflow failover skips PreConditionFailed and runtime-local + // callers still benefit from immediate teardown close to the runtime. + cleanupCtx, cancel := context.WithTimeout(context.Background(), envdInitCleanupTimeout) + defer cancel() + if sandBox.Namespace != "" { + cleanupCtx = namespaces.WithNamespace(cleanupCtx, sandBox.Namespace) + } + cleanupCtx = constants.WithFailoverOperation(cleanupCtx) + if flowOpts.CubeBoxCreated { + cleanupCtx = constants.WithCubeboxCreated(cleanupCtx) + } + return l.destroySandboxAfterEnvdInitFailure(cleanupCtx, &workflow.DestroyContext{ + BaseWorkflowInfo: workflow.BaseWorkflowInfo{ + SandboxID: sandBox.ID, + }, + DestroyInfo: &cubebox.DestroyCubeSandboxRequest{ + RequestID: realReq.RequestID, + SandboxID: sandBox.ID, + }, + }) +} + +func (l *local) destroySandboxAfterEnvdInitFailure(ctx context.Context, opts *workflow.DestroyContext) error { + if l != nil && l.destroyFn != nil { + return l.destroyFn(ctx, opts) + } + return l.Destroy(ctx, opts) +} + func (l *local) genSandboxOptions(ctx context.Context, realReq *cubebox.RunCubeSandboxRequest, sandBox *cubeboxstore.CubeBox, flowOpts *workflow.CreateContext) ([]oci.SpecOpts, error) { var ( additionalSandboxOpt []oci.SpecOpts diff --git a/Cubelet/services/cubebox/cube_container_create_test.go b/Cubelet/services/cubebox/cube_container_create_test.go index e9af3c640..9716393f4 100644 --- a/Cubelet/services/cubebox/cube_container_create_test.go +++ b/Cubelet/services/cubebox/cube_container_create_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/containerd/v2/pkg/namespaces" "github.com/containerd/containerd/v2/pkg/oci" imagespec "github.com/opencontainers/image-spec/specs-go/v1" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -324,6 +325,58 @@ func TestAppendExt4NetfileMounts(t *testing.T) { }) } +func TestCleanupAfterEnvdInitFailurePassesExpectedContextAndRequest(t *testing.T) { + flowOpts := &workflow.CreateContext{CubeBoxCreated: true} + req := &cubebox.RunCubeSandboxRequest{RequestID: "req-cleanup"} + sb := &cubeboxstore.CubeBox{ + Namespace: "ns-cleanup", + Metadata: cubeboxstore.Metadata{ + ID: "sb-cleanup", + }, + } + + called := false + l := &local{ + destroyFn: func(ctx context.Context, opts *workflow.DestroyContext) error { + called = true + assert.True(t, constants.IsFailoverOperation(ctx)) + assert.True(t, constants.IsCubeboxCreated(ctx)) + ns, err := namespaces.NamespaceRequired(ctx) + require.NoError(t, err) + assert.Equal(t, "ns-cleanup", ns) + require.NotNil(t, opts) + assert.Equal(t, "sb-cleanup", opts.SandboxID) + require.NotNil(t, opts.DestroyInfo) + assert.Equal(t, "sb-cleanup", opts.DestroyInfo.SandboxID) + assert.Equal(t, "req-cleanup", opts.DestroyInfo.RequestID) + return nil + }, + } + + err := l.cleanupAfterEnvdInitFailure(flowOpts, req, sb) + require.NoError(t, err) + require.True(t, called) +} + +func TestCleanupAfterEnvdInitFailureReturnsDestroyError(t *testing.T) { + flowOpts := &workflow.CreateContext{Failover: true} + req := &cubebox.RunCubeSandboxRequest{RequestID: "req-cleanup"} + sb := &cubeboxstore.CubeBox{ + Metadata: cubeboxstore.Metadata{ + ID: "sb-cleanup", + }, + } + + l := &local{ + destroyFn: func(ctx context.Context, opts *workflow.DestroyContext) error { + return fmt.Errorf("destroy failed") + }, + } + + err := l.cleanupAfterEnvdInitFailure(flowOpts, req, sb) + require.EqualError(t, err, "destroy failed") +} + func TestAppendExt4NetfileMountsNoopWithoutNetfile(t *testing.T) { mountsConfig := &virtiofs.CubeRootfsInfo{ PmemFile: "/pmem/rootfs.ext4", diff --git a/Cubelet/services/cubebox/local.go b/Cubelet/services/cubebox/local.go index b0c6c08df..98387f23b 100644 --- a/Cubelet/services/cubebox/local.go +++ b/Cubelet/services/cubebox/local.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "maps" + "net/http" "os" "path" "path/filepath" @@ -203,13 +204,15 @@ func init() { } l := &local{ - client: client, - localTask: i.(tasks.TasksClient), - config: config, - criImage: obj.(*cubeimages.CubeImageService), - cbriManager: cbriManager, - cubeboxManger: cubeboxAPIObj.(cubes.CubeboxAPI), - shims: shimPlugin.(*v2.ShimManager), + client: client, + localTask: i.(tasks.TasksClient), + config: config, + criImage: obj.(*cubeimages.CubeImageService), + cbriManager: cbriManager, + cubeboxManger: cubeboxAPIObj.(cubes.CubeboxAPI), + shims: shimPlugin.(*v2.ShimManager), + envdHTTPClient: newEnvdHTTPClient(), + envdInitPort: defaultEnvdInitPort, } CubeLog.Info("Start recovering state") @@ -254,9 +257,12 @@ type local struct { config *CubeConfig shims *v2.ShimManager - criImage *cubeimages.CubeImageService - cbriManager cbri.APIManager - cubeboxManger cubes.CubeboxAPI + criImage *cubeimages.CubeImageService + cbriManager cbri.APIManager + cubeboxManger cubes.CubeboxAPI + envdHTTPClient *http.Client + envdInitPort int + destroyFn func(context.Context, *workflow.DestroyContext) error } const ( diff --git a/Cubelet/services/cubebox/probe.go b/Cubelet/services/cubebox/probe.go index 88eb2f9d3..a8e3693a7 100644 --- a/Cubelet/services/cubebox/probe.go +++ b/Cubelet/services/cubebox/probe.go @@ -5,7 +5,9 @@ package cubebox import ( + "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -14,6 +16,7 @@ import ( "net/url" "runtime/debug" "strconv" + "strings" "time" "github.com/tencentcloud/CubeSandbox/Cubelet/api/services/cubebox/v1" @@ -27,8 +30,55 @@ import ( "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/utils" "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/version" "github.com/tencentcloud/CubeSandbox/Cubelet/plugins/workflow" + "github.com/tencentcloud/CubeSandbox/cubelog" ) +const ( + envdInitPath = "/init" + // Keep create-time envd init within a bounded sub-second budget so the + // safeguard absorbs brief restore jitter without turning sandbox create + // into an unbounded slow path. + envdInitAttemptTimeout = 150 * time.Millisecond + envdInitMaxAttempts = 3 + envdInitRetryDelay = 25 * time.Millisecond + defaultEnvdInitPort = 49983 + missingEnvdSupportAnnotationDetail = "template does not carry envd support annotation" +) + +func newEnvdInitFailure(msg string, hasEnvdCapability bool, err error) error { + if !hasEnvdCapability { + return ret.Errorf(errorcode.ErrorCode_ExecCommandInSandboxFailed, + "%s; %s: %v", msg, missingEnvdSupportAnnotationDetail, err) + } + return ret.Errorf(errorcode.ErrorCode_ExecCommandInSandboxFailed, + "%s: %v", msg, err) +} + +func (l *local) getEnvdInitPort() int { + if l != nil && l.envdInitPort > 0 { + return l.envdInitPort + } + return defaultEnvdInitPort +} + +func (l *local) getEnvdHTTPClient() *http.Client { + if l != nil && l.envdHTTPClient != nil { + return l.envdHTTPClient + } + return newEnvdHTTPClient() +} + +func newEnvdHTTPClient() *http.Client { + return &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + Transport: &http.Transport{ + DisableKeepAlives: true, + }, + } +} + func (l *local) doProbe(ctx context.Context, c *cubebox.ContainerConfig, ci *cubeboxstore.Container) (retErr error) { startTime := time.Now() defer func() { @@ -142,6 +192,128 @@ func (l *local) doProbe(ctx context.Context, c *cubebox.ContainerConfig, ci *cub return nil } +func (l *local) doCreateTimeEnvdInit(ctx context.Context, req *cubebox.RunCubeSandboxRequest, sandBox *cubeboxstore.CubeBox) error { + if req == nil || sandBox == nil || req.Annotations == nil { + return nil + } + raw := strings.TrimSpace(req.Annotations[constants.MasterAnnotationCreateTimeEnvVars]) + if raw == "" { + return nil + } + envVars := map[string]string{} + if err := json.Unmarshal([]byte(raw), &envVars); err != nil { + return ret.Errorf(errorcode.ErrorCode_InvalidParamFormat, "invalid create_time_env_vars annotation: %v", err) + } + if len(envVars) == 0 { + return nil + } + if strings.TrimSpace(sandBox.IP) == "" { + return ret.Err(errorcode.ErrorCode_CreateNetworkFailed, "sandbox IP is empty for create_time_env_vars init") + } + + body, err := json.Marshal(struct { + EnvVars map[string]string `json:"envVars"` + }{ + EnvVars: envVars, + }) + if err != nil { + return ret.Errorf(errorcode.ErrorCode_InvalidParamFormat, "marshal create_time_env_vars init request failed: %v", err) + } + + port := l.getEnvdInitPort() + hasEnvdCapability := strings.TrimSpace(req.Annotations[constants.MasterAnnotationComponentEnvdVersion]) != "" + if !hasEnvdCapability { + // Templates built before envd capability propagation do not carry the + // annotation. Keep them backward-compatible by probing the default envd + // init endpoint with the same bounded retry instead of rejecting upfront. + log.G(ctx).WithFields(CubeLog.Fields{ + "sandboxID": sandBox.ID, + "templateID": strings.TrimSpace(req.Annotations[constants.MasterAnnotationAppSnapshotTemplateID]), + "envdInitPort": port, + }).Warnf("missing envd support annotation; probing default envd init endpoint with bounded retry") + } + + return l.doCreateTimeEnvdInitWithRetry(ctx, sandBox.IP, port, hasEnvdCapability, body) +} + +func (l *local) doCreateTimeEnvdInitWithRetry(ctx context.Context, sandboxIP string, port int, hasEnvdCapability bool, body []byte) error { + var lastErr error + for attempt := 1; attempt <= envdInitMaxAttempts; attempt++ { + innerCtx, cancel := context.WithTimeout(ctx, envdInitAttemptTimeout) + retryable, err := l.doCreateTimeEnvdInitAttempt(innerCtx, sandboxIP, port, body) + cancel() + if err == nil { + return nil + } + lastErr = err + if !retryable || attempt == envdInitMaxAttempts { + return newEnvdInitFailure("create_time_env_vars init failed after bounded retry", hasEnvdCapability, err) + } + select { + case <-time.After(envdInitRetryDelay): + case <-ctx.Done(): + return newEnvdInitFailure("create_time_env_vars init canceled during bounded retry", hasEnvdCapability, lastErr) + } + } + return newEnvdInitFailure("create_time_env_vars init failed after bounded retry", hasEnvdCapability, lastErr) +} + +func (l *local) doCreateTimeEnvdInitAttempt(ctx context.Context, sandboxIP string, port int, body []byte) (bool, error) { + reqURL := formatURL("http", sandboxIP, port, envdInitPath) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL.String(), bytes.NewReader(body)) + if err != nil { + return false, ret.Errorf(errorcode.ErrorCode_InvalidParamFormat, "build envd init request failed: %v", err) + } + httpReq.Header.Set("Content-Type", "application/json") + + resp, err := l.getEnvdHTTPClient().Do(httpReq) + if err != nil { + return isRetryableEnvdInitTransportErr(err), ret.Errorf(errorcode.ErrorCode_ExecCommandInSandboxFailed, "envd init request failed: %v", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return false, ret.Errorf(errorcode.ErrorCode_ExecCommandInSandboxFailed, "read envd init response body failed: %v", err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return isRetryableEnvdInitStatusCode(resp.StatusCode), ret.Errorf( + errorcode.ErrorCode_ExecCommandInSandboxFailed, + "envd init request returned HTTP %d: %s", + resp.StatusCode, + strings.TrimSpace(string(respBody)), + ) + } + return false, nil +} + +func isRetryableEnvdInitStatusCode(statusCode int) bool { + switch statusCode { + case http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: + return true + default: + return false + } +} + +func isRetryableEnvdInitTransportErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return true + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "connection refused") || + strings.Contains(msg, "connection reset") || + strings.Contains(msg, "broken pipe") || + strings.Contains(msg, "eof") +} + func doPreStop(ctx context.Context, ci *cubeboxstore.Container) { c := ci.Config if c.GetPrestop() == nil || c.GetPrestop().GetLifecyleHandler() == nil { diff --git a/Cubelet/services/cubebox/probe_test.go b/Cubelet/services/cubebox/probe_test.go index ec2de757c..587956300 100644 --- a/Cubelet/services/cubebox/probe_test.go +++ b/Cubelet/services/cubebox/probe_test.go @@ -5,10 +5,16 @@ package cubebox import ( + "bytes" "context" + "encoding/json" "fmt" + "io" "net" "net/http" + "net/http/httptest" + neturl "net/url" + "strconv" "testing" "time" @@ -18,12 +24,19 @@ import ( "github.com/tencentcloud/CubeSandbox/Cubelet/api/services/cubebox/v1" "github.com/tencentcloud/CubeSandbox/Cubelet/api/services/errorcode/v1" "github.com/tencentcloud/CubeSandbox/Cubelet/network/proto" + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/constants" "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/ret" cubeboxstore "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/store/cubebox" "github.com/tencentcloud/CubeSandbox/Cubelet/plugins/workflow" "k8s.io/utils/pointer" ) +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + func TestProbeErrIp(t *testing.T) { cnt := &cubebox.ContainerConfig{ Probe: &cubebox.Probe{ @@ -146,6 +159,216 @@ func TestProbeErrAction(t *testing.T) { assert.Equal(t, errorcode.ErrorCode_InvalidParamFormat, err.Code()) } +func TestDoCreateTimeEnvdInitPostsEnvVars(t *testing.T) { + var gotBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/init" { + t.Fatalf("path=%q, want /init", r.URL.Path) + } + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode request body: %v", err) + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + u, err := neturl.Parse(server.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + port, err := strconv.Atoi(u.Port()) + if err != nil { + t.Fatalf("parse server port: %v", err) + } + + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationComponentEnvdVersion: "0.2.0", + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test","USER_ID":"42"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: server.Client(), envdInitPort: port} + if err := l.doCreateTimeEnvdInit(context.Background(), req, sandBox); err != nil { + t.Fatalf("doCreateTimeEnvdInit err=%v", err) + } + envVars, ok := gotBody["envVars"].(map[string]any) + if !ok { + t.Fatalf("envVars payload missing: %#v", gotBody) + } + if envVars["SESSION_ID"] != "user-session-test" { + t.Fatalf("SESSION_ID=%v, want user-session-test", envVars["SESSION_ID"]) + } + if envVars["USER_ID"] != "42" { + t.Fatalf("USER_ID=%v, want 42", envVars["USER_ID"]) + } +} + +func TestDoCreateTimeEnvdInitFailsOnHTTPError(t *testing.T) { + callCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + callCount++ + http.Error(w, "envd refused init", http.StatusInternalServerError) + })) + defer server.Close() + + u, err := neturl.Parse(server.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + port, err := strconv.Atoi(u.Port()) + if err != nil { + t.Fatalf("parse server port: %v", err) + } + + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationComponentEnvdVersion: "0.2.0", + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: server.Client(), envdInitPort: port} + retErr := l.doCreateTimeEnvdInit(context.Background(), req, sandBox) + if retErr == nil { + t.Fatal("expected init failure") + } + errInfo, _ := ret.FromError(retErr) + assert.Equal(t, errorcode.ErrorCode_ExecCommandInSandboxFailed, errInfo.Code()) + assert.Contains(t, errInfo.Message(), "envd refused init") + assert.Equal(t, 1, callCount) +} + +func TestDoCreateTimeEnvdInitRetriesTransientHTTPError(t *testing.T) { + callCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + callCount++ + if callCount < envdInitMaxAttempts { + http.Error(w, "envd warming up", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + u, err := neturl.Parse(server.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + port, err := strconv.Atoi(u.Port()) + if err != nil { + t.Fatalf("parse server port: %v", err) + } + + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationComponentEnvdVersion: "0.2.0", + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: server.Client(), envdInitPort: port} + if err := l.doCreateTimeEnvdInit(context.Background(), req, sandBox); err != nil { + t.Fatalf("doCreateTimeEnvdInit err=%v", err) + } + assert.Equal(t, envdInitMaxAttempts, callCount) +} + +func TestDoCreateTimeEnvdInitRetriesTransportError(t *testing.T) { + callCount := 0 + client := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + callCount++ + if callCount < envdInitMaxAttempts { + return nil, fmt.Errorf("dial tcp %s: connection refused", r.URL.Host) + } + return &http.Response{ + StatusCode: http.StatusNoContent, + Header: make(http.Header), + Body: io.NopCloser(bytes.NewReader(nil)), + Request: r, + }, nil + }), + } + + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationComponentEnvdVersion: "0.2.0", + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: client, envdInitPort: 49983} + if err := l.doCreateTimeEnvdInit(context.Background(), req, sandBox); err != nil { + t.Fatalf("doCreateTimeEnvdInit err=%v", err) + } + assert.Equal(t, envdInitMaxAttempts, callCount) +} + +func TestDoCreateTimeEnvdInitFallsBackWithoutEnvdSupportAnnotation(t *testing.T) { + called := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + if r.URL.Path != "/init" { + t.Fatalf("path=%q, want /init", r.URL.Path) + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + u, err := neturl.Parse(server.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + port, err := strconv.Atoi(u.Port()) + if err != nil { + t.Fatalf("parse server port: %v", err) + } + + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: server.Client(), envdInitPort: port} + if err := l.doCreateTimeEnvdInit(context.Background(), req, sandBox); err != nil { + t.Fatalf("doCreateTimeEnvdInit err=%v", err) + } + if !called { + t.Fatal("expected missing envd support annotation to still issue envd init request") + } +} + +func TestDoCreateTimeEnvdInitFailsWithoutEnvdSupportAnnotationWhenEnvdUnavailable(t *testing.T) { + client := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("dial tcp %s: connection refused", r.URL.Host) + }), + } + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: client, envdInitPort: 49983} + retErr := l.doCreateTimeEnvdInit(context.Background(), req, sandBox) + if retErr == nil { + t.Fatal("expected init failure when envd init cannot be reached without envd support annotation") + } + errInfo, _ := ret.FromError(retErr) + assert.Equal(t, errorcode.ErrorCode_ExecCommandInSandboxFailed, errInfo.Code()) + assert.Contains(t, errInfo.Message(), "connection refused") +} + func TestProbe(t *testing.T) { testPort := 7997 testHost := "127.0.0.1" diff --git a/examples/code-sandbox-quickstart/README.md b/examples/code-sandbox-quickstart/README.md index e8214221e..b74f75628 100644 --- a/examples/code-sandbox-quickstart/README.md +++ b/examples/code-sandbox-quickstart/README.md @@ -122,10 +122,11 @@ hello cube | `exec_code.py` | `sandbox.run_code()` — execute Python code inside a sandbox | | `cmd.py` | `sandbox.commands.run()` — execute shell commands | | `create.py` | `sandbox.get_info()` — retrieve sandbox metadata | +| `create_with_envs.py` | `Sandbox.create(envs=...)` — pass create-time environment variables | | `read.py` | `sandbox.files.read()` — read a file from the sandbox filesystem | | `pause.py` | `sandbox.pause()` / `sandbox.connect()` — snapshot and restore | -| `auto_resume.py` | `lifecycle={"on_timeout": "pause", "auto_resume": True}` — let the platform pause idle sandboxes and resume them on the next request | -| `auto_kill.py` | `lifecycle={"on_timeout": "kill"}` — let the platform tear down idle sandboxes (the default — destruction is irreversible, the sandbox cannot be resumed) | +| `auto-resume.py` | `lifecycle={"on_timeout": "pause", "auto_resume": True}` — let the platform pause idle sandboxes and resume them on the next request | +| `auto-kill.py` | `lifecycle={"on_timeout": "kill"}` — let the platform tear down idle sandboxes (the default — destruction is irreversible, the sandbox cannot be resumed) | | `network_no_internet.py` | `allow_internet_access=False` — fully air-gapped sandbox | | `network_allowlist.py` | `allow_out` — whitelist specific CIDRs, block everything else | | `network_denylist.py` | `deny_out` — block specific CIDRs, allow the rest | @@ -146,6 +147,21 @@ with Sandbox.create(template=template_id) as sandbox: print(result.stdout) ``` +### Create-Time Environment Variables + +You can pass environment variables when creating a sandbox. They are then +available to subsequent command execution in that sandbox: + +```python +python create_with_envs.py +``` + +Expected output: + +```text +user-session-test +``` + ### pause.py — Pause & Resume Snapshot a running sandbox to free compute resources, then restore it later: @@ -158,7 +174,7 @@ with Sandbox.create(template=template_id) as sandbox: print(sandbox.get_info()) ``` -### auto_resume.py — Auto Pause & Auto Resume +### auto-resume.py — Auto Pause & Auto Resume Like `pause.py`, but the platform handles the pause/resume cycle on its own. The `lifecycle` argument mirrors the e2b SDK @@ -178,9 +194,9 @@ sandbox.run_code("print('back from a transparent resume')") sandbox.kill() ``` -### auto_kill.py — Auto Kill on Idle Timeout +### auto-kill.py — Auto Kill on Idle Timeout -The destructive twin of `auto_resume.py`. Setting `on_timeout="kill"` (also the +The destructive twin of `auto-resume.py`. Setting `on_timeout="kill"` (also the default when no `lifecycle` is passed) tells the platform to tear the sandbox down once it idles past `timeout` — no snapshot is kept, the next request fails fast with **410 Gone**: @@ -258,10 +274,12 @@ code-sandbox-quickstart/ ├── exec_code.py # Run Python code inside a sandbox ├── cmd.py # Execute shell commands ├── create.py # Create sandbox and inspect metadata +├── create_with_envs.py # Create sandbox with create-time env vars +├── env_utils.py # Shared .env loader helper ├── read.py # Read files from the sandbox filesystem ├── pause.py # Pause and resume a sandbox -├── auto_resume.py # Auto-pause / auto-resume on idle timeout -├── auto_kill.py # Auto-kill on idle timeout (destruction is final) +├── auto-resume.py # Auto-pause / auto-resume on idle timeout +├── auto-kill.py # Auto-kill on idle timeout (destruction is final) ├── network_no_internet.py # Fully air-gapped sandbox ├── network_allowlist.py # Outbound CIDR allowlist ├── network_denylist.py # Outbound CIDR denylist diff --git a/examples/code-sandbox-quickstart/README_zh.md b/examples/code-sandbox-quickstart/README_zh.md index faae1d009..db81ec06e 100644 --- a/examples/code-sandbox-quickstart/README_zh.md +++ b/examples/code-sandbox-quickstart/README_zh.md @@ -117,10 +117,11 @@ hello cube | `exec_code.py` | `sandbox.run_code()` — 在沙箱中执行 Python 代码 | | `cmd.py` | `sandbox.commands.run()` — 执行 Shell 命令 | | `create.py` | `sandbox.get_info()` — 获取沙箱元数据 | +| `create_with_envs.py` | `Sandbox.create(envs=...)` — 创建时注入环境变量 | | `read.py` | `sandbox.files.read()` — 读取沙箱文件系统中的文件 | | `pause.py` | `sandbox.pause()` / `sandbox.connect()` — 快照与恢复 | -| `auto_resume.py` | `lifecycle={"on_timeout": "pause", "auto_resume": True}` — 平台在空闲超时后自动暂停沙箱,下一次请求自动恢复 | -| `auto_kill.py` | `lifecycle={"on_timeout": "kill"}` — 平台在空闲超时后直接销毁沙箱(默认行为,销毁不可逆,沙箱无法恢复) | +| `auto-resume.py` | `lifecycle={"on_timeout": "pause", "auto_resume": True}` — 平台在空闲超时后自动暂停沙箱,下一次请求自动恢复 | +| `auto-kill.py` | `lifecycle={"on_timeout": "kill"}` — 平台在空闲超时后直接销毁沙箱(默认行为,销毁不可逆,沙箱无法恢复) | | `network_no_internet.py` | `allow_internet_access=False` — 完全断网沙箱 | | `network_allowlist.py` | `allow_out` — 白名单 CIDR,拦截其余所有出口 | | `network_denylist.py` | `deny_out` — 黑名单 CIDR,其余放行 | @@ -141,6 +142,20 @@ with Sandbox.create(template=template_id) as sandbox: print(result.stdout) ``` +### 创建时注入环境变量 + +可以在创建沙箱时传入环境变量,后续在该沙箱中的命令执行也可以读取到这些变量: + +```python +python create_with_envs.py +``` + +预期输出: + +```text +user-session-test +``` + ### pause.py — 暂停与恢复 将运行中的沙箱快照以释放计算资源,之后恢复: @@ -153,7 +168,7 @@ with Sandbox.create(template=template_id) as sandbox: print(sandbox.get_info()) ``` -### auto_resume.py — 自动暂停与自动恢复 +### auto-resume.py — 自动暂停与自动恢复 与 `pause.py` 类似,但暂停/恢复完全交给平台自动管理。`lifecycle` 参数与 e2b SDK 对齐 (参考 [e2b 文档](https://e2b.dev/docs/sandbox/auto-resume)):`on_timeout="pause"` @@ -172,9 +187,9 @@ sandbox.run_code("print('back from a transparent resume')") sandbox.kill() ``` -### auto_kill.py — 空闲超时后自动销毁 +### auto-kill.py — 空闲超时后自动销毁 -`auto_resume.py` 的孪生销毁版本。`on_timeout="kill"`(不传 `lifecycle` +`auto-resume.py` 的孪生销毁版本。`on_timeout="kill"`(不传 `lifecycle` 时的默认值)告诉平台:沙箱空闲超过 `timeout` 后直接拆除 VM,不 保留快照,下一次请求会以 **410 Gone** 快速失败: @@ -251,10 +266,12 @@ code-sandbox-quickstart/ ├── exec_code.py # 在沙箱中运行 Python 代码 ├── cmd.py # 执行 Shell 命令 ├── create.py # 创建沙箱并查看元数据 +├── create_with_envs.py # 创建时注入环境变量 +├── env_utils.py # 共享的 .env 加载辅助脚本 ├── read.py # 读取沙箱文件系统中的文件 ├── pause.py # 暂停与恢复沙箱 -├── auto_resume.py # 自动暂停 / 自动恢复(基于空闲超时) -├── auto_kill.py # 空闲超时后自动销毁(不可恢复) +├── auto-resume.py # 自动暂停 / 自动恢复(基于空闲超时) +├── auto-kill.py # 空闲超时后自动销毁(不可恢复) ├── network_no_internet.py # 完全断网沙箱 ├── network_allowlist.py # 出口 CIDR 白名单 ├── network_denylist.py # 出口 CIDR 黑名单 diff --git a/examples/code-sandbox-quickstart/create_with_envs.py b/examples/code-sandbox-quickstart/create_with_envs.py new file mode 100644 index 000000000..d2056f10b --- /dev/null +++ b/examples/code-sandbox-quickstart/create_with_envs.py @@ -0,0 +1,20 @@ +# Copyright (c) 2024 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 + +import os +from e2b_code_interpreter import Sandbox +from env_utils import load_local_dotenv + +load_local_dotenv() + +template_id = os.environ["CUBE_TEMPLATE_ID"] + +with Sandbox.create( + template=template_id, + envs={ + "API_TOKEN": "demo-token", + "SESSION_ID": "user-session-test", + }, +) as sandbox: + result = sandbox.commands.run("echo $SESSION_ID") + print(result.stdout) From 48a958c596729400deccf30b2bfef717e1773b3a Mon Sep 17 00:00:00 2001 From: ls-ggg <335814617@qq.com> Date: Wed, 1 Jul 2026 14:15:40 +0800 Subject: [PATCH 2/2] docs: add sandbox logs guide (cubecli logs) Introduce a new guide page in both English and Chinese explaining how to retrieve sandbox and template build logs via cubecli logs. Key points covered: - WIP notice: log retrieval is actively iterated; current interface is temporary - cubecli logs reads the container init-process stdout/stderr; must be run on the node (accesses Cubelet mount namespace) - --tpl flag for template build logs (host filesystem, no ns entry) - Flag reference: --stderr, --all, --tail N, --head N - Log file paths for sandbox vs template - Scope: init-process only; envd sub-task logs retrieved via E2B SDK (on_stdout / on_stderr callbacks), refer to E2B SDK docs - Limitations: no --follow, logs deleted with sandbox Add nav entries in config.mjs under the Operations section (en & zh). Signed-off-by: coolli <335814617@qq.com> --- docs/.vitepress/config.mjs | 2 + docs/guide/sandbox-logs.md | 92 +++++++++++++++++++++++++++++++++++ docs/zh/guide/sandbox-logs.md | 92 +++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 docs/guide/sandbox-logs.md create mode 100644 docs/zh/guide/sandbox-logs.md diff --git a/docs/.vitepress/config.mjs b/docs/.vitepress/config.mjs index 5cc8a7d51..4da6f1c4c 100644 --- a/docs/.vitepress/config.mjs +++ b/docs/.vitepress/config.mjs @@ -148,6 +148,7 @@ export default withMermaid(defineConfig({ items: [ { text: 'WebUI Dashboard', link: '/guide/webui' }, { text: 'Service Management & Logs', link: '/guide/service-management' }, + { text: 'Sandbox Logs', link: '/guide/sandbox-logs' }, { text: 'Template Inspection & Request Preview', link: '/guide/template-inspection-and-preview' }, { text: 'HTTPS & Domain Resolution', link: '/guide/https-and-domain' }, { text: 'Network Hardening', link: '/guide/network-hardening' }, @@ -273,6 +274,7 @@ export default withMermaid(defineConfig({ items: [ { text: 'WebUI 控制台', link: '/zh/guide/webui' }, { text: '服务管理与日志', link: '/zh/guide/service-management' }, + { text: '沙箱日志', link: '/zh/guide/sandbox-logs' }, { text: '模板检查与请求预览', link: '/zh/guide/template-inspection-and-preview' }, { text: 'HTTPS 证书与域名解析', link: '/zh/guide/https-and-domain' }, { text: '网络加固', link: '/zh/guide/network-hardening' }, diff --git a/docs/guide/sandbox-logs.md b/docs/guide/sandbox-logs.md new file mode 100644 index 000000000..7af835ce3 --- /dev/null +++ b/docs/guide/sandbox-logs.md @@ -0,0 +1,92 @@ +--- +title: Sandbox Logs +lang: en-US +--- + +# Sandbox Logs + +::: warning Work in progress +Log retrieval is still being actively iterated. The `cubecli logs` command described here is a **temporary solution** — the interface and underlying storage layout may change in future releases. +::: + +CubeSandbox exposes two complementary logging layers: + +| Layer | What it captures | How to access | +|---|---|---| +| **Sandbox log** | stdout/stderr of the container init process (the main entrypoint) | `cubecli logs` (this page) | +| **`envd` task log** | stdout/stderr of individual `exec` sub-tasks spawned inside the sandbox | E2B SDK (`on_stdout` / `on_stderr` callbacks) | + +This page covers the **sandbox-level log** only. For `envd` sub-task logs, refer to the [E2B SDK documentation](https://e2b.dev/docs). + +## Prerequisites + +`cubecli` is built alongside Cubelet and installed as part of the standard one-click deployment. The `logs` sub-command accesses log files that live inside the **Cubelet mount namespace**, so it must be run **directly on the compute node** — it cannot be executed remotely via the API or from a non-node host. + +## Reading sandbox logs + +```bash +# Last 100 lines of stdout (default) +cubecli logs + +# Last 100 lines of stderr +cubecli logs --stderr + +# Full log (all lines) +cubecli logs --all + +# Last N lines +cubecli logs --tail 50 +# Short form +cubecli logs -t 50 + +# First N lines +cubecli logs --head 20 +# Short form +cubecli logs -H 20 +``` + +### Flag reference + +| Flag | Short | Description | +|---|---|---| +| `--stderr` | `-e` | Read stderr instead of stdout | +| `--all` | `-a` | Print all lines; cannot be combined with `--tail` or `--head` | +| `--tail N` | `-t N` | Print the last N lines (default: 100 when no other flag is set) | +| `--head N` | `-H N` | Print the first N lines | + +## Reading template build logs + +During template construction the container's stdout/stderr are saved to the host filesystem under `/data/log/template/_0/`. These files do **not** require entering the Cubelet mount namespace, so `--tpl` skips the namespace re-exec: + +```bash +# Last 100 lines of template build stdout +cubecli logs --tpl + +# Full build stderr +cubecli logs --tpl --all --stderr +``` + +## Where the log files live + +| Context | Path | +|---|---| +| Sandbox stdout | `/data/cubelet/state/io.containerd.runtime.v2.task/default//stdout` (inside Cubelet mount namespace) | +| Sandbox stderr | `/data/cubelet/state/io.containerd.runtime.v2.task/default//stderr` (inside Cubelet mount namespace) | +| Template stdout | `/data/log/template/_0/stdout` (host filesystem) | +| Template stderr | `/data/log/template/_0/stderr` (host filesystem) | + +::: tip Why the mount namespace? +Sandbox log files are written by CubeShim into the bundle directory which is only visible inside Cubelet's private mount namespace. `cubecli logs` automatically re-execs itself into that namespace before reading — you do not need to do anything special beyond running the command on the node. +::: + +## Scope and limitations + +- These logs capture only the **init process** (PID 1 inside the container). Output from processes spawned via `exec` calls is captured through the E2B SDK's `on_stdout` / `on_stderr` callbacks — refer to the [E2B SDK documentation](https://e2b.dev/docs) for details. +- Log forwarding must be enabled on the running CubeShim version (available since v0.4.0). On older deployments the log file will be missing. +- Logs are not streamed in real time — there is no `--follow` flag yet. Re-run the command to see new output. +- Log files are removed when the sandbox is deleted. + +## Related + +- [Service Management & Logs](./service-management.md) — host-side service logs, journalctl, and the diagnostic bundle +- [Template Inspection & Request Preview](./template-inspection-and-preview.md) diff --git a/docs/zh/guide/sandbox-logs.md b/docs/zh/guide/sandbox-logs.md new file mode 100644 index 000000000..0ff19125b --- /dev/null +++ b/docs/zh/guide/sandbox-logs.md @@ -0,0 +1,92 @@ +--- +title: 沙箱日志 +lang: zh-CN +--- + +# 沙箱日志 + +::: warning 功能迭代中 +日志能力目前仍在持续迭代,本文介绍的 `cubecli logs` 命令是**临时使用版本**——后续版本可能对接口及底层存储结构进行调整。 +::: + +CubeSandbox 提供两个互补的日志层: + +| 层级 | 记录内容 | 获取方式 | +|---|---|---| +| **沙箱日志** | 容器 init 进程(主入口)的 stdout/stderr | `cubecli logs`(本文) | +| **`envd` 任务日志** | 在沙箱内通过 `exec` 接口启动的子任务的 stdout/stderr | E2B SDK(`on_stdout` / `on_stderr` 回调) | + +本文仅介绍**沙箱级别的日志**。`envd` 子任务的日志获取方式请参阅 [E2B SDK 文档](https://e2b.dev/docs)。 + +## 前置条件 + +`cubecli` 随 Cubelet 一同构建,并在一键部署时自动安装。`logs` 子命令访问的日志文件位于 **Cubelet 挂载命名空间**内,因此必须**直接在计算节点上执行**,无法通过 API 或非节点主机远程调用。 + +## 读取沙箱日志 + +```bash +# 最后 100 行 stdout(默认) +cubecli logs + +# 最后 100 行 stderr +cubecli logs --stderr + +# 完整日志(全部行) +cubecli logs --all + +# 最后 N 行 +cubecli logs --tail 50 +# 简写 +cubecli logs -t 50 + +# 前 N 行 +cubecli logs --head 20 +# 简写 +cubecli logs -H 20 +``` + +### 参数说明 + +| 参数 | 简写 | 说明 | +|---|---|---| +| `--stderr` | `-e` | 读取 stderr,默认读取 stdout | +| `--all` | `-a` | 输出全部行;不可与 `--tail` 或 `--head` 同时使用 | +| `--tail N` | `-t N` | 输出最后 N 行(未指定其他标志时默认为 100) | +| `--head N` | `-H N` | 输出前 N 行 | + +## 读取模板构建日志 + +模板构建过程中,容器的 stdout/stderr 会写入宿主机文件系统 `/data/log/template/_0/`。这些文件无需进入 Cubelet 挂载命名空间,使用 `--tpl` 标志可跳过命名空间切换: + +```bash +# 最后 100 行模板构建 stdout +cubecli logs --tpl + +# 完整模板构建 stderr +cubecli logs --tpl --all --stderr +``` + +## 日志文件路径 + +| 场景 | 路径 | +|---|---| +| 沙箱 stdout | `/data/cubelet/state/io.containerd.runtime.v2.task/default//stdout`(Cubelet 挂载命名空间内) | +| 沙箱 stderr | `/data/cubelet/state/io.containerd.runtime.v2.task/default//stderr`(Cubelet 挂载命名空间内) | +| 模板 stdout | `/data/log/template/_0/stdout`(宿主机文件系统) | +| 模板 stderr | `/data/log/template/_0/stderr`(宿主机文件系统) | + +::: tip 为什么需要挂载命名空间? +沙箱日志文件由 CubeShim 写入 bundle 目录,该目录仅在 Cubelet 的私有挂载命名空间内可见。`cubecli logs` 会在读取前自动重新进入该命名空间——你无需任何额外操作,只需在节点上直接运行该命令即可。 +::: + +## 范围与限制 + +- 这些日志仅记录 **init 进程**(容器内 PID 1)的输出。通过 `exec` 接口启动的进程输出需通过 E2B SDK 的 `on_stdout` / `on_stderr` 回调获取,详见 [E2B SDK 文档](https://e2b.dev/docs)。 +- 日志转发需要 v0.4.0 及以上版本的 CubeShim。在更旧的部署上,日志文件将不存在。 +- 日志目前不支持实时流式读取,暂无 `--follow` 标志。如需查看最新输出,请重新执行命令。 +- 沙箱删除后,对应的日志文件会一并清除。 + +## 相关文档 + +- [服务管理与日志](./service-management.md) — 宿主机服务日志、journalctl 及诊断包 +- [模板检查与请求预览](./template-inspection-and-preview.md)