diff --git a/CubeAPI/src/cubemaster/mod.rs b/CubeAPI/src/cubemaster/mod.rs index bff23c375..02702fc75 100644 --- a/CubeAPI/src/cubemaster/mod.rs +++ b/CubeAPI/src/cubemaster/mod.rs @@ -564,7 +564,10 @@ impl CubeMasterError { /// as a potential source of routing ambiguity; /// * `.` and `..` are reserved for relative path resolution and easily slip /// through naive equality checks. -fn validate_path_segment(name: &'static str, value: &str) -> Result<(), CubeMasterError> { +pub(crate) fn validate_path_segment( + name: &'static str, + value: &str, +) -> Result<(), CubeMasterError> { let is_valid = !value.is_empty() && value .bytes() @@ -637,6 +640,15 @@ pub struct CreateSandboxRequest { #[serde(skip_serializing_if = "Option::is_none")] pub labels: Option>, + #[serde( + rename = "create_time_env_vars", + skip_serializing_if = "Option::is_none" + )] + /// Sandbox-level env vars requested at create time. CubeMaster forwards + /// them to cubelet via an internal annotation, and cubelet uses them to + /// initialize envd after sandbox startup. + pub create_time_env_vars: Option>, + #[serde(rename = "distribution_scope", skip_serializing_if = "Option::is_none")] pub distribution_scope: Option>, diff --git a/CubeAPI/src/models/mod.rs b/CubeAPI/src/models/mod.rs index 9061215e3..b71338b18 100644 --- a/CubeAPI/src/models/mod.rs +++ b/CubeAPI/src/models/mod.rs @@ -163,6 +163,8 @@ pub struct SandboxVolumeMount { /// Rule: ID abbreviations → uppercase (templateID, sandboxID, envVars); /// allow_internet_access is a known SDK snake_case quirk; /// lifecycle is a nested object — see SandboxLifecycleConfig. +/// `envVars` is the canonical field name; `envs` is accepted as a compatibility +/// alias for E2B SDK callers. #[derive(Debug, Deserialize, Validate, ToSchema)] #[allow(dead_code)] pub struct NewSandbox { @@ -199,7 +201,11 @@ pub struct NewSandbox { )] pub distribution_scope: Option>, - #[serde(rename = "envVars", skip_serializing_if = "Option::is_none")] + #[serde( + alias = "envs", + rename = "envVars", + skip_serializing_if = "Option::is_none" + )] pub env_vars: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -510,7 +516,7 @@ fn default_page_limit() -> i32 { #[cfg(test)] mod tests { - use super::SandboxNetworkConfig; + use super::{NewSandbox, SandboxNetworkConfig}; #[test] fn sandbox_network_config_accepts_snake_case_policy_fields() { @@ -526,6 +532,25 @@ mod tests { ); assert_eq!(cfg.deny_out, Some(vec!["0.0.0.0/0".to_string()])); } + + #[test] + fn new_sandbox_accepts_e2b_envs_alias() { + let req: NewSandbox = serde_json::from_value(serde_json::json!({ + "templateID": "tpl-1", + "envs": { + "CUBE_TEST_ENV": "value" + } + })) + .expect("new sandbox request should deserialize"); + + assert_eq!( + req.env_vars + .as_ref() + .and_then(|envs| envs.get("CUBE_TEST_ENV")) + .map(String::as_str), + Some("value") + ); + } } // ─── Templates ───────────────────────────────────────────────────────────── diff --git a/CubeAPI/src/services/sandboxes.rs b/CubeAPI/src/services/sandboxes.rs index f590088aa..9b631155c 100644 --- a/CubeAPI/src/services/sandboxes.rs +++ b/CubeAPI/src/services/sandboxes.rs @@ -3,7 +3,6 @@ // use std::collections::HashMap; - use uuid::Uuid; use super::validate_allow_out_domains_require_deny_all; @@ -28,6 +27,34 @@ const RET_CODE_HTTP_OK: i32 = 200; const RET_CODE_NOT_FOUND: i32 = 130404; const RET_CODE_CONFLICT: i32 = 130409; const HOSTDIR_MOUNT_KEY: &str = "host-mount"; +const ENV_VAR_NAME_MAX_LEN: usize = 256; +const ENV_VAR_VALUE_MAX_LEN: usize = 4096; + +/// Environment variable names that may compromise sandbox isolation if injected +/// at the runtime level (loader overrides, language runtime paths). +const FORBIDDEN_ENV_NAMES: &[&str] = &[ + "BASH_ENV", + "ENV", + "LD_PRELOAD", + "LD_AUDIT", + "LD_LIBRARY_PATH", + "LD_ORIGIN_PATH", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + "GCONV_PATH", + "PATH", + "PYTHONPATH", + "NODE_PATH", + "JAVA_TOOL_OPTIONS", + "_JAVA_OPTIONS", + "GEM_PATH", + "RUBYOPT", + "RUBYLIB", + "PERL5LIB", + "PERLLIB", + "CLASSPATH", + "IFS", +]; #[derive(Clone)] pub struct SandboxService { @@ -117,7 +144,20 @@ impl SandboxService { } pub async fn create_sandbox(&self, body: NewSandbox) -> AppResult { - let template_id = body.template_id.clone(); + let NewSandbox { + template_id, + timeout, + lifecycle, + allow_internet_access, + network, + metadata, + distribution_scope, + env_vars, + .. + } = body; + if let Some(env_vars) = env_vars.as_ref() { + validate_env_vars(env_vars)?; + } let mut annotations = HashMap::from([ ( "cube.master.appsnapshot.template.id".to_string(), @@ -129,7 +169,7 @@ impl SandboxService { ), ]); - let labels = body.metadata.map(|mut meta| { + let labels = metadata.map(|mut meta| { if let Some(value) = meta.remove(HOSTDIR_MOUNT_KEY) { annotations.insert(HOSTDIR_MOUNT_KEY.to_string(), value); } @@ -137,13 +177,12 @@ impl SandboxService { }); let cube_network_config = - build_cube_network_config(body.allow_internet_access, body.network.as_ref())?; + build_cube_network_config(allow_internet_access, network.as_ref())?; // Derive the two CubeMaster-side bools from the e2b-shaped lifecycle // object. Absent lifecycle keeps today's behaviour: idle sandboxes // are killed (auto_pause = false), and auto_resume defaults off. - let (auto_pause, auto_resume) = body - .lifecycle + let (auto_pause, auto_resume) = lifecycle .as_ref() .map(|lc| { use crate::models::SandboxOnTimeout; @@ -157,10 +196,11 @@ impl SandboxService { let req = CreateSandboxRequest { request_id: new_request_id(), instance_type: self.instance_type.clone(), - timeout: Some(body.timeout), + timeout: Some(timeout), annotations, labels, - distribution_scope: body.distribution_scope, + create_time_env_vars: env_vars, + distribution_scope, volumes: None, containers: vec![], exposed_ports: vec![], @@ -519,6 +559,56 @@ impl SandboxService { } } +/// Validate environment variable names against the POSIX name convention +/// and a deny-list of runtime-loader / path-override names that could +/// compromise sandbox isolation. +fn validate_env_vars(env_vars: &HashMap) -> AppResult<()> { + for (name, value) in env_vars { + if name.is_empty() || name.len() > ENV_VAR_NAME_MAX_LEN { + return Err(AppError::BadRequest(format!( + "invalid env var name length: {name:?}" + ))); + } + let bytes = name.as_bytes(); + if !bytes + .first() + .map_or(false, |b| b.is_ascii_alphabetic() || *b == b'_') + || !bytes + .iter() + .all(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + return Err(AppError::BadRequest(format!( + "env var name must match [a-zA-Z_][a-zA-Z0-9_]*: {name:?}" + ))); + } + if FORBIDDEN_ENV_NAMES + .iter() + .any(|forbidden| name.eq_ignore_ascii_case(forbidden)) + { + return Err(AppError::BadRequest(format!( + "env var name not allowed: {name}" + ))); + } + if value.len() > ENV_VAR_VALUE_MAX_LEN { + return Err(AppError::BadRequest(format!( + "env var value too large for {name:?}: {} bytes", + value.len() + ))); + } + if value.contains('\0') { + return Err(AppError::BadRequest(format!( + "env var value contains NUL byte: {name:?}" + ))); + } + if value.chars().any(|ch| ch != '\t' && ch.is_control()) { + return Err(AppError::BadRequest(format!( + "env var value contains control character: {name:?}" + ))); + } + } + Ok(()) +} + fn internal_error(error: impl std::fmt::Display) -> AppError { AppError::Internal(anyhow::anyhow!(error.to_string())) } @@ -780,13 +870,21 @@ fn map_egress_rule(rule: &EgressRule) -> CubeEgressRule { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Arc; - use super::{build_cube_network_config, filter_by_metadata, from_cubemaster_info}; - use crate::cubemaster::{CreateSandboxRequest, ListSandboxResponse, SandboxInfo}; + use super::{ + build_cube_network_config, filter_by_metadata, from_cubemaster_info, SandboxService, + }; + use crate::cubemaster::{ + CreateSandboxRequest, CubeMasterClient, ListSandboxResponse, SandboxInfo, + }; use crate::models::{ - EgressRule, EgressRuleAction, EgressRuleInject, EgressRuleMatch, SandboxNetworkConfig, - SandboxState, + EgressRule, EgressRuleAction, EgressRuleInject, EgressRuleMatch, NewSandbox, + SandboxNetworkConfig, SandboxState, }; + use axum::{extract::State, routing::post, Json, Router}; + use serde_json::Value; + use tokio::sync::Mutex; #[test] fn metadata_filter_matches_all_pairs() { @@ -1030,12 +1128,13 @@ mod tests { timeout: Some(60), annotations: HashMap::new(), labels: None, + create_time_env_vars: None, + distribution_scope: None, volumes: None, containers: vec![], exposed_ports: vec![], network_type: None, cube_network_config: None, - distribution_scope: None, auto_pause: false, auto_resume: false, }; @@ -1126,4 +1225,258 @@ mod tests { .unwrap(); assert_eq!(translate(&empty), (false, false)); } + + #[tokio::test] + async fn create_sandbox_forwards_create_time_env_vars_to_cubemaster() { + #[derive(Clone, Default)] + struct Capture { + create_body: Arc>>, + } + + async fn create_handler( + State(capture): State, + Json(body): Json, + ) -> Json { + *capture.create_body.lock().await = Some(body); + Json(serde_json::json!({ + "requestID": "req-1", + "sandbox_id": "sb-123", + "ret": { "ret_code": 0, "ret_msg": "ok" } + })) + } + + async fn spawn_server(app: Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("server should run"); + }); + format!("http://{}", addr) + } + + let capture = Capture::default(); + let cubemaster_url = spawn_server( + Router::new() + .route("/cube/sandbox", post(create_handler)) + .with_state(capture.clone()), + ) + .await; + + let service = SandboxService::new( + CubeMasterClient::new(cubemaster_url, reqwest::Client::new()), + "cubebox".to_string(), + "cube.app".to_string(), + ); + + let env_vars = HashMap::from([( + "CUBE_TEST_CREATE_ENV".to_string(), + "from-create".to_string(), + )]); + let sandbox = service + .create_sandbox(NewSandbox { + template_id: "tpl-1".to_string(), + timeout: 15, + lifecycle: None, + secure: None, + allow_internet_access: None, + network: None, + metadata: None, + distribution_scope: None, + env_vars: Some(env_vars), + mcp: None, + volume_mounts: None, + }) + .await + .expect("sandbox create should succeed"); + + assert_eq!(sandbox.sandbox_id, "sb-123"); + let create_body = capture + .create_body + .lock() + .await + .clone() + .expect("create body"); + assert_eq!( + create_body["create_time_env_vars"]["CUBE_TEST_CREATE_ENV"], + serde_json::json!("from-create") + ); + assert!(create_body.get("envVars").is_none()); + } + + #[tokio::test] + async fn create_sandbox_omits_create_time_env_vars_when_absent() { + #[derive(Clone, Default)] + struct Capture { + create_body: Arc>>, + } + + async fn create_handler( + State(capture): State, + Json(body): Json, + ) -> Json { + *capture.create_body.lock().await = Some(body); + Json(serde_json::json!({ + "requestID": "req-1", + "sandbox_id": "sb-no-envs", + "ret": { "ret_code": 0, "ret_msg": "ok" } + })) + } + + async fn spawn_server(app: Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("server should run"); + }); + format!("http://{}", addr) + } + + let capture = Capture::default(); + let cubemaster_url = spawn_server( + Router::new() + .route("/cube/sandbox", post(create_handler)) + .with_state(capture.clone()), + ) + .await; + + let service = SandboxService::new( + CubeMasterClient::new(cubemaster_url, reqwest::Client::new()), + "cubebox".to_string(), + "cube.app".to_string(), + ); + + let sandbox = service + .create_sandbox(NewSandbox { + template_id: "tpl-1".to_string(), + timeout: 15, + lifecycle: None, + secure: None, + allow_internet_access: None, + network: None, + metadata: None, + distribution_scope: None, + env_vars: None, + mcp: None, + volume_mounts: None, + }) + .await + .expect("sandbox create should succeed"); + + assert_eq!(sandbox.sandbox_id, "sb-no-envs"); + let create_body = capture + .create_body + .lock() + .await + .clone() + .expect("create body"); + assert!( + create_body.get("create_time_env_vars").is_none(), + "create_time_env_vars should be omitted when caller did not provide envs" + ); + } + + #[test] + fn create_sandbox_rejects_dangerous_env_var_names() { + for name in super::FORBIDDEN_ENV_NAMES { + let err = super::validate_env_vars(&HashMap::from([( + (*name).to_string(), + "val".to_string(), + )])) + .expect_err("dangerous env var name should be rejected"); + assert!( + err.to_string().contains("not allowed"), + "error for {name} should say 'not allowed': {err}" + ); + } + } + + #[test] + fn create_sandbox_rejects_dangerous_env_var_names_case_insensitive() { + for name in ["ld_preload", "Ld_Preload", "LD_PRELOAD"] { + let err = + super::validate_env_vars(&HashMap::from([(name.to_string(), "val".to_string())])) + .expect_err(&format!( + "dangerous env var name {name} should be rejected case-insensitively" + )); + assert!( + err.to_string().contains("not allowed"), + "error for {name} should say 'not allowed': {err}" + ); + } + } + + #[test] + fn create_sandbox_rejects_invalid_env_var_name_format() { + for (name, desc) in [ + ("", "empty"), + ("9VAR", "starts with digit"), + ("MY-VAR", "contains hyphen"), + ("MY.VAR", "contains dot"), + ] { + let err = + super::validate_env_vars(&HashMap::from([(name.to_string(), "v".to_string())])) + .expect_err(&format!("{desc} should be rejected: {name}")); + let msg = err.to_string(); + assert!( + msg.contains("must match") || msg.contains("invalid env var name"), + "error for {desc} ({name}) should mention name validation: {err}" + ); + } + } + + #[test] + fn create_sandbox_rejects_invalid_env_var_value() { + let too_large = "x".repeat(super::ENV_VAR_VALUE_MAX_LEN + 1); + let err = super::validate_env_vars(&HashMap::from([("TOO_LARGE".to_string(), too_large)])) + .expect_err("oversized env var value should be rejected"); + assert!( + err.to_string().contains("value too large"), + "oversized env var value error should mention size: {err}" + ); + + let err = super::validate_env_vars(&HashMap::from([( + "HAS_NUL".to_string(), + "abc\0def".to_string(), + )])) + .expect_err("env var value with NUL should be rejected"); + assert!( + err.to_string().contains("contains NUL"), + "NUL-containing env var value error should mention NUL: {err}" + ); + + let err = super::validate_env_vars(&HashMap::from([( + "HAS_ESC".to_string(), + "line\x1b[31mred".to_string(), + )])) + .expect_err("env var value with control character should be rejected"); + assert!( + err.to_string().contains("control character"), + "control-character env var value error should mention control character: {err}" + ); + + let err = super::validate_env_vars(&HashMap::from([( + "HAS_NEWLINE".to_string(), + "line1\nline2".to_string(), + )])) + .expect_err("env var value with newline should be rejected"); + assert!( + err.to_string().contains("control character"), + "newline env var value error should mention control character: {err}" + ); + } + + #[test] + fn create_sandbox_accepts_valid_env_var_names() { + super::validate_env_vars(&HashMap::from([ + ("MY_VAR".to_string(), "val".to_string()), + ("_underscore_prefix".to_string(), "val".to_string()), + ("CUBE_TEST_ENV".to_string(), "val".to_string()), + ("TAB_OK".to_string(), "hello\tworld".to_string()), + ])) + .expect("valid env var names should be accepted"); + } } diff --git a/CubeMaster/pkg/base/constants/constants.go b/CubeMaster/pkg/base/constants/constants.go index 7386edbe0..ec03736ee 100644 --- a/CubeMaster/pkg/base/constants/constants.go +++ b/CubeMaster/pkg/base/constants/constants.go @@ -79,6 +79,9 @@ const ( CubeAnnotationRootfsArtifactSizeBytes = "cube.master.rootfs.artifact.size_bytes" CubeAnnotationWritableLayerSize = "cube.master.rootfs.writable_layer_size" CubeAnnotationTemplateSpecFingerprint = "cube.master.template.spec_fingerprint" + // CubeAnnotationCreateTimeEnvVars stores the serialized create-time env map + // that CubeMaster passes to cubelet for envd initialization. + CubeAnnotationCreateTimeEnvVars = "cube.master.internal.create_time_env_vars" CubeAnnotationsVirtiofsCache = "cube.master.virtiofs.cache" diff --git a/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go b/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go index fe734606a..51e2a7b6e 100644 --- a/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go +++ b/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go @@ -530,7 +530,6 @@ func dealCubeboxCreateReqWithTemplateCenter(ctx context.Context, templateID stri return err } } - if templateReq.NetworkType != "" { reqInOut.NetworkType = templateReq.NetworkType } diff --git a/CubeMaster/pkg/service/sandbox/types/types.go b/CubeMaster/pkg/service/sandbox/types/types.go index 0068750d5..9c0c51c9f 100644 --- a/CubeMaster/pkg/service/sandbox/types/types.go +++ b/CubeMaster/pkg/service/sandbox/types/types.go @@ -44,8 +44,12 @@ type CreateCubeSandboxReq struct { Containers []*Container `json:"containers,omitempty"` - Annotations map[string]string `json:"annotations,omitempty" ` - Labels map[string]string `json:"labels,omitempty" ` + Annotations map[string]string `json:"annotations,omitempty" ` + Labels map[string]string `json:"labels,omitempty" ` + // CreateTimeEnvVars carries sandbox-level env vars requested at create + // time. CubeMaster serializes them into an internal annotation so cubelet + // can initialize envd after sandbox startup. + CreateTimeEnvVars map[string]string `json:"create_time_env_vars,omitempty"` DistributionScope []string `json:"distribution_scope,omitempty"` InstanceType string `json:"instance_type,omitempty"` NetworkType string `json:"network_type,omitempty"` diff --git a/CubeMaster/pkg/service/sandbox/util.go b/CubeMaster/pkg/service/sandbox/util.go index 01e2f8a00..0a4946d31 100644 --- a/CubeMaster/pkg/service/sandbox/util.go +++ b/CubeMaster/pkg/service/sandbox/util.go @@ -26,6 +26,8 @@ import ( "github.com/tencentcloud/CubeSandbox/CubeMaster/pkg/base/log" ) +const maxCreateTimeEnvVarsAnnotationBytes = 16 * 1024 + func checkAndGetReqResource(req *types.CreateCubeSandboxReq) (*selctx.RequestResource, error) { res := &selctx.RequestResource{ Cpu: resource.MustParse("0"), @@ -683,6 +685,33 @@ func checkAndGetAnnotation(req *types.CreateCubeSandboxReq, out *cubebox.RunCube if v, ok := req.Annotations[constants.CubeAnnotationsInsRegion]; !ok || v == "" { out.Annotations[constants.CubeAnnotationsInsRegion] = config.GetConfig().Log.Region } + if err := setCreateTimeEnvVarsAnnotation(out.Annotations, req.CreateTimeEnvVars); err != nil { + return err + } + return nil +} + +func setCreateTimeEnvVarsAnnotation(out map[string]string, envVars map[string]string) error { + if len(envVars) == 0 { + return nil + } + if out == nil { + return errors.New("annotation output map is nil") + } + // Carry the create-time env map to cubelet so the sandbox runtime can + // initialize envd after startup for envd-backed command execution. + payload, err := utils.JSONTool.Marshal(envVars) + if err != nil { + return fmt.Errorf("marshal create_time_env_vars failed: %w", err) + } + if len(payload) > maxCreateTimeEnvVarsAnnotationBytes { + return fmt.Errorf( + "create_time_env_vars annotation payload too large: %d bytes exceeds limit %d", + len(payload), + maxCreateTimeEnvVarsAnnotationBytes, + ) + } + out[constants.CubeAnnotationCreateTimeEnvVars] = string(payload) return nil } diff --git a/CubeMaster/pkg/service/sandbox/util_env_test.go b/CubeMaster/pkg/service/sandbox/util_env_test.go new file mode 100644 index 000000000..9fdd5f8d3 --- /dev/null +++ b/CubeMaster/pkg/service/sandbox/util_env_test.go @@ -0,0 +1,51 @@ +package sandbox + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/tencentcloud/CubeSandbox/CubeMaster/pkg/base/constants" +) + +func TestSetCreateTimeEnvVarsAnnotation(t *testing.T) { + out := map[string]string{} + envVars := map[string]string{ + "SESSION_ID": "user-session-test", + "USER_ID": "42", + } + + if err := setCreateTimeEnvVarsAnnotation(out, envVars); err != nil { + t.Fatalf("setCreateTimeEnvVarsAnnotation err=%v", err) + } + + raw := out[constants.CubeAnnotationCreateTimeEnvVars] + if raw == "" { + t.Fatalf("missing %s annotation", constants.CubeAnnotationCreateTimeEnvVars) + } + var decoded map[string]string + if err := json.Unmarshal([]byte(raw), &decoded); err != nil { + t.Fatalf("unmarshal create time env vars annotation: %v", err) + } + if decoded["SESSION_ID"] != "user-session-test" { + t.Fatalf("SESSION_ID=%q, want user-session-test", decoded["SESSION_ID"]) + } + if decoded["USER_ID"] != "42" { + t.Fatalf("USER_ID=%q, want 42", decoded["USER_ID"]) + } +} + +func TestSetCreateTimeEnvVarsAnnotationRejectsOversizedPayload(t *testing.T) { + out := map[string]string{} + envVars := map[string]string{ + "OVERSIZED": strings.Repeat("x", maxCreateTimeEnvVarsAnnotationBytes), + } + + err := setCreateTimeEnvVarsAnnotation(out, envVars) + if err == nil { + t.Fatal("expected oversized payload error") + } + if !strings.Contains(err.Error(), "annotation payload too large") { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/Cubelet/pkg/constants/const.go b/Cubelet/pkg/constants/const.go index 414614d07..29e2e53c3 100644 --- a/Cubelet/pkg/constants/const.go +++ b/Cubelet/pkg/constants/const.go @@ -206,6 +206,8 @@ const ( MasterAnnotationRootfsArtifactSizeBytes = "cube.master.rootfs.artifact.size_bytes" MasterAnnotationWritableLayerSize = "cube.master.rootfs.writable_layer_size" MasterAnnotationTemplateSpecFingerprint = "cube.master.template.spec_fingerprint" + MasterAnnotationComponentEnvdVersion = "cube.master.components.envd.version" + MasterAnnotationCreateTimeEnvVars = "cube.master.internal.create_time_env_vars" MasterAnnotationInstanceType = "cube.master.instance.type" MasterAnnotationNetworkPolicyBlockAll = "cube.master.network.policy.block_all" MasterAnnotationNetworkPolicyAllowPublicServices = "cube.master.network.policy.allow_public_services" diff --git a/Cubelet/services/cubebox/cube_container_create.go b/Cubelet/services/cubebox/cube_container_create.go index a4cbcd00c..858b85663 100644 --- a/Cubelet/services/cubebox/cube_container_create.go +++ b/Cubelet/services/cubebox/cube_container_create.go @@ -75,7 +75,8 @@ import ( const ( cubeSharedBindRootPath = "/run/cube-bind-share" - K8sEmptyDirPath = "kubernetes.io~empty-dir" + K8sEmptyDirPath = "kubernetes.io~empty-dir" + envdInitCleanupTimeout = 10 * time.Second ) func init() { @@ -283,58 +284,75 @@ func (l *local) createContainers(ctx context.Context, flowOpts *workflow.CreateC }) } - sandBox.Lock() - defer func() { - if err := l.cubeboxManger.Save(ctx, sandBox); err != nil { - log.G(ctx).Warnf("saveSandBoxInfo failed.%s", err.Error()) - } - sandBox.Unlock() - }() + if err := func() error { + sandBox.Lock() + defer func() { + if err := l.cubeboxManger.Save(ctx, sandBox); err != nil { + log.G(ctx).Warnf("saveSandBoxInfo failed.%s", err.Error()) + } + sandBox.Unlock() + }() - for _, param := range params { - ci := param.ci - containerLog := sanboxlog.WithFields(CubeLog.Fields{ - "containerID": ci.ID, - "isPod": ci.IsPod, - }) - err = func() (retE error) { - containerLog := log.G(ctx).WithField("container-id", ci.ID) - retE = l.runContainer(param.ctxTmp, sandBox, param.ci, param.cOpts, ociRuntime) - withOciSpec := log.IsDebug() || retE != nil - if ci.Container != nil && withOciSpec { - info, err := ci.Container.Info(ctx, containerd.WithoutRefreshedMetadata) - if err == nil { - v, err := typeurl.UnmarshalAny(info.Spec) - if err != nil { - return fmt.Errorf("failed to unmarshal container spec with url %s: %w", info.Spec.GetTypeUrl(), err) + for _, param := range params { + ci := param.ci + containerLog := sanboxlog.WithFields(CubeLog.Fields{ + "containerID": ci.ID, + "isPod": ci.IsPod, + }) + err = func() (retE error) { + containerLog := log.G(ctx).WithField("container-id", ci.ID) + retE = l.runContainer(param.ctxTmp, sandBox, param.ci, param.cOpts, ociRuntime) + withOciSpec := log.IsDebug() || retE != nil + if ci.Container != nil && withOciSpec { + info, err := ci.Container.Info(ctx, containerd.WithoutRefreshedMetadata) + if err == nil { + v, err := typeurl.UnmarshalAny(info.Spec) + if err != nil { + return fmt.Errorf("failed to unmarshal container spec with url %s: %w", info.Spec.GetTypeUrl(), err) + } + jsonstr := log.WithJsonValue(struct { + containers.Container + Spec interface{} `json:"Spec,omitempty"` + }{ + Container: info, + Spec: v, + }) + containerLog.Debugf("container-oci-spec: %s", jsonstr) } - jsonstr := log.WithJsonValue(struct { - containers.Container - Spec interface{} `json:"Spec,omitempty"` - }{ - Container: info, - Spec: v, - }) - containerLog.Debugf("container-oci-spec: %s", jsonstr) } + if retE != nil { + containerLog.Errorf("run container failed.%s", retE.Error()) + } else { + containerLog.Debug("run container success") + } + return retE + }() + if err != nil { + return fmt.Errorf("failed to run container %s: %w", param.ci.ID, err) } - if retE != nil { - containerLog.Errorf("run container failed.%s", retE.Error()) - } else { - containerLog.Debug("run container success") + if err := l.doProbe(param.ctxTmp, param.cntrReq, param.ci); err != nil { + return err + } + err = l.cbriManager.PostCreateContainer(ctx, sandBox, param.ci) + if err != nil { + containerLog.Errorf("post create container failed, err: %v", err) } - return retE - }() - if err != nil { - return fmt.Errorf("failed to run container %s: %w", param.ci.ID, err) - } - if err := l.doProbe(param.ctxTmp, param.cntrReq, param.ci); err != nil { - return err } - err = l.cbriManager.PostCreateContainer(ctx, sandBox, param.ci) - if err != nil { - containerLog.Errorf("post create container failed, err: %v", err) + return nil + }(); err != nil { + return err + } + + if err := l.doCreateTimeEnvdInit(ctx, realReq, sandBox); err != nil { + cleanupErr := l.cleanupAfterEnvdInitFailure(flowOpts, realReq, sandBox) + if cleanupErr == nil { + // The sandbox has already been torn down synchronously, so the outer + // workflow failover does not need to repeat the same destroy path. + flowOpts.Failover = false + } else { + sanboxlog.Errorf("cleanup sandbox after envd init failure failed: %v", cleanupErr) } + return err } pid := sandBox.Endpoint.Pid @@ -357,6 +375,38 @@ func (l *local) createContainers(ctx context.Context, flowOpts *workflow.CreateC return nil } +func (l *local) cleanupAfterEnvdInitFailure(flowOpts *workflow.CreateContext, + realReq *cubebox.RunCubeSandboxRequest, sandBox *cubeboxstore.CubeBox) error { + // CubeMaster already compensates create failures on the main path, but + // cubelet workflow failover skips PreConditionFailed and runtime-local + // callers still benefit from immediate teardown close to the runtime. + cleanupCtx, cancel := context.WithTimeout(context.Background(), envdInitCleanupTimeout) + defer cancel() + if sandBox.Namespace != "" { + cleanupCtx = namespaces.WithNamespace(cleanupCtx, sandBox.Namespace) + } + cleanupCtx = constants.WithFailoverOperation(cleanupCtx) + if flowOpts.CubeBoxCreated { + cleanupCtx = constants.WithCubeboxCreated(cleanupCtx) + } + return l.destroySandboxAfterEnvdInitFailure(cleanupCtx, &workflow.DestroyContext{ + BaseWorkflowInfo: workflow.BaseWorkflowInfo{ + SandboxID: sandBox.ID, + }, + DestroyInfo: &cubebox.DestroyCubeSandboxRequest{ + RequestID: realReq.RequestID, + SandboxID: sandBox.ID, + }, + }) +} + +func (l *local) destroySandboxAfterEnvdInitFailure(ctx context.Context, opts *workflow.DestroyContext) error { + if l != nil && l.destroyFn != nil { + return l.destroyFn(ctx, opts) + } + return l.Destroy(ctx, opts) +} + func (l *local) genSandboxOptions(ctx context.Context, realReq *cubebox.RunCubeSandboxRequest, sandBox *cubeboxstore.CubeBox, flowOpts *workflow.CreateContext) ([]oci.SpecOpts, error) { var ( additionalSandboxOpt []oci.SpecOpts diff --git a/Cubelet/services/cubebox/cube_container_create_test.go b/Cubelet/services/cubebox/cube_container_create_test.go index e9af3c640..9716393f4 100644 --- a/Cubelet/services/cubebox/cube_container_create_test.go +++ b/Cubelet/services/cubebox/cube_container_create_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/containerd/v2/pkg/namespaces" "github.com/containerd/containerd/v2/pkg/oci" imagespec "github.com/opencontainers/image-spec/specs-go/v1" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -324,6 +325,58 @@ func TestAppendExt4NetfileMounts(t *testing.T) { }) } +func TestCleanupAfterEnvdInitFailurePassesExpectedContextAndRequest(t *testing.T) { + flowOpts := &workflow.CreateContext{CubeBoxCreated: true} + req := &cubebox.RunCubeSandboxRequest{RequestID: "req-cleanup"} + sb := &cubeboxstore.CubeBox{ + Namespace: "ns-cleanup", + Metadata: cubeboxstore.Metadata{ + ID: "sb-cleanup", + }, + } + + called := false + l := &local{ + destroyFn: func(ctx context.Context, opts *workflow.DestroyContext) error { + called = true + assert.True(t, constants.IsFailoverOperation(ctx)) + assert.True(t, constants.IsCubeboxCreated(ctx)) + ns, err := namespaces.NamespaceRequired(ctx) + require.NoError(t, err) + assert.Equal(t, "ns-cleanup", ns) + require.NotNil(t, opts) + assert.Equal(t, "sb-cleanup", opts.SandboxID) + require.NotNil(t, opts.DestroyInfo) + assert.Equal(t, "sb-cleanup", opts.DestroyInfo.SandboxID) + assert.Equal(t, "req-cleanup", opts.DestroyInfo.RequestID) + return nil + }, + } + + err := l.cleanupAfterEnvdInitFailure(flowOpts, req, sb) + require.NoError(t, err) + require.True(t, called) +} + +func TestCleanupAfterEnvdInitFailureReturnsDestroyError(t *testing.T) { + flowOpts := &workflow.CreateContext{Failover: true} + req := &cubebox.RunCubeSandboxRequest{RequestID: "req-cleanup"} + sb := &cubeboxstore.CubeBox{ + Metadata: cubeboxstore.Metadata{ + ID: "sb-cleanup", + }, + } + + l := &local{ + destroyFn: func(ctx context.Context, opts *workflow.DestroyContext) error { + return fmt.Errorf("destroy failed") + }, + } + + err := l.cleanupAfterEnvdInitFailure(flowOpts, req, sb) + require.EqualError(t, err, "destroy failed") +} + func TestAppendExt4NetfileMountsNoopWithoutNetfile(t *testing.T) { mountsConfig := &virtiofs.CubeRootfsInfo{ PmemFile: "/pmem/rootfs.ext4", diff --git a/Cubelet/services/cubebox/local.go b/Cubelet/services/cubebox/local.go index b0c6c08df..98387f23b 100644 --- a/Cubelet/services/cubebox/local.go +++ b/Cubelet/services/cubebox/local.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "maps" + "net/http" "os" "path" "path/filepath" @@ -203,13 +204,15 @@ func init() { } l := &local{ - client: client, - localTask: i.(tasks.TasksClient), - config: config, - criImage: obj.(*cubeimages.CubeImageService), - cbriManager: cbriManager, - cubeboxManger: cubeboxAPIObj.(cubes.CubeboxAPI), - shims: shimPlugin.(*v2.ShimManager), + client: client, + localTask: i.(tasks.TasksClient), + config: config, + criImage: obj.(*cubeimages.CubeImageService), + cbriManager: cbriManager, + cubeboxManger: cubeboxAPIObj.(cubes.CubeboxAPI), + shims: shimPlugin.(*v2.ShimManager), + envdHTTPClient: newEnvdHTTPClient(), + envdInitPort: defaultEnvdInitPort, } CubeLog.Info("Start recovering state") @@ -254,9 +257,12 @@ type local struct { config *CubeConfig shims *v2.ShimManager - criImage *cubeimages.CubeImageService - cbriManager cbri.APIManager - cubeboxManger cubes.CubeboxAPI + criImage *cubeimages.CubeImageService + cbriManager cbri.APIManager + cubeboxManger cubes.CubeboxAPI + envdHTTPClient *http.Client + envdInitPort int + destroyFn func(context.Context, *workflow.DestroyContext) error } const ( diff --git a/Cubelet/services/cubebox/probe.go b/Cubelet/services/cubebox/probe.go index 88eb2f9d3..a8e3693a7 100644 --- a/Cubelet/services/cubebox/probe.go +++ b/Cubelet/services/cubebox/probe.go @@ -5,7 +5,9 @@ package cubebox import ( + "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -14,6 +16,7 @@ import ( "net/url" "runtime/debug" "strconv" + "strings" "time" "github.com/tencentcloud/CubeSandbox/Cubelet/api/services/cubebox/v1" @@ -27,8 +30,55 @@ import ( "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/utils" "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/version" "github.com/tencentcloud/CubeSandbox/Cubelet/plugins/workflow" + "github.com/tencentcloud/CubeSandbox/cubelog" ) +const ( + envdInitPath = "/init" + // Keep create-time envd init within a bounded sub-second budget so the + // safeguard absorbs brief restore jitter without turning sandbox create + // into an unbounded slow path. + envdInitAttemptTimeout = 150 * time.Millisecond + envdInitMaxAttempts = 3 + envdInitRetryDelay = 25 * time.Millisecond + defaultEnvdInitPort = 49983 + missingEnvdSupportAnnotationDetail = "template does not carry envd support annotation" +) + +func newEnvdInitFailure(msg string, hasEnvdCapability bool, err error) error { + if !hasEnvdCapability { + return ret.Errorf(errorcode.ErrorCode_ExecCommandInSandboxFailed, + "%s; %s: %v", msg, missingEnvdSupportAnnotationDetail, err) + } + return ret.Errorf(errorcode.ErrorCode_ExecCommandInSandboxFailed, + "%s: %v", msg, err) +} + +func (l *local) getEnvdInitPort() int { + if l != nil && l.envdInitPort > 0 { + return l.envdInitPort + } + return defaultEnvdInitPort +} + +func (l *local) getEnvdHTTPClient() *http.Client { + if l != nil && l.envdHTTPClient != nil { + return l.envdHTTPClient + } + return newEnvdHTTPClient() +} + +func newEnvdHTTPClient() *http.Client { + return &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + Transport: &http.Transport{ + DisableKeepAlives: true, + }, + } +} + func (l *local) doProbe(ctx context.Context, c *cubebox.ContainerConfig, ci *cubeboxstore.Container) (retErr error) { startTime := time.Now() defer func() { @@ -142,6 +192,128 @@ func (l *local) doProbe(ctx context.Context, c *cubebox.ContainerConfig, ci *cub return nil } +func (l *local) doCreateTimeEnvdInit(ctx context.Context, req *cubebox.RunCubeSandboxRequest, sandBox *cubeboxstore.CubeBox) error { + if req == nil || sandBox == nil || req.Annotations == nil { + return nil + } + raw := strings.TrimSpace(req.Annotations[constants.MasterAnnotationCreateTimeEnvVars]) + if raw == "" { + return nil + } + envVars := map[string]string{} + if err := json.Unmarshal([]byte(raw), &envVars); err != nil { + return ret.Errorf(errorcode.ErrorCode_InvalidParamFormat, "invalid create_time_env_vars annotation: %v", err) + } + if len(envVars) == 0 { + return nil + } + if strings.TrimSpace(sandBox.IP) == "" { + return ret.Err(errorcode.ErrorCode_CreateNetworkFailed, "sandbox IP is empty for create_time_env_vars init") + } + + body, err := json.Marshal(struct { + EnvVars map[string]string `json:"envVars"` + }{ + EnvVars: envVars, + }) + if err != nil { + return ret.Errorf(errorcode.ErrorCode_InvalidParamFormat, "marshal create_time_env_vars init request failed: %v", err) + } + + port := l.getEnvdInitPort() + hasEnvdCapability := strings.TrimSpace(req.Annotations[constants.MasterAnnotationComponentEnvdVersion]) != "" + if !hasEnvdCapability { + // Templates built before envd capability propagation do not carry the + // annotation. Keep them backward-compatible by probing the default envd + // init endpoint with the same bounded retry instead of rejecting upfront. + log.G(ctx).WithFields(CubeLog.Fields{ + "sandboxID": sandBox.ID, + "templateID": strings.TrimSpace(req.Annotations[constants.MasterAnnotationAppSnapshotTemplateID]), + "envdInitPort": port, + }).Warnf("missing envd support annotation; probing default envd init endpoint with bounded retry") + } + + return l.doCreateTimeEnvdInitWithRetry(ctx, sandBox.IP, port, hasEnvdCapability, body) +} + +func (l *local) doCreateTimeEnvdInitWithRetry(ctx context.Context, sandboxIP string, port int, hasEnvdCapability bool, body []byte) error { + var lastErr error + for attempt := 1; attempt <= envdInitMaxAttempts; attempt++ { + innerCtx, cancel := context.WithTimeout(ctx, envdInitAttemptTimeout) + retryable, err := l.doCreateTimeEnvdInitAttempt(innerCtx, sandboxIP, port, body) + cancel() + if err == nil { + return nil + } + lastErr = err + if !retryable || attempt == envdInitMaxAttempts { + return newEnvdInitFailure("create_time_env_vars init failed after bounded retry", hasEnvdCapability, err) + } + select { + case <-time.After(envdInitRetryDelay): + case <-ctx.Done(): + return newEnvdInitFailure("create_time_env_vars init canceled during bounded retry", hasEnvdCapability, lastErr) + } + } + return newEnvdInitFailure("create_time_env_vars init failed after bounded retry", hasEnvdCapability, lastErr) +} + +func (l *local) doCreateTimeEnvdInitAttempt(ctx context.Context, sandboxIP string, port int, body []byte) (bool, error) { + reqURL := formatURL("http", sandboxIP, port, envdInitPath) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL.String(), bytes.NewReader(body)) + if err != nil { + return false, ret.Errorf(errorcode.ErrorCode_InvalidParamFormat, "build envd init request failed: %v", err) + } + httpReq.Header.Set("Content-Type", "application/json") + + resp, err := l.getEnvdHTTPClient().Do(httpReq) + if err != nil { + return isRetryableEnvdInitTransportErr(err), ret.Errorf(errorcode.ErrorCode_ExecCommandInSandboxFailed, "envd init request failed: %v", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return false, ret.Errorf(errorcode.ErrorCode_ExecCommandInSandboxFailed, "read envd init response body failed: %v", err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return isRetryableEnvdInitStatusCode(resp.StatusCode), ret.Errorf( + errorcode.ErrorCode_ExecCommandInSandboxFailed, + "envd init request returned HTTP %d: %s", + resp.StatusCode, + strings.TrimSpace(string(respBody)), + ) + } + return false, nil +} + +func isRetryableEnvdInitStatusCode(statusCode int) bool { + switch statusCode { + case http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: + return true + default: + return false + } +} + +func isRetryableEnvdInitTransportErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return true + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "connection refused") || + strings.Contains(msg, "connection reset") || + strings.Contains(msg, "broken pipe") || + strings.Contains(msg, "eof") +} + func doPreStop(ctx context.Context, ci *cubeboxstore.Container) { c := ci.Config if c.GetPrestop() == nil || c.GetPrestop().GetLifecyleHandler() == nil { diff --git a/Cubelet/services/cubebox/probe_test.go b/Cubelet/services/cubebox/probe_test.go index ec2de757c..587956300 100644 --- a/Cubelet/services/cubebox/probe_test.go +++ b/Cubelet/services/cubebox/probe_test.go @@ -5,10 +5,16 @@ package cubebox import ( + "bytes" "context" + "encoding/json" "fmt" + "io" "net" "net/http" + "net/http/httptest" + neturl "net/url" + "strconv" "testing" "time" @@ -18,12 +24,19 @@ import ( "github.com/tencentcloud/CubeSandbox/Cubelet/api/services/cubebox/v1" "github.com/tencentcloud/CubeSandbox/Cubelet/api/services/errorcode/v1" "github.com/tencentcloud/CubeSandbox/Cubelet/network/proto" + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/constants" "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/ret" cubeboxstore "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/store/cubebox" "github.com/tencentcloud/CubeSandbox/Cubelet/plugins/workflow" "k8s.io/utils/pointer" ) +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + func TestProbeErrIp(t *testing.T) { cnt := &cubebox.ContainerConfig{ Probe: &cubebox.Probe{ @@ -146,6 +159,216 @@ func TestProbeErrAction(t *testing.T) { assert.Equal(t, errorcode.ErrorCode_InvalidParamFormat, err.Code()) } +func TestDoCreateTimeEnvdInitPostsEnvVars(t *testing.T) { + var gotBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/init" { + t.Fatalf("path=%q, want /init", r.URL.Path) + } + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode request body: %v", err) + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + u, err := neturl.Parse(server.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + port, err := strconv.Atoi(u.Port()) + if err != nil { + t.Fatalf("parse server port: %v", err) + } + + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationComponentEnvdVersion: "0.2.0", + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test","USER_ID":"42"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: server.Client(), envdInitPort: port} + if err := l.doCreateTimeEnvdInit(context.Background(), req, sandBox); err != nil { + t.Fatalf("doCreateTimeEnvdInit err=%v", err) + } + envVars, ok := gotBody["envVars"].(map[string]any) + if !ok { + t.Fatalf("envVars payload missing: %#v", gotBody) + } + if envVars["SESSION_ID"] != "user-session-test" { + t.Fatalf("SESSION_ID=%v, want user-session-test", envVars["SESSION_ID"]) + } + if envVars["USER_ID"] != "42" { + t.Fatalf("USER_ID=%v, want 42", envVars["USER_ID"]) + } +} + +func TestDoCreateTimeEnvdInitFailsOnHTTPError(t *testing.T) { + callCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + callCount++ + http.Error(w, "envd refused init", http.StatusInternalServerError) + })) + defer server.Close() + + u, err := neturl.Parse(server.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + port, err := strconv.Atoi(u.Port()) + if err != nil { + t.Fatalf("parse server port: %v", err) + } + + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationComponentEnvdVersion: "0.2.0", + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: server.Client(), envdInitPort: port} + retErr := l.doCreateTimeEnvdInit(context.Background(), req, sandBox) + if retErr == nil { + t.Fatal("expected init failure") + } + errInfo, _ := ret.FromError(retErr) + assert.Equal(t, errorcode.ErrorCode_ExecCommandInSandboxFailed, errInfo.Code()) + assert.Contains(t, errInfo.Message(), "envd refused init") + assert.Equal(t, 1, callCount) +} + +func TestDoCreateTimeEnvdInitRetriesTransientHTTPError(t *testing.T) { + callCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + callCount++ + if callCount < envdInitMaxAttempts { + http.Error(w, "envd warming up", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + u, err := neturl.Parse(server.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + port, err := strconv.Atoi(u.Port()) + if err != nil { + t.Fatalf("parse server port: %v", err) + } + + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationComponentEnvdVersion: "0.2.0", + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: server.Client(), envdInitPort: port} + if err := l.doCreateTimeEnvdInit(context.Background(), req, sandBox); err != nil { + t.Fatalf("doCreateTimeEnvdInit err=%v", err) + } + assert.Equal(t, envdInitMaxAttempts, callCount) +} + +func TestDoCreateTimeEnvdInitRetriesTransportError(t *testing.T) { + callCount := 0 + client := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + callCount++ + if callCount < envdInitMaxAttempts { + return nil, fmt.Errorf("dial tcp %s: connection refused", r.URL.Host) + } + return &http.Response{ + StatusCode: http.StatusNoContent, + Header: make(http.Header), + Body: io.NopCloser(bytes.NewReader(nil)), + Request: r, + }, nil + }), + } + + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationComponentEnvdVersion: "0.2.0", + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: client, envdInitPort: 49983} + if err := l.doCreateTimeEnvdInit(context.Background(), req, sandBox); err != nil { + t.Fatalf("doCreateTimeEnvdInit err=%v", err) + } + assert.Equal(t, envdInitMaxAttempts, callCount) +} + +func TestDoCreateTimeEnvdInitFallsBackWithoutEnvdSupportAnnotation(t *testing.T) { + called := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + if r.URL.Path != "/init" { + t.Fatalf("path=%q, want /init", r.URL.Path) + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + u, err := neturl.Parse(server.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + port, err := strconv.Atoi(u.Port()) + if err != nil { + t.Fatalf("parse server port: %v", err) + } + + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: server.Client(), envdInitPort: port} + if err := l.doCreateTimeEnvdInit(context.Background(), req, sandBox); err != nil { + t.Fatalf("doCreateTimeEnvdInit err=%v", err) + } + if !called { + t.Fatal("expected missing envd support annotation to still issue envd init request") + } +} + +func TestDoCreateTimeEnvdInitFailsWithoutEnvdSupportAnnotationWhenEnvdUnavailable(t *testing.T) { + client := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("dial tcp %s: connection refused", r.URL.Host) + }), + } + req := &cubebox.RunCubeSandboxRequest{ + Annotations: map[string]string{ + constants.MasterAnnotationCreateTimeEnvVars: `{"SESSION_ID":"user-session-test"}`, + }, + } + sandBox := &cubeboxstore.CubeBox{IP: "127.0.0.1"} + + l := &local{envdHTTPClient: client, envdInitPort: 49983} + retErr := l.doCreateTimeEnvdInit(context.Background(), req, sandBox) + if retErr == nil { + t.Fatal("expected init failure when envd init cannot be reached without envd support annotation") + } + errInfo, _ := ret.FromError(retErr) + assert.Equal(t, errorcode.ErrorCode_ExecCommandInSandboxFailed, errInfo.Code()) + assert.Contains(t, errInfo.Message(), "connection refused") +} + func TestProbe(t *testing.T) { testPort := 7997 testHost := "127.0.0.1" diff --git a/docs/.vitepress/config.mjs b/docs/.vitepress/config.mjs index 5cc8a7d51..4da6f1c4c 100644 --- a/docs/.vitepress/config.mjs +++ b/docs/.vitepress/config.mjs @@ -148,6 +148,7 @@ export default withMermaid(defineConfig({ items: [ { text: 'WebUI Dashboard', link: '/guide/webui' }, { text: 'Service Management & Logs', link: '/guide/service-management' }, + { text: 'Sandbox Logs', link: '/guide/sandbox-logs' }, { text: 'Template Inspection & Request Preview', link: '/guide/template-inspection-and-preview' }, { text: 'HTTPS & Domain Resolution', link: '/guide/https-and-domain' }, { text: 'Network Hardening', link: '/guide/network-hardening' }, @@ -273,6 +274,7 @@ export default withMermaid(defineConfig({ items: [ { text: 'WebUI 控制台', link: '/zh/guide/webui' }, { text: '服务管理与日志', link: '/zh/guide/service-management' }, + { text: '沙箱日志', link: '/zh/guide/sandbox-logs' }, { text: '模板检查与请求预览', link: '/zh/guide/template-inspection-and-preview' }, { text: 'HTTPS 证书与域名解析', link: '/zh/guide/https-and-domain' }, { text: '网络加固', link: '/zh/guide/network-hardening' }, diff --git a/docs/guide/sandbox-logs.md b/docs/guide/sandbox-logs.md new file mode 100644 index 000000000..7af835ce3 --- /dev/null +++ b/docs/guide/sandbox-logs.md @@ -0,0 +1,92 @@ +--- +title: Sandbox Logs +lang: en-US +--- + +# Sandbox Logs + +::: warning Work in progress +Log retrieval is still being actively iterated. The `cubecli logs` command described here is a **temporary solution** — the interface and underlying storage layout may change in future releases. +::: + +CubeSandbox exposes two complementary logging layers: + +| Layer | What it captures | How to access | +|---|---|---| +| **Sandbox log** | stdout/stderr of the container init process (the main entrypoint) | `cubecli logs` (this page) | +| **`envd` task log** | stdout/stderr of individual `exec` sub-tasks spawned inside the sandbox | E2B SDK (`on_stdout` / `on_stderr` callbacks) | + +This page covers the **sandbox-level log** only. For `envd` sub-task logs, refer to the [E2B SDK documentation](https://e2b.dev/docs). + +## Prerequisites + +`cubecli` is built alongside Cubelet and installed as part of the standard one-click deployment. The `logs` sub-command accesses log files that live inside the **Cubelet mount namespace**, so it must be run **directly on the compute node** — it cannot be executed remotely via the API or from a non-node host. + +## Reading sandbox logs + +```bash +# Last 100 lines of stdout (default) +cubecli logs + +# Last 100 lines of stderr +cubecli logs --stderr + +# Full log (all lines) +cubecli logs --all + +# Last N lines +cubecli logs --tail 50 +# Short form +cubecli logs -t 50 + +# First N lines +cubecli logs --head 20 +# Short form +cubecli logs -H 20 +``` + +### Flag reference + +| Flag | Short | Description | +|---|---|---| +| `--stderr` | `-e` | Read stderr instead of stdout | +| `--all` | `-a` | Print all lines; cannot be combined with `--tail` or `--head` | +| `--tail N` | `-t N` | Print the last N lines (default: 100 when no other flag is set) | +| `--head N` | `-H N` | Print the first N lines | + +## Reading template build logs + +During template construction the container's stdout/stderr are saved to the host filesystem under `/data/log/template/_0/`. These files do **not** require entering the Cubelet mount namespace, so `--tpl` skips the namespace re-exec: + +```bash +# Last 100 lines of template build stdout +cubecli logs --tpl + +# Full build stderr +cubecli logs --tpl --all --stderr +``` + +## Where the log files live + +| Context | Path | +|---|---| +| Sandbox stdout | `/data/cubelet/state/io.containerd.runtime.v2.task/default//stdout` (inside Cubelet mount namespace) | +| Sandbox stderr | `/data/cubelet/state/io.containerd.runtime.v2.task/default//stderr` (inside Cubelet mount namespace) | +| Template stdout | `/data/log/template/_0/stdout` (host filesystem) | +| Template stderr | `/data/log/template/_0/stderr` (host filesystem) | + +::: tip Why the mount namespace? +Sandbox log files are written by CubeShim into the bundle directory which is only visible inside Cubelet's private mount namespace. `cubecli logs` automatically re-execs itself into that namespace before reading — you do not need to do anything special beyond running the command on the node. +::: + +## Scope and limitations + +- These logs capture only the **init process** (PID 1 inside the container). Output from processes spawned via `exec` calls is captured through the E2B SDK's `on_stdout` / `on_stderr` callbacks — refer to the [E2B SDK documentation](https://e2b.dev/docs) for details. +- Log forwarding must be enabled on the running CubeShim version (available since v0.4.0). On older deployments the log file will be missing. +- Logs are not streamed in real time — there is no `--follow` flag yet. Re-run the command to see new output. +- Log files are removed when the sandbox is deleted. + +## Related + +- [Service Management & Logs](./service-management.md) — host-side service logs, journalctl, and the diagnostic bundle +- [Template Inspection & Request Preview](./template-inspection-and-preview.md) diff --git a/docs/zh/guide/sandbox-logs.md b/docs/zh/guide/sandbox-logs.md new file mode 100644 index 000000000..0ff19125b --- /dev/null +++ b/docs/zh/guide/sandbox-logs.md @@ -0,0 +1,92 @@ +--- +title: 沙箱日志 +lang: zh-CN +--- + +# 沙箱日志 + +::: warning 功能迭代中 +日志能力目前仍在持续迭代，本文介绍的 `cubecli logs` 命令是**临时使用版本**——后续版本可能对接口及底层存储结构进行调整。 +::: + +CubeSandbox 提供两个互补的日志层： + +| 层级 | 记录内容 | 获取方式 | +|---|---|---| +| **沙箱日志** | 容器 init 进程（主入口）的 stdout/stderr | `cubecli logs`（本文） | +| **`envd` 任务日志** | 在沙箱内通过 `exec` 接口启动的子任务的 stdout/stderr | E2B SDK（`on_stdout` / `on_stderr` 回调） | + +本文仅介绍**沙箱级别的日志**。`envd` 子任务的日志获取方式请参阅 [E2B SDK 文档](https://e2b.dev/docs)。 + +## 前置条件 + +`cubecli` 随 Cubelet 一同构建，并在一键部署时自动安装。`logs` 子命令访问的日志文件位于 **Cubelet 挂载命名空间**内，因此必须**直接在计算节点上执行**，无法通过 API 或非节点主机远程调用。 + +## 读取沙箱日志 + +```bash +# 最后 100 行 stdout（默认） +cubecli logs + +# 最后 100 行 stderr +cubecli logs --stderr + +# 完整日志（全部行） +cubecli logs --all + +# 最后 N 行 +cubecli logs --tail 50 +# 简写 +cubecli logs -t 50 + +# 前 N 行 +cubecli logs --head 20 +# 简写 +cubecli logs -H 20 +``` + +### 参数说明 + +| 参数 | 简写 | 说明 | +|---|---|---| +| `--stderr` | `-e` | 读取 stderr，默认读取 stdout | +| `--all` | `-a` | 输出全部行；不可与 `--tail` 或 `--head` 同时使用 | +| `--tail N` | `-t N` | 输出最后 N 行（未指定其他标志时默认为 100） | +| `--head N` | `-H N` | 输出前 N 行 | + +## 读取模板构建日志 + +模板构建过程中，容器的 stdout/stderr 会写入宿主机文件系统 `/data/log/template/_0/`。这些文件无需进入 Cubelet 挂载命名空间，使用 `--tpl` 标志可跳过命名空间切换： + +```bash +# 最后 100 行模板构建 stdout +cubecli logs --tpl + +# 完整模板构建 stderr +cubecli logs --tpl --all --stderr +``` + +## 日志文件路径 + +| 场景 | 路径 | +|---|---| +| 沙箱 stdout | `/data/cubelet/state/io.containerd.runtime.v2.task/default//stdout`（Cubelet 挂载命名空间内） | +| 沙箱 stderr | `/data/cubelet/state/io.containerd.runtime.v2.task/default//stderr`（Cubelet 挂载命名空间内） | +| 模板 stdout | `/data/log/template/_0/stdout`（宿主机文件系统） | +| 模板 stderr | `/data/log/template/_0/stderr`（宿主机文件系统） | + +::: tip 为什么需要挂载命名空间？ +沙箱日志文件由 CubeShim 写入 bundle 目录，该目录仅在 Cubelet 的私有挂载命名空间内可见。`cubecli logs` 会在读取前自动重新进入该命名空间——你无需任何额外操作，只需在节点上直接运行该命令即可。 +::: + +## 范围与限制 + +- 这些日志仅记录 **init 进程**（容器内 PID 1）的输出。通过 `exec` 接口启动的进程输出需通过 E2B SDK 的 `on_stdout` / `on_stderr` 回调获取，详见 [E2B SDK 文档](https://e2b.dev/docs)。 +- 日志转发需要 v0.4.0 及以上版本的 CubeShim。在更旧的部署上，日志文件将不存在。 +- 日志目前不支持实时流式读取，暂无 `--follow` 标志。如需查看最新输出，请重新执行命令。 +- 沙箱删除后，对应的日志文件会一并清除。 + +## 相关文档 + +- [服务管理与日志](./service-management.md) — 宿主机服务日志、journalctl 及诊断包 +- [模板检查与请求预览](./template-inspection-and-preview.md) diff --git a/examples/code-sandbox-quickstart/README.md b/examples/code-sandbox-quickstart/README.md index e8214221e..b74f75628 100644 --- a/examples/code-sandbox-quickstart/README.md +++ b/examples/code-sandbox-quickstart/README.md @@ -122,10 +122,11 @@ hello cube | `exec_code.py` | `sandbox.run_code()` — execute Python code inside a sandbox | | `cmd.py` | `sandbox.commands.run()` — execute shell commands | | `create.py` | `sandbox.get_info()` — retrieve sandbox metadata | +| `create_with_envs.py` | `Sandbox.create(envs=...)` — pass create-time environment variables | | `read.py` | `sandbox.files.read()` — read a file from the sandbox filesystem | | `pause.py` | `sandbox.pause()` / `sandbox.connect()` — snapshot and restore | -| `auto_resume.py` | `lifecycle={"on_timeout": "pause", "auto_resume": True}` — let the platform pause idle sandboxes and resume them on the next request | -| `auto_kill.py` | `lifecycle={"on_timeout": "kill"}` — let the platform tear down idle sandboxes (the default — destruction is irreversible, the sandbox cannot be resumed) | +| `auto-resume.py` | `lifecycle={"on_timeout": "pause", "auto_resume": True}` — let the platform pause idle sandboxes and resume them on the next request | +| `auto-kill.py` | `lifecycle={"on_timeout": "kill"}` — let the platform tear down idle sandboxes (the default — destruction is irreversible, the sandbox cannot be resumed) | | `network_no_internet.py` | `allow_internet_access=False` — fully air-gapped sandbox | | `network_allowlist.py` | `allow_out` — whitelist specific CIDRs, block everything else | | `network_denylist.py` | `deny_out` — block specific CIDRs, allow the rest | @@ -146,6 +147,21 @@ with Sandbox.create(template=template_id) as sandbox: print(result.stdout) ``` +### Create-Time Environment Variables + +You can pass environment variables when creating a sandbox. They are then +available to subsequent command execution in that sandbox: + +```python +python create_with_envs.py +``` + +Expected output: + +```text +user-session-test +``` + ### pause.py — Pause & Resume Snapshot a running sandbox to free compute resources, then restore it later: @@ -158,7 +174,7 @@ with Sandbox.create(template=template_id) as sandbox: print(sandbox.get_info()) ``` -### auto_resume.py — Auto Pause & Auto Resume +### auto-resume.py — Auto Pause & Auto Resume Like `pause.py`, but the platform handles the pause/resume cycle on its own. The `lifecycle` argument mirrors the e2b SDK @@ -178,9 +194,9 @@ sandbox.run_code("print('back from a transparent resume')") sandbox.kill() ``` -### auto_kill.py — Auto Kill on Idle Timeout +### auto-kill.py — Auto Kill on Idle Timeout -The destructive twin of `auto_resume.py`. Setting `on_timeout="kill"` (also the +The destructive twin of `auto-resume.py`. Setting `on_timeout="kill"` (also the default when no `lifecycle` is passed) tells the platform to tear the sandbox down once it idles past `timeout` — no snapshot is kept, the next request fails fast with **410 Gone**: @@ -258,10 +274,12 @@ code-sandbox-quickstart/ ├── exec_code.py # Run Python code inside a sandbox ├── cmd.py # Execute shell commands ├── create.py # Create sandbox and inspect metadata +├── create_with_envs.py # Create sandbox with create-time env vars +├── env_utils.py # Shared .env loader helper ├── read.py # Read files from the sandbox filesystem ├── pause.py # Pause and resume a sandbox -├── auto_resume.py # Auto-pause / auto-resume on idle timeout -├── auto_kill.py # Auto-kill on idle timeout (destruction is final) +├── auto-resume.py # Auto-pause / auto-resume on idle timeout +├── auto-kill.py # Auto-kill on idle timeout (destruction is final) ├── network_no_internet.py # Fully air-gapped sandbox ├── network_allowlist.py # Outbound CIDR allowlist ├── network_denylist.py # Outbound CIDR denylist diff --git a/examples/code-sandbox-quickstart/README_zh.md b/examples/code-sandbox-quickstart/README_zh.md index faae1d009..db81ec06e 100644 --- a/examples/code-sandbox-quickstart/README_zh.md +++ b/examples/code-sandbox-quickstart/README_zh.md @@ -117,10 +117,11 @@ hello cube | `exec_code.py` | `sandbox.run_code()` — 在沙箱中执行 Python 代码 | | `cmd.py` | `sandbox.commands.run()` — 执行 Shell 命令 | | `create.py` | `sandbox.get_info()` — 获取沙箱元数据 | +| `create_with_envs.py` | `Sandbox.create(envs=...)` — 创建时注入环境变量 | | `read.py` | `sandbox.files.read()` — 读取沙箱文件系统中的文件 | | `pause.py` | `sandbox.pause()` / `sandbox.connect()` — 快照与恢复 | -| `auto_resume.py` | `lifecycle={"on_timeout": "pause", "auto_resume": True}` — 平台在空闲超时后自动暂停沙箱，下一次请求自动恢复 | -| `auto_kill.py` | `lifecycle={"on_timeout": "kill"}` — 平台在空闲超时后直接销毁沙箱（默认行为，销毁不可逆，沙箱无法恢复） | +| `auto-resume.py` | `lifecycle={"on_timeout": "pause", "auto_resume": True}` — 平台在空闲超时后自动暂停沙箱，下一次请求自动恢复 | +| `auto-kill.py` | `lifecycle={"on_timeout": "kill"}` — 平台在空闲超时后直接销毁沙箱（默认行为，销毁不可逆，沙箱无法恢复） | | `network_no_internet.py` | `allow_internet_access=False` — 完全断网沙箱 | | `network_allowlist.py` | `allow_out` — 白名单 CIDR，拦截其余所有出口 | | `network_denylist.py` | `deny_out` — 黑名单 CIDR，其余放行 | @@ -141,6 +142,20 @@ with Sandbox.create(template=template_id) as sandbox: print(result.stdout) ``` +### 创建时注入环境变量 + +可以在创建沙箱时传入环境变量，后续在该沙箱中的命令执行也可以读取到这些变量： + +```python +python create_with_envs.py +``` + +预期输出: + +```text +user-session-test +``` + ### pause.py — 暂停与恢复 将运行中的沙箱快照以释放计算资源，之后恢复： @@ -153,7 +168,7 @@ with Sandbox.create(template=template_id) as sandbox: print(sandbox.get_info()) ``` -### auto_resume.py — 自动暂停与自动恢复 +### auto-resume.py — 自动暂停与自动恢复 与 `pause.py` 类似，但暂停/恢复完全交给平台自动管理。`lifecycle` 参数与 e2b SDK 对齐 （参考 [e2b 文档](https://e2b.dev/docs/sandbox/auto-resume)）：`on_timeout="pause"` @@ -172,9 +187,9 @@ sandbox.run_code("print('back from a transparent resume')") sandbox.kill() ``` -### auto_kill.py — 空闲超时后自动销毁 +### auto-kill.py — 空闲超时后自动销毁 -`auto_resume.py` 的孪生销毁版本。`on_timeout="kill"`（不传 `lifecycle` +`auto-resume.py` 的孪生销毁版本。`on_timeout="kill"`（不传 `lifecycle` 时的默认值）告诉平台：沙箱空闲超过 `timeout` 后直接拆除 VM，不 保留快照，下一次请求会以 **410 Gone** 快速失败： @@ -251,10 +266,12 @@ code-sandbox-quickstart/ ├── exec_code.py # 在沙箱中运行 Python 代码 ├── cmd.py # 执行 Shell 命令 ├── create.py # 创建沙箱并查看元数据 +├── create_with_envs.py # 创建时注入环境变量 +├── env_utils.py # 共享的 .env 加载辅助脚本 ├── read.py # 读取沙箱文件系统中的文件 ├── pause.py # 暂停与恢复沙箱 -├── auto_resume.py # 自动暂停 / 自动恢复（基于空闲超时） -├── auto_kill.py # 空闲超时后自动销毁（不可恢复） +├── auto-resume.py # 自动暂停 / 自动恢复（基于空闲超时） +├── auto-kill.py # 空闲超时后自动销毁（不可恢复） ├── network_no_internet.py # 完全断网沙箱 ├── network_allowlist.py # 出口 CIDR 白名单 ├── network_denylist.py # 出口 CIDR 黑名单 diff --git a/examples/code-sandbox-quickstart/create_with_envs.py b/examples/code-sandbox-quickstart/create_with_envs.py new file mode 100644 index 000000000..d2056f10b --- /dev/null +++ b/examples/code-sandbox-quickstart/create_with_envs.py @@ -0,0 +1,20 @@ +# Copyright (c) 2024 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 + +import os +from e2b_code_interpreter import Sandbox +from env_utils import load_local_dotenv + +load_local_dotenv() + +template_id = os.environ["CUBE_TEMPLATE_ID"] + +with Sandbox.create( + template=template_id, + envs={ + "API_TOKEN": "demo-token", + "SESSION_ID": "user-session-test", + }, +) as sandbox: + result = sandbox.commands.run("echo $SESSION_ID") + print(result.stdout)