From eaeadabc80f3279cdcbad7f81888b321b2d05626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Mon, 15 Jun 2026 21:26:27 +0800 Subject: [PATCH 1/2] feat(server): support injecting env vars into egress container via OPENSANDBOX_EGRESS_ prefix Env vars in CreateSandboxRequest.env with the OPENSANDBOX_EGRESS_ prefix are now automatically routed to the egress sidecar container instead of the main sandbox container. This enables users to configure egress behavior (e.g., mitmproxy scripts, log level, DNS settings) at sandbox creation time without requiring a custom egress image. Reserved internal vars (RULES, MODE, TOKEN) are rejected with 400 to prevent overriding server-managed state. Closes #968 Co-Authored-By: Claude Opus 4.6 --- .../opensandbox_server/services/constants.py | 9 ++ .../services/k8s/agent_sandbox_provider.py | 4 + .../services/k8s/batchsandbox_provider.py | 2 + .../services/k8s/create_helpers.py | 34 ++++++ .../services/k8s/egress_helper.py | 4 + .../services/k8s/kubernetes_service.py | 3 +- .../services/k8s/workload_provider.py | 1 + server/tests/k8s/test_egress_helper.py | 106 ++++++++++++++++++ 8 files changed, 162 insertions(+), 1 deletion(-) diff --git a/server/opensandbox_server/services/constants.py b/server/opensandbox_server/services/constants.py index 4859158a8..af78e6758 100644 --- a/server/opensandbox_server/services/constants.py +++ b/server/opensandbox_server/services/constants.py @@ -42,6 +42,13 @@ # Must match components/egress/pkg/constants/configuration.go EnvEgressToken OPENSANDBOX_EGRESS_TOKEN = "OPENSANDBOX_EGRESS_TOKEN" OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT = "OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT" + +EGRESS_ENV_PREFIX = "OPENSANDBOX_EGRESS_" +RESERVED_EGRESS_ENV_VARS = frozenset({ + EGRESS_RULES_ENV, + EGRESS_MODE_ENV, + OPENSANDBOX_EGRESS_TOKEN, +}) OPENSANDBOX_RUNTIME_VOLUME_NAME = "opensandbox-bin" OPENSANDBOX_RUNTIME_MOUNT_PATH = "/opt/opensandbox" @@ -146,6 +153,8 @@ class SnapshotErrorCodes: "EGRESS_MODE_ENV", "OPENSANDBOX_EGRESS_TOKEN", "OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT", + "EGRESS_ENV_PREFIX", + "RESERVED_EGRESS_ENV_VARS", "OPENSANDBOX_RUNTIME_VOLUME_NAME", "OPENSANDBOX_RUNTIME_MOUNT_PATH", "SandboxErrorCodes", diff --git a/server/opensandbox_server/services/k8s/agent_sandbox_provider.py b/server/opensandbox_server/services/k8s/agent_sandbox_provider.py index 0247940ca..d8d891a78 100644 --- a/server/opensandbox_server/services/k8s/agent_sandbox_provider.py +++ b/server/opensandbox_server/services/k8s/agent_sandbox_provider.py @@ -139,6 +139,7 @@ def create_workload( egress_auth_token: Optional[str] = None, egress_mode: str = EGRESS_MODE_DNS, credential_proxy_enabled: bool = False, + egress_env: Optional[Dict[str, Optional[str]]] = None, ) -> Dict[str, Any]: """Create an agent-sandbox Sandbox CRD workload.""" if is_windows_profile(platform): @@ -158,6 +159,7 @@ def create_workload( egress_auth_token=egress_auth_token, egress_mode=egress_mode, credential_proxy_enabled=credential_proxy_enabled, + egress_env=egress_env, ) if volumes: @@ -245,6 +247,7 @@ def _build_pod_spec( egress_auth_token: Optional[str] = None, egress_mode: str = EGRESS_MODE_DNS, credential_proxy_enabled: bool = False, + egress_env: Optional[Dict[str, Optional[str]]] = None, ) -> Dict[str, Any]: """Build pod spec dict for the Sandbox CRD.""" disable_ipv6_for_egress = ( @@ -291,6 +294,7 @@ def _build_pod_spec( egress_auth_token=egress_auth_token, egress_mode=egress_mode, credential_proxy_enabled=credential_proxy_enabled, + extra_env=egress_env, ) return pod_spec diff --git a/server/opensandbox_server/services/k8s/batchsandbox_provider.py b/server/opensandbox_server/services/k8s/batchsandbox_provider.py index 2fba857b8..6933eefdb 100644 --- a/server/opensandbox_server/services/k8s/batchsandbox_provider.py +++ b/server/opensandbox_server/services/k8s/batchsandbox_provider.py @@ -118,6 +118,7 @@ def create_workload( egress_auth_token: Optional[str] = None, egress_mode: str = EGRESS_MODE_DNS, credential_proxy_enabled: bool = False, + egress_env: Optional[Dict[str, Optional[str]]] = None, ) -> Dict[str, Any]: """Create a BatchSandbox in template mode or pool mode.""" extensions = extensions or {} @@ -225,6 +226,7 @@ def create_workload( egress_auth_token=egress_auth_token, egress_mode=egress_mode, credential_proxy_enabled=credential_proxy_enabled, + extra_env=egress_env, ) if volumes: diff --git a/server/opensandbox_server/services/k8s/create_helpers.py b/server/opensandbox_server/services/k8s/create_helpers.py index 07eb788ce..cd6ba1621 100644 --- a/server/opensandbox_server/services/k8s/create_helpers.py +++ b/server/opensandbox_server/services/k8s/create_helpers.py @@ -21,6 +21,8 @@ from opensandbox_server.api.schema import CreateSandboxRequest from opensandbox_server.config import AppConfig, EGRESS_MODE_DNS from opensandbox_server.services.constants import ( + EGRESS_ENV_PREFIX, + RESERVED_EGRESS_ENV_VARS, SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY, SANDBOX_SECURE_ACCESS_TOKEN_METADATA_KEY, SANDBOX_ID_LABEL, @@ -29,6 +31,32 @@ ) from opensandbox_server.services.validators import calculate_expiration_or_raise +Pair = tuple[Dict[str, Optional[str]], Dict[str, Optional[str]]] + + +def _split_egress_env( + env: Optional[Dict[str, Optional[str]]], +) -> Pair: + """Split request env into (sandbox_env, egress_env) by OPENSANDBOX_EGRESS_ prefix. + + Raises ValueError if a user-supplied key collides with a reserved internal var. + """ + if not env: + return {}, {} + + sandbox_env: Dict[str, Optional[str]] = {} + egress_env: Dict[str, Optional[str]] = {} + for key, value in env.items(): + if key.startswith(EGRESS_ENV_PREFIX): + if key in RESERVED_EGRESS_ENV_VARS: + raise ValueError( + f"Environment variable '{key}' is reserved and cannot be overridden" + ) + egress_env[key] = value + else: + sandbox_env[key] = value + return sandbox_env, egress_env + @dataclass class _CreateWorkloadContext: @@ -41,6 +69,8 @@ class _CreateWorkloadContext: egress_auth_token: Optional[str] credential_proxy_enabled: bool secure_access_token: Optional[str] + sandbox_env: Dict[str, Optional[str]] + egress_env: Dict[str, Optional[str]] def _build_create_workload_context( @@ -84,6 +114,8 @@ def _build_create_workload_context( if request.resource_limits and request.resource_limits.root: resource_limits = request.resource_limits.root + sandbox_env, egress_env = _split_egress_env(request.env) + return _CreateWorkloadContext( labels=labels, annotations=annotations, @@ -94,4 +126,6 @@ def _build_create_workload_context( egress_auth_token=egress_auth_token, credential_proxy_enabled=credential_proxy_enabled, secure_access_token=secure_access_token, + sandbox_env=sandbox_env, + egress_env=egress_env, ) diff --git a/server/opensandbox_server/services/k8s/egress_helper.py b/server/opensandbox_server/services/k8s/egress_helper.py index c728c8a10..e146bd1d0 100644 --- a/server/opensandbox_server/services/k8s/egress_helper.py +++ b/server/opensandbox_server/services/k8s/egress_helper.py @@ -76,6 +76,7 @@ def apply_egress_to_spec( egress_auth_token: Optional[str] = None, egress_mode: str = EGRESS_MODE_DNS, credential_proxy_enabled: bool = False, + extra_env: Optional[Dict[str, Optional[str]]] = None, ) -> None: """ Append the egress sidecar to ``containers``. When ``egress.disable_ipv6`` is enabled, @@ -94,6 +95,9 @@ def apply_egress_to_spec( env.append({"name": OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT, "value": "true"}) if egress_auth_token: env.append({"name": OPENSANDBOX_EGRESS_TOKEN, "value": egress_auth_token}) + if extra_env: + for name, value in extra_env.items(): + env.append({"name": name, "value": value or ""}) sidecar: Dict[str, Any] = { "name": "egress", diff --git a/server/opensandbox_server/services/k8s/kubernetes_service.py b/server/opensandbox_server/services/k8s/kubernetes_service.py index 37e0710ee..8de99f357 100644 --- a/server/opensandbox_server/services/k8s/kubernetes_service.py +++ b/server/opensandbox_server/services/k8s/kubernetes_service.py @@ -453,7 +453,7 @@ async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxRe namespace=self.namespace, image_spec=request.image, entrypoint=request.entrypoint, - env=request.env or {}, + env=context.sandbox_env, resource_limits=context.resource_limits, labels=context.labels, annotations=context.annotations or None, @@ -465,6 +465,7 @@ async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxRe egress_auth_token=context.egress_auth_token, egress_mode=context.egress_mode, credential_proxy_enabled=context.credential_proxy_enabled, + egress_env=context.egress_env, volumes=request.volumes, platform=request.platform, ) diff --git a/server/opensandbox_server/services/k8s/workload_provider.py b/server/opensandbox_server/services/k8s/workload_provider.py index 4bfb55c5d..dcc941d8a 100644 --- a/server/opensandbox_server/services/k8s/workload_provider.py +++ b/server/opensandbox_server/services/k8s/workload_provider.py @@ -53,6 +53,7 @@ def create_workload( egress_auth_token: Optional[str] = None, egress_mode: str = EGRESS_MODE_DNS, credential_proxy_enabled: bool = False, + egress_env: Optional[Dict[str, Optional[str]]] = None, ) -> Dict[str, Any]: """ Create a new workload resource. diff --git a/server/tests/k8s/test_egress_helper.py b/server/tests/k8s/test_egress_helper.py index e17ad0922..e37bba29c 100644 --- a/server/tests/k8s/test_egress_helper.py +++ b/server/tests/k8s/test_egress_helper.py @@ -15,6 +15,8 @@ import json from typing import Optional +import pytest + from opensandbox_server.api.schema import NetworkPolicy, NetworkRule from opensandbox_server.config import EGRESS_MODE_DNS, EGRESS_MODE_DNS_NFT from opensandbox_server.services.constants import ( @@ -26,6 +28,7 @@ OPENSANDBOX_RUNTIME_MOUNT_PATH, OPENSANDBOX_RUNTIME_VOLUME_NAME, ) +from opensandbox_server.services.k8s.create_helpers import _split_egress_env from opensandbox_server.services.k8s.egress_helper import ( apply_egress_to_spec, build_security_context_for_sandbox_container, @@ -379,6 +382,63 @@ def test_no_op_when_no_egress_image(self): assert len(containers) == 0 + def test_extra_env_injected_into_sidecar(self): + containers: list = [] + network_policy = NetworkPolicy( + default_action="deny", + egress=[NetworkRule(action="allow", target="example.com")], + ) + extra = { + "OPENSANDBOX_EGRESS_MITMPROXY_SCRIPT": "/scripts/auth.py", + "OPENSANDBOX_EGRESS_LOG_LEVEL": "debug", + } + + apply_egress_to_spec( + containers, + network_policy, + "opensandbox/egress:v1.1.0", + extra_env=extra, + ) + + env_by_name = {e["name"]: e["value"] for e in containers[0]["env"]} + assert env_by_name["OPENSANDBOX_EGRESS_MITMPROXY_SCRIPT"] == "/scripts/auth.py" + assert env_by_name["OPENSANDBOX_EGRESS_LOG_LEVEL"] == "debug" + + def test_extra_env_none_value_becomes_empty_string(self): + containers: list = [] + network_policy = NetworkPolicy( + default_action="deny", + egress=[NetworkRule(action="allow", target="example.com")], + ) + + apply_egress_to_spec( + containers, + network_policy, + "opensandbox/egress:v1.1.0", + extra_env={"OPENSANDBOX_EGRESS_LOG_LEVEL": None}, + ) + + env_by_name = {e["name"]: e["value"] for e in containers[0]["env"]} + assert env_by_name["OPENSANDBOX_EGRESS_LOG_LEVEL"] == "" + + def test_extra_env_empty_dict_is_noop(self): + containers: list = [] + network_policy = NetworkPolicy( + default_action="deny", + egress=[NetworkRule(action="allow", target="example.com")], + ) + + apply_egress_to_spec( + containers, + network_policy, + "opensandbox/egress:v1.1.0", + extra_env={}, + ) + + env_names = {e["name"] for e in containers[0]["env"]} + assert env_names == {EGRESS_RULES_ENV, EGRESS_MODE_ENV} + + class TestPrepExecdInitForEgress: def test_returns_privileged_security_dict_and_prefixed_script(self): base = "cp ./execd /opt/opensandbox/execd" @@ -386,3 +446,49 @@ def test_returns_privileged_security_dict_and_prefixed_script(self): assert sc == {"privileged": True} assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" in script assert script.endswith(base) + + +class TestSplitEgressEnv: + def test_splits_by_prefix(self): + env = { + "MY_VAR": "hello", + "OPENSANDBOX_EGRESS_LOG_LEVEL": "debug", + "OTHER": "world", + } + sandbox_env, egress_env = _split_egress_env(env) + assert sandbox_env == {"MY_VAR": "hello", "OTHER": "world"} + assert egress_env == {"OPENSANDBOX_EGRESS_LOG_LEVEL": "debug"} + + def test_none_returns_empty_dicts(self): + sandbox_env, egress_env = _split_egress_env(None) + assert sandbox_env == {} + assert egress_env == {} + + def test_empty_returns_empty_dicts(self): + sandbox_env, egress_env = _split_egress_env({}) + assert sandbox_env == {} + assert egress_env == {} + + def test_no_egress_vars(self): + env = {"FOO": "bar", "BAZ": "qux"} + sandbox_env, egress_env = _split_egress_env(env) + assert sandbox_env == env + assert egress_env == {} + + def test_rejects_reserved_rules(self): + with pytest.raises(ValueError, match="reserved"): + _split_egress_env({"OPENSANDBOX_EGRESS_RULES": "evil"}) + + def test_rejects_reserved_mode(self): + with pytest.raises(ValueError, match="reserved"): + _split_egress_env({"OPENSANDBOX_EGRESS_MODE": "evil"}) + + def test_rejects_reserved_token(self): + with pytest.raises(ValueError, match="reserved"): + _split_egress_env({"OPENSANDBOX_EGRESS_TOKEN": "evil"}) + + def test_allows_mitmproxy_transparent(self): + env = {"OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT": "true"} + sandbox_env, egress_env = _split_egress_env(env) + assert sandbox_env == {} + assert egress_env == {"OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT": "true"} From dee470ea2c363072ca965c57a0637f29028820a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Mon, 15 Jun 2026 21:54:05 +0800 Subject: [PATCH 2/2] fix(server): address egress env injection review findings - Mirror OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT to sandbox container so execd bootstrap.sh can trust the MITM CA in manual MITM path - Move _build_create_workload_context into try block so reserved env var ValueError is caught and returned as 400 instead of 500 - Skip user-supplied MITMPROXY_TRANSPARENT in extra_env when credential_proxy_enabled to prevent overriding the internal value - Reserve OPENSANDBOX_EGRESS_HTTP_ADDR to prevent users from changing the egress listen address and breaking readiness probes Co-Authored-By: Claude Opus 4.6 --- .../opensandbox_server/services/constants.py | 4 +++ .../services/k8s/create_helpers.py | 3 ++ .../services/k8s/egress_helper.py | 2 ++ .../services/k8s/kubernetes_service.py | 18 ++++++------ server/tests/k8s/test_egress_helper.py | 28 ++++++++++++++++++- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/server/opensandbox_server/services/constants.py b/server/opensandbox_server/services/constants.py index af78e6758..f39ea9037 100644 --- a/server/opensandbox_server/services/constants.py +++ b/server/opensandbox_server/services/constants.py @@ -44,10 +44,13 @@ OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT = "OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT" EGRESS_ENV_PREFIX = "OPENSANDBOX_EGRESS_" +# Must match components/egress/pkg/constants/configuration.go EnvEgressHTTPAddr +OPENSANDBOX_EGRESS_HTTP_ADDR = "OPENSANDBOX_EGRESS_HTTP_ADDR" RESERVED_EGRESS_ENV_VARS = frozenset({ EGRESS_RULES_ENV, EGRESS_MODE_ENV, OPENSANDBOX_EGRESS_TOKEN, + OPENSANDBOX_EGRESS_HTTP_ADDR, }) OPENSANDBOX_RUNTIME_VOLUME_NAME = "opensandbox-bin" OPENSANDBOX_RUNTIME_MOUNT_PATH = "/opt/opensandbox" @@ -154,6 +157,7 @@ class SnapshotErrorCodes: "OPENSANDBOX_EGRESS_TOKEN", "OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT", "EGRESS_ENV_PREFIX", + "OPENSANDBOX_EGRESS_HTTP_ADDR", "RESERVED_EGRESS_ENV_VARS", "OPENSANDBOX_RUNTIME_VOLUME_NAME", "OPENSANDBOX_RUNTIME_MOUNT_PATH", diff --git a/server/opensandbox_server/services/k8s/create_helpers.py b/server/opensandbox_server/services/k8s/create_helpers.py index cd6ba1621..41736baed 100644 --- a/server/opensandbox_server/services/k8s/create_helpers.py +++ b/server/opensandbox_server/services/k8s/create_helpers.py @@ -22,6 +22,7 @@ from opensandbox_server.config import AppConfig, EGRESS_MODE_DNS from opensandbox_server.services.constants import ( EGRESS_ENV_PREFIX, + OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT, RESERVED_EGRESS_ENV_VARS, SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY, SANDBOX_SECURE_ACCESS_TOKEN_METADATA_KEY, @@ -53,6 +54,8 @@ def _split_egress_env( f"Environment variable '{key}' is reserved and cannot be overridden" ) egress_env[key] = value + if key == OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT: + sandbox_env[key] = value else: sandbox_env[key] = value return sandbox_env, egress_env diff --git a/server/opensandbox_server/services/k8s/egress_helper.py b/server/opensandbox_server/services/k8s/egress_helper.py index e146bd1d0..2320ee65a 100644 --- a/server/opensandbox_server/services/k8s/egress_helper.py +++ b/server/opensandbox_server/services/k8s/egress_helper.py @@ -97,6 +97,8 @@ def apply_egress_to_spec( env.append({"name": OPENSANDBOX_EGRESS_TOKEN, "value": egress_auth_token}) if extra_env: for name, value in extra_env.items(): + if credential_proxy_enabled and name == OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT: + continue env.append({"name": name, "value": value or ""}) sidecar: Dict[str, Any] = { diff --git a/server/opensandbox_server/services/k8s/kubernetes_service.py b/server/opensandbox_server/services/k8s/kubernetes_service.py index 8de99f357..4965e152d 100644 --- a/server/opensandbox_server/services/k8s/kubernetes_service.py +++ b/server/opensandbox_server/services/k8s/kubernetes_service.py @@ -423,16 +423,16 @@ async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxRe sandbox_id = self.generate_sandbox_id() created_at = datetime.now(timezone.utc) - context = _build_create_workload_context( - app_config=self.app_config, - request=request, - sandbox_id=sandbox_id, - created_at=created_at, - egress_token_factory=generate_egress_token, - secure_access_token_factory=generate_secure_access_token, - ) - + try: + context = _build_create_workload_context( + app_config=self.app_config, + request=request, + sandbox_id=sandbox_id, + created_at=created_at, + egress_token_factory=generate_egress_token, + secure_access_token_factory=generate_secure_access_token, + ) apply_access_renew_extend_seconds_to_mapping(context.annotations, request.extensions) apply_extensions_to_annotations(context.annotations, request.extensions) diff --git a/server/tests/k8s/test_egress_helper.py b/server/tests/k8s/test_egress_helper.py index e37bba29c..16131c672 100644 --- a/server/tests/k8s/test_egress_helper.py +++ b/server/tests/k8s/test_egress_helper.py @@ -421,6 +421,28 @@ def test_extra_env_none_value_becomes_empty_string(self): env_by_name = {e["name"]: e["value"] for e in containers[0]["env"]} assert env_by_name["OPENSANDBOX_EGRESS_LOG_LEVEL"] == "" + def test_extra_env_mitm_ignored_when_credential_proxy_enabled(self): + containers: list = [] + network_policy = NetworkPolicy( + default_action="deny", + egress=[NetworkRule(action="allow", target="example.com")], + ) + + apply_egress_to_spec( + containers, + network_policy, + "opensandbox/egress:v1.1.0", + credential_proxy_enabled=True, + extra_env={"OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT": "false"}, + ) + + mitm_vals = [ + e["value"] + for e in containers[0]["env"] + if e["name"] == OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT + ] + assert mitm_vals == ["true"] + def test_extra_env_empty_dict_is_noop(self): containers: list = [] network_policy = NetworkPolicy( @@ -490,5 +512,9 @@ def test_rejects_reserved_token(self): def test_allows_mitmproxy_transparent(self): env = {"OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT": "true"} sandbox_env, egress_env = _split_egress_env(env) - assert sandbox_env == {} + assert sandbox_env == {"OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT": "true"} assert egress_env == {"OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT": "true"} + + def test_rejects_reserved_http_addr(self): + with pytest.raises(ValueError, match="reserved"): + _split_egress_env({"OPENSANDBOX_EGRESS_HTTP_ADDR": "0.0.0.0:9999"})