diff --git a/.github/workflows/unit-test-check.yml b/.github/workflows/unit-test-check.yml index 77a1f33e4..7b53dc65c 100644 --- a/.github/workflows/unit-test-check.yml +++ b/.github/workflows/unit-test-check.yml @@ -182,6 +182,12 @@ jobs: CubeShim/*) add_component "CubeShim" ;; + CubeNet/cubevs/*) + # cubevs is its own Go module with its own unit tests; run + # those, and keep cubelet (it embeds the cubevs runtime). + add_component "cubevs" + add_component "cubelet" + ;; CubeNet/*) # Embedded network runtime lives under Cubelet; CubeNet eBPF # helper changes still need the cubelet test set. @@ -231,6 +237,7 @@ jobs: elif . == "cubedb" then "CubeDB" elif . == "cube-lifecycle-manager" then "Cube Lifecycle Manager" elif . == "cubelet" then "Cubelet" + elif . == "cubevs" then "CubeVS" elif . == "agent" then "Agent" elif . == "hypervisor" then "Hypervisor" else error("unsupported component: " + .) @@ -247,6 +254,7 @@ jobs: elif . == "cubedb" then "cubedb-test" elif . == "cube-lifecycle-manager" then "cube-lifecycle-manager-test" elif . == "cubelet" then "cubelet-pkg-test" + elif . == "cubevs" then "cubevs-test" elif . == "agent" then "agent-test" elif . == "hypervisor" then "hypervisor-test" else error("unsupported component: " + .) diff --git a/CubeAPI/src/cubemaster/mod.rs b/CubeAPI/src/cubemaster/mod.rs index 37a0e505f..682a30bb1 100644 --- a/CubeAPI/src/cubemaster/mod.rs +++ b/CubeAPI/src/cubemaster/mod.rs @@ -821,6 +821,8 @@ pub struct CubeEgressRuleMatch { pub path: Option, #[serde(skip_serializing_if = "Option::is_none")] pub scheme: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub port: Option, } #[derive(Debug, Serialize, Clone)] diff --git a/CubeAPI/src/models/mod.rs b/CubeAPI/src/models/mod.rs index 1e9051e7c..77e081dc9 100644 --- a/CubeAPI/src/models/mod.rs +++ b/CubeAPI/src/models/mod.rs @@ -78,6 +78,11 @@ pub struct EgressRule { /// /// Multi-field semantics: AND across fields, OR within `method`. /// Comparisons on sni/host/scheme are case-insensitive. +/// +/// `port` + `scheme` together pin the (host, port) tuple CubeEgress intercepts. +/// Both nil keeps the legacy default {80/http, 443/https}. When `port` is set, +/// `scheme` MUST also be set — same-`(host, port)` rules across the policy +/// must agree on `scheme` (the server rejects the whole policy on mismatch). #[derive(Debug, Clone, Serialize, Deserialize, Default, ToSchema)] pub struct EgressRuleMatch { #[serde(skip_serializing_if = "Option::is_none")] @@ -90,6 +95,8 @@ pub struct EgressRuleMatch { pub path: Option, #[serde(skip_serializing_if = "Option::is_none")] pub scheme: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub port: Option, } /// Rule action. diff --git a/CubeAPI/src/services/sandboxes.rs b/CubeAPI/src/services/sandboxes.rs index b0560f435..691de88a3 100644 --- a/CubeAPI/src/services/sandboxes.rs +++ b/CubeAPI/src/services/sandboxes.rs @@ -17,9 +17,9 @@ use crate::{ }, error::{AppError, AppResult}, models::{ - EgressRule, LogLevel as ModelLogLevel, NewSandbox, Sandbox, SandboxAutoResume, - SandboxDetail, SandboxLifecycleConfig, SandboxLog, SandboxLogEntry, SandboxLogs, - SandboxLogsV2Response, SandboxNetworkConfig, SandboxOnTimeout, SandboxState, + EgressRule, EgressRuleMatch, LogLevel as ModelLogLevel, NewSandbox, Sandbox, + SandboxAutoResume, SandboxDetail, SandboxLifecycleConfig, SandboxLog, SandboxLogEntry, + SandboxLogs, SandboxLogsV2Response, SandboxNetworkConfig, SandboxOnTimeout, SandboxState, SandboxVolumeMount, }, }; @@ -1049,6 +1049,12 @@ pub(crate) fn build_cube_network_config( allow_internet_access == Some(false), )?; + if let Some(rs) = network.and_then(|n| n.rules.as_ref()) { + for (index, rule) in rs.iter().enumerate() { + validate_egress_rule_match(&rule.r#match, index)?; + } + } + let rules: Vec = network .and_then(|n| n.rules.as_ref()) .map(|rs| rs.iter().map(map_egress_rule).collect()) @@ -1080,6 +1086,33 @@ pub(crate) fn build_cube_network_config( })) } +/// Validate the port/scheme pair on one egress rule match, mirroring the +/// SDK client-side contract and the CubeEgress Lua validation: a set port +/// must be in [1, 65535] and must be paired with a scheme, and a set scheme +/// must be http or https (case-insensitive — downstream normalizes). +fn validate_egress_rule_match(rule_match: &EgressRuleMatch, index: usize) -> AppResult<()> { + if let Some(port) = rule_match.port { + if !(1..=65535).contains(&port) { + return Err(AppError::BadRequest(format!( + "network.rules[{index}].match.port must be in [1, 65535], got {port}" + ))); + } + if rule_match.scheme.is_none() { + return Err(AppError::BadRequest(format!( + "network.rules[{index}].match.port requires match.scheme to be set" + ))); + } + } + if let Some(scheme) = rule_match.scheme.as_deref() { + if !scheme.eq_ignore_ascii_case("http") && !scheme.eq_ignore_ascii_case("https") { + return Err(AppError::BadRequest(format!( + "network.rules[{index}].match.scheme must be 'http' or 'https', got {scheme:?}" + ))); + } + } + Ok(()) +} + fn map_egress_rule(rule: &EgressRule) -> CubeEgressRule { CubeEgressRule { name: rule.name.clone(), @@ -1089,6 +1122,7 @@ fn map_egress_rule(rule: &EgressRule) -> CubeEgressRule { method: rule.r#match.method.clone(), path: rule.r#match.path.clone(), scheme: rule.r#match.scheme.clone(), + port: rule.r#match.port, }, action: CubeEgressRuleAction { allow: rule.action.allow, @@ -1342,6 +1376,7 @@ mod tests { method: Some(vec!["POST".to_string()]), path: Some("/v1/chat".to_string()), sni: Some("api.deepseek.com".to_string()), + port: None, }, action: EgressRuleAction { allow: true, @@ -1410,6 +1445,91 @@ mod tests { assert!(rule["action"].get("inject").is_none()); } + fn network_with_match(rule_match: EgressRuleMatch) -> SandboxNetworkConfig { + SandboxNetworkConfig { + allow_public_traffic: None, + allow_out: None, + deny_out: None, + mask_request_host: None, + rules: Some(vec![EgressRule { + name: "r1".to_string(), + r#match: rule_match, + action: EgressRuleAction { + allow: true, + audit: None, + inject: None, + }, + }]), + } + } + + #[test] + fn egress_match_port_requires_scheme() { + let err = build_cube_network_config( + None, + Some(&network_with_match(EgressRuleMatch { + port: Some(8443), + ..Default::default() + })), + ) + .expect_err("port without scheme must be rejected"); + assert!(err.to_string().contains("requires match.scheme"), "{err}"); + } + + #[test] + fn egress_match_port_range_enforced() { + for port in [0, -1, 65536, 99999] { + let err = build_cube_network_config( + None, + Some(&network_with_match(EgressRuleMatch { + port: Some(port), + scheme: Some("https".to_string()), + ..Default::default() + })), + ) + .expect_err("out-of-range port must be rejected"); + assert!(err.to_string().contains("[1, 65535]"), "{err}"); + } + } + + #[test] + fn egress_match_invalid_scheme_rejected() { + let err = build_cube_network_config( + None, + Some(&network_with_match(EgressRuleMatch { + scheme: Some("ftp".to_string()), + ..Default::default() + })), + ) + .expect_err("non-http(s) scheme must be rejected"); + assert!( + err.to_string().contains("must be 'http' or 'https'"), + "{err}" + ); + } + + #[test] + fn egress_match_valid_port_scheme_accepted() { + build_cube_network_config( + None, + Some(&network_with_match(EgressRuleMatch { + port: Some(8443), + scheme: Some("https".to_string()), + ..Default::default() + })), + ) + .expect("valid port+scheme pair"); + // Case variants are accepted (downstream normalizes to lowercase). + build_cube_network_config( + None, + Some(&network_with_match(EgressRuleMatch { + scheme: Some("HTTPS".to_string()), + ..Default::default() + })), + ) + .expect("uppercase scheme is accepted"); + } + #[test] fn listed_sandbox_preserves_resources_from_cubemaster_list() { let listed = from_cubemaster_info(SandboxInfo { diff --git a/CubeEgress/Makefile b/CubeEgress/Makefile index e9200b070..1b7ea4b9e 100644 --- a/CubeEgress/Makefile +++ b/CubeEgress/Makefile @@ -27,6 +27,13 @@ else MAKEFLAGS += --no-print-directory endif +.PHONY: test-lua +test-lua: + $(Q)LUA_BIN=$$(command -v luajit || command -v lua); \ + test -n "$$LUA_BIN" || { echo "error: lua or luajit not installed"; exit 1; }; \ + $$LUA_BIN tests/port_scheme_test.lua && \ + $$LUA_BIN tests/port_scheme_extra_test.lua + .PHONY: build build: $(call msg,BUILD IMAGE $(IMAGE_LOCAL):$(IMAGE_TAG)-$(ARCH)) diff --git a/CubeEgress/lua/access_phase.lua b/CubeEgress/lua/access_phase.lua index 9b30b9005..e6e680ab7 100644 --- a/CubeEgress/lua/access_phase.lua +++ b/CubeEgress/lua/access_phase.lua @@ -68,6 +68,7 @@ -- deployments use "skipped" + admin API for policy installation. local policy = require "policy" +local port_scheme = require "port_scheme" local _M = {} @@ -122,8 +123,12 @@ end -- ---------- match evaluation ---------- --- Returns true if every constraint in `m` passes against `ctx`. -local function rule_matches(m, ctx) +-- Returns true if every constraint in `m` passes against `ctx`. `is_allow` +-- selects the port/scheme semantics: an allow rule with an omitted port is +-- narrowed to the default set {80/http,443/https} (fail-closed), while a deny +-- rule with an omitted port is port-agnostic within the host (also fail-closed +-- — a custom-port allow must not bypass a broader host deny). +local function rule_matches(m, ctx, is_allow) if type(m) ~= "table" then return false end if m.sni ~= nil then @@ -143,8 +148,16 @@ local function rule_matches(m, ctx) if m.path ~= nil then if not path_match(m.path, ctx.path) then return false end end - if m.scheme ~= nil then - if string.lower(m.scheme) ~= ctx.scheme then return false end + -- Port and scheme form one semantic constraint whose meaning depends on + -- whether this rule allows or denies (see port_scheme.matches_deny). + local port_scheme_ok + if is_allow then + port_scheme_ok = port_scheme.matches(m.port, m.scheme, ctx.dst_port, ctx.scheme) + else + port_scheme_ok = port_scheme.matches_deny(m.port, m.scheme, ctx.dst_port, ctx.scheme) + end + if not port_scheme_ok then + return false end return true end @@ -294,6 +307,13 @@ local function build_ctx() method = ngx.var.request_method, path = ngx.var.uri, dst_ip = dst_ip, + -- dst_port is the original sandbox-side destination port preserved + -- by TPROXY. In an IP_TRANSPARENT listener, nginx's $server_port is + -- read via getsockname() on the tproxy socket, which reports the + -- ORIGINAL dst — not the 8080/8443 listener port we bind to. Used + -- by rule_matches() to enforce match.port constraints on rules that + -- pin to a custom port (e.g. tcp/8443 for an internal API). + dst_port = tonumber(ngx.var.server_port), scheme = ngx.var.scheme, } end @@ -408,6 +428,7 @@ function _M.decide() method = ctx.method, path = ctx.path, dst_ip = ctx.dst_ip, + dst_port = ctx.dst_port, scheme = ctx.scheme, policy_id = nil, rule_id = nil, @@ -467,7 +488,7 @@ function _M.decide() decision.policy_id = p.policy_id for _, r in ipairs(p.rules or {}) do - if rule_matches(r.match, ctx) then + if rule_matches(r.match, ctx, r.action ~= nil and r.action.allow == true) then decision.rule_id = r.id decision.audit_level = (r.action and r.action.audit) or "metadata" decision.inject = r.action and r.action.inject -- consumed in Pγ diff --git a/CubeEgress/lua/policy.lua b/CubeEgress/lua/policy.lua index bc0127e7a..5d9e96502 100644 --- a/CubeEgress/lua/policy.lua +++ b/CubeEgress/lua/policy.lua @@ -19,6 +19,7 @@ local cjson = require "cjson.safe" -- resty.openssl.digest is already used by cert_signer.lua (sha256 leaf -- signing), so the runtime cost of pulling it in here is amortized. local digest_lib = require "resty.openssl.digest" +local port_scheme = require "port_scheme" local _M = {} @@ -31,6 +32,7 @@ local INDEX_LOCK_TIMEOUT_MS = 1000 -- per-entry limit; anything larger is almost certainly a misconfiguration -- (and would bloat policy_store fast). local SECRET_MAX_BYTES = 65536 +local MAX_L7_PORTS_PER_HOST = 8 -- ---------- validation ---------- @@ -50,8 +52,114 @@ local function is_valid_sandbox_ip(s) return true end +-- Strict IPv4 (+optional /prefix) parse, mirroring Go's netip acceptance: +-- no leading zeros, octets <= 255, prefix <= 32. Returns the canonical +-- dotted-quad and prefix (default 32), or nil. +local function parse_ipv4_cidr(s) + local a, b, c, d, prefix = string.match(s, "^([0-9]+)%.([0-9]+)%.([0-9]+)%.([0-9]+)/([0-9]+)$") + if not a then + a, b, c, d = string.match(s, IPV4_PATTERN) + end + if not a then return nil end + local octets = {} + for _, octet in ipairs({a, b, c, d}) do + if #octet > 1 and string.sub(octet, 1, 1) == "0" then return nil end + local n = tonumber(octet) + if not n or n > 255 then return nil end + octets[#octets + 1] = tostring(n) + end + if prefix then + prefix = tonumber(prefix) + if not prefix or prefix > 32 then return nil end + else + prefix = 32 + end + return table.concat(octets, "."), prefix +end + +-- Canonicalize a rule host/sni for conflict-detection aggregation. +-- Returns the canonical identity, or (nil, reason) when the value must be +-- rejected outright. Kept in lock-step with Go's l7GroupKey/classifyL7Target: +-- - DNS names: lowercase + strip one trailing dot (no whitespace trim on +-- either side — padded values are rejected, not repaired). +-- - "1.2.3.4" and "1.2.3.4/32" map to the same identity (Go groups both +-- via parseCIDR), so a per-(host,port) scheme conflict between the two +-- spellings is detected instead of bypassed. +-- - Subnet CIDRs (prefixlen < 32) are rejected, as Go's classifyL7Target +-- does: an L7 rule pins exactly one (host, port) listener tuple, and a +-- subnet can never appear in an HTTP Host header or TLS SNI anyway. +local function normalize_identity(value) + if type(value) ~= "string" or value == "" then return nil, "empty" end + -- Reject whitespace-padded values: request-time matching compares the + -- raw host/sni, so a padded identity would validate yet never match — + -- fail fast instead of accepting a dead rule. + if string.find(value, "^%s") or string.find(value, "%s$") then + return nil, "leading or trailing whitespace" + end + local identity = string.gsub(string.lower(value), "%.$", "") + if identity == "" then return nil, "empty" end + + if string.find(identity, "/", 1, true) then + -- CIDR notation: only a /32 IPv4 literal is meaningful here. + local ip, prefix = parse_ipv4_cidr(identity) + if not ip then return nil, "invalid CIDR notation" end + if prefix < 32 then return nil, "subnet CIDR is not a valid L7 host" end + return ip + end + -- Dotted-quad-shaped values (digits and dots only) must be valid IPv4; + -- canonicalize so bare and /32 spellings aggregate identically. + if string.match(identity, "^[0-9.]+$") then + local ip = parse_ipv4_cidr(identity) + if not ip then return nil, "invalid IPv4 address" end + return ip + end + return identity +end + +local function validate_match_tuples(match, rule_index, identities) + local tuples, err = port_scheme.expand(match.port, match.scheme) + if not tuples then + return false, string.format("rules[%d].match.%s", rule_index, err) + end + + -- When both identities are present, Host is the policy aggregation key; + -- otherwise SNI is used. Request-time matching still evaluates both fields. + local raw_identity = match.host + if raw_identity == nil then raw_identity = match.sni end + if raw_identity == nil then return true end + local identity, identity_err = normalize_identity(raw_identity) + if not identity then + return false, string.format("rules[%d].match host/sni %q is invalid: %s", + rule_index, raw_identity, identity_err or "empty") + end + + local state = identities[identity] + if not state then + state = {count = 0, ports = {}} + identities[identity] = state + end + for _, tuple in ipairs(tuples) do + local existing = state.ports[tuple.port] + if existing ~= nil and existing ~= tuple.scheme then + return false, string.format( + "rules[%d].match conflicts for host %q port %d: %s vs %s", + rule_index, identity, tuple.port, existing, tuple.scheme) + end + if existing == nil then + state.ports[tuple.port] = tuple.scheme + state.count = state.count + 1 + if state.count > MAX_L7_PORTS_PER_HOST then + return false, string.format( + "rules[%d].match exceeds %d L7 port tuples for host %q", + rule_index, MAX_L7_PORTS_PER_HOST, identity) + end + end + end + return true +end + -- Validate policy structure. Returns (true, nil) or (false, err_string). --- Strict on required fields; permissive on optional (forward-compat). +-- Strict on required fields; permissive on unrelated optional fields. local function validate_policy(p) if type(p) ~= "table" then return false, "policy must be an object" end if type(p.policy_id) ~= "string" or p.policy_id == "" then @@ -67,6 +175,7 @@ local function validate_policy(p) return false, "policy.rules must have at least one rule" end local seen_ids = {} + local identities = {} for i = 1, n do local r = p.rules[i] if type(r) ~= "table" then @@ -82,6 +191,8 @@ local function validate_policy(p) if type(r.match) ~= "table" then return false, "rules[" .. i .. "].match required (object; empty {} allowed)" end + local match_ok, match_err = validate_match_tuples(r.match, i, identities) + if not match_ok then return false, match_err end if type(r.action) ~= "table" then return false, "rules[" .. i .. "].action required (object)" end diff --git a/CubeEgress/lua/port_scheme.lua b/CubeEgress/lua/port_scheme.lua new file mode 100644 index 000000000..b2a2d3b77 --- /dev/null +++ b/CubeEgress/lua/port_scheme.lua @@ -0,0 +1,104 @@ +-- CubeEgress L7 destination port/scheme semantics shared by validation and +-- request matching. +local _M = {} + +local function trim(s) + return (string.gsub(s, "^%s*(.-)%s*$", "%1")) +end + +function _M.normalize_scheme(value) + if value == nil then return nil end + if type(value) ~= "string" then + return nil, "scheme must be a string" + end + local scheme = string.lower(trim(value)) + if scheme ~= "http" and scheme ~= "https" then + return nil, "scheme must be http or https" + end + return scheme +end + +local function normalize_port(value) + if value == nil then return nil end + if type(value) ~= "number" or value ~= math.floor(value) then + return nil, "port must be an integer" + end + if value < 1 or value > 65535 then + return nil, "port must be in [1, 65535]" + end + return value +end + +-- expand returns the effective destination tuples for a rule: +-- nil/nil -> 80/http + 443/https +-- nil/scheme -> the scheme's conventional port +-- port/scheme -> the exact tuple +-- port/nil -> invalid +function _M.expand(port_value, scheme_value) + local port, perr = normalize_port(port_value) + if perr then return nil, perr end + local scheme, serr = _M.normalize_scheme(scheme_value) + if serr then return nil, serr end + + if port ~= nil and scheme == nil then + return nil, "port requires scheme" + end + if port ~= nil then + return {{port = port, scheme = scheme}} + end + if scheme == "http" then + return {{port = 80, scheme = "http"}} + end + if scheme == "https" then + return {{port = 443, scheme = "https"}} + end + return { + {port = 80, scheme = "http"}, + {port = 443, scheme = "https"}, + } +end + +function _M.matches(port_value, scheme_value, dst_port, request_scheme) + local tuples = _M.expand(port_value, scheme_value) + if not tuples then return false end + local scheme = _M.normalize_scheme(request_scheme) + local port = tonumber(dst_port) + if not scheme or not port then return false end + + for _, tuple in ipairs(tuples) do + if tuple.port == port and tuple.scheme == scheme then + return true + end + end + return false +end + +-- matches_deny evaluates the port/scheme constraint for a DENY rule. An allow +-- rule with an omitted port narrows to the default set {80/http,443/https} +-- (fail-closed: a custom-port flow needs an explicit allow). A deny rule with +-- an omitted port is instead port-AGNOSTIC within the host: it matches any +-- intercepted flow to the host. Narrowing a deny to the default set would be +-- fail-open — a custom-port allow rule (e.g. allow host="api.example.com" +-- port=8443 scheme=https) would bypass a broader host deny (deny +-- host="*.example.com") because the deny never matched port 8443. An +-- explicitly-set port+scheme still matches exactly that tuple. +function _M.matches_deny(port_value, scheme_value, dst_port, request_scheme) + local port, perr = normalize_port(port_value) + if perr then return false end + local scheme, serr = _M.normalize_scheme(scheme_value) + if serr then return false end + if port ~= nil then + -- Explicit port (+scheme): exact tuple, same as the allow path. + return _M.matches(port_value, scheme_value, dst_port, request_scheme) + end + if scheme == nil then + -- No port and no scheme: deny every intercepted flow to the host. + return true + end + -- No port but a scheme: deny every intercepted flow of that scheme, + -- regardless of which port carried it. + local req_scheme = _M.normalize_scheme(request_scheme) + return req_scheme ~= nil and req_scheme == scheme +end + +return _M diff --git a/CubeEgress/scripts/cube-proxy-iptables-init.sh b/CubeEgress/scripts/cube-proxy-iptables-init.sh index bf73063af..77658453e 100755 --- a/CubeEgress/scripts/cube-proxy-iptables-init.sh +++ b/CubeEgress/scripts/cube-proxy-iptables-init.sh @@ -3,12 +3,19 @@ # CubeSandbox transparent proxy — host-side network setup. # Phase 1: full MITM via OpenResty + TPROXY. # -# Selection model: traffic is matched purely by ingress interface and -# destination port — `iif cube-dev` + tcp dport 80/443. There is NO -# fwmark involved in either the forward (sandbox→OpenResty) or the -# return (OpenResty→sandbox) direction. The cube-egress worker's -# replies are routed naturally by the kernel and re-injected into the -# sandbox tap by the from_envoy BPF program on cube-dev egress. +# Selection model: traffic is matched by ingress interface and by an +# skb->mark stamped from the sandbox tap's eBPF datapath. The mvmtap +# program reads allow_out_v3 to find the (host, port) → scheme mapping +# for the outgoing SYN, then writes CUBE_L7_MARK_HTTP (0xCE010000) or +# CUBE_L7_MARK_HTTPS (0xCE020000) so this chain can steer the packet at +# 8080 (nginx HTTP listener) or 8443 (nginx HTTPS listener) regardless +# of the original destination port. This lets users configure L7 +# capture on arbitrary ports (e.g. an API on tcp/3000) without teaching +# iptables about their port map. +# +# The cube-owned mark uses the CUBE_L7_MARK_MASK (0xFFFF0000) high-16 +# range so the low 16 bits remain free for host-level marks users may +# set for other purposes. # # Idempotent: safe to re-run. Rules live in a dedicated TRANSPROXY # sub-chain so 'down' tears down our config without touching anything @@ -64,6 +71,38 @@ SANDBOX_NETWORK_CIDR="${CUBE_SANDBOX_NETWORK_CIDR:-192.168.0.0/18}" TPROXY_ON_IP="$(sandbox_gateway_ip_from_cidr "${SANDBOX_NETWORK_CIDR}")" # cube-dev IP TPROXY_PORT_HTTP=8080 TPROXY_PORT_HTTPS=8443 +# skb->mark values written by the mvmtap L7 proxy path (the cube_l7_mark_http / +# cube_l7_mark_https / cube_l7_mark_mask globals in CubeNet/src/cubevs.h). Only +# the high 16 bits are cube-owned; the mask lets users co-exist with other +# host-level fwmark schemes on the low 16 bits. +# +# Defaults match the shipped values; a deployment may override them via +# /etc/cubeegress/l7-marks.conf so these rules and the dataplane (which reads +# the same file into its eBPF globals) stay in lock-step. +CUBE_L7_MARK_HTTP=0xCE010000 +CUBE_L7_MARK_HTTPS=0xCE020000 +CUBE_L7_MARK_MASK=0xFFFF0000 +if [ -f /etc/cubeegress/l7-marks.conf ]; then + # shellcheck disable=SC1091 + . /etc/cubeegress/l7-marks.conf +fi +# Validate the (possibly overridden) marks: http must differ from https, and +# both may only set bits inside the mask. Compare arithmetically (not as +# strings) so the same value in different notations — 0xCE010000 vs +# 0xce010000 vs 3456172032 — is still rejected, matching the uint32 +# comparison in cubevs.resolveL7Marks. +validate_l7_marks() { + if (( CUBE_L7_MARK_HTTP == CUBE_L7_MARK_HTTPS )); then + echo "cube-proxy-iptables-init: CUBE_L7_MARK_HTTP (${CUBE_L7_MARK_HTTP}) must differ from CUBE_L7_MARK_HTTPS" >&2 + exit 1 + fi + if [ $(( CUBE_L7_MARK_HTTP & ~CUBE_L7_MARK_MASK )) -ne 0 ] || \ + [ $(( CUBE_L7_MARK_HTTPS & ~CUBE_L7_MARK_MASK )) -ne 0 ]; then + echo "cube-proxy-iptables-init: L7 marks must set bits only within CUBE_L7_MARK_MASK (${CUBE_L7_MARK_MASK})" >&2 + exit 1 + fi +} +validate_l7_marks ROUTE_TABLE=100 INGRESS_IFACE="${CUBE_INGRESS_IFACE:-cube-dev}" CHAIN="TRANSPROXY" @@ -81,33 +120,73 @@ require_modules() { done } -# Create-or-flush our sub-chain, then ensure PREROUTING jumps to it once. +# Build the steering rules into a scratch chain and only swap it into the live +# chain once every rule is verified present. Flushing and rebuilding the live +# chain in place would be non-atomic: if the second (HTTPS) rule failed after +# the first succeeded, the script would abort before install_routing() while +# PREROUTING still jumped at a half-built chain — a silent fail-open that +# bypasses L7 interception/deny for that scheme. Building in an unreferenced +# scratch chain keeps the currently-live config intact until the swap. install_chain() { - iptables -t mangle -N "${CHAIN}" 2>/dev/null || true - iptables -t mangle -F "${CHAIN}" + local scratch="${CHAIN}.new" - iptables -t mangle -C PREROUTING -j "${CHAIN}" 2>/dev/null \ - || iptables -t mangle -A PREROUTING -j "${CHAIN}" + # Start from a clean scratch chain. It is not referenced by PREROUTING yet, + # so a failure below never disturbs the currently-live chain. + iptables -t mangle -F "${scratch}" 2>/dev/null || true + iptables -t mangle -X "${scratch}" 2>/dev/null || true + iptables -t mangle -N "${scratch}" - iptables -t mangle -A "${CHAIN}" \ - -i "${INGRESS_IFACE}" -p tcp --dport 80 \ + # HTTP: mvmtap stamps CUBE_L7_MARK_HTTP → steer to nginx HTTP listener. + iptables -t mangle -A "${scratch}" \ + -i "${INGRESS_IFACE}" -p tcp \ + -m mark --mark "${CUBE_L7_MARK_HTTP}/${CUBE_L7_MARK_MASK}" \ -j TPROXY --on-ip "${TPROXY_ON_IP}" --on-port "${TPROXY_PORT_HTTP}" - iptables -t mangle -A "${CHAIN}" \ - -i "${INGRESS_IFACE}" -p tcp --dport 443 \ + # HTTPS: mvmtap stamps CUBE_L7_MARK_HTTPS → steer to nginx HTTPS listener. + iptables -t mangle -A "${scratch}" \ + -i "${INGRESS_IFACE}" -p tcp \ + -m mark --mark "${CUBE_L7_MARK_HTTPS}/${CUBE_L7_MARK_MASK}" \ -j TPROXY --on-ip "${TPROXY_ON_IP}" --on-port "${TPROXY_PORT_HTTPS}" - iptables -t mangle -A "${CHAIN}" -j RETURN + iptables -t mangle -A "${scratch}" -j RETURN + + # Post-condition: both steering rules must exist before the swap, else a + # silently dropped rule would fail open for that scheme. + iptables -t mangle -C "${scratch}" \ + -i "${INGRESS_IFACE}" -p tcp \ + -m mark --mark "${CUBE_L7_MARK_HTTP}/${CUBE_L7_MARK_MASK}" \ + -j TPROXY --on-ip "${TPROXY_ON_IP}" --on-port "${TPROXY_PORT_HTTP}" \ + || fatal "HTTP TPROXY rule missing from ${scratch} after build" + iptables -t mangle -C "${scratch}" \ + -i "${INGRESS_IFACE}" -p tcp \ + -m mark --mark "${CUBE_L7_MARK_HTTPS}/${CUBE_L7_MARK_MASK}" \ + -j TPROXY --on-ip "${TPROXY_ON_IP}" --on-port "${TPROXY_PORT_HTTPS}" \ + || fatal "HTTPS TPROXY rule missing from ${scratch} after build" + + # Swap: detach the previously-live chain from PREROUTING, drop it, rename + # the fully-built scratch chain into the live name, then point PREROUTING + # at it. The old config keeps serving until it is detached here, so there + # is no window with a half-configured chain. + while iptables -t mangle -C PREROUTING -j "${CHAIN}" 2>/dev/null; do + iptables -t mangle -D PREROUTING -j "${CHAIN}" || break + done + iptables -t mangle -F "${CHAIN}" 2>/dev/null || true + iptables -t mangle -X "${CHAIN}" 2>/dev/null || true + iptables -t mangle -E "${scratch}" "${CHAIN}" + iptables -t mangle -C PREROUTING -j "${CHAIN}" 2>/dev/null \ + || iptables -t mangle -A PREROUTING -j "${CHAIN}" } install_routing() { - # Two ip rules: tcp/80 and tcp/443 from cube-dev → table 100. - # Match by selectors (iif/ipproto/dport), not fwmark. - local proto port - for port in 80 443; do + # Two ip rules: cube-owned mark bits → table 100. Match on fwmark so the + # rule set stays independent of the original destination port; user rules + # may attach L7 handling to arbitrary ports (e.g. tcp/3000) and mvmtap + # writes the same mark for all of them. + local mark + for mark in "${CUBE_L7_MARK_HTTP}" "${CUBE_L7_MARK_HTTPS}"; do if ! ip rule show \ - | grep -q "iif ${INGRESS_IFACE} ipproto tcp dport ${port} lookup ${ROUTE_TABLE}"; then - ip rule add iif "${INGRESS_IFACE}" ipproto tcp dport "${port}" \ + | grep -qiE "fwmark ${mark}/${CUBE_L7_MARK_MASK} lookup ${ROUTE_TABLE}[[:space:]]*$"; then + ip rule add fwmark "${mark}/${CUBE_L7_MARK_MASK}" \ table "${ROUTE_TABLE}" fi done @@ -124,20 +203,36 @@ remove_chain() { done iptables -t mangle -F "${CHAIN}" 2>/dev/null || true iptables -t mangle -X "${CHAIN}" 2>/dev/null || true + # Drop any scratch chain left over from an aborted install. + iptables -t mangle -F "${CHAIN}.new" 2>/dev/null || true + iptables -t mangle -X "${CHAIN}.new" 2>/dev/null || true } remove_routing() { - local port - for port in 80 443; do + local mark + for mark in "${CUBE_L7_MARK_HTTP}" "${CUBE_L7_MARK_HTTPS}"; do while ip rule show \ - | grep -q "iif ${INGRESS_IFACE} ipproto tcp dport ${port} lookup ${ROUTE_TABLE}"; do - ip rule del iif "${INGRESS_IFACE}" ipproto tcp dport "${port}" \ + | grep -qiE "fwmark ${mark}/${CUBE_L7_MARK_MASK} lookup ${ROUTE_TABLE}[[:space:]]*$"; do + ip rule del fwmark "${mark}/${CUBE_L7_MARK_MASK}" \ table "${ROUTE_TABLE}" || break done done ip route flush table "${ROUTE_TABLE}" 2>/dev/null || true } +# Remove policy-routing selectors installed by the pre-fwmark implementation. +# Delete only the exact cube-dev TCP/80 and TCP/443 selectors; unrelated rules +# using table 100 are outside this script's ownership. +remove_legacy_dport_routing() { + local port + for port in 80 443; do + while ip rule del iif "${INGRESS_IFACE}" ipproto tcp dport "${port}" \ + table "${ROUTE_TABLE}" 2>/dev/null; do + : + done + done +} + show_status() { log "=== mangle/${CHAIN} ===" iptables -t mangle -L "${CHAIN}" -n -v --line-numbers 2>/dev/null \ @@ -163,6 +258,7 @@ main() { require_modules install_chain install_routing + remove_legacy_dport_routing log "cube-proxy iptables/route rules installed" show_status ;; @@ -170,6 +266,7 @@ main() { require_root remove_chain remove_routing + remove_legacy_dport_routing log "cube-proxy iptables/route rules removed" ;; status) @@ -182,4 +279,6 @@ main() { esac } -main "$@" +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/CubeEgress/tests/cube-proxy-iptables-init_test.sh b/CubeEgress/tests/cube-proxy-iptables-init_test.sh new file mode 100644 index 000000000..6df11e1cc --- /dev/null +++ b/CubeEgress/tests/cube-proxy-iptables-init_test.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=../scripts/cube-proxy-iptables-init.sh +source "${ROOT_DIR}/scripts/cube-proxy-iptables-init.sh" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT +CALLS="${TMP_DIR}/calls" + +# Each legacy selector exists twice. The third deletion fails and terminates +# the loop. Record every attempted exact command. +declare -A DELETE_COUNT=([80]=0 [443]=0) +ip() { + printf '%s\n' "$*" >> "${CALLS}" + if [[ "$1 $2" == "rule del" ]]; then + local port="" + local i + for ((i = 1; i <= $#; i++)); do + if [[ "${!i}" == "dport" ]]; then + local next=$((i + 1)) + port="${!next}" + break + fi + done + [[ -n "${port}" ]] || return 1 + DELETE_COUNT["${port}"]=$((DELETE_COUNT["${port}"] + 1)) + (( DELETE_COUNT["${port}"] <= 2 )) + return + fi + return 1 +} + +remove_legacy_dport_routing +[[ "${DELETE_COUNT[80]}" -eq 3 ]] +[[ "${DELETE_COUNT[443]}" -eq 3 ]] +grep -Fxq "rule del iif cube-dev ipproto tcp dport 80 table 100" "${CALLS}" +grep -Fxq "rule del iif cube-dev ipproto tcp dport 443 table 100" "${CALLS}" +if grep -Ev '^rule del iif cube-dev ipproto tcp dport (80|443) table 100$' "${CALLS}"; then + echo "unexpected legacy cleanup command" >&2 + exit 1 +fi + +# --- fwmark ip-rule idempotency is case-insensitive -------------------------- +# iproute2 prints fwmark hex in lowercase (0xce010000/0xffff0000), but the +# script greps for the uppercase CUBE_L7_MARK_* constants. The grep must be +# case-insensitive, or install_routing re-adds duplicate rules on every run and +# remove_routing never matches anything to delete. +RULES_STATE="${TMP_DIR}/rules" +: > "${RULES_STATE}" +: > "${CALLS}" +ip() { + printf '%s\n' "$*" >> "${CALLS}" + case "$1 $2" in + "rule show") + cat "${RULES_STATE}" + return 0 + ;; + "rule add") + # $3=fwmark $4=mark/mask $5=table $6=; store iproute2-style + # lowercase show output, as the real `ip rule show` would print. + printf 'from all fwmark %s lookup %s\n' "$4" "$6" \ + | tr '[:upper:]' '[:lower:]' >> "${RULES_STATE}" + return 0 + ;; + "rule del") + local needle + needle="$(printf '%s' "$4" | tr '[:upper:]' '[:lower:]')" + grep -viF "fwmark ${needle} " "${RULES_STATE}" > "${RULES_STATE}.tmp" || true + mv "${RULES_STATE}.tmp" "${RULES_STATE}" + return 0 + ;; + "route "*) + return 0 + ;; + esac + return 0 +} + +# Re-running install must not duplicate rules: the grep matches the lowercase +# `ip rule show` output, so the second run is a no-op. +install_routing +install_routing +[[ "$(grep -c 'rule add fwmark' "${CALLS}" || true)" -eq 2 ]] +[[ "$(wc -l < "${RULES_STATE}")" -eq 2 ]] + +# remove must match (and so delete) every installed rule. +remove_routing +[[ "$(grep -c 'rule del fwmark' "${CALLS}" || true)" -eq 2 ]] +[[ ! -s "${RULES_STATE}" ]] + +# --- install_chain builds into a scratch chain and swaps atomically ---------- +# A steering-rule failure must abort BEFORE the live chain / PREROUTING jump is +# touched, so a partial install never leaves a fail-open gap. We mock iptables +# and record every call; chain contents live in IPT_STATE, the PREROUTING jump +# in JUMP_STATE. +: > "${CALLS}" +IPT_STATE="${TMP_DIR}/iptables_rules" +JUMP_STATE="${TMP_DIR}/prerouting_jump" +: > "${IPT_STATE}" +: > "${JUMP_STATE}" +FAIL_ON_MARK="" + +iptables() { + printf '%s\n' "$*" >> "${CALLS}" + local op="$3" target="$4" + local rest="${*:5}" + case "${op}" in + -N|-F|-X) return 0 ;; + -E) # rename chain $4 -> $5, carrying its rules + local from="$4" to="$5" + { grep -vF "${from}|" "${IPT_STATE}" || true + grep -F "${from}|" "${IPT_STATE}" | sed "s|^${from}|${to}|" || true + } > "${IPT_STATE}.new" + mv "${IPT_STATE}.new" "${IPT_STATE}" + return 0 ;; + -A) + if [[ "${target}" == "PREROUTING" ]]; then + printf '%s\n' "$6" > "${JUMP_STATE}" + return 0 + fi + if [[ -n "${FAIL_ON_MARK}" && "${rest}" == *"${FAIL_ON_MARK}"* ]]; then + return 1 + fi + printf '%s|%s\n' "${target}" "${rest}" >> "${IPT_STATE}" + return 0 ;; + -C) + if [[ "${target}" == "PREROUTING" ]]; then + [[ "$(cat "${JUMP_STATE}")" == "$6" ]] && return 0 || return 1 + fi + grep -qF "${target}|${rest}" "${IPT_STATE}" && return 0 || return 1 ;; + -D) + if [[ "${target}" == "PREROUTING" ]]; then : > "${JUMP_STATE}"; fi + return 0 ;; + esac + return 0 +} + +# Success path: both steering rules are verified in the scratch chain BEFORE it +# is renamed into the live chain, and PREROUTING only jumps at the live chain +# after the swap. +install_chain +http_v="$(grep -n -- "-t mangle -C ${CHAIN}.new .*${CUBE_L7_MARK_HTTP}" "${CALLS}" | head -1 | cut -d: -f1)" +https_v="$(grep -n -- "-t mangle -C ${CHAIN}.new .*${CUBE_L7_MARK_HTTPS}" "${CALLS}" | head -1 | cut -d: -f1)" +swap="$(grep -n -- "-t mangle -E ${CHAIN}.new ${CHAIN}" "${CALLS}" | head -1 | cut -d: -f1)" +jump="$(grep -n -- "-t mangle -A PREROUTING -j ${CHAIN}" "${CALLS}" | head -1 | cut -d: -f1)" +[[ -n "${http_v}" && -n "${https_v}" && -n "${swap}" && -n "${jump}" ]] +[[ "${http_v}" -lt "${swap}" && "${https_v}" -lt "${swap}" && "${swap}" -lt "${jump}" ]] +# Post-swap, the live chain holds both steering rules. +grep -qF "${CHAIN}|-i ${INGRESS_IFACE} -p tcp -m mark --mark ${CUBE_L7_MARK_HTTP}/${CUBE_L7_MARK_MASK} -j TPROXY --on-ip ${TPROXY_ON_IP} --on-port ${TPROXY_PORT_HTTP}" "${IPT_STATE}" +grep -qF "${CHAIN}|-i ${INGRESS_IFACE} -p tcp -m mark --mark ${CUBE_L7_MARK_HTTPS}/${CUBE_L7_MARK_MASK} -j TPROXY --on-ip ${TPROXY_ON_IP} --on-port ${TPROXY_PORT_HTTPS}" "${IPT_STATE}" + +# Failure path: the HTTPS rule fails to add. The subshell aborts before the +# swap and before any PREROUTING jump, so the live path is never half-built. +: > "${CALLS}"; : > "${IPT_STATE}"; : > "${JUMP_STATE}" +if ( FAIL_ON_MARK="${CUBE_L7_MARK_HTTPS}"; install_chain ); then + echo "install_chain succeeded despite HTTPS rule failure" >&2 + exit 1 +fi +if grep -q -- "-t mangle -E ${CHAIN}.new ${CHAIN}" "${CALLS}"; then + echo "live chain swapped in despite failed HTTPS rule (fail-open)" >&2 + exit 1 +fi +if grep -q -- "-t mangle -A PREROUTING -j ${CHAIN}" "${CALLS}"; then + echo "PREROUTING jump installed despite failed HTTPS rule (fail-open)" >&2 + exit 1 +fi +unset -f iptables + +# Verify migration ordering without touching the host network. +: > "${CALLS}" +require_root() { :; } +require_iface() { :; } +require_modules() { :; } +install_chain() { echo install_chain >> "${CALLS}"; } +install_routing() { echo install_routing >> "${CALLS}"; } +remove_legacy_dport_routing() { echo remove_legacy >> "${CALLS}"; } +show_status() { echo show_status >> "${CALLS}"; } +main up +EXPECTED=$'install_chain\ninstall_routing\nremove_legacy\nshow_status' +[[ "$(<"${CALLS}")" == "${EXPECTED}" ]] + +# --- marks validation compares arithmetically, not as strings ---------------- +# Go resolveL7Marks compares uint32 values; the shell check must reject the +# same mark written in a different notation (hex case variant or decimal), +# otherwise iptables and the dataplane diverge on identical marks. +# validate_l7_marks exits the (sub)shell on rejection, so a subshell that +# survives means the value was (wrongly) accepted. +if ( + CUBE_L7_MARK_HTTP=0xCE010000 + CUBE_L7_MARK_HTTPS=0xce010000 # same value, different case + validate_l7_marks +); then + echo "case-variant identical marks were not rejected" >&2 + exit 1 +fi +if ( + CUBE_L7_MARK_HTTP=0xCE010000 + CUBE_L7_MARK_HTTPS=3456172032 # decimal equivalent of 0xCE010000 + validate_l7_marks +); then + echo "decimal-equivalent identical marks were not rejected" >&2 + exit 1 +fi +# distinct values within the mask still pass +( + CUBE_L7_MARK_HTTP=0xCE010000 + CUBE_L7_MARK_HTTPS=0xCE020000 + CUBE_L7_MARK_MASK=0xFFFF0000 + validate_l7_marks +) +# bits outside the mask are still rejected (0x0000CE00 sets low-16 bits) +if ( + CUBE_L7_MARK_HTTP=0x0000CE00 + CUBE_L7_MARK_HTTPS=0xCE020000 + CUBE_L7_MARK_MASK=0xFFFF0000 + validate_l7_marks +); then + echo "out-of-mask mark was not rejected" >&2 + exit 1 +fi + +# --- sandbox_gateway_ip_from_cidr: TPROXY --on-ip = network address + 1 ------- +# The TPROXY target IP is derived from CUBE_SANDBOX_NETWORK_CIDR by this +# arithmetic; a bug here would silently steer intercepted traffic to a wrong +# gateway while CI stayed green. +[[ "$(sandbox_gateway_ip_from_cidr 192.168.0.0/18)" == "192.168.0.1" ]] +[[ "$(sandbox_gateway_ip_from_cidr 10.0.0.0/8)" == "10.0.0.1" ]] +[[ "$(sandbox_gateway_ip_from_cidr 172.16.0.0/12)" == "172.16.0.1" ]] +[[ "$(sandbox_gateway_ip_from_cidr 192.168.1.0/24)" == "192.168.1.1" ]] +# Non-/8-aligned mask. +[[ "$(sandbox_gateway_ip_from_cidr 10.128.0.0/9)" == "10.128.0.1" ]] +# Host bits set: masked off before +1 (gateway is the network address + 1, not +# the literal address + 1). +[[ "$(sandbox_gateway_ip_from_cidr 192.168.5.7/24)" == "192.168.5.1" ]] +# Invalid CIDRs are rejected (fatal exits the subshell, so a surviving subshell +# means the value was wrongly accepted). +for bad in "10.0.0.0/33" "10.0.0.0/0" "notacidr" "1.2.3.256/24" "10.0.0.0"; do + if ( sandbox_gateway_ip_from_cidr "${bad}" >/dev/null 2>&1 ); then + echo "invalid CIDR ${bad} was accepted" >&2 + exit 1 + fi +done + +printf 'cube-proxy-iptables-init_test: PASS\n' diff --git a/CubeEgress/tests/port_scheme_extra_test.lua b/CubeEgress/tests/port_scheme_extra_test.lua new file mode 100644 index 000000000..0e78cdf9a --- /dev/null +++ b/CubeEgress/tests/port_scheme_extra_test.lua @@ -0,0 +1,81 @@ +-- Complementary tests for CubeEgress custom-port semantics. +-- Focuses on port_scheme.expand (the branch table behind every +-- rule match) which port_scheme_test.lua exercises only indirectly +-- via port_scheme.matches. Run: lua tests/port_scheme_extra_test.lua +package.path = "lua/?.lua;" .. package.path + +local port_scheme = require "port_scheme" + +local function assert_true(value, message) + if not value then error(message or "expected true") end +end + +local function assert_false(value, message) + if value then error(message or "expected false") end +end + +-- Find a tuple by port in an expand() result. +local function tuple_for(tuples, port) + if not tuples then return nil end + for _, t in ipairs(tuples) do + if t.port == port then return t end + end + return nil +end + +-- Default rule (no port/scheme) -> {80/http, 443/https}. +local def, err = port_scheme.expand(nil, nil) +assert_false(err, "expand(nil, nil) should succeed") +assert_true(#def == 2, "default should yield 2 tuples, got " .. #def) +assert_true(tuple_for(def, 80) and tuple_for(def, 80).scheme == "http", "default missing 80/http") +assert_true(tuple_for(def, 443) and tuple_for(def, 443).scheme == "https", "default missing 443/https") + +-- scheme-only http -> {80/http}. +local only_http = port_scheme.expand(nil, "http") +assert_true(#only_http == 1 and only_http[1].port == 80 and only_http[1].scheme == "http", + "scheme=http should yield exactly {80/http}") + +-- scheme-only https -> {443/https}. +local only_https = port_scheme.expand(nil, "https") +assert_true(#only_https == 1 and only_https[1].port == 443 and only_https[1].scheme == "https", + "scheme=https should yield exactly {443/https}") + +-- explicit custom port + scheme -> exact single tuple. +local custom = port_scheme.expand(8080, "http") +assert_true(#custom == 1 and custom[1].port == 8080 and custom[1].scheme == "http", + "port=8080+scheme=http should yield exactly {8080/http}") +local custom_https = port_scheme.expand(8443, "https") +assert_true(#custom_https == 1 and custom_https[1].port == 8443 and custom_https[1].scheme == "https", + "port=8443+scheme=https should yield exactly {8443/https}") + +-- port without scheme is invalid. +local bad, bad_err = port_scheme.expand(8080, nil) +assert_false(bad, "port without scheme must be rejected") +assert_true(bad_err and string.find(bad_err, "port requires scheme", 1, true) ~= nil, + "port-without-scheme error must mention 'port requires scheme'") + +-- unknown scheme is invalid. +local bad_scheme, bs_err = port_scheme.expand(nil, "ftp") +assert_false(bad_scheme, "scheme=ftp must be rejected") +assert_true(bs_err and string.find(bs_err, "scheme must be http or https", 1, true) ~= nil, + "unknown scheme error must mention allowed values") + +-- out-of-range ports are invalid. +local zero, z_err = port_scheme.expand(0, "http") +assert_false(zero, "port=0 must be rejected") +local too_big, tb_err = port_scheme.expand(70000, "http") +assert_false(too_big, "port=70000 must be rejected") + +-- scheme normalization is case/whitespace insensitive. +local upper = port_scheme.expand(nil, "HTTPS") +assert_true(#upper == 1 and upper[1].port == 443 and upper[1].scheme == "https", + "scheme='HTTPS' should normalize to https/443") +local spaced = port_scheme.expand(nil, " http ") +assert_true(#spaced == 1 and spaced[1].port == 80 and spaced[1].scheme == "http", + "scheme=' http ' should normalize to http/80") + +-- matches() honors case-insensitive scheme on the request too. +assert_true(port_scheme.matches(nil, "HTTPS", 443, "HTTPS"), + "matches must accept case-insensitive scheme match") + +print("port_scheme_extra_test: PASS") diff --git a/CubeEgress/tests/port_scheme_test.lua b/CubeEgress/tests/port_scheme_test.lua new file mode 100644 index 000000000..da9c10857 --- /dev/null +++ b/CubeEgress/tests/port_scheme_test.lua @@ -0,0 +1,147 @@ +package.path = "lua/?.lua;" .. package.path + +package.preload["cjson.safe"] = function() + return {encode = function() return "{}" end, decode = function() return {} end} +end +package.preload["resty.openssl.digest"] = function() + return {new = function() return nil, "not used by validation tests" end} +end + +local port_scheme = require "port_scheme" +local policy = require "policy" +local access = require "access_phase" + +local function assert_true(value, message) + if not value then error(message or "expected true") end +end + +local function assert_false(value, message) + if value then error(message or "expected false") end +end + +local function rule(id, match) + return {id = id, match = match, action = {allow = true}} +end + +assert_true(port_scheme.matches(nil, nil, 80, "http")) +assert_true(port_scheme.matches(nil, nil, 443, "https")) +assert_false(port_scheme.matches(nil, nil, 8080, "http")) +assert_true(port_scheme.matches(nil, "http", 80, "http")) +assert_false(port_scheme.matches(nil, "http", 8080, "http")) +assert_true(port_scheme.matches(8443, "https", 8443, "https")) +assert_false(port_scheme.matches(8443, "https", 8443, "http")) +assert_false(port_scheme.matches(8443, nil, 8443, "https")) + +-- matches_deny: a port-less deny is port-agnostic within the host, so a +-- custom-port flow is still caught (fail-closed). +assert_true(port_scheme.matches_deny(nil, nil, 8443, "https")) +assert_true(port_scheme.matches_deny(nil, nil, 80, "http")) +assert_true(port_scheme.matches_deny(nil, "https", 8443, "https")) +assert_false(port_scheme.matches_deny(nil, "https", 8080, "http")) +assert_true(port_scheme.matches_deny(8443, "https", 8443, "https")) +assert_false(port_scheme.matches_deny(8443, "https", 443, "https")) + +-- Regression: a broad host rule must not shadow the custom-port rule. +-- (Third arg is_allow=true: an allow rule with no port narrows to the default +-- set, so it does NOT match the custom-port flow.) +local ctx = { + host = "api.example.com", sni = "api.example.com", method = "GET", + path = "/", scheme = "https", dst_port = 8443, +} +assert_false(access._rule_matches({host = "api.example.com"}, ctx, true)) +assert_true(access._rule_matches({host = "api.example.com", port = 8443, scheme = "https"}, ctx, true)) + +-- Regression: a port-less DENY rule is port-agnostic within the host +-- (fail-closed), so a custom-port allow cannot bypass a broader host deny. +-- Same ctx: dst_port=8443, scheme=https, host=api.example.com. +assert_true(access._rule_matches({host = "api.example.com"}, ctx, false), + "port-less deny should match the custom-port flow") +assert_true(access._rule_matches({host = "*.example.com"}, ctx, false), + "wildcard port-less deny should match the custom-port flow") +-- Scheme-only deny matches any intercepted port of that scheme... +assert_true(access._rule_matches({host = "api.example.com", scheme = "https"}, ctx, false), + "scheme-only https deny should match :8443 https") +-- ...but not the other scheme. +local http_ctx = { + host = "api.example.com", sni = "api.example.com", method = "GET", + path = "/", scheme = "http", dst_port = 8080, +} +assert_false(access._rule_matches({host = "api.example.com", scheme = "https"}, http_ctx, false), + "scheme-only https deny should not match http flow") +-- An explicit port+scheme deny still matches exactly that tuple. +assert_true(access._rule_matches({host = "api.example.com", port = 8443, scheme = "https"}, ctx, false)) +assert_false(access._rule_matches({host = "api.example.com", port = 8443, scheme = "https"}, http_ctx, false), + "explicit-port deny should not match a different port/scheme") + +local function validate(rules) + return policy.validate_policy({policy_id = "sandbox-1", rules = rules}) +end + +local ok, err = validate({rule("port-only", {host = "api.example.com", port = 8080})}) +assert_false(ok, "port-only policy unexpectedly valid") +assert_true(string.find(err, "port requires scheme", 1, true) ~= nil) + +ok = validate({ + rule("default", {host = "api.example.com"}), + rule("conflict", {host = "api.example.com", port = 443, scheme = "http"}), +}) +assert_false(ok, "default/explicit scheme conflict unexpectedly valid") + +local many = {} +for i = 1, 9 do + many[i] = rule("r" .. i, {host = "api.example.com", port = 8000 + i, scheme = "http"}) +end +ok = validate(many) +assert_false(ok, "nine tuples unexpectedly valid") + +ok, err = validate({ + rule("host-a", {host = "a.example.com", port = 8443, scheme = "http"}), + rule("host-b", {host = "b.example.com", port = 8443, scheme = "https"}), +}) +assert_true(ok, err) + +-- normalize_identity: bare IP and /32 spelling aggregate under one identity, +-- so a per-(host,port) scheme conflict across the two spellings is caught +-- (Go l7GroupKey groups both via parseCIDR). +ok, err = validate({ + rule("ip-bare", {host = "1.2.3.4", port = 8443, scheme = "https"}), + rule("ip-cidr32", {host = "1.2.3.4/32", port = 8443, scheme = "http"}), +}) +assert_false(ok, "bare-IP vs /32 scheme conflict unexpectedly valid") +assert_true(string.find(err, "conflicts", 1, true) ~= nil) + +-- Same tuple under both spellings is a no-op duplicate, not a conflict. +ok, err = validate({ + rule("ip-bare", {host = "1.2.3.4", port = 8443, scheme = "https"}), + rule("ip-cidr32", {host = "1.2.3.4/32", port = 8443, scheme = "https"}), +}) +assert_true(ok, err) + +-- Subnet CIDRs are rejected as L7 hosts (mirrors Go classifyL7Target). +ok, err = validate({rule("subnet", {host = "1.2.3.0/24", port = 8443, scheme = "https"})}) +assert_false(ok, "subnet CIDR host unexpectedly valid") +assert_true(string.find(err, "subnet CIDR", 1, true) ~= nil) + +ok = validate({rule("bad-prefix", {host = "1.2.3.4/33", port = 8443, scheme = "https"})}) +assert_false(ok, "invalid CIDR prefix unexpectedly valid") + +ok = validate({rule("leading-zero", {host = "01.2.3.4", port = 8443, scheme = "https"})}) +assert_false(ok, "leading-zero IPv4 unexpectedly valid") + +ok = validate({rule("sni-subnet", {sni = "10.0.0.0/8", port = 443, scheme = "https"})}) +assert_false(ok, "subnet CIDR sni unexpectedly valid") + +-- Whitespace-padded host is rejected, not trimmed (request-time matching +-- compares the raw value, so trimming would accept a dead rule). +ok, err = validate({rule("padded", {host = " example.com ", port = 8443, scheme = "https"})}) +assert_false(ok, "whitespace-padded host unexpectedly valid") +assert_true(string.find(err, "whitespace", 1, true) ~= nil) + +-- Case + trailing-dot canonicalization still groups DNS names. +ok = validate({ + rule("dns-upper", {host = "API.example.com", port = 8443, scheme = "https"}), + rule("dns-dot", {host = "api.example.com.", port = 8443, scheme = "http"}), +}) +assert_false(ok, "case/dot DNS scheme conflict unexpectedly valid") + +print("port_scheme_test: PASS") diff --git a/CubeMaster/api/services/cubebox/v1/cubebox.pb.go b/CubeMaster/api/services/cubebox/v1/cubebox.pb.go index 12e239418..77413d739 100644 --- a/CubeMaster/api/services/cubebox/v1/cubebox.pb.go +++ b/CubeMaster/api/services/cubebox/v1/cubebox.pb.go @@ -3152,12 +3152,15 @@ func (x *EgressRule) GetAction() *EgressRuleAction { } type EgressRuleMatch struct { - state protoimpl.MessageState `protogen:"open.v1"` - Sni *string `protobuf:"bytes,1,opt,name=sni,proto3,oneof" json:"sni,omitempty"` - Host *string `protobuf:"bytes,3,opt,name=host,proto3,oneof" json:"host,omitempty"` - Method []string `protobuf:"bytes,4,rep,name=method,proto3" json:"method,omitempty"` - Path *string `protobuf:"bytes,5,opt,name=path,proto3,oneof" json:"path,omitempty"` - Scheme *string `protobuf:"bytes,7,opt,name=scheme,proto3,oneof" json:"scheme,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Sni *string `protobuf:"bytes,1,opt,name=sni,proto3,oneof" json:"sni,omitempty"` + Host *string `protobuf:"bytes,3,opt,name=host,proto3,oneof" json:"host,omitempty"` + Method []string `protobuf:"bytes,4,rep,name=method,proto3" json:"method,omitempty"` + Path *string `protobuf:"bytes,5,opt,name=path,proto3,oneof" json:"path,omitempty"` + Scheme *string `protobuf:"bytes,7,opt,name=scheme,proto3,oneof" json:"scheme,omitempty"` + // L7 destination port. When set, `scheme` MUST also be set — together they + // pin the (host, port, scheme) tuple CubeEgress intercepts. + Port *int32 `protobuf:"varint,8,opt,name=port,proto3,oneof" json:"port,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -3227,6 +3230,13 @@ func (x *EgressRuleMatch) GetScheme() string { return "" } +func (x *EgressRuleMatch) GetPort() int32 { + if x != nil && x.Port != nil { + return *x.Port + } + return 0 +} + type EgressRuleAction struct { state protoimpl.MessageState `protogen:"open.v1"` Allow bool `protobuf:"varint,1,opt,name=allow,proto3" json:"allow,omitempty"` @@ -6792,17 +6802,19 @@ const file_api_services_cubebox_v1_cubebox_proto_rawDesc = "" + "\x05match\x18\x02 \x01(\v2,.cubelet.services.cubebox.v1.EgressRuleMatchH\x00R\x05match\x88\x01\x01\x12J\n" + "\x06action\x18\x03 \x01(\v2-.cubelet.services.cubebox.v1.EgressRuleActionH\x01R\x06action\x88\x01\x01B\b\n" + "\x06_matchB\t\n" + - "\a_action\"\xb4\x01\n" + + "\a_action\"\xd6\x01\n" + "\x0fEgressRuleMatch\x12\x15\n" + "\x03sni\x18\x01 \x01(\tH\x00R\x03sni\x88\x01\x01\x12\x17\n" + "\x04host\x18\x03 \x01(\tH\x01R\x04host\x88\x01\x01\x12\x16\n" + "\x06method\x18\x04 \x03(\tR\x06method\x12\x17\n" + "\x04path\x18\x05 \x01(\tH\x02R\x04path\x88\x01\x01\x12\x1b\n" + - "\x06scheme\x18\a \x01(\tH\x03R\x06scheme\x88\x01\x01B\x06\n" + + "\x06scheme\x18\a \x01(\tH\x03R\x06scheme\x88\x01\x01\x12\x17\n" + + "\x04port\x18\b \x01(\x05H\x04R\x04port\x88\x01\x01B\x06\n" + "\x04_sniB\a\n" + "\x05_hostB\a\n" + "\x05_pathB\t\n" + - "\a_scheme\"\x94\x01\n" + + "\a_schemeB\a\n" + + "\x05_port\"\x94\x01\n" + "\x10EgressRuleAction\x12\x14\n" + "\x05allow\x18\x01 \x01(\bR\x05allow\x12\x19\n" + "\x05audit\x18\x02 \x01(\tH\x00R\x05audit\x88\x01\x01\x12E\n" + diff --git a/CubeMaster/api/services/cubebox/v1/cubebox.proto b/CubeMaster/api/services/cubebox/v1/cubebox.proto index ae3599adc..13ba0279f 100644 --- a/CubeMaster/api/services/cubebox/v1/cubebox.proto +++ b/CubeMaster/api/services/cubebox/v1/cubebox.proto @@ -620,6 +620,9 @@ message EgressRuleMatch { repeated string method = 4; optional string path = 5; optional string scheme = 7; + // L7 destination port. When set, `scheme` MUST also be set — together they + // pin the (host, port, scheme) tuple CubeEgress intercepts. + optional int32 port = 8; } message EgressRuleAction { diff --git a/CubeMaster/cmd/cubemastercli/commands/cubebox/template_test.go b/CubeMaster/cmd/cubemastercli/commands/cubebox/template_test.go index e8c377acc..62be7737a 100644 --- a/CubeMaster/cmd/cubemastercli/commands/cubebox/template_test.go +++ b/CubeMaster/cmd/cubemastercli/commands/cubebox/template_test.go @@ -344,6 +344,72 @@ func TestMergeCubeNetworkConfigValuesPreservesExistingCIDRs(t *testing.T) { } } +func TestMergeCubeNetworkConfigValuesPreservesRulesAndAllowPublicTraffic(t *testing.T) { + host := "api.internal.example.com" + scheme := "https" + port := 8443 + audit := "full" + format := "Bearer ${SECRET}" + allowPublic := false + existing := &types.CubeNetworkConfig{ + AllowPublicTraffic: &allowPublic, + Rules: []*types.EgressRule{ + { + Name: "api-8443-https", + Match: &types.EgressRuleMatch{ + Host: &host, + Scheme: &scheme, + Port: &port, + }, + Action: &types.EgressRuleAction{ + Allow: true, + Audit: &audit, + Inject: []*types.EgressRuleInject{ + {Header: "Authorization", Secret: "s3cret", Format: &format}, + }, + }, + }, + }, + } + + // Any --allow-* flag triggers the merge path, which clones `existing`. + got := mergeCubeNetworkConfigValues(existing, false, false, []string{"10.0.0.0/8"}, nil) + if got == nil { + t.Fatal("got nil merged config") + } + if got.AllowPublicTraffic == nil || *got.AllowPublicTraffic != false { + t.Fatalf("AllowPublicTraffic=%v, want pointer to false (template value preserved)", got.AllowPublicTraffic) + } + if len(got.Rules) != 1 { + t.Fatalf("Rules=%v, want the template rule preserved", got.Rules) + } + rule := got.Rules[0] + if rule.Name != "api-8443-https" || rule.Match == nil || rule.Match.Port == nil || *rule.Match.Port != 8443 { + t.Fatalf("rule=%+v, want port-pinned rule preserved", rule) + } + if rule.Action == nil || len(rule.Action.Inject) != 1 || rule.Action.Inject[0].Header != "Authorization" { + t.Fatalf("action=%+v, want inject preserved", rule.Action) + } + + // The clone must be deep: mutating the merged copy must not touch the template. + *got.AllowPublicTraffic = true + *rule.Match.Port = 443 + *rule.Match.Host = "mutated.example.com" + *rule.Action.Inject[0].Format = "mutated" + if *existing.AllowPublicTraffic != false { + t.Fatal("mutation of merged AllowPublicTraffic leaked into template") + } + if *existing.Rules[0].Match.Port != 8443 { + t.Fatal("mutation of merged Match.Port leaked into template") + } + if *existing.Rules[0].Match.Host != "api.internal.example.com" { + t.Fatal("mutation of merged Match.Host leaked into template") + } + if *existing.Rules[0].Action.Inject[0].Format != "Bearer ${SECRET}" { + t.Fatal("mutation of merged Inject.Format leaked into template") + } +} + func TestRedoCommandParsesNodeScope(t *testing.T) { ctx := newRedoContext(t, []string{ "--template-id", "tpl-1", diff --git a/CubeMaster/pkg/service/sandbox/types/types.go b/CubeMaster/pkg/service/sandbox/types/types.go index 6b37d1d02..7658c379f 100644 --- a/CubeMaster/pkg/service/sandbox/types/types.go +++ b/CubeMaster/pkg/service/sandbox/types/types.go @@ -157,12 +157,21 @@ type EgressRule struct { // EgressRuleMatch holds the per-request match conditions for an EgressRule. // All fields are optional; an empty match matches any request. +// +// Port + Scheme together select which TCP port CubeEgress intercepts: +// - both nil: legacy behavior — CubeEgress captures the default {80/http, +// 443/https} pair. +// - both set: CubeEgress captures the specific (host, port) tuple, routing +// via skb->mark to the HTTP (scheme="http") or HTTPS (scheme="https") +// TPROXY listener. Every rule sharing the same (host, port) MUST agree on +// scheme; the server rejects the whole policy if it detects a mismatch. type EgressRuleMatch struct { SNI *string `json:"sni,omitempty"` Host *string `json:"host,omitempty"` Method []string `json:"method,omitempty"` Path *string `json:"path,omitempty"` Scheme *string `json:"scheme,omitempty"` + Port *int `json:"port,omitempty"` } // EgressRuleAction holds the action taken when an EgressRule matches. @@ -215,6 +224,7 @@ func (r *EgressRule) DeepCopy() *EgressRule { Method: append([]string(nil), r.Match.Method...), Path: cloneStringPtr(r.Match.Path), Scheme: cloneStringPtr(r.Match.Scheme), + Port: cloneIntPtr(r.Match.Port), } } if r.Action != nil { @@ -255,6 +265,14 @@ func cloneStringPtr(value *string) *string { return &cloned } +func cloneIntPtr(value *int) *int { + if value == nil { + return nil + } + cloned := *value + return &cloned +} + type Volume struct { Name string `json:"name,omitempty"` diff --git a/CubeMaster/pkg/service/sandbox/util.go b/CubeMaster/pkg/service/sandbox/util.go index 219226bec..6ac4dd44f 100644 --- a/CubeMaster/pkg/service/sandbox/util.go +++ b/CubeMaster/pkg/service/sandbox/util.go @@ -87,15 +87,51 @@ func checkParam(req *types.CreateCubeSandboxReq) error { return ret.Err(errorcode.ErrorCode_MasterParamsError, "containers param is nil") } - if req.CubeNetworkConfig != nil && req.CubeNetworkConfig.MaskRequestHost != nil { - if err := validateMaskRequestHost(*req.CubeNetworkConfig.MaskRequestHost); err != nil { - return ret.Err(errorcode.ErrorCode_MasterParamsError, err.Error()) + if req.CubeNetworkConfig != nil { + if req.CubeNetworkConfig.MaskRequestHost != nil { + if err := validateMaskRequestHost(*req.CubeNetworkConfig.MaskRequestHost); err != nil { + return ret.Err(errorcode.ErrorCode_MasterParamsError, err.Error()) + } + } + for i, rule := range req.CubeNetworkConfig.Rules { + if rule == nil { + continue + } + if err := validateEgressRuleMatch(rule.Match, i); err != nil { + return ret.Err(errorcode.ErrorCode_MasterParamsError, err.Error()) + } } } return nil } +// validateEgressRuleMatch enforces the port/scheme contract on one egress +// rule match, mirroring the SDK client-side check and the CubeEgress Lua +// validation: a set Port must be in [1, 65535] and must be paired with +// Scheme, and a set Scheme must be http or https (case-insensitive — +// downstream normalizes to lowercase). +func validateEgressRuleMatch(match *types.EgressRuleMatch, index int) error { + if match == nil { + return nil + } + if match.Port != nil { + if *match.Port < 1 || *match.Port > 65535 { + return fmt.Errorf("network.rules[%d].match.port must be in [1, 65535], got %d", index, *match.Port) + } + if match.Scheme == nil { + return fmt.Errorf("network.rules[%d].match.port requires match.scheme to be set", index) + } + } + if match.Scheme != nil { + scheme := strings.ToLower(strings.TrimSpace(*match.Scheme)) + if scheme != "http" && scheme != "https" { + return fmt.Errorf("network.rules[%d].match.scheme must be 'http' or 'https', got %q", index, *match.Scheme) + } + } + return nil +} + func validateMaskRequestHost(value string) error { invalid := func(reason string) error { return fmt.Errorf("network.maskRequestHost is invalid: %s", reason) @@ -317,13 +353,18 @@ func mapEgressRuleMatch(in *types.EgressRuleMatch) *cubebox.EgressRuleMatch { if in == nil { return nil } - return &cubebox.EgressRuleMatch{ + out := &cubebox.EgressRuleMatch{ Sni: in.SNI, Host: in.Host, Method: append([]string(nil), in.Method...), Path: in.Path, Scheme: in.Scheme, } + if in.Port != nil { + p := int32(*in.Port) + out.Port = &p + } + return out } func mapEgressRuleAction(in *types.EgressRuleAction) *cubebox.EgressRuleAction { diff --git a/CubeMaster/pkg/service/sandbox/util_test.go b/CubeMaster/pkg/service/sandbox/util_test.go index 2a5c0a8f5..130c5bc59 100644 --- a/CubeMaster/pkg/service/sandbox/util_test.go +++ b/CubeMaster/pkg/service/sandbox/util_test.go @@ -36,6 +36,49 @@ func ensureSandboxTestConfig(t *testing.T) *config.Config { return cfg } +func TestValidateEgressRuleMatch(t *testing.T) { + strPtr := func(s string) *string { return &s } + intPtr := func(i int) *int { return &i } + + valid := []struct { + name string + match *types.EgressRuleMatch + }{ + {"nil match", nil}, + {"empty match", &types.EgressRuleMatch{}}, + {"scheme only", &types.EgressRuleMatch{Scheme: strPtr("https")}}, + {"scheme case variant", &types.EgressRuleMatch{Scheme: strPtr("HTTPS")}}, + {"scheme with spaces", &types.EgressRuleMatch{Scheme: strPtr(" http ")}}, + {"port with scheme", &types.EgressRuleMatch{Port: intPtr(8443), Scheme: strPtr("https")}}, + {"port boundary low", &types.EgressRuleMatch{Port: intPtr(1), Scheme: strPtr("http")}}, + {"port boundary high", &types.EgressRuleMatch{Port: intPtr(65535), Scheme: strPtr("https")}}, + } + for _, tt := range valid { + t.Run(tt.name, func(t *testing.T) { + assert.NoError(t, validateEgressRuleMatch(tt.match, 0)) + }) + } + + invalid := []struct { + name string + match *types.EgressRuleMatch + wantErr string + }{ + {"port without scheme", &types.EgressRuleMatch{Port: intPtr(8443)}, "requires match.scheme"}, + {"port zero", &types.EgressRuleMatch{Port: intPtr(0), Scheme: strPtr("http")}, "[1, 65535]"}, + {"port negative", &types.EgressRuleMatch{Port: intPtr(-1), Scheme: strPtr("http")}, "[1, 65535]"}, + {"port too large", &types.EgressRuleMatch{Port: intPtr(65536), Scheme: strPtr("http")}, "[1, 65535]"}, + {"scheme not http", &types.EgressRuleMatch{Scheme: strPtr("ftp")}, "must be 'http' or 'https'"}, + } + for _, tt := range invalid { + t.Run(tt.name, func(t *testing.T) { + err := validateEgressRuleMatch(tt.match, 0) + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + func TestValidateMaskRequestHost(t *testing.T) { for _, value := range []string{ "localhost", diff --git a/CubeMaster/pkg/templatecenter/template_request_test.go b/CubeMaster/pkg/templatecenter/template_request_test.go new file mode 100644 index 000000000..a48b8cd5b --- /dev/null +++ b/CubeMaster/pkg/templatecenter/template_request_test.go @@ -0,0 +1,33 @@ +package templatecenter + +import ( + "testing" + + "github.com/tencentcloud/CubeSandbox/CubeMaster/pkg/service/sandbox/types" +) + +func TestCloneEgressRuleDeepCopiesPort(t *testing.T) { + port := 8443 + rule := &types.EgressRule{ + Name: "custom-https", + Match: &types.EgressRuleMatch{ + Port: &port, + }, + } + + cloned := rule.DeepCopy() + if cloned == nil || cloned.Match == nil || cloned.Match.Port == nil { + t.Fatalf("cloned rule lost port: %+v", cloned) + } + if *cloned.Match.Port != port { + t.Fatalf("cloned port=%d, want %d", *cloned.Match.Port, port) + } + if cloned.Match.Port == rule.Match.Port { + t.Fatal("cloned port aliases source pointer") + } + + *cloned.Match.Port = 443 + if *rule.Match.Port != 8443 { + t.Fatalf("source port changed through clone: %d", *rule.Match.Port) + } +} diff --git a/CubeNet/cubevs/.gitignore b/CubeNet/cubevs/.gitignore index 4f4cb290c..715f524f3 100644 --- a/CubeNet/cubevs/.gitignore +++ b/CubeNet/cubevs/.gitignore @@ -1,4 +1,4 @@ .coverage coverage.html -/*_bpfel.go -/*_bpfel.o +/*_bpfel*.go +/*_bpfel*.o diff --git a/CubeNet/cubevs/Makefile b/CubeNet/cubevs/Makefile index 1f89d8c7e..78ad1acf7 100644 --- a/CubeNet/cubevs/Makefile +++ b/CubeNet/cubevs/Makefile @@ -1,8 +1,8 @@ COVERAGE_PROFILE=.coverage COVERAGE_REPORT=coverage.html -BPF_GENS := $(wildcard *_bpfel.go) -BPF_OBJS := $(wildcard *_bpfel.o) +BPF_GENS := $(wildcard *_bpfel*.go) +BPF_OBJS := $(wildcard *_bpfel*.o) ifeq ($(V),1) Q = @@ -38,7 +38,7 @@ lint: $(Q)golangci-lint run --fix .PHONY: test -test: +test: gen $(call msg,RUN TESTS) $(Q)go clean -testcache $(Q)go test $(VERBOSE) -cover -coverprofile=$(COVERAGE_PROFILE) ./... diff --git a/CubeNet/cubevs/cubevs.go b/CubeNet/cubevs/cubevs.go index b131af6ae..8d87a242d 100644 --- a/CubeNet/cubevs/cubevs.go +++ b/CubeNet/cubevs/cubevs.go @@ -37,6 +37,12 @@ type Params struct { NodeMacAddr net.HardwareAddr // MAC address of the Node gateway (next hop) NodeGatewayMacAddr net.HardwareAddr + // L7 skb->mark values stamped by the dataplane and matched by the iptables + // TPROXY rules. Zero means "use the shipped default"; override from the + // install-time config shared with the iptables init script. + L7MarkHTTP uint32 + L7MarkHTTPS uint32 + L7MarkMask uint32 } // TAPDevice contains info about a TAP device. @@ -81,24 +87,64 @@ type lpmKey struct { IP uint32 } -// netPolicyValueV2 mirrors struct net_policy_value_v2 on the BPF side. +// l7PortEntry mirrors struct l7_port_entry on the BPF side. Port is stored in +// network byte order to match tcphdr->dest so the datapath can compare without +// an endianness conversion. Scheme is one of L7SchemeHTTP / L7SchemeHTTPS. +type l7PortEntry struct { + Port uint16 // network byte order + Scheme uint8 + Pad uint8 +} + +// netPolicyValueV2 mirrors struct net_policy_value_v2 on the BPF side. This is +// the legacy 16-byte layout, read only when migrating a pre-v3 allow_out_v2 +// map to allow_out_v3; the current dataplane uses netPolicyValueV3. type netPolicyValueV2 struct { ExpiresAtNS uint64 Flags uint8 Reserved [7]uint8 } +// lpmKeyV3 mirrors struct lpm_key_v3 on the BPF side. IP and port are +// in network byte order; a single longest-prefix lookup resolves exact +// (ip, port) (prefixlen 48), ip-only (prefixlen 32), or ip/mask +// (prefixlen < 32) rules. Pad keeps the LPM data payload 4-byte aligned. +type lpmKeyV3 struct { + Prefixlen uint32 + IP uint32 + Port uint16 + Pad uint16 +} + +// netPolicyValueV3 mirrors struct net_policy_value_v3 on the BPF side. +// Unlike netPolicyValueV2, the port lives in the key, so the scheme is +// resolved at insert time and stored directly here. KeyPrefixlen records +// the prefixlen of the key this value was written under: LPM lookups are +// longest-prefix, so writers merging with an existing entry for the EXACT +// same key must compare it against their key's prefixlen first. +type netPolicyValueV3 struct { + ExpiresAtNS uint64 + Flags uint8 + Scheme uint8 + KeyPrefixlen uint8 + Reserved [5]uint8 +} + // dnsAllowKey mirrors struct dns_allow_key on the BPF side. type dnsAllowKey struct { Prefixlen uint32 Name [maxDNSNameLen]byte } -// dnsAllowValue mirrors struct dns_allow_value on the BPF side. +// dnsAllowValue mirrors struct dns_allow_value on the BPF side. Ports carries +// the (port, scheme) tuples the userspace built from all rules sharing this +// host. PortCount == 0 means "unspecified, default 80/443". type dnsAllowValue struct { - NameLen uint32 - Flags uint8 - Reserved [3]uint8 + NameLen uint32 + Flags uint8 + PortCount uint8 + Reserved [2]uint8 + Ports [maxL7PortsPerHost]l7PortEntry } // dnsQueryTrackKey mirrors struct dns_query_track_key on the BPF side. @@ -112,10 +158,14 @@ type dnsQueryTrackKey struct { } // dnsQueryTrackValue mirrors struct dns_query_track_value on the BPF side. +// Ports is copied from the matched dns_allow_value at query time so the +// response handler can rebuild net_policy_value_v3 without a second lookup. type dnsQueryTrackValue struct { ExpiresAtNS uint64 Flags uint8 - Reserved [7]uint8 + PortCount uint8 + Reserved [6]uint8 + Ports [maxL7PortsPerHost]l7PortEntry } const ( @@ -128,6 +178,18 @@ const ( dnsPolicyFlagLearningEnabled = 1 << 0 // Network policy flags. Must match src/cubevs.h. netPolicyFlagL7Required = 1 << 0 + // netPolicyFlagL3Allowed marks a domain present in both plain allow_out + // and an L7 rule, so the datapath learns the plain /32 any-port entry + // alongside the L7 /48 entries. Must match src/cubevs.h. + netPolicyFlagL3Allowed = 1 << 1 + // L7 scheme values in dns_allow_value / net_policy_value_v3 per-port + // entries. Must match L7_SCHEME_* in src/cubevs.h. + L7SchemeNone uint8 = 0 + L7SchemeHTTP uint8 = 1 + L7SchemeHTTPS uint8 = 2 + // Maximum number of (port, scheme) tuples per host. Must match + // MAX_L7_PORTS_PER_HOST in src/cubevs.h. + maxL7PortsPerHost = 8 // Network policy value marker. Must match src/cubevs.h. netPolicyValueStatic = 1 // programs that power CubeVS. @@ -155,9 +217,11 @@ const ( MapNameLocalPortMapping = "local_port_mapping" // MapNameAllowOut is the cube-v0.2.0 legacy migration source. MapNameAllowOut = "allow_out" - MapNameAllowOutV2 = "allow_out_v2" + MapNameAllowOutV2 = "allow_out_v2" // legacy 16-byte policy value + MapNameAllowOutV3 = "allow_out_v3" // current 16-byte policy value MapNameDenyOut = "deny_out" - MapNameDNSAllow = "dns_allow" + MapNameDNSAllow = "dns_allow" // legacy 8-byte DNS value + MapNameDNSAllowV2 = "dns_allow_v2" // current 40-byte DNS value MapNameDNSQueryTrack = "dns_query_track" // constants referenced by BPF programs. globalNameMVMInnerIP = "mvm_inner_ip" @@ -179,8 +243,9 @@ const ( globalNameNodeMacaddrP2 = "nodenic_macaddr_p2" globalNameNodeGatewayMacaddrP1 = "nodegw_macaddr_p1" globalNameNodeGatewayMacaddrP2 = "nodegw_macaddr_p2" - // for bpffs. - bpfFSPath = "/sys/fs/bpf" + globalNameCubeL7MarkHTTP = "cube_l7_mark_http" + globalNameCubeL7MarkHTTPS = "cube_l7_mark_https" + globalNameCubeL7MarkMask = "cube_l7_mark_mask" // for TC. tcFlagDirectAction = 1 tcFilterHandle = 1 @@ -264,6 +329,24 @@ func _() { _ = arr[size-8] // error if size < 8 } + { + // static assert, make sure LpmKeyV3 is of size 12 + var arr [12]struct{} + var obj lpmKeyV3 + const size = unsafe.Sizeof(obj) + _ = arr[size-1] // error if size > 12 + _ = arr[size-12] // error if size < 12 + } + + { + // static assert, make sure l7PortEntry is of size 4 + var arr [4]struct{} + var obj l7PortEntry + const size = unsafe.Sizeof(obj) + _ = arr[size-1] // error if size > 4 + _ = arr[size-4] // error if size < 4 + } + { // static assert, make sure netPolicyValueV2 is of size 16 var arr [16]struct{} @@ -283,12 +366,12 @@ func _() { } { - // static assert, make sure dnsAllowValue is of size 8 - var arr [8]struct{} + // static assert, make sure dnsAllowValue is of size 40 + var arr [40]struct{} var obj dnsAllowValue const size = unsafe.Sizeof(obj) - _ = arr[size-1] // error if size > 8 - _ = arr[size-8] // error if size < 8 + _ = arr[size-1] // error if size > 40 + _ = arr[size-40] // error if size < 40 } { @@ -301,11 +384,11 @@ func _() { } { - // static assert, make sure dnsQueryTrackValue is of size 16 - var arr [16]struct{} + // static assert, make sure dnsQueryTrackValue is of size 48 + var arr [48]struct{} var obj dnsQueryTrackValue const size = unsafe.Sizeof(obj) - _ = arr[size-1] // error if size > 16 - _ = arr[size-16] // error if size < 16 + _ = arr[size-1] // error if size > 48 + _ = arr[size-48] // error if size < 48 } } diff --git a/CubeNet/cubevs/custom_port_test.go b/CubeNet/cubevs/custom_port_test.go new file mode 100644 index 000000000..9747d412e --- /dev/null +++ b/CubeNet/cubevs/custom_port_test.go @@ -0,0 +1,134 @@ +package cubevs + +import "testing" + +// ------- v3 custom-port encoding (buildV3Entries) ------------------- + +func TestExpandDefaultPortSet(t *testing.T) { + // The default (host, port) set used when an L7 rule omits port. + ports := expandDefaultPortSet() + if got, want := len(ports), 2; got != want { + t.Fatalf("len(expandDefaultPortSet())=%d, want %d", got, want) + } + want := map[uint16]uint8{ + htonsPort(80): L7SchemeHTTP, + htonsPort(443): L7SchemeHTTPS, + } + for _, p := range ports { + scheme, ok := want[p.Port] + if !ok { + t.Fatalf("unexpected default port 0x%04x", p.Port) + } + if p.Scheme != scheme { + t.Fatalf("port 0x%04x scheme=%d, want %d", p.Port, p.Scheme, scheme) + } + } +} + +func TestBuildV3EntriesL7ExplicitPorts(t *testing.T) { + // L7 with explicit (port, scheme) tuples -> one /48 entry per tuple, + // each carrying scheme + expiry, ip in network byte order. + const ip uint32 = 0x01020304 + const expires uint64 = 1234 + entries := buildV3Entries(lpmKey{Prefixlen: 32, IP: ip}, netPolicyFlagL7Required, + []l7PortEntry{ + {Port: htonsPort(8080), Scheme: L7SchemeHTTP}, + {Port: htonsPort(8443), Scheme: L7SchemeHTTPS}, + }, expires) + if got, want := len(entries), 2; got != want { + t.Fatalf("len(entries)=%d, want %d", got, want) + } + got := map[uint16]uint8{} + for _, e := range entries { + if e.key.Prefixlen != 48 { + t.Fatalf("key.Prefixlen=%d, want 48 (exact ip+port)", e.key.Prefixlen) + } + if e.key.IP != ip { + t.Fatalf("key.IP=0x%08x, want 0x%08x", e.key.IP, ip) + } + if e.value.Flags != netPolicyFlagL7Required { + t.Fatalf("value.Flags=%d, want %d", e.value.Flags, netPolicyFlagL7Required) + } + if e.value.ExpiresAtNS != expires { + t.Fatalf("value.ExpiresAtNS=%d, want %d", e.value.ExpiresAtNS, expires) + } + got[e.key.Port] = e.value.Scheme + } + if got[htonsPort(8080)] != L7SchemeHTTP { + t.Fatalf("port 8080 scheme=%d, want HTTP", got[htonsPort(8080)]) + } + if got[htonsPort(8443)] != L7SchemeHTTPS { + t.Fatalf("port 8443 scheme=%d, want HTTPS", got[htonsPort(8443)]) + } +} + +func TestBuildV3EntriesL7DefaultExpansion(t *testing.T) { + // L7 with no port set -> default {80/http, 443/https} expansion, + // both as /48 exact entries. + entries := buildV3Entries(lpmKey{Prefixlen: 32, IP: 0x01020304}, netPolicyFlagL7Required, nil, 0) + if got, want := len(entries), 2; got != want { + t.Fatalf("len(entries)=%d, want 2 (default {80,443})", got) + } + for _, e := range entries { + if e.key.Prefixlen != 48 { + t.Fatalf("key.Prefixlen=%d, want 48", e.key.Prefixlen) + } + } +} + +func TestBuildV3EntriesPlainAllow(t *testing.T) { + // Non-L7 allow -> single ip-only /32 entry, port=0, scheme=NONE. + entries := buildV3Entries(lpmKey{Prefixlen: 32, IP: 0x01020304}, 0, nil, 0) + if got, want := len(entries), 1; got != want { + t.Fatalf("len(entries)=%d, want 1", got) + } + e := entries[0] + if e.key.Prefixlen != 32 { + t.Fatalf("key.Prefixlen=%d, want 32 (ip-only)", e.key.Prefixlen) + } + if e.key.Port != 0 { + t.Fatalf("key.Port=%d, want 0 (non-L7)", e.key.Port) + } + if e.value.Scheme != L7SchemeNone { + t.Fatalf("value.Scheme=%d, want NONE", e.value.Scheme) + } +} + +func TestBuildV3EntriesSubnet(t *testing.T) { + // Non-L7 subnet -> single entry at the source prefixlen (< 32). + entries := buildV3Entries(lpmKey{Prefixlen: 24, IP: 0x01020304}, 0, nil, 0) + if got, want := len(entries), 1; got != want { + t.Fatalf("len(entries)=%d, want 1", got) + } + if entries[0].key.Prefixlen != 24 { + t.Fatalf("key.Prefixlen=%d, want 24 (subnet)", entries[0].key.Prefixlen) + } + if entries[0].key.Port != 0 { + t.Fatalf("key.Port=%d, want 0", entries[0].key.Port) + } +} + +func TestBuildV3EntriesExpiryCopied(t *testing.T) { + // Expiry is carried verbatim into every expanded entry. + entries := buildV3Entries(lpmKey{Prefixlen: 32, IP: 1}, netPolicyFlagL7Required, + []l7PortEntry{{Port: htonsPort(8080), Scheme: L7SchemeHTTP}}, 999) + for _, e := range entries { + if e.value.ExpiresAtNS != 999 { + t.Fatalf("value.ExpiresAtNS=%d, want 999", e.value.ExpiresAtNS) + } + } +} + +func TestLpmKeyV3NetworkByteOrder(t *testing.T) { + // A custom port must be stored in network byte order in the LPM key, + // so the datapath (which compares against tcphdr->dest) matches. + const port uint16 = 0x1234 + entries := buildV3Entries(lpmKey{IP: 1}, netPolicyFlagL7Required, + []l7PortEntry{{Port: htonsPort(port), Scheme: L7SchemeHTTP}}, 0) + if len(entries) != 1 { + t.Fatalf("len(entries)=%d, want 1", len(entries)) + } + if got := entries[0].key.Port; got != htonsPort(port) { + t.Fatalf("key.Port=0x%04x, want network byte order 0x%04x", got, htonsPort(port)) + } +} diff --git a/CubeNet/cubevs/dns_learn_test.go b/CubeNet/cubevs/dns_learn_test.go new file mode 100644 index 000000000..a7796dba7 --- /dev/null +++ b/CubeNet/cubevs/dns_learn_test.go @@ -0,0 +1,418 @@ +package cubevs + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target $GOARCH dnslearn ../src/dns_learn_test.bpf.c -- -I../vmlinux/$GOARCH + +import ( + "encoding/binary" + "testing" + + "github.com/cilium/ebpf" +) + +const dnsLearnTestCaseLen = 12 + +type dnsLearnTestEnv struct { + program *ebpf.Program + allowOut *ebpf.Map + queryStore *ebpf.Map + allowInnerSpec *ebpf.MapSpec +} + +func loadDNSLearnTestEnv(t *testing.T) *dnsLearnTestEnv { + t.Helper() + + spec, err := loadDnslearn() + if err != nil { + t.Fatalf("load dns learn test spec: %v", err) + } + allowSpec := spec.Maps["allow_out_v3"] + if allowSpec == nil || allowSpec.InnerMap == nil { + t.Fatal("allow_out_v3 spec or inner template missing") + } + allowInnerSpec := allowSpec.InnerMap.Copy() + + for name, mapSpec := range spec.Maps { + switch name { + case ".rodata", "allow_out_v3", "test_query_store": + mapSpec.Pinning = ebpf.PinNone + default: + delete(spec.Maps, name) + } + } + + coll, err := ebpf.NewCollection(spec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF dns learn test unavailable: %v", err) + } + t.Fatalf("load dns learn test collection: %v", err) + } + t.Cleanup(coll.Close) + + env := &dnsLearnTestEnv{ + program: coll.Programs["test_dns_learn"], + allowOut: coll.Maps["allow_out_v3"], + queryStore: coll.Maps["test_query_store"], + allowInnerSpec: allowInnerSpec, + } + if env.program == nil || env.allowOut == nil || env.queryStore == nil { + t.Fatal("loaded dns learn program or maps missing") + } + return env +} + +// runDNSLearn drives dns_learn_response_ip with the given query against the +// sandbox's allow_out_v3 inner map, returning that inner map for assertions. +// seeds (if any) are written into the fresh inner map BEFORE the program +// runs, simulating pre-existing static/learned entries. +func runDNSLearn(t *testing.T, env *dnsLearnTestEnv, ifindex uint32, ip uint32, + ttl uint32, query dnsQueryTrackValue, seeds ...allowOutV3Entry, +) *ebpf.Map { + t.Helper() + + innerSpec := env.allowInnerSpec.Copy() + innerSpec.Name = "allow_dns_learn" + innerSpec.Pinning = ebpf.PinNone + inner, err := ebpf.NewMap(innerSpec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF LPM trie unavailable: %v", err) + } + t.Fatalf("create allow inner: %v", err) + } + t.Cleanup(func() { _ = inner.Close() }) + if err := env.allowOut.Put(&ifindex, inner); err != nil { + t.Fatalf("attach allow inner map: %v", err) + } + + for i, s := range seeds { + if err := inner.Update(&s.key, &s.value, ebpf.UpdateAny); err != nil { + t.Fatalf("seed allow inner entry %d: %v", i, err) + } + } + + qkey := uint32(0) + if err := env.queryStore.Put(&qkey, &query); err != nil { + t.Fatalf("seed query store: %v", err) + } + + // Pad the packet to 16 bytes: the skb test-run path rejects very small + // buffers (a 12-byte packet fails with EINVAL), while the program only + // reads the leading dns_learn_case struct. + data := make([]byte, 16) + binary.LittleEndian.PutUint32(data[0:4], ifindex) + binary.LittleEndian.PutUint32(data[4:8], ip) + binary.LittleEndian.PutUint32(data[8:12], ttl) + ret, _, err := env.program.Test(data) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF dns learn test-run unavailable: %v", err) + } + t.Fatalf("run dns learn test: %v", err) + } + if ret != 0 { + t.Fatalf("test_dns_learn returned %d, want TC_ACT_OK", ret) + } + return inner +} + +func lookupAllowV3(t *testing.T, inner *ebpf.Map, key lpmKeyV3) (netPolicyValueV3, bool) { + t.Helper() + var value netPolicyValueV3 + if err := inner.Lookup(&key, &value); err != nil { + return netPolicyValueV3{}, false + } + return value, true +} + +// TestDNSLearnPlainAllowWritesIPOnlyEntry is the B3 regression test: a plain +// (non-L7) domain allow rule must still be learned as a /32 (any-port) entry, +// exactly as the pre-v3 dataplane did. The regression wrote nothing, so the +// resolved IP was rejected by classify_egress_flow under default-deny. +func TestDNSLearnPlainAllowWritesIPOnlyEntry(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(300) + ip := mustParseCIDRForTest(t, "192.0.2.50").IP + + // Plain (non-L7) allow: flags=0, port_count=0. + inner := runDNSLearn(t, env, ifindex, ip, 300, dnsQueryTrackValue{Flags: 0, PortCount: 0}) + + value, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0}) + if !ok { + t.Fatal("plain allow did not learn a /32 entry (B3 regression)") + } + if value.Flags&uint8(netPolicyFlagL7Required) != 0 { + t.Fatalf("plain allow /32 has unexpected L7 flag: %#x", value.Flags) + } + if value.Scheme != L7SchemeNone { + t.Fatalf("plain allow /32 scheme=%d, want L7SchemeNone", value.Scheme) + } + if value.ExpiresAtNS == 0 { + t.Fatal("plain allow /32 has zero expiry, want temporary (DNS TTL)") + } +} + +// TestDNSLearnL7DefaultPortSet covers the L7 path with no explicit ports: +// it must learn the default {80/http, 443/https} set as /48 entries. +func TestDNSLearnL7DefaultPortSet(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(301) + ip := mustParseCIDRForTest(t, "192.0.2.60").IP + + query := dnsQueryTrackValue{Flags: uint8(netPolicyFlagL7Required), PortCount: 0} + inner := runDNSLearn(t, env, ifindex, ip, 300, query) + + for _, tc := range []struct { + port uint16 + scheme uint8 + }{ + {htonsPort(80), L7SchemeHTTP}, + {htonsPort(443), L7SchemeHTTPS}, + } { + value, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: tc.port}) + if !ok { + t.Fatalf("L7 default missing /48 entry for port %d", ntohsPort(tc.port)) + } + if value.Scheme != tc.scheme { + t.Fatalf("port %d scheme=%d, want %d", ntohsPort(tc.port), value.Scheme, tc.scheme) + } + if value.Flags&uint8(netPolicyFlagL7Required) == 0 { + t.Fatalf("port %d missing L7 flag", ntohsPort(tc.port)) + } + } +} + +// TestDNSLearnL7ExplicitPorts covers the L7 path with explicit ports: only the +// declared (port, scheme) tuples are learned, no others. +func TestDNSLearnL7ExplicitPorts(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(302) + ip := mustParseCIDRForTest(t, "192.0.2.70").IP + + query := dnsQueryTrackValue{ + Flags: uint8(netPolicyFlagL7Required), + PortCount: 2, + Ports: [maxL7PortsPerHost]l7PortEntry{ + {Port: htonsPort(8443), Scheme: L7SchemeHTTPS}, + {Port: htonsPort(8080), Scheme: L7SchemeHTTP}, + }, + } + inner := runDNSLearn(t, env, ifindex, ip, 300, query) + + if _, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(8443)}); !ok { + t.Fatal("missing /48 entry for explicit port 8443") + } + if _, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(8080)}); !ok { + t.Fatal("missing /48 entry for explicit port 8080") + } + if _, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(80)}); ok { + t.Fatal("unexpected /48 entry for non-configured port 80") + } +} + +// TestDNSLearnL7WithL3AlsoWritesPlainAndL7Entries covers the coexistence path: +// a domain present in both plain allow_out and an L7 rule must learn BOTH the +// /32 any-port entry (plain SNAT for non-rule ports) AND the /48 L7 entry +// (interception for the rule's port). Previously the L7 flag subsumed the plain +// allow, so only the rule's port was admitted and the domain lost plain L3 +// access on every other port. +func TestDNSLearnL7WithL3AlsoWritesPlainAndL7Entries(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(304) + ip := mustParseCIDRForTest(t, "192.0.2.80").IP + + query := dnsQueryTrackValue{ + Flags: uint8(netPolicyFlagL7Required) | uint8(netPolicyFlagL3Allowed), + PortCount: 1, + Ports: [maxL7PortsPerHost]l7PortEntry{ + {Port: htonsPort(8443), Scheme: L7SchemeHTTPS}, + }, + } + inner := runDNSLearn(t, env, ifindex, ip, 300, query) + + // The /48 L7 entry for the rule's port is present and intercepted. + l7, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(8443)}) + if !ok { + t.Fatal("missing /48 L7 entry for rule port 8443") + } + if l7.Flags&uint8(netPolicyFlagL7Required) == 0 { + t.Fatal("/48 entry missing L7 flag") + } + if l7.Scheme != L7SchemeHTTPS { + t.Fatalf("/48 scheme=%d, want https", l7.Scheme) + } + + // The /32 any-port plain entry is also present for everything else, and it + // must be a plain allow (no L7 / L3 marker bits leaked into the value). + plain, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0}) + if !ok { + t.Fatal("missing /32 plain entry for L3-allowed domain") + } + if plain.Flags&uint8(netPolicyFlagL7Required) != 0 { + t.Fatalf("/32 plain entry has unexpected L7 flag: %#x", plain.Flags) + } + if plain.Flags&uint8(netPolicyFlagL3Allowed) != 0 { + t.Fatalf("/32 plain entry leaked L3_ALLOWED marker: %#x", plain.Flags) + } + if plain.Scheme != L7SchemeNone { + t.Fatalf("/32 plain scheme=%d, want none", plain.Scheme) + } + + // A lookup for a non-rule port must NOT match a /48 L7 entry; it falls back + // via LPM longest-prefix to the /32 plain entry (which is exactly how + // classify_egress_flow admits the flow via plain SNAT). + fallback, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if !ok { + t.Fatal("non-rule port 443 matched nothing, want fallback to the /32 plain entry") + } + if fallback.KeyPrefixlen == 48 && fallback.Flags&uint8(netPolicyFlagL7Required) != 0 { + t.Fatalf("non-rule port 443 unexpectedly matched a /48 L7 entry: %+v", fallback) + } + if fallback.KeyPrefixlen != 32 { + t.Fatalf("non-rule port 443 fell back to key_prefixlen=%d, want 32 (plain)", fallback.KeyPrefixlen) + } +} + +// testFlagMarker is a high-bit flag used only by tests to detect improper +// flag inheritance from COVERING entries (it is not a real netPolicyFlag*). +const testFlagMarker = 0x40 + +// TestDNSLearnCoveringStaticCIDRDoesNotImmortalize is the exact-key-match +// regression test: a static CIDR COVERING the resolved IP must NOT make the +// DNS-learned entry inherit the static zero expiry (never ages) or the +// covering entry's flags. The LPM lookup inside dns_learn_response_ip is +// longest-prefix, so without the key_prefixlen check the covering static +// entry was treated as "an existing entry for the same key". +func TestDNSLearnCoveringStaticCIDRDoesNotImmortalize(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(303) + + // Static /24 (never expires) covering both test IPs. + staticSubnet := allowOutV3Entry{ + key: lpmKeyV3{Prefixlen: 24, IP: mustParseCIDRForTest(t, "192.0.2.0").IP, Port: 0}, + value: netPolicyValueV3{Flags: testFlagMarker, KeyPrefixlen: 24}, // ExpiresAtNS: 0 = static + } + + // Plain (non-L7) learn of 192.0.2.50 under the covering /24. + ipPlain := mustParseCIDRForTest(t, "192.0.2.50").IP + inner := runDNSLearn(t, env, ifindex, ipPlain, 300, + dnsQueryTrackValue{Flags: 0, PortCount: 0}, staticSubnet) + value, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 32, IP: ipPlain, Port: 0}) + if !ok { + t.Fatal("plain allow did not learn a /32 entry") + } + if value.KeyPrefixlen != 32 { + t.Fatalf("learned /32 KeyPrefixlen=%d, want 32 (lookup may have hit the covering /24)", value.KeyPrefixlen) + } + if value.ExpiresAtNS == 0 { + t.Fatal("learned /32 inherited static zero expiry from covering /24 (would never age)") + } + if value.Flags&testFlagMarker != 0 { + t.Fatalf("learned /32 inherited flags from covering /24: %#x", value.Flags) + } + + // L7 learn of 192.0.2.60 under the covering /24. + ipL7 := mustParseCIDRForTest(t, "192.0.2.60").IP + inner = runDNSLearn(t, env, ifindex, ipL7, 300, + dnsQueryTrackValue{Flags: uint8(netPolicyFlagL7Required), PortCount: 0}, staticSubnet) + value, ok = lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ipL7, Port: htonsPort(443)}) + if !ok { + t.Fatal("L7 allow did not learn a /48 entry for 443") + } + if value.KeyPrefixlen != 48 { + t.Fatalf("learned /48 KeyPrefixlen=%d, want 48 (lookup may have hit the covering /24)", value.KeyPrefixlen) + } + if value.ExpiresAtNS == 0 { + t.Fatal("learned /48 inherited static zero expiry from covering /24 (would never age)") + } + if value.Flags&testFlagMarker != 0 { + t.Fatalf("learned /48 inherited flags from covering /24: %#x", value.Flags) + } +} + +// TestDNSLearnExactStaticEntrySurvivesRefresh is the counterpart: an existing +// static entry for the EXACT same (ip, port)/48 key must survive a DNS +// refresh — its zero expiry and its extra flags are preserved. Other ports +// of the same learn get a normal TTL entry. +func TestDNSLearnExactStaticEntrySurvivesRefresh(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(304) + ip := mustParseCIDRForTest(t, "192.0.2.70").IP + + staticExact := allowOutV3Entry{ + key: lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}, + value: netPolicyValueV3{Flags: uint8(netPolicyFlagL7Required) | testFlagMarker, Scheme: L7SchemeHTTP, KeyPrefixlen: 48}, + } + inner := runDNSLearn(t, env, ifindex, ip, 300, + dnsQueryTrackValue{Flags: uint8(netPolicyFlagL7Required), PortCount: 0}, staticExact) + + // Exact static /48(443): zero expiry + marker flag preserved. + value, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if !ok { + t.Fatal("missing /48 entry for 443") + } + if value.ExpiresAtNS != 0 { + t.Fatal("exact static /48 lost its zero expiry on DNS refresh") + } + if value.Flags&testFlagMarker == 0 { + t.Fatalf("exact static /48 lost its flags on DNS refresh: %#x", value.Flags) + } + if value.Scheme != L7SchemeHTTPS { + t.Fatalf("exact static /48 scheme=%d after refresh, want HTTPS (scheme is last-write-wins)", value.Scheme) + } + + // /48(80) has no exact static entry: normal learned TTL entry. + value, ok = lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(80)}) + if !ok { + t.Fatal("missing /48 entry for 80") + } + if value.ExpiresAtNS == 0 { + t.Fatal("/48(80) wrongly became static (no exact static entry exists)") + } + if value.Flags&testFlagMarker != 0 { + t.Fatalf("/48(80) inherited marker flag from the 443 entry: %#x", value.Flags) + } +} + +// TestDNSLearnRefreshRenewsTTLMergesFlagsAndOverwritesScheme covers the +// remaining same-key merge cell: the OLD entry is itself a LEARNED entry for +// the exact same key. The refresh must (a) merge flags (old marker retained), +// (b) RENEW the expiry to the new DNS TTL (not keep the stale one), and +// (c) overwrite the scheme with the latest learn (last write wins). +func TestDNSLearnRefreshRenewsTTLMergesFlagsAndOverwritesScheme(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(307) + ip := mustParseCIDRForTest(t, "192.0.2.100").IP + + staleLearned := allowOutV3Entry{ + key: lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}, + value: netPolicyValueV3{ + ExpiresAtNS: 1, // stale, about to expire + Flags: uint8(netPolicyFlagL7Required) | testFlagMarker, + Scheme: L7SchemeHTTP, // wrong scheme on 443: refresh must overwrite + KeyPrefixlen: 48, + }, + } + inner := runDNSLearn(t, env, ifindex, ip, 300, + dnsQueryTrackValue{Flags: uint8(netPolicyFlagL7Required), PortCount: 0}, staleLearned) + + value, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if !ok { + t.Fatal("missing /48 entry for 443") + } + if value.KeyPrefixlen != 48 { + t.Fatalf("KeyPrefixlen=%d, want 48", value.KeyPrefixlen) + } + if value.ExpiresAtNS == 0 { + t.Fatal("refresh of a learned entry must keep a non-zero TTL (not become static)") + } + if value.ExpiresAtNS == 1 { + t.Fatal("refresh kept the stale expiry instead of renewing to the new DNS TTL") + } + if value.Flags&testFlagMarker == 0 { + t.Fatalf("refresh lost old learned flags: %#x", value.Flags) + } + if value.Scheme != L7SchemeHTTPS { + t.Fatalf("scheme=%d after refresh, want HTTPS (last write wins)", value.Scheme) + } +} diff --git a/CubeNet/cubevs/dns_reaper.go b/CubeNet/cubevs/dns_reaper.go index 48970f8a2..bad5dbe90 100644 --- a/CubeNet/cubevs/dns_reaper.go +++ b/CubeNet/cubevs/dns_reaper.go @@ -23,13 +23,13 @@ func reapDNSState() { reapDNSQueryTrack(now) } -// reapDNSLearnedPolicies scans allow_out_v2 and removes expired DNS-learned entries. +// reapDNSLearnedPolicies scans allow_out_v3 and removes expired DNS-learned entries. func reapDNSLearnedPolicies(now uint64) { - allowOut, err := loadPinnedMap(MapNameAllowOutV2) + allowOut, err := loadPinnedMap(MapNameAllowOutV3) if err != nil { enqueueEvent(Event{ Error: err, - Message: "failed to load allow_out_v2 map", + Message: "failed to load allow_out_v3 map", }) return } @@ -51,13 +51,13 @@ func reapDNSLearnedPolicies(now uint64) { if err := iter.Err(); err != nil { enqueueEvent(Event{ Error: err, - Message: "failed to iterate allow_out_v2 map", + Message: "failed to iterate allow_out_v3 map", }) return } } -// reapDNSLearnedPoliciesForInnerMap deletes expired DNS-learned entries from one allow_out_v2 inner map. +// reapDNSLearnedPoliciesForInnerMap deletes expired DNS-learned entries from one allow_out_v3 inner map. func reapDNSLearnedPoliciesForInnerMap(innerMapID uint32, now uint64) error { inner, err := ebpf.NewMapFromID(ebpf.MapID(innerMapID)) if err != nil { @@ -66,12 +66,12 @@ func reapDNSLearnedPoliciesForInnerMap(innerMapID uint32, now uint64) error { defer inner.Close() var ( - key lpmKey - value netPolicyValueV2 + key lpmKeyV3 + value netPolicyValueV3 ) iter := inner.Iterate() for iter.Next(&key, &value) { - if !netPolicyValueV2Expired(value, now) { + if !netPolicyValueV3Expired(value, now) { continue } if err := inner.Delete(&key); err != nil && !errors.Is(err, ebpf.ErrKeyNotExist) { @@ -79,7 +79,7 @@ func reapDNSLearnedPoliciesForInnerMap(innerMapID uint32, now uint64) error { } } if err := iter.Err(); err != nil { - return fmt.Errorf("failed to iterate allow_out_v2 inner map: %w", err) + return fmt.Errorf("failed to iterate allow_out_v3 inner map: %w", err) } return nil } diff --git a/CubeNet/cubevs/dnspolicy.go b/CubeNet/cubevs/dnspolicy.go index 9079ffa0b..52d1bad50 100644 --- a/CubeNet/cubevs/dnspolicy.go +++ b/CubeNet/cubevs/dnspolicy.go @@ -31,11 +31,11 @@ func newInnerDNSAllowMap() (*ebpf.Map, error) { // ensureDNSAllowInnerMap creates the per-sandbox DNS allow map when it is absent. func ensureDNSAllowInnerMap(outerMap *ebpf.Map, ifindex uint32) error { - return ensureInnerMapWithFactory(outerMap, ifindex, MapNameDNSAllow, newInnerDNSAllowMap) + return ensureInnerMapWithFactory(outerMap, ifindex, MapNameDNSAllowV2, newInnerDNSAllowMap) } func initDNSAllow(ifindex uint32) error { - dnsAllow, err := loadPinnedMap(MapNameDNSAllow) + dnsAllow, err := loadPinnedMap(MapNameDNSAllowV2) if err != nil { return err } @@ -89,35 +89,25 @@ type dnsAllowRule struct { domain string } -func buildDNSAllowRules(domains, l7Domains []string) ([]dnsAllowRule, error) { - rules := make([]dnsAllowRule, 0, len(domains)+len(l7Domains)) - indexByKey := make(map[dnsAllowKey]int, len(domains)+len(l7Domains)) +func buildDNSAllowRules(domains []string) ([]dnsAllowRule, error) { + rules := make([]dnsAllowRule, 0, len(domains)) + indexByKey := make(map[dnsAllowKey]int, len(domains)) - add := func(domains []string, flags uint8) error { - for _, domain := range domains { - key, value, err := makeDNSAllowRule(domain, flags) - if err != nil { - return err - } - if idx, ok := indexByKey[key]; ok { - rules[idx].value.Flags |= flags - continue - } - indexByKey[key] = len(rules) - rules = append(rules, dnsAllowRule{ - key: key, - value: value, - domain: domain, - }) + for _, domain := range domains { + key, value, err := makeDNSAllowRule(domain, 0) + if err != nil { + return nil, err } - return nil - } - - if err := add(domains, 0); err != nil { - return nil, err - } - if err := add(l7Domains, uint8(netPolicyFlagL7Required)); err != nil { - return nil, err + if idx, ok := indexByKey[key]; ok { + rules[idx].value.Flags |= value.Flags + continue + } + indexByKey[key] = len(rules) + rules = append(rules, dnsAllowRule{ + key: key, + value: value, + domain: domain, + }) } return rules, nil } @@ -150,6 +140,12 @@ func updateDNSAllowRule(inner *ebpf.Map, rule dnsAllowRule) error { var oldValue dnsAllowValue if err := inner.Lookup(&rule.key, &oldValue); err == nil { value.Flags |= oldValue.Flags + // Preserve any port tuples already installed for this key. Rule + // order should not cause a later rule with a subset of the port + // set to clobber an earlier rule's ports. buildL7Plan is the + // single point where scheme conflicts are detected, so any port + // present in both rules must agree on scheme by construction. + mergePortsIntoDNSValue(&value, oldValue.Ports[:oldValue.PortCount]) } else if !errors.Is(err, ebpf.ErrKeyNotExist) { return fmt.Errorf("dns allow lookup failed: %w, domain: %s", err, rule.domain) } @@ -160,6 +156,26 @@ func updateDNSAllowRule(inner *ebpf.Map, rule dnsAllowRule) error { return nil } +// mergePortsIntoDNSValue unions src into v.Ports without exceeding +// maxL7PortsPerHost. Silently drops overflow — buildL7Plan already enforces +// the budget in userspace. +func mergePortsIntoDNSValue(v *dnsAllowValue, src []l7PortEntry) { + for _, p := range src { + exists := false + for i := uint8(0); i < v.PortCount; i++ { + if v.Ports[i].Port == p.Port { + exists = true + break + } + } + if exists || v.PortCount >= maxL7PortsPerHost { + continue + } + v.Ports[v.PortCount] = p + v.PortCount++ + } +} + func flushDNSAllowInnerMap(inner *ebpf.Map) error { var oldKey dnsAllowKey var oldValue dnsAllowValue @@ -177,7 +193,7 @@ func flushDNSAllowInnerMap(inner *ebpf.Map) error { // cleanupDNSAllow clears the sandbox DNS allow inner map while keeping it preallocated. func cleanupDNSAllow(ifindex uint32) error { - dnsAllow, err := loadPinnedMap(MapNameDNSAllow) + dnsAllow, err := loadPinnedMap(MapNameDNSAllowV2) if err != nil { return err } @@ -201,7 +217,7 @@ func applyDNSAllow(ifindex uint32, rules []dnsAllowRule, replace bool) error { return nil } - dnsAllow, err := loadPinnedMap(MapNameDNSAllow) + dnsAllow, err := loadPinnedMap(MapNameDNSAllowV2) if err != nil { return err } diff --git a/CubeNet/cubevs/dump.go b/CubeNet/cubevs/dump.go index de6c0bf1e..22288fb98 100644 --- a/CubeNet/cubevs/dump.go +++ b/CubeNet/cubevs/dump.go @@ -91,6 +91,10 @@ type EgressSessionDump struct { StateRaw uint8 `json:"state_raw"` ActiveClose bool `json:"active_close"` ActiveCloseRaw uint8 `json:"active_close_raw"` + PacketClass string `json:"packet_class"` + PacketClassRaw uint8 `json:"packet_class_raw"` + L7Scheme string `json:"l7_scheme"` + L7SchemeRaw uint8 `json:"l7_scheme_raw"` } type IngressSessionDump struct { @@ -113,8 +117,14 @@ type PolicyEntryDump struct { ExpiresIn string `json:"expires_in,omitempty"` Expired bool `json:"expired"` L7Required bool `json:"l7_required"` + L3Allowed bool `json:"l3_allowed"` Flags uint8 `json:"flags"` Static bool `json:"static"` + // L7 port tuples inherited from a matched dns_allow_value at DNS-learn + // time (or attached directly to an L7 IP/CIDR rule). Empty when the + // datapath falls back to the default {80/http, 443/https} port set. + PortCount uint8 `json:"port_count"` + Ports []L7PortEntryDump `json:"ports,omitempty"` } type DenyPolicyMapDump struct { @@ -139,9 +149,14 @@ type DNSAllowRuleDump struct { Domain string `json:"domain"` Wildcard bool `json:"wildcard"` L7Required bool `json:"l7_required"` + L3Allowed bool `json:"l3_allowed"` Flags uint8 `json:"flags"` NameLen uint32 `json:"name_len"` Prefixlen uint32 `json:"prefixlen"` + // L7 port tuples the userspace built from rules sharing this host. + // Empty when the datapath falls back to the default {80/http, 443/https}. + PortCount uint8 `json:"port_count"` + Ports []L7PortEntryDump `json:"ports,omitempty"` } type DNSQueryTrackDump struct { @@ -155,7 +170,21 @@ type DNSQueryTrackDump struct { ExpiresIn string `json:"expires_in"` Expired bool `json:"expired"` L7Required bool `json:"l7_required"` + L3Allowed bool `json:"l3_allowed"` Flags uint8 `json:"flags"` + // L7 port tuples copied verbatim from the matched dns_allow_value so the + // response handler can rebuild net_policy_value_v3 without a second + // dns_allow lookup. + PortCount uint8 `json:"port_count"` + Ports []L7PortEntryDump `json:"ports,omitempty"` +} + +// L7PortEntryDump is the JSON-friendly view of one (port, scheme) tuple. +// Port is emitted in host byte order (matches the port number users write in +// their policy YAML) even though it is stored in NBO on the datapath. +type L7PortEntryDump struct { + Port uint16 `json:"port"` + Scheme string `json:"scheme"` // "http" | "https" | ""(未知) } const mapNameSNATIPList = mapSNATIPList @@ -168,9 +197,9 @@ var businessMapDumpOrder = []string{ MapNameEgressSessions, MapNameIngressSessions, mapNameSNATIPList, - MapNameAllowOutV2, + MapNameAllowOutV3, MapNameDenyOut, - MapNameDNSAllow, + MapNameDNSAllowV2, MapNameDNSQueryTrack, } @@ -184,9 +213,9 @@ var businessMapDumpers = map[string]businessMapDumper{ MapNameEgressSessions: dumpEgressSessions, MapNameIngressSessions: dumpIngressSessions, mapNameSNATIPList: dumpSNATIPList, - MapNameAllowOutV2: dumpAllowOutV2, + MapNameAllowOutV3: dumpAllowOutV3, MapNameDenyOut: dumpDenyOut, - MapNameDNSAllow: dumpDNSAllow, + MapNameDNSAllowV2: dumpDNSAllow, MapNameDNSQueryTrack: dumpDNSQueryTrack, } @@ -411,6 +440,10 @@ func dumpEgressSessions(opts DumpOptions, now uint64) (any, error) { StateRaw: value.State, ActiveClose: value.ActiveClose != 0, ActiveCloseRaw: value.ActiveClose, + PacketClass: packetClassToString(value.PacketClass), + PacketClassRaw: value.PacketClass, + L7Scheme: l7SchemeToString(value.L7Scheme), + L7SchemeRaw: value.L7Scheme, }) } return entries, wrapIterErr(iter.Err(), MapNameEgressSessions) @@ -448,8 +481,8 @@ func dumpIngressSessions(opts DumpOptions, _ uint64) (any, error) { return entries, wrapIterErr(iter.Err(), MapNameIngressSessions) } -func dumpAllowOutV2(opts DumpOptions, now uint64) (any, error) { - return dumpPolicyMap(opts, now, MapNameAllowOutV2) +func dumpAllowOutV3(opts DumpOptions, now uint64) (any, error) { + return dumpPolicyMap(opts, now, MapNameAllowOutV3) } func dumpPolicyMap(opts DumpOptions, now uint64, mapName string) (any, error) { @@ -482,13 +515,13 @@ func dumpPolicyInnerMap(innerMapID uint32, now uint64) ([]PolicyEntryDump, error defer inner.Close() entries := make([]PolicyEntryDump, 0) - seen := make(map[lpmKey]struct{}) - var key lpmKey - var value netPolicyValueV2 + seen := make(map[lpmKeyV3]struct{}) + var key lpmKeyV3 + var value netPolicyValueV3 iter := inner.Iterate() for iter.Next(&key, &value) { if _, ok := seen[key]; ok { - return nil, fmt.Errorf("policy inner map iteration returned duplicate key: %s", dumpLPMCIDR(key)) //nolint:err113 + return nil, fmt.Errorf("policy inner map iteration returned duplicate key: %s", dumpLPMCIDRV3(key)) //nolint:err113 } seen[key] = struct{}{} if len(seen) > maxNetPolicyEntries { @@ -496,12 +529,23 @@ func dumpPolicyInnerMap(innerMapID uint32, now uint64) ([]PolicyEntryDump, error } entry := PolicyEntryDump{ - CIDR: dumpLPMCIDR(key), - Expired: netPolicyValueV2Expired(value, now), + CIDR: dumpLPMCIDRV3(key), + Expired: netPolicyValueV3Expired(value, now), L7Required: value.Flags&uint8(netPolicyFlagL7Required) != 0, + L3Allowed: value.Flags&uint8(netPolicyFlagL3Allowed) != 0, Flags: value.Flags, Static: value.ExpiresAtNS == 0, } + // v3 carries the port in the key and the scheme in the value, so a + // single (ip, port)/48 entry is reported as one L7 (port, scheme) + // tuple; ip-only / subnet entries carry no port. + if key.Port != 0 && value.Scheme != L7SchemeNone { + entry.PortCount = 1 + entry.Ports = []L7PortEntryDump{{ + Port: ntohsPort(key.Port), + Scheme: l7SchemeToString(value.Scheme), + }} + } if value.ExpiresAtNS != 0 { entry.ExpiresAtNS = value.ExpiresAtNS entry.ExpiresInNS = remainingNS(value.ExpiresAtNS, now) @@ -512,6 +556,13 @@ func dumpPolicyInnerMap(innerMapID uint32, now uint64) ([]PolicyEntryDump, error return entries, wrapIterErr(iter.Err(), "policy inner") } +func dumpLPMCIDRV3(key lpmKeyV3) string { + if key.Port != 0 { + return fmt.Sprintf("%s/%d:%d", uint32ToIP(key.IP).String(), key.Prefixlen, ntohsPort(key.Port)) + } + return fmt.Sprintf("%s/%d", uint32ToIP(key.IP).String(), key.Prefixlen) +} + func dumpDenyOut(opts DumpOptions, _ uint64) (any, error) { outer, err := loadPinnedMap(MapNameDenyOut) if err != nil { @@ -564,7 +615,7 @@ func dumpDenyInnerMap(innerMapID uint32) ([]DenyPolicyEntryDump, error) { } func dumpDNSAllow(opts DumpOptions, _ uint64) (any, error) { - outer, err := loadPinnedMap(MapNameDNSAllow) + outer, err := loadPinnedMap(MapNameDNSAllowV2) if err != nil { return nil, err } @@ -576,10 +627,10 @@ func dumpDNSAllow(opts DumpOptions, _ uint64) (any, error) { } entries := make([]DNSAllowMapDump, 0) - err = dumpSelectedInnerMapIDs(opts, outer, MapNameDNSAllow, func(ifindex uint32, innerMapID uint32) error { + err = dumpSelectedInnerMapIDs(opts, outer, MapNameDNSAllowV2, func(ifindex uint32, innerMapID uint32) error { innerDump, err := dumpDNSAllowInnerMap(innerMapID) if err != nil { - return fmt.Errorf("dump %s inner map failed: %w, ifindex: %d", MapNameDNSAllow, err, ifindex) + return fmt.Errorf("dump %s inner map failed: %w, ifindex: %d", MapNameDNSAllowV2, err, ifindex) } innerDump.Ifindex = ifindex if meta, ok := metadata[ifindex]; ok { @@ -659,7 +710,10 @@ func dumpDNSQueryTrack(opts DumpOptions, now uint64) (any, error) { ExpiresIn: remainingDuration(value.ExpiresAtNS, now), Expired: value.ExpiresAtNS <= now, L7Required: value.Flags&uint8(netPolicyFlagL7Required) != 0, + L3Allowed: value.Flags&uint8(netPolicyFlagL3Allowed) != 0, Flags: value.Flags, + PortCount: value.PortCount, + Ports: l7PortEntriesToDump(value.Ports[:], value.PortCount), }) } return entries, wrapIterErr(iter.Err(), MapNameDNSQueryTrack) @@ -827,9 +881,12 @@ func dumpDNSAllowRule(key dnsAllowKey, value dnsAllowValue) (DNSAllowRuleDump, e Domain: domain, Wildcard: wildcard, L7Required: value.Flags&uint8(netPolicyFlagL7Required) != 0, + L3Allowed: value.Flags&uint8(netPolicyFlagL3Allowed) != 0, Flags: value.Flags, NameLen: value.NameLen, Prefixlen: key.Prefixlen, + PortCount: value.PortCount, + Ports: l7PortEntriesToDump(value.Ports[:], value.PortCount), }, nil } @@ -837,6 +894,52 @@ func ifindexMatches(opts DumpOptions, ifindex uint32) bool { return !opts.FilterIfindex || opts.Ifindex == ifindex } +// l7PortEntriesToDump converts the fixed-size ports array embedded in the +// datapath value into a JSON-friendly slice sized to port_count. Ports are +// stored in network byte order on the datapath (matching tcphdr->dest) but +// emitted in host byte order here so the JSON is directly comparable to the +// port numbers users write in their rules. +func l7PortEntriesToDump(ports []l7PortEntry, count uint8) []L7PortEntryDump { + if count == 0 { + return nil + } + if int(count) > len(ports) { + count = uint8(len(ports)) + } + out := make([]L7PortEntryDump, 0, count) + for i := uint8(0); i < count; i++ { + out = append(out, L7PortEntryDump{ + Port: ntohs(ports[i].Port), + Scheme: l7SchemeToString(ports[i].Scheme), + }) + } + return out +} + +func packetClassToString(class uint8) string { + switch class { + case 0: + return "snat" + case 1: + return "l7_proxy" + default: + return fmt.Sprintf("unknown(%d)", class) + } +} + +func l7SchemeToString(s uint8) string { + switch s { + case L7SchemeNone: + return "none" + case L7SchemeHTTP: + return "http" + case L7SchemeHTTPS: + return "https" + default: + return fmt.Sprintf("unknown(%d)", s) + } +} + func remainingNS(expiresAt, now uint64) int64 { if expiresAt >= now { return int64(expiresAt - now) diff --git a/CubeNet/cubevs/dump_test.go b/CubeNet/cubevs/dump_test.go index 239583dd0..4f5d849eb 100644 --- a/CubeNet/cubevs/dump_test.go +++ b/CubeNet/cubevs/dump_test.go @@ -7,6 +7,35 @@ import ( "golang.org/x/sys/unix" ) +func TestSessionClassificationStrings(t *testing.T) { + for _, tt := range []struct { + value uint8 + want string + }{ + {0, "snat"}, + {1, "l7_proxy"}, + {9, "unknown(9)"}, + } { + if got := packetClassToString(tt.value); got != tt.want { + t.Fatalf("packetClassToString(%d)=%q, want %q", tt.value, got, tt.want) + } + } + + for _, tt := range []struct { + value uint8 + want string + }{ + {L7SchemeNone, "none"}, + {L7SchemeHTTP, "http"}, + {L7SchemeHTTPS, "https"}, + {9, "unknown(9)"}, + } { + if got := l7SchemeToString(tt.value); got != tt.want { + t.Fatalf("l7SchemeToString(%d)=%q, want %q", tt.value, got, tt.want) + } + } +} + func TestBusinessMapNamesReturnsCopy(t *testing.T) { names := BusinessMapNames() if len(names) == 0 { @@ -21,15 +50,15 @@ func TestBusinessMapNamesReturnsCopy(t *testing.T) { func TestNormalizeBusinessMapNames(t *testing.T) { got, err := normalizeBusinessMapNames([]string{ - MapNameDNSAllow, - MapNameDNSAllow, - MapNameAllowOutV2, + MapNameDNSAllowV2, + MapNameDNSAllowV2, + MapNameAllowOutV3, }) if err != nil { t.Fatalf("normalizeBusinessMapNames returned error: %v", err) } - want := []string{MapNameDNSAllow, MapNameAllowOutV2} + want := []string{MapNameDNSAllowV2, MapNameAllowOutV3} if !reflect.DeepEqual(got, want) { t.Fatalf("normalizeBusinessMapNames()=%v, want %v", got, want) } @@ -141,3 +170,30 @@ func TestDumpSessionKey(t *testing.T) { t.Fatalf("ProtocolName=%q, want tcp", got.ProtocolName) } } + +// TestL7PortEntriesToDumpRendersHostByteOrder guards the NBO→host port +// conversion: ports are stored in network byte order on the datapath, and a +// regression that drops the ntohs() would render the raw wire value (e.g. +// 0xEB20) instead of the user-facing host-order port (8443). +func TestL7PortEntriesToDumpRendersHostByteOrder(t *testing.T) { + ports := []l7PortEntry{ + {Port: htonsPort(8443), Scheme: L7SchemeHTTPS}, + {Port: htonsPort(8080), Scheme: L7SchemeHTTP}, + } + + got := l7PortEntriesToDump(ports, 2) + if len(got) != 2 { + t.Fatalf("len(got)=%d, want 2", len(got)) + } + if got[0].Port != 8443 || got[0].Scheme != "https" { + t.Fatalf("entry[0]=%+v, want port 8443 scheme https", got[0]) + } + if got[1].Port != 8080 || got[1].Scheme != "http" { + t.Fatalf("entry[1]=%+v, want port 8080 scheme http", got[1]) + } + + // count == 0 renders nothing (default-set fallback is implicit). + if got := l7PortEntriesToDump(ports, 0); got != nil { + t.Fatalf("count=0 rendered %v, want nil", got) + } +} diff --git a/CubeNet/cubevs/egress_policy_test.go b/CubeNet/cubevs/egress_policy_test.go new file mode 100644 index 000000000..16eb34562 --- /dev/null +++ b/CubeNet/cubevs/egress_policy_test.go @@ -0,0 +1,309 @@ +package cubevs + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target $GOARCH egresspolicy ../src/egress_policy_test.bpf.c -- -I../vmlinux/$GOARCH + +import ( + "encoding/binary" + "testing" + + "github.com/cilium/ebpf" +) + +const ( + egressPolicyTestCaseLen = 16 + flowVerdictReject = uint8(0) + flowVerdictSNAT = uint8(1) + flowVerdictHTTP = uint8(2) + flowVerdictHTTPS = uint8(3) +) + +type egressPolicyTestEnv struct { + program *ebpf.Program + allowOut *ebpf.Map + denyOut *ebpf.Map + allowInnerSpec *ebpf.MapSpec + denyInnerSpec *ebpf.MapSpec +} + +func loadEgressPolicyTestEnv(t *testing.T) *egressPolicyTestEnv { + t.Helper() + + spec, err := loadEgresspolicy() + if err != nil { + t.Fatalf("load egress policy test spec: %v", err) + } + allowSpec := spec.Maps["allow_out_v3"] + denySpec := spec.Maps["deny_out"] + if allowSpec == nil || allowSpec.InnerMap == nil || denySpec == nil || denySpec.InnerMap == nil { + t.Fatal("egress policy map specs or inner templates missing") + } + allowInnerSpec := allowSpec.InnerMap.Copy() + denyInnerSpec := denySpec.InnerMap.Copy() + + for name, mapSpec := range spec.Maps { + switch name { + case ".rodata", "allow_out_v3", "deny_out": + mapSpec.Pinning = ebpf.PinNone + default: + delete(spec.Maps, name) + } + } + + coll, err := ebpf.NewCollection(spec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF policy test unavailable: %v", err) + } + t.Fatalf("load egress policy test collection: %v", err) + } + t.Cleanup(coll.Close) + + env := &egressPolicyTestEnv{ + program: coll.Programs["test_classify_egress_flow"], + allowOut: coll.Maps["allow_out_v3"], + denyOut: coll.Maps["deny_out"], + allowInnerSpec: allowInnerSpec, + denyInnerSpec: denyInnerSpec, + } + if env.program == nil || env.allowOut == nil || env.denyOut == nil { + t.Fatal("loaded egress policy program or maps missing") + } + return env +} + +func newEgressPolicyInnerMap(t *testing.T, spec *ebpf.MapSpec, name string) *ebpf.Map { + t.Helper() + + innerSpec := spec.Copy() + innerSpec.Name = name + innerSpec.Pinning = ebpf.PinNone + inner, err := ebpf.NewMap(innerSpec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF LPM trie unavailable: %v", err) + } + t.Fatalf("create %s: %v", name, err) + } + t.Cleanup(func() { _ = inner.Close() }) + return inner +} + +func (env *egressPolicyTestEnv) attachInnerMaps(t *testing.T, ifindex uint32) (*ebpf.Map, *ebpf.Map) { + t.Helper() + + allowInner := newEgressPolicyInnerMap(t, env.allowInnerSpec, "allow_test") + denyInner := newEgressPolicyInnerMap(t, env.denyInnerSpec, "deny_test") + if err := env.allowOut.Put(&ifindex, allowInner); err != nil { + t.Fatalf("attach allow inner map: %v", err) + } + if err := env.denyOut.Put(&ifindex, denyInner); err != nil { + t.Fatalf("attach deny inner map: %v", err) + } + return allowInner, denyInner +} + +func runEgressPolicyCase(t *testing.T, prog *ebpf.Program, ifindex, daddr uint32, + dport uint16, +) uint8 { + t.Helper() + + data := make([]byte, egressPolicyTestCaseLen) + binary.LittleEndian.PutUint32(data[0:4], ifindex) + binary.LittleEndian.PutUint32(data[4:8], daddr) + binary.LittleEndian.PutUint16(data[8:10], dport) + ret, out, err := prog.Test(data) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF policy test-run unavailable: %v", err) + } + t.Fatalf("run egress policy test: %v", err) + } + if ret != 0 { + t.Fatalf("test_classify_egress_flow returned %d, want TC_ACT_OK", ret) + } + if len(out) < egressPolicyTestCaseLen { + t.Fatalf("test output length=%d, want >=%d", len(out), egressPolicyTestCaseLen) + } + return out[10] +} + +func mustParseCIDRForTest(t *testing.T, cidr string) lpmKey { + t.Helper() + + key, err := parseCIDR(cidr) + if err != nil { + t.Fatalf("parse %q: %v", cidr, err) + } + return key +} + +func TestClassifyEgressFlowLPMFallback(t *testing.T) { + env := loadEgressPolicyTestEnv(t) + tests := []struct { + name string + allowCIDR string + daddr string + }{ + {name: "IP rule", allowCIDR: "203.0.113.9/32", daddr: "203.0.113.9"}, + {name: "subnet rule", allowCIDR: "198.51.100.0/24", daddr: "198.51.100.42"}, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ifindex := uint32(100 + i) + allowInner, denyInner := env.attachInnerMaps(t, ifindex) + allowKey := mustParseCIDRForTest(t, tt.allowCIDR) + if err := allowInner.Put(&lpmKeyV3{ + Prefixlen: allowKey.Prefixlen, + IP: allowKey.IP, + }, &netPolicyValueV3{}); err != nil { + t.Fatalf("insert allow rule %s: %v", tt.allowCIDR, err) + } + + // A deny-all backstop proves that FLOW_SNAT came from the /48 + // lookup falling back to the allow rule, not from default allow. + denyAll := mustParseCIDRForTest(t, "0.0.0.0/0") + denyValue := uint32(1) + if err := denyInner.Put(&denyAll, &denyValue); err != nil { + t.Fatalf("insert deny-all rule: %v", err) + } + + daddr := mustParseCIDRForTest(t, tt.daddr).IP + got := runEgressPolicyCase(t, env.program, ifindex, daddr, htonsPort(443)) + if got != flowVerdictSNAT { + t.Fatalf("verdict=%d, want FLOW_SNAT", got) + } + }) + } +} + +func TestClassifyEgressFlowExactL7Match(t *testing.T) { + env := loadEgressPolicyTestEnv(t) + ifindex := uint32(150) + allowInner, denyInner := env.attachInnerMaps(t, ifindex) + daddr := mustParseCIDRForTest(t, "192.0.2.20").IP + dport := htonsPort(8443) + allowKey := lpmKeyV3{Prefixlen: 48, IP: daddr, Port: dport} + allowValue := netPolicyValueV3{ + Flags: uint8(netPolicyFlagL7Required), + Scheme: L7SchemeHTTPS, + } + if err := allowInner.Put(&allowKey, &allowValue); err != nil { + t.Fatalf("insert exact L7 allow rule: %v", err) + } + + denyAll := mustParseCIDRForTest(t, "0.0.0.0/0") + denyValue := uint32(1) + if err := denyInner.Put(&denyAll, &denyValue); err != nil { + t.Fatalf("insert deny-all rule: %v", err) + } + + if got := runEgressPolicyCase(t, env.program, ifindex, daddr, dport); got != flowVerdictHTTPS { + t.Fatalf("exact-port verdict=%d, want FLOW_HTTPS", got) + } + if got := runEgressPolicyCase(t, env.program, ifindex, daddr, htonsPort(443)); got != flowVerdictReject { + t.Fatalf("different-port verdict=%d, want FLOW_REJECT", got) + } +} + +// TestClassifyEgressFlowExactL7MatchHTTP covers the plaintext HTTP interception +// verdict (FLOW_HTTP) — the primary new data path the HTTPS-only cases did not +// exercise. eBPF returns FLOW_HTTP for an L7 entry whose scheme is http. +func TestClassifyEgressFlowExactL7MatchHTTP(t *testing.T) { + env := loadEgressPolicyTestEnv(t) + ifindex := uint32(152) + allowInner, denyInner := env.attachInnerMaps(t, ifindex) + daddr := mustParseCIDRForTest(t, "192.0.2.22").IP + dport := htonsPort(8080) + allowKey := lpmKeyV3{Prefixlen: 48, IP: daddr, Port: dport} + allowValue := netPolicyValueV3{ + Flags: uint8(netPolicyFlagL7Required), + Scheme: L7SchemeHTTP, + } + if err := allowInner.Put(&allowKey, &allowValue); err != nil { + t.Fatalf("insert exact L7 HTTP allow rule: %v", err) + } + + denyAll := mustParseCIDRForTest(t, "0.0.0.0/0") + denyValue := uint32(1) + if err := denyInner.Put(&denyAll, &denyValue); err != nil { + t.Fatalf("insert deny-all rule: %v", err) + } + + if got := runEgressPolicyCase(t, env.program, ifindex, daddr, dport); got != flowVerdictHTTP { + t.Fatalf("exact-port verdict=%d, want FLOW_HTTP", got) + } + if got := runEgressPolicyCase(t, env.program, ifindex, daddr, htonsPort(80)); got != flowVerdictReject { + t.Fatalf("different-port verdict=%d, want FLOW_REJECT", got) + } +} + +func TestClassifyEgressFlowL7UnknownSchemeFailsClosed(t *testing.T) { + env := loadEgressPolicyTestEnv(t) + ifindex := uint32(151) + allowInner, denyInner := env.attachInnerMaps(t, ifindex) + daddr := mustParseCIDRForTest(t, "192.0.2.21").IP + dport := htonsPort(8443) + // L7_REQUIRED set but scheme is NONE — a corrupt or half-written entry. + // Must fail closed (FLOW_REJECT) rather than downgrade to FLOW_SNAT and + // silently bypass the TPROXY intercept the rule asked for. + allowKey := lpmKeyV3{Prefixlen: 48, IP: daddr, Port: dport} + allowValue := netPolicyValueV3{ + Flags: uint8(netPolicyFlagL7Required), + Scheme: L7SchemeNone, + } + if err := allowInner.Put(&allowKey, &allowValue); err != nil { + t.Fatalf("insert L7 allow rule with unknown scheme: %v", err) + } + + denyAll := mustParseCIDRForTest(t, "0.0.0.0/0") + denyValue := uint32(1) + if err := denyInner.Put(&denyAll, &denyValue); err != nil { + t.Fatalf("insert deny-all rule: %v", err) + } + + if got := runEgressPolicyCase(t, env.program, ifindex, daddr, dport); got != flowVerdictReject { + t.Fatalf("verdict=%d, want FLOW_REJECT (fail closed on unknown scheme)", got) + } +} + +func TestClassifyEgressFlowExpiredAllow(t *testing.T) { + env := loadEgressPolicyTestEnv(t) + tests := []struct { + name string + prefixlen uint32 + }{ + {name: "expired exact L7 then deny", prefixlen: 48}, + {name: "expired fallback IP then deny", prefixlen: 32}, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ifindex := uint32(200 + i) + allowInner, denyInner := env.attachInnerMaps(t, ifindex) + daddr := mustParseCIDRForTest(t, "192.0.2.10").IP + dport := htonsPort(443) + allowKey := lpmKeyV3{Prefixlen: tt.prefixlen, IP: daddr} + allowValue := netPolicyValueV3{ExpiresAtNS: 1} + if tt.prefixlen == 48 { + allowKey.Port = dport + allowValue.Flags = uint8(netPolicyFlagL7Required) + allowValue.Scheme = L7SchemeHTTPS + } + if err := allowInner.Put(&allowKey, &allowValue); err != nil { + t.Fatalf("insert expired /%d allow rule: %v", tt.prefixlen, err) + } + + denyAll := mustParseCIDRForTest(t, "0.0.0.0/0") + denyValue := uint32(1) + if err := denyInner.Put(&denyAll, &denyValue); err != nil { + t.Fatalf("insert deny-all rule: %v", err) + } + + got := runEgressPolicyCase(t, env.program, ifindex, daddr, dport) + if got != flowVerdictReject { + t.Fatalf("verdict=%d, want FLOW_REJECT", got) + } + }) + } +} diff --git a/CubeNet/cubevs/l7_mark_test.go b/CubeNet/cubevs/l7_mark_test.go new file mode 100644 index 000000000..388623d7d --- /dev/null +++ b/CubeNet/cubevs/l7_mark_test.go @@ -0,0 +1,117 @@ +package cubevs + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target $GOARCH l7mark ../src/l7_mark_test.bpf.c -- -I../vmlinux/$GOARCH + +import ( + "encoding/binary" + "testing" + + "github.com/cilium/ebpf" +) + +// loadL7MarkProgram loads the l7_mark test object (only its .rodata and the +// test program), applies the given L7 mark overrides to the const volatile +// globals exactly as rewriteConstants would, and returns the loaded program. +func loadL7MarkProgram(t *testing.T, httpMark, httpsMark, mask uint32) *ebpf.Program { + t.Helper() + spec, err := loadL7mark() + if err != nil { + t.Fatalf("load l7 mark test spec: %v", err) + } + for name, mapSpec := range spec.Maps { + if name != ".rodata" { + delete(spec.Maps, name) + } else { + mapSpec.Pinning = ebpf.PinNone + } + } + setVar := func(name string, value uint32) { + v := spec.Variables[name] + if v == nil { + t.Fatalf("variable %s missing from l7 mark test object", name) + } + if err := v.Set(value); err != nil { + t.Fatalf("set %s: %v", name, err) + } + } + setVar(globalNameCubeL7MarkHTTP, httpMark) + setVar(globalNameCubeL7MarkHTTPS, httpsMark) + setVar(globalNameCubeL7MarkMask, mask) + + coll, err := ebpf.NewCollection(spec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF l7 mark test unavailable: %v", err) + } + t.Fatalf("load l7 mark test collection: %v", err) + } + t.Cleanup(coll.Close) + prog := coll.Programs["test_l7_mark"] + if prog == nil { + t.Fatal("test_l7_mark program missing") + } + return prog +} + +func runL7MarkCase(t *testing.T, prog *ebpf.Program, inMark uint32) uint32 { + t.Helper() + data := make([]byte, 16) // pad past the 4-byte case; tiny packets are rejected + binary.LittleEndian.PutUint32(data[0:4], inMark) + ret, out, err := prog.Test(data) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF l7 mark test-run unavailable: %v", err) + } + t.Fatalf("run l7 mark test: %v", err) + } + if ret != 0 { + t.Fatalf("test_l7_mark returned %d, want TC_ACT_OK", ret) + } + return binary.LittleEndian.Uint32(out[0:4]) +} + +// TestL7MarkStampedMatchesConfiguredValue proves that an overridden +// cube_l7_mark_http reaches the running dataplane and is the value stamped +// onto skb->mark (masked), exactly as mvmtap.bpf.c does for L7 traffic. +func TestL7MarkStampedMatchesConfiguredValue(t *testing.T) { + const httpMark = uint32(0xABCD0000) + const mask = uint32(0xFFFF0000) + prog := loadL7MarkProgram(t, httpMark, defaultL7MarkHTTPS, mask) + + // Low (user) bits of the incoming mark are preserved; the configured HTTP + // mark is OR'd into the cube-owned bits. + const inMark = uint32(0x00001234) + want := (inMark &^ mask) | httpMark + if got := runL7MarkCase(t, prog, inMark); got != want { + t.Fatalf("stamped mark=%#x, want %#x (configured http mark %#x OR'd over low bits %#x)", + got, want, httpMark, inMark&^mask) + } +} + +// TestResolveL7Marks covers the defaults-and-validation helper used by +// rewriteConstants: zero fields get shipped defaults, overrides are honored, +// and invalid combinations (http==https, bits outside the mask) are rejected. +func TestResolveL7Marks(t *testing.T) { + h, s, m, err := resolveL7Marks(Params{}) + if err != nil { + t.Fatalf("defaults: %v", err) + } + if h != defaultL7MarkHTTP || s != defaultL7MarkHTTPS || m != defaultL7MarkMask { + t.Fatalf("defaults: got http=%#x https=%#x mask=%#x", h, s, m) + } + + h, s, m, err = resolveL7Marks(Params{L7MarkHTTP: 0xAB010000, L7MarkHTTPS: 0xAB020000}) + if err != nil { + t.Fatalf("override: %v", err) + } + if h != 0xAB010000 || s != 0xAB020000 || m != defaultL7MarkMask { + t.Fatalf("override: got http=%#x https=%#x mask=%#x", h, s, m) + } + + if _, _, _, err := resolveL7Marks(Params{L7MarkHTTP: 0xAB010000, L7MarkHTTPS: 0xAB010000}); err == nil { + t.Fatal("http==https was not rejected") + } + if _, _, _, err := resolveL7Marks(Params{L7MarkHTTP: 0xAB010001}); err == nil { + t.Fatal("mark with bits outside the mask was not rejected") + } +} diff --git a/CubeNet/cubevs/map.go b/CubeNet/cubevs/map.go index 92437b8bd..46b7a3666 100644 --- a/CubeNet/cubevs/map.go +++ b/CubeNet/cubevs/map.go @@ -7,6 +7,11 @@ import ( "github.com/cilium/ebpf" ) +// bpfFSPath is the bpffs mount point used for pinning maps. It is a var (not a +// const) so tests can point it at a temporary bpffs mount instead of the real +// /sys/fs/bpf. +var bpfFSPath = "/sys/fs/bpf" + func pinPath(name string) string { path := filepath.Join(bpfFSPath, name) diff --git a/CubeNet/cubevs/migration.go b/CubeNet/cubevs/migration.go index 19c2ab359..1baa5500a 100644 --- a/CubeNet/cubevs/migration.go +++ b/CubeNet/cubevs/migration.go @@ -3,6 +3,7 @@ package cubevs import ( "errors" "fmt" + "log" "os" "unsafe" @@ -11,24 +12,173 @@ import ( const legacyAllowOutValueSize = uint32(4) -// migrateAllowOutV1ToV2 copies static v0.2.0 allow_out entries into -// allow_out_v2. The legacy value is only a presence marker, so migrated -// entries become static v2 entries with ExpiresAtNS set to zero. -func migrateAllowOutV1ToV2() error { - legacy, err := ebpf.LoadPinnedMap(pinPath(MapNameAllowOut), nil) +// skipUnsupportedL7Subnet reports whether a legacy allow entry is an L7 rule +// keyed by a subnet (prefixlen<32). v3 cannot express subnet+port in a single +// LPM key (its /48 key matches exact (ip, port) pairs), and new L7 rules now +// reject subnet hosts outright (classifyL7Target). Carrying such a legacy rule +// forward would silently narrow it to the network address, so it is dropped +// here with a warning instead; the operator should recreate it as /32 host or +// domain rules. +func skipUnsupportedL7Subnet(key lpmKey, flags uint8, sourceName string) bool { + if flags&netPolicyFlagL7Required == 0 || key.Prefixlen >= 32 { + return false + } + log.Printf("cubevs migration: dropping unsupported L7 subnet rule %s/%d from %s: v3 cannot express subnet+port; recreate as /32 host or domain rules", + uint32ToIP(key.IP).String(), key.Prefixlen, sourceName) + return true +} + +type legacyDNSAllowValue struct { + NameLen uint32 + Flags uint8 + Reserved [3]uint8 +} + +// allowOutV3Entry is one (key, value) pair in the new allow_out_v3 +// layout, produced by expanding an older allow_out entry. +type allowOutV3Entry struct { + key lpmKeyV3 + value netPolicyValueV3 +} + +// buildV3Entries expands one legacy (ip CIDR, flags, ports, expires) into +// the new allow_out_v3 entries: +// - L7 flag set: one exact (ip, port)/48 entry per (port, scheme); +// a default port set (port_count == 0) expands to {80/http, 443/https}. +// - otherwise: one ip-only (or subnet) /32 entry, scheme = NONE. +// +// The legacy net_policy_value_v2 ports[] (if any) are passed by the caller +// as the ports slice; a nil slice triggers the default-set expansion. +func buildV3Entries(ipKey lpmKey, flags uint8, ports []l7PortEntry, expires uint64) []allowOutV3Entry { + if flags&netPolicyFlagL7Required != 0 { + if len(ports) == 0 { + ports = expandDefaultPortSet() + } + out := make([]allowOutV3Entry, 0, len(ports)) + for _, p := range ports { + out = append(out, allowOutV3Entry{ + key: lpmKeyV3{Prefixlen: 48, IP: ipKey.IP, Port: p.Port}, + value: netPolicyValueV3{Flags: flags, Scheme: p.Scheme, ExpiresAtNS: expires, KeyPrefixlen: 48}, + }) + } + return out + } + return []allowOutV3Entry{{ + key: lpmKeyV3{Prefixlen: ipKey.Prefixlen, IP: ipKey.IP, Port: 0}, + value: netPolicyValueV3{Flags: flags, ExpiresAtNS: expires, KeyPrefixlen: uint8(ipKey.Prefixlen)}, + }} +} + +// applyV3Entries writes expanded entries into a v3 inner LPM trie. +func applyV3Entries(dest *ebpf.Map, entries []allowOutV3Entry) error { + for _, e := range entries { + if err := dest.Update(&e.key, &e.value, ebpf.UpdateAny); err != nil { + return fmt.Errorf("update %s inner map failed: %w", MapNameAllowOutV3, err) + } + } + return nil +} + +func pinnedMapExists(name string) (bool, error) { + _, err := os.Stat(pinPath(name)) + if err == nil { + return true, nil + } + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, err +} + +// migratePersistentPolicyMaps performs a one-way migration into the current +// generation. Both legacy maps are migrated first; the legacy pins are only +// unlinked after both have been copied into the current maps. This ordering +// matters for rollback: Init removes the freshly created allow_out_v3 / +// dns_allow_v2 pins when migration fails, so if the allow pin were unlinked +// before the DNS migration ran, a DNS failure would leave the allow_out +// policy unmigratable (legacy pin gone, new pin rolled back). The current +// datapath and userspace only read allow_out_v3 and dns_allow_v2. +func migratePersistentPolicyMaps() error { + allowSrc, err := allowOutMigrationSource() + if err != nil { + return err + } + if allowSrc != "" { + if err := migrateAllowOutMap(allowSrc); err != nil { + // Leave the legacy pin in place so the migration can be + // retried on the next restart. + return err + } + } + + if err := migrateDNSAllowMap(MapNameDNSAllow); err != nil { + // Leave the legacy allow pin in place too, so both migrations are + // retried together on the next restart (see function comment). + return err + } + + // Both migrations succeeded, so the current maps now hold the policy and + // dropping the legacy pins is pure cleanup. Make it best-effort: a failed + // unlink must not fail Init (which would roll back the freshly migrated + // maps and lose the policy); a lingering pin is simply re-migrated + // (idempotently) on the next restart. + if allowSrc != "" { + if err := removePinnedMap(allowSrc); err != nil { + log.Printf("cubevs migration: leaving legacy pin %s after successful migration: %v", allowSrc, err) + } + } + if err := removePinnedMap(MapNameDNSAllow); err != nil { + log.Printf("cubevs migration: leaving legacy pin %s after successful migration: %v", MapNameDNSAllow, err) + } + return nil +} + +// allowOutMigrationSource returns the name of the legacy allow_out pin to +// migrate from, preferring allow_out_v2 and falling back to the v0.2.0 +// allow_out pin. It returns "" when no legacy pin is present. +func allowOutMigrationSource() (string, error) { + v2Exists, err := pinnedMapExists(MapNameAllowOutV2) + if err != nil { + return "", err + } + if v2Exists { + return MapNameAllowOutV2, nil + } + legacyExists, err := pinnedMapExists(MapNameAllowOut) + if err != nil { + return "", err + } + if legacyExists { + return MapNameAllowOut, nil + } + return "", nil +} + +// removePinnedMap unlinks a bpffs pin. It is safe to call for a map whose +// inner maps are not separately pinned (they live only inside the outer +// hash-of-maps and are released by the kernel once the outer pin is gone), +// so a single-file unlink suffices. A missing pin is treated as success. +func removePinnedMap(name string) error { + if err := os.Remove(pinPath(name)); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove legacy pin %s failed: %w", name, err) + } + return nil +} + +func migrateAllowOutMap(sourceName string) error { + source, err := ebpf.LoadPinnedMap(pinPath(sourceName), nil) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil } - return fmt.Errorf("load legacy %s failed: %w", MapNameAllowOut, err) + return fmt.Errorf("load legacy %s failed: %w", sourceName, err) } - defer legacy.Close() + defer source.Close() - if err := verifyMapLayout(legacy, MapNameAllowOut, ebpf.HashOfMaps, uint32(unsafe.Sizeof(uint32(0))), uint32(unsafe.Sizeof(uint32(0)))); err != nil { + if err := verifyMapLayout(source, sourceName, ebpf.HashOfMaps, uint32(unsafe.Sizeof(uint32(0))), uint32(unsafe.Sizeof(uint32(0)))); err != nil { return err } - - current, err := loadPinnedMap(MapNameAllowOutV2) + current, err := loadPinnedMap(MapNameAllowOutV3) if err != nil { return err } @@ -36,59 +186,160 @@ func migrateAllowOutV1ToV2() error { var ifindex uint32 var innerMapID uint32 - iter := legacy.Iterate() + iter := source.Iterate() for iter.Next(&ifindex, &innerMapID) { - if err := migrateAllowOutInnerMap(current, ifindex, ebpf.MapID(innerMapID)); err != nil { + if err := migrateAllowOutInnerMap(current, ifindex, sourceName, ebpf.MapID(innerMapID)); err != nil { return err } } if err := iter.Err(); err != nil { - return fmt.Errorf("iterate legacy %s failed: %w", MapNameAllowOut, err) + return fmt.Errorf("iterate legacy %s failed: %w", sourceName, err) } return nil } -func migrateAllowOutInnerMap(current *ebpf.Map, ifindex uint32, legacyInnerID ebpf.MapID) error { - legacyInner, err := ebpf.NewMapFromID(legacyInnerID) +func migrateAllowOutInnerMap(current *ebpf.Map, ifindex uint32, sourceName string, sourceInnerID ebpf.MapID) error { + source, err := ebpf.NewMapFromID(sourceInnerID) + if err != nil { + return fmt.Errorf("open legacy %s inner map failed: %w, id: %d", sourceName, err, sourceInnerID) + } + defer source.Close() + info, err := source.Info() if err != nil { - return fmt.Errorf("open legacy %s inner map failed: %w, id: %d", MapNameAllowOut, err, legacyInnerID) + return fmt.Errorf("get legacy %s inner map info failed: %w", sourceName, err) + } + if info.Type != ebpf.LPMTrie || info.KeySize != uint32(unsafe.Sizeof(lpmKey{})) { + return fmt.Errorf("%s inner map has incompatible ABI: type=%s key_size=%d", sourceName, info.Type, info.KeySize) //nolint:err113 + } + if info.ValueSize != legacyAllowOutValueSize && + info.ValueSize != uint32(unsafe.Sizeof(netPolicyValueV2{})) { + return fmt.Errorf("%s inner map has unsupported value_size=%d", sourceName, info.ValueSize) //nolint:err113 } - defer legacyInner.Close() - if err := verifyMapLayout(legacyInner, MapNameAllowOut, ebpf.LPMTrie, uint32(unsafe.Sizeof(lpmKey{})), legacyAllowOutValueSize); err != nil { + if err := ensureAllowOutV3InnerMap(current, ifindex); err != nil { return err } + destination, err := lookupInnerMap(current, ifindex) + if err != nil { + return err + } + defer destination.Close() + + switch info.ValueSize { + case legacyAllowOutValueSize: + // v0.2.0 static allow marker: plain /32, no L7. + var key lpmKey + var oldValue uint32 + iter := source.Iterate() + for iter.Next(&key, &oldValue) { + if err := applyV3Entries(destination, buildV3Entries(key, 0, nil, 0)); err != nil { + return fmt.Errorf("update %s inner map failed: %w", MapNameAllowOutV3, err) + } + } + return wrapIterErr(iter.Err(), sourceName) + case uint32(unsafe.Sizeof(netPolicyValueV2{})): + // 16-byte legacy net_policy_value_v2: flags + expires, no ports. + var key lpmKey + var oldValue netPolicyValueV2 + iter := source.Iterate() + for iter.Next(&key, &oldValue) { + if skipUnsupportedL7Subnet(key, oldValue.Flags, sourceName) { + continue + } + if err := applyV3Entries(destination, buildV3Entries(key, oldValue.Flags, nil, oldValue.ExpiresAtNS)); err != nil { + return fmt.Errorf("update %s inner map failed: %w", MapNameAllowOutV3, err) + } + } + return wrapIterErr(iter.Err(), sourceName) + default: + return fmt.Errorf("%s inner map has unsupported value_size=%d", sourceName, info.ValueSize) //nolint:err113 + } +} - if err := ensureAllowOutV2InnerMap(current, ifindex); err != nil { +func migrateDNSAllowMap(sourceName string) error { + source, err := ebpf.LoadPinnedMap(pinPath(sourceName), nil) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("load legacy %s failed: %w", sourceName, err) + } + defer source.Close() + if err := verifyMapLayout(source, sourceName, ebpf.HashOfMaps, uint32(unsafe.Sizeof(uint32(0))), uint32(unsafe.Sizeof(uint32(0)))); err != nil { return err } - inner, err := lookupInnerMap(current, ifindex) + current, err := loadPinnedMap(MapNameDNSAllowV2) if err != nil { return err } - defer inner.Close() - - var key lpmKey - var oldValue uint32 - iter := legacyInner.Iterate() - for iter.Next(&key, &oldValue) { - value := netPolicyValueV2{} - var existing netPolicyValueV2 - if err := inner.Lookup(&key, &existing); err == nil { - value.Flags |= existing.Flags - } else if !errors.Is(err, ebpf.ErrKeyNotExist) { - return fmt.Errorf("lookup %s inner map failed: %w", MapNameAllowOutV2, err) - } - if err := inner.Update(&key, &value, ebpf.UpdateAny); err != nil { - return fmt.Errorf("update %s inner map failed: %w", MapNameAllowOutV2, err) + defer current.Close() + + var ifindex uint32 + var innerMapID uint32 + iter := source.Iterate() + for iter.Next(&ifindex, &innerMapID) { + if err := migrateDNSAllowInnerMap(current, ifindex, sourceName, ebpf.MapID(innerMapID)); err != nil { + return err } } if err := iter.Err(); err != nil { - return fmt.Errorf("iterate legacy %s inner map failed: %w", MapNameAllowOut, err) + return fmt.Errorf("iterate legacy %s failed: %w", sourceName, err) } return nil } +func migrateDNSAllowInnerMap(current *ebpf.Map, ifindex uint32, sourceName string, sourceInnerID ebpf.MapID) error { + source, err := ebpf.NewMapFromID(sourceInnerID) + if err != nil { + return fmt.Errorf("open legacy %s inner map failed: %w, id: %d", sourceName, err, sourceInnerID) + } + defer source.Close() + info, err := source.Info() + if err != nil { + return err + } + if info.Type != ebpf.LPMTrie || info.KeySize != uint32(unsafe.Sizeof(dnsAllowKey{})) { + return fmt.Errorf("%s inner map has incompatible ABI: type=%s key_size=%d", sourceName, info.Type, info.KeySize) //nolint:err113 + } + legacySize := uint32(unsafe.Sizeof(legacyDNSAllowValue{})) + currentSize := uint32(unsafe.Sizeof(dnsAllowValue{})) + if info.ValueSize != legacySize && info.ValueSize != currentSize { + return fmt.Errorf("%s inner map has unsupported value_size=%d", sourceName, info.ValueSize) //nolint:err113 + } + + if err := ensureDNSAllowInnerMap(current, ifindex); err != nil { + return err + } + destination, err := lookupInnerMap(current, ifindex) + if err != nil { + return err + } + defer destination.Close() + + if info.ValueSize == legacySize { + var key dnsAllowKey + var oldValue legacyDNSAllowValue + iter := source.Iterate() + for iter.Next(&key, &oldValue) { + value := dnsAllowValue{NameLen: oldValue.NameLen, Flags: oldValue.Flags} + if err := destination.Update(&key, &value, ebpf.UpdateAny); err != nil { + return fmt.Errorf("update %s inner map failed: %w", MapNameDNSAllowV2, err) + } + } + return wrapIterErr(iter.Err(), sourceName) + } + + var key dnsAllowKey + var value dnsAllowValue + iter := source.Iterate() + for iter.Next(&key, &value) { + if err := destination.Update(&key, &value, ebpf.UpdateAny); err != nil { + return fmt.Errorf("update %s inner map failed: %w", MapNameDNSAllowV2, err) + } + } + return wrapIterErr(iter.Err(), sourceName) +} + func verifyMapLayout(m *ebpf.Map, name string, wantType ebpf.MapType, wantKeySize, wantValueSize uint32) error { info, err := m.Info() if err != nil { diff --git a/CubeNet/cubevs/migration_test.go b/CubeNet/cubevs/migration_test.go new file mode 100644 index 000000000..493b7be10 --- /dev/null +++ b/CubeNet/cubevs/migration_test.go @@ -0,0 +1,629 @@ +package cubevs + +import ( + "strings" + "testing" + "unsafe" + + "github.com/cilium/ebpf" + "golang.org/x/sys/unix" +) + +func TestSkipUnsupportedL7Subnet(t *testing.T) { + ip := uint32(0x0a000000) // 10.0.0.0 + tests := []struct { + name string + prefixlen uint32 + flags uint8 + want bool + }{ + {name: "L7 subnet dropped", prefixlen: 24, flags: uint8(netPolicyFlagL7Required), want: true}, + {name: "L7 /32 host kept", prefixlen: 32, flags: uint8(netPolicyFlagL7Required), want: false}, + {name: "plain subnet kept (non-L7)", prefixlen: 24, flags: 0, want: false}, + {name: "plain /32 kept (non-L7)", prefixlen: 32, flags: 0, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + key := lpmKey{Prefixlen: tt.prefixlen, IP: ip} + if got := skipUnsupportedL7Subnet(key, tt.flags, "test"); got != tt.want { + t.Fatalf("skipUnsupportedL7Subnet(prefixlen=%d flags=%#x)=%v, want %v", + tt.prefixlen, tt.flags, got, tt.want) + } + }) + } +} + +// newDNSAllowOuterMapWithValueSize creates a dns_allow outer hash-of-maps whose +// inner LPM-trie template carries values of the given size (8-byte legacy or +// 40-byte current). +func newDNSAllowOuterMapWithValueSize(t *testing.T, valueSize uint32) *ebpf.Map { + t.Helper() + outer, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.HashOfMaps, + KeySize: uint32(unsafe.Sizeof(uint32(0))), + ValueSize: uint32(unsafe.Sizeof(uint32(0))), + MaxEntries: 1024, + InnerMap: &ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(dnsAllowKey{})), + ValueSize: valueSize, + MaxEntries: maxDNSAllowEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }, + }) + if err != nil { + t.Fatalf("create dns_allow outer map (value_size=%d): %v", valueSize, err) + } + t.Cleanup(func() { outer.Close() }) + return outer +} + +// newDNSAllowOuterMap creates the dns_allow_v2 outer hash-of-maps with an inner +// LPM-trie template matching the current (40-byte) dns_allow_value. +func newDNSAllowOuterMap(t *testing.T) *ebpf.Map { + return newDNSAllowOuterMapWithValueSize(t, uint32(unsafe.Sizeof(dnsAllowValue{}))) +} + +// newLPMInner creates a standalone LPM-trie inner map with the given value size. +func newLPMInner(t *testing.T, valueSize uint32) *ebpf.Map { + t.Helper() + m, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(dnsAllowKey{})), + ValueSize: valueSize, + MaxEntries: maxDNSAllowEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }) + if err != nil { + t.Fatalf("create LPM inner (value_size=%d): %v", valueSize, err) + } + t.Cleanup(func() { m.Close() }) + return m +} + +func mustInnerMapID(t *testing.T, m *ebpf.Map) ebpf.MapID { + t.Helper() + info, err := m.Info() + if err != nil { + t.Fatalf("get inner map info: %v", err) + } + id, ok := info.ID() + if !ok { + t.Fatal("inner map has no id") + } + return id +} + +// TestMigrateDNSAllowInnerMapFromLegacy drives the legacy (8-byte) dns_allow +// inner-map migration: a legacy value (NameLen + Flags, no ports) must land in +// the new dns_allow_v2 inner map as a 40-byte value with the same NameLen and +// Flags and PortCount == 0 (default 80/443 at match time). +func TestMigrateDNSAllowInnerMapFromLegacy(t *testing.T) { + ifindex := uint32(42) + current := newDNSAllowOuterMap(t) + + // Seed a legacy (8-byte) inner map with an L7 rule and a plain rule. + legacy := newLPMInner(t, uint32(unsafe.Sizeof(legacyDNSAllowValue{}))) + type entry struct { + domain string + flags uint8 + } + entries := []entry{ + {"api.example.com", uint8(netPolicyFlagL7Required)}, + {"static.example.com", 0}, + } + for _, e := range entries { + key, val, err := makeDNSAllowRule(e.domain, e.flags) + if err != nil { + t.Fatalf("makeDNSAllowRule %s: %v", e.domain, err) + } + lv := legacyDNSAllowValue{NameLen: val.NameLen, Flags: val.Flags} + if err := legacy.Update(&key, &lv, ebpf.UpdateAny); err != nil { + t.Fatalf("seed legacy entry %s: %v", e.domain, err) + } + } + + if err := migrateDNSAllowInnerMap(current, ifindex, "test-legacy", mustInnerMapID(t, legacy)); err != nil { + t.Fatalf("migrateDNSAllowInnerMap: %v", err) + } + + dest, err := lookupInnerMap(current, ifindex) + if err != nil { + t.Fatalf("lookupInnerMap: %v", err) + } + defer dest.Close() + + for _, e := range entries { + key, want, err := makeDNSAllowRule(e.domain, e.flags) + if err != nil { + t.Fatalf("makeDNSAllowRule %s: %v", e.domain, err) + } + var got dnsAllowValue + if err := dest.Lookup(&key, &got); err != nil { + t.Fatalf("migrated entry %s missing: %v", e.domain, err) + } + if got.NameLen != want.NameLen || got.Flags != want.Flags { + t.Fatalf("entry %s: got NameLen=%d Flags=%d, want NameLen=%d Flags=%d", + e.domain, got.NameLen, got.Flags, want.NameLen, want.Flags) + } + if got.PortCount != 0 { + t.Fatalf("entry %s: PortCount=%d, want 0 (legacy rules carry no ports)", e.domain, got.PortCount) + } + } +} + +// TestMigrateDNSAllowInnerMapFromCurrent drives the already-current (40-byte) +// path: an entry carrying an explicit (port, scheme) set must be copied through +// with its ports preserved. +func TestMigrateDNSAllowInnerMapFromCurrent(t *testing.T) { + ifindex := uint32(43) + current := newDNSAllowOuterMap(t) + + cur := newLPMInner(t, uint32(unsafe.Sizeof(dnsAllowValue{}))) + key, val, err := makeDNSAllowRule("api.example.com", uint8(netPolicyFlagL7Required)) + if err != nil { + t.Fatalf("makeDNSAllowRule: %v", err) + } + val.PortCount = 2 + val.Ports[0] = l7PortEntry{Port: htonsPort(8080), Scheme: L7SchemeHTTP} + val.Ports[1] = l7PortEntry{Port: htonsPort(8443), Scheme: L7SchemeHTTPS} + if err := cur.Update(&key, &val, ebpf.UpdateAny); err != nil { + t.Fatalf("seed current entry: %v", err) + } + + if err := migrateDNSAllowInnerMap(current, ifindex, "test-current", mustInnerMapID(t, cur)); err != nil { + t.Fatalf("migrateDNSAllowInnerMap: %v", err) + } + + dest, err := lookupInnerMap(current, ifindex) + if err != nil { + t.Fatalf("lookupInnerMap: %v", err) + } + defer dest.Close() + + var got dnsAllowValue + if err := dest.Lookup(&key, &got); err != nil { + t.Fatalf("migrated entry missing: %v", err) + } + if got.NameLen != val.NameLen || got.Flags != val.Flags || got.PortCount != val.PortCount { + t.Fatalf("got NameLen=%d Flags=%d PortCount=%d, want NameLen=%d Flags=%d PortCount=%d", + got.NameLen, got.Flags, got.PortCount, val.NameLen, val.Flags, val.PortCount) + } + if got.Ports[0] != val.Ports[0] || got.Ports[1] != val.Ports[1] { + t.Fatalf("ports not preserved: got %+v, want %+v", got.Ports[:2], val.Ports[:2]) + } +} + +// mountBpffs mounts a fresh bpffs at a temp dir, points bpfFSPath at it, and +// registers cleanup to restore the path and unmount. Skips when unavailable. +func mountBpffs(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := unix.Mount("bpf", dir, "bpf", 0, ""); err != nil { + t.Skipf("bpffs mount unavailable: %v", err) + } + t.Cleanup(func() { _ = unix.Unmount(dir, 0) }) + oldPath := bpfFSPath + bpfFSPath = dir + t.Cleanup(func() { bpfFSPath = oldPath }) + return dir +} + +// TestMigrateDNSAllowMapFailureKeepsLegacyPin covers the rollback path: when +// migration fails partway, the legacy source pin must be preserved so the next +// restart can retry. It pins a legacy dns_allow whose inner uses an +// unsupported 48-byte value layout (the abandoned intermediate format), so +// migrateDNSAllowInnerMap's layout check fails — then asserts +// migratePersistentPolicyMaps errors AND the legacy pin is still present. +func TestMigrateDNSAllowMapFailureKeepsLegacyPin(t *testing.T) { + mountBpffs(t) + ifindex := uint32(42) + + // Pin a legacy dns_allow outer whose inner carries an unsupported 48-byte + // value layout (abandoned intermediate format, not migratable). + badOuter := newDNSAllowOuterMapWithValueSize(t, 48) + badInner := newLPMInner(t, 48) + key, _, err := makeDNSAllowRule("api.example.com", uint8(netPolicyFlagL7Required)) + if err != nil { + t.Fatalf("makeDNSAllowRule: %v", err) + } + var raw48 [48]byte + if err := badInner.Update(&key, &raw48, ebpf.UpdateAny); err != nil { + t.Fatalf("seed 48-byte entry: %v", err) + } + if err := badOuter.Put(&ifindex, badInner); err != nil { + t.Fatalf("attach bad inner: %v", err) + } + badInner.Close() + if err := badOuter.Pin(pinPath(MapNameDNSAllow)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameDNSAllow, err) + } + + // Pin a fresh dns_allow_v2 outer as the (never-to-be-populated) destination. + newOuter := newDNSAllowOuterMap(t) + if err := newOuter.Pin(pinPath(MapNameDNSAllowV2)); err != nil { + t.Fatalf("pin %s: %v", MapNameDNSAllowV2, err) + } + + // Migration must fail on the unsupported 48-byte inner layout... + err = migratePersistentPolicyMaps() + if err == nil { + t.Fatal("migratePersistentPolicyMaps succeeded, want failure for unsupported inner value_size") + } + + // ...and the legacy source pin must be preserved for retry (rollback). + src, err := ebpf.LoadPinnedMap(pinPath(MapNameDNSAllow), nil) + if err != nil { + t.Fatalf("legacy %s pin was removed despite failed migration (rollback broken): %v", + MapNameDNSAllow, err) + } + src.Close() +} + +// TestMigrateDNSAllowMapOuterWithBpffs exercises the full outer-map migration: +// a legacy dns_allow hash-of-maps pinned on bpffs (with an 8-byte inner LPM +// trie) is migrated by migrateDNSAllowMap into a fresh dns_allow_v2 outer map, +// and the legacy pin is then removed by removePinnedMap. It mounts a fresh +// bpffs at a temp dir and points bpfFSPath at it, so it runs without touching +// the real /sys/fs/bpf. +func TestMigrateDNSAllowMapOuterWithBpffs(t *testing.T) { + mountBpffs(t) + ifindex := uint32(42) + + // Pin a legacy dns_allow outer map (8-byte inner) seeded with two rules. + legacyOuter := newDNSAllowOuterMapWithValueSize(t, uint32(unsafe.Sizeof(legacyDNSAllowValue{}))) + legacyInner := newLPMInner(t, uint32(unsafe.Sizeof(legacyDNSAllowValue{}))) + type entry struct { + domain string + flags uint8 + } + entries := []entry{ + {"api.example.com", uint8(netPolicyFlagL7Required)}, + {"static.example.com", 0}, + } + for _, e := range entries { + key, val, err := makeDNSAllowRule(e.domain, e.flags) + if err != nil { + t.Fatalf("makeDNSAllowRule %s: %v", e.domain, err) + } + lv := legacyDNSAllowValue{NameLen: val.NameLen, Flags: val.Flags} + if err := legacyInner.Update(&key, &lv, ebpf.UpdateAny); err != nil { + t.Fatalf("seed legacy entry %s: %v", e.domain, err) + } + } + if err := legacyOuter.Put(&ifindex, legacyInner); err != nil { + t.Fatalf("attach legacy inner: %v", err) + } + legacyInner.Close() // inner survives via the outer hash-of-maps reference + if err := legacyOuter.Pin(pinPath(MapNameDNSAllow)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameDNSAllow, err) + } + + // Pin a fresh dns_allow_v2 outer map (40-byte inner template) for the copy. + newOuter := newDNSAllowOuterMap(t) + if err := newOuter.Pin(pinPath(MapNameDNSAllowV2)); err != nil { + t.Fatalf("pin %s: %v", MapNameDNSAllowV2, err) + } + + // Run the real outer-map migration. + if err := migrateDNSAllowMap(MapNameDNSAllow); err != nil { + t.Fatalf("migrateDNSAllowMap(%s): %v", MapNameDNSAllow, err) + } + + // The new outer must now hold the migrated rules (NameLen+Flags, PortCount=0). + dest, err := lookupInnerMap(newOuter, ifindex) + if err != nil { + t.Fatalf("lookupInnerMap: %v", err) + } + defer dest.Close() + for _, e := range entries { + key, want, err := makeDNSAllowRule(e.domain, e.flags) + if err != nil { + t.Fatalf("makeDNSAllowRule %s: %v", e.domain, err) + } + var got dnsAllowValue + if err := dest.Lookup(&key, &got); err != nil { + t.Fatalf("migrated entry %s missing: %v", e.domain, err) + } + if got.NameLen != want.NameLen || got.Flags != want.Flags || got.PortCount != 0 { + t.Fatalf("entry %s: got %+v, want NameLen=%d Flags=%d PortCount=0", + e.domain, got, want.NameLen, want.Flags) + } + } + + // removePinnedMap must delete the legacy pin from the mounted bpffs. + if err := removePinnedMap(MapNameDNSAllow); err != nil { + t.Fatalf("removePinnedMap(%s): %v", MapNameDNSAllow, err) + } + if _, err := ebpf.LoadPinnedMap(pinPath(MapNameDNSAllow), nil); err == nil { + t.Fatalf("legacy %s pin still present after removePinnedMap", MapNameDNSAllow) + } +} + +// newAllowOutOuterMapWithValueSize creates an allow_out outer hash-of-maps +// whose inner LPM-trie template carries values of the given size. +func newAllowOutOuterMapWithValueSize(t *testing.T, valueSize uint32) *ebpf.Map { + t.Helper() + outer, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.HashOfMaps, + KeySize: uint32(unsafe.Sizeof(uint32(0))), + ValueSize: uint32(unsafe.Sizeof(uint32(0))), + MaxEntries: maxNetPolicyEntries, + InnerMap: &ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(lpmKey{})), + ValueSize: valueSize, + MaxEntries: maxNetPolicyEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }, + }) + if err != nil { + t.Fatalf("create allow_out outer map (value_size=%d): %v", valueSize, err) + } + t.Cleanup(func() { outer.Close() }) + return outer +} + +// newAllowOutLPMInner creates a standalone LPM-trie inner map keyed by lpmKey +// with the given value size. +func newAllowOutLPMInner(t *testing.T, valueSize uint32) *ebpf.Map { + t.Helper() + m, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(lpmKey{})), + ValueSize: valueSize, + MaxEntries: maxNetPolicyEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }) + if err != nil { + t.Fatalf("create allow_out LPM inner (value_size=%d): %v", valueSize, err) + } + t.Cleanup(func() { m.Close() }) + return m +} + +// newAllowOutV3OuterMap creates the current allow_out_v3 outer hash-of-maps +// with an inner template matching net_policy_value_v3. +func newAllowOutV3OuterMap(t *testing.T) *ebpf.Map { + t.Helper() + outer, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.HashOfMaps, + KeySize: uint32(unsafe.Sizeof(uint32(0))), + ValueSize: uint32(unsafe.Sizeof(uint32(0))), + MaxEntries: maxNetPolicyEntries, + InnerMap: &ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(lpmKeyV3{})), + ValueSize: uint32(unsafe.Sizeof(netPolicyValueV3{})), + MaxEntries: maxNetPolicyEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }, + }) + if err != nil { + t.Fatalf("create allow_out_v3 outer map: %v", err) + } + t.Cleanup(func() { outer.Close() }) + return outer +} + +// TestMigrateAllowOutMapFailureKeepsLegacyPin is the allow_out counterpart of +// TestMigrateDNSAllowMapFailureKeepsLegacyPin: a legacy allow_out_v2 outer +// pinned on bpffs whose inner carries the unsupported 48-byte value layout +// (the abandoned intermediate net_policy_value_v2 format) must make +// migratePersistentPolicyMaps fail, and the legacy source pin must survive +// for the next restart's retry (rollback). +func TestMigrateAllowOutMapFailureKeepsLegacyPin(t *testing.T) { + mountBpffs(t) + ifindex := uint32(42) + + // Pin a legacy allow_out_v2 outer whose inner carries an unsupported + // 48-byte value layout (abandoned intermediate format, not migratable). + badOuter := newAllowOutOuterMapWithValueSize(t, 48) + badInner := newAllowOutLPMInner(t, 48) + key := lpmKey{Prefixlen: 32, IP: mustParseCIDRForTest(t, "192.0.2.44/32").IP} + var raw48 [48]byte + if err := badInner.Update(&key, &raw48, ebpf.UpdateAny); err != nil { + t.Fatalf("seed 48-byte entry: %v", err) + } + if err := badOuter.Put(&ifindex, badInner); err != nil { + t.Fatalf("attach bad inner: %v", err) + } + badInner.Close() // inner survives via the outer hash-of-maps reference + if err := badOuter.Pin(pinPath(MapNameAllowOutV2)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameAllowOutV2, err) + } + + // Pin a fresh allow_out_v3 outer as the (never-to-be-populated) destination. + newOuter := newAllowOutV3OuterMap(t) + if err := newOuter.Pin(pinPath(MapNameAllowOutV3)); err != nil { + t.Fatalf("pin %s: %v", MapNameAllowOutV3, err) + } + + // Migration must fail on the unsupported 48-byte inner layout... + err := migratePersistentPolicyMaps() + if err == nil { + t.Fatal("migratePersistentPolicyMaps succeeded, want failure for unsupported inner value_size") + } + if !strings.Contains(err.Error(), "unsupported value_size") { + t.Fatalf("migration failed for an unexpected reason: %v", err) + } + + // ...and the legacy source pin must be preserved for retry (rollback). + src, err := ebpf.LoadPinnedMap(pinPath(MapNameAllowOutV2), nil) + if err != nil { + t.Fatalf("legacy %s pin was removed despite failed migration (rollback broken): %v", + MapNameAllowOutV2, err) + } + src.Close() +} + +// TestMigrateDNSFailureKeepsAllowOutLegacyPin covers the combined-failure +// rollback case the single-map tests miss: the allow_out migration succeeds +// but the DNS migration then fails. The legacy allow pin must survive the +// failure — if migratePersistentPolicyMaps unlinked it before the DNS +// migration ran, Init's rollback would drop the freshly populated +// allow_out_v3 and the next restart would skip re-migration (legacy pin +// gone), permanently losing the allow_out policy. +func TestMigrateDNSFailureKeepsAllowOutLegacyPin(t *testing.T) { + mountBpffs(t) + ifindex := uint32(42) + + // A good legacy allow_out_v2 (16-byte net_policy_value_v2 inner) that + // migrates cleanly. + allowOuter := newAllowOutOuterMapWithValueSize(t, uint32(unsafe.Sizeof(netPolicyValueV2{}))) + allowInner := newAllowOutLPMInner(t, uint32(unsafe.Sizeof(netPolicyValueV2{}))) + allowKey := lpmKey{Prefixlen: 32, IP: mustParseCIDRForTest(t, "192.0.2.44/32").IP} + allowVal := netPolicyValueV2{Flags: 0, ExpiresAtNS: 0} + if err := allowInner.Update(&allowKey, &allowVal, ebpf.UpdateAny); err != nil { + t.Fatalf("seed legacy allow entry: %v", err) + } + if err := allowOuter.Put(&ifindex, allowInner); err != nil { + t.Fatalf("attach legacy allow inner: %v", err) + } + allowInner.Close() + if err := allowOuter.Pin(pinPath(MapNameAllowOutV2)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameAllowOutV2, err) + } + // Destination allow_out_v3 outer. + newAllowOuter := newAllowOutV3OuterMap(t) + if err := newAllowOuter.Pin(pinPath(MapNameAllowOutV3)); err != nil { + t.Fatalf("pin %s: %v", MapNameAllowOutV3, err) + } + + // A bad legacy dns_allow whose inner carries the unsupported 48-byte value + // layout, so the DNS migration fails. + badDNSOuter := newDNSAllowOuterMapWithValueSize(t, 48) + badDNSInner := newLPMInner(t, 48) + dnsKey, _, err := makeDNSAllowRule("api.example.com", uint8(netPolicyFlagL7Required)) + if err != nil { + t.Fatalf("makeDNSAllowRule: %v", err) + } + var raw48 [48]byte + if err := badDNSInner.Update(&dnsKey, &raw48, ebpf.UpdateAny); err != nil { + t.Fatalf("seed 48-byte dns entry: %v", err) + } + if err := badDNSOuter.Put(&ifindex, badDNSInner); err != nil { + t.Fatalf("attach bad dns inner: %v", err) + } + badDNSInner.Close() + if err := badDNSOuter.Pin(pinPath(MapNameDNSAllow)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameDNSAllow, err) + } + // Destination dns_allow_v2 outer. + newDNSOuter := newDNSAllowOuterMap(t) + if err := newDNSOuter.Pin(pinPath(MapNameDNSAllowV2)); err != nil { + t.Fatalf("pin %s: %v", MapNameDNSAllowV2, err) + } + + // Migration must fail on the unsupported DNS inner layout... + err = migratePersistentPolicyMaps() + if err == nil { + t.Fatal("migratePersistentPolicyMaps succeeded, want failure for unsupported DNS inner value_size") + } + + // ...and the legacy allow pin must survive so the next restart retries + // both migrations together (this is the regression the fix addresses). + src, err := ebpf.LoadPinnedMap(pinPath(MapNameAllowOutV2), nil) + if err != nil { + t.Fatalf("legacy %s pin was removed after allow migration but DNS migration failed; "+ + "allow_out policy would be lost on restart: %v", MapNameAllowOutV2, err) + } + src.Close() +} + +// TestMigrateAllowOutMapOuterWithBpffs covers the allow_out success path the +// rollback tests miss: a legacy allow_out_v2 outer (16-byte +// net_policy_value_v2 inner, flags+expires but no ports) is migrated into a +// fresh allow_out_v3 outer. An L7 entry expands to the default {80,443} /48 +// set — keeping the L7 flag, the per-port scheme, and the expiry — while a +// plain entry becomes a /32 any-port entry. +func TestMigrateAllowOutMapOuterWithBpffs(t *testing.T) { + mountBpffs(t) + ifindex := uint32(42) + + l7IP := mustParseCIDRForTest(t, "192.0.2.44/32").IP + plainIP := mustParseCIDRForTest(t, "192.0.2.45/32").IP + expires := uint64(999999999) + + // Legacy allow_out_v2 outer with a 16-byte net_policy_value_v2 inner, + // seeded with one L7 entry and one plain entry (both with an expiry). + legacyOuter := newAllowOutOuterMapWithValueSize(t, uint32(unsafe.Sizeof(netPolicyValueV2{}))) + legacyInner := newAllowOutLPMInner(t, uint32(unsafe.Sizeof(netPolicyValueV2{}))) + seed := []struct { + ip uint32 + flags uint8 + }{ + {l7IP, uint8(netPolicyFlagL7Required)}, + {plainIP, 0}, + } + for _, s := range seed { + key := lpmKey{Prefixlen: 32, IP: s.ip} + val := netPolicyValueV2{Flags: s.flags, ExpiresAtNS: expires} + if err := legacyInner.Update(&key, &val, ebpf.UpdateAny); err != nil { + t.Fatalf("seed legacy entry: %v", err) + } + } + if err := legacyOuter.Put(&ifindex, legacyInner); err != nil { + t.Fatalf("attach legacy inner: %v", err) + } + legacyInner.Close() + if err := legacyOuter.Pin(pinPath(MapNameAllowOutV2)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameAllowOutV2, err) + } + + // Fresh allow_out_v3 outer as the destination. + newOuter := newAllowOutV3OuterMap(t) + if err := newOuter.Pin(pinPath(MapNameAllowOutV3)); err != nil { + t.Fatalf("pin %s: %v", MapNameAllowOutV3, err) + } + + if err := migrateAllowOutMap(MapNameAllowOutV2); err != nil { + t.Fatalf("migrateAllowOutMap: %v", err) + } + + dest, err := lookupInnerMap(newOuter, ifindex) + if err != nil { + t.Fatalf("lookupInnerMap: %v", err) + } + defer dest.Close() + + // The L7 entry expands to the default {80/http, 443/https} /48 set. + for _, tc := range []struct { + port uint16 + scheme uint8 + }{ + {htonsPort(80), L7SchemeHTTP}, + {htonsPort(443), L7SchemeHTTPS}, + } { + key := lpmKeyV3{Prefixlen: 48, IP: l7IP, Port: tc.port} + var got netPolicyValueV3 + if err := dest.Lookup(&key, &got); err != nil { + t.Fatalf("migrated L7 /48 entry for port %d missing: %v", ntohsPort(tc.port), err) + } + if got.Flags&uint8(netPolicyFlagL7Required) == 0 { + t.Fatalf("port %d lost L7 flag: %#x", ntohsPort(tc.port), got.Flags) + } + if got.Scheme != tc.scheme { + t.Fatalf("port %d scheme=%d, want %d", ntohsPort(tc.port), got.Scheme, tc.scheme) + } + if got.ExpiresAtNS != expires { + t.Fatalf("port %d expiry=%d, want %d (preserved)", ntohsPort(tc.port), got.ExpiresAtNS, expires) + } + } + + // The plain entry becomes a single /32 any-port entry with no L7 flag. + plainKey := lpmKeyV3{Prefixlen: 32, IP: plainIP, Port: 0} + var plain netPolicyValueV3 + if err := dest.Lookup(&plainKey, &plain); err != nil { + t.Fatalf("migrated plain /32 entry missing: %v", err) + } + if plain.Flags&uint8(netPolicyFlagL7Required) != 0 { + t.Fatalf("plain /32 has unexpected L7 flag: %#x", plain.Flags) + } + if plain.Scheme != L7SchemeNone { + t.Fatalf("plain /32 scheme=%d, want none", plain.Scheme) + } + if plain.ExpiresAtNS != expires { + t.Fatalf("plain /32 expiry=%d, want %d (preserved)", plain.ExpiresAtNS, expires) + } +} diff --git a/CubeNet/cubevs/miscs.go b/CubeNet/cubevs/miscs.go index 27545d997..6f1125bc1 100644 --- a/CubeNet/cubevs/miscs.go +++ b/CubeNet/cubevs/miscs.go @@ -3,6 +3,7 @@ package cubevs import ( "errors" "fmt" + "log" "os" "strings" @@ -13,8 +14,8 @@ import ( const ( typeNameU32 = "__u32" - typeNameLPMKey = "lpm_key" - typeNamePolicyValue = "net_policy_value_v2" + typeNameLPMKey = "lpm_key_v3" + typeNamePolicyValue = "net_policy_value_v3" typeNameDNSAllowKey = "dns_allow_key" typeNameDNSAllowValue = "dns_allow_value" ) @@ -31,8 +32,53 @@ func init() { _ = rlimit.RemoveMemlock() } +// Shipped defaults for the L7 skb->mark values. A zero Params field means +// "use the default"; deployments override via the install-time config. +const ( + defaultL7MarkHTTP = 0xCE010000 + defaultL7MarkHTTPS = 0xCE020000 + defaultL7MarkMask = 0xFFFF0000 +) + +// resolveL7Marks returns the effective L7 mark values shared by the dataplane +// and iptables, applying shipped defaults for any field left at zero and +// validating the result: http must differ from https, and both may only set +// bits inside the mask. +func resolveL7Marks(p Params) (http, https, mask uint32, err error) { + http, https, mask = p.L7MarkHTTP, p.L7MarkHTTPS, p.L7MarkMask + if http == 0 { + http = defaultL7MarkHTTP + } + if https == 0 { + https = defaultL7MarkHTTPS + } + if mask == 0 { + mask = defaultL7MarkMask + } + if http == https { + return 0, 0, 0, fmt.Errorf("l7 mark http %#x must differ from https %#x", http, https) //nolint:err113 + } + if http&^mask != 0 || https&^mask != 0 { + return 0, 0, 0, fmt.Errorf("l7 marks http %#x https %#x must set bits only within mask %#x", http, https, mask) //nolint:err113 + } + return http, https, mask, nil +} + func rewriteConstants(vars map[string]*ebpf.VariableSpec, params Params) error { var err error + l7HTTP, l7HTTPS, l7Mask, l7Err := resolveL7Marks(params) + if l7Err != nil { + return l7Err + } + if v := vars[globalNameCubeL7MarkHTTP]; v != nil { + err = errors.Join(err, v.Set(l7HTTP)) + } + if v := vars[globalNameCubeL7MarkHTTPS]; v != nil { + err = errors.Join(err, v.Set(l7HTTPS)) + } + if v := vars[globalNameCubeL7MarkMask]; v != nil { + err = errors.Join(err, v.Set(l7Mask)) + } err = errors.Join(err, vars[globalNameMVMInnerIP].Set(ipToUint32(params.MVMInnerIP))) err = errors.Join(err, vars[globalNameMVMMacaddrP1].Set(hardwareAddrToUint32(params.MVMMacAddr))) err = errors.Join(err, vars[globalNameMVMMacaddrP2].Set(hardwareAddrToUint16(params.MVMMacAddr))) @@ -226,13 +272,59 @@ func attachTCFilter(progName string, ifindex uint32, direction TCDirection) erro return nil } +// persistentPolicyGenerationExists reports whether a complete current-generation +// policy map set (both allow_out_v3 and dns_allow_v2) is pinned on bpffs. +// +// A half-pinned set (exactly one of the two) means a previous boot died between +// pinning the two maps. That orphan is an empty, not-yet-migrated map, so we +// remove it and report "no generation": Init then rebuilds a consistent pair +// and re-migrates from the legacy pins (which are still present in this +// scenario). Returning an error here instead would permanently brick startup — +// Init's recovery defer is only registered on the generationExists==false path, +// which an early error return skips, so the orphan would survive every restart. +func persistentPolicyGenerationExists() (bool, error) { + allowExists, err := pinnedMapExists(MapNameAllowOutV3) + if err != nil { + return false, err + } + dnsExists, err := pinnedMapExists(MapNameDNSAllowV2) + if err != nil { + return false, err + } + if allowExists == dnsExists { + return allowExists, nil + } + + log.Printf("cubevs: incomplete policy map generation (%s exists=%t, %s exists=%t); removing orphaned pin and rebuilding", + MapNameAllowOutV3, allowExists, MapNameDNSAllowV2, dnsExists) + if allowExists { + _ = os.Remove(pinPath(MapNameAllowOutV3)) // NOCC:Path Traversal() + } + if dnsExists { + _ = os.Remove(pinPath(MapNameDNSAllowV2)) // NOCC:Path Traversal() + } + return false, nil +} + // Init should be called once before invoking any other CubeVS APIs. -func Init(params Params) error { +func Init(params Params) (retErr error) { + generationExists, err := persistentPolicyGenerationExists() + if err != nil { + return err + } + if !generationExists { + defer func() { + if retErr != nil { + _ = os.Remove(pinPath(MapNameAllowOutV3)) // NOCC:Path Traversal() + _ = os.Remove(pinPath(MapNameDNSAllowV2)) // NOCC:Path Traversal() + } + }() + } _ = os.Remove(pinPath("tungrp_to_tuns")) // NOCC:Path Traversal() // dns_query_track is runtime pending-query state, not persisted policy. _ = os.Remove(pinPath(MapNameDNSQueryTrack)) // NOCC:Path Traversal() - err := loadObject(params, loadLocalgw, "loadLocalgw") + err = loadObject(params, loadLocalgw, "loadLocalgw") if err != nil { return err } @@ -255,8 +347,12 @@ func Init(params Params) error { return err } - if err := migrateAllowOutV1ToV2(); err != nil { - return err + if !generationExists { + if err := migratePersistentPolicyMaps(); err != nil { + // The deferred cleanup above removes the new pins on error; + // the legacy source pins are intentionally left for retry. + return err + } } // attach TC filter to cube-dev diff --git a/CubeNet/cubevs/miscs_test.go b/CubeNet/cubevs/miscs_test.go index 0f2bc8dfa..6cffb502c 100644 --- a/CubeNet/cubevs/miscs_test.go +++ b/CubeNet/cubevs/miscs_test.go @@ -1,6 +1,8 @@ package cubevs import ( + "errors" + "os" "testing" "github.com/cilium/ebpf" @@ -109,3 +111,84 @@ func TestPopulateDNSTailCallsEmptyWhenObjectDoesNotOwnDNSPrograms(t *testing.T) t.Fatalf("contents=%#v, want empty", contents) } } + +// TestPersistentPolicyGenerationRecoversHalfPinned covers the F1 availability +// regression: a boot that died between pinning allow_out_v3 and dns_allow_v2 +// leaves a half-pinned generation. persistentPolicyGenerationExists must treat +// it as an incomplete generation — remove the orphan pin and report "no +// generation" so Init rebuilds a consistent pair — rather than returning an +// error that permanently bricks startup (Init's recovery defer is only +// registered on the generationExists==false path, which an early error return +// skips). +func TestPersistentPolicyGenerationRecoversHalfPinned(t *testing.T) { + cases := []struct { + name string + pinAllow bool // pin allow_out_v3 + pinDNS bool // pin dns_allow_v2 + wantExist bool + }{ + {"neither pinned (fresh)", false, false, false}, + {"both pinned (complete)", true, true, true}, + {"only allow_out_v3 pinned (orphan)", true, false, false}, + {"only dns_allow_v2 pinned (orphan)", false, true, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mountBpffs(t) + if tc.pinAllow { + m := newAllowOutV3OuterMap(t) + if err := m.Pin(pinPath(MapNameAllowOutV3)); err != nil { + t.Fatalf("pin %s: %v", MapNameAllowOutV3, err) + } + } + if tc.pinDNS { + m := newDNSAllowOuterMap(t) + if err := m.Pin(pinPath(MapNameDNSAllowV2)); err != nil { + t.Fatalf("pin %s: %v", MapNameDNSAllowV2, err) + } + } + + exists, err := persistentPolicyGenerationExists() + if err != nil { + t.Fatalf("returned error (would brick startup): %v", err) + } + if exists != tc.wantExist { + t.Fatalf("exists=%t, want %t", exists, tc.wantExist) + } + + // After the call the two pins must be consistent: both present + // (complete) or both absent (fresh / orphan recovered). + allowExists, err := pinnedMapExists(MapNameAllowOutV3) + if err != nil { + t.Fatalf("pinnedMapExists(%s): %v", MapNameAllowOutV3, err) + } + dnsExists, err := pinnedMapExists(MapNameDNSAllowV2) + if err != nil { + t.Fatalf("pinnedMapExists(%s): %v", MapNameDNSAllowV2, err) + } + if allowExists != dnsExists { + t.Fatalf("pins still inconsistent after recovery: %s=%t %s=%t", + MapNameAllowOutV3, allowExists, MapNameDNSAllowV2, dnsExists) + } + }) + } +} + +// TestPersistentPolicyGenerationRemovesOrphanPin asserts the orphan pin is +// actually unlinked (not merely reported as absent) so the next fresh load +// rebuilds a consistent pair. +func TestPersistentPolicyGenerationRemovesOrphanPin(t *testing.T) { + mountBpffs(t) + + orphan := newAllowOutV3OuterMap(t) + if err := orphan.Pin(pinPath(MapNameAllowOutV3)); err != nil { + t.Fatalf("pin %s: %v", MapNameAllowOutV3, err) + } + + if _, err := persistentPolicyGenerationExists(); err != nil { + t.Fatalf("returned error (would brick startup): %v", err) + } + if _, err := os.Stat(pinPath(MapNameAllowOutV3)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("orphan %s pin still present after recovery: err=%v", MapNameAllowOutV3, err) + } +} diff --git a/CubeNet/cubevs/netpolicy.go b/CubeNet/cubevs/netpolicy.go index 07c6192bd..fb6099635 100644 --- a/CubeNet/cubevs/netpolicy.go +++ b/CubeNet/cubevs/netpolicy.go @@ -25,8 +25,13 @@ var alwaysDeniedSandboxCIDRs = []string{ var alwaysDeniedSandboxEntries = mustBuildDenyOutPolicyEntries(alwaysDeniedSandboxCIDRs) type allowOutPolicyEntry struct { - key lpmKey - flags uint8 + key lpmKey + flags uint8 + // ports is inherited by dns_learn_response_ip into net_policy_value_v3.ports + // when a domain rule with an explicit (port, scheme) set is learned into + // allow_out_v3. Empty for legacy port-agnostic rules (the datapath falls + // back to the default 80/443 set in that case). + ports []l7PortEntry source string } @@ -64,11 +69,19 @@ func newInnerLPMMapWithValueSize(valueSize uint32, keyType, valueType btf.Type) } func newInnerAllowOutMap() (*ebpf.Map, error) { - return newInnerLPMMapWithValueSize(uint32(unsafe.Sizeof(netPolicyValueV2{})), btfTypeLPMKey, btfTypePolicyValue) + return ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(lpmKeyV3{})), + ValueSize: uint32(unsafe.Sizeof(netPolicyValueV3{})), + MaxEntries: maxNetPolicyEntries, + Flags: unix.BPF_F_NO_PREALLOC, + Key: btfTypeLPMKey, + Value: btfTypePolicyValue, + }) } -func ensureAllowOutV2InnerMap(outerMap *ebpf.Map, ifindex uint32) error { - return ensureInnerMapWithFactory(outerMap, ifindex, MapNameAllowOutV2, newInnerAllowOutMap) +func ensureAllowOutV3InnerMap(outerMap *ebpf.Map, ifindex uint32) error { + return ensureInnerMapWithFactory(outerMap, ifindex, MapNameAllowOutV3, newInnerAllowOutMap) } func ensureDenyOutInnerMap(outerMap *ebpf.Map, ifindex uint32) error { @@ -104,16 +117,16 @@ func ensureInnerMapWithFactory(outerMap *ebpf.Map, ifindex uint32, mapName strin } // initNetPolicy creates inner LPM trie maps for the given ifindex -// in allow_out_v2, deny_out and dns_allow hash-of-maps, if not already present. +// in allow_out_v3, deny_out and dns_allow_v2 hash-of-maps, if not already present. // This should be called during AttachFilter. func initNetPolicy(ifindex uint32) error { - allowOut, err := loadPinnedMap(MapNameAllowOutV2) + allowOut, err := loadPinnedMap(MapNameAllowOutV3) if err != nil { return err } defer allowOut.Close() - err = ensureAllowOutV2InnerMap(allowOut, ifindex) + err = ensureAllowOutV3InnerMap(allowOut, ifindex) if err != nil { return err } @@ -135,14 +148,14 @@ func initNetPolicy(ifindex uint32) error { // flushInnerMap removes all entries from the inner LPM trie map // associated with the given ifindex in the outer hash-of-maps. func flushInnerMap(outerMap *ebpf.Map, ifindex uint32) error { - return flushInnerMapWithValue(outerMap, ifindex, new(uint32)) + return flushInnerMapWithValue(outerMap, ifindex, new(lpmKey), new(uint32)) } func flushAllowOutInnerMap(outerMap *ebpf.Map, ifindex uint32) error { - return flushInnerMapWithValue(outerMap, ifindex, new(netPolicyValueV2)) + return flushInnerMapWithValue(outerMap, ifindex, new(lpmKeyV3), new(netPolicyValueV3)) } -func flushInnerMapWithValue(outerMap *ebpf.Map, ifindex uint32, value any) error { +func flushInnerMapWithValue(outerMap *ebpf.Map, ifindex uint32, key, value any) error { var innerMapID uint32 err := outerMap.Lookup(&ifindex, &innerMapID) if err != nil { @@ -158,10 +171,9 @@ func flushInnerMapWithValue(outerMap *ebpf.Map, ifindex uint32, value any) error } defer inner.Close() - var key lpmKey iter := inner.Iterate() - for iter.Next(&key, value) { - if err := inner.Delete(&key); err != nil && !errors.Is(err, ebpf.ErrKeyNotExist) { + for iter.Next(key, value) { + if err := inner.Delete(key); err != nil && !errors.Is(err, ebpf.ErrKeyNotExist) { return fmt.Errorf("inner map delete failed: %w", err) } } @@ -186,10 +198,10 @@ func lookupInnerMap(outerMap *ebpf.Map, ifindex uint32) (*ebpf.Map, error) { } // cleanupNetPolicy flushes all entries in the inner LPM trie maps -// for the given ifindex in both allow_out_v2 and deny_out. +// for the given ifindex in both allow_out_v3 and deny_out. // This should be called during DeleteTAPDevice. func cleanupNetPolicy(ifindex uint32) error { - allowOut, err := loadPinnedMap(MapNameAllowOutV2) + allowOut, err := loadPinnedMap(MapNameAllowOutV3) if err != nil { return err } @@ -197,7 +209,7 @@ func cleanupNetPolicy(ifindex uint32) error { err = flushAllowOutInnerMap(allowOut, ifindex) if err != nil { - return fmt.Errorf("flush %s failed: %w", MapNameAllowOutV2, err) + return fmt.Errorf("flush %s failed: %w", MapNameAllowOutV3, err) } denyOut, err := loadPinnedMap(MapNameDenyOut) @@ -289,37 +301,255 @@ func mustBuildDenyOutPolicyEntries(cidrs []string) []denyOutPolicyEntry { return entries } -func buildAllowOutPolicyEntries(allowOutCIDRs, l7AllowOutCIDRs []string) ([]allowOutPolicyEntry, error) { - entries := make([]allowOutPolicyEntry, 0, len(allowOutCIDRs)+len(l7AllowOutCIDRs)) - indexByKey := make(map[lpmKey]int, len(allowOutCIDRs)+len(l7AllowOutCIDRs)) +func buildAllowOutPolicyEntries(allowOutCIDRs []string) ([]allowOutPolicyEntry, error) { + entries := make([]allowOutPolicyEntry, 0, len(allowOutCIDRs)) + indexByKey := make(map[lpmKey]int, len(allowOutCIDRs)) - add := func(cidrs []string, flags uint8) error { - for _, cidr := range cidrs { - key, err := parseCIDR(cidr) - if err != nil { - return err + for _, cidr := range allowOutCIDRs { + key, err := parseCIDR(cidr) + if err != nil { + return nil, err + } + if _, ok := indexByKey[key]; ok { + continue + } + indexByKey[key] = len(entries) + entries = append(entries, allowOutPolicyEntry{ + key: key, + source: cidr, + }) + } + return entries, nil +} + +// ntohsPort converts a network-order uint16 back to a host-order integer for +// use in error messages. eBPF stores ports in NBO to avoid conversions on hot +// paths; users see host order in their config. +func ntohsPort(p uint16) uint16 { + return (p>>8)&0xff | (p<<8)&0xff00 +} + +// htonsPort converts host-order integer to network byte order for storage +// alongside the eBPF datapath's tcphdr->dest comparisons. +func htonsPort(p uint16) uint16 { + return (p>>8)&0xff | (p<<8)&0xff00 +} + +// expandDefaultPortSet returns the (port, scheme) tuples applied when a user +// rule omits both port and scheme: the classic {80/http, 443/https}. Kept as a +// helper so the datapath fallback in mvmtap.bpf.c (port_count==0 branch) and +// the userspace expansion stay in lockstep. +func expandDefaultPortSet() []l7PortEntry { + return []l7PortEntry{ + {Port: htonsPort(80), Scheme: L7SchemeHTTP}, + {Port: htonsPort(443), Scheme: L7SchemeHTTPS}, + } +} + +// l7TargetKind separates L7Target hosts into the two datapath maps: CIDR +// literals go to allow_out_v3 as static entries; domain names go to dns_allow_v2 +// so their (port, scheme) set follows learned IPs into allow_out_v3 at +// response time. +type l7TargetKind int + +const ( + l7KindCIDR l7TargetKind = iota + l7KindDomain +) + +func classifyL7Target(host string) (l7TargetKind, error) { + if isIPv4Target(host) { + // An L7 host must be a single host, not a subnet CIDR: the datapath + // matches exact (ip, port)/48 pairs and cannot express a subnet+port + // rule, so reject network blocks here instead of silently narrowing + // them to the network address downstream. + key, err := parseCIDR(host) + if err != nil { + return 0, err + } + if key.Prefixlen < 32 { + return 0, fmt.Errorf("invalid l7_allow_out host %s: subnet CIDR not supported for L7 rules, use a single host IP or a domain name", host) //nolint:err113 + } + return l7KindCIDR, nil + } + if strings.Contains(host, "/") { + return 0, fmt.Errorf("invalid l7_allow_out CIDR target: %s", host) //nolint:err113 + } + if net.ParseIP(host) != nil || isDottedDecimalLikeTarget(host) { + return 0, fmt.Errorf("unsupported l7_allow_out IP target: %s", host) //nolint:err113 + } + if !isDNSAllowTarget(host) { + return 0, fmt.Errorf("invalid l7_allow_out domain target: %s", host) //nolint:err113 + } + return l7KindDomain, nil +} + +// l7GroupKey returns a canonical grouping key for an L7 target host. Raw +// variants that resolve to the same datapath key — DNS names differing only by +// case or a trailing dot, CIDRs differing only by notation (e.g. "1.2.3.4" vs +// "1.2.3.4/32") — map to one key so buildL7Plan aggregates their port sets and +// detects (host, port) scheme conflicts, instead of letting them collide +// later in a last-write-wins same-key merge that silently drops ports and +// bypasses conflict detection. +func l7GroupKey(host string) (string, error) { + kind, err := classifyL7Target(host) + if err != nil { + return "", err + } + if kind == l7KindCIDR { + key, err := parseCIDR(host) + if err != nil { + return "", err + } + return fmt.Sprintf("cidr:%d:%08x", key.Prefixlen, key.IP), nil + } + // Match makeDNSAllowRule's case / trailing-dot normalisation. + return "dns:" + strings.ToLower(strings.TrimSuffix(host, ".")), nil +} + +// buildL7Plan merges rule targets by host, enforces (host, port) scheme +// consistency, and produces: +// - allow_out_v3 static entries for IP/CIDR hosts (with port_count / ports +// inherited from the merged tuple set), +// - dns_allow_v2 rules for domain hosts (with the same port set). +// +// A target with Port == 0 && Scheme == L7SchemeNone means "unspecified": it +// expands to {80/http, 443/https} for merging purposes. Empty port_count is +// only preserved when *every* rule for that host is unspecified — the moment a +// user attaches an explicit (port, scheme) the port set becomes explicit and +// the datapath skips the default-set fallback. +func buildL7Plan(targets []L7Target) (l7CIDRs []allowOutPolicyEntry, + l7DNS []dnsAllowRule, err error) { + + // Group by canonical host key (see l7GroupKey), not the raw host string, + // so that raw variants resolving to the same datapath key aggregate into + // one host. Track scheme per port to detect conflicts early. + type hostState struct { + raw string // first-seen raw host, used for output + errors + portScheme map[uint16]uint8 // key: NBO port, value: scheme + portOrder []uint16 // first appearance order for deterministic overflow + anyExplicit bool + } + byHost := make(map[string]*hostState, len(targets)) + hostOrder := make([]string, 0, len(targets)) + + for _, tgt := range targets { + host := tgt.Host + gkey, gerr := l7GroupKey(host) + if gerr != nil { + return nil, nil, gerr + } + st, ok := byHost[gkey] + if !ok { + st = &hostState{raw: host, portScheme: make(map[uint16]uint8)} + byHost[gkey] = st + hostOrder = append(hostOrder, gkey) + } + if tgt.Port == 0 && tgt.Scheme == L7SchemeNone { + // Unspecified rule — expand to default set for conflict checking. + for _, def := range expandDefaultPortSet() { + if existing, ok := st.portScheme[def.Port]; ok { + if existing != def.Scheme { + return nil, nil, fmt.Errorf( + "l7 rule conflict for %s port %d: scheme %d vs %d (via default set)", //nolint:err113 + host, ntohsPort(def.Port), existing, def.Scheme) + } + } else { + st.portOrder = append(st.portOrder, def.Port) + } + st.portScheme[def.Port] = def.Scheme } - if idx, ok := indexByKey[key]; ok { - entries[idx].flags |= flags - continue + continue + } + if tgt.Port == 0 || (tgt.Scheme != L7SchemeHTTP && tgt.Scheme != L7SchemeHTTPS) { + return nil, nil, fmt.Errorf( + "invalid l7 target %s: port and scheme must both be set (port=%d scheme=%d)", //nolint:err113 + host, tgt.Port, tgt.Scheme) + } + port := htonsPort(tgt.Port) + if existing, ok := st.portScheme[port]; ok { + if existing != tgt.Scheme { + return nil, nil, fmt.Errorf( + "l7 rule conflict for %s port %d: scheme %d vs %d", //nolint:err113 + host, tgt.Port, existing, tgt.Scheme) } - indexByKey[key] = len(entries) - entries = append(entries, allowOutPolicyEntry{ + } else { + st.portOrder = append(st.portOrder, port) + } + st.portScheme[port] = tgt.Scheme + st.anyExplicit = true + } + + // Materialise per-host port entries. Order host iteration to keep output + // deterministic (test assertions compare slices directly). hostOrder holds + // canonical group keys; the first-seen raw host drives classification and + // key construction (it resolves to the same datapath key as the group). + for _, gkey := range hostOrder { + st := byHost[gkey] + host := st.raw + var ports []l7PortEntry + if st.anyExplicit || len(st.portScheme) != len(expandDefaultPortSet()) { + // Emit the full port list when an explicit (port, scheme) rule is + // attached. The second disjunct is defensive: it is unreachable + // today (a pure-default host always has exactly the default set's + // two tuples) but guards against expandDefaultPortSet() changing + // cardinality, which would otherwise silently drop a tuple here. + ports = make([]l7PortEntry, 0, len(st.portOrder)) + for _, p := range st.portOrder { + ports = append(ports, l7PortEntry{Port: p, Scheme: st.portScheme[p]}) + } + if len(ports) > maxL7PortsPerHost { + return nil, nil, fmt.Errorf( + "l7 rule exceeds %d port tuples for %s", //nolint:err113 + maxL7PortsPerHost, host) + } + } + // ports == nil for pure-default hosts: signal "use default set" to the + // datapath (port_count = 0). + + kind, cerr := classifyL7Target(host) + if cerr != nil { + return nil, nil, cerr + } + switch kind { + case l7KindCIDR: + key, perr := parseCIDR(host) + if perr != nil { + return nil, nil, perr + } + l7CIDRs = append(l7CIDRs, allowOutPolicyEntry{ + key: key, + flags: uint8(netPolicyFlagL7Required), + ports: ports, + source: host, + }) + case l7KindDomain: + key, value, merr := makeDNSAllowRule(host, uint8(netPolicyFlagL7Required)) + if merr != nil { + return nil, nil, merr + } + applyPortsToDNSAllowValue(&value, ports) + l7DNS = append(l7DNS, dnsAllowRule{ key: key, - flags: flags, - source: cidr, + value: value, + domain: host, }) } - return nil } + return l7CIDRs, l7DNS, nil +} - if err := add(allowOutCIDRs, 0); err != nil { - return nil, err +// applyPortsToDNSAllowValue copies ports into the DNS allow value. len(ports)==0 +// leaves PortCount=0 so the datapath falls back to {80,443} at match time. +func applyPortsToDNSAllowValue(v *dnsAllowValue, ports []l7PortEntry) { + if len(ports) == 0 { + v.PortCount = 0 + return } - if err := add(l7AllowOutCIDRs, uint8(netPolicyFlagL7Required)); err != nil { - return nil, err + v.PortCount = uint8(len(ports)) + for i, p := range ports { + v.Ports[i] = p } - return entries, nil } func buildDenyOutPolicyEntries(cidrs []string) ([]denyOutPolicyEntry, error) { @@ -352,23 +582,35 @@ func buildNetPolicyPlan(opts MVMOptions) (*netPolicyPlan, error) { return nil, err } - var l7AllowOut []string + var l7Targets []L7Target if opts.L7AllowOut != nil { - l7AllowOut = *opts.L7AllowOut + l7Targets = *opts.L7AllowOut } - l7AllowOutCIDRs, l7DNSAllowDomains, err := splitAllowOutTargets(l7AllowOut) + l7CIDRs, l7DNSRules, err := buildL7Plan(l7Targets) if err != nil { return nil, err } - allowOutEntries, err := buildAllowOutPolicyEntries(allowOutCIDRs, l7AllowOutCIDRs) + baseAllowOutEntries, err := buildAllowOutPolicyEntries(allowOutCIDRs) if err != nil { return nil, err } - dnsAllowRules, err := buildDNSAllowRules(dnsAllowDomains, l7DNSAllowDomains) + // Merge non-L7 allow_out CIDRs with L7 CIDRs. If the same key appears in + // both, keep the L7 entry (its flags + ports are strictly a superset). + allowOutEntries := mergeAllowOutWithL7(baseAllowOutEntries, l7CIDRs) + + // Domain rules: non-L7 domains get flags=0 and no port set; L7 domain + // rules already carry flags + ports from buildL7Plan. Merge by key. + dnsAllowRules, err := buildDNSAllowRules(dnsAllowDomains) if err != nil { return nil, err } + dnsAllowRules = mergeDNSAllowRules(dnsAllowRules, l7DNSRules) + // Mark each L7 domain rule that is also covered by a plain (non-L7) + // allow_out domain — an exact same-host entry or a leading-"*." wildcard — + // so the datapath keeps the plain /32 L3 access alongside the L7 /48 + // interception for that host. + markL3AllowedByPlainCover(dnsAllowRules, dnsAllowDomains) var denyOutEntries []denyOutPolicyEntry if opts.AllowInternetAccess != nil && !*opts.AllowInternetAccess { @@ -389,7 +631,7 @@ func buildNetPolicyPlan(opts MVMOptions) (*netPolicyPlan, error) { allowOutEntries: allowOutEntries, dnsAllowRules: dnsAllowRules, denyOutEntries: denyOutEntries, - dnsPolicyFlags: dnsPolicyFlagsForDomains(dnsAllowDomains, l7DNSAllowDomains), + dnsPolicyFlags: dnsPolicyFlagsForDomains(dnsAllowDomains, l7DNSDomainNames(l7DNSRules)), } if err := validateNetPolicyPlan(plan); err != nil { return nil, err @@ -397,6 +639,119 @@ func buildNetPolicyPlan(opts MVMOptions) (*netPolicyPlan, error) { return plan, nil } +// l7DNSDomainNames pulls the raw domain string out of each L7 dns rule so +// dnsPolicyFlagsForDomains can decide whether to enable DNS learning. Only the +// count matters — the caller only tests len>0. +func l7DNSDomainNames(rules []dnsAllowRule) []string { + out := make([]string, 0, len(rules)) + for _, r := range rules { + out = append(out, r.domain) + } + return out +} + +// mergeAllowOutWithL7 combines the non-L7 base entries with the L7 entries. +// When the same LPM key appears in both, the L7 version's flags and port set +// win, and the merged entry is also marked netPolicyFlagL3Allowed so +// populateAllowOutInnerMap writes the plain /32 any-port entry alongside the +// L7 /48 entries — otherwise an L7 rule would silently narrow a same-host +// plain allow_out to only the rule's ports. (Domain hosts get the same +// treatment in mergeDNSAllowRules.) +func mergeAllowOutWithL7(base, l7 []allowOutPolicyEntry) []allowOutPolicyEntry { + if len(l7) == 0 { + return base + } + byKey := make(map[lpmKey]int, len(base)+len(l7)) + out := make([]allowOutPolicyEntry, 0, len(base)+len(l7)) + for _, e := range base { + byKey[e.key] = len(out) + out = append(out, e) + } + for _, e := range l7 { + if idx, ok := byKey[e.key]; ok { + out[idx].flags |= e.flags | netPolicyFlagL3Allowed + out[idx].ports = e.ports + continue + } + byKey[e.key] = len(out) + out = append(out, e) + } + return out +} + +// mergeDNSAllowRules combines the non-L7 domain rules with the L7 domain rules. +// Same-key entries merge flags (OR) and adopt the L7 rule's port set. The +// netPolicyFlagL3Allowed marker (plain-L3 + L7 coexistence) is applied +// separately by markL3AllowedByPlainCover, which handles both the exact +// same-host case and a leading-"*." wildcard allow_out covering the L7 host. +func mergeDNSAllowRules(base, l7 []dnsAllowRule) []dnsAllowRule { + if len(l7) == 0 { + return base + } + byKey := make(map[dnsAllowKey]int, len(base)+len(l7)) + out := make([]dnsAllowRule, 0, len(base)+len(l7)) + for _, r := range base { + byKey[r.key] = len(out) + out = append(out, r) + } + for _, r := range l7 { + if idx, ok := byKey[r.key]; ok { + out[idx].value.Flags |= r.value.Flags + out[idx].value.PortCount = r.value.PortCount + out[idx].value.Ports = r.value.Ports + continue + } + byKey[r.key] = len(out) + out = append(out, r) + } + return out +} + +// l7DomainHasPlainCover reports whether an L7 rule host is covered by a plain +// (non-L7) allow_out domain — either an exact same-host entry or a +// leading-"*." wildcard whose base the host is a subdomain of. Matching follows +// the same semantics as makeDNSAllowRule / domain_match: case-insensitive, a +// trailing dot is ignored, and "*.base" covers any-depth subdomains of base but +// not the apex itself. +func l7DomainHasPlainCover(host string, plainDomains []string) bool { + h := strings.ToLower(strings.TrimSuffix(host, ".")) + for _, raw := range plainDomains { + d := strings.ToLower(strings.TrimSuffix(raw, ".")) + if strings.HasPrefix(d, "*.") { + base := d[2:] + if h != base && strings.HasSuffix(h, "."+base) { + return true + } + continue + } + if h == d { + return true + } + } + return false +} + +// markL3AllowedByPlainCover sets netPolicyFlagL3Allowed on each L7 dns_allow +// rule whose host is also covered by a plain (non-L7) allow_out domain. This +// is what lets a host keep its plain /32 L3 access (SNAT on non-rule ports) +// alongside the L7 /48 interception on the rule's ports — both when the host +// appears verbatim in allow_out and when a leading-"*." wildcard covers it +// (the exact rule otherwise shadows the wildcard in the DNS LPM match, which +// would silently deny the host's non-rule ports under deny-all). +func markL3AllowedByPlainCover(rules []dnsAllowRule, plainDomains []string) { + for i := range rules { + if rules[i].value.Flags&uint8(netPolicyFlagL7Required) == 0 { + continue + } + if rules[i].value.Flags&uint8(netPolicyFlagL3Allowed) != 0 { + continue + } + if l7DomainHasPlainCover(rules[i].domain, plainDomains) { + rules[i].value.Flags |= uint8(netPolicyFlagL3Allowed) + } + } +} + func appendDenyOutPolicyEntries(dst, src []denyOutPolicyEntry) []denyOutPolicyEntry { if len(src) == 0 { return dst @@ -424,8 +779,36 @@ func effectiveDenyOutEntriesForReplace(plan *netPolicyPlan) []denyOutPolicyEntry return appendDenyOutPolicyEntries(entries, alwaysDeniedSandboxEntries) } +// expandedAllowOutEntryCount returns the number of allow_out_v3 inner-map +// entries a plan actually occupies: one per plain (non-L7) allow, and +// len(ports) per L7 rule — or the default {80, 443} set when the L7 rule has +// no explicit ports. The inner LPM trie is bounded by maxNetPolicyEntries, so +// budget validation must use this expanded count, not len(allowOutEntries) +// (which counts one per host and undercounts multi-port L7 rules, letting +// population overflow mid-write with E2BIG and leave a half-populated map). +func expandedAllowOutEntryCount(entries []allowOutPolicyEntry) int { + defaultPorts := len(expandDefaultPortSet()) + total := 0 + for _, e := range entries { + if e.flags&netPolicyFlagL7Required != 0 { + n := len(e.ports) + if n == 0 { + n = defaultPorts + } + if e.flags&netPolicyFlagL3Allowed != 0 { + // The host also gets a plain /32 any-port entry (coexistence). + n++ + } + total += n + continue + } + total++ + } + return total +} + func validateNetPolicyPlan(plan *netPolicyPlan) error { - if err := validateNetPolicyEntryCount("network.allow_out_v2", len(plan.allowOutEntries), maxNetPolicyEntries); err != nil { + if err := validateNetPolicyEntryCount("network.allow_out_v3", expandedAllowOutEntryCount(plan.allowOutEntries), maxNetPolicyEntries); err != nil { return err } if err := validateNetPolicyEntryCount("network.dns_allow", len(plan.dnsAllowRules), maxDNSAllowDomains); err != nil { @@ -434,6 +817,56 @@ func validateNetPolicyPlan(plan *netPolicyPlan) error { return validateNetPolicyEntryCount("network.deny_out", len(effectiveDenyOutEntriesForReplace(plan)), maxNetPolicyEntries) } +func validateNetPolicyEntryCounts(allowOutCIDRs, l7AllowOutCIDRs, dnsAllowDomains, l7DNSAllowDomains, denyOut []string) error { + if count, err := countUniqueLPMEntries(allowOutCIDRs, l7AllowOutCIDRs); err != nil { + return err + } else if err := validateNetPolicyEntryCount("network.allow_out_v3", count, maxNetPolicyEntries); err != nil { + return err + } + + if count, err := countUniqueDNSAllowEntries(dnsAllowDomains, l7DNSAllowDomains); err != nil { + return err + } else if err := validateNetPolicyEntryCount("network.dns_allow", count, maxDNSAllowDomains); err != nil { + return err + } + + if count, err := countUniqueLPMEntries(denyOut); err != nil { + return err + } else if err := validateNetPolicyEntryCount("network.deny_out", count, maxNetPolicyEntries); err != nil { + return err + } + + return nil +} + +func countUniqueLPMEntries(groups ...[]string) (int, error) { + seen := make(map[lpmKey]struct{}) + for _, group := range groups { + for _, cidr := range group { + key, err := parseCIDR(cidr) + if err != nil { + return 0, err + } + seen[key] = struct{}{} + } + } + return len(seen), nil +} + +func countUniqueDNSAllowEntries(groups ...[]string) (int, error) { + seen := make(map[dnsAllowKey]struct{}) + for _, group := range groups { + for _, domain := range group { + key, _, err := makeDNSAllowRule(domain, 0) + if err != nil { + return 0, err + } + seen[key] = struct{}{} + } + } + return len(seen), nil +} + func validateNetPolicyEntryCount(field string, count int, maxEntries int) error { if count <= maxEntries { return nil @@ -474,7 +907,7 @@ func setDNSPolicyFlags(ifindex uint32, flags uint8) error { } // splitAllowOutTargets separates user-facing allow_out targets into IPv4/CIDR -// entries for allow_out_v2 and DNS names for dns_allow. +// entries for allow_out_v3 and DNS names for dns_allow_v2. func splitAllowOutTargets(targets []string) ([]string, []string, error) { cidrs := make([]string, 0, len(targets)) domains := make([]string, 0, len(targets)) @@ -585,7 +1018,18 @@ func populateInnerMap(outerMap *ebpf.Map, ifindex uint32, entries []denyOutPolic return nil } -// populateAllowOutInnerMap inserts pre-parsed static allow_out_v2 entries. +// populateAllowOutInnerMap inserts pre-parsed static allow_out_v3 entries. +// +// v3 carries the port in the LPM key, so an L7 entry (flags & +// netPolicyFlagL7Required) is materialised as one exact (ip, port)/48 entry +// per (port, scheme) tuple — a default port set expands to {80/http, +// 443/https}. A plain allow is a single ip-only (or subnet) /32 entry +// with scheme = NONE. When a static (ip, port) key already holds a +// DNS-learned entry, the learned flags are merged in but the static (zero) +// expiry wins — the entry becomes permanent. Keeping the learned TTL +// instead would let the reaper delete the entry at the old TTL and +// silently drop the static verdict; a later DNS refresh preserves the +// static zero expiry (dns_response.h same-key rule). func populateAllowOutInnerMap(outerMap *ebpf.Map, ifindex uint32, entries []allowOutPolicyEntry) error { var innerMapID uint32 err := outerMap.Lookup(&ifindex, &innerMapID) @@ -600,26 +1044,69 @@ func populateAllowOutInnerMap(outerMap *ebpf.Map, ifindex uint32, entries []allo defer inner.Close() for _, entry := range entries { - val := netPolicyValueV2{Flags: entry.flags} - var oldVal netPolicyValueV2 - if err := inner.Lookup(&entry.key, &oldVal); err == nil { - // Static allow entries never expire, but they must preserve existing flags. - val.Flags |= oldVal.Flags - } else if !errors.Is(err, ebpf.ErrKeyNotExist) { - return fmt.Errorf("inner map lookup failed: %w, cidr: %s", err, entry.source) + if entry.flags&netPolicyFlagL7Required != 0 { + ports := entry.ports + if len(ports) == 0 { + ports = expandDefaultPortSet() + } + for _, p := range ports { + key := lpmKeyV3{Prefixlen: 48, IP: entry.key.IP, Port: p.Port} + val := netPolicyValueV3{Flags: entry.flags, Scheme: p.Scheme, KeyPrefixlen: 48} + var oldVal netPolicyValueV3 + lerr := inner.Lookup(&key, &oldVal) + switch { + case lerr == nil: + // LPM lookup is longest-prefix: only merge with + // an entry written under the EXACT same key, + // never with a shorter covering entry (whose + // flags would otherwise leak into this /48). + // Flags only: the static (zero) expiry wins over + // a learned same-key entry, so the entry becomes + // permanent rather than ageing out at the old TTL. + if oldVal.KeyPrefixlen == uint8(key.Prefixlen) { + val.Flags |= oldVal.Flags + } + case !errors.Is(lerr, ebpf.ErrKeyNotExist): + return fmt.Errorf("inner map lookup failed: %w, cidr: %s", lerr, entry.source) + } + if uerr := inner.Update(&key, &val, ebpf.UpdateAny); uerr != nil { + return fmt.Errorf("inner map update failed: %w, cidr: %s", uerr, entry.source) + } + } + + // Coexistence: this host is also in plain allow_out, so write a + // plain /32 any-port entry alongside the /48 L7 entries. The /48 + // (longest-prefix) match wins for the rule's ports; the /32 covers + // all other ports via plain SNAT. Strip the L7/L3 marker bits so + // the entry reads as a plain allow. + if entry.flags&netPolicyFlagL3Allowed != 0 { + plainKey := lpmKeyV3{Prefixlen: entry.key.Prefixlen, IP: entry.key.IP, Port: 0} + plainVal := netPolicyValueV3{ + Flags: entry.flags &^ (netPolicyFlagL7Required | netPolicyFlagL3Allowed), + Scheme: L7SchemeNone, + KeyPrefixlen: uint8(plainKey.Prefixlen), + } + if uerr := inner.Update(&plainKey, &plainVal, ebpf.UpdateAny); uerr != nil { + return fmt.Errorf("inner map update failed: %w, cidr: %s", uerr, entry.source) + } + } + continue } - err = inner.Update(&entry.key, &val, ebpf.UpdateAny) - if err != nil { - return fmt.Errorf("inner map update failed: %w, cidr: %s", err, entry.source) + // Plain allow: ip-only / subnet key with port = 0, scheme = NONE. + key := lpmKeyV3{Prefixlen: entry.key.Prefixlen, IP: entry.key.IP, Port: 0} + val := netPolicyValueV3{Flags: entry.flags, KeyPrefixlen: uint8(key.Prefixlen)} + if uerr := inner.Update(&key, &val, ebpf.UpdateAny); uerr != nil { + return fmt.Errorf("inner map update failed: %w, cidr: %s", uerr, entry.source) } } return nil } -// netPolicyValueV2Expired reports whether a v2 allow entry is a dynamic entry -// whose DNS-learned TTL has expired. Static entries have ExpiresAtNS set to 0. -func netPolicyValueV2Expired(value netPolicyValueV2, now uint64) bool { +// netPolicyValueV3Expired reports whether a v3 allow entry is a dynamic +// entry whose DNS-learned TTL has expired. Static entries have ExpiresAtNS +// set to 0. +func netPolicyValueV3Expired(value netPolicyValueV3, now uint64) bool { return value.ExpiresAtNS != 0 && value.ExpiresAtNS <= now } @@ -627,10 +1114,10 @@ func netPolicyValueV2Expired(value netPolicyValueV2, now uint64) bool { // based on MVMOptions. // // Rules: -// - AllowOut IP/CIDR targets are inserted into allow_out_v2 inner map. -// - L7AllowOut IP/CIDR targets are inserted into allow_out_v2 with the L7 flag. -// - AllowOut domain targets are inserted into dns_allow inner map. -// - L7AllowOut domain targets are inserted into dns_allow with the L7 flag. +// - AllowOut IP/CIDR targets are inserted into allow_out_v3 inner map. +// - L7AllowOut IP/CIDR targets are inserted into allow_out_v3 with the L7 flag. +// - AllowOut domain targets are inserted into dns_allow_v2 inner map. +// - L7AllowOut domain targets are inserted into dns_allow_v2 with the L7 flag. // - Default private/link-local DenyOut ranges are preloaded when a TAP enters // the free pool. Replace paths replay them after flushing policy maps. // - AllowInternetAccess=false: DenyOut is set to "0.0.0.0/0" (deny all). @@ -640,7 +1127,7 @@ func applyNetPolicy(ifindex uint32, opts MVMOptions) error { // replaceNetPolicy replaces all configured egress policy for an ifindex. // It is used by TAP upsert/recovery paths so removed policy entries do not -// survive cubelet network runtime restart/recovery. +// survive a network-agent restart. func replaceNetPolicy(ifindex uint32, opts MVMOptions) error { return applyNetPolicyWithMode(ifindex, opts, true) } @@ -652,27 +1139,27 @@ func applyNetPolicyWithMode(ifindex uint32, opts MVMOptions, replace bool) error } if replace || len(plan.allowOutEntries) > 0 { - allowOutMap, err := loadPinnedMap(MapNameAllowOutV2) + allowOutMap, err := loadPinnedMap(MapNameAllowOutV3) if err != nil { return err } defer allowOutMap.Close() - if err := ensureAllowOutV2InnerMap(allowOutMap, ifindex); err != nil { + if err := ensureAllowOutV3InnerMap(allowOutMap, ifindex); err != nil { return err } if replace { if err := flushAllowOutInnerMap(allowOutMap, ifindex); err != nil { - return fmt.Errorf("flush %s failed: %w", MapNameAllowOutV2, err) + return fmt.Errorf("flush %s failed: %w", MapNameAllowOutV3, err) } } err = populateAllowOutInnerMap(allowOutMap, ifindex, plan.allowOutEntries) if err != nil { - return fmt.Errorf("populate %s failed: %w", MapNameAllowOutV2, err) + return fmt.Errorf("populate %s failed: %w", MapNameAllowOutV3, err) } } if err := applyDNSAllow(ifindex, plan.dnsAllowRules, replace); err != nil { - return fmt.Errorf("populate %s failed: %w", MapNameDNSAllow, err) + return fmt.Errorf("populate %s failed: %w", MapNameDNSAllowV2, err) } denyOutEntries := plan.denyOutEntries diff --git a/CubeNet/cubevs/netpolicy_test.go b/CubeNet/cubevs/netpolicy_test.go index 8b47e7955..ffa615aeb 100644 --- a/CubeNet/cubevs/netpolicy_test.go +++ b/CubeNet/cubevs/netpolicy_test.go @@ -3,8 +3,12 @@ package cubevs import ( "fmt" "reflect" + "strings" "testing" "unsafe" + + "github.com/cilium/ebpf" + "golang.org/x/sys/unix" ) func TestSplitAllowOutTargets(t *testing.T) { @@ -50,7 +54,7 @@ func TestSplitAllowOutTargetsRejectsInvalidTargets(t *testing.T) { } } -func TestBuildNetPolicyPlanValidatesFinalMapTargets(t *testing.T) { +func TestValidateNetPolicyEntryCountsUsesFinalMapTargets(t *testing.T) { allowOutCIDRs := repeatedCIDRs(maxNetPolicyEntries) l7AllowOutCIDRs := []string{"198.51.100.1"} dnsAllowDomains := repeatedDomains(maxDNSAllowDomains) @@ -59,84 +63,75 @@ func TestBuildNetPolicyPlanValidatesFinalMapTargets(t *testing.T) { tests := []struct { name string - opts MVMOptions + err error want string }{ { - name: "allow out v2 counts allow and l7 cidrs", - opts: MVMOptions{AllowOut: &allowOutCIDRs, L7AllowOut: &l7AllowOutCIDRs}, - want: "network.allow_out_v2 exceeds maximum entries: got 8193, max 8192", + name: "allow out v3 counts allow and l7 cidrs", + err: validateNetPolicyEntryCounts(allowOutCIDRs, l7AllowOutCIDRs, nil, nil, nil), + want: "network.allow_out_v3 exceeds maximum entries: got 8193, max 8192", }, { name: "dns allow counts allow and l7 domains", - opts: MVMOptions{AllowOut: &dnsAllowDomains, L7AllowOut: &l7DNSAllowDomains}, + err: validateNetPolicyEntryCounts(nil, nil, dnsAllowDomains, l7DNSAllowDomains, nil), want: "network.dns_allow exceeds maximum entries: got 1025, max 1024", }, { - name: "deny out counts effective deny cidrs on replace", - opts: MVMOptions{DenyOut: &denyOut}, + name: "deny out counts effective deny cidrs", + err: validateNetPolicyEntryCounts(nil, nil, nil, nil, denyOut), want: "network.deny_out exceeds maximum entries: got 8193, max 8192", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := buildNetPolicyPlan(tt.opts) - if err == nil { - t.Fatalf("buildNetPolicyPlan returned nil error") + if tt.err == nil { + t.Fatalf("validateNetPolicyEntryCounts returned nil error") } - if got := err.Error(); got != tt.want { + if got := tt.err.Error(); got != tt.want { t.Fatalf("error=%q, want %q", got, tt.want) } }) } } -func TestBuildNetPolicyPlanDeduplicatesValidationTargetsByMapKey(t *testing.T) { - allowOut := []string{"198.51.100.1", "198.51.100.1/32", "API.Example.COM."} - l7AllowOut := []string{"198.51.100.1", "api.example.com"} - denyOut := []string{"203.0.113.1", "203.0.113.1/32"} - - if _, err := buildNetPolicyPlan(MVMOptions{AllowOut: &allowOut, L7AllowOut: &l7AllowOut, DenyOut: &denyOut}); err != nil { - t.Fatalf("buildNetPolicyPlan returned error: %v", err) +func TestValidateNetPolicyEntryCountsDeduplicatesByMapKey(t *testing.T) { + err := validateNetPolicyEntryCounts( + []string{"198.51.100.1", "198.51.100.1/32"}, + []string{"198.51.100.1"}, + []string{"API.Example.COM."}, + []string{"api.example.com"}, + []string{"203.0.113.1", "203.0.113.1/32"}, + ) + if err != nil { + t.Fatalf("validateNetPolicyEntryCounts returned error: %v", err) } } -func TestBuildDNSAllowRulesDeduplicatesAndMergesFlags(t *testing.T) { - rules, err := buildDNSAllowRules( - []string{"api.example.com", "*.github.com"}, - []string{"API.Example.COM.", "*.GitHub.com."}, - ) +func TestBuildDNSAllowRulesDeduplicatesFlagsAndPorts(t *testing.T) { + // Plain L3 domain rules go through buildDNSAllowRules; L7 domain rules + // with per-host port sets go through buildL7Plan and are merged in with + // mergeDNSAllowRules. Verify each half separately here. + base, err := buildDNSAllowRules([]string{"api.example.com", "*.github.com", "API.Example.COM."}) if err != nil { t.Fatalf("buildDNSAllowRules returned error: %v", err) } - if got, want := len(rules), 2; got != want { - t.Fatalf("len(rules)=%d, want %d", got, want) - } - - byKey := make(map[dnsAllowKey]dnsAllowRule, len(rules)) - for _, rule := range rules { - byKey[rule.key] = rule + if got, want := len(base), 2; got != want { + t.Fatalf("len(base)=%d, want %d", got, want) } - for _, domain := range []string{"api.example.com", "*.github.com"} { - key, _, err := makeDNSAllowRule(domain, 0) - if err != nil { - t.Fatalf("makeDNSAllowRule(%q) returned error: %v", domain, err) - } - rule, ok := byKey[key] - if !ok { - t.Fatalf("missing DNS allow rule for %q", domain) + for _, rule := range base { + if rule.value.Flags != 0 { + t.Fatalf("base rule %q should have no flags, got %d", rule.domain, rule.value.Flags) } - if rule.value.Flags != uint8(netPolicyFlagL7Required) { - t.Fatalf("rule %q flags=%d, want %d", domain, rule.value.Flags, netPolicyFlagL7Required) + if rule.value.PortCount != 0 { + t.Fatalf("base rule %q should have no port set, got %d", rule.domain, rule.value.PortCount) } } } -func TestBuildAllowOutPolicyEntriesDeduplicatesAndMergesFlags(t *testing.T) { +func TestBuildAllowOutPolicyEntriesDeduplicatesByKey(t *testing.T) { entries, err := buildAllowOutPolicyEntries( - []string{"198.51.100.1", "203.0.113.0/24"}, - []string{"198.51.100.1/32", "203.0.113.0/24"}, + []string{"198.51.100.1", "203.0.113.0/24", "198.51.100.1/32"}, ) if err != nil { t.Fatalf("buildAllowOutPolicyEntries returned error: %v", err) @@ -144,17 +139,21 @@ func TestBuildAllowOutPolicyEntriesDeduplicatesAndMergesFlags(t *testing.T) { if got, want := len(entries), 2; got != want { t.Fatalf("len(entries)=%d, want %d", got, want) } - - for _, entry := range entries { - if entry.flags != uint8(netPolicyFlagL7Required) { - t.Fatalf("entry %q flags=%d, want %d", entry.source, entry.flags, netPolicyFlagL7Required) - } - } + // First occurrence wins for the source label. if entries[0].source != "198.51.100.1" { - t.Fatalf("first duplicate source=%q, want first source", entries[0].source) + t.Fatalf("first source=%q, want first occurrence", entries[0].source) } if entries[1].source != "203.0.113.0/24" { - t.Fatalf("second duplicate source=%q, want first source", entries[1].source) + t.Fatalf("second source=%q", entries[1].source) + } + // Non-L7 allow_out entries carry no flags and no port set. + for _, e := range entries { + if e.flags != 0 { + t.Fatalf("entry %q flags=%d, want 0", e.source, e.flags) + } + if len(e.ports) != 0 { + t.Fatalf("entry %q ports=%v, want empty", e.source, e.ports) + } } } @@ -199,7 +198,10 @@ func TestAppendDenyOutPolicyEntriesDeduplicatesExisting(t *testing.T) { func TestBuildNetPolicyPlanDeduplicatesAndMergesFlags(t *testing.T) { allowOut := []string{"api.example.com", "198.51.100.1"} - l7AllowOut := []string{"API.Example.COM.", "198.51.100.1/32"} + l7AllowOut := []L7Target{ + {Host: "API.Example.COM."}, + {Host: "198.51.100.1/32"}, + } denyOut := []string{"192.168.0.0/16", "203.0.113.0/24"} plan, err := buildNetPolicyPlan(MVMOptions{ @@ -214,13 +216,19 @@ func TestBuildNetPolicyPlanDeduplicatesAndMergesFlags(t *testing.T) { if got, want := len(plan.allowOutEntries), 1; got != want { t.Fatalf("len(plan.allowOutEntries)=%d, want %d", got, want) } - if got, want := plan.allowOutEntries[0].flags, uint8(netPolicyFlagL7Required); got != want { + // 198.51.100.1 is in BOTH plain allow_out and an L7 rule, so the merged + // static entry carries L7Required|L3Allowed (the L3 bit keeps the plain /32 + // any-port entry alongside the L7 /48 entries). + if got, want := plan.allowOutEntries[0].flags, uint8(netPolicyFlagL7Required|netPolicyFlagL3Allowed); got != want { t.Fatalf("allow out flags=%d, want %d", got, want) } if got, want := len(plan.dnsAllowRules), 1; got != want { t.Fatalf("len(plan.dnsAllowRules)=%d, want %d", got, want) } - if got, want := plan.dnsAllowRules[0].value.Flags, uint8(netPolicyFlagL7Required); got != want { + // api.example.com is in BOTH plain allow_out and an L7 rule, so the merged + // entry carries L7Required|L3Allowed (the L3 bit keeps the plain /32 + // any-port entry alongside the L7 /48 entries). + if got, want := plan.dnsAllowRules[0].value.Flags, uint8(netPolicyFlagL7Required|netPolicyFlagL3Allowed); got != want { t.Fatalf("dns allow flags=%d, want %d", got, want) } if got, want := plan.dnsPolicyFlags, uint8(dnsPolicyFlagLearningEnabled); got != want { @@ -234,6 +242,129 @@ func TestBuildNetPolicyPlanDeduplicatesAndMergesFlags(t *testing.T) { } } +// TestBuildNetPolicyPlanL3AllowedCoexistence pins the plain-allow_out + +// L7-rule coexistence semantics for a shared domain: the merged dns_allow +// entry must carry netPolicyFlagL3Allowed (so the datapath learns both the +// plain /32 any-port entry and the L7 /48 entries), while a domain present in +// only one of them must not set it. +func TestBuildNetPolicyPlanL3AllowedCoexistence(t *testing.T) { + allowOut := []string{"both.example.com", "plain.example.com"} + l7AllowOut := []L7Target{ + {Host: "both.example.com", Port: 8443, Scheme: L7SchemeHTTPS}, + {Host: "l7only.example.com", Port: 9090, Scheme: L7SchemeHTTP}, + } + + plan, err := buildNetPolicyPlan(MVMOptions{ + AllowOut: &allowOut, + L7AllowOut: &l7AllowOut, + }) + if err != nil { + t.Fatalf("buildNetPolicyPlan returned error: %v", err) + } + + flagsByDomain := make(map[string]uint8, len(plan.dnsAllowRules)) + for _, r := range plan.dnsAllowRules { + flagsByDomain[r.domain] = r.value.Flags + } + + l7bit := uint8(netPolicyFlagL7Required) + l3bit := uint8(netPolicyFlagL3Allowed) + + if got := flagsByDomain["both.example.com"]; got != l7bit|l3bit { + t.Fatalf("both.example.com flags=%#x, want L7Required|L3Allowed (%#x)", got, l7bit|l3bit) + } + if got := flagsByDomain["l7only.example.com"]; got != l7bit { + t.Fatalf("l7only.example.com flags=%#x, want L7Required only (%#x)", got, l7bit) + } + if got := flagsByDomain["plain.example.com"]; got != 0 { + t.Fatalf("plain.example.com flags=%#x, want 0 (plain allow)", got) + } +} + +// TestBuildNetPolicyPlanStaticL3AllowedCoexistence is the static-IP +// counterpart of TestBuildNetPolicyPlanL3AllowedCoexistence: a host present in +// both plain allow_out and an L7 rule must be marked netPolicyFlagL3Allowed in +// allowOutEntries, so populateAllowOutInnerMap writes the plain /32 any-port +// entry alongside the L7 /48 entries. +func TestBuildNetPolicyPlanStaticL3AllowedCoexistence(t *testing.T) { + allowOut := []string{"198.51.100.10", "198.51.100.20"} + l7AllowOut := []L7Target{ + {Host: "198.51.100.10", Port: 8443, Scheme: L7SchemeHTTPS}, + {Host: "198.51.100.30", Port: 9090, Scheme: L7SchemeHTTP}, + } + + plan, err := buildNetPolicyPlan(MVMOptions{AllowOut: &allowOut, L7AllowOut: &l7AllowOut}) + if err != nil { + t.Fatalf("buildNetPolicyPlan returned error: %v", err) + } + + flagsByIP := make(map[string]uint8, len(plan.allowOutEntries)) + for _, e := range plan.allowOutEntries { + flagsByIP[uint32ToIP(e.key.IP).String()] = e.flags + } + + l7bit := uint8(netPolicyFlagL7Required) + l3bit := uint8(netPolicyFlagL3Allowed) + + if got := flagsByIP["198.51.100.10"]; got != l7bit|l3bit { + t.Fatalf("198.51.100.10 flags=%#x, want L7Required|L3Allowed (%#x)", got, l7bit|l3bit) + } + if got := flagsByIP["198.51.100.30"]; got != l7bit { + t.Fatalf("198.51.100.30 flags=%#x, want L7Required only (%#x)", got, l7bit) + } + if got := flagsByIP["198.51.100.20"]; got != 0 { + t.Fatalf("198.51.100.20 flags=%#x, want 0 (plain allow)", got) + } +} + +// TestBuildNetPolicyPlanWildcardPlainCoversL7Host covers the wildcard L3 +// fallback: an L7 rule host covered by a leading-"*." plain allow_out domain +// must be marked netPolicyFlagL3Allowed, so the host keeps plain /32 L3 access +// on non-rule ports even though the exact rule shadows the wildcard in the DNS +// LPM match. Apex and unrelated domains must NOT be marked. +func TestBuildNetPolicyPlanWildcardPlainCoversL7Host(t *testing.T) { + l7bit := uint8(netPolicyFlagL7Required) + l3bit := uint8(netPolicyFlagL3Allowed) + + cases := []struct { + name string + allowOut []string + l7Host string + wantFlags uint8 + }{ + {"wildcard covers subdomain", []string{"*.qq.com"}, "a.qq.com", l7bit | l3bit}, + {"wildcard covers deep subdomain", []string{"*.qq.com"}, "a.b.qq.com", l7bit | l3bit}, + {"wildcard does not cover apex", []string{"*.qq.com"}, "qq.com", l7bit}, + {"wildcard does not cover unrelated", []string{"*.qq.com"}, "other.com", l7bit}, + {"case + trailing dot normalized", []string{"*.QQ.com."}, "A.QQ.com", l7bit | l3bit}, + {"exact plain covers same host", []string{"a.qq.com"}, "a.qq.com", l7bit | l3bit}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + allowOut := tc.allowOut + l7AllowOut := []L7Target{{Host: tc.l7Host, Port: 8443, Scheme: L7SchemeHTTPS}} + plan, err := buildNetPolicyPlan(MVMOptions{AllowOut: &allowOut, L7AllowOut: &l7AllowOut}) + if err != nil { + t.Fatalf("buildNetPolicyPlan: %v", err) + } + + want := strings.ToLower(strings.TrimSuffix(tc.l7Host, ".")) + got, found := uint8(0), false + for _, r := range plan.dnsAllowRules { + if strings.ToLower(strings.TrimSuffix(r.domain, ".")) == want { + got, found = r.value.Flags, true + } + } + if !found { + t.Fatalf("L7 host %s not found in dnsAllowRules", tc.l7Host) + } + if got != tc.wantFlags { + t.Fatalf("%s flags=%#x, want %#x", tc.l7Host, got, tc.wantFlags) + } + }) + } +} + func TestBuildNetPolicyPlanBlockAllKeepsDefaultDenyOutOnReplace(t *testing.T) { allowInternetAccess := false plan, err := buildNetPolicyPlan(MVMOptions{AllowInternetAccess: &allowInternetAccess}) @@ -256,7 +387,7 @@ func TestPolicyEntryBuildersRejectBlankCIDR(t *testing.T) { { name: "allow out builder", fn: func() error { - _, err := buildAllowOutPolicyEntries([]string{" "}, nil) + _, err := buildAllowOutPolicyEntries([]string{" "}) return err }, }, @@ -315,6 +446,375 @@ func repeatedCIDRs(count int) []string { return entries } +// -------- L7 port + scheme merge tests ------------------------------------- + +func TestBuildL7Plan_DefaultRuleImpliesEmptyPortSet(t *testing.T) { + // A single rule without port/scheme should be encoded as PortCount=0 in + // the dns_allow value so the datapath falls back to {80, 443} at match + // time. This is the backward-compat path. + _, dns, err := buildL7Plan([]L7Target{{Host: "api.example.com"}}) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(dns) != 1 { + t.Fatalf("len(dns)=%d, want 1", len(dns)) + } + if got := dns[0].value.PortCount; got != 0 { + t.Fatalf("PortCount=%d, want 0 (unspecified rule keeps default set)", got) + } + if got := dns[0].value.Flags; got != uint8(netPolicyFlagL7Required) { + t.Fatalf("Flags=%d, want %d", got, netPolicyFlagL7Required) + } +} + +func TestBuildL7Plan_ExplicitPortsSetPortCount(t *testing.T) { + // Two rules: default 80/443 + explicit 8080/http. The default gets + // expanded for conflict checking, so the resulting port set is + // {80, 443, 8080}, all http/https appropriately. + _, dns, err := buildL7Plan([]L7Target{ + {Host: "api.example.com"}, + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + }) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(dns) != 1 { + t.Fatalf("len(dns)=%d, want 1 (rules for same host merge)", len(dns)) + } + if got, want := int(dns[0].value.PortCount), 3; got != want { + t.Fatalf("PortCount=%d, want %d", got, want) + } + + // Verify each expected port is present with the correct scheme. + found := map[uint16]uint8{} + for i := uint8(0); i < dns[0].value.PortCount; i++ { + p := dns[0].value.Ports[i] + found[ntohsPort(p.Port)] = p.Scheme + } + for _, expect := range []struct { + port uint16 + scheme uint8 + }{ + {80, L7SchemeHTTP}, + {443, L7SchemeHTTPS}, + {8080, L7SchemeHTTP}, + } { + if got, ok := found[expect.port]; !ok || got != expect.scheme { + t.Fatalf("port %d: got scheme %d (present=%v), want %d", + expect.port, got, ok, expect.scheme) + } + } +} + +func TestBuildL7Plan_PreservesFirstAppearancePortOrder(t *testing.T) { + _, dns, err := buildL7Plan([]L7Target{ + {Host: "api.example.com", Port: 9002, Scheme: L7SchemeHTTP}, + {Host: "api.example.com", Port: 9001, Scheme: L7SchemeHTTPS}, + {Host: "api.example.com", Port: 9003, Scheme: L7SchemeHTTP}, + {Host: "api.example.com", Port: 9001, Scheme: L7SchemeHTTPS}, + }) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(dns) != 1 { + t.Fatalf("len(dns)=%d, want 1", len(dns)) + } + want := []uint16{9002, 9001, 9003} + if got := int(dns[0].value.PortCount); got != len(want) { + t.Fatalf("PortCount=%d, want %d", got, len(want)) + } + for i, port := range want { + if got := ntohsPort(dns[0].value.Ports[i].Port); got != port { + t.Fatalf("Ports[%d]=%d, want %d", i, got, port) + } + } +} + +func TestBuildL7Plan_SchemeConflictOnSamePortRejected(t *testing.T) { + // (host, port=443) with scheme=http conflicts with the default rule's + // implicit (host, 443, https). iptables can only steer 443 to one + // listener, so this must be rejected at rule-build time. + _, _, err := buildL7Plan([]L7Target{ + {Host: "api.example.com"}, + {Host: "api.example.com", Port: 443, Scheme: L7SchemeHTTP}, + }) + if err == nil { + t.Fatal("expected scheme conflict error, got nil") + } + if !strings.Contains(err.Error(), "conflict") { + t.Fatalf("error should mention conflict, got %v", err) + } +} + +func TestBuildL7Plan_SchemeConflictOnExplicitPortRejected(t *testing.T) { + // Two explicit rules for the same (host, port) with different schemes. + _, _, err := buildL7Plan([]L7Target{ + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTPS}, + }) + if err == nil { + t.Fatal("expected scheme conflict error, got nil") + } +} + +func TestBuildL7Plan_MergesDNSHostCaseVariants(t *testing.T) { + // "API.example.com", "api.example.com." and "api.example.com" all + // normalise to the same dns_allow key (case / trailing dot). They must + // aggregate into one host so every port survives — previously each raw + // string formed its own group and the later same-key merge silently + // dropped the earlier port set. + _, dns, err := buildL7Plan([]L7Target{ + {Host: "API.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + {Host: "api.example.com.", Port: 9090, Scheme: L7SchemeHTTP}, + }) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(dns) != 1 { + t.Fatalf("len(dns)=%d, want 1 (case/dot variants merge into one host)", len(dns)) + } + found := map[uint16]uint8{} + for i := uint8(0); i < dns[0].value.PortCount; i++ { + p := dns[0].value.Ports[i] + found[ntohsPort(p.Port)] = p.Scheme + } + for _, port := range []uint16{8080, 9090} { + if _, ok := found[port]; !ok { + t.Fatalf("port %d dropped after merging host variants (got %v)", port, found) + } + } +} + +func TestBuildL7Plan_DetectsDNSSchemeConflictAcrossCaseVariants(t *testing.T) { + // Same normalised host, same port, different scheme => conflict, even + // though the raw host strings differ only by case. Previously the two + // groups passed conflict detection independently and the conflict was + // silently resolved last-write-wins. + _, _, err := buildL7Plan([]L7Target{ + {Host: "X.com", Port: 443, Scheme: L7SchemeHTTP}, + {Host: "x.com", Port: 443, Scheme: L7SchemeHTTPS}, + }) + if err == nil { + t.Fatal("buildL7Plan did not detect scheme conflict across case variants") + } + if !strings.Contains(err.Error(), "conflict") { + t.Fatalf("error should mention conflict, got %v", err) + } +} + +func TestBuildL7Plan_MergesCIDRNotationVariants(t *testing.T) { + // "1.2.3.4" and "1.2.3.4/32" parse to the same lpm key. They must + // aggregate into one CIDR entry so both ports survive. + cidrs, _, err := buildL7Plan([]L7Target{ + {Host: "1.2.3.4", Port: 8080, Scheme: L7SchemeHTTP}, + {Host: "1.2.3.4/32", Port: 9090, Scheme: L7SchemeHTTP}, + }) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(cidrs) != 1 { + t.Fatalf("len(cidrs)=%d, want 1 (notation variants merge into one host)", len(cidrs)) + } + found := map[uint16]uint8{} + for _, p := range cidrs[0].ports { + found[ntohsPort(p.Port)] = p.Scheme + } + for _, port := range []uint16{8080, 9090} { + if _, ok := found[port]; !ok { + t.Fatalf("port %d dropped after merging CIDR notation variants (got %v)", port, found) + } + } +} + +func TestBuildL7Plan_DetectsCIDRSchemeConflictAcrossNotationVariants(t *testing.T) { + // Same lpm key, same port, different scheme => conflict, even though the + // raw CIDR strings differ only by notation. + _, _, err := buildL7Plan([]L7Target{ + {Host: "1.2.3.4", Port: 443, Scheme: L7SchemeHTTP}, + {Host: "1.2.3.4/32", Port: 443, Scheme: L7SchemeHTTPS}, + }) + if err == nil { + t.Fatal("buildL7Plan did not detect scheme conflict across CIDR notation variants") + } + if !strings.Contains(err.Error(), "conflict") { + t.Fatalf("error should mention conflict, got %v", err) + } +} + +func TestBuildL7Plan_SubnetCIDRRejected(t *testing.T) { + // A subnet CIDR (prefixlen<32) is not a valid L7 host: the datapath + // matches exact (ip, port)/48 pairs and cannot express a subnet+port + // rule, so it must be rejected rather than silently narrowed to the + // network address. A /32 host and a domain remain valid. + if _, _, err := buildL7Plan([]L7Target{{Host: "10.0.0.0/24", Port: 443, Scheme: L7SchemeHTTPS}}); err == nil { + t.Fatal("buildL7Plan accepted a subnet CIDR L7 host, want rejection") + } + if _, _, err := buildL7Plan([]L7Target{{Host: "1.2.3.4", Port: 443, Scheme: L7SchemeHTTPS}}); err != nil { + t.Fatalf("single host IP (/32) must remain valid, got %v", err) + } + if _, _, err := buildL7Plan([]L7Target{{Host: "api.example.com", Port: 443, Scheme: L7SchemeHTTPS}}); err != nil { + t.Fatalf("domain host must remain valid, got %v", err) + } +} + +func TestExpandedAllowOutEntryCount(t *testing.T) { + // Plain allow -> 1 entry; default-port L7 -> 2 (80/443); explicit-port L7 + // -> len(ports). Budget validation must use this expanded count, not one + // per host. + defaultL7 := allowOutPolicyEntry{flags: uint8(netPolicyFlagL7Required)} + plain := allowOutPolicyEntry{flags: 0} + explicitL7 := allowOutPolicyEntry{ + flags: uint8(netPolicyFlagL7Required), + ports: []l7PortEntry{ + {Port: htonsPort(8080), Scheme: L7SchemeHTTP}, + {Port: htonsPort(8443), Scheme: L7SchemeHTTPS}, + }, + } + entries := []allowOutPolicyEntry{defaultL7, plain, explicitL7} + // 2 (default) + 1 (plain) + 2 (explicit) = 5 + if got := expandedAllowOutEntryCount(entries); got != 5 { + t.Fatalf("expandedAllowOutEntryCount=%d, want 5", got) + } +} + +func TestValidateNetPolicyPlanCountsExpandedL7Ports(t *testing.T) { + // Each L7 host with 8 explicit ports occupies 8 inner-map entries, not 1. + // 1025 such hosts need 8200 entries > maxNetPolicyEntries (8192); the old + // len(allowOutEntries) check counted 1025 and passed, undercounting the + // /48 expansion and letting population overflow mid-write (E2BIG). + mkPorts := func() []l7PortEntry { + ports := make([]l7PortEntry, 0, maxL7PortsPerHost) + for i := 0; i < maxL7PortsPerHost; i++ { + ports = append(ports, l7PortEntry{Port: htonsPort(uint16(8000 + i)), Scheme: L7SchemeHTTP}) + } + return ports + } + l7Entry := allowOutPolicyEntry{ + key: lpmKey{Prefixlen: 32, IP: 0x0a000001}, + flags: uint8(netPolicyFlagL7Required), + ports: mkPorts(), + } + + // 1024 hosts -> 8192 entries == budget: allowed. + plan := &netPolicyPlan{} + for i := 0; i < 1024; i++ { + plan.allowOutEntries = append(plan.allowOutEntries, l7Entry) + } + if err := validateNetPolicyPlan(plan); err != nil { + t.Fatalf("1024 8-port L7 hosts (8192 entries) should fit the budget, got %v", err) + } + + // 1025 hosts -> 8200 entries > budget: must reject. + plan.allowOutEntries = append(plan.allowOutEntries, l7Entry) + if err := validateNetPolicyPlan(plan); err == nil { + t.Fatal("1025 8-port L7 hosts (8200 entries) should exceed the budget, got nil") + } +} + +func TestBuildL7Plan_SameHostPortSchemeIsIdempotent(t *testing.T) { + // The same (host, port, scheme) tuple appearing twice is fine — different + // rules may share the tuple to attach different lua policy actions in + // CubeEgress. Only the shared datapath entry needs to be deduplicated. + _, dns, err := buildL7Plan([]L7Target{ + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + }) + if err != nil { + t.Fatalf("duplicate tuple must be idempotent, got err %v", err) + } + if len(dns) != 1 || dns[0].value.PortCount != 1 { + t.Fatalf("dns=%+v, want single rule with 1 port", dns) + } + if got := ntohsPort(dns[0].value.Ports[0].Port); got != 8080 { + t.Fatalf("port=%d, want 8080", got) + } + if dns[0].value.Ports[0].Scheme != L7SchemeHTTP { + t.Fatalf("scheme=%d, want L7SchemeHTTP", dns[0].value.Ports[0].Scheme) + } +} + +func TestBuildL7Plan_MultiHostIndependent(t *testing.T) { + // Two hosts sharing a port at different schemes must NOT conflict — the + // scheme-consistency rule is per (host, port), not per port. + _, dns, err := buildL7Plan([]L7Target{ + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + {Host: "other.example.com", Port: 8080, Scheme: L7SchemeHTTPS}, + }) + if err != nil { + t.Fatalf("cross-host same-port with different schemes must be allowed: %v", err) + } + if len(dns) != 2 { + t.Fatalf("expected two dns rules, got %d", len(dns)) + } +} + +func TestBuildL7Plan_IPClassifiedAsCIDR(t *testing.T) { + // An IP-literal L7 target lands in allow_out_v3 as a static entry with + // L7_REQUIRED and the explicit (port, scheme) set copied into ports[]. + cidrs, dns, err := buildL7Plan([]L7Target{ + {Host: "1.2.3.4", Port: 8443, Scheme: L7SchemeHTTPS}, + }) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(dns) != 0 { + t.Fatalf("dns entries=%d, want 0 for IP-literal host", len(dns)) + } + if len(cidrs) != 1 { + t.Fatalf("cidrs=%d, want 1", len(cidrs)) + } + if got := cidrs[0].flags; got != uint8(netPolicyFlagL7Required) { + t.Fatalf("flags=%d, want %d", got, netPolicyFlagL7Required) + } + if len(cidrs[0].ports) != 1 || ntohsPort(cidrs[0].ports[0].Port) != 8443 { + t.Fatalf("ports=%+v, want [{Port:8443, Scheme:HTTPS}]", cidrs[0].ports) + } + if cidrs[0].ports[0].Scheme != L7SchemeHTTPS { + t.Fatalf("scheme=%d, want L7SchemeHTTPS", cidrs[0].ports[0].Scheme) + } +} + +func TestBuildL7Plan_PortBudgetExceededRejected(t *testing.T) { + // More distinct (port, scheme) tuples than maxL7PortsPerHost — must fail + // rather than silently truncate, so users see a real error at policy + // submission time instead of dropped rules at runtime. + targets := make([]L7Target, 0, maxL7PortsPerHost+1) + for i := 0; i < maxL7PortsPerHost+1; i++ { + targets = append(targets, L7Target{ + Host: "api.example.com", + Port: uint16(9000 + i), + // Half http, half https — no scheme conflict on distinct ports. + Scheme: L7SchemeHTTP, + }) + } + if _, _, err := buildL7Plan(targets); err == nil { + t.Fatal("expected budget-exceeded error, got nil") + } +} + +func TestBuildL7Plan_ExplicitRuleWithoutSchemeRejected(t *testing.T) { + // A rule with Port set but Scheme == None (or vice versa) is a schema + // error — the caller (extractL7PortScheme) is supposed to reject partial + // specifications before we get here, but defense-in-depth is cheap. + _, _, err := buildL7Plan([]L7Target{ + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeNone}, + }) + if err == nil { + t.Fatal("expected error for port-without-scheme, got nil") + } +} + +func TestApplyPortsToDNSAllowValue_Empty(t *testing.T) { + // Empty ports slice must leave PortCount=0 so the datapath falls back to + // the default set. Regression guard: don't accidentally set PortCount to + // len(ports) when ports is nil. + var v dnsAllowValue + applyPortsToDNSAllowValue(&v, nil) + if v.PortCount != 0 { + t.Fatalf("PortCount=%d, want 0", v.PortCount) + } +} + func repeatedDomains(count int) []string { entries := make([]string, count) for i := range entries { @@ -324,6 +824,8 @@ func repeatedDomains(count int) []string { } func TestNetPolicyValueV2Layout(t *testing.T) { + // netPolicyValueV2 is the legacy 16-byte allow_out_v2 ABI, read only when + // migrating a pre-v3 allow_out_v2 map to allow_out_v3. var value netPolicyValueV2 if got, want := unsafe.Sizeof(value), uintptr(16); got != want { t.Fatalf("unsafe.Sizeof(netPolicyValueV2{})=%d, want %d", got, want) @@ -339,22 +841,45 @@ func TestNetPolicyValueV2Layout(t *testing.T) { } } -func TestNetPolicyValueV2Expired(t *testing.T) { +func TestNetPolicyValueV3Layout(t *testing.T) { + // netPolicyValueV3 is the current on-disk ABI for allow_out_v3. + // The port lives in the LPM key, so the value only carries the + // scheme plus the DNS-learned expiry. Any layout drift silently + // corrupts the persisted policy, so assert the exact offsets. + var value netPolicyValueV3 + if got, want := unsafe.Sizeof(value), uintptr(16); got != want { + t.Fatalf("unsafe.Sizeof(netPolicyValueV3{})=%d, want %d", got, want) + } + if got, want := unsafe.Offsetof(value.ExpiresAtNS), uintptr(0); got != want { + t.Fatalf("ExpiresAtNS offset=%d, want %d", got, want) + } + if got, want := unsafe.Offsetof(value.Flags), uintptr(8); got != want { + t.Fatalf("Flags offset=%d, want %d", got, want) + } + if got, want := unsafe.Offsetof(value.Scheme), uintptr(9); got != want { + t.Fatalf("Scheme offset=%d, want %d", got, want) + } + if got, want := unsafe.Offsetof(value.KeyPrefixlen), uintptr(10); got != want { + t.Fatalf("KeyPrefixlen offset=%d, want %d", got, want) + } +} + +func TestNetPolicyValueV3Expired(t *testing.T) { now := uint64(100) tests := []struct { name string - value netPolicyValueV2 + value netPolicyValueV3 want bool }{ - {name: "static", value: netPolicyValueV2{ExpiresAtNS: 0}, want: false}, - {name: "dynamic valid", value: netPolicyValueV2{ExpiresAtNS: now + 1}, want: false}, - {name: "dynamic expired", value: netPolicyValueV2{ExpiresAtNS: now}, want: true}, + {name: "static", value: netPolicyValueV3{ExpiresAtNS: 0}, want: false}, + {name: "dynamic valid", value: netPolicyValueV3{ExpiresAtNS: now + 1}, want: false}, + {name: "dynamic expired", value: netPolicyValueV3{ExpiresAtNS: now}, want: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := netPolicyValueV2Expired(tt.value, now); got != tt.want { - t.Fatalf("netPolicyValueV2Expired()=%t, want %t", got, tt.want) + if got := netPolicyValueV3Expired(tt.value, now); got != tt.want { + t.Fatalf("netPolicyValueV3Expired()=%t, want %t", got, tt.want) } }) } @@ -368,7 +893,7 @@ func TestMakeDNSAllowRuleSetsL7Flag(t *testing.T) { if value.Flags != uint8(netPolicyFlagL7Required) { t.Fatalf("value.Flags=%d, want %d", value.Flags, netPolicyFlagL7Required) } - if got, want := unsafe.Sizeof(value), uintptr(8); got != want { + if got, want := unsafe.Sizeof(value), uintptr(40); got != want { t.Fatalf("unsafe.Sizeof(dnsAllowValue{})=%d, want %d", got, want) } if key.Name[int(value.NameLen)-1] != 0 { @@ -427,7 +952,7 @@ func TestMVMMetadataLayoutAndDNSPolicyFlags(t *testing.T) { func TestDNSAllowValueLayoutAndFlags(t *testing.T) { var value dnsAllowValue - if got, want := unsafe.Sizeof(value), uintptr(8); got != want { + if got, want := unsafe.Sizeof(value), uintptr(40); got != want { t.Fatalf("unsafe.Sizeof(dnsAllowValue{})=%d, want %d", got, want) } if got, want := unsafe.Offsetof(value.NameLen), uintptr(0); got != want { @@ -453,3 +978,252 @@ func TestDNSAllowDuplicateRulesMergeFlags(t *testing.T) { t.Fatalf("merged Flags=%d, want %d", allowValue.Flags, netPolicyFlagL7Required) } } + +// attachAllowOutV3Inner creates an allow_out_v3 inner LPM trie and attaches +// it to the outer hash-of-maps under ifindex, returning the inner map. +func attachAllowOutV3Inner(t *testing.T, outer *ebpf.Map, ifindex uint32) *ebpf.Map { + t.Helper() + inner, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(lpmKeyV3{})), + ValueSize: uint32(unsafe.Sizeof(netPolicyValueV3{})), + MaxEntries: maxNetPolicyEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF LPM trie unavailable: %v", err) + } + t.Fatalf("create allow_out_v3 inner: %v", err) + } + t.Cleanup(func() { _ = inner.Close() }) + if err := outer.Put(&ifindex, inner); err != nil { + t.Fatalf("attach allow_out_v3 inner: %v", err) + } + return inner +} + +func mustLookupV3(t *testing.T, inner *ebpf.Map, key lpmKeyV3) netPolicyValueV3 { + t.Helper() + var value netPolicyValueV3 + if err := inner.Lookup(&key, &value); err != nil { + t.Fatalf("lookup %+v: %v", key, err) + } + return value +} + +// TestPopulateAllowOutStaticV3OverCoveringLearnedStaysStatic is the +// exact-key-match regression test for the userspace writer: a static /48 +// written while a DNS-learned /32 (non-zero TTL) covers the same IP must +// stay static. The inner LPM lookup is longest-prefix, so without the +// KeyPrefixlen check the learned /32's TTL was wrongly inherited and the +// "static" /48 aged out at the learned TTL. +func TestPopulateAllowOutStaticV3OverCoveringLearnedStaysStatic(t *testing.T) { + outer := newAllowOutV3OuterMap(t) + ifindex := uint32(305) + inner := attachAllowOutV3Inner(t, outer, ifindex) + ip := mustParseCIDRForTest(t, "192.0.2.80").IP + + // Pre-existing DNS-learned /32 (temporary) covering the IP. + learnedTTL := uint64(123456789) + seed := lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0} + if err := inner.Update(&seed, &netPolicyValueV3{ + ExpiresAtNS: learnedTTL, Flags: testFlagMarker, KeyPrefixlen: 32, + }, ebpf.UpdateAny); err != nil { + t.Fatalf("seed learned /32: %v", err) + } + + entries := []allowOutPolicyEntry{{ + key: lpmKey{Prefixlen: 32, IP: ip}, + flags: netPolicyFlagL7Required, + ports: []l7PortEntry{{Port: htonsPort(443), Scheme: L7SchemeHTTPS}}, + }} + if err := populateAllowOutInnerMap(outer, ifindex, entries); err != nil { + t.Fatalf("populateAllowOutInnerMap: %v", err) + } + + got := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if got.KeyPrefixlen != 48 { + t.Fatalf("static /48 KeyPrefixlen=%d, want 48", got.KeyPrefixlen) + } + if got.ExpiresAtNS != 0 { + t.Fatalf("static /48 inherited learned TTL %d from covering /32, want static (0)", got.ExpiresAtNS) + } + if got.Flags&testFlagMarker != 0 { + t.Fatalf("static /48 inherited flags from covering /32: %#x", got.Flags) + } +} + +// TestPopulateAllowOutStaticV3OverExactLearnedStaticWins is the counterpart: +// a static /48 written over a learned entry for the EXACT same key merges +// the learned flags but the static (zero) expiry WINS — the entry becomes +// permanent. Keeping the learned TTL instead would let the reaper delete +// the entry at the old TTL and silently drop the static verdict. +func TestPopulateAllowOutStaticV3OverExactLearnedStaticWins(t *testing.T) { + outer := newAllowOutV3OuterMap(t) + ifindex := uint32(306) + inner := attachAllowOutV3Inner(t, outer, ifindex) + ip := mustParseCIDRForTest(t, "192.0.2.90").IP + + learnedTTL := uint64(123456789) + seed := lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)} + if err := inner.Update(&seed, &netPolicyValueV3{ + ExpiresAtNS: learnedTTL, Flags: uint8(netPolicyFlagL7Required) | testFlagMarker, + Scheme: L7SchemeHTTP, KeyPrefixlen: 48, // stale scheme: static write must overwrite + }, ebpf.UpdateAny); err != nil { + t.Fatalf("seed learned /48: %v", err) + } + + entries := []allowOutPolicyEntry{{ + key: lpmKey{Prefixlen: 32, IP: ip}, + flags: netPolicyFlagL7Required, + ports: []l7PortEntry{{Port: htonsPort(443), Scheme: L7SchemeHTTPS}}, + }} + if err := populateAllowOutInnerMap(outer, ifindex, entries); err != nil { + t.Fatalf("populateAllowOutInnerMap: %v", err) + } + + got := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if got.ExpiresAtNS != 0 { + t.Fatalf("static write over learned /48 must win with zero expiry, got %d (learned TTL was %d)", got.ExpiresAtNS, learnedTTL) + } + if got.Flags&testFlagMarker == 0 { + t.Fatalf("exact learned /48 flags not merged: %#x", got.Flags) + } + if got.Scheme != L7SchemeHTTPS { + t.Fatalf("scheme=%d after static write, want HTTPS (last write wins)", got.Scheme) + } +} + +// TestPopulateAllowOutStaticV3OverExactStaticMergesFlags covers the +// static-over-static same-key cell: the write must union the old entry's +// flags and keep the zero (static) expiry. +func TestPopulateAllowOutStaticV3OverExactStaticMergesFlags(t *testing.T) { + outer := newAllowOutV3OuterMap(t) + ifindex := uint32(308) + inner := attachAllowOutV3Inner(t, outer, ifindex) + ip := mustParseCIDRForTest(t, "192.0.2.100").IP + + seed := lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)} + if err := inner.Update(&seed, &netPolicyValueV3{ + Flags: uint8(netPolicyFlagL7Required) | testFlagMarker, + Scheme: L7SchemeHTTPS, KeyPrefixlen: 48, + }, ebpf.UpdateAny); err != nil { + t.Fatalf("seed static /48: %v", err) + } + + entries := []allowOutPolicyEntry{{ + key: lpmKey{Prefixlen: 32, IP: ip}, + flags: netPolicyFlagL7Required, + ports: []l7PortEntry{{Port: htonsPort(443), Scheme: L7SchemeHTTPS}}, + }} + if err := populateAllowOutInnerMap(outer, ifindex, entries); err != nil { + t.Fatalf("populateAllowOutInnerMap: %v", err) + } + + got := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if got.ExpiresAtNS != 0 { + t.Fatalf("static-over-static must stay static, got expiry %d", got.ExpiresAtNS) + } + if got.Flags&testFlagMarker == 0 { + t.Fatalf("static-over-static lost old flags: %#x", got.Flags) + } + if got.Flags&uint8(netPolicyFlagL7Required) == 0 { + t.Fatalf("static-over-static lost rule flags: %#x", got.Flags) + } +} + +// TestPopulateAllowOutStaticL3AlsoWritesPlainAndL7Entries is the static-IP +// counterpart of the DNS-learn coexistence test: a host in both plain +// allow_out and an L7 rule must be written as BOTH a plain /32 any-port entry +// (marker bits stripped) and the L7 /48 entries. +func TestPopulateAllowOutStaticL3AlsoWritesPlainAndL7Entries(t *testing.T) { + outer := newAllowOutV3OuterMap(t) + ifindex := uint32(310) + inner := attachAllowOutV3Inner(t, outer, ifindex) + ip := mustParseCIDRForTest(t, "192.0.2.120").IP + + entries := []allowOutPolicyEntry{{ + key: lpmKey{Prefixlen: 32, IP: ip}, + flags: uint8(netPolicyFlagL7Required) | uint8(netPolicyFlagL3Allowed), + ports: []l7PortEntry{{Port: htonsPort(8443), Scheme: L7SchemeHTTPS}}, + }} + if err := populateAllowOutInnerMap(outer, ifindex, entries); err != nil { + t.Fatalf("populateAllowOutInnerMap: %v", err) + } + + // The /48 L7 entry for the rule port is present and intercepted. + l7 := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(8443)}) + if l7.KeyPrefixlen != 48 || l7.Flags&uint8(netPolicyFlagL7Required) == 0 { + t.Fatalf("/48 = %+v, want L7 /48 entry", l7) + } + if l7.Scheme != L7SchemeHTTPS { + t.Fatalf("/48 scheme=%d, want https", l7.Scheme) + } + + // The /32 plain entry is present for everything else, with marker bits + // stripped and static (zero) expiry. + plain := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0}) + if plain.KeyPrefixlen != 32 { + t.Fatalf("/32 KeyPrefixlen=%d, want 32", plain.KeyPrefixlen) + } + if plain.Flags != 0 { + t.Fatalf("/32 flags=%#x, want 0 (plain, marker bits stripped)", plain.Flags) + } + if plain.Scheme != L7SchemeNone { + t.Fatalf("/32 scheme=%d, want none", plain.Scheme) + } + if plain.ExpiresAtNS != 0 { + t.Fatalf("/32 ExpiresAtNS=%d, want 0 (static)", plain.ExpiresAtNS) + } + + // A lookup for a non-rule port must fall back to the /32 plain entry, not + // match a /48 L7 entry (this is how classify_egress_flow admits it as SNAT). + fallback := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(9090)}) + if fallback.KeyPrefixlen == 48 && fallback.Flags&uint8(netPolicyFlagL7Required) != 0 { + t.Fatalf("non-rule port unexpectedly matched a /48 L7 entry: %+v", fallback) + } + if fallback.KeyPrefixlen != 32 { + t.Fatalf("non-rule port fell back to KeyPrefixlen=%d, want 32 (plain)", fallback.KeyPrefixlen) + } +} + +// TestPopulateAllowOutPlainStaticOverwritesLearnedUnconditionally pins the +// plain (non-L7) path's NO-merge semantics: a static /32 overwrites a learned +// /32 outright — last write wins for flags, scheme AND expiry, with no +// same-key preservation (the plain path does not read the old entry at all). +func TestPopulateAllowOutPlainStaticOverwritesLearnedUnconditionally(t *testing.T) { + outer := newAllowOutV3OuterMap(t) + ifindex := uint32(309) + inner := attachAllowOutV3Inner(t, outer, ifindex) + ip := mustParseCIDRForTest(t, "192.0.2.110").IP + + seed := lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0} + if err := inner.Update(&seed, &netPolicyValueV3{ + ExpiresAtNS: 123456789, Flags: testFlagMarker, KeyPrefixlen: 32, + }, ebpf.UpdateAny); err != nil { + t.Fatalf("seed learned /32: %v", err) + } + + entries := []allowOutPolicyEntry{{ + key: lpmKey{Prefixlen: 32, IP: ip}, + flags: 0, // plain allow, no L7 + }} + if err := populateAllowOutInnerMap(outer, ifindex, entries); err != nil { + t.Fatalf("populateAllowOutInnerMap: %v", err) + } + + got := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0}) + if got.KeyPrefixlen != 32 { + t.Fatalf("KeyPrefixlen=%d, want 32", got.KeyPrefixlen) + } + if got.ExpiresAtNS != 0 { + t.Fatalf("plain static overwrite kept learned TTL %d", got.ExpiresAtNS) + } + if got.Flags != 0 { + t.Fatalf("plain static overwrite kept old flags: %#x", got.Flags) + } + if got.Scheme != L7SchemeNone { + t.Fatalf("scheme=%d, want NONE", got.Scheme) + } +} diff --git a/CubeNet/cubevs/reaper.go b/CubeNet/cubevs/reaper.go index 8cc70f912..ee91c3aa3 100644 --- a/CubeNet/cubevs/reaper.go +++ b/CubeNet/cubevs/reaper.go @@ -165,7 +165,9 @@ type natSession struct { VMPort uint16 State uint8 ActiveClose uint8 - Reserved [34]uint8 + PacketClass uint8 + L7Scheme uint8 + Reserved [32]uint8 } // timeout returns the timeout for the session in nanoseconds. @@ -221,6 +223,14 @@ type ingressSessionValue struct { Reserved [3]uint16 } +//nolint:unused +func ingressSession(key *sessionKey, value *ingressSessionValue) string { + return fmt.Sprintf("%s:%d->%s:%d(%s:%d)", + uint32ToIP(key.SourceIP), ntohs(key.SourcePort), + uint32ToIP(key.TargetIP), ntohs(key.TargetPort), + uint32ToIP(value.VMIP), ntohs(value.VMPort)) +} + // StartSessionReaper starts a goroutine that will periodically // check for sessions and DNS-learned policies that have expired and remove them. func StartSessionReaper() <-chan Event { diff --git a/CubeNet/cubevs/tap.go b/CubeNet/cubevs/tap.go index 5df0491f8..afca5d27b 100644 --- a/CubeNet/cubevs/tap.go +++ b/CubeNet/cubevs/tap.go @@ -8,11 +8,22 @@ import ( "github.com/cilium/ebpf" ) +// L7Target describes one (host-or-CIDR, port, scheme) tuple carried inside +// MVMOptions.L7AllowOut. Host is either a domain (with optional wildcard) or +// an IPv4 / IPv4-CIDR literal. Port == 0 with SchemeNone signals the legacy +// default port set {80/http, 443/https}; when Port > 0, Scheme must be +// L7SchemeHTTP or L7SchemeHTTPS. +type L7Target struct { + Host string + Port uint16 // 0 = unspecified (default port set) + Scheme uint8 // one of L7SchemeNone / L7SchemeHTTP / L7SchemeHTTPS +} + type MVMOptions struct { AllowInternetAccess *bool - AllowOut *[]string // CIDR, IP, or domain - L7AllowOut *[]string // CIDR, IP, or domain that requires L7 policy handling - DenyOut *[]string // CIDR or IP + AllowOut *[]string // CIDR, IP, or domain + L7AllowOut *[]L7Target // Host + optional (port, scheme) for L7 policy handling + DenyOut *[]string // CIDR or IP } type tapMetadataMapOps interface { @@ -61,7 +72,7 @@ func AddTAPDevice(ifindex uint32, ip net.IP, id string, version uint32, opts MVM // UpsertTAPDeviceMetadata registers or refreshes TAP metadata without touching // per-sandbox policy maps. Recovery paths use this to repair metadata while -// preserving allow_out_v2, deny_out and dns_allow contents. +// preserving allow_out_v3, deny_out and dns_allow_v2 contents. func UpsertTAPDeviceMetadata(ifindex uint32, ip net.IP, id string, version uint32) error { if len(id) > maxIDLength { return ErrTooLong diff --git a/CubeNet/cubevs/tcp_state_test.go b/CubeNet/cubevs/tcp_state_test.go new file mode 100644 index 000000000..c22ccd7f9 --- /dev/null +++ b/CubeNet/cubevs/tcp_state_test.go @@ -0,0 +1,287 @@ +package cubevs + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target $GOARCH tcpstate ../src/tcp_state_test.bpf.c -- -I../vmlinux/$GOARCH + +import ( + "encoding/binary" + "errors" + "testing" + + "github.com/cilium/ebpf" + "golang.org/x/sys/unix" +) + +const ( + tcpDirOriginal = 0 + tcpDirReply = 1 + tcpTestCaseLen = 40 +) + +type tcpUpdateStep struct { + dir uint8 + syn uint8 + ack uint8 + fin uint8 + rst uint8 +} + +type tcpUpdateCase struct { + accessTime uint64 + nowNS uint64 + state uint8 + activeClose uint8 + steps []tcpUpdateStep +} + +func encodeTCPUpdateCase(tc tcpUpdateCase) []byte { + data := make([]byte, tcpTestCaseLen) + binary.LittleEndian.PutUint64(data[0:8], tc.accessTime) + binary.LittleEndian.PutUint64(data[8:16], tc.nowNS) + data[16] = tc.state + data[17] = tc.activeClose + data[18] = uint8(len(tc.steps)) + for i, step := range tc.steps { + if i >= 2 { + break + } + off := 24 + i*8 + data[off] = step.dir + data[off+1] = step.syn + data[off+2] = step.ack + data[off+3] = step.fin + data[off+4] = step.rst + } + return data +} + +func decodeTCPUpdateCase(data []byte) tcpUpdateCase { + return tcpUpdateCase{ + accessTime: binary.LittleEndian.Uint64(data[0:8]), + nowNS: binary.LittleEndian.Uint64(data[8:16]), + state: data[16], + activeClose: data[17], + } +} + +func bpfTestUnavailable(err error) bool { + var verifierErr *ebpf.VerifierError + if errors.As(err, &verifierErr) { + return false + } + return errors.Is(err, unix.EPERM) || errors.Is(err, unix.EACCES) || + errors.Is(err, ebpf.ErrNotSupported) +} + +func loadTCPStateTestProgram(t *testing.T) *ebpf.Program { + t.Helper() + + spec, err := loadTcpstate() + if err != nil { + t.Fatalf("load tcp state test spec: %v", err) + } + for name := range spec.Maps { + if name != ".rodata" { + delete(spec.Maps, name) + } + } + coll, err := ebpf.NewCollection(spec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF test unavailable: %v", err) + } + t.Fatalf("load tcp state test collection: %v", err) + } + t.Cleanup(coll.Close) + + prog := coll.Programs["test_update_session"] + if prog == nil { + t.Fatal("test_update_session program missing") + } + return prog +} + +func runTCPUpdateCase(t *testing.T, prog *ebpf.Program, tc tcpUpdateCase) tcpUpdateCase { + t.Helper() + + ret, out, err := prog.Test(encodeTCPUpdateCase(tc)) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF test-run unavailable: %v", err) + } + t.Fatalf("run tcp state test: %v", err) + } + if ret != 0 { + t.Fatalf("test_update_session returned %d, want TC_ACT_OK", ret) + } + if len(out) < tcpTestCaseLen { + t.Fatalf("test output length=%d, want >=%d", len(out), tcpTestCaseLen) + } + return decodeTCPUpdateCase(out) +} + +// TestTCPUpdateSessionBidirectionalFINOrderings deterministically covers both +// serializations of near-simultaneous FINs. It does not prove atomicity when +// two CPUs update the same map value at exactly the same time. +func TestTCPUpdateSessionBidirectionalFINOrderings(t *testing.T) { + prog := loadTCPStateTestProgram(t) + fin := func(dir uint8) tcpUpdateStep { return tcpUpdateStep{dir: dir, ack: 1, fin: 1} } + + tests := []struct { + name string + steps []tcpUpdateStep + wantState tcpConntrackState + wantActiveClose uint8 + }{ + { + name: "original FIN then reply FIN", + steps: []tcpUpdateStep{fin(tcpDirOriginal), fin(tcpDirReply)}, + wantState: tcpCTLastAck, + wantActiveClose: 1, + }, + { + name: "reply FIN then original FIN", + steps: []tcpUpdateStep{fin(tcpDirReply), fin(tcpDirOriginal)}, + wantState: tcpCTLastAck, + wantActiveClose: 0, + }, + { + name: "original FIN retransmission", + steps: []tcpUpdateStep{fin(tcpDirOriginal), fin(tcpDirOriginal)}, + wantState: tcpCTFinWait, + wantActiveClose: 1, + }, + { + name: "reply FIN retransmission", + steps: []tcpUpdateStep{fin(tcpDirReply), fin(tcpDirReply)}, + wantState: tcpCTFinWait, + wantActiveClose: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := runTCPUpdateCase(t, prog, tcpUpdateCase{ + accessTime: 1, + nowNS: uint64(2 * 1e9), + state: uint8(tcpCTEstablished), + steps: tt.steps, + }) + if got.state != uint8(tt.wantState) { + t.Fatalf("state=%s, want %s", tcpConntrackState(got.state), tt.wantState) + } + if got.activeClose != tt.wantActiveClose { + t.Fatalf("active_close=%d, want %d", got.activeClose, tt.wantActiveClose) + } + }) + } +} + +func TestTCPUpdateSessionFINRetransmissionAfterACK(t *testing.T) { + prog := loadTCPStateTestProgram(t) + fin := func(dir uint8) tcpUpdateStep { return tcpUpdateStep{dir: dir, ack: 1, fin: 1} } + ack := func(dir uint8) tcpUpdateStep { return tcpUpdateStep{dir: dir, ack: 1} } + + tests := []struct { + name string + first []tcpUpdateStep + retransmit tcpUpdateStep + wantActiveClose uint8 + }{ + { + name: "original FIN retransmission in close-wait", + first: []tcpUpdateStep{fin(tcpDirOriginal), ack(tcpDirReply)}, + retransmit: fin(tcpDirOriginal), + wantActiveClose: 1, + }, + { + name: "reply FIN retransmission in close-wait", + first: []tcpUpdateStep{fin(tcpDirReply), ack(tcpDirOriginal)}, + retransmit: fin(tcpDirReply), + wantActiveClose: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + closeWait := runTCPUpdateCase(t, prog, tcpUpdateCase{ + accessTime: 1, + nowNS: uint64(2 * 1e9), + state: uint8(tcpCTEstablished), + steps: tt.first, + }) + if closeWait.state != uint8(tcpCTCloseWait) { + t.Fatalf("precondition state=%s, want %s", tcpConntrackState(closeWait.state), tcpCTCloseWait) + } + got := runTCPUpdateCase(t, prog, tcpUpdateCase{ + accessTime: closeWait.accessTime, + nowNS: uint64(3 * 1e9), + state: closeWait.state, + activeClose: closeWait.activeClose, + steps: []tcpUpdateStep{tt.retransmit}, + }) + if got.state != uint8(tcpCTCloseWait) { + t.Fatalf("state=%s, want %s", tcpConntrackState(got.state), tcpCTCloseWait) + } + if got.activeClose != tt.wantActiveClose { + t.Fatalf("active_close=%d, want %d", got.activeClose, tt.wantActiveClose) + } + }) + } +} + +func TestTCPUpdateSessionBidirectionalFINCompletionTimeout(t *testing.T) { + prog := loadTCPStateTestProgram(t) + fin := func(dir uint8) tcpUpdateStep { return tcpUpdateStep{dir: dir, ack: 1, fin: 1} } + ack := func(dir uint8) tcpUpdateStep { return tcpUpdateStep{dir: dir, ack: 1} } + + tests := []struct { + name string + first []tcpUpdateStep + lastACK tcpUpdateStep + wantActiveClose uint8 + wantTimeout uint64 + }{ + { + name: "original initiated close", + first: []tcpUpdateStep{fin(tcpDirOriginal), fin(tcpDirReply)}, + lastACK: ack(tcpDirOriginal), + wantActiveClose: 1, + wantTimeout: uint64(tcpTimeouts[tcpCTClose].Nanoseconds()), + }, + { + name: "reply initiated close", + first: []tcpUpdateStep{fin(tcpDirReply), fin(tcpDirOriginal)}, + lastACK: ack(tcpDirReply), + wantActiveClose: 0, + wantTimeout: uint64(tcpTimeouts[tcpCTTimeWait].Nanoseconds()), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + closing := runTCPUpdateCase(t, prog, tcpUpdateCase{ + accessTime: 1, + nowNS: uint64(2 * 1e9), + state: uint8(tcpCTEstablished), + steps: tt.first, + }) + closed := runTCPUpdateCase(t, prog, tcpUpdateCase{ + accessTime: closing.accessTime, + nowNS: uint64(3 * 1e9), + state: closing.state, + activeClose: closing.activeClose, + steps: []tcpUpdateStep{tt.lastACK}, + }) + if closed.state != uint8(tcpCTTimeWait) { + t.Fatalf("state=%s, want %s", tcpConntrackState(closed.state), tcpCTTimeWait) + } + if closed.activeClose != tt.wantActiveClose { + t.Fatalf("active_close=%d, want %d", closed.activeClose, tt.wantActiveClose) + } + sess := natSession{State: closed.state, ActiveClose: closed.activeClose} + if got := sess.tcpTimeout(); got != tt.wantTimeout { + t.Fatalf("timeout=%d, want %d", got, tt.wantTimeout) + } + }) + } +} diff --git a/CubeNet/src/cubevs.h b/CubeNet/src/cubevs.h index 264d305b2..60bbf7284 100644 --- a/CubeNet/src/cubevs.h +++ b/CubeNet/src/cubevs.h @@ -36,7 +36,45 @@ #define MAX_DNS_NAME_LEN 256 #define DNS_POLICY_FLAG_LEARNING_ENABLED 1 #define NET_POLICY_FLAG_L7_REQUIRED 1 +/* Set alongside NET_POLICY_FLAG_L7_REQUIRED when a domain is present in BOTH + * a plain (L3) allow_out rule and an L7 rule. Tells dns_learn_response_ip to + * learn the plain /32 any-port entry in addition to the L7 (ip, port)/48 + * entries, so non-rule ports keep plain SNAT access while the rule's ports are + * L7-intercepted. Without it an L7 rule silently narrows a same-domain plain + * allow_out to only the rule's ports. + */ +#define NET_POLICY_FLAG_L3_ALLOWED 2 #define NSEC_PER_SEC 1000000000ULL + +/* L7 scheme values embedded in dns_allow_value / net_policy_value_v2 per-port + * entries. Used by the eBPF datapath to compute skb->mark so CubeEgress's + * iptables TPROXY rules can steer HTTP vs HTTPS traffic to distinct listeners + * without depending on the destination port number. Keep in sync with + * cubevs/cubevs.go (L7SchemeHTTP / L7SchemeHTTPS). + */ +#define L7_SCHEME_NONE 0 +#define L7_SCHEME_HTTP 1 +#define L7_SCHEME_HTTPS 2 + +/* Maximum number of (port, scheme) tuples a single L7 rule host may declare. + * Bounded so the map value size stays small and BPF verifier can unroll the + * lookup loop. Users needing more should merge rules or reuse ports. + */ +#define MAX_L7_PORTS_PER_HOST 8 + +/* skb->mark encoding for L7 redirect. The high 16 bits carry a cube-owned + * prefix (0xCE?? masked by cube_l7_mark_mask) so cube marks do not collide + * with unrelated mark bits users may set elsewhere. iptables uses + * `-m mark --mark VAL/MASK` to match on cube-owned bits only. + * + * These are const volatile globals (not macros) so a deployment can override + * them at load time from userspace (rewriteConstants, sourced from the same + * install-time config the iptables init script reads), keeping the dataplane + * and iptables in lock-step. Defaults match the shipped values. + */ +const volatile __u32 cube_l7_mark_mask = 0xFFFF0000u; +const volatile __u32 cube_l7_mark_http = 0xCE010000u; +const volatile __u32 cube_l7_mark_https = 0xCE020000u; #define DNS_QUERY_TRACK_TTL_NS (10ULL * NSEC_PER_SEC) /* https://en.wikipedia.org/wiki/IPv4#Header @@ -142,15 +180,53 @@ struct lpm_key { __u32 ip; }; +/* LPM key for allow_out_v3. Carries the destination IP (32-bit, network + * byte order) and an optional destination port (16-bit, network byte order) + * so a single longest-prefix lookup resolves: + * - exact (ip, port): prefixlen = 48 (ip[4] + port[2]) + * - ip only (any port): prefixlen = 32 (ip[4], port ignored) + * - ip/mask subnet: prefixlen < 32 (only top bits of ip matter) + * The struct is padded to 12 bytes (8-byte data payload) so the trie's + * word-wise compare stays 4-byte aligned on every kernel. Insert and + * lookup MUST both fill ip/port from network-byte-order bytes (matching + * iphdr->daddr / tcphdr->dest), never from a host-byte-order integer + * shift — otherwise the exact (ip, port) match silently fails. + */ +struct lpm_key_v3 { + __u32 prefixlen; + __u32 ip; /* network byte order */ + __u16 port; /* network byte order; 0 when key is ip-only/subnet */ + __u16 _pad; /* 0; keeps the data payload at 8 bytes */ +}; + struct dns_allow_key { __u32 prefixlen; char name[MAX_DNS_NAME_LEN]; }; +/* Per-host L7 port entry. Attached inline to both dns_allow_value and + * net_policy_value_v2 so the datapath can pick the right scheme for a given + * destination port without a second map lookup. port is in network byte order + * (matches tcphdr->dest), scheme is one of L7_SCHEME_HTTP / L7_SCHEME_HTTPS. + */ +struct l7_port_entry { + __u16 port; /* network byte order */ + __u8 scheme; + __u8 _pad; +}; + +/* dns_allow_value carries the L7 policy attached to a matched DNS name. + * port_count = 0 is the "unspecified" case: the datapath applies the default + * port set {80/http, 443/https} for backward compatibility with rules that + * omit port. port_count > 0 restricts L7 handling to the listed (port, scheme) + * tuples only. + */ struct dns_allow_value { __u32 name_len; __u8 flags; - __u8 reserved[3]; + __u8 port_count; + __u8 reserved[2]; + struct l7_port_entry ports[MAX_L7_PORTS_PER_HOST]; }; struct dns_query_track_key { @@ -165,7 +241,9 @@ struct dns_query_track_key { struct dns_query_track_value { __u64 expires_at_ns; __u8 flags; - __u8 reserved[7]; + __u8 port_count; + __u8 reserved[6]; + struct l7_port_entry ports[MAX_L7_PORTS_PER_HOST]; }; /* Per-packet query parser state shared by the DNS tail-call pipeline. */ @@ -182,12 +260,36 @@ struct dns_query_state { char name[MAX_DNS_NAME_LEN]; }; +/* net_policy_value_v2 stores the per-sandbox allow_out_v2 verdict. This is the + * legacy 16-byte layout read only when migrating a pre-v3 allow_out_v2 map to + * allow_out_v3; the current dataplane uses net_policy_value_v3. + */ struct net_policy_value_v2 { __u64 expires_at_ns; __u8 flags; __u8 reserved[7]; }; +/* Per-sandbox allow_out_v3 verdict. Unlike v2, the port lives in the + * LPM key (see lpm_key_v3), so the value no longer needs the 8-tuple + * (port, scheme) array: the scheme is resolved at insert time and + * stored here directly. A zero expires_at_ns is a static entry; a + * non-zero expires_at_ns is a temporary DNS-learned entry. + */ +struct net_policy_value_v3 { + __u64 expires_at_ns; + __u8 flags; + __u8 scheme; /* L7_SCHEME_* */ + /* prefixlen of the lpm_key_v3 this value was written under. LPM trie + * lookups are longest-prefix, so a lookup for key K may return an + * entry written under a SHORTER covering key; writers that mean to + * merge with an existing entry for the EXACT same key must compare + * this field against their key's prefixlen first. + */ + __u8 key_prefixlen; + __u8 reserved[5]; +}; + struct mvm_port { __u32 ifindex; __u16 listen_port; @@ -214,7 +316,9 @@ struct nat_session { __u16 vm_port; __u8 state; __u8 active_close; - __u8 reserved[34]; + __u8 packet_class; /* SNAT_PACKET or L7PROXY_PACKET */ + __u8 l7_scheme; /* L7_SCHEME_*; NONE for non-L7 sessions */ + __u8 reserved[32]; }; struct ingress_session { @@ -275,24 +379,29 @@ static __always_inline int _() { int b[sizeof(struct mvm_meta) == 128 ? 1 : -1] = {}; int d[sizeof(struct lpm_key) == 8 ? 1 : -1] = {}; + int dv3[sizeof(struct lpm_key_v3) == 12 ? 1 : -1] = {}; int r[sizeof(struct net_policy_value_v2) == 16 ? 1 : -1] = {}; + int rv3[sizeof(struct net_policy_value_v3) == 16 ? 1 : -1] = {}; int f[sizeof(struct dns_allow_key) == MAX_DNS_NAME_LEN + 4 ? 1 : -1] = {}; - int g[sizeof(struct dns_allow_value) == 8 ? 1 : -1] = {}; + int g[sizeof(struct dns_allow_value) == 40 ? 1 : -1] = {}; int h[sizeof(struct dns_query_track_key) == 24 ? 1 : -1] = {}; - int i[sizeof(struct dns_query_track_value) == 16 ? 1 : -1] = {}; + int i[sizeof(struct dns_query_track_value) == 48 ? 1 : -1] = {}; int l[sizeof(struct mvm_port) == 8 ? 1 : -1] = {}; int n[sizeof(struct session_key) % 20 == 0 ? 1 : -1] = {}; - int o[sizeof(struct nat_session) % 64 == 0 ? 1 : -1] = {}; + int o[sizeof(struct nat_session) == 64 ? 1 : -1] = {}; int p[sizeof(struct ingress_session) % 16 == 0 ? 1 : -1] = {}; int q[sizeof(struct snat_ip) % 16 == 0 ? 1 : -1] = {}; + int s[sizeof(struct l7_port_entry) == 4 ? 1 : -1] = {}; - return b[0] + d[0] + r[0] + f[0] + g[0] + h[0] + i[0] + l[0] + n[0] + o[0] + p[0] + q[0]; + return b[0] + d[0] + dv3[0] + r[0] + rv3[0] + f[0] + g[0] + h[0] + i[0] + l[0] + n[0] + o[0] + p[0] + q[0] + s[0]; } static __always_inline __attribute__((used)) __u32 __btf_pin(void) { return __builtin_btf_type_id(*(struct lpm_key *)0, BPF_TYPE_ID_LOCAL) + __builtin_btf_type_id(*(struct net_policy_value_v2 *)0, BPF_TYPE_ID_LOCAL) + + __builtin_btf_type_id(*(struct lpm_key_v3 *)0, BPF_TYPE_ID_LOCAL) + + __builtin_btf_type_id(*(struct net_policy_value_v3 *)0, BPF_TYPE_ID_LOCAL) + __builtin_btf_type_id(*(struct dns_allow_key *)0, BPF_TYPE_ID_LOCAL) + __builtin_btf_type_id(*(struct dns_allow_value *)0, BPF_TYPE_ID_LOCAL); } diff --git a/CubeNet/src/dns_learn_test.bpf.c b/CubeNet/src/dns_learn_test.bpf.c new file mode 100644 index 000000000..e88e2ba02 --- /dev/null +++ b/CubeNet/src/dns_learn_test.bpf.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (c) 2026 Cube Authors */ +#include +#include +#include +#include + +#include "dns_response.h" + +/* Single-slot store for the dns_query_track_value under test. The dataplane + * reads it as a map-value pointer (mirroring production, where the query is a + * dns_query_track map value) so the verifier accepts the per-port loop's + * bounded offsets — a stack copy would risk rejection as a variable-offset + * stack access when the loop is not fully unrolled. + */ +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct dns_query_track_value); +} test_query_store SEC(".maps"); + +struct dns_learn_case { + __u32 ifindex; + __u32 ip; /* network byte order */ + __u32 ttl; /* seconds */ +}; + +SEC("tc") +int test_dns_learn(struct __sk_buff *skb) +{ + struct dns_learn_case tc = {}; + __u32 qkey = 0; + struct dns_query_track_value *query; + + if (bpf_skb_load_bytes(skb, 0, &tc, sizeof(tc))) + return TC_ACT_SHOT; + + query = bpf_map_lookup_elem(&test_query_store, &qkey); + if (!query) + return TC_ACT_SHOT; + + dns_learn_response_ip(tc.ifindex, tc.ip, tc.ttl, query); + return TC_ACT_OK; +} + +char __license[] SEC("license") = "Dual BSD/GPL"; diff --git a/CubeNet/src/dns_query.h b/CubeNet/src/dns_query.h index 021a48c2c..a50f16445 100644 --- a/CubeNet/src/dns_query.h +++ b/CubeNet/src/dns_query.h @@ -178,10 +178,15 @@ static __always_inline bool dns_query_should_filter_ipv4_a(struct __sk_buff *skb return is_ipv4_a; } -/* Track an allowed IPv4 A query so the response can inherit its L7 flags. */ +/* Track an allowed IPv4 A query so the response can inherit its L7 flags + * and per-host port set. Copies port_count + ports[] verbatim from the matched + * dns_allow_value so dns_learn_response_ip can rebuild net_policy_value_v3 + * without a second dns_allow_v2 lookup at response time. + */ static __always_inline void dns_track_allowed_query(struct __sk_buff *skb, const struct dns_query_state *state, - __u8 flags, __u64 qname_hash) + const struct dns_allow_value *matched, + __u64 qname_hash) { struct dns_query_track_key track_key = {}; struct dns_query_track_value track_value = {}; @@ -189,6 +194,7 @@ static __always_inline void dns_track_allowed_query(struct __sk_buff *skb, struct ethhdr *l2; struct iphdr *l3; struct udphdr *udp; + int i; if (!__pull_headers_udp(skb, &l2, &l3, &udp)) return; @@ -200,8 +206,26 @@ static __always_inline void dns_track_allowed_query(struct __sk_buff *skb, track_key.source_port = udp->source; track_key.dns_id = hdr.id; track_key.qname_hash = qname_hash; - track_value.flags = flags; + track_value.flags = matched->flags; track_value.expires_at_ns = bpf_ktime_get_ns() + DNS_QUERY_TRACK_TTL_NS; + track_value.port_count = matched->port_count; + if (track_value.port_count > MAX_L7_PORTS_PER_HOST) + track_value.port_count = MAX_L7_PORTS_PER_HOST; + +#pragma unroll + for (i = 0; i < MAX_L7_PORTS_PER_HOST; i++) { + /* Guard inside body (not `break`) so clang keeps this as a + * fixed-trip-count loop and fully unrolls it — otherwise the BPF + * verifier rejects ports[i] writes as variable-offset stack + * accesses. Field-by-field assignment sidesteps clang's memcpy + * lowering for whole-struct copies (which is disallowed in BPF). + */ + if (i < track_value.port_count) { + track_value.ports[i].port = matched->ports[i].port; + track_value.ports[i].scheme = matched->ports[i].scheme; + track_value.ports[i]._pad = matched->ports[i]._pad; + } + } bpf_map_update_elem(&dns_query_track, &track_key, &track_value, BPF_ANY); } @@ -226,7 +250,7 @@ static __always_inline void dns_init_query_state(struct dns_query_state *state, /* Query hook for sandbox-originated UDP/53 traffic. * - * Each sandbox owns one precreated DNS allow LPM trie via dns_allow[ifindex]. + * Each sandbox owns one precreated DNS allow LPM trie via dns_allow_v2[ifindex]. * Callers only invoke this hook when per-sandbox metadata flags in * ifindex_to_mvmmeta enable DNS policy processing. The DNS allow trie stores * only reversed lower-case domain rules and their rule-specific flags. @@ -243,7 +267,7 @@ static __always_inline int dns_handle_query(struct __sk_buff *skb, __u32 dns_off if (!dns_read_query_header(skb, dns_off, &hdr, &flags)) return CUBE_DNS_PASS; - inner_map = bpf_map_lookup_elem(&dns_allow, &ifindex); + inner_map = bpf_map_lookup_elem(&dns_allow_v2, &ifindex); if (!inner_map) return CUBE_DNS_PASS; diff --git a/CubeNet/src/dns_response.h b/CubeNet/src/dns_response.h index 511acb631..133000279 100644 --- a/CubeNet/src/dns_response.h +++ b/CubeNet/src/dns_response.h @@ -98,31 +98,113 @@ static __always_inline bool dns_response_learning_enabled(__u32 ifindex) return mvm_meta && (mvm_meta->dns_policy_flags & DNS_POLICY_FLAG_LEARNING_ENABLED); } -/* Add an IPv4 A-record address as a temporary DNS-learned allow_out_v2 entry. */ +/* Add an IPv4 A-record address as temporary DNS-learned allow_out_v3 + * entries. + * + * Two shapes, mirroring how the rule was installed: + * - Plain (non-L7) allow: a single /32 (any-port) entry, exactly as the + * pre-v3 dataplane did. Without this a plain domain-based allow rule + * would resolve via DNS yet never be admitted by classify_egress_flow, + * regressing to a default-deny drop. + * - L7 allow: v3 carries the destination port in the LPM key, so we write + * one exact (ip, port)/48 entry per (port, scheme) the query inherited + * from its matched dns_allow_v2 rule. port_count == 0 means the default + * {80/http, 443/https} set. + * Each entry is refreshed independently; an existing entry for the EXACT + * same key keeps its flags and its (zero) expiry, so a static rule + * survives a later DNS refresh of the same IP. "Exact" is checked via + * old->key_prefixlen: the LPM lookup alone is longest-prefix and would + * otherwise match a shorter COVERING entry (e.g. a static /24), making + * the learned entry wrongly inherit the static zero expiry (never ages) + * or the covering entry's flags. + */ static __always_inline void dns_learn_response_ip(__u32 ifindex, __u32 ip, __u32 ttl, - __u8 flags) + const struct dns_query_track_value *query) { - struct lpm_key key = { .prefixlen = 32, .ip = ip }; - struct net_policy_value_v2 value = { - .expires_at_ns = bpf_ktime_get_ns() + ((__u64)ttl * NSEC_PER_SEC), - .flags = flags, - }; - struct net_policy_value_v2 *old_value; + __u64 now = bpf_ktime_get_ns(); + __u64 expires = now + ((__u64)ttl * NSEC_PER_SEC); void *inner_map; - inner_map = bpf_map_lookup_elem(&allow_out_v2, &ifindex); + inner_map = bpf_map_lookup_elem(&allow_out_v3, &ifindex); if (!inner_map) return; - /* DNS learning must not downgrade flags or shorten static allow rules. */ - old_value = bpf_map_lookup_elem(inner_map, &key); - if (old_value) { - value.flags |= old_value->flags; - if (old_value->expires_at_ns == 0) - value.expires_at_ns = 0; + /* Learn the plain /32 (any-port) entry whenever the domain is + * plain-allowed. For a non-L7 allow this is the whole policy; for a domain + * that is both plain-allowed and L7-ruled (NET_POLICY_FLAG_L3_ALLOWED) it + * coexists with the /48 L7 entries written below — a non-rule port falls + * back to this /32 (plain SNAT) while the rule's ports match the longer + * /48 prefix (L7 intercept) in classify_egress_flow. The L7/L3 marker bits + * are stripped so the entry reads as a plain allow; for a non-L7 query + * (flags==0) the mask is a no-op, so both cases share this one block. + */ + if (!(query->flags & NET_POLICY_FLAG_L7_REQUIRED) || + (query->flags & NET_POLICY_FLAG_L3_ALLOWED)) { + struct lpm_key_v3 key = { .prefixlen = 32, .ip = ip, .port = 0 }; + struct net_policy_value_v3 value = { + .expires_at_ns = expires, + .flags = query->flags & + ~(__u8)(NET_POLICY_FLAG_L7_REQUIRED | NET_POLICY_FLAG_L3_ALLOWED), + .scheme = L7_SCHEME_NONE, + .key_prefixlen = 32, + }; + struct net_policy_value_v3 *old = bpf_map_lookup_elem(inner_map, &key); + if (old && old->key_prefixlen == key.prefixlen) { + value.flags |= old->flags; + if (old->expires_at_ns == 0) + value.expires_at_ns = 0; + } + bpf_map_update_elem(inner_map, &key, &value, BPF_ANY); } - bpf_map_update_elem(inner_map, &key, &value, BPF_ANY); + /* A non-L7 allow is fully handled by the /32 entry above. */ + if (!(query->flags & NET_POLICY_FLAG_L7_REQUIRED)) + return; + + /* L7 allow: one exact (ip, port)/48 entry per (port, scheme). + * + * The loop uses a fixed trip count with the bound as a guard inside the + * body (not `break`), so clang fully unrolls it. That turns each + * query->ports[i] into a constant-offset access the verifier accepts. A + * `break`-bound loop is not unrolled and was rejected by the verifier as + * a variable-offset map-value read (off beyond the 48-byte value). + */ + __u8 count = query->port_count; + bool use_defaults = count == 0; + __u8 limit = use_defaults ? 2 : count; + if (limit > MAX_L7_PORTS_PER_HOST) + limit = MAX_L7_PORTS_PER_HOST; + +#pragma unroll + for (__u8 i = 0; i < MAX_L7_PORTS_PER_HOST; i++) { + if (i >= limit) + continue; + + __u16 port; + __u8 scheme; + if (use_defaults) { + port = (i == 0) ? bpf_htons(80) : bpf_htons(443); + scheme = (i == 0) ? L7_SCHEME_HTTP : L7_SCHEME_HTTPS; + } else { + port = query->ports[i].port; + scheme = query->ports[i].scheme; + } + + struct lpm_key_v3 key = { .prefixlen = 48, .ip = ip, .port = port }; + struct net_policy_value_v3 value = { + .expires_at_ns = expires, + .flags = query->flags, + .scheme = scheme, + .key_prefixlen = 48, + }; + struct net_policy_value_v3 *old = bpf_map_lookup_elem(inner_map, &key); + if (old && old->key_prefixlen == key.prefixlen) { + value.flags |= old->flags; + if (old->expires_at_ns == 0) + value.expires_at_ns = 0; + } + bpf_map_update_elem(inner_map, &key, &value, BPF_ANY); + } } /* Return true when an answer RR carries an IN A record payload. */ @@ -142,7 +224,7 @@ static __always_inline bool dns_response_record_is_ipv4_a(const struct dns_rr_he */ static __always_inline bool dns_process_response_answer(struct __sk_buff *skb, __u32 *cursor, __u32 ifindex, - __u8 flags) + const struct dns_query_track_value *query) { struct dns_rr_header rr; __u16 rdlength; @@ -161,7 +243,7 @@ static __always_inline bool dns_process_response_answer(struct __sk_buff *skb, return false; ttl = bpf_ntohl(rr.ttl); if (ttl < 300) ttl = 300; - dns_learn_response_ip(ifindex, ip, ttl, flags); + dns_learn_response_ip(ifindex, ip, ttl, query); } /* Keep cursor advancement bounded even for unsupported RR types. */ @@ -187,7 +269,7 @@ static __always_inline struct dns_query_track_value *dns_lookup_response_query( /* Response hook for DNS replies returning to a sandbox. * - * The path learns IPv4 A records into allow_out_v2 as temporary DNS-learned IP + * The path learns IPv4 A records into allow_out_v3 as temporary DNS-learned IP * policy entries. It intentionally preserves the existing filtering semantics. * * Marked __always_inline so the calling SEC("tc") program contains no @@ -238,7 +320,7 @@ static __always_inline void dns_handle_response(struct __sk_buff *skb, __u32 dns for (i = 0; i < DNS_MAX_RESPONSE_ANSWERS; i++) { if (i >= ancount) break; - if (!dns_process_response_answer(skb, &cursor, ifindex, query->flags)) + if (!dns_process_response_answer(skb, &cursor, ifindex, query)) goto delete_query; } diff --git a/CubeNet/src/egress_policy_test.bpf.c b/CubeNet/src/egress_policy_test.bpf.c new file mode 100644 index 000000000..ca3f3d8b5 --- /dev/null +++ b/CubeNet/src/egress_policy_test.bpf.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (c) 2026 Cube Authors */ +#include +#include +#include +#include + +#include "session.h" + +struct egress_policy_case { + __u32 ifindex; + __u32 daddr; + __u16 dport; + __u8 verdict; + __u8 reserved[5]; +}; + +SEC("tc") +int test_classify_egress_flow(struct __sk_buff *skb) +{ + struct egress_policy_case tc = {}; + + if (bpf_skb_load_bytes(skb, 0, &tc, sizeof(tc))) + return TC_ACT_SHOT; + + tc.verdict = classify_egress_flow(tc.ifindex, tc.daddr, tc.dport); + if (bpf_skb_store_bytes(skb, 0, &tc, sizeof(tc), 0)) + return TC_ACT_SHOT; + + return TC_ACT_OK; +} + +char __license[] SEC("license") = "Dual BSD/GPL"; diff --git a/CubeNet/src/icmp.h b/CubeNet/src/icmp.h index 0bfaeca81..dbacb70e1 100644 --- a/CubeNet/src/icmp.h +++ b/CubeNet/src/icmp.h @@ -40,7 +40,7 @@ static __always_inline bool create_icmp_sessions(struct __sk_buff *skb, struct snat_ip *snat_ip, __u16 snat_id) { return create_nat_session(skb, ekey, now_ns, vm_ifindex, snat_ip, snat_id, - ICMP_CT_UNREPLIED); + ICMP_CT_UNREPLIED, SNAT_PACKET, L7_SCHEME_NONE); } #endif /* __ICMP_H */ diff --git a/CubeNet/src/l7_mark_test.bpf.c b/CubeNet/src/l7_mark_test.bpf.c new file mode 100644 index 000000000..c56bd5367 --- /dev/null +++ b/CubeNet/src/l7_mark_test.bpf.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (c) 2026 Cube Authors */ +#include +#include +#include +#include + +#include "cubevs.h" + +struct l7_mark_case { + __u32 mark; +}; + +SEC("tc") +int test_l7_mark(struct __sk_buff *skb) +{ + struct l7_mark_case tc = {}; + + if (bpf_skb_load_bytes(skb, 0, &tc, sizeof(tc))) + return TC_ACT_SHOT; + + /* Mirror mvmtap.bpf.c's L7 stamp: keep the low (user) bits of skb->mark, + * then OR in the configured cube HTTP mark. + */ + tc.mark = (tc.mark & ~cube_l7_mark_mask) | cube_l7_mark_http; + + if (bpf_skb_store_bytes(skb, 0, &tc, sizeof(tc), 0)) + return TC_ACT_SHOT; + return TC_ACT_OK; +} + +char __license[] SEC("license") = "Dual BSD/GPL"; diff --git a/CubeNet/src/localgw.bpf.c b/CubeNet/src/localgw.bpf.c index 898fc4d7c..3b6b52183 100644 --- a/CubeNet/src/localgw.bpf.c +++ b/CubeNet/src/localgw.bpf.c @@ -10,6 +10,40 @@ #include "map.h" #include "nat.h" #include "skb.h" +#include "tcp.h" + +/* Update a CubeEgress TCP session from the original reply tuple before DNAT + * rewrites the sandbox-assigned destination address to mvm_inner_ip. A missing + * or non-L7 session is intentionally non-blocking. + */ +static __always_inline void update_l7_session_from_cubedev(struct __sk_buff *skb) +{ + struct session_key key = {}; + struct nat_session *sess; + struct ethhdr *l2; + struct iphdr *l3; + struct tcphdr *l4; + __u64 now; + + if (!__pull_headers(skb, &l2, &l3, &l4)) + return; + if (l3->protocol != IPPROTO_TCP) + return; + + key.src_ip = l3->saddr; + key.dst_ip = l3->daddr; + key.src_port = l4->source; + key.dst_port = l4->dest; + key.version = 0; + key.protocol = IPPROTO_TCP; + sess = lookup_session(&key); + if (!sess || sess->packet_class != L7PROXY_PACKET) + return; + + now = bpf_ktime_get_ns(); + update_session(IP_CT_DIR_REPLY, sess, now, l4->syn, l4->ack, + l4->fin, l4->rst); +} /* This filter will be attached to the egress path of cube-dev device. * It performs a DNAT and then redirect the traffics to Sandbox TAP devices. @@ -26,6 +60,8 @@ int from_envoy(struct __sk_buff *skb) if (skb->protocol != bpf_htons(ETH_P_IP)) return TC_ACT_OK; + update_l7_session_from_cubedev(skb); + ret = pull_headers(skb, &l2, &l3); if (ret != TC_ACT_OK) return ret; diff --git a/CubeNet/src/map.h b/CubeNet/src/map.h index 6f93c3519..4ea2d3a48 100644 --- a/CubeNet/src/map.h +++ b/CubeNet/src/map.h @@ -96,13 +96,18 @@ struct { __uint(pinning, LIBBPF_PIN_BY_NAME); } snat_iplist SEC(".maps"); -/* Egress allow list v2 (hash of maps) +/* Egress allow list v3 (hash of maps) * * key: ifindex of the TAP device - * value: fd of inner LPM trie map (destination IP allow list) + * value: fd of inner LPM trie map (destination ip[:port] allow list) * - * Inner values use net_policy_value_v2. A zero expires_at_ns means a static - * allow entry; a non-zero expires_at_ns means a temporary DNS-learned entry. + * Inner keys use lpm_key_v3 so a single longest-prefix lookup resolves + * exact (ip, port) (prefixlen 48), ip-only / any-port (prefixlen 32), + * or ip/mask subnet (prefixlen < 32) rules. Inner values use + * net_policy_value_v3, which marks the L7 scheme directly (the port is + * now part of the key, so no per-packet (port, scheme) array scan). + * A zero expires_at_ns means a static entry; a non-zero expires_at_ns + * means a temporary DNS-learned entry. */ struct { __uint(type, BPF_MAP_TYPE_HASH_OF_MAPS); @@ -112,11 +117,11 @@ struct { __array(values, struct { __uint(type, BPF_MAP_TYPE_LPM_TRIE); __uint(max_entries, MAX_IP_RULE_ENTRIES); - __type(key, struct lpm_key); - __type(value, struct net_policy_value_v2); + __type(key, struct lpm_key_v3); + __type(value, struct net_policy_value_v3); __uint(map_flags, BPF_F_NO_PREALLOC); }); -} allow_out_v2 SEC(".maps"); +} allow_out_v3 SEC(".maps"); /* Egress deny list (hash of maps) * @@ -146,7 +151,7 @@ struct { * value: fd of inner LPM trie map for this sandbox's DNS policy rules * * Inner keys are reversed lower-case domain name prefixes. DNS policy mode is - * stored in ifindex_to_mvmmeta, while dns_allow stores only domain rules. + * stored in ifindex_to_mvmmeta, while dns_allow_v2 stores only domain rules. * Exact rule "qq.com" is encoded as "moc.qq\0" with the trailing NUL included * in prefixlen. Wildcard rule "*.qq.com" is encoded as "moc.qq." without NUL, * so only subdomains such as "a.qq.com" can match it. @@ -163,13 +168,13 @@ struct { __type(value, struct dns_allow_value); __uint(map_flags, BPF_F_NO_PREALLOC); }); -} dns_allow SEC(".maps"); +} dns_allow_v2 SEC(".maps"); /* Pending DNS queries waiting for responses. * * key: sandbox ifindex + DNS server IP + sandbox UDP source port + DNS id * + raw DNS QNAME hash - * value: L7 flags inherited from dns_allow and pending expiration time + * value: L7 flags inherited from dns_allow_v2 and pending expiration time */ struct { __uint(type, BPF_MAP_TYPE_LRU_HASH); diff --git a/CubeNet/src/mvmtap.bpf.c b/CubeNet/src/mvmtap.bpf.c index 1d71f5a74..c265d6bc6 100644 --- a/CubeNet/src/mvmtap.bpf.c +++ b/CubeNet/src/mvmtap.bpf.c @@ -106,42 +106,17 @@ static __always_inline bool should_do_nat(const struct iphdr *l3) return true; } -/* - * Check whether a TCP flow should be redirected to the L7 proxy. - * - * Looks up allow_out_v2 for the given ifindex/daddr and returns true iff - * the entry carries NET_POLICY_FLAG_L7_REQUIRED and the destination port - * is 80 or 443. This is a fast, self-contained lookup — the general - * egress policy check (allow / deny) is enforced later inside - * create_nat_session(). +/* Egress flow classification now lives in classify_egress_flow() (session.h), + * which merges the former l7_scheme_for_flow() and session_policy_allowed() + * into a single policy verdict (reject / accept-SNAT / accept-HTTP / + * accept-HTTPS). It is applied once, when a new flow is created, and the + * result is cached in nat_session for reuse on every later packet. */ -static __always_inline bool should_redirect_to_l7_proxy(__u32 ifindex, __u32 daddr, - const struct tcphdr *l4) -{ - struct lpm_key key = { .prefixlen = 32, .ip = daddr }; - struct net_policy_value_v2 *value; - void *inner_map; - - if (l4->dest != bpf_htons(80) && l4->dest != bpf_htons(443)) - return false; - - inner_map = bpf_map_lookup_elem(&allow_out_v2, &ifindex); - if (!inner_map) - return false; - - value = bpf_map_lookup_elem(inner_map, &key); - if (!value) - return false; - if (value->expires_at_ns != 0 && value->expires_at_ns <= bpf_ktime_get_ns()) - return false; - - return value->flags & NET_POLICY_FLAG_L7_REQUIRED; -} - enum tcp_nat_result { TCP_NAT_DROP = 0, TCP_NAT_OK, TCP_NAT_RESET, + TCP_L7PROXY_OK, }; /* do_tcp_nat() returns a 64-bit value that encodes both the status enum @@ -410,6 +385,31 @@ static __always_inline struct snat_ip *pick_snat_ip_port(__u32 mvm_ip, const str return NULL; } +/* Reserve the reverse-flow key for an L7 proxy session. L7 traffic is not + * source-NATed, so the reply tuple is the exact reverse of the sandbox's + * original tuple. create_nat_session() later inserts the matching egress value + * and rolls this reservation back if that insertion fails. + */ +static __always_inline bool create_l7_ingress_session(const struct session_key *ekey) +{ + struct ingress_session isess = { + .version = ekey->version, + .vm_ip = ekey->src_ip, + .vm_port = ekey->src_port, + }; + struct session_key ikey = { + .src_ip = ekey->dst_ip, + .dst_ip = ekey->src_ip, + .src_port = ekey->dst_port, + .dst_port = ekey->src_port, + .version = 0, + .protocol = ekey->protocol, + }; + + return bpf_map_update_elem(&ingress_sessions, &ikey, &isess, + BPF_NOEXIST) == 0; +} + static __always_inline void del_session(struct session_key *ekey, struct nat_session *sess) { struct session_key ikey = { @@ -470,6 +470,9 @@ static __always_inline __u32 do_icmp_nat(struct __sk_buff *skb, struct mvm_meta } /* create new session */ + if (classify_egress_flow(skb->ingress_ifindex, key.dst_ip, + key.dst_port) == FLOW_REJECT) + return 0; snat_ip = pick_snat_ip_port(mvm_meta->ip, &key, &snat_id); if (!snat_ip || !snat_ip->ip || !snat_id) return 0; @@ -568,6 +571,9 @@ static __always_inline __u32 do_udp_nat_inline(struct __sk_buff *skb, } /* create new session */ + if (classify_egress_flow(skb->ingress_ifindex, key.dst_ip, + key.dst_port) == FLOW_REJECT) + return 0; snat_ip = pick_snat_ip_port(mvm_meta->ip, &key, &snat_port); if (!snat_ip || !snat_ip->ip || !snat_port) return 0; @@ -681,6 +687,7 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * struct session_key key = {}; struct nat_session *sess; struct snat_ip *snat_ip; + struct snat_ip l7_endpoint = {}; bool syn, ack, fin, rst; struct ethhdr *l2; struct iphdr *l3; @@ -689,6 +696,10 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * __u16 snat_port; __u64 flags; __u64 now; + __u32 l7_mark; + __u8 packet_class = SNAT_PACKET; + __u8 l7_scheme = L7_SCHEME_NONE; + __u8 verdict = FLOW_SNAT; long err; bool ok; @@ -711,6 +722,16 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * sess = bpf_map_lookup_elem(&egress_sessions, &key); if (sess) { if (sess->state == TCP_CONNTRACK_CLOSE || sess->state == TCP_CONNTRACK_TIME_WAIT) { + /* L7 sessions use an identity reverse tuple, so immediately + * replacing a terminal session would let delayed packets from + * the old connection mutate the new session. Keep the old pair + * until the userspace reaper removes it and reject premature + * tuple reuse. Ordinary SNAT sessions remain safe to recreate + * because they allocate a fresh reverse-side source port. + */ + if (sess->packet_class == L7PROXY_PACKET) + return TCP_NAT_PACK(0, TCP_NAT_RESET); + /* guest kernel reuse source port too fast */ del_session(&key, sess); goto do_create; @@ -719,28 +740,87 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * goto do_update; } do_create: - /* create new session */ + /* Classify the flow with the unified egress policy. The verdict is + * cached in nat_session and reused for every later packet. + */ + verdict = classify_egress_flow(skb->ingress_ifindex, key.dst_ip, + key.dst_port); + switch (verdict) { + case FLOW_HTTP: + case FLOW_HTTPS: + if (!create_l7_ingress_session(&key)) + return TCP_NAT_DROP; + l7_endpoint.ifindex = cubegw0_ifindex; + l7_endpoint.ip = key.src_ip; + snat_ip = &l7_endpoint; + snat_port = key.src_port; + packet_class = L7PROXY_PACKET; + l7_scheme = (verdict == FLOW_HTTP) ? L7_SCHEME_HTTP : + L7_SCHEME_HTTPS; + break; + case FLOW_REJECT: + /* Denied by egress policy: signal the caller so it can emit an + * RST, exactly as the old create_nat_session path did. + */ + nat_cb_set(skb, NAT_CB_DENIED_BY_POLICY); + return TCP_NAT_PACK(0, TCP_NAT_RESET); + case FLOW_SNAT: + default: snat_ip = pick_snat_ip_port(mvm_meta->ip, &key, &snat_port); if (!snat_ip || !snat_ip->ip || !snat_port) return TCP_NAT_DROP; - ok = create_new_sessions(skb, &key, now, skb->ingress_ifindex, snat_ip, snat_port); - if (!ok) { - /* Preserve RST-on-deny: create_nat_session stamps - * skb->cb when the failure is due to net policy. - */ - if (nat_cb_get(skb) == NAT_CB_DENIED_BY_POLICY) - return TCP_NAT_PACK(0, TCP_NAT_RESET); - return TCP_NAT_DROP; - } - sess = bpf_map_lookup_elem(&egress_sessions, &key); - if (!sess) - return TCP_NAT_DROP; - goto do_nat; + break; + } + ok = create_new_sessions(skb, &key, now, skb->ingress_ifindex, + snat_ip, snat_port, packet_class, l7_scheme); + if (!ok) + return TCP_NAT_DROP; + sess = bpf_map_lookup_elem(&egress_sessions, &key); + if (!sess) + return TCP_NAT_DROP; + goto do_nat; } else { /* lookup existing session */ sess = bpf_map_lookup_elem(&egress_sessions, &key); - if (!sess) + if (!sess) { + /* Legacy default-port (80/443) connection drain: the eBPF session + * entry was lost (agent restart, map eviction, or expiry) AND the + * allow_out_v3 /48 entry for this (ip, port) has aged out, so + * classify_egress_flow would return FLOW_REJECT. But the proxy + * still holds an established TPROXY socket for this 4-tuple — the + * connection was legitimately opened when the policy allowed it. + * Re-stamp the mark so iptables TPROXY steers the packet to the + * proxy, keeping the connection alive instead of resetting it. + * Custom-port connections are intentionally excluded: they should + * respect the current policy when their allow_out_v3 entry expires. + * No session is re-created, so each packet on the drained flow + * re-enters this path (per-packet socket lookup — acceptable for + * draining connections that will eventually close). + */ + if (l4->dest == bpf_htons(80) || l4->dest == bpf_htons(443)) { + struct bpf_sock *sk; + struct bpf_sock_tuple tuple = {}; + tuple.ipv4.saddr = key.src_ip; + tuple.ipv4.daddr = key.dst_ip; + tuple.ipv4.sport = l4->source; + tuple.ipv4.dport = l4->dest; + sk = bpf_skc_lookup_tcp(skb, &tuple, sizeof(tuple.ipv4), BPF_F_CURRENT_NETNS, 0); + if (sk) { + __u32 state = sk->state; + + bpf_sk_release(sk); + if (state == BPF_TCP_ESTABLISHED) { + if (l4->dest == bpf_htons(80)) { + skb->mark = (skb->mark & ~cube_l7_mark_mask) | cube_l7_mark_http; + } else { + skb->mark = (skb->mark & ~cube_l7_mark_mask) | cube_l7_mark_https; + } + return TCP_NAT_PACK(cubegw0_ifindex, TCP_L7PROXY_OK); + } + } + } return rst ? TCP_NAT_DROP : TCP_NAT_RESET; + } } do_update: @@ -748,6 +828,18 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * update_session(IP_CT_DIR_ORIGINAL, sess, now, syn, ack, fin, rst); do_nat: + if (sess->packet_class == L7PROXY_PACKET) { + if (sess->l7_scheme == L7_SCHEME_HTTP) + l7_mark = cube_l7_mark_http; + else if (sess->l7_scheme == L7_SCHEME_HTTPS) + l7_mark = cube_l7_mark_https; + else + return TCP_NAT_DROP; + + skb->mark = (skb->mark & ~cube_l7_mark_mask) | l7_mark; + return TCP_NAT_PACK(sess->node_ifindex, TCP_L7PROXY_OK); + } + old_saddr = l3->saddr; new_saddr = sess->node_ip; old_sport = l4->source; @@ -880,7 +972,7 @@ int dns_finish(struct __sk_buff *skb) if (!dns_policy_enabled(mvm_meta)) return finish_udp_nat(skb, mvm_meta); - inner_map = bpf_map_lookup_elem(&dns_allow, &ifindex); + inner_map = bpf_map_lookup_elem(&dns_allow_v2, &ifindex); if (!inner_map) return finish_udp_nat(skb, mvm_meta); @@ -895,7 +987,7 @@ int dns_finish(struct __sk_buff *skb) if (!matched) return finish_udp_nat(skb, mvm_meta); - dns_track_allowed_query(skb, state, matched->flags, qname_hash); + dns_track_allowed_query(skb, state, matched, qname_hash); return finish_udp_nat(skb, mvm_meta); } @@ -907,10 +999,8 @@ int from_cube(struct __sk_buff *skb) { __u32 daddr, ifindex, dst_ifindex; __u64 tcp_ret; - struct bpf_sock_tuple tuple = {}; struct mvm_port mvm_port = {}; struct mvm_meta *mvm_meta; - struct bpf_sock *sk; struct ethhdr *l2; struct iphdr *l3; struct tcphdr *l4; @@ -994,23 +1084,6 @@ int from_cube(struct __sk_buff *skb) } } - if (proto == IPPROTO_TCP && - __pull_headers(skb, &l2, &l3, &l4) && - (l4->dest == bpf_htons(80) || l4->dest == bpf_htons(443))) { - tuple.ipv4.saddr = mvm_meta->ip; - tuple.ipv4.daddr = daddr; - tuple.ipv4.sport = l4->source; - tuple.ipv4.dport = l4->dest; - sk = bpf_skc_lookup_tcp(skb, &tuple, sizeof(tuple.ipv4), BPF_F_CURRENT_NETNS, 0); - if (sk) { - __u32 state = sk->state; - - bpf_sk_release(sk); - if (state == BPF_TCP_ESTABLISHED) - return bpf_redirect(cubegw0_ifindex, BPF_F_INGRESS); - } - } - ret = pull_headers(skb, &l2, &l3); if (ret != TC_ACT_OK) return ret; @@ -1020,26 +1093,27 @@ int from_cube(struct __sk_buff *skb) if (l3->daddr == nodenic_ip) { /* This branch bypasses do_*_nat() and therefore the policy - * check inside create_nat_session(). Enforce policy inline. - * TCP callers get an RST to match the guest-visible behavior - * of the do_tcp_nat() path; UDP/ICMP silently drop. + * check applied there. Enforce the unified egress policy + * inline. TCP callers get an RST to match the guest-visible + * behavior of the do_tcp_nat() path; UDP/ICMP silently drop. + * dport is 0 because the original check was port-agnostic. */ - if (!session_policy_allowed(ifindex, daddr)) { + switch (classify_egress_flow(ifindex, daddr, 0)) { + case FLOW_REJECT: if (proto == IPPROTO_TCP) return tcp_reply_reset(skb, ifindex); return TC_ACT_SHOT; + default: + return bpf_redirect(cubegw0_ifindex, BPF_F_INGRESS); } - return bpf_redirect(cubegw0_ifindex, BPF_F_INGRESS); } if (proto == IPPROTO_TCP) { - if (!__pull_headers(skb, &l2, &l3, &l4)) - return TC_ACT_SHOT; - if (should_redirect_to_l7_proxy(ifindex, daddr, l4)) - return bpf_redirect(cubegw0_ifindex, BPF_F_INGRESS); tcp_ret = do_tcp_nat(skb, mvm_meta); if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_OK) return bpf_redirect(TCP_NAT_IFINDEX(tcp_ret), egress_redirect_flags); + if (TCP_NAT_STATUS(tcp_ret) == TCP_L7PROXY_OK) + return bpf_redirect(TCP_NAT_IFINDEX(tcp_ret), BPF_F_INGRESS); if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_RESET) return tcp_reply_reset(skb, ifindex); } diff --git a/CubeNet/src/nodenic.bpf.c b/CubeNet/src/nodenic.bpf.c index c8e1cdfc3..32c2380d4 100644 --- a/CubeNet/src/nodenic.bpf.c +++ b/CubeNet/src/nodenic.bpf.c @@ -68,24 +68,6 @@ static int tcp_nat_proxy(struct __sk_buff *skb, struct ethhdr *l2, struct iphdr return bpf_redirect(mvm_port->ifindex, 0); } -static __always_inline struct nat_session *lookup_session(const struct session_key *ikey) -{ - struct ingress_session *isess; - struct session_key key = {}; - - isess = bpf_map_lookup_elem(&ingress_sessions, ikey); - if (!isess) - return NULL; - - key.src_ip = isess->vm_ip; - key.dst_ip = ikey->src_ip; - key.src_port = isess->vm_port; - key.dst_port = ikey->src_port; - key.version = isess->version; - key.protocol = ikey->protocol; - return bpf_map_lookup_elem(&egress_sessions, &key); -} - static int tcp_nat_session(struct __sk_buff *skb, struct ethhdr *l2, struct iphdr *l3, struct tcphdr *l4) { __u32 old_daddr, new_daddr, tcp_csum_off; @@ -447,7 +429,7 @@ int dns_handle_response_prog(struct __sk_buff *skb) if (!rstate) return TC_ACT_OK; - /* Learn A records into allow_out_v2 before reverse NAT. */ + /* Learn A records into allow_out_v3 before reverse NAT. */ dns_handle_response(skb, rstate->dns_off, rstate->ifindex, rstate->server_ip, rstate->source_port); diff --git a/CubeNet/src/session.h b/CubeNet/src/session.h index 6b14a5ada..04c59f9e5 100644 --- a/CubeNet/src/session.h +++ b/CubeNet/src/session.h @@ -7,6 +7,11 @@ #include "cubevs.h" #include "map.h" +enum packet_class { + SNAT_PACKET = 0, + L7PROXY_PACKET, +}; + /* Lazy refresh threshold: 1 second in nanoseconds */ #define SESSION_REFRESH_INTERVAL_NS (1000 * 1000 * 1000UL) @@ -38,43 +43,131 @@ static __always_inline void session_mark_replied(enum ip_conntrack_dir dir, } /** - * session_policy_allowed - check egress network policy for a candidate flow - * @vm_ifindex: TAP ifindex of the originating MVM (policy key) - * @daddr: destination IP address in network byte order + * lookup_session - resolve a reverse-flow key to its egress NAT session + * @ikey: ingress/reply-direction session key + * + * ingress_sessions stores the sandbox identity needed to reconstruct the + * original-direction egress_sessions key. Both keys use network-byte-order + * addresses and ports. Returns the live map value, or NULL when either side + * of the session pair is missing. + */ +static __always_inline struct nat_session *lookup_session(const struct session_key *ikey) +{ + struct ingress_session *isess; + struct session_key ekey = {}; + + isess = bpf_map_lookup_elem(&ingress_sessions, ikey); + if (!isess) + return NULL; + + ekey.src_ip = isess->vm_ip; + ekey.dst_ip = ikey->src_ip; + ekey.src_port = isess->vm_port; + ekey.dst_port = ikey->src_port; + ekey.version = isess->version; + ekey.protocol = ikey->protocol; + + return bpf_map_lookup_elem(&egress_sessions, &ekey); +} + +/* Unified egress policy verdict for a candidate flow. + * + * Replaces the former pair l7_scheme_for_flow() (allow_out_v3 /48 L7 + * lookup) and session_policy_allowed() (allow_out_v3 /32 + deny_out /32). + * Callers classify once and act on the verdict: + * FLOW_REJECT - deny (drop / RST, never create a session) + * FLOW_SNAT - plain SNAT egress is allowed + * FLOW_HTTP - L7 proxy over HTTP is required + * FLOW_HTTPS - L7 proxy over HTTPS is required + */ +enum flow_verdict { + FLOW_REJECT = 0, + FLOW_SNAT, + FLOW_HTTP, + FLOW_HTTPS, +}; + +/** + * classify_egress_flow - single egress policy decision for a candidate flow + * @ifindex: TAP ifindex of the originating MVM (policy key) + * @daddr: destination IP address in network byte order + * @dport: destination port in network byte order (0 for port-agnostic) * - * Priority: allow_out_v2 > deny_out > default allow. + * Priority: allow_out_v3 > deny_out > default allow. * - * 1. If allow_out_v2 has an inner map for this ifindex and daddr matches - * a non-expired entry, the flow is explicitly allowed. - * 2. If deny_out has an inner map for this ifindex and daddr matches, - * the flow is denied. - * 3. Otherwise the flow is allowed. + * 1. Look up (daddr, dport)/48 in allow_out_v3. LPM automatically falls + * back to a matching /32 or subnet entry. A non-expired L7_REQUIRED + * entry returns FLOW_HTTP / FLOW_HTTPS; any other non-expired allow + * entry returns FLOW_SNAT. + * 2. Else if deny_out matches a /32 (or wider) entry, the flow is + * rejected (FLOW_REJECT). + * 3. Otherwise the flow is allowed via SNAT (default allow). * * Traffic to mvm_gateway_ip is internal (destined for cube-dev) and always * allowed regardless of policy. + * + * The L7 (/48) lookup is performed FIRST, before the deny check. This is + * the key fix for DNS-learned L7 entries: those are stored as (ip, port)/48 + * in allow_out_v3, but the old session_policy_allowed() used a hardcoded + * /32 key and could never match them, so an already-authorized L7 flow fell + * through to deny_out (e.g. 0.0.0.0/0) and was silently dropped. */ -static __always_inline bool session_policy_allowed(__u32 vm_ifindex, __u32 daddr) +static __always_inline __u8 classify_egress_flow(__u32 ifindex, __u32 daddr, + __u16 dport) { - struct lpm_key key = { .prefixlen = 32, .ip = daddr }; - struct net_policy_value_v2 *value; + struct lpm_key_v3 key = {}; + struct net_policy_value_v3 *value; void *inner_map; + __u64 now = bpf_ktime_get_ns(); + /* internal traffic destined for the MVM gateway is always allowed */ if (daddr == mvm_gateway_ip) - return true; + return FLOW_SNAT; - inner_map = bpf_map_lookup_elem(&allow_out_v2, &vm_ifindex); + /* 1) Allow: the /48 lookup resolves an exact L7 rule or falls back + * to a plain /32/subnet rule in the same LPM trie. + */ + inner_map = bpf_map_lookup_elem(&allow_out_v3, &ifindex); if (inner_map) { + key.prefixlen = 48; + key.ip = daddr; + key.port = dport; value = bpf_map_lookup_elem(inner_map, &key); if (value && (value->expires_at_ns == 0 || - value->expires_at_ns > bpf_ktime_get_ns())) - return true; + value->expires_at_ns > now)) { + if (value->flags & NET_POLICY_FLAG_L7_REQUIRED) { + if (value->scheme == L7_SCHEME_HTTP) + return FLOW_HTTP; + if (value->scheme == L7_SCHEME_HTTPS) + return FLOW_HTTPS; + /* L7 required but scheme unknown: fail closed rather + * than silently downgrading to plain SNAT, which would + * bypass the TPROXY intercept the rule asked for. A + * well-formed entry always carries a scheme (userspace + * populate and DNS-learn both set it), so reaching this + * branch means a corrupt or half-written map value. + */ + return FLOW_REJECT; + } + return FLOW_SNAT; + } } - inner_map = bpf_map_lookup_elem(&deny_out, &vm_ifindex); - if (inner_map && bpf_map_lookup_elem(inner_map, &key)) - return false; + /* 2) Deny: /32 (or wider) lookup in deny_out. deny_out inner maps are + * keyed by the 8-byte struct lpm_key (see map.h), so use a dedicated key + * rather than reusing the 12-byte lpm_key_v3 above — passing a v3 key to + * an 8-byte-key map only works because the kernel reads map->key_size + * bytes, which is an accident of struct layout, not a contract. + */ + inner_map = bpf_map_lookup_elem(&deny_out, &ifindex); + if (inner_map) { + struct lpm_key deny_key = { .prefixlen = 32, .ip = daddr }; + if (bpf_map_lookup_elem(inner_map, &deny_key)) + return FLOW_REJECT; + } - return true; + /* 3) Default: allow via SNAT */ + return FLOW_SNAT; } /** @@ -86,11 +179,17 @@ static __always_inline bool session_policy_allowed(__u32 vm_ifindex, __u32 daddr * @snat_ip: selected SNAT IP entry * @snat_port: selected SNAT port/identifier in network byte order * @initial_state: protocol-specific initial conntrack state + * @packet_class: SNAT_PACKET or L7PROXY_PACKET + * @l7_scheme: L7_SCHEME_*; NONE for non-L7 sessions + * + * packet_class and l7_scheme are initialized in the stack value before the + * single BPF_NOEXIST insertion. This prevents another CPU from observing a + * partially classified session. * - * Enforces egress network policy before creating the session. On policy - * deny, stamps skb->cb with NAT_CB_DENIED_BY_POLICY so the caller can - * distinguish "deny" from "resource exhaustion" (e.g. TCP callers use it - * to trigger tcp_reply_reset). + * Egress network policy is NOT enforced here. Callers must classify the + * flow with classify_egress_flow() first and reject denied flows (stamping + * skb->cb with NAT_CB_DENIED_BY_POLICY) before reaching this point. This + * keeps the policy verdict a single decision taken once per new flow. * * Returns true on success, false otherwise (ingress session cleaned up). */ @@ -98,7 +197,8 @@ static __always_inline bool create_nat_session(struct __sk_buff *skb, struct session_key *ekey, __u64 now_ns, __u32 vm_ifindex, struct snat_ip *snat_ip, __u16 snat_port, - __u8 initial_state) + __u8 initial_state, __u8 packet_class, + __u8 l7_scheme) { struct nat_session sess = {}; struct session_key ikey = {}; @@ -117,13 +217,6 @@ static __always_inline bool create_nat_session(struct __sk_buff *skb, */ nat_cb_set(skb, NAT_CB_OK); - if (!session_policy_allowed(vm_ifindex, ekey->dst_ip)) { - nat_cb_set(skb, NAT_CB_DENIED_BY_POLICY); - /* release the ingress slot reserved in pick_snat_ip_port */ - bpf_map_delete_elem(&ingress_sessions, &ikey); - return false; - } - sess.access_time = now_ns; sess.node_ifindex = snat_ip->ifindex; sess.node_ip = snat_ip->ip; @@ -132,6 +225,8 @@ static __always_inline bool create_nat_session(struct __sk_buff *skb, sess.node_port = snat_port; sess.vm_port = ekey->src_port; sess.state = initial_state; + sess.packet_class = packet_class; + sess.l7_scheme = l7_scheme; err = bpf_map_update_elem(&egress_sessions, ekey, &sess, BPF_NOEXIST); if (err) { /* on failure, clean up the ingress slot we reserved earlier */ diff --git a/CubeNet/src/tcp.h b/CubeNet/src/tcp.h index cba5953b5..574f44931 100644 --- a/CubeNet/src/tcp.h +++ b/CubeNet/src/tcp.h @@ -195,7 +195,15 @@ static const u8 tcp_conntracks[2][6][TCP_CONNTRACK_MAX] = { } }; -static unsigned int get_conntrack_index(bool syn, bool ack, bool fin, bool rst) +/* Force inlining: with a single call site inside update_session clang would + * usually inline anyway, but if it ever emits a real BPF subprog the verifier + * must track the returned index through the subprog boundary — extra burden + * on the already-fragile 3D tcp_conntracks[dir][index][old_state] access. + * Inlining lets clang constant-propagate index when the flag inputs are + * compile-time constants (as in the tcp_state test macro and the production + * do_tcp_nat literal-dir call), collapsing one array dimension. + */ +static __always_inline unsigned int get_conntrack_index(bool syn, bool ack, bool fin, bool rst) { if (rst) return TCP_RST_SET; else if (syn) return (ack ? TCP_SYNACK_SET : TCP_SYN_SET); @@ -261,7 +269,13 @@ static __always_inline long snat_tcp(struct __sk_buff *skb, static __always_inline void update_session(enum ip_conntrack_dir dir, struct nat_session *sess, __u64 now_ns, bool syn, bool ack, bool fin, bool rst) { - enum tcp_conntrack old_state, new_state; + /* __u8 (not enum): keeps old_state in a single unsigned register. With a + * signed enum, older clang (14) narrows the value with `&= 255` masks that + * split it into a bounds-checked copy and a separate index copy, so the + * verifier sees the tcp_conntracks index register as unbounded (umax=255) + * and rejects the .rodata read. A plain __u8 keeps check and index unified. + */ + __u8 old_state, new_state; unsigned int index; session_lazy_refresh(sess, now_ns); @@ -290,6 +304,31 @@ static __always_inline void update_session(enum ip_conntrack_dir dir, struct nat } new_state = tcp_conntracks[dir][index][old_state]; + + if (index == TCP_FIN_SET) { + /* A retransmitted FIN from the side that initiated close must not be + * mistaken for the peer's FIN. The generic conntrack table cannot + * distinguish direction once it reaches FIN_WAIT/CLOSE_WAIT, so use + * active_close to retain the state until the opposite side sends FIN. + */ + if ((old_state == TCP_CONNTRACK_FIN_WAIT || + old_state == TCP_CONNTRACK_CLOSE_WAIT) && + ((sess->active_close && dir == IP_CT_DIR_ORIGINAL) || + (!sess->active_close && dir == IP_CT_DIR_REPLY))) + new_state = old_state; + + /* Record only a real original-direction transition that initiates + * close. Checking the classified packet and computed transition avoids + * marking RST|FIN or SYN|FIN packets as active closes. If reply sent + * FIN first, the state is already FIN_WAIT/CLOSE_WAIT when original + * later sends FIN and active_close remains zero. + */ + if (dir == IP_CT_DIR_ORIGINAL && + new_state == TCP_CONNTRACK_FIN_WAIT && + new_state != old_state) + sess->active_close = 1; + } + /* no store if state remain unchanged */ if (new_state != old_state) sess->state = new_state; @@ -298,10 +337,11 @@ static __always_inline void update_session(enum ip_conntrack_dir dir, struct nat static __always_inline bool create_new_sessions(struct __sk_buff *skb, struct session_key *ekey, __u64 now_ns, __u32 vm_ifindex, - struct snat_ip *snat_ip, __u16 snat_port) + struct snat_ip *snat_ip, __u16 snat_port, + __u8 packet_class, __u8 l7_scheme) { return create_nat_session(skb, ekey, now_ns, vm_ifindex, snat_ip, snat_port, - TCP_CONNTRACK_SYN_SENT); + TCP_CONNTRACK_SYN_SENT, packet_class, l7_scheme); } #endif /* __TCP_H */ diff --git a/CubeNet/src/tcp_state_test.bpf.c b/CubeNet/src/tcp_state_test.bpf.c new file mode 100644 index 000000000..e959e4a12 --- /dev/null +++ b/CubeNet/src/tcp_state_test.bpf.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (c) 2026 Cube Authors */ +#include +#include +#include +#include +#include + +#include "tcp.h" + +struct tcp_update_step { + __u8 dir; + __u8 syn; + __u8 ack; + __u8 fin; + __u8 rst; + __u8 reserved[3]; +}; + +struct tcp_update_case { + __u64 access_time; + __u64 now_ns; + __u8 state; + __u8 active_close; + __u8 step_count; + __u8 reserved[5]; + struct tcp_update_step steps[2]; +}; + +#define APPLY_TCP_STEP(DIR, SESS, NOW, STEP) do { \ + if ((STEP).rst) \ + update_session((DIR), (SESS), (NOW), false, false, false, true); \ + else if ((STEP).syn && (STEP).ack) \ + update_session((DIR), (SESS), (NOW), true, true, false, false); \ + else if ((STEP).syn) \ + update_session((DIR), (SESS), (NOW), true, false, false, false); \ + else if ((STEP).fin) \ + update_session((DIR), (SESS), (NOW), false, false, true, false); \ + else if ((STEP).ack) \ + update_session((DIR), (SESS), (NOW), false, true, false, false); \ + else \ + update_session((DIR), (SESS), (NOW), false, false, false, false); \ +} while (0) + +SEC("tc") +int test_update_session(struct __sk_buff *skb) +{ + struct tcp_update_case tc = {}; + struct nat_session sess = {}; + int i; + + if (bpf_skb_load_bytes(skb, 0, &tc, sizeof(tc))) + return TC_ACT_SHOT; + if (tc.state > TCP_CONNTRACK_SYN_SENT2 || tc.step_count > 2) + return TC_ACT_SHOT; + + sess.access_time = tc.access_time; + sess.state = tc.state; + sess.active_close = tc.active_close; + +#pragma unroll + for (i = 0; i < 2; i++) { + if (i < tc.step_count) { + if (tc.steps[i].dir == IP_CT_DIR_ORIGINAL) + APPLY_TCP_STEP(IP_CT_DIR_ORIGINAL, &sess, tc.now_ns + i, + tc.steps[i]); + else if (tc.steps[i].dir == IP_CT_DIR_REPLY) + APPLY_TCP_STEP(IP_CT_DIR_REPLY, &sess, tc.now_ns + i, + tc.steps[i]); + else + return TC_ACT_SHOT; + } + } + + tc.access_time = sess.access_time; + tc.state = sess.state; + tc.active_close = sess.active_close; + if (bpf_skb_store_bytes(skb, 0, &tc, sizeof(tc), 0)) + return TC_ACT_SHOT; + + return TC_ACT_OK; +} + +char __license[] SEC("license") = "Dual BSD/GPL"; diff --git a/CubeNet/src/udp.h b/CubeNet/src/udp.h index fc17c7482..0b06bde7f 100644 --- a/CubeNet/src/udp.h +++ b/CubeNet/src/udp.h @@ -24,7 +24,7 @@ static __always_inline bool create_udp_sessions(struct __sk_buff *skb, struct snat_ip *snat_ip, __u16 snat_port) { return create_nat_session(skb, ekey, now_ns, vm_ifindex, snat_ip, snat_port, - UDP_CT_UNREPLIED); + UDP_CT_UNREPLIED, SNAT_PACKET, L7_SCHEME_NONE); } #endif /* __UDP_H */ diff --git a/Cubelet/api/services/cubebox/v1/cubebox.pb.go b/Cubelet/api/services/cubebox/v1/cubebox.pb.go index 3505a187f..9c88b780b 100644 --- a/Cubelet/api/services/cubebox/v1/cubebox.pb.go +++ b/Cubelet/api/services/cubebox/v1/cubebox.pb.go @@ -3095,12 +3095,17 @@ func (x *EgressRule) GetAction() *EgressRuleAction { } type EgressRuleMatch struct { - state protoimpl.MessageState `protogen:"open.v1"` - Sni *string `protobuf:"bytes,1,opt,name=sni,proto3,oneof" json:"sni,omitempty"` - Host *string `protobuf:"bytes,3,opt,name=host,proto3,oneof" json:"host,omitempty"` - Method []string `protobuf:"bytes,4,rep,name=method,proto3" json:"method,omitempty"` - Path *string `protobuf:"bytes,5,opt,name=path,proto3,oneof" json:"path,omitempty"` - Scheme *string `protobuf:"bytes,7,opt,name=scheme,proto3,oneof" json:"scheme,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Sni *string `protobuf:"bytes,1,opt,name=sni,proto3,oneof" json:"sni,omitempty"` + Host *string `protobuf:"bytes,3,opt,name=host,proto3,oneof" json:"host,omitempty"` + Method []string `protobuf:"bytes,4,rep,name=method,proto3" json:"method,omitempty"` + Path *string `protobuf:"bytes,5,opt,name=path,proto3,oneof" json:"path,omitempty"` + Scheme *string `protobuf:"bytes,7,opt,name=scheme,proto3,oneof" json:"scheme,omitempty"` + // L7 destination port. When set, `scheme` MUST also be set — together they + // pin the (host, port, scheme) tuple CubeEgress intercepts via skb->mark + + // iptables TPROXY. Both omitted keeps the legacy default {80/http, 443/https} + // behavior. + Port *int32 `protobuf:"varint,8,opt,name=port,proto3,oneof" json:"port,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -3170,6 +3175,13 @@ func (x *EgressRuleMatch) GetScheme() string { return "" } +func (x *EgressRuleMatch) GetPort() int32 { + if x != nil && x.Port != nil { + return *x.Port + } + return 0 +} + type EgressRuleAction struct { state protoimpl.MessageState `protogen:"open.v1"` Allow bool `protobuf:"varint,1,opt,name=allow,proto3" json:"allow,omitempty"` @@ -6757,17 +6769,19 @@ const file_api_services_cubebox_v1_cubebox_proto_rawDesc = "" + "\x05match\x18\x02 \x01(\v2,.cubelet.services.cubebox.v1.EgressRuleMatchH\x00R\x05match\x88\x01\x01\x12J\n" + "\x06action\x18\x03 \x01(\v2-.cubelet.services.cubebox.v1.EgressRuleActionH\x01R\x06action\x88\x01\x01B\b\n" + "\x06_matchB\t\n" + - "\a_action\"\xb4\x01\n" + + "\a_action\"\xd6\x01\n" + "\x0fEgressRuleMatch\x12\x15\n" + "\x03sni\x18\x01 \x01(\tH\x00R\x03sni\x88\x01\x01\x12\x17\n" + "\x04host\x18\x03 \x01(\tH\x01R\x04host\x88\x01\x01\x12\x16\n" + "\x06method\x18\x04 \x03(\tR\x06method\x12\x17\n" + "\x04path\x18\x05 \x01(\tH\x02R\x04path\x88\x01\x01\x12\x1b\n" + - "\x06scheme\x18\a \x01(\tH\x03R\x06scheme\x88\x01\x01B\x06\n" + + "\x06scheme\x18\a \x01(\tH\x03R\x06scheme\x88\x01\x01\x12\x17\n" + + "\x04port\x18\b \x01(\x05H\x04R\x04port\x88\x01\x01B\x06\n" + "\x04_sniB\a\n" + "\x05_hostB\a\n" + "\x05_pathB\t\n" + - "\a_scheme\"\x94\x01\n" + + "\a_schemeB\a\n" + + "\x05_port\"\x94\x01\n" + "\x10EgressRuleAction\x12\x14\n" + "\x05allow\x18\x01 \x01(\bR\x05allow\x12\x19\n" + "\x05audit\x18\x02 \x01(\tH\x00R\x05audit\x88\x01\x01\x12E\n" + diff --git a/Cubelet/api/services/cubebox/v1/cubebox.proto b/Cubelet/api/services/cubebox/v1/cubebox.proto index 96b303a4c..8ef1c14a7 100644 --- a/Cubelet/api/services/cubebox/v1/cubebox.proto +++ b/Cubelet/api/services/cubebox/v1/cubebox.proto @@ -610,6 +610,11 @@ message EgressRuleMatch { repeated string method = 4; optional string path = 5; optional string scheme = 7; + // L7 destination port. When set, `scheme` MUST also be set — together they + // pin the (host, port, scheme) tuple CubeEgress intercepts via skb->mark + + // iptables TPROXY. Both omitted keeps the legacy default {80/http, 443/https} + // behavior. + optional int32 port = 8; } message EgressRuleAction { diff --git a/Cubelet/doc/cubelet-api.md b/Cubelet/doc/cubelet-api.md index 90e606138..35c7cbd7a 100644 --- a/Cubelet/doc/cubelet-api.md +++ b/Cubelet/doc/cubelet-api.md @@ -860,6 +860,7 @@ Device specifies a host device to mount into a container. | method | [string](#string) | repeated | | | path | [string](#string) | optional | | | scheme | [string](#string) | optional | | +| port | [int32](#int32) | optional | L7 destination port. When set, `scheme` MUST also be set — together they pin the (host, port, scheme) tuple CubeEgress intercepts via skb->mark + iptables TPROXY. Both omitted keeps the legacy default {80/http, 443/https} behavior. | diff --git a/Cubelet/network/plugin_policy.go b/Cubelet/network/plugin_policy.go index 8b0f64474..1dec27c7b 100644 --- a/Cubelet/network/plugin_policy.go +++ b/Cubelet/network/plugin_policy.go @@ -67,13 +67,18 @@ func mapRunRequestEgressRuleMatch(in *cubebox.EgressRuleMatch) *networkruntime.E if in == nil { return nil } - return &networkruntime.EgressRuleMatch{ + out := &networkruntime.EgressRuleMatch{ SNI: in.Sni, Host: in.Host, Method: append([]string(nil), in.GetMethod()...), Path: in.Path, Scheme: in.Scheme, } + if in.Port != nil { + p := int(*in.Port) + out.Port = &p + } + return out } func mapRunRequestEgressRuleAction(in *cubebox.EgressRuleAction) *networkruntime.EgressRuleAction { diff --git a/Cubelet/network/runtime/controller.go b/Cubelet/network/runtime/controller.go index bf2ae04ee..e9cdcc348 100644 --- a/Cubelet/network/runtime/controller.go +++ b/Cubelet/network/runtime/controller.go @@ -11,6 +11,7 @@ import ( "net" "os" "slices" + "strconv" "strings" "sync" "time" @@ -292,6 +293,9 @@ func initCubeVS(cfg Config, device *systemnet.HostDevice, cubeDev *systemnet.Cub NodeMacAddr: device.Mac, NodeGatewayMacAddr: device.GatewayMac, } + if err := loadL7MarksConfig(¶ms); err != nil { + return nil, err + } if err := cubevs.Init(params); err != nil { return nil, err } @@ -307,6 +311,64 @@ func initCubeVS(cfg Config, device *systemnet.HostDevice, cubeDev *systemnet.Cub return cubeRouter, nil } +// l7MarksConfigPath is the install-time config shared with the +// cube-proxy-iptables-init script, so the dataplane (eBPF globals) and the +// iptables TPROXY rules stamp/match the same skb->mark values. It is a var so +// tests can point it at a temp file instead of the real /etc path. +var l7MarksConfigPath = "/etc/cubeegress/l7-marks.conf" + +// loadL7MarksConfig overlays CUBE_L7_MARK_{HTTP,HTTPS,MASK} from +// l7MarksConfigPath onto params.L7Mark*. A missing file leaves the shipped +// defaults (cubevs.resolveL7Marks applies them); unset keys likewise fall +// back to defaults. Values are hex (e.g. 0xCE010000), matching the shell +// KEY=VALUE format the iptables script sources. Because the iptables script +// sources the same file as POSIX shell, hand-edited but shell-legal lines +// are tolerated here too: an "export " key prefix and a trailing +// " # comment" on the value. +func loadL7MarksConfig(params *cubevs.Params) error { + data, err := os.ReadFile(l7MarksConfigPath) // NOCC:Path Traversal() + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, raw, ok := strings.Cut(line, "=") + if !ok { + continue + } + key = strings.TrimSpace(key) + if strings.HasPrefix(key, "export") { + // `export KEY=value` — without this the key would match no case + // below and the override would be silently ignored, diverging + // the dataplane marks from the iptables rules. + key = strings.TrimSpace(strings.TrimPrefix(key, "export")) + } + // Strip a trailing ` # comment` before parsing; without this the + // ParseUint below would fail and block controller startup. + raw, _, _ = strings.Cut(raw, "#") + text := strings.Trim(strings.TrimSpace(raw), `"'`) + value, perr := strconv.ParseUint(text, 0, 32) + if perr != nil { + return fmt.Errorf("parse %q in %s: %w", line, l7MarksConfigPath, perr) + } + switch key { + case "CUBE_L7_MARK_HTTP": + params.L7MarkHTTP = uint32(value) + case "CUBE_L7_MARK_HTTPS": + params.L7MarkHTTPS = uint32(value) + case "CUBE_L7_MARK_MASK": + params.L7MarkMask = uint32(value) + } + } + return nil +} + func startCubeVSSessionLogDrain() { sessionEvents := cubevs.StartSessionReaper() go func() { diff --git a/Cubelet/network/runtime/controller_test.go b/Cubelet/network/runtime/controller_test.go index e5f06cec1..b83d9e0d7 100644 --- a/Cubelet/network/runtime/controller_test.go +++ b/Cubelet/network/runtime/controller_test.go @@ -5,9 +5,11 @@ import ( "errors" "net" "os" + "path/filepath" "reflect" "testing" + "github.com/tencentcloud/CubeSandbox/CubeNet/cubevs" "github.com/tencentcloud/CubeSandbox/Cubelet/network/runtime/systemnet" ) @@ -760,3 +762,91 @@ func TestNewNetworkControllerFromDepsRequiresAdapters(t *testing.T) { t.Fatal("expected missing adapter error") } } + +func TestLoadL7MarksConfig(t *testing.T) { + withPath := func(t *testing.T, content *string) { + t.Helper() + old := l7MarksConfigPath + t.Cleanup(func() { l7MarksConfigPath = old }) + if content == nil { + l7MarksConfigPath = filepath.Join(t.TempDir(), "absent.conf") + return + } + p := filepath.Join(t.TempDir(), "l7-marks.conf") + if err := os.WriteFile(p, []byte(*content), 0o644); err != nil { + t.Fatalf("write temp conf: %v", err) + } + l7MarksConfigPath = p + } + + t.Run("absent file leaves defaults", func(t *testing.T) { + withPath(t, nil) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err != nil { + t.Fatalf("absent file: %v", err) + } + if p.L7MarkHTTP != 0 || p.L7MarkHTTPS != 0 || p.L7MarkMask != 0 { + t.Fatalf("absent file set marks: %+v", p) + } + }) + + t.Run("override is applied", func(t *testing.T) { + conf := "# comment\nCUBE_L7_MARK_HTTP=0xCF010000\nCUBE_L7_MARK_HTTPS=0xCF020000\nCUBE_L7_MARK_MASK=0xFFFF0000\n" + withPath(t, &conf) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err != nil { + t.Fatalf("override: %v", err) + } + if p.L7MarkHTTP != 0xCF010000 || p.L7MarkHTTPS != 0xCF020000 || p.L7MarkMask != 0xFFFF0000 { + t.Fatalf("override: got http=%#x https=%#x mask=%#x", p.L7MarkHTTP, p.L7MarkHTTPS, p.L7MarkMask) + } + }) + + t.Run("partial override keeps others zero", func(t *testing.T) { + conf := "CUBE_L7_MARK_HTTP=0xCF030000\n" + withPath(t, &conf) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err != nil { + t.Fatalf("partial: %v", err) + } + if p.L7MarkHTTP != 0xCF030000 || p.L7MarkHTTPS != 0 || p.L7MarkMask != 0 { + t.Fatalf("partial: got http=%#x https=%#x mask=%#x", p.L7MarkHTTP, p.L7MarkHTTPS, p.L7MarkMask) + } + }) + + t.Run("malformed value errors", func(t *testing.T) { + conf := "CUBE_L7_MARK_HTTP=not-a-number\n" + withPath(t, &conf) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err == nil { + t.Fatal("malformed value was not rejected") + } + }) + + t.Run("export-prefixed key is honored", func(t *testing.T) { + // Shell-legal `export KEY=value` must not be silently ignored — + // otherwise the dataplane stamps a different mark than iptables + // matches. + conf := "export CUBE_L7_MARK_HTTP=0xCF010000\nexport CUBE_L7_MARK_MASK=0xFFFF0000\n" + withPath(t, &conf) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err != nil { + t.Fatalf("export-prefixed: %v", err) + } + if p.L7MarkHTTP != 0xCF010000 || p.L7MarkMask != 0xFFFF0000 { + t.Fatalf("export-prefixed: got http=%#x mask=%#x", p.L7MarkHTTP, p.L7MarkMask) + } + }) + + t.Run("inline comment on value is stripped", func(t *testing.T) { + conf := "CUBE_L7_MARK_HTTP=0xCF030000 # http listener mark\n" + withPath(t, &conf) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err != nil { + t.Fatalf("inline comment: %v", err) + } + if p.L7MarkHTTP != 0xCF030000 { + t.Fatalf("inline comment: got http=%#x", p.L7MarkHTTP) + } + }) +} diff --git a/Cubelet/network/runtime/cubeegress/wire.go b/Cubelet/network/runtime/cubeegress/wire.go index ca22697ac..c25fbc3a5 100644 --- a/Cubelet/network/runtime/cubeegress/wire.go +++ b/Cubelet/network/runtime/cubeegress/wire.go @@ -59,6 +59,7 @@ type MatchInput struct { Method []string Path *string Scheme *string + Port *int } // ActionInput mirrors runtime.EgressRuleAction. @@ -175,6 +176,12 @@ func renderMatch(m *MatchInput) map[string]any { if m.Scheme != nil { out["scheme"] = *m.Scheme } + if m.Port != nil { + // CubeEgress's access_phase.lua rule_matches compares match.port + // against ctx.dst_port (from tproxy's $server_port). Emit as a + // number so cjson decodes it to a numeric type in openresty. + out["port"] = *m.Port + } return out } diff --git a/Cubelet/network/runtime/cubeegress_adapter.go b/Cubelet/network/runtime/cubeegress_adapter.go index 2fcd3a48d..006ba88a5 100644 --- a/Cubelet/network/runtime/cubeegress_adapter.go +++ b/Cubelet/network/runtime/cubeegress_adapter.go @@ -73,13 +73,18 @@ func toMatchInput(m *EgressRuleMatch) *cubeegress.MatchInput { if m == nil { return nil } - return &cubeegress.MatchInput{ + out := &cubeegress.MatchInput{ SNI: m.SNI, Host: m.Host, Method: append([]string(nil), m.Method...), Path: m.Path, Scheme: m.Scheme, } + if m.Port != nil { + p := *m.Port + out.Port = &p + } + return out } // toActionInput deep-copies the action section, including header-injection diff --git a/Cubelet/network/runtime/cubevs_adapter.go b/Cubelet/network/runtime/cubevs_adapter.go index e371e6ee3..8b69c1385 100644 --- a/Cubelet/network/runtime/cubevs_adapter.go +++ b/Cubelet/network/runtime/cubevs_adapter.go @@ -86,7 +86,10 @@ func (realCubeVSAdapter) DeletePortMappingsByIfindex(ifindex uint32) error { // registerCubeVSTap writes the complete TAP metadata and policy options into // CubeVS for a newly-created sandbox. func (s *NetworkController) registerCubeVSTap(ifindex int, ip net.IP, sandboxID string, cfg *CubeNetworkConfig) (err error) { - opts := cubeVSTapRegistration(cfg) + opts, err := cubeVSTapRegistration(cfg) + if err != nil { + return err + } CubeLog.WithContext(context.Background()).Infof( "network runtime register cubevs tap: sandbox_id=%s ifindex=%d sandbox_ip=%s cube_network_config=%s allow_internet_access=%v allow_out=%v l7_allow_out=%v deny_out=%v", sandboxID, @@ -115,7 +118,10 @@ func (s *NetworkController) registerCubeVSTap(ifindex int, ip net.IP, sandboxID // Legacy recovery uses this to avoid carrying old allow/deny/DNS residue into // the recovered Active sandbox. func (s *NetworkController) replaceCubeVSTap(ifindex int, ip net.IP, sandboxID string, cfg *CubeNetworkConfig) error { - opts := cubeVSTapRegistration(cfg) + opts, err := cubeVSTapRegistration(cfg) + if err != nil { + return err + } CubeLog.WithContext(context.Background()).Infof( "network runtime replace cubevs tap: sandbox_id=%s ifindex=%d sandbox_ip=%s cube_network_config=%s allow_internet_access=%v allow_out=%v l7_allow_out=%v deny_out=%v", sandboxID, diff --git a/Cubelet/network/runtime/policy_builder.go b/Cubelet/network/runtime/policy_builder.go index 75095bf8c..681c2afb2 100644 --- a/Cubelet/network/runtime/policy_builder.go +++ b/Cubelet/network/runtime/policy_builder.go @@ -5,6 +5,7 @@ package runtime import ( + "errors" "fmt" "net" "strings" @@ -46,6 +47,14 @@ func cloneEgressRules(in []*EgressRule) []*EgressRule { if r.Match != nil { match := *r.Match match.Method = append([]string(nil), r.Match.Method...) + // The pointer fields must be deep-copied too: a shallow struct copy + // would alias the caller's request, so a later mutation of the + // request would leak into the stored (supposedly immutable) copy. + match.SNI = cloneStringPtr(r.Match.SNI) + match.Host = cloneStringPtr(r.Match.Host) + match.Path = cloneStringPtr(r.Match.Path) + match.Scheme = cloneStringPtr(r.Match.Scheme) + match.Port = cloneIntPtr(r.Match.Port) cp.Match = &match } if r.Action != nil { @@ -75,6 +84,26 @@ func cloneEgressRules(in []*EgressRule) []*EgressRule { return out } +// cloneStringPtr returns a copy of a *string, or nil. Mirrors the CubeMaster +// helper of the same name (pkg/service/sandbox/types/types.go). +func cloneStringPtr(value *string) *string { + if value == nil { + return nil + } + cloned := *value + return &cloned +} + +// cloneIntPtr returns a copy of a *int, or nil. Mirrors the CubeMaster helper +// of the same name (pkg/service/sandbox/types/types.go). +func cloneIntPtr(value *int) *int { + if value == nil { + return nil + } + cloned := *value + return &cloned +} + // formatCubeNetworkConfig renders a compact log-only view of the policy. It // deliberately avoids dumping full L7 rule bodies, which may contain secrets in // header-injection rules. @@ -94,10 +123,10 @@ func formatCubeNetworkConfig(in *CubeNetworkConfig) string { // allow_internet_access / allow_out / deny_out, and it also receives network // targets extracted from L7 rules as L7 allow targets. The complete L7 rules // are still pushed to CubeEgress separately. -func cubeVSTapRegistration(cfg *CubeNetworkConfig) cubevs.MVMOptions { +func cubeVSTapRegistration(cfg *CubeNetworkConfig) (cubevs.MVMOptions, error) { if cfg == nil { allowInternetAccess := true - return cubevs.MVMOptions{AllowInternetAccess: &allowInternetAccess} + return cubevs.MVMOptions{AllowInternetAccess: &allowInternetAccess}, nil } opts := cubevs.MVMOptions{} if cfg.AllowInternetAccess != nil { @@ -111,46 +140,135 @@ func cubeVSTapRegistration(cfg *CubeNetworkConfig) cubevs.MVMOptions { allowOut := append([]string(nil), cfg.AllowOut...) opts.AllowOut = &allowOut } - if l7AllowOut := extractL7AllowOutTargetsFromRules(cfg.Rules); len(l7AllowOut) > 0 { + l7AllowOut, err := extractL7AllowOutTargetsFromRules(cfg.Rules) + if err != nil { + return cubevs.MVMOptions{}, err + } + if len(l7AllowOut) > 0 { opts.L7AllowOut = &l7AllowOut } if len(cfg.DenyOut) > 0 { denyOut := append([]string(nil), cfg.DenyOut...) opts.DenyOut = &denyOut } - return opts + return opts, nil } -// extractL7AllowOutTargetsFromRules converts SNI/Host matches into the coarse -// network targets that cubevs needs to allow before CubeEgress can inspect L7 -// traffic. Invalid or non-IPv4-looking targets are ignored rather than failing -// sandbox creation; the full rule is still validated by CubeEgress on push. -func extractL7AllowOutTargetsFromRules(rules []*EgressRule) []string { - seen := make(map[string]struct{}) - targets := make([]string, 0, len(rules)) - add := func(target string, ok bool) { +// extractL7AllowOutTargetsFromRules walks the L7 rule list and produces the +// (host, port, scheme) tuples cubevs needs for its per-host dns_allow_v2 / +// allow_out_v3 port set. SNI-only rules and rules without a Match.Host are +// treated as legacy port-agnostic entries — they expand to {80/http, 443/https} +// downstream in buildL7Plan. Rules that specify a Port MUST also specify a +// scheme ("http" or "https"). Any invalid port/scheme pair rejects the whole +// projection so CubeVS and CubeEgress cannot observe different policies. +// +// Duplicate (host, port, scheme) tuples are deduplicated to keep the map +// value's port_count within maxL7PortsPerHost when a user attaches the same +// port to several rules. +func extractL7AllowOutTargetsFromRules(rules []*EgressRule) ([]cubevs.L7Target, error) { + type key struct { + host string + port uint16 + scheme uint8 + } + seen := make(map[key]struct{}) + targets := make([]cubevs.L7Target, 0, len(rules)) + add := func(host string, ok bool, port uint16, scheme uint8) { if !ok { return } - if _, exists := seen[target]; exists { + k := key{host, port, scheme} + if _, exists := seen[k]; exists { return } - seen[target] = struct{}{} - targets = append(targets, target) + seen[k] = struct{}{} + targets = append(targets, cubevs.L7Target{Host: host, Port: port, Scheme: scheme}) } - for _, rule := range rules { + for i, rule := range rules { if rule == nil || rule.Match == nil { continue } + port, scheme, err := extractL7PortScheme(rule.Match) + if err != nil { + return nil, fmt.Errorf("network.rules[%d] %q: %w", i, rule.Name, err) + } + // A rule carrying both SNI and Host projects BOTH as L7 targets (SNI + // first, then Host), not Host alone. if rule.Match.SNI != nil { - add(normalizeL7DomainTarget(*rule.Match.SNI)) + host, ok := normalizeL7DomainTarget(*rule.Match.SNI) + add(host, ok, port, scheme) } if rule.Match.Host != nil { - add(normalizeL7HostTarget(*rule.Match.Host)) + host, ok := normalizeL7HostTarget(*rule.Match.Host) + add(host, ok, port, scheme) } } - return targets + return targets, nil +} + +// extractL7PortScheme reads Match.Port + Match.Scheme and normalises them into +// cubevs.L7Target's numeric representation. +// +// - Port set, Scheme nil → invalid (port without scheme cannot decide the +// nginx listener); reject. +// - Port nil, Scheme set → fill in the scheme's default port (http → 80, +// https → 443). Callers that only want to say "https on this host" can +// omit port and get the conventional default. +// - Port set, Scheme set → new port-scoped feature: exact tuple. +// - Both nil → legacy: buildL7Plan expands to +// {80/http, 443/https}. +// +// Any recognised, non-empty Scheme string is normalised to lowercase and +// stripped of surrounding whitespace before comparison — the wire form is +// case-insensitive. +func extractL7PortScheme(match *EgressRuleMatch) (uint16, uint8, error) { + if match.Port == nil && match.Scheme == nil { + // Legacy: buildL7Plan will expand to {80/http, 443/https}. + return 0, cubevs.L7SchemeNone, nil + } + + // Scheme presence guides port validation. Decode it first (nil is fine). + var schemeValue uint8 = cubevs.L7SchemeNone + if match.Scheme != nil { + switch strings.ToLower(strings.TrimSpace(*match.Scheme)) { + case "http": + schemeValue = cubevs.L7SchemeHTTP + case "https": + schemeValue = cubevs.L7SchemeHTTPS + default: + return 0, 0, fmt.Errorf("scheme must be http or https, got %q", *match.Scheme) + } + } + + if match.Port == nil { + // Scheme only → fill in scheme's canonical default port. + return defaultPortForScheme(schemeValue), schemeValue, nil + } + + if match.Scheme == nil { + return 0, 0, errors.New("port requires scheme") + } + + p := *match.Port + if p <= 0 || p > 65535 { + return 0, 0, fmt.Errorf("port must be in [1, 65535], got %d", p) + } + return uint16(p), schemeValue, nil +} + +// defaultPortForScheme returns the conventional port for a bare scheme. +// http → 80, https → 443. Any other scheme value returns 0 (should never be +// reached: extractL7PortScheme rejects unknown schemes before calling this). +func defaultPortForScheme(scheme uint8) uint16 { + switch scheme { + case cubevs.L7SchemeHTTP: + return 80 + case cubevs.L7SchemeHTTPS: + return 443 + default: + return 0 + } } // normalizeL7DomainTarget canonicalizes a DNS name or wildcard suffix for the diff --git a/Cubelet/network/runtime/policy_builder_test.go b/Cubelet/network/runtime/policy_builder_test.go index 911c5bfeb..2a645c5df 100644 --- a/Cubelet/network/runtime/policy_builder_test.go +++ b/Cubelet/network/runtime/policy_builder_test.go @@ -1,6 +1,11 @@ package runtime -import "testing" +import ( + "strings" + "testing" + + "github.com/tencentcloud/CubeSandbox/CubeNet/cubevs" +) func TestCubeVSTapRegistrationBuildsL3L4AndL7Options(t *testing.T) { allowInternet := false @@ -19,7 +24,10 @@ func TestCubeVSTapRegistrationBuildsL3L4AndL7Options(t *testing.T) { }}, } - opts := cubeVSTapRegistration(cfg) + opts, err := cubeVSTapRegistration(cfg) + if err != nil { + t.Fatalf("cubeVSTapRegistration error=%v", err) + } if opts.AllowInternetAccess == nil || *opts.AllowInternetAccess != false { t.Fatalf("AllowInternetAccess = %#v, want false", opts.AllowInternetAccess) } @@ -32,8 +40,188 @@ func TestCubeVSTapRegistrationBuildsL3L4AndL7Options(t *testing.T) { if opts.L7AllowOut == nil || len(*opts.L7AllowOut) != 2 { t.Fatalf("L7AllowOut = %#v", opts.L7AllowOut) } - if (*opts.L7AllowOut)[0] != "api.example.com" || (*opts.L7AllowOut)[1] != "10.1.2.3" { - t.Fatalf("L7AllowOut = %#v", *opts.L7AllowOut) + got := *opts.L7AllowOut + if got[0].Host != "api.example.com" || got[1].Host != "10.1.2.3" { + t.Fatalf("L7AllowOut = %#v", got) + } + // No explicit port/scheme on the rules: legacy default targets. + for i, tgt := range got { + if tgt.Port != 0 || tgt.Scheme != cubevs.L7SchemeNone { + t.Fatalf("L7AllowOut[%d] = {Port:%d Scheme:%d}, want legacy default {0/L7SchemeNone}", + i, tgt.Port, tgt.Scheme) + } + } +} + +func TestCubeVSTapRegistrationBlockAll(t *testing.T) { + allowInternet := false + opts, err := cubeVSTapRegistration(&CubeNetworkConfig{ + AllowInternetAccess: &allowInternet, + }) + if err != nil { + t.Fatalf("cubeVSTapRegistration error=%v", err) + } + if opts.AllowInternetAccess == nil || *opts.AllowInternetAccess != false { + t.Fatalf("opts.AllowInternetAccess=%v, want false", opts.AllowInternetAccess) + } +} + +func TestCubeVSTapRegistrationExtractsL7AllowOut(t *testing.T) { + sni := "API.Example.COM." + sniWildcard := "*.SNI.Example.COM" + hostIP := "1.2.3.4:443" + hostCIDR := "10.1.2.3/8" + hostDomain := "Gateway.Example.COM:8443" + hostWildcard := "*.Gateway.Example.COM" + duplicateHost := "gateway.example.com" + invalidHost := "999.999.999.999" + opts, err := cubeVSTapRegistration(&CubeNetworkConfig{ + AllowOut: []string{"8.8.8.8"}, + Rules: []*EgressRule{ + {Match: &EgressRuleMatch{SNI: &sni, Host: &hostIP}}, + {Match: &EgressRuleMatch{Host: &hostCIDR}}, + {Match: &EgressRuleMatch{Host: &hostDomain}}, + {Match: &EgressRuleMatch{SNI: &sni}}, + {Match: &EgressRuleMatch{SNI: &sniWildcard}}, + {Match: &EgressRuleMatch{Host: &hostWildcard}}, + {Match: &EgressRuleMatch{Host: &duplicateHost}}, + {Match: &EgressRuleMatch{Host: &invalidHost}}, + {Match: &EgressRuleMatch{Path: stringPtr("/v1/chat")}}, + }, + }) + if err != nil { + t.Fatalf("cubeVSTapRegistration error=%v", err) + } + if opts.AllowOut == nil || len(*opts.AllowOut) != 1 || (*opts.AllowOut)[0] != "8.8.8.8" { + t.Fatalf("opts.AllowOut=%v, want [8.8.8.8]", opts.AllowOut) + } + if opts.L7AllowOut == nil { + t.Fatal("opts.L7AllowOut=nil, want extracted targets") + } + // Rules without explicit Port + Scheme land as legacy default targets + // (Port=0, Scheme=L7SchemeNone) — downstream buildL7Plan expands them + // to {80/http, 443/https}. Assert host projection only; port/scheme + // coverage is in TestExtractL7PortScheme. + // The first rule carries both SNI and Host: BOTH are projected (SNI + // first, then Host), so "api.example.com" (from SNI) and "1.2.3.4" (from + // Host) both appear; the later duplicate SNI is deduplicated. + wantHosts := []string{ + "api.example.com", "1.2.3.4", "10.0.0.0/8", "gateway.example.com", + "*.sni.example.com", "*.gateway.example.com", + } + gotTargets := *opts.L7AllowOut + if len(gotTargets) != len(wantHosts) { + t.Fatalf("opts.L7AllowOut host count=%d, want %d (%+v)", + len(gotTargets), len(wantHosts), gotTargets) + } + for i, w := range wantHosts { + if gotTargets[i].Host != w { + t.Fatalf("opts.L7AllowOut[%d].Host=%q, want %q", i, gotTargets[i].Host, w) + } + if gotTargets[i].Port != 0 || gotTargets[i].Scheme != cubevs.L7SchemeNone { + t.Fatalf("opts.L7AllowOut[%d] expected legacy default (0/L7SchemeNone), got Port=%d Scheme=%d", + i, gotTargets[i].Port, gotTargets[i].Scheme) + } + } +} + +// TestExtractL7PortScheme covers the four accepted rule shapes plus the +// error paths. Rules missing one of {Port, Scheme}, using an unknown scheme +// string, or an out-of-range port must be dropped rather than silently +// promoted to the legacy default set — that would let a typo like "htps" fall +// back to port 443 instead of surfacing. +func TestExtractL7PortScheme(t *testing.T) { + intPtr := func(i int) *int { return &i } + + tests := []struct { + name string + match EgressRuleMatch + wantPort uint16 + wantScheme uint8 + wantErr bool + }{ + {"both nil (legacy)", EgressRuleMatch{}, 0, cubevs.L7SchemeNone, false}, + {"port only rejected", + EgressRuleMatch{Port: intPtr(8080)}, 0, 0, true}, + {"http alone fills default port 80", + EgressRuleMatch{Scheme: stringPtr("http")}, 80, cubevs.L7SchemeHTTP, false}, + {"https alone fills default port 443", + EgressRuleMatch{Scheme: stringPtr("https")}, 443, cubevs.L7SchemeHTTPS, false}, + {"https mixed case alone fills 443", + EgressRuleMatch{Scheme: stringPtr("HTTPS")}, 443, cubevs.L7SchemeHTTPS, false}, + {"http lowercase", + EgressRuleMatch{Port: intPtr(8080), Scheme: stringPtr("http")}, + 8080, cubevs.L7SchemeHTTP, false}, + {"https mixed case", + EgressRuleMatch{Port: intPtr(8443), Scheme: stringPtr("HTTPS")}, + 8443, cubevs.L7SchemeHTTPS, false}, + {"scheme with whitespace", + EgressRuleMatch{Port: intPtr(80), Scheme: stringPtr(" http ")}, + 80, cubevs.L7SchemeHTTP, false}, + {"unknown scheme rejected", + EgressRuleMatch{Port: intPtr(80), Scheme: stringPtr("gopher")}, + 0, 0, true}, + {"unknown scheme alone rejected", + EgressRuleMatch{Scheme: stringPtr("gopher")}, 0, 0, true}, + {"port too low rejected", + EgressRuleMatch{Port: intPtr(0), Scheme: stringPtr("http")}, + 0, 0, true}, + {"port too high rejected", + EgressRuleMatch{Port: intPtr(65536), Scheme: stringPtr("http")}, + 0, 0, true}, + {"port negative rejected", + EgressRuleMatch{Port: intPtr(-1), Scheme: stringPtr("http")}, + 0, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + port, scheme, err := extractL7PortScheme(&tt.match) + if (err != nil) != tt.wantErr { + t.Fatalf("err=%v, wantErr=%v", err, tt.wantErr) + } + if err != nil { + return + } + if port != tt.wantPort { + t.Fatalf("port=%d, want %d", port, tt.wantPort) + } + if scheme != tt.wantScheme { + t.Fatalf("scheme=%d, want %d", scheme, tt.wantScheme) + } + }) + } +} + +func TestCubeVSTapRegistrationRejectsEntireInvalidL7Policy(t *testing.T) { + host := "api.example.com" + https := "https" + validPort := 8443 + invalidPort := 8080 + + opts, err := cubeVSTapRegistration(&CubeNetworkConfig{ + AllowOut: []string{"8.8.8.8"}, + Rules: []*EgressRule{ + { + Name: "valid-rule", + Match: &EgressRuleMatch{ + Host: &host, Port: &validPort, Scheme: &https, + }, + }, + { + Name: "invalid-rule", + Match: &EgressRuleMatch{Host: &host, Port: &invalidPort}, + }, + }, + }) + if err == nil { + t.Fatal("cubeVSTapRegistration error=nil, want invalid policy error") + } + if !strings.Contains(err.Error(), `network.rules[1] "invalid-rule"`) || + !strings.Contains(err.Error(), "port requires scheme") { + t.Fatalf("error=%q, want rule location and reason", err) + } + if opts.AllowOut != nil || opts.L7AllowOut != nil || opts.DenyOut != nil { + t.Fatalf("opts=%+v, want empty options on whole-policy rejection", opts) } } @@ -82,3 +270,41 @@ func TestToEgressInputTranslatesL7Rules(t *testing.T) { func stringPtr(value string) *string { return &value } + +// TestCloneEgressRulesDeepCopiesMatchPointers guards against the shallow-copy +// regression: cloneEgressRules must not alias the caller's match pointers, so +// mutating the request after the clone must not leak into the stored copy. +func TestCloneEgressRulesDeepCopiesMatchPointers(t *testing.T) { + port := 8443 + original := &EgressRule{ + Name: "rule", + Match: &EgressRuleMatch{ + SNI: stringPtr("api.example.com"), + Host: stringPtr("api.example.com"), + Path: stringPtr("/v1/chat"), + Scheme: stringPtr("https"), + Port: &port, + Method: []string{"GET"}, + }, + Action: &EgressRuleAction{Allow: true}, + } + + cloned := cloneEgressRules([]*EgressRule{original}) + if len(cloned) != 1 || cloned[0].Match == nil { + t.Fatalf("cloneEgressRules returned %#v", cloned) + } + + // Mutate every pointer field on the original; the clone must be unaffected. + *original.Match.SNI = "mutated.example.com" + *original.Match.Host = "mutated.example.com" + *original.Match.Path = "/mutated" + *original.Match.Scheme = "http" + *original.Match.Port = 9999 + + cm := cloned[0].Match + if *cm.SNI != "api.example.com" || *cm.Host != "api.example.com" || + *cm.Path != "/v1/chat" || *cm.Scheme != "https" || *cm.Port != 8443 { + t.Fatalf("clone aliased caller pointers: SNI=%q Host=%q Path=%q Scheme=%q Port=%d", + *cm.SNI, *cm.Host, *cm.Path, *cm.Scheme, *cm.Port) + } +} diff --git a/Cubelet/network/runtime/types.go b/Cubelet/network/runtime/types.go index cbe0ccccc..6ee398105 100644 --- a/Cubelet/network/runtime/types.go +++ b/Cubelet/network/runtime/types.go @@ -122,12 +122,20 @@ type EgressRule struct { // EgressRuleMatch holds the per-request match conditions for an EgressRule. // All fields are optional; an empty match matches any request. +// +// Port and Scheme together control which TCP port CubeEgress intercepts on the +// sandbox side. When both are omitted the rule applies to the default set +// {80/http, 443/https}. When Port is set, Scheme MUST also be set to "http" or +// "https" — every rule for the same (host, port) tuple must agree on the +// scheme, because iptables can only steer a single tuple to one TPROXY +// listener. type EgressRuleMatch struct { SNI *string `json:"sni,omitempty"` Host *string `json:"host,omitempty"` Method []string `json:"method,omitempty"` Path *string `json:"path,omitempty"` Scheme *string `json:"scheme,omitempty"` + Port *int `json:"port,omitempty"` } // EgressRuleAction holds the action taken when an EgressRule matches. diff --git a/Makefile b/Makefile index 51068e845..2bb96fec7 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,10 @@ BUILDER_CONTAINER_HOME ?= /home/builder TMP_GIT_CREDENTIALS ?= /tmp/.cube-sandbox-builder-tmp-git-credentials BUILDER_CMD ?= bash BUILDER_RUN_EXTRA_MOUNTS ?= +# User the builder container runs as. Defaults to the host user so bind-mounted +# outputs stay host-writable; privileged test targets (e.g. cubevs-test, which +# loads eBPF / mounts bpffs) override this to 0:0 along with --privileged. +BUILDER_USER ?= $(UID):$(GID) ROOT_DIR := $(shell pwd) UID := $(shell id -u) GID := $(shell id -g) @@ -205,7 +209,7 @@ ifeq ($(strip $(BUILDER_CMD)),) $(error BUILDER_CMD must not be empty) endif docker run --rm -i \ - --user "$(UID):$(GID)" \ + --user "$(BUILDER_USER)" \ -e HOME=$(BUILDER_CONTAINER_HOME) \ -e CARGO_HOME=$(BUILDER_CONTAINER_HOME)/.cargo \ -e RUSTUP_HOME=/usr/local/rustup \ @@ -371,6 +375,19 @@ cube-lifecycle-manager-test: builder-image cubelet-pkg-test: builder-image $(MAKE) builder-run BUILDER_CMD='cd /workspace && IN_CUBE_SANDBOX_BUILDER=1 make cubecow-sdk && cd /workspace/Cubelet && go mod download && make proto && go test -short ./pkg/...' +# cubevs-test runs the CubeNet/cubevs module's own unit tests (dataplane policy, +# DNS learning, migration, dump, classify), which the cubelet targets never +# compile. It regenerates the BPF objects first (make gen) since the test files +# embed them, then runs the full module test set. The eBPF-loading tests need a +# privileged root container (CAP_BPF/CAP_SYS_ADMIN for bpf() and the bpffs +# mount), so this runs the builder privileged as root. Note: the generated .o +# files under CubeNet/cubevs become root-owned in the bind-mounted workspace; +# that is harmless in CI (fresh checkout) but may require sudo to clean locally. +# Use plain `go test ./...` (no -coverprofile): the builder lacks covdata. +.PHONY: cubevs-test +cubevs-test: builder-image + $(MAKE) builder-run BUILDER_USER=0:0 BUILDER_RUN_EXTRA_MOUNTS='--privileged' BUILDER_CMD='cd /workspace/CubeNet/cubevs && make gen && go test ./...' + .PHONY: agent-test agent-test: builder-image $(MAKE) builder-run BUILDER_CMD='cd /workspace/agent && make test' diff --git a/deploy/one-click/README.md b/deploy/one-click/README.md index 0c1b9b6de..27b523185 100644 --- a/deploy/one-click/README.md +++ b/deploy/one-click/README.md @@ -170,6 +170,7 @@ One-click does not create an extra global `configs/` layer on the target machine - `cubelet_conf.default_timeout_insec`: cluster default sandbox idle TTL when the client omits `timeout`; unset or `<= 0` means **no cluster-wide idle timeout** (shipped default `-1`). See [lifecycle — Operational Notes](../../docs/guide/lifecycle.md#cluster-default-idle-timeout-default_timeout_insec). - `Cubelet/config/` → `Cubelet/config/` - `Cubelet/dynamicconf/` → `Cubelet/dynamicconf/` +- `CUBE_L7_MARK_{HTTP,HTTPS,MASK}` (env) → `/etc/cubeegress/l7-marks.conf` — the L7 egress skb->mark values shared by Cubelet's embedded network runtime eBPF dataplane (which stamps `skb->mark`) and the `cube-proxy-iptables-init` TPROXY rules (which match it). Both read the same file, and both validate the values (`HTTP != HTTPS`, bits confined to the mask). See `env.example` for the shipped defaults and how to override them. - `CubeAPI/bin/cube-api` → `/usr/local/services/cubetoolbox/CubeAPI/bin/cube-api` - `support/` → `/usr/local/services/cubetoolbox/support/` - `cubeproxy/` → `/usr/local/services/cubetoolbox/cubeproxy/` diff --git a/deploy/one-click/README_zh.md b/deploy/one-click/README_zh.md index 0def14894..8975f7211 100644 --- a/deploy/one-click/README_zh.md +++ b/deploy/one-click/README_zh.md @@ -159,6 +159,7 @@ one-click 不会在目标机额外创建一层全局 `configs/`，而是直接 - `cubelet_conf.default_timeout_insec`: cluster default sandbox idle TTL when the client omits `timeout`; unset or `<= 0` means **no cluster-wide idle timeout** (shipped default `-1`). See [lifecycle — 设计与运维要点](../../docs/zh/guide/lifecycle.md#集群默认空闲超时default_timeout_insec)。 - `Cubelet/config/` -> `Cubelet/config/` - `Cubelet/dynamicconf/` -> `Cubelet/dynamicconf/` +- `CUBE_L7_MARK_{HTTP,HTTPS,MASK}`（环境变量） -> `/etc/cubeegress/l7-marks.conf` —— L7 egress 的 skb->mark 值，由 Cubelet 内置 network runtime 的 eBPF 数据面（负责打标）与 `cube-proxy-iptables-init` 的 TPROXY 规则（负责匹配）共用。两侧读取同一文件并各自校验（`HTTP != HTTPS`、取值只能落在 mask 位内）。默认值与覆盖方式见 `env.example`。 - `CubeAPI/bin/cube-api` -> `/usr/local/services/cubetoolbox/CubeAPI/bin/cube-api` - `support/` -> `/usr/local/services/cubetoolbox/support/` - `cubeproxy/` -> `/usr/local/services/cubetoolbox/cubeproxy/` diff --git a/deploy/one-click/env.example b/deploy/one-click/env.example index 7b5e3f391..05da46262 100644 --- a/deploy/one-click/env.example +++ b/deploy/one-click/env.example @@ -178,6 +178,21 @@ CUBE_SANDBOX_CUBE_ROUTER_ENABLE=0 # usable IPs from CUBE_SANDBOX_NETWORK_CIDR. # CUBE_SANDBOX_CUBE_ROUTER_CIDR=10.254.0.0/24 +# ---- L7 egress skb->mark overrides ---- +# The L7 egress dataplane (Cubelet embedded network runtime eBPF) and the +# iptables TPROXY rules steer intercepted HTTP/HTTPS traffic by skb->mark. +# install.sh writes /etc/cubeegress/l7-marks.conf from these env vars (with the +# shipped defaults) before the consuming units start, and both Cubelet and the +# iptables init script read that same file, so the stamped and matched marks +# stay in lock-step. Override only to dodge a collision with an existing fwmark +# scheme in your environment: HTTP and HTTPS must differ and may only set bits +# inside CUBE_L7_MARK_MASK (keep the high 16 bits cube-owned). Leave unset to +# keep the shipped defaults; install.sh validates and refuses to persist +# invalid values. +# CUBE_L7_MARK_HTTP=0xCE010000 +# CUBE_L7_MARK_HTTPS=0xCE020000 +# CUBE_L7_MARK_MASK=0xFFFF0000 + # Cube proxy options. CUBE_PROXY_ENABLE=1 # Optional full image override (otherwise selected by MIRROR=cn|int): diff --git a/deploy/one-click/install.sh b/deploy/one-click/install.sh index 03a687800..4dda1e4f4 100755 --- a/deploy/one-click/install.sh +++ b/deploy/one-click/install.sh @@ -108,6 +108,37 @@ warn_default_external_credentials() { fi } +# Write /etc/cubeegress/l7-marks.conf so the L7 skb->mark values used by the +# dataplane (Cubelet embedded network runtime's eBPF globals) and the iptables +# TPROXY rules stay in lock-step. Override via CUBE_L7_MARK_{HTTP,HTTPS,MASK} +# in your env/.env; defaults match the shipped values. +write_l7_marks_conf() { + local http="${CUBE_L7_MARK_HTTP:-0xCE010000}" + local https="${CUBE_L7_MARK_HTTPS:-0xCE020000}" + local mask="${CUBE_L7_MARK_MASK:-0xFFFF0000}" + + # Validate before persisting: http must differ from https, and both may only + # set bits inside the mask. Compare arithmetically (not as strings) so the + # same value in different notations — 0xCE010000 vs 0xce010000 vs decimal — + # is still rejected, matching cubevs.resolveL7Marks. + if (( http == https )); then + die "CUBE_L7_MARK_HTTP (${http}) must differ from CUBE_L7_MARK_HTTPS" + fi + if (( (http & ~mask) != 0 || (https & ~mask) != 0 )); then + die "CUBE_L7_MARK_* values must set bits only within CUBE_L7_MARK_MASK (${mask})" + fi + + mkdir -p /etc/cubeegress + cat > /etc/cubeegress/l7-marks.conf <mark values. +CUBE_L7_MARK_HTTP=${http} +CUBE_L7_MARK_HTTPS=${https} +CUBE_L7_MARK_MASK=${mask} +EOF + log "wrote /etc/cubeegress/l7-marks.conf (http=${http} https=${https} mask=${mask})" +} + INSTALL_PREFIX="${CUBE_SANDBOX_INSTALL_ROOT}" # Resolve install vs upgrade mode and, for upgrades, run preflight + backup and @@ -1712,6 +1743,8 @@ else fi restore_selinux_contexts +# Persist the L7 skb->mark config before the units that consume it start. +write_l7_marks_conf install_systemd_units mask_external_dep_services check_runtime_file_paths_not_directories diff --git a/docs/guide/security-proxy.md b/docs/guide/security-proxy.md index 8841cd08b..dc343060d 100644 --- a/docs/guide/security-proxy.md +++ b/docs/guide/security-proxy.md @@ -21,14 +21,22 @@ rule list attached at sandbox-creation time: ## How it intercepts CubeEgress runs as a host-network container and binds two TPROXY -listeners on the sandbox-facing IP: +listeners on the sandbox-facing IP — an HTTP listener on 8080 and +an HTTPS listener on 8443. Which traffic reaches each listener is +decided per-rule by the port/scheme mapping (see +[Custom L7 ports](#custom-l7-ports)), not by a fixed destination +port: ``` sandbox ──→ cube-dev (host iface) │ - ├─ iptables mangle/PREROUTING -j TPROXY - │ port 80 → 192.168.0.1:8080 (HTTP listener) - │ port 443 → 192.168.0.1:8443 (HTTPS listener) + ├─ eBPF (mvmtap) resolves the outbound (host, port) to + │ a scheme via allow_out_v3 and stamps an skb->mark + │ (HTTP or HTTPS) on the SYN + │ + ├─ iptables mangle/PREROUTING -m mark -j TPROXY + │ HTTP mark → 192.168.0.1:8080 (HTTP listener) + │ HTTPS mark → 192.168.0.1:8443 (HTTPS listener) │ ▼ CubeEgress (OpenResty + lua) @@ -84,7 +92,8 @@ Match fields (all optional, AND'd together): | Field | Type | Notes | | --- | --- | --- | -| `scheme` | `"http"` / `"https"` | | +| `scheme` | `"http"` / `"https"` | Case-insensitive. See [Custom L7 ports](#custom-l7-ports) for how `scheme` interacts with `port`. | +| `port` | int | Destination TCP port to intercept, `1`–`65535`. Must be paired with `scheme`; see [Custom L7 ports](#custom-l7-ports). | | `sni` | string | TLS ClientHello SNI; supports leading `*.` for "any subdomain" — `*.example.com` matches `www.example.com` and `foo.bar.example.com`, but not the apex | | `host` | string | Match against the HTTP `Host:` header (port stripped); same semantics as `sni` — exact match, or leading `*.` for "any subdomain" (case-insensitive) | | `method` | list of methods | OR within the list (`["GET", "POST"]`) | @@ -93,6 +102,86 @@ Match fields (all optional, AND'd together): A request must match every present field; absent fields are wildcarded. +### Custom L7 ports + +By default a rule intercepts the classic `{80/http, 443/https}` +set. The optional `port` + `scheme` pair narrows or extends which +TCP port gets steered through the proxy: + +| `port` | `scheme` | Intercepts | +| --- | --- | --- | +| omitted | omitted | `{80/http, 443/https}` — the default set (backward compatible) | +| omitted | `"http"` / `"https"` | only that scheme on its default port (`http` → 80, `https` → 443) | +| set | set | exactly that `(host, port, scheme)` tuple — e.g. an API on `tcp/8443` | +| set | omitted | invalid — `port` requires `scheme` | + +```python +from cubesandbox import Sandbox, Rule, Match, Action + +rules = [ + # Intercept an internal API on a non-standard HTTPS port. + Rule( + name="internal_api", + match=Match(host="api.internal.example", port=8443, scheme="https"), + action=Action(allow=True), + ), + # Intercept plain HTTP on a custom port. + Rule( + name="custom_http", + match=Match(host="metrics.internal.example", port=18080, scheme="http"), + action=Action(allow=True), + ), + # scheme alone keeps the classic port but only matches one side. + Rule( + name="https_only", + match=Match(host="public.example", scheme="https"), + action=Action(allow=True), + ), +] + +with Sandbox.create(network={"rules": rules}) as sb: + sb.commands.run("curl -s https://api.internal.example:8443/health") + sb.commands.run("curl -s http://metrics.internal.example:18080/") + sb.commands.run("curl -s https://public.example/") # → proxied +``` + +Constraints (enforced by the SDK client and re-checked server-side): + +- `port` must be in `[1, 65535]`; `scheme` must be `http` or + `https` (case-insensitive). +- An L7 host must be a domain name or a single IP — subnet CIDRs + are rejected (a subnet can't appear in an HTTP `Host:` header or + TLS SNI). +- Every rule sharing the same `(host, port)` must agree on + `scheme`; a conflicting policy is rejected outright. +- At most 8 distinct `(port, scheme)` tuples per host. + +Traffic to a port no rule covers still falls back to the L3/L4 +`allow_out` / `deny_out` policy and never reaches CubeEgress. + +An omitted `port` means different things for allow and deny rules. +An **allow** rule without `port` narrows to the default set +`{80/http, 443/https}` — fail-closed, so a custom-port flow is only +proxied when a rule names that port. A **deny** rule without `port` +is port-agnostic within the host — it matches every intercepted flow +to the host regardless of port. This keeps a broad host deny from +being bypassed by a narrower custom-port allow: with +`deny host="*.example.com"` and `allow host="api.example.com", +port=8443, scheme="https"`, the deny still matches +`api.example.com:8443`, so the outcome is decided by rule order +(first match wins). List the more specific allow rule first to grant +the exception. + +The same domain may appear in both the L3/L4 `allow_out` list and an +L7 `rule`. In that case the two coexist: the domain's learned IPs get +a plain `/32` any-port allow entry (so non-rule ports keep ordinary +L3/L4 SNAT access) **and** an L7 `(ip, port)` entry per rule port (so +those ports are steered through CubeEgress). The more specific +`(ip, port)` match wins for the rule's ports; the `/32` covers the +rest. Put another way, adding an L7 rule for a domain already in +`allow_out` does **not** remove its plain L3 access — it only adds L7 +interception on the rule's ports. + ::: tip Single-level vs multi-level subdomain `*.example.com` matches **all** subdomains regardless of label depth. To allow only single-level subdomains (e.g. `www`, @@ -209,10 +298,12 @@ about: - **Internal `cube-dev` traffic** — sandbox-to-sandbox traffic and traffic to in-cluster services (Cube API, etc.) doesn't enter the TPROXY chain, so rules don't apply. -- **Non-HTTP egress on TCP/UDP** — the TPROXY chain only redirects - ports 80 and 443. Direct TCP to other ports still goes out - subject to the L3/L4 `allow_out` / `deny_out` policy on the - CubeNet data plane, but is invisible to CubeEgress. +- **TCP/UDP not covered by an L7 rule** — the TPROXY chain only + redirects traffic an L7 rule marked: the default `{80/http, + 443/https}` set plus any custom `(port, scheme)` a rule declares. + Direct TCP to ports no rule covers still goes out subject to the + L3/L4 `allow_out` / `deny_out` policy on the CubeNet data plane, + but is invisible to CubeEgress. - **Sandboxes built from templates without the CA bake** — if the template was created with `--with-cube-ca=false`, the sandbox's TLS clients don't trust CubeEgress's leaf certs and HTTPS calls diff --git a/docs/zh/guide/security-proxy.md b/docs/zh/guide/security-proxy.md index 0d3775bc2..2284a36b5 100644 --- a/docs/zh/guide/security-proxy.md +++ b/docs/zh/guide/security-proxy.md @@ -16,14 +16,20 @@ Cube Sandbox 在每台宿主机上部署一个透明出网代理 —— **CubeEg ## 拦截链路 CubeEgress 是一个 host-network 容器，在面向沙箱的 IP 上 bind 两个 -TPROXY listener： +TPROXY listener —— 8080 上的 HTTP listener 和 8443 上的 HTTPS +listener。哪些流量进哪个 listener，由每条规则声明的 port/scheme +映射决定（见[自定义 L7 端口](#自定义-l7-端口)），而不再绑定固定的 +目的端口： ``` sandbox ──→ cube-dev (主机网卡) │ - ├─ iptables mangle/PREROUTING -j TPROXY - │ port 80 → 192.168.0.1:8080 (HTTP) - │ port 443 → 192.168.0.1:8443 (HTTPS) + ├─ eBPF (mvmtap) 按 allow_out_v3 里的 (host, port)→scheme + │ 映射,在出方向 SYN 上打 skb->mark (HTTP 或 HTTPS) + │ + ├─ iptables mangle/PREROUTING -m mark -j TPROXY + │ HTTP mark → 192.168.0.1:8080 (HTTP) + │ HTTPS mark → 192.168.0.1:8443 (HTTPS) │ ▼ CubeEgress (OpenResty + lua) @@ -75,7 +81,8 @@ with Sandbox.create(network={"rules": rules}) as sb: | 字段 | 类型 | 说明 | | --- | --- | --- | -| `scheme` | `"http"` / `"https"` | | +| `scheme` | `"http"` / `"https"` | 大小写不敏感。与 `port` 的搭配语义见[自定义 L7 端口](#自定义-l7-端口)。 | +| `port` | int | 要拦截的目的 TCP 端口，`1`–`65535`。必须与 `scheme` 同时设置；见[自定义 L7 端口](#自定义-l7-端口)。 | | `sni` | string | TLS ClientHello 的 SNI；以 `*.` 开头时表示"任意子域"——`*.example.com` 同时命中 `www.example.com` 和 `foo.bar.example.com`，但**不**命中 apex | | `host` | string | 匹配 HTTP `Host:` 头（自动去除端口部分）；语义与 `sni` 相同 —— 支持精确匹配，或以 `*.` 开头的子域通配（大小写不敏感）| | `method` | 方法列表 | 列表内 OR 关系（`["GET", "POST"]`） | @@ -83,6 +90,81 @@ with Sandbox.create(network={"rules": rules}) as sb: 请求要同时满足所有出现的字段；未出现的字段视作通配。 +### 自定义 L7 端口 + +默认情况下，一条规则拦截经典的 `{80/http, 443/https}` 集合。可选 +的 `port` + `scheme` 组合可以收窄或扩展哪个 TCP 端口会被送进代理： + +| `port` | `scheme` | 拦截范围 | +| --- | --- | --- | +| 省略 | 省略 | `{80/http, 443/https}` —— 默认集合（向后兼容） | +| 省略 | `"http"` / `"https"` | 仅该 scheme 的默认端口（`http` → 80，`https` → 443） | +| 设置 | 设置 | 精确的 `(host, port, scheme)` 组合 —— 例如 `tcp/8443` 上的 API | +| 设置 | 省略 | 非法 —— `port` 必须搭配 `scheme` | + +```python +from cubesandbox import Sandbox, Rule, Match, Action + +rules = [ + # 拦截一个非标准 HTTPS 端口上的内部 API + Rule( + name="internal_api", + match=Match(host="api.internal.example", port=8443, scheme="https"), + action=Action(allow=True), + ), + # 拦截自定义端口上的明文 HTTP + Rule( + name="custom_http", + match=Match(host="metrics.internal.example", port=18080, scheme="http"), + action=Action(allow=True), + ), + # 只给 scheme 时仍走经典端口,但只匹配其中一侧 + Rule( + name="https_only", + match=Match(host="public.example", scheme="https"), + action=Action(allow=True), + ), +] + +with Sandbox.create(network={"rules": rules}) as sb: + sb.commands.run("curl -s https://api.internal.example:8443/health") + sb.commands.run("curl -s http://metrics.internal.example:18080/") + sb.commands.run("curl -s https://public.example/") # → 被代理 +``` + +约束（SDK 客户端先校验，服务端再校验一次）： + +- `port` 必须在 `[1, 65535]`；`scheme` 必须是 `http` 或 + `https`（大小写不敏感）。 +- L7 规则的 host 必须是域名或单个 IP —— 不支持子网 CIDR（子网 + 无法出现在 HTTP `Host:` 头或 TLS SNI 里）。 +- 共享同一个 `(host, port)` 的所有规则必须对 `scheme` 达成一致； + 冲突的 policy 会被整体拒绝。 +- 每个 host 最多 8 个不同的 `(port, scheme)` 组合。 + +任何规则都没有覆盖的端口，仍回落到 L3/L4 的 `allow_out` / +`deny_out` 策略，不会进入 CubeEgress。 + +省略 `port` 对 allow 和 deny 规则的含义不同。**allow** 规则省略 +`port` 时会收窄到默认集合 `{80/http, 443/https}` —— 这是 +fail-closed 的选择，即自定义端口的流量只有被规则显式点名才会进 +代理。**deny** 规则省略 `port` 时则在该 host 内与端口无关 —— +无论哪个端口进来的被拦截流量都会命中。这样可防止"较宽的 host +deny 被较窄的自定义端口 allow 绕过"：例如 `deny +host="*.example.com"` 与 `allow host="api.example.com", port=8443, +scheme="https"` 同时存在时，deny 仍会命中 +`api.example.com:8443`，最终结果由规则顺序（先命中先生效）决定。 +若想放行这个例外，把更具体的 allow 规则放在前面。 + +同一个域名可以同时出现在 L3/L4 的 `allow_out` 列表和某条 L7 +`rule` 里，此时两者**并存**：该域名学到的 IP 会同时得到一条纯 +`/32` 任意端口放行条目（让非 rule 端口保留普通的 L3/L4 SNAT 访 +问）**和**每个 rule 端口一条 L7 `(ip, port)` 条目（让这些端口被 +送进 CubeEgress 代理）。更精确的 `(ip, port)` 匹配对 rule 端口生 +效，`/32` 兜底其余端口。换句话说，给一个已在 `allow_out` 里的域 +名再加 L7 rule，**不会**移除它原有的纯 L3 放行 —— 只是在 rule 的 +端口上叠加了 L7 拦截。 + ::: tip 单层 vs 多层子域 `*.example.com` 不区分子域层数，**所有**结尾命中的子域都算。 若想只放行单层子域（如 `www`、`api`），需要为不希望放过的嵌套 @@ -187,9 +269,11 @@ inject 的 `secret` 值在写入任何日志路径前都会被剥除。审计日 - **`cube-dev` 内部流量** —— 沙箱到沙箱、沙箱到集群内服务（Cube API 等）不进 TPROXY 链路，不受规则约束。 -- **80/443 之外的 TCP/UDP** —— TPROXY 链路只重定向 80 和 443。 - 直连其它端口的 TCP 仍受 CubeNet 数据面的 L3/L4 `allow_out` / - `deny_out` 策略约束，但 CubeEgress 看不到。 +- **没有被 L7 规则覆盖的 TCP/UDP** —— TPROXY 链路只重定向被 L7 + 规则标记的流量：默认的 `{80/http, 443/https}` 集合，加上规则 + 声明的任意自定义 `(port, scheme)`。直连任何规则都没覆盖的端口 + 的 TCP，仍受 CubeNet 数据面的 L3/L4 `allow_out` / `deny_out` + 策略约束，但 CubeEgress 看不到。 - **没烘 CA 的模板** —— 如果模板用 `--with-cube-ca=false` 创建， 沙箱里的 TLS 客户端**不**信任 CubeEgress 签的 leaf 证书， HTTPS 在规则评估之前就会因 self-signed cert 报错。 diff --git a/examples/code-sandbox-quickstart/network_l7_custom_port_echo.py b/examples/code-sandbox-quickstart/network_l7_custom_port_echo.py new file mode 100644 index 000000000..a56901e7e --- /dev/null +++ b/examples/code-sandbox-quickstart/network_l7_custom_port_echo.py @@ -0,0 +1,215 @@ +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 + +""" +network_l7_custom_port_echo.py — L7 rule demo across the four port quadrants. + +One sandbox, five rules, six probes: + + 1. default http — rule without port (scheme="http" only): intercepts :80 + 2. default https — rule without port (scheme="https" only): intercepts :443 + 3. default both — rule with NEITHER port NOR scheme: one rule intercepts + both :80 and :443 (the classic {80/http, 443/https} set) + 4. custom http — rule with port=18080, scheme="http" + 5. custom https — rule with port=1012, scheme="https" + +How it works: + Each L7 rule opens the L3 path for the (host, port) tuples it matches and + tells CubeEgress which traffic to intercept. The HTTP legs assert the + injected marker header inside the echoed response body. Leg 4's target is + a local echo server started by this script (self-contained); leg 1 uses + httpbingo.org because binding :80 on the host is impractical. Leg 3 + (postman-echo.com) proves a bare host-only rule fans out to the whole + default {80/http, 443/https} set — both probes go through the SAME rule. + Rule evaluation is first-match-wins, so each leg deliberately uses its own + host: same-host rules would shadow each other and blur attribution. + + The HTTPS legs (2, 3b, 5) need an upstream certificate the CubeEgress proxy + can verify — it runs ``proxy_ssl_verify on`` against the system CA bundle, + so a local self-signed echo would be rejected by design. They therefore use + public endpoints: httpbingo.org:443 and postman-echo.com:443 (echo → + marker asserted) plus tls-v1-2.badssl.com:1012 (non-standard port, no + header echo → 200 + non-empty body asserted). + +Prerequisites: + - CUBE_TEMPLATE_ID, with the template built so the sandbox trusts the + cluster interception CA (same requirement as cube-test-network.py). + - The cluster can reach httpbingo.org / tls-v1-2.badssl.com (override via + the env vars below if your environment needs different endpoints). + +Env: + CUBE_TEMPLATE_ID (required) template to create the sandbox from + EXAMPLE_L7_TARGET_HOST (optional) bridge IP for the local echo leg + EXAMPLE_L7_HTTP_PORT (optional) local echo port, default 18080 + EXAMPLE_L7_DEFAULT_HOST (optional) host for legs 1-2, default httpbingo.org + EXAMPLE_L7_BOTH_HOST (optional) host for leg 3, default postman-echo.com + (must differ from EXAMPLE_L7_DEFAULT_HOST — + first-match-wins would otherwise shadow rules) + EXAMPLE_L7_CUSTOM_HTTPS_URL (optional) URL for leg 4, default + https://tls-v1-2.badssl.com:1012/ + +Constraints (SDK- and server-enforced): + - port requires scheme; set both or neither (SDK raises ValueError). + scheme alone stays on the classic {80, 443} set and only qualifies + whether HTTP or HTTPS traffic matches (legs 1-2 use exactly that). + - host must be a domain or a single IP — subnet CIDRs are rejected. + - Scheme must be consistent per (host, port) across rules; the server + rejects a conflicting policy outright. + - At most 8 distinct (port, scheme) tuples per host. + - host/scheme matching is case-insensitive. +""" + +import json +import os +import subprocess +import sys +import threading +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + +from cubesandbox import Action, Inject, Match, Rule, Sandbox +from env_utils import load_local_dotenv + +load_local_dotenv() + +template_id = os.environ["CUBE_TEMPLATE_ID"] + +HTTP_PORT = int(os.environ.get("EXAMPLE_L7_HTTP_PORT", "18080")) +DEFAULT_HOST = os.environ.get("EXAMPLE_L7_DEFAULT_HOST", "httpbingo.org") +BOTH_HOST = os.environ.get("EXAMPLE_L7_BOTH_HOST", "postman-echo.com") +assert BOTH_HOST != DEFAULT_HOST, ( + "EXAMPLE_L7_BOTH_HOST must differ from EXAMPLE_L7_DEFAULT_HOST " + "(first-match-wins would shadow the rules)" +) +CUSTOM_HTTPS_URL = os.environ.get( + "EXAMPLE_L7_CUSTOM_HTTPS_URL", "https://tls-v1-2.badssl.com:1012/" +) +custom_https = urlparse(CUSTOM_HTTPS_URL) +assert custom_https.scheme == "https" and custom_https.hostname and custom_https.port, ( + f"EXAMPLE_L7_CUSTOM_HTTPS_URL must be an https URL with an explicit port: {CUSTOM_HTTPS_URL!r}" +) +assert custom_https.port != 443, "custom https leg requires a non-standard port" + +MARKER = f"cube-l7-echo-{uuid.uuid4().hex[:12]}" + + +class HeaderEchoHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + payload = json.dumps({"path": self.path, "headers": dict(self.headers)}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_args: object) -> None: + pass + + +def detect_target_host() -> str: + """Pick a host bridge IP reachable from sandboxes (never the node IP).""" + override = os.environ.get("EXAMPLE_L7_TARGET_HOST") + if override: + return override + try: + out = subprocess.check_output(["ip", "-4", "-o", "addr", "show", "up"], text=True) + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 4 and parts[1].startswith(("docker", "br-")): + return parts[3].split("/")[0] + except Exception: + pass + print( + "WARNING: no UP docker*/br-* bridge found; falling back to 172.17.0.1, " + "which is unreachable from sandboxes when docker0 is down. If the " + "custom-http leg fails, set EXAMPLE_L7_TARGET_HOST to the bridge IP of " + "the network your sandboxes attach to (see `ip -4 addr`).", + file=sys.stderr, + ) + return "172.17.0.1" # docker0 default; override via EXAMPLE_L7_TARGET_HOST if down + + +target_host = detect_target_host() + +server = ThreadingHTTPServer(("0.0.0.0", HTTP_PORT), HeaderEchoHandler) +thread = threading.Thread(target=server.serve_forever, daemon=True) +thread.start() +print(f"echo server on 0.0.0.0:{HTTP_PORT}; custom-http leg targets {target_host}:{HTTP_PORT}") + +inject = [Inject(header="X-Cube-L7-Demo", secret=MARKER)] +rules = [ + # 1+2. Default set via scheme-only rules (no port): :80 http / :443 https. + Rule( + name="default-http", + match=Match(host=DEFAULT_HOST, scheme="http"), + action=Action(allow=True, inject=inject), + ), + Rule( + name="default-https", + match=Match(host=DEFAULT_HOST, scheme="https"), + action=Action(allow=True, inject=inject), + ), + # 3. Neither port nor scheme: one rule fans out to {80/http, 443/https}. + Rule( + name="default-both", + match=Match(host=BOTH_HOST), + action=Action(allow=True, inject=inject), + ), + # 4. Custom HTTP port against the local echo server. + Rule( + name="custom-http", + match=Match(host=target_host, port=HTTP_PORT, scheme="http"), + action=Action(allow=True, inject=inject), + ), + # 5. Custom HTTPS port against a real publicly-signed upstream. + Rule( + name="custom-https", + match=Match(host=custom_https.hostname, port=custom_https.port, scheme="https"), + action=Action(allow=True, audit="metadata"), + ), +] + +results = [] + +try: + with Sandbox.create( + template=template_id, + timeout=120, + allow_internet_access=False, + network={"rules": rules}, + ) as sandbox: + + def probe_marker(label: str, url: str) -> None: + # httpbingo wraps every echoed header value in a single-element + # JSON array; a plain substring grep still matches the marker. + r = sandbox.commands.run(f"curl -sS --max-time 20 '{url}'", timeout=30) + out = r.stdout.strip() + ok = MARKER in out + snippet = out if len(out) <= 300 else out[:300] + "..." + print(f"{label}: {'OK' if ok else 'FAIL'} — {snippet or r.stderr.strip()}") + results.append(ok) + + def probe_status(label: str, url: str) -> None: + r = sandbox.commands.run( + f"curl -sS --max-time 20 '{url}' -o /dev/null -w 'code=%{{http_code}} len=%{{size_download}}'", + timeout=30, + ) + out = r.stdout.strip() + ok = "code=200" in out and "len=0" not in out + print(f"{label}: {'OK' if ok else 'FAIL'} — {out or r.stderr.strip()}") + results.append(ok) + + probe_marker(f"1 default http :{80:<5}", f"http://{DEFAULT_HOST}/headers") + probe_marker(f"2 default https :{443:<5}", f"https://{DEFAULT_HOST}/headers") + probe_marker(f"3a default both :{80:<5}", f"http://{BOTH_HOST}/headers") + probe_marker(f"3b default both :{443:<5}", f"https://{BOTH_HOST}/headers") + probe_marker(f"4 custom http :{HTTP_PORT:<5}", f"http://{target_host}:{HTTP_PORT}/headers") + probe_status(f"5 custom https :{custom_https.port:<5}", CUSTOM_HTTPS_URL) +finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + +passed = sum(results) +print(f"summary: {passed}/{len(results)} legs passed (marker={MARKER})") diff --git a/sdk/go/aligned_test.go b/sdk/go/aligned_test.go index 3f259dba7..fd7c0c6ce 100644 --- a/sdk/go/aligned_test.go +++ b/sdk/go/aligned_test.go @@ -69,6 +69,97 @@ func TestCreateSerializesPolicyAndPublicTraffic(t *testing.T) { } } +func TestCreateSerializesRulePortAndNormalizesScheme(t *testing.T) { + var got map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&got) + w.WriteHeader(http.StatusCreated) + fmt.Fprint(w, sandboxJSON(testSandboxID, "tpl-env")) + })) + defer server.Close() + + client := NewClient(Config{APIURL: server.URL, TemplateID: "tpl-env", Timeout: 300 * time.Second}) + _, err := client.Create(context.Background(), CreateOptions{ + Network: NetworkOptions{ + Rules: []Rule{{ + Name: "custom-https", + Match: Match{Host: "api.example.com", Port: 8443, Scheme: " HTTPS "}, + Action: Action{Allow: true}, + }}, + }, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + + network := got["network"].(map[string]any) + rules := network["rules"].([]any) + rule := rules[0].(map[string]any) + match := rule["match"].(map[string]any) + if match["port"] != float64(8443) { + t.Fatalf("match.port=%#v, want 8443", match["port"]) + } + if match["scheme"] != "https" { + t.Fatalf("match.scheme=%#v, want normalized https", match["scheme"]) + } +} + +func TestCreateRejectsInvalidRulePortScheme(t *testing.T) { + called := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + w.WriteHeader(http.StatusCreated) + fmt.Fprint(w, sandboxJSON(testSandboxID, "tpl-env")) + })) + defer server.Close() + + tests := []struct { + name string + match Match + wantErr string + }{ + {"port without scheme", Match{Host: "a.com", Port: 8443}, "port requires match.scheme"}, + {"port too low", Match{Host: "a.com", Port: 0, Scheme: "https"}, "not-reached"}, // 0 means unset + {"port negative", Match{Host: "a.com", Port: -1, Scheme: "https"}, "[1, 65535]"}, + {"port too high", Match{Host: "a.com", Port: 65536, Scheme: "https"}, "[1, 65535]"}, + {"unknown scheme", Match{Host: "a.com", Scheme: "gopher"}, "must be 'http' or 'https'"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == "not-reached" { + return // Port 0 is the zero value (unset); covered by the "port without scheme" case + } + client := NewClient(Config{APIURL: server.URL, TemplateID: "tpl-env"}) + _, err := client.Create(context.Background(), CreateOptions{ + Network: NetworkOptions{ + Rules: []Rule{{Name: "r1", Match: tt.match, Action: Action{Allow: true}}}, + }, + }) + if err == nil { + t.Fatalf("err=nil, want %q", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err=%q, want substring %q", err, tt.wantErr) + } + }) + } + if called { + t.Fatal("server was called despite client-side validation failure") + } +} + +func TestMatchValidateAcceptsLegacySchemeOnly(t *testing.T) { + // Scheme alone (no port) filters HTTP vs HTTPS on the default {80, 443} + // set — the classic behavior, not the port-scoped feature. + m := Match{Host: "api.example.com", Scheme: "HTTPS"} + if err := m.validate(); err != nil { + t.Fatalf("validate: %v", err) + } + if got := m.normalized().Scheme; got != "https" { + t.Fatalf("normalized scheme=%q, want https", got) + } +} + func TestCreateRejectsAllowOutDomainWithoutDenyAll(t *testing.T) { called := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/sdk/go/client.go b/sdk/go/client.go index 3d72208c3..411f026a2 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -160,7 +160,15 @@ func (c *Client) createPayload(opts CreateOptions) (map[string]any, error) { network["denyOut"] = opts.Network.DenyOut } if len(opts.Network.Rules) > 0 { - network["rules"] = opts.Network.Rules + rules := make([]Rule, len(opts.Network.Rules)) + for i, rule := range opts.Network.Rules { + if err := rule.Match.validate(); err != nil { + return nil, fmt.Errorf("network.rules[%d] %q: %w", i, rule.Name, err) + } + rule.Match = rule.Match.normalized() + rules[i] = rule + } + network["rules"] = rules } if len(network) > 0 { payload["network"] = network diff --git a/sdk/go/policy.go b/sdk/go/policy.go index 3a5846d7f..721770b92 100644 --- a/sdk/go/policy.go +++ b/sdk/go/policy.go @@ -4,6 +4,7 @@ package cubesandbox import ( + "fmt" "net" "strings" ) @@ -16,12 +17,52 @@ import ( // Match holds rule match conditions. All fields are optional; an empty Match // matches any request. Fields are AND-ed; Method values are OR-ed; // sni/host/scheme are compared case-insensitively server-side. +// +// Port + Scheme together drive which TCP port CubeEgress intercepts on the +// sandbox side. Both must be set together or both omitted: +// - Both omitted → default set {80/http, 443/https} (backward compatible). +// - Both set → CubeEgress intercepts only that (host, port, scheme) tuple. +// +// Every rule sharing the same (host, port) MUST agree on Scheme — a port can +// only route to one nginx listener. The server rejects the whole policy on +// mismatch. type Match struct { SNI string `json:"sni,omitempty"` Host string `json:"host,omitempty"` Method []string `json:"method,omitempty"` Path string `json:"path,omitempty"` Scheme string `json:"scheme,omitempty"` + Port int `json:"port,omitempty"` +} + +// validate checks the port/scheme pair, mirroring the Python and TypeScript +// SDKs and the CubeAPI/CubeMaster server-side contract: a set Port must be in +// [1, 65535] and must be paired with Scheme, and a set Scheme must be http or +// https (case-insensitive). Client-side validation catches typos before the +// network round-trip; the server rejects the same shapes. +func (m Match) validate() error { + if m.Scheme != "" { + scheme := strings.ToLower(strings.TrimSpace(m.Scheme)) + if scheme != "http" && scheme != "https" { + return fmt.Errorf("match.scheme must be 'http' or 'https', got %q", m.Scheme) + } + } + if m.Port != 0 { + if m.Port < 1 || m.Port > 65535 { + return fmt.Errorf("match.port must be in [1, 65535], got %d", m.Port) + } + if m.Scheme == "" { + return fmt.Errorf("match.port requires match.scheme to be set") + } + } + return nil +} + +// normalized returns a copy with Scheme canonicalized (stripped, lowercased) +// so the wire form is consistent regardless of caller casing. +func (m Match) normalized() Match { + m.Scheme = strings.ToLower(strings.TrimSpace(m.Scheme)) + return m } // Inject injects a credential header on an allowed HTTPS request whose diff --git a/sdk/node/src/policy.ts b/sdk/node/src/policy.ts index aa6df7bda..6b6103fc1 100644 --- a/sdk/node/src/policy.ts +++ b/sdk/node/src/policy.ts @@ -31,6 +31,16 @@ export type AuditLevel = "full" | "metadata" | "none"; * Rule match conditions. All fields optional; an empty match matches any * request. Semantics: AND across fields, OR within ``method``. Comparisons on * sni/host/scheme are case-insensitive (server-enforced). + * + * ``port`` + ``scheme`` together drive which TCP port CubeEgress intercepts + * on the sandbox side. Both must be set together or both omitted: + * + * - Both omitted → default set ``{80/http, 443/https}`` (backward compatible). + * - Both set → CubeEgress intercepts only that (host, port, scheme) tuple. + * + * Every rule sharing the same ``(host, port)`` MUST agree on ``scheme`` — + * a port can only route to one nginx listener (HTTP → 8080, HTTPS → 8443). + * The server rejects the whole policy if it detects a mismatch. */ export interface Match { sni?: string; @@ -38,6 +48,7 @@ export interface Match { method?: Method[]; path?: string; scheme?: Scheme; + port?: number; } /** @@ -87,13 +98,65 @@ export function renderInject(inject: Inject): string { return fmt.replace("${SECRET}", inject.secret); } +/** + * Normalize *value* (strip + lowercase) and validate against http/https. + * + * Returns the normalized scheme, or ``undefined`` when *value* is absent. + * Scheme matching is case-insensitive across the stack (Lua + * ``normalize_scheme`` and cubevs both lowercase/strip), so the SDK + * normalizes before the membership check and propagates the clean lowercase + * value downstream. + */ +function validateScheme(value: unknown, field: string): Scheme | undefined { + if (value === undefined) { + return undefined; + } + if (typeof value !== "string") { + throw new Error(`${field} must be 'http' or 'https', got ${JSON.stringify(value)}`); + } + const normalized = value.trim().toLowerCase(); + if (normalized !== "http" && normalized !== "https") { + throw new Error(`${field} must be 'http' or 'https', got ${JSON.stringify(value)}`); + } + return normalized; +} + +/** + * Client-side validation of the port/scheme pair, mirroring the Python SDK + * contract and the CubeAPI/CubeMaster server-side checks: a set port must be + * an integer in [1, 65535] and must be paired with a scheme. Scheme alone + * (no port) stays legal — it filters HTTP vs HTTPS on the default {80, 443} + * set, which is the classic behavior, not the port-scoped feature. + * + * Not strictly required (the server would reject the same shape) but catches + * typos before the network round-trip. + */ +function validateMatchPortScheme(match: Match): void { + if (match.port !== undefined) { + if (typeof match.port !== "number" || !Number.isInteger(match.port)) { + throw new Error(`match.port must be an int, got ${typeof match.port}`); + } + if (match.port < 1 || match.port > 65535) { + throw new Error(`match.port must be in [1, 65535], got ${match.port}`); + } + if (match.scheme === undefined) { + throw new Error("match.port requires match.scheme to be set"); + } + } +} + function serializeMatch(match: Match): Record { + // Validate and normalize before emitting the wire form so the canonical + // lowercase scheme reaches the server regardless of caller casing. + const scheme = validateScheme(match.scheme, "match.scheme"); + validateMatchPortScheme(match); const out: Record = {}; if (match.sni !== undefined) out.sni = match.sni; if (match.host !== undefined) out.host = match.host; if (match.method !== undefined) out.method = [...match.method]; if (match.path !== undefined) out.path = match.path; - if (match.scheme !== undefined) out.scheme = match.scheme; + if (scheme !== undefined) out.scheme = scheme; + if (match.port !== undefined) out.port = match.port; return out; } diff --git a/sdk/node/test/policy.test.ts b/sdk/node/test/policy.test.ts index 3a560c4e0..135ff7aac 100644 --- a/sdk/node/test/policy.test.ts +++ b/sdk/node/test/policy.test.ts @@ -66,6 +66,66 @@ describe("serializeRule", () => { { header: "Authorization", secret: "sk_xxx", format: "Bearer ${SECRET}" }, ]); }); + + it("emits port and normalizes scheme casing on the wire", () => { + const wire = serializeRule({ + name: "r1", + match: { host: "api.example.com", port: 8443, scheme: "HTTPS" as "https" }, + action: { allow: true }, + }); + expect(wire.match).toEqual({ host: "api.example.com", port: 8443, scheme: "https" }); + }); + + it("rejects port without scheme", () => { + expect(() => + serializeRule({ + name: "r1", + match: { host: "api.example.com", port: 8443 }, + action: { allow: true }, + }), + ).toThrow("match.port requires match.scheme to be set"); + }); + + it("rejects out-of-range ports", () => { + for (const port of [0, -1, 65536, 99999]) { + expect(() => + serializeRule({ + name: "r1", + match: { host: "api.example.com", port, scheme: "https" }, + action: { allow: true }, + }), + ).toThrow("match.port must be in [1, 65535]"); + } + }); + + it("rejects non-integer ports", () => { + expect(() => + serializeRule({ + name: "r1", + match: { host: "api.example.com", port: 443.5, scheme: "https" }, + action: { allow: true }, + }), + ).toThrow("match.port must be an int"); + }); + + it("rejects unknown schemes", () => { + expect(() => + serializeRule({ + name: "r1", + match: { host: "api.example.com", scheme: "gopher" as "http" }, + action: { allow: true }, + }), + ).toThrow("match.scheme must be 'http' or 'https'"); + }); + + it("accepts scheme alone (legacy default-set filter)", () => { + const wire = serializeRule({ + name: "r1", + match: { host: "api.example.com", scheme: "https" }, + action: { allow: true }, + }); + expect(wire.match).toEqual({ host: "api.example.com", scheme: "https" }); + }); }); describe("renderInject", () => { diff --git a/sdk/python/cubesandbox/_policy.py b/sdk/python/cubesandbox/_policy.py index a0536ad21..cb15d7e2d 100644 --- a/sdk/python/cubesandbox/_policy.py +++ b/sdk/python/cubesandbox/_policy.py @@ -26,6 +26,25 @@ AuditLevel = Literal["full", "metadata", "none"] +def _validate_scheme(value: Any, field: str) -> Optional[str]: + """Normalize *value* (strip + lowercase) and validate against http/https. + + Returns the normalized scheme, or ``None`` when *value* is ``None``. + Scheme matching is case-insensitive across the stack (Lua + ``normalize_scheme`` and cubevs both lowercase/strip), so the SDK + normalizes before the membership check and propagates the clean + lowercase value downstream. + """ + if value is None: + return None + if not isinstance(value, str): + raise ValueError(f"{field} must be 'http' or 'https', got {value!r}") + normalized = value.strip().lower() + if normalized not in ("http", "https"): + raise ValueError(f"{field} must be 'http' or 'https', got {value!r}") + return normalized + + @dataclass class Match: """ @@ -33,12 +52,44 @@ class Match: Multi-field semantics: AND across fields, OR within ``method``. Comparisons on sni/host/scheme are case-insensitive. + + ``port`` + ``scheme`` together drive which TCP port CubeEgress intercepts + on the sandbox side. Both must be set together or both omitted: + + - Both omitted → default set ``{80/http, 443/https}`` (backward compatible). + - Both set → CubeEgress intercepts only that (host, port, scheme) tuple. + + Every rule sharing the same ``(host, port)`` MUST agree on ``scheme`` — + a port can only route to one nginx listener (HTTP → 8080, HTTPS → 8443). + The server rejects the whole policy if it detects a mismatch. """ sni: Optional[str] = None host: Optional[str] = None method: Optional[List[Method]] = None path: Optional[str] = None scheme: Optional[Scheme] = None + port: Optional[int] = None + + def __post_init__(self) -> None: + # Client-side pre-validation. Not strictly required (the server would + # reject the same shape) but catches typos before the network round-trip + # and produces a Pythonic error path (ValueError) instead of a 400. + # _validate_scheme returns the normalized scheme; store it so + # to_wire() emits the canonical lowercase form. + self.scheme = _validate_scheme(self.scheme, "Match.scheme") + if self.port is not None: + if not isinstance(self.port, int) or isinstance(self.port, bool): + raise ValueError(f"Match.port must be an int, got {type(self.port).__name__}") + if self.port < 1 or self.port > 65535: + raise ValueError(f"Match.port must be in [1, 65535], got {self.port}") + if self.scheme is None: + raise ValueError("Match.port requires Match.scheme to be set") + if self.scheme is not None and self.port is None: + # Legacy shape: scheme alone (no port) is still accepted server-side + # as a match qualifier that filters by HTTP vs HTTPS on the default + # {80, 443} set. This is not the new port-scoped feature, so we do + # NOT raise — but callers relying on it get the classic behavior. + pass def to_wire(self) -> Dict[str, Any]: out: Dict[str, Any] = {} @@ -52,6 +103,8 @@ def to_wire(self) -> Dict[str, Any]: out["path"] = self.path if self.scheme is not None: out["scheme"] = self.scheme + if self.port is not None: + out["port"] = self.port return out @@ -125,7 +178,28 @@ def to_wire(self) -> Dict[str, Any]: def _normalize_match_dict(m: Dict[str, Any]) -> Dict[str, Any]: - return dict(m) + out = dict(m) + # Best-effort client-side validation for the port + scheme pair. Mirrors + # Match.__post_init__ so dict-shaped and dataclass-shaped rules get the + # same behavior. Errors here would otherwise surface as a 400 from the + # server; catching them locally gives users a Pythonic ValueError with a + # stack trace pointing at the offending call site. + port = out.get("port") + scheme = _validate_scheme(out.get("scheme"), "match.scheme") + if scheme is not None: + # Propagate the normalized (stripped, lowercased) scheme so the wire + # form is canonical regardless of caller casing/whitespace. + out["scheme"] = scheme + if port is not None: + if not isinstance(port, int) or isinstance(port, bool): + raise ValueError( + f"match.port must be an int, got {type(port).__name__}" + ) + if port < 1 or port > 65535: + raise ValueError(f"match.port must be in [1, 65535], got {port}") + if scheme is None: + raise ValueError("match.port requires match.scheme to be set") + return out def _normalize_inject_dict(i: Dict[str, Any]) -> Dict[str, Any]: @@ -267,6 +341,14 @@ def _convert_e2b_per_host_rules(rules: Dict[str, Any]) -> List[Dict[str, Any]]: f"network.rules[{host!r}] must be a list of transform entries, " f"got {type(entries).__name__}" ) + if not entries: + # An empty list would fan out to zero rules and silently drop the + # host the caller keyed in — the exact "silent drop" this + # compatibility layer exists to prevent. + raise ValueError( + f"network.rules[{host!r}] is an empty list; every host must " + "declare at least one transform entry" + ) for index, entry in enumerate(entries): if not isinstance(entry, dict): raise ValueError( diff --git a/sdk/python/tests/test_l7_custom_port_e2e.py b/sdk/python/tests/test_l7_custom_port_e2e.py new file mode 100644 index 000000000..9af501f42 --- /dev/null +++ b/sdk/python/tests/test_l7_custom_port_e2e.py @@ -0,0 +1,194 @@ +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Live custom-port CubeEgress dataplane tests. + +The test starts a host-side HTTP echo target, creates one sandbox with an +HTTP custom-port inject rule and an HTTPS custom-port deny rule, then executes +both requests from inside the sandbox. + +Required opt-in: + +- ``CUBE_E2E=1`` or pytest ``--run-e2e`` +- ``CUBE_L7_E2E_HTTP_TARGET_HOST``: a host address reachable from sandboxes + that is not CubeVS's node-IP fast path (a Docker bridge gateway is suitable) +- ``CUBE_TEMPLATE_ID`` or pytest ``--cube-template-id`` + +Optional: + +- ``CUBE_L7_E2E_HTTP_BIND_HOST`` (default ``0.0.0.0``) +- ``CUBE_L7_E2E_HTTP_PORT`` (default ``18080``) +- ``CUBE_L7_E2E_HTTPS_URL`` + (default ``https://tls-v1-2.badssl.com:1012/``) +- ``CUBE_L7_E2E_HTTPS_ALLOW_URL`` + (defaults to the same non-standard-port HTTPS URL) +""" + +from __future__ import annotations + +import base64 +import json +import os +import threading +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + +import pytest + +from cubesandbox import Action, Config, Inject, Match, Rule, Sandbox + +pytestmark = pytest.mark.e2e + + +class _HeaderEchoHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + payload = json.dumps({"path": self.path, "headers": dict(self.headers)}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, _format: str, *_args: object) -> None: + return + + +def _option(pytestconfig: pytest.Config, option: str, env: str) -> str | None: + return pytestconfig.getoption(option) or os.environ.get(env) + + +def _require_custom_port_e2e(pytestconfig: pytest.Config) -> tuple[str, str]: + if not pytestconfig.getoption("--run-e2e") and os.environ.get("CUBE_E2E") != "1": + pytest.skip("use --run-e2e or set CUBE_E2E=1") + target_host = os.environ.get("CUBE_L7_E2E_HTTP_TARGET_HOST") + if not target_host: + pytest.skip("set CUBE_L7_E2E_HTTP_TARGET_HOST to a sandbox-reachable host address") + template_id = _option(pytestconfig, "--cube-template-id", "CUBE_TEMPLATE_ID") + if not template_id: + pytest.skip("set CUBE_TEMPLATE_ID or --cube-template-id") + return target_host, template_id + + +def _python_command(source: str) -> str: + encoded = base64.b64encode(source.encode()).decode() + return f"python3 -c \"import base64;exec(base64.b64decode('{encoded}'))\"" + + +def _https_probe_script(url: str) -> str: + return f""" +import urllib.error +import urllib.request +opener = urllib.request.build_opener(urllib.request.ProxyHandler({{}})) +try: + with opener.open({url!r}, timeout=30) as response: + print('STATUS=' + str(response.status)) + print('FINAL_URL=' + response.geturl()) + print('BODY_LEN=' + str(len(response.read()))) +except urllib.error.HTTPError as error: + print('STATUS=' + str(error.code)) +""" + + +def test_l7_custom_http_inject_and_https_deny(pytestconfig: pytest.Config) -> None: + target_host, template_id = _require_custom_port_e2e(pytestconfig) + bind_host = os.environ.get("CUBE_L7_E2E_HTTP_BIND_HOST", "0.0.0.0") + http_port = int(os.environ.get("CUBE_L7_E2E_HTTP_PORT", "18080")) + https_url = os.environ.get( + "CUBE_L7_E2E_HTTPS_URL", "https://tls-v1-2.badssl.com:1012/" + ) + parsed_https = urlparse(https_url) + assert parsed_https.scheme == "https" and parsed_https.hostname and parsed_https.port + + marker = f"cube-l7-{uuid.uuid4().hex}" + server = ThreadingHTTPServer((bind_host, http_port), _HeaderEchoHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + config = Config(api_url=os.environ.get("CUBE_API_URL", "http://127.0.0.1:3000")) + rules = [ + Rule( + name="e2e-custom-http-inject", + match=Match(host=target_host, port=http_port, scheme="http"), + action=Action( + allow=True, + inject=[Inject(header="X-Cube-L7-E2E", secret=marker)], + ), + ), + Rule( + name="e2e-custom-https-deny", + match=Match(host=parsed_https.hostname, port=parsed_https.port, scheme="https"), + action=Action(allow=False), + ), + ] + + try: + with Sandbox.create( + template=template_id, + timeout=120, + allow_internet_access=False, + network={"rules": rules}, + config=config, + ) as sandbox: + http_url = f"http://{target_host}:{http_port}/headers" + http_script = f""" +import urllib.request +opener = urllib.request.build_opener(urllib.request.ProxyHandler({{}})) +with opener.open({http_url!r}, timeout=20) as response: + print('STATUS=' + str(response.status)) + print(response.read().decode()) +""" + http_result = sandbox.commands.run(_python_command(http_script), timeout=30) + assert http_result.exit_code == 0, http_result.stderr + assert "STATUS=200" in http_result.stdout + assert marker in http_result.stdout, ( + "custom HTTP request reached the target without the injected header: " + + http_result.stdout + ) + + https_result = sandbox.commands.run( + _python_command(_https_probe_script(https_url)), timeout=40 + ) + assert https_result.exit_code == 0, https_result.stderr + assert "STATUS=403" in https_result.stdout, ( + "custom HTTPS deny rule was not enforced: " + https_result.stdout + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def test_l7_custom_https_allow_reaches_real_upstream(pytestconfig: pytest.Config) -> None: + _, template_id = _require_custom_port_e2e(pytestconfig) + https_url = os.environ.get( + "CUBE_L7_E2E_HTTPS_ALLOW_URL", + os.environ.get("CUBE_L7_E2E_HTTPS_URL", "https://tls-v1-2.badssl.com:1012/"), + ) + parsed = urlparse(https_url) + assert parsed.scheme == "https" and parsed.hostname and parsed.port + assert parsed.port != 443, "custom HTTPS allow E2E requires a non-standard port" + + config = Config(api_url=os.environ.get("CUBE_API_URL", "http://127.0.0.1:3000")) + rule = Rule( + name="e2e-custom-https-allow", + match=Match(host=parsed.hostname, port=parsed.port, scheme="https"), + action=Action(allow=True, audit="metadata"), + ) + + with Sandbox.create( + template=template_id, + timeout=120, + allow_internet_access=False, + network={"rules": [rule]}, + config=config, + ) as sandbox: + result = sandbox.commands.run( + _python_command(_https_probe_script(https_url)), timeout=40 + ) + assert result.exit_code == 0, result.stderr + assert "STATUS=200" in result.stdout, ( + "custom HTTPS allow did not reach the real upstream: " + result.stdout + ) + assert "BODY_LEN=0" not in result.stdout, ( + "custom HTTPS upstream returned no response body: " + result.stdout + ) diff --git a/sdk/python/tests/test_l7_custom_port_validation_e2e.py b/sdk/python/tests/test_l7_custom_port_validation_e2e.py new file mode 100644 index 000000000..1d2b3ae80 --- /dev/null +++ b/sdk/python/tests/test_l7_custom_port_validation_e2e.py @@ -0,0 +1,107 @@ +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Negative / validation e2e cases for custom-port CubeEgress rules. + +These complement ``test_l7_custom_port_e2e.py`` (which covers the happy +paths: custom HTTP inject, custom HTTPS allow/deny). Here we assert the +*rejection* contract for malformed or conflicting custom-port specs: + +* two rules pinning the same ``(host, port)`` to different schemes are + rejected as a whole policy by CubeMaster/CubeEgress; +* more than ``MAX_L7_PORTS_PER_HOST`` (8) distinct ``(port, scheme)`` + tuples on one host are rejected. + +(The client-side "port requires scheme" contract is covered +unconditionally by ``tests/test_policy.py::test_port_without_scheme_rejected``; +both cases below drive a real ``Sandbox.create`` and therefore require a +live cluster.) + +Required opt-in (same as the happy-path e2e): + +- ``CUBE_E2E=1`` or pytest ``--run-e2e`` +- ``CUBE_TEMPLATE_ID`` or pytest ``--cube-template-id`` +""" + +from __future__ import annotations + +import os + +import pytest + +from cubesandbox import Action, Config, Match, Rule, Sandbox +from cubesandbox._exceptions import ApiError + +pytestmark = pytest.mark.e2e + + +def _skip_unless_e2e(pytestconfig: pytest.Config) -> str | None: + if not pytestconfig.getoption("--run-e2e") and os.environ.get("CUBE_E2E") != "1": + pytest.skip("use --run-e2e or set CUBE_E2E=1") + template_id = ( + pytestconfig.getoption("--cube-template-id") + or os.environ.get("CUBE_TEMPLATE_ID") + ) + if not template_id: + pytest.skip("set CUBE_TEMPLATE_ID or --cube-template-id") + return template_id + + +def test_l7_custom_scheme_conflict_rejected( + pytestconfig: pytest.Config, +) -> None: + # (host, port) pinned to two different schemes is a whole-policy + # rejection: iptables can only steer one port to one listener. + template_id = _skip_unless_e2e(pytestconfig) + config = Config(api_url=os.environ.get("CUBE_API_URL", "http://127.0.0.1:3000")) + rules = [ + Rule( + name="custom-8080-http", + match=Match(host="api.example.com", port=8080, scheme="http"), + action=Action(allow=True), + ), + Rule( + name="custom-8080-https", + match=Match(host="api.example.com", port=8080, scheme="https"), + action=Action(allow=True), + ), + ] + # Assert the *policy* rejection specifically (ApiError mentioning the + # conflict), not just any failure — a connection error or unrelated 500 + # must not satisfy this. + with pytest.raises(ApiError, match="conflict"): + with Sandbox.create( + template=template_id, + timeout=120, + allow_internet_access=False, + network={"rules": rules}, + config=config, + ): + pass + + +def test_l7_custom_port_budget_exceeded_rejected( + pytestconfig: pytest.Config, +) -> None: + # More than 8 distinct (port, scheme) tuples on one host must be + # rejected rather than silently truncated. + template_id = _skip_unless_e2e(pytestconfig) + config = Config(api_url=os.environ.get("CUBE_API_URL", "http://127.0.0.1:3000")) + rules = [ + Rule( + name=f"custom-{i}", + match=Match(host="api.example.com", port=8000 + i, scheme="http"), + action=Action(allow=True), + ) + for i in range(9) + ] + # Assert the *budget* rejection specifically (ApiError mentioning the + # exceeded tuple limit), not just any failure. + with pytest.raises(ApiError, match="exceeds"): + with Sandbox.create( + template=template_id, + timeout=120, + allow_internet_access=False, + network={"rules": rules}, + config=config, + ): + pass diff --git a/sdk/python/tests/test_policy.py b/sdk/python/tests/test_policy.py new file mode 100644 index 000000000..e84ce7e23 --- /dev/null +++ b/sdk/python/tests/test_policy.py @@ -0,0 +1,155 @@ +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the ``cubesandbox._policy`` module — L7 egress rule dataclasses.""" + +from __future__ import annotations + +import pytest + +from cubesandbox import Action, Inject, Match, Rule +from cubesandbox._policy import ( + _convert_e2b_per_host_rules, + _normalize_match_dict, + _serialize_rule, +) + + +class TestMatchWireShape: + def test_empty_match_produces_empty_dict(self): + assert Match().to_wire() == {} + + def test_host_only(self): + assert Match(host="api.example.com").to_wire() == {"host": "api.example.com"} + + def test_default_port_scheme_omitted(self): + # No port/scheme → wire dict does not carry those keys, keeping + # backward compatibility with servers that predate the L7 port field. + wire = Match(host="api.example.com").to_wire() + assert "port" not in wire + assert "scheme" not in wire + + def test_explicit_port_and_scheme_serialized(self): + wire = Match(host="api.example.com", port=8080, scheme="http").to_wire() + assert wire == {"host": "api.example.com", "port": 8080, "scheme": "http"} + + def test_scheme_alone_uses_default_port(self): + # SDK does not expand scheme-only rules on the client side — the wire + # form is left untouched and the server (network-agent) fills in the + # scheme's default port (http → 80, https → 443) when building the + # cubevs L7 plan. Verify only that the wire shape is stable. + wire = Match(host="api.example.com", scheme="https").to_wire() + assert wire == {"host": "api.example.com", "scheme": "https"} + + def test_scheme_normalized_strip_lowercase(self): + # Scheme matching is case-insensitive across the stack (Lua + # normalize_scheme and cubevs both lowercase/strip) — the SDK + # normalizes before validating and stores the canonical lowercase + # form on the wire. + wire = Match(host="api.example.com", scheme=" HTTPS ").to_wire() + assert wire == {"host": "api.example.com", "scheme": "https"} + + +class TestMatchPortValidation: + def test_port_without_scheme_rejected(self): + with pytest.raises(ValueError, match="scheme"): + Match(host="api.example.com", port=8080) + + def test_port_out_of_range_low(self): + with pytest.raises(ValueError, match=r"\[1, 65535\]"): + Match(host="api.example.com", port=0, scheme="http") + + def test_port_out_of_range_high(self): + with pytest.raises(ValueError, match=r"\[1, 65535\]"): + Match(host="api.example.com", port=65536, scheme="http") + + def test_port_negative(self): + with pytest.raises(ValueError, match=r"\[1, 65535\]"): + Match(host="api.example.com", port=-1, scheme="http") + + def test_port_wrong_type_string(self): + with pytest.raises(ValueError, match="int"): + Match(host="api.example.com", port="8080", scheme="http") # type: ignore[arg-type] + + def test_port_wrong_type_bool_rejected(self): + # bool is a subclass of int in Python — must be filtered out + # explicitly, otherwise Match(port=True) would slip through as port=1. + with pytest.raises(ValueError, match="int"): + Match(host="api.example.com", port=True, scheme="http") # type: ignore[arg-type] + + @pytest.mark.parametrize("scheme", ["htps", "", 1]) + def test_invalid_scheme_rejected(self, scheme): + with pytest.raises(ValueError, match="http.*https"): + Match(host="api.example.com", scheme=scheme) # type: ignore[arg-type] + + +class TestNormalizeMatchDict: + def test_dict_pass_through(self): + out = _normalize_match_dict({"host": "foo", "port": 8080, "scheme": "http"}) + assert out == {"host": "foo", "port": 8080, "scheme": "http"} + + def test_dict_port_without_scheme_rejected(self): + with pytest.raises(ValueError, match="scheme"): + _normalize_match_dict({"host": "foo", "port": 8080}) + + def test_dict_port_out_of_range_rejected(self): + with pytest.raises(ValueError, match=r"\[1, 65535\]"): + _normalize_match_dict({"host": "foo", "port": 0, "scheme": "http"}) + + def test_dict_input_not_mutated(self): + # Regression guard: normalization returns a new dict so caller-owned + # data structures are not silently changed. + original = {"host": "foo"} + out = _normalize_match_dict(original) + assert out is not original + + def test_dict_scheme_normalized(self): + # Same normalize-then-validate semantics as the Match dataclass: + # mixed case and surrounding whitespace are accepted, and the wire + # form carries the canonical lowercase scheme. + out = _normalize_match_dict({"host": "foo", "port": 8443, "scheme": " HTTPS "}) + assert out["scheme"] == "https" + + @pytest.mark.parametrize("scheme", ["ftp", "", False]) + def test_dict_invalid_scheme_rejected(self, scheme): + with pytest.raises(ValueError, match="http.*https"): + _normalize_match_dict({"host": "foo", "scheme": scheme}) + + +class TestRuleSerializerHandlesPortScheme: + def test_rule_dataclass_carries_port(self): + rule = Rule( + name="api-inject", + match=Match(host="api.example.com", port=8443, scheme="https"), + action=Action(allow=True), + ) + wire = _serialize_rule(rule) + assert wire["match"]["port"] == 8443 + assert wire["match"]["scheme"] == "https" + + def test_rule_dict_carries_port(self): + wire = _serialize_rule({ + "name": "api-inject", + "match": {"host": "api.example.com", "port": 8443, "scheme": "https"}, + "action": {"allow": True}, + }) + assert wire["match"]["port"] == 8443 + + +class TestE2BPerHostRulesCompat: + def test_transform_still_works_without_port(self): + # E2B compat layer does not use port; generated rules should not carry + # port/scheme so they get the legacy default {80, 443} treatment. + rules = _convert_e2b_per_host_rules({ + "api.example.com": [ + {"transform": {"headers": {"Authorization": "Bearer x"}}} + ], + }) + assert len(rules) == 1 + assert "port" not in rules[0]["match"] + assert "scheme" not in rules[0]["match"] + + def test_empty_transform_list_rejected(self): + # An empty entries list fans out to zero rules and would silently + # drop the host the caller keyed in — must raise, not no-op. + with pytest.raises(ValueError, match="empty list"): + _convert_e2b_per_host_rules({"api.example.com": []}) diff --git a/tests/e2e/sdk_compat/cases/network/test_dns_allow.py b/tests/e2e/sdk_compat/cases/network/test_dns_allow.py index 392dd810d..c0f595345 100644 --- a/tests/e2e/sdk_compat/cases/network/test_dns_allow.py +++ b/tests/e2e/sdk_compat/cases/network/test_dns_allow.py @@ -31,18 +31,33 @@ ] -def _resolve_and_probe_command(host: str, port: int, timeout: int) -> str: - """Resolve *host*, print addrs, then TCP-probe the first IPv4 address.""" +def _resolve_and_probe_command(host: str, port: int, timeout: int, attempts: int = 3) -> str: + """Resolve *host* (retrying transient DNS failures), print addrs, then + TCP-probe the first IPv4 address. + + Only the resolution step is retried (temporary resolver failure / rate + limit); the TCP probe verdict (OK / FAIL) is a policy outcome and is never + retried, so a blocked domain still reports PROBE:FAIL immediately. + """ return ( "python3 - <<'PY'\n" - "import socket\n" + "import socket, time\n" f"host = {host!r}\n" f"port = {port!r}\n" f"timeout = {timeout!r}\n" - "try:\n" - " infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)\n" - "except Exception as exc:\n" - " print(f'RESOLVE:ERROR:{type(exc).__name__}:{exc}')\n" + f"attempts = {attempts!r}\n" + "infos = None\n" + "last = None\n" + "for attempt in range(attempts):\n" + " try:\n" + " infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)\n" + " break\n" + " except Exception as exc:\n" + " last = exc\n" + " if attempt + 1 < attempts:\n" + " time.sleep(min(2 ** attempt, 5))\n" + "if infos is None:\n" + " print(f'RESOLVE:ERROR:{type(last).__name__}:{last}')\n" " raise SystemExit(0)\n" "addrs = []\n" "for family, _, _, _, sockaddr in infos:\n" diff --git a/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py b/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py new file mode 100644 index 000000000..a9046adf9 --- /dev/null +++ b/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py @@ -0,0 +1,491 @@ +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 + +"""E2E coverage for network.rules with custom L7 ports (feat/custom_l7_port). + +The custom L7 port feature lets an egress rule pin the TCP port CubeEgress +intercepts on, via ``Match.port`` + ``Match.scheme``. This module exercises: + + 1. Custom HTTP port interception + credential injection — a rule with + ``(host, port, scheme=http)`` redirects a non-standard sandbox egress + port through CubeEgress and injects the rule's ``Inject`` header; the + upstream echo response reflects the marker back to the sandbox. + 2. Backward-compatible scheme-only rule — ``Match`` with ``scheme`` but no + ``port`` still intercepts the classic default set (``:80``/``:443``) and + injects on ``:80``. + 3. Create-time rejection of ``port`` without ``scheme`` (SDK ValueError). + 4. Create-time rejection of a subnet-CIDR ``host`` (server-side policy error). + +Topology notes +-------------- +* The custom-port leg runs a tiny HTTP echo server **in the test process** and + points the sandbox egress at ``SDK_E2E_L7_TARGET_HOST`` (a host IP reachable + from sandboxes, e.g. the bridge the sandbox pod attaches to). Set that env var + or the test is skipped — there is no portable default for "an IP the sandbox + can reach that is not the node IP". +* Plaintext HTTP legs are used on purpose: they need no TLS interception CA, so + they pass in any cluster that runs CubeEgress. HTTPS custom-port legs (which + require the sandbox image to trust the interception CA) are covered by + ``examples/code-sandbox-quickstart/network_l7_custom_port_echo.py``. +* All probing is done *inside* the sandbox via ``curl``; we grep the echoed + response for the injected marker secret. +""" + +from __future__ import annotations + +import json +import os +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + +import pytest + +from adapters import create_adapter +from framework.assertions import assert_command_ok +from framework.capabilities import NETWORK_L7_CUSTOM_PORT, capabilities_for_backend +from framework.cleanup import safe_kill +from framework.config import SdkE2EConfig + +# Host the sandbox can reach for the custom-port egress leg. Required; the test +# skips when unset (no safe portable default — see module docstring). +L7_TARGET_HOST = os.environ.get("SDK_E2E_L7_TARGET_HOST") +# Custom plaintext HTTP port the echo server listens on (+ the rule pins). +# 0 = let the OS assign a free port. A fixed default can collide with a +# leftover echo server from an interrupted run (EADDRINUSE); overriding via +# SDK_E2E_L7_CUSTOM_HTTP_PORT is still possible when a fixed port is required. +L7_CUSTOM_HTTP_PORT = int(os.environ.get("SDK_E2E_L7_CUSTOM_HTTP_PORT", "0")) +# Public host for the scheme-only default-set leg (echoes request headers). +L7_DEFAULT_HOST = os.environ.get("SDK_E2E_L7_DEFAULT_HOST", "httpbingo.org") +L7_MARKER_HEADER = "X-Cube-L7-E2E" +# Per-session unique secret so the grep cannot match ambient traffic. +L7_MARKER_SECRET = f"e2e-{uuid.uuid4().hex[:12]}" + +# HTTPS custom-port leg. TLS MITM means the sandbox image must trust the L7 +# interception CA (same prerequisite as the SDK quickstart example). There is no +# portable default for "a publicly-signed HTTPS endpoint on a non-standard +# port whose upstream cert CubeEgress can verify", so the URL is configurable +# and the whole leg is skipped unless the CA-trust prerequisite is declared. +L7_CUSTOM_HTTPS_URL = os.environ.get( + "SDK_E2E_L7_CUSTOM_HTTPS_URL", "https://tls-v1-2.badssl.com:1012/" +) +_l7_https = urlparse(L7_CUSTOM_HTTPS_URL) +L7_CUSTOM_HTTPS_HOST = _l7_https.hostname +L7_CUSTOM_HTTPS_PORT = _l7_https.port # None when the URL omits the port +L7_HTTPS_CAP_TRUSTED = os.environ.get("SDK_E2E_L7_HTTPS_CAP_TRUSTED", "").strip().lower() in { + "1", + "true", + "yes", + "on", +} +# Subject CN of the cluster's MITM interception CA (generated by +# deploy/one-click .../cube-egress-prepare.sh as "CubeSandbox Egress MITM CA"). +# An intercepted TLS connection is terminated by a leaf cert issued by this CA; +# a plain-SNAT passthrough would present the upstream's real publicly-issued +# cert instead. Asserting the leaf issuer proves interception actually happened. +L7_INTERCEPTION_ISSUER = os.environ.get( + "SDK_E2E_L7_INTERCEPTION_ISSUER", "CubeSandbox Egress MITM CA" +) + +pytestmark = [ + pytest.mark.e2e, + pytest.mark.sdk_compat, + pytest.mark.network, + pytest.mark.p1, +] + + +def _skip_without_capability(sdk_backend: str) -> None: + if NETWORK_L7_CUSTOM_PORT not in capabilities_for_backend(sdk_backend): + pytest.skip( + f"backend {sdk_backend!r} does not support {NETWORK_L7_CUSTOM_PORT}" + ) + + +class _HeaderEchoHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + payload = json.dumps( + {"path": self.path, "headers": dict(self.headers)} + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_args: object) -> None: + pass + + +def _detect_bridge_ip() -> str | None: + """Best-effort mirror of the example's detect_target_host() for local runs. + + Only used as a fallback when SDK_E2E_L7_TARGET_HOST is unset; the test still + skips unless the caller sets the env var (we never assume reachability). + """ + try: + out = ( + __import__("subprocess") + .check_output(["ip", "-4", "-o", "addr", "show", "up"], text=True) + ) + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 4 and parts[1].startswith(("docker", "br-")): + return parts[3].split("/")[0] + except Exception: + pass + return None + + +@pytest.fixture(scope="module") +def l7_echo_server(): + """Start the in-process HTTP echo server for the custom-port leg. + + Skips the dependant test when no reachable target host is configured, so we + never bind a port or run a sandbox in environments that can't route to it. + """ + if not L7_TARGET_HOST and not _detect_bridge_ip(): + pytest.skip( + "SDK_E2E_L7_TARGET_HOST is required for the custom L7 port egress " + "leg (a host IP reachable from sandboxes)" + ) + server = ThreadingHTTPServer(("0.0.0.0", L7_CUSTOM_HTTP_PORT), _HeaderEchoHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + # server.server_address[1] is the OS-assigned port when + # L7_CUSTOM_HTTP_PORT == 0 (the default). + yield L7_TARGET_HOST or _detect_bridge_ip(), server.server_address[1] + finally: + server.shutdown() + server.server_close() + + +def _l7_rules_custom_http(target_host: str, port: int) -> list[dict]: + return [ + { + "name": "e2e-custom-http", + "match": { + "host": target_host, + "port": port, + "scheme": "http", + }, + "action": { + "allow": True, + "inject": [{"header": L7_MARKER_HEADER, "secret": L7_MARKER_SECRET}], + }, + } + ] + + +def _l7_rules_scheme_only_default() -> list[dict]: + return [ + { + "name": "e2e-default-http", + "match": {"host": L7_DEFAULT_HOST, "scheme": "http"}, + "action": { + "allow": True, + "inject": [{"header": L7_MARKER_HEADER, "secret": L7_MARKER_SECRET}], + }, + } + ] + + +def _l7_rules_custom_https(host: str, port: int) -> list[dict]: + return [ + { + "name": "e2e-custom-https", + "match": {"host": host, "port": port, "scheme": "https"}, + "action": {"allow": True}, + } + ] + + +@pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) +def test_l7_custom_port_injects_marker_on_custom_http_port( + sdk_backend: str, + sdk_e2e_config: SdkE2EConfig, + l7_echo_server: tuple[str, int], +): + """A (host, port, scheme=http) rule intercepts a non-standard egress port + and injects the rule's credential header; the echo reflects it back.""" + _skip_without_capability(sdk_backend) + target_host, http_port = l7_echo_server + + adapter = None + try: + adapter = create_adapter( + sdk_backend, + sdk_e2e_config, + metadata={ + "test_suite": "sdk_compat", + "test_backend": sdk_backend, + "test_case": "l7_custom_port_injects_marker", + }, + create_options={ + "allow_internet_access": False, + "network": {"rules": _l7_rules_custom_http(target_host, http_port)}, + }, + ) + + url = f"http://{target_host}:{http_port}/headers" + result = adapter.run_command( + f"curl -sS --max-time 20 '{url}'", + timeout=sdk_e2e_config.command_timeout, + ) + assert_command_ok(result) + assert L7_MARKER_SECRET in result.stdout, ( + f"custom-port L7 rule should inject {L7_MARKER_HEADER!r} at {url}; " + f"response={result.stdout[:400]!r} stderr={result.stderr[:200]!r}" + ) + finally: + if adapter is not None: + safe_kill(adapter, sdk_e2e_config) + + +@pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) +@pytest.mark.requires_internet +def test_l7_custom_port_scheme_only_intercepts_default_http_set( + sdk_backend: str, + sdk_e2e_config: SdkE2EConfig, +): + """A scheme-only rule (no port) keeps the classic :80 default behavior and + still injects on the default HTTP port — backward compatibility.""" + _skip_without_capability(sdk_backend) + + adapter = None + try: + adapter = create_adapter( + sdk_backend, + sdk_e2e_config, + metadata={ + "test_suite": "sdk_compat", + "test_backend": sdk_backend, + "test_case": "l7_custom_port_scheme_only_default", + }, + create_options={ + "allow_internet_access": False, + "network": {"rules": _l7_rules_scheme_only_default()}, + }, + ) + + url = f"http://{L7_DEFAULT_HOST}/headers" + result = adapter.run_command( + # --retry absorbs transient failures (timeout / connreset / + # temporary DNS) against the public echo endpoint. + f"curl -sS --max-time 20 --retry 3 --retry-delay 1 --retry-all-errors '{url}'", + timeout=sdk_e2e_config.command_timeout, + ) + assert_command_ok(result) + assert L7_MARKER_SECRET in result.stdout, ( + f"scheme-only L7 rule should intercept :80 and inject " + f"{L7_MARKER_HEADER!r} at {url}; response={result.stdout[:400]!r} " + f"stderr={result.stderr[:200]!r}" + ) + finally: + if adapter is not None: + safe_kill(adapter, sdk_e2e_config) + + +@pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) +def test_l7_custom_port_rejects_port_without_scheme( + sdk_backend: str, + sdk_e2e_config: SdkE2EConfig, +): + """Match.port requires Match.scheme — the SDK raises ValueError before any + network round-trip (mirrors Match.__post_init__ / _normalize_match_dict).""" + _skip_without_capability(sdk_backend) + + adapter = None + try: + with pytest.raises(Exception) as exc_info: + adapter = create_adapter( + sdk_backend, + sdk_e2e_config, + metadata={ + "test_suite": "sdk_compat", + "test_backend": sdk_backend, + "test_case": "l7_custom_port_port_without_scheme", + }, + create_options={ + "network": { + "rules": [ + { + "name": "bad-port-no-scheme", + "match": {"host": "198.51.100.7", "port": 1234}, + "action": {"allow": True}, + } + ] + } + }, + ) + message = str(exc_info.value).lower() + assert "port" in message and "scheme" in message, ( + f"create failure should mention the port/scheme pairing constraint; " + f"got={exc_info.value!r}" + ) + finally: + if adapter is not None: + safe_kill(adapter, sdk_e2e_config) + + +@pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) +def test_l7_custom_port_rejects_subnet_host( + sdk_backend: str, + sdk_e2e_config: SdkE2EConfig, +): + """An L7 rule host must be a single host IP or a domain — a subnet CIDR is + rejected server-side (cubevs netpolicy: subnet CIDR not supported for L7).""" + _skip_without_capability(sdk_backend) + + adapter = None + try: + with pytest.raises(Exception) as exc_info: + adapter = create_adapter( + sdk_backend, + sdk_e2e_config, + metadata={ + "test_suite": "sdk_compat", + "test_backend": sdk_backend, + "test_case": "l7_custom_port_subnet_host", + }, + create_options={ + "network": { + "rules": [ + { + "name": "bad-subnet-host", + "match": { + "host": "10.0.0.0/24", + "scheme": "http", + }, + "action": {"allow": True}, + } + ] + } + }, + ) + message = str(exc_info.value).lower() + assert "subnet" in message or "cidr" in message or "host" in message, ( + f"create failure should mention the subnet/host constraint; " + f"got={exc_info.value!r}" + ) + finally: + if adapter is not None: + safe_kill(adapter, sdk_e2e_config) + + +def _fetch_leaf_issuer(adapter, sdk_e2e_config, attempts: int = 3) -> str: + """Return the issuer of the TLS leaf cert the sandbox is presented with. + + Uses ``openssl s_client`` (no verification, so the handshake completes even + for the interception CA) and extracts the leaf issuer via ``openssl x509``. + An intercepted connection shows the cluster interception CA; a plain-SNAT + passthrough shows the upstream's real public CA. + + Retries when the handshake fails to produce a cert (transient connect/timeout + against the public endpoint), so a slow upstream does not flake the leg. + """ + cmd = ( + f"openssl s_client -connect {L7_CUSTOM_HTTPS_HOST}:{L7_CUSTOM_HTTPS_PORT} " + f"-servername {L7_CUSTOM_HTTPS_HOST} /dev/null " + f"| openssl x509 -noout -issuer" + ) + out = "" + for attempt in range(attempts): + result = adapter.run_command(cmd, timeout=sdk_e2e_config.command_timeout) + assert_command_ok(result) + out = result.stdout.strip() + if out: + return out + if attempt + 1 < attempts: + time.sleep(min(2 ** attempt, 5)) + return out + + +@pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) +@pytest.mark.requires_internet +def test_l7_custom_port_https_intercepts_custom_port( + sdk_backend: str, + sdk_e2e_config: SdkE2EConfig, +): + """A (host, port, scheme=https) rule intercepts a non-standard HTTPS egress + port: CubeEgress does TLS MITM and forwards to the upstream. Proves the + port-scoped scheme=https interception path on a non-443 port. + + Interception is proven by asserting the TLS leaf issuer: an intercepted + connection is terminated by CubeEgress's MITM cert, issued by the cluster + interception CA (L7_INTERCEPTION_ISSUER), whereas a plain-SNAT passthrough + would present the upstream's real publicly-issued cert. This distinguishes + interception from mere reachability — a bare 200/body check could not. + """ + _skip_without_capability(sdk_backend) + if not L7_HTTPS_CAP_TRUSTED: + pytest.skip( + "HTTPS L7 interception needs the sandbox image to trust the L7 " + "interception CA; set SDK_E2E_L7_HTTPS_CAP_TRUSTED=1 to enable " + "(see examples/code-sandbox-quickstart/network_l7_custom_port_echo.py)" + ) + if not L7_CUSTOM_HTTPS_HOST or not L7_CUSTOM_HTTPS_PORT: + pytest.skip( + f"SDK_E2E_L7_CUSTOM_HTTPS_URL must include an explicit port; " + f"got {L7_CUSTOM_HTTPS_URL!r}" + ) + if L7_CUSTOM_HTTPS_PORT == 443: + pytest.skip( + "HTTPS custom-port leg exercises a non-standard port; " + f"{L7_CUSTOM_HTTPS_URL!r} resolves to :443 (use a non-443 port)" + ) + + adapter = None + try: + adapter = create_adapter( + sdk_backend, + sdk_e2e_config, + metadata={ + "test_suite": "sdk_compat", + "test_backend": sdk_backend, + "test_case": "l7_custom_port_https_custom_port", + }, + create_options={ + "allow_internet_access": False, + "network": { + "rules": _l7_rules_custom_https( + L7_CUSTOM_HTTPS_HOST, L7_CUSTOM_HTTPS_PORT + ) + }, + }, + ) + + result = adapter.run_command( + f"curl -sS --max-time 20 --retry 3 --retry-delay 1 --retry-all-errors -o /dev/null " + f"-w 'code=%{{http_code}} len=%{{size_download}}' " + f"'{L7_CUSTOM_HTTPS_URL}'", + timeout=sdk_e2e_config.command_timeout, + ) + assert_command_ok(result) + out = result.stdout.strip() + assert "code=200" in out, ( + f"custom-port HTTPS L7 rule should proxy {L7_CUSTOM_HTTPS_URL} to " + f"HTTP 200; curl output={out!r} stderr={result.stderr[:200]!r}" + ) + assert "len=0" not in out, ( + f"custom-port HTTPS L7 rule proxied {L7_CUSTOM_HTTPS_URL} but the " + f"upstream returned an empty body; curl output={out!r}" + ) + + # The 200/body check above only proves reachability. Assert the TLS + # leaf was issued by the interception CA to prove the connection was + # actually MITM-intercepted (not plain-SNAT passthrough, which would + # present the upstream's real public CA). + issuer = _fetch_leaf_issuer(adapter, sdk_e2e_config) + assert L7_INTERCEPTION_ISSUER in issuer, ( + f"expected interception CA {L7_INTERCEPTION_ISSUER!r} as the TLS " + f"leaf issuer (proves MITM interception); got issuer={issuer!r}. " + f"A public CA here means the custom-port HTTPS flow bypassed " + f"CubeEgress (plain SNAT passthrough)." + ) + finally: + if adapter is not None: + safe_kill(adapter, sdk_e2e_config) diff --git a/tests/e2e/sdk_compat/cases/network/test_l7_egress.py b/tests/e2e/sdk_compat/cases/network/test_l7_egress.py index b565f060b..bd72a0c67 100644 --- a/tests/e2e/sdk_compat/cases/network/test_l7_egress.py +++ b/tests/e2e/sdk_compat/cases/network/test_l7_egress.py @@ -18,9 +18,12 @@ OTHER_HOST = os.environ.get("SDK_E2E_L7_OTHER_HOST", "example.com") INJECT_HEADER = os.environ.get("SDK_E2E_L7_INJECT_HEADER", "X-Cube-E2E-Inject") INJECT_SECRET = os.environ.get("SDK_E2E_L7_INJECT_SECRET", "e2e-inject-secret-not-a-real-key") -# CubeEgress MITM CA CN varies by install path (one-click prepare vs chart bake). -# Match a stable substring; override with SDK_E2E_L7_MITM_CA_CN when needed. -MITM_CA_CN = os.environ.get("SDK_E2E_L7_MITM_CA_CN", "Cube Sandbox Egress CA") +# CubeEgress MITM CA CN. The authoritative value (one-click prepare and the +# k8s chart, deploy/.../cube-egress-prepare.sh + chart values caCommonName) is +# "CubeSandbox Egress MITM CA". Override with SDK_E2E_L7_MITM_CA_CN if a +# deployment customized it. A regex fallback in the test also tolerates +# "Cube Sandbox Egress"-style spacing variants. +MITM_CA_CN = os.environ.get("SDK_E2E_L7_MITM_CA_CN", "CubeSandbox Egress MITM CA") # Cold MITM path (DNS learn + TLS + upstream) needs more headroom than the # shared TCP network_probe_timeout default (5s). L7_HTTP_TIMEOUT = int(os.environ.get("SDK_E2E_L7_HTTP_TIMEOUT", "15")) @@ -55,27 +58,41 @@ def _https_rule( return {"name": name, "match": match, "action": action} -def _http_json_command(url: str, *, method: str = "GET", timeout: int = 15) -> str: - """Fetch URL without TLS verify; print STATUS + body (or error).""" +def _http_json_command(url: str, *, method: str = "GET", timeout: int = 15, attempts: int = 3) -> str: + """Fetch URL without TLS verify; print STATUS + body (or error). + + Retries transient transport failures (timeout / connection reset / temporary + DNS) with backoff, since the public echo endpoints can be slow or rate-limit. + An HTTP 4xx/5xx is a real policy verdict and is returned as-is, never retried. + """ return ( "python3 - <<'PY'\n" - "import json, ssl, urllib.error, urllib.request\n" + "import ssl, time, urllib.error, urllib.request\n" f"url = {url!r}\n" f"method = {method!r}\n" f"timeout = {timeout!r}\n" + f"attempts = {attempts!r}\n" "ctx = ssl._create_unverified_context()\n" - "req = urllib.request.Request(url, method=method)\n" - "try:\n" - " with urllib.request.urlopen(req, context=ctx, timeout=timeout) as resp:\n" - " body = resp.read().decode('utf-8', errors='replace')\n" - " print(f'STATUS:{resp.status}')\n" + "last = None\n" + "for attempt in range(attempts):\n" + " try:\n" + " req = urllib.request.Request(url, method=method)\n" + " with urllib.request.urlopen(req, context=ctx, timeout=timeout) as resp:\n" + " body = resp.read().decode('utf-8', errors='replace')\n" + " print(f'STATUS:{resp.status}')\n" + " print(body)\n" + " break\n" + " except urllib.error.HTTPError as exc:\n" + " body = exc.read().decode('utf-8', errors='replace')\n" + " print(f'STATUS:{exc.code}')\n" " print(body)\n" - "except urllib.error.HTTPError as exc:\n" - " body = exc.read().decode('utf-8', errors='replace')\n" - " print(f'STATUS:{exc.code}')\n" - " print(body)\n" - "except Exception as exc:\n" - " print(f'ERROR:{type(exc).__name__}:{exc}')\n" + " break\n" + " except Exception as exc:\n" + " last = exc\n" + " if attempt + 1 < attempts:\n" + " time.sleep(min(2 ** attempt, 5))\n" + "else:\n" + " print(f'ERROR:{type(last).__name__}:{last}')\n" "PY" ) @@ -118,23 +135,35 @@ def _header_ci(headers: dict[str, str], name: str) -> str | None: return None -def _tls_issuer_command(host: str, timeout: int = 15) -> str: +def _tls_issuer_command(host: str, timeout: int = 15, attempts: int = 3) -> str: return ( "python3 - <<'PY'\n" - "import shutil, socket, ssl, subprocess, tempfile\n" + "import shutil, socket, ssl, subprocess, tempfile, time\n" f"host = {host!r}\n" f"timeout = {timeout!r}\n" + f"attempts = {attempts!r}\n" "if shutil.which('openssl') is None:\n" " print('ISSUER:ERROR:openssl_not_found')\n" " raise SystemExit(0)\n" "ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)\n" "ctx.check_hostname = False\n" "ctx.verify_mode = ssl.CERT_NONE\n" - "with socket.create_connection((host, 443), timeout=timeout) as sock:\n" - " with ctx.wrap_socket(sock, server_hostname=host) as ssock:\n" - " der = ssock.getpeercert(binary_form=True)\n" + "der = None\n" + "last = None\n" + # Retry transient connect/handshake failures (timeout / temp DNS) with + # backoff; the public echo endpoint can be slow or rate-limit. + "for attempt in range(attempts):\n" + " try:\n" + " with socket.create_connection((host, 443), timeout=timeout) as sock:\n" + " with ctx.wrap_socket(sock, server_hostname=host) as ssock:\n" + " der = ssock.getpeercert(binary_form=True)\n" + " break\n" + " except Exception as exc:\n" + " last = exc\n" + " if attempt + 1 < attempts:\n" + " time.sleep(min(2 ** attempt, 5))\n" "if not der:\n" - " print('ISSUER:ERROR:empty_peer_cert')\n" + " print(f'ISSUER:ERROR:connect_failed:{type(last).__name__}:{last}')\n" " raise SystemExit(0)\n" "try:\n" " with tempfile.NamedTemporaryFile() as tmp:\n" diff --git a/tests/e2e/sdk_compat/framework/capabilities.py b/tests/e2e/sdk_compat/framework/capabilities.py index d6e4c0492..64999fc26 100644 --- a/tests/e2e/sdk_compat/framework/capabilities.py +++ b/tests/e2e/sdk_compat/framework/capabilities.py @@ -12,6 +12,7 @@ NETWORK_ALLOW_DENY = "network_allow_deny" NETWORK_PUBLIC_ACCESS = "network_public_access" NETWORK_MASK_REQUEST_HOST = "network_mask_request_host" +NETWORK_L7_CUSTOM_PORT = "network_l7_custom_port" # CubeVS domain allow_out + DNS A learning (exact / leading "*."). NETWORK_DNS_ALLOW = "network_dns_allow" # Built-in deny of sandbox-private / link-local CIDRs when public egress is on. @@ -54,6 +55,7 @@ HOST_MOUNT, VOLUME_PLUGIN, AUTH_SIMPLE_KEY, + NETWORK_L7_CUSTOM_PORT, } ) diff --git a/tests/unittest/run.sh b/tests/unittest/run.sh index 97a868c1b..4bcc118ed 100755 --- a/tests/unittest/run.sh +++ b/tests/unittest/run.sh @@ -88,6 +88,12 @@ WITH_TESTS=( # and need the generated CubeNet/cubevs code but no cubecow/CGO, so build cubevs # then run just the runtime package rather than the whole Cubelet suite. "cubelet-network|Go|0|make builder-run BUILDER_CMD='cd /workspace/CubeNet/cubevs && make gen && cd /workspace/Cubelet && go mod download && go test ./network/runtime/...'" + # cubevs: the CubeNet/cubevs module's OWN unit tests (dataplane policy, DNS + # learning, migration, dump, classify). cubelet-network builds cubevs's + # generated code but runs Cubelet's tests, so these never ran. cubevs-test + # regenerates the BPF objects (make gen) and runs the full module set in a + # privileged root builder (eBPF load + bpffs mount need the privilege). + "cubevs|Go|0|make cubevs-test" "cubecow|Go+CGO|0|make cubecow-test-native" "cube-lifecycle-manager|Go|0|make builder-run BUILDER_CMD='cd /workspace/cube-lifecycle-manager && go mod download && go test ./...'" "cube-api|Rust|0|make cube-api-test"