From ce88facb1ad247902efabc02724d46c43c5ac30a Mon Sep 17 00:00:00 2001 From: hankzhou Date: Sun, 9 Aug 2026 21:31:51 +0800 Subject: [PATCH 01/31] feat(cubenet): port L7 custom-port dataplane and spec tests Port the L7 custom-port feature dataplane from L7_support_portconfig (gitwoa repo) onto the restructured master (#1285): - V2 -> V3 allow_out migration: net_policy_value_v3 with port in the LPM key, key_prefixlen exact-key merge, static-wins expiry semantics - dns_allow -> dns_allow_v2 rename with legacy/current migration and crash-safe rollback - L7 (host, port, scheme) plan builder: canonical host grouping, per-port scheme conflict detection, /48 entry budget accounting - BPF dataplane: mvmtap/session/tcp/dns_response/dns_query updates, legacy default-port drain shim, L7 mark configurability - Spec tests: DNS learn (kernel BPF), migration rollback over bpffs, buildL7Plan matrix, static-wins merge matrix (100 pass, 0 fail) Reconciliation with #1285: keep the restructured lifecycle split (CleanupTAPDevicePolicy / InstallTAPDefaultDenyPolicy / cleanupDNSPolicyFlags, UpsertTAPDeviceMetadata / DeleteTAPDevice) and re-apply the branch changes on top. Note: Cubelet/network/runtime does not compile at this commit because MVMOptions.L7AllowOut changed from *[]string to *[]L7Target; fixed in the follow-up Cubelet runtime port. Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- CubeNet/cubevs/.gitignore | 4 +- CubeNet/cubevs/cubevs.go | 115 ++++- CubeNet/cubevs/custom_port_test.go | 134 +++++ CubeNet/cubevs/dns_learn_test.go | 355 +++++++++++++ CubeNet/cubevs/dns_reaper.go | 18 +- CubeNet/cubevs/dnspolicy.go | 78 +-- CubeNet/cubevs/dump.go | 127 ++++- CubeNet/cubevs/dump_test.go | 37 +- CubeNet/cubevs/egress_policy_test.go | 276 +++++++++++ CubeNet/cubevs/l7_mark_test.go | 117 +++++ CubeNet/cubevs/map.go | 5 + CubeNet/cubevs/migration.go | 307 ++++++++++-- CubeNet/cubevs/migration_test.go | 458 +++++++++++++++++ CubeNet/cubevs/miscs.go | 92 +++- CubeNet/cubevs/netpolicy.go | 542 +++++++++++++++++--- CubeNet/cubevs/netpolicy_test.go | 716 ++++++++++++++++++++++++--- CubeNet/cubevs/reaper.go | 12 +- CubeNet/cubevs/tap.go | 17 +- CubeNet/cubevs/tcp_state_test.go | 287 +++++++++++ CubeNet/src/cubevs.h | 115 ++++- CubeNet/src/dns_learn_test.bpf.c | 47 ++ CubeNet/src/dns_query.h | 34 +- CubeNet/src/dns_response.h | 109 +++- CubeNet/src/egress_policy_test.bpf.c | 33 ++ CubeNet/src/icmp.h | 2 +- CubeNet/src/l7_mark_test.bpf.c | 32 ++ CubeNet/src/localgw.bpf.c | 36 ++ CubeNet/src/map.h | 25 +- CubeNet/src/mvmtap.bpf.c | 226 ++++++--- CubeNet/src/nodenic.bpf.c | 20 +- CubeNet/src/session.h | 156 ++++-- CubeNet/src/tcp.h | 40 +- CubeNet/src/tcp_state_test.bpf.c | 84 ++++ CubeNet/src/udp.h | 2 +- 34 files changed, 4230 insertions(+), 428 deletions(-) create mode 100644 CubeNet/cubevs/custom_port_test.go create mode 100644 CubeNet/cubevs/dns_learn_test.go create mode 100644 CubeNet/cubevs/egress_policy_test.go create mode 100644 CubeNet/cubevs/l7_mark_test.go create mode 100644 CubeNet/cubevs/migration_test.go create mode 100644 CubeNet/cubevs/tcp_state_test.go create mode 100644 CubeNet/src/dns_learn_test.bpf.c create mode 100644 CubeNet/src/egress_policy_test.bpf.c create mode 100644 CubeNet/src/l7_mark_test.bpf.c create mode 100644 CubeNet/src/tcp_state_test.bpf.c diff --git a/CubeNet/cubevs/.gitignore b/CubeNet/cubevs/.gitignore index 4f4cb290c..715f524f3 100644 --- a/CubeNet/cubevs/.gitignore +++ b/CubeNet/cubevs/.gitignore @@ -1,4 +1,4 @@ .coverage coverage.html -/*_bpfel.go -/*_bpfel.o +/*_bpfel*.go +/*_bpfel*.o diff --git a/CubeNet/cubevs/cubevs.go b/CubeNet/cubevs/cubevs.go index b131af6ae..ad53ff32c 100644 --- a/CubeNet/cubevs/cubevs.go +++ b/CubeNet/cubevs/cubevs.go @@ -37,6 +37,12 @@ type Params struct { NodeMacAddr net.HardwareAddr // MAC address of the Node gateway (next hop) NodeGatewayMacAddr net.HardwareAddr + // L7 skb->mark values stamped by the dataplane and matched by the iptables + // TPROXY rules. Zero means "use the shipped default"; override from the + // install-time config shared with the iptables init script. + L7MarkHTTP uint32 + L7MarkHTTPS uint32 + L7MarkMask uint32 } // TAPDevice contains info about a TAP device. @@ -81,24 +87,64 @@ type lpmKey struct { IP uint32 } -// netPolicyValueV2 mirrors struct net_policy_value_v2 on the BPF side. +// l7PortEntry mirrors struct l7_port_entry on the BPF side. Port is stored in +// network byte order to match tcphdr->dest so the datapath can compare without +// an endianness conversion. Scheme is one of L7SchemeHTTP / L7SchemeHTTPS. +type l7PortEntry struct { + Port uint16 // network byte order + Scheme uint8 + Pad uint8 +} + +// netPolicyValueV2 mirrors struct net_policy_value_v2 on the BPF side. This is +// the legacy 16-byte layout, read only when migrating a pre-v3 allow_out_v2 +// map to allow_out_v3; the current dataplane uses netPolicyValueV3. type netPolicyValueV2 struct { ExpiresAtNS uint64 Flags uint8 Reserved [7]uint8 } +// lpmKeyV3 mirrors struct lpm_key_v3 on the BPF side. IP and port are +// in network byte order; a single longest-prefix lookup resolves exact +// (ip, port) (prefixlen 48), ip-only (prefixlen 32), or ip/mask +// (prefixlen < 32) rules. Pad keeps the LPM data payload 4-byte aligned. +type lpmKeyV3 struct { + Prefixlen uint32 + IP uint32 + Port uint16 + Pad uint16 +} + +// netPolicyValueV3 mirrors struct net_policy_value_v3 on the BPF side. +// Unlike netPolicyValueV2, the port lives in the key, so the scheme is +// resolved at insert time and stored directly here. KeyPrefixlen records +// the prefixlen of the key this value was written under: LPM lookups are +// longest-prefix, so writers merging with an existing entry for the EXACT +// same key must compare it against their key's prefixlen first. +type netPolicyValueV3 struct { + ExpiresAtNS uint64 + Flags uint8 + Scheme uint8 + KeyPrefixlen uint8 + Reserved [5]uint8 +} + // dnsAllowKey mirrors struct dns_allow_key on the BPF side. type dnsAllowKey struct { Prefixlen uint32 Name [maxDNSNameLen]byte } -// dnsAllowValue mirrors struct dns_allow_value on the BPF side. +// dnsAllowValue mirrors struct dns_allow_value on the BPF side. Ports carries +// the (port, scheme) tuples the userspace built from all rules sharing this +// host. PortCount == 0 means "unspecified, default 80/443". type dnsAllowValue struct { - NameLen uint32 - Flags uint8 - Reserved [3]uint8 + NameLen uint32 + Flags uint8 + PortCount uint8 + Reserved [2]uint8 + Ports [maxL7PortsPerHost]l7PortEntry } // dnsQueryTrackKey mirrors struct dns_query_track_key on the BPF side. @@ -112,10 +158,14 @@ type dnsQueryTrackKey struct { } // dnsQueryTrackValue mirrors struct dns_query_track_value on the BPF side. +// Ports is copied from the matched dns_allow_value at query time so the +// response handler can rebuild net_policy_value_v3 without a second lookup. type dnsQueryTrackValue struct { ExpiresAtNS uint64 Flags uint8 - Reserved [7]uint8 + PortCount uint8 + Reserved [6]uint8 + Ports [maxL7PortsPerHost]l7PortEntry } const ( @@ -128,6 +178,14 @@ const ( dnsPolicyFlagLearningEnabled = 1 << 0 // Network policy flags. Must match src/cubevs.h. netPolicyFlagL7Required = 1 << 0 + // L7 scheme values in dns_allow_value / net_policy_value_v3 per-port + // entries. Must match L7_SCHEME_* in src/cubevs.h. + L7SchemeNone uint8 = 0 + L7SchemeHTTP uint8 = 1 + L7SchemeHTTPS uint8 = 2 + // Maximum number of (port, scheme) tuples per host. Must match + // MAX_L7_PORTS_PER_HOST in src/cubevs.h. + maxL7PortsPerHost = 8 // Network policy value marker. Must match src/cubevs.h. netPolicyValueStatic = 1 // programs that power CubeVS. @@ -155,9 +213,11 @@ const ( MapNameLocalPortMapping = "local_port_mapping" // MapNameAllowOut is the cube-v0.2.0 legacy migration source. MapNameAllowOut = "allow_out" - MapNameAllowOutV2 = "allow_out_v2" + MapNameAllowOutV2 = "allow_out_v2" // legacy 16-byte policy value + MapNameAllowOutV3 = "allow_out_v3" // current 16-byte policy value MapNameDenyOut = "deny_out" - MapNameDNSAllow = "dns_allow" + MapNameDNSAllow = "dns_allow" // legacy 8-byte DNS value + MapNameDNSAllowV2 = "dns_allow_v2" // current 40-byte DNS value MapNameDNSQueryTrack = "dns_query_track" // constants referenced by BPF programs. globalNameMVMInnerIP = "mvm_inner_ip" @@ -179,8 +239,9 @@ const ( globalNameNodeMacaddrP2 = "nodenic_macaddr_p2" globalNameNodeGatewayMacaddrP1 = "nodegw_macaddr_p1" globalNameNodeGatewayMacaddrP2 = "nodegw_macaddr_p2" - // for bpffs. - bpfFSPath = "/sys/fs/bpf" + globalNameCubeL7MarkHTTP = "cube_l7_mark_http" + globalNameCubeL7MarkHTTPS = "cube_l7_mark_https" + globalNameCubeL7MarkMask = "cube_l7_mark_mask" // for TC. tcFlagDirectAction = 1 tcFilterHandle = 1 @@ -264,6 +325,24 @@ func _() { _ = arr[size-8] // error if size < 8 } + { + // static assert, make sure LpmKeyV3 is of size 12 + var arr [12]struct{} + var obj lpmKeyV3 + const size = unsafe.Sizeof(obj) + _ = arr[size-1] // error if size > 12 + _ = arr[size-12] // error if size < 12 + } + + { + // static assert, make sure l7PortEntry is of size 4 + var arr [4]struct{} + var obj l7PortEntry + const size = unsafe.Sizeof(obj) + _ = arr[size-1] // error if size > 4 + _ = arr[size-4] // error if size < 4 + } + { // static assert, make sure netPolicyValueV2 is of size 16 var arr [16]struct{} @@ -283,12 +362,12 @@ func _() { } { - // static assert, make sure dnsAllowValue is of size 8 - var arr [8]struct{} + // static assert, make sure dnsAllowValue is of size 40 + var arr [40]struct{} var obj dnsAllowValue const size = unsafe.Sizeof(obj) - _ = arr[size-1] // error if size > 8 - _ = arr[size-8] // error if size < 8 + _ = arr[size-1] // error if size > 40 + _ = arr[size-40] // error if size < 40 } { @@ -301,11 +380,11 @@ func _() { } { - // static assert, make sure dnsQueryTrackValue is of size 16 - var arr [16]struct{} + // static assert, make sure dnsQueryTrackValue is of size 48 + var arr [48]struct{} var obj dnsQueryTrackValue const size = unsafe.Sizeof(obj) - _ = arr[size-1] // error if size > 16 - _ = arr[size-16] // error if size < 16 + _ = arr[size-1] // error if size > 48 + _ = arr[size-48] // error if size < 48 } } diff --git a/CubeNet/cubevs/custom_port_test.go b/CubeNet/cubevs/custom_port_test.go new file mode 100644 index 000000000..9747d412e --- /dev/null +++ b/CubeNet/cubevs/custom_port_test.go @@ -0,0 +1,134 @@ +package cubevs + +import "testing" + +// ------- v3 custom-port encoding (buildV3Entries) ------------------- + +func TestExpandDefaultPortSet(t *testing.T) { + // The default (host, port) set used when an L7 rule omits port. + ports := expandDefaultPortSet() + if got, want := len(ports), 2; got != want { + t.Fatalf("len(expandDefaultPortSet())=%d, want %d", got, want) + } + want := map[uint16]uint8{ + htonsPort(80): L7SchemeHTTP, + htonsPort(443): L7SchemeHTTPS, + } + for _, p := range ports { + scheme, ok := want[p.Port] + if !ok { + t.Fatalf("unexpected default port 0x%04x", p.Port) + } + if p.Scheme != scheme { + t.Fatalf("port 0x%04x scheme=%d, want %d", p.Port, p.Scheme, scheme) + } + } +} + +func TestBuildV3EntriesL7ExplicitPorts(t *testing.T) { + // L7 with explicit (port, scheme) tuples -> one /48 entry per tuple, + // each carrying scheme + expiry, ip in network byte order. + const ip uint32 = 0x01020304 + const expires uint64 = 1234 + entries := buildV3Entries(lpmKey{Prefixlen: 32, IP: ip}, netPolicyFlagL7Required, + []l7PortEntry{ + {Port: htonsPort(8080), Scheme: L7SchemeHTTP}, + {Port: htonsPort(8443), Scheme: L7SchemeHTTPS}, + }, expires) + if got, want := len(entries), 2; got != want { + t.Fatalf("len(entries)=%d, want %d", got, want) + } + got := map[uint16]uint8{} + for _, e := range entries { + if e.key.Prefixlen != 48 { + t.Fatalf("key.Prefixlen=%d, want 48 (exact ip+port)", e.key.Prefixlen) + } + if e.key.IP != ip { + t.Fatalf("key.IP=0x%08x, want 0x%08x", e.key.IP, ip) + } + if e.value.Flags != netPolicyFlagL7Required { + t.Fatalf("value.Flags=%d, want %d", e.value.Flags, netPolicyFlagL7Required) + } + if e.value.ExpiresAtNS != expires { + t.Fatalf("value.ExpiresAtNS=%d, want %d", e.value.ExpiresAtNS, expires) + } + got[e.key.Port] = e.value.Scheme + } + if got[htonsPort(8080)] != L7SchemeHTTP { + t.Fatalf("port 8080 scheme=%d, want HTTP", got[htonsPort(8080)]) + } + if got[htonsPort(8443)] != L7SchemeHTTPS { + t.Fatalf("port 8443 scheme=%d, want HTTPS", got[htonsPort(8443)]) + } +} + +func TestBuildV3EntriesL7DefaultExpansion(t *testing.T) { + // L7 with no port set -> default {80/http, 443/https} expansion, + // both as /48 exact entries. + entries := buildV3Entries(lpmKey{Prefixlen: 32, IP: 0x01020304}, netPolicyFlagL7Required, nil, 0) + if got, want := len(entries), 2; got != want { + t.Fatalf("len(entries)=%d, want 2 (default {80,443})", got) + } + for _, e := range entries { + if e.key.Prefixlen != 48 { + t.Fatalf("key.Prefixlen=%d, want 48", e.key.Prefixlen) + } + } +} + +func TestBuildV3EntriesPlainAllow(t *testing.T) { + // Non-L7 allow -> single ip-only /32 entry, port=0, scheme=NONE. + entries := buildV3Entries(lpmKey{Prefixlen: 32, IP: 0x01020304}, 0, nil, 0) + if got, want := len(entries), 1; got != want { + t.Fatalf("len(entries)=%d, want 1", got) + } + e := entries[0] + if e.key.Prefixlen != 32 { + t.Fatalf("key.Prefixlen=%d, want 32 (ip-only)", e.key.Prefixlen) + } + if e.key.Port != 0 { + t.Fatalf("key.Port=%d, want 0 (non-L7)", e.key.Port) + } + if e.value.Scheme != L7SchemeNone { + t.Fatalf("value.Scheme=%d, want NONE", e.value.Scheme) + } +} + +func TestBuildV3EntriesSubnet(t *testing.T) { + // Non-L7 subnet -> single entry at the source prefixlen (< 32). + entries := buildV3Entries(lpmKey{Prefixlen: 24, IP: 0x01020304}, 0, nil, 0) + if got, want := len(entries), 1; got != want { + t.Fatalf("len(entries)=%d, want 1", got) + } + if entries[0].key.Prefixlen != 24 { + t.Fatalf("key.Prefixlen=%d, want 24 (subnet)", entries[0].key.Prefixlen) + } + if entries[0].key.Port != 0 { + t.Fatalf("key.Port=%d, want 0", entries[0].key.Port) + } +} + +func TestBuildV3EntriesExpiryCopied(t *testing.T) { + // Expiry is carried verbatim into every expanded entry. + entries := buildV3Entries(lpmKey{Prefixlen: 32, IP: 1}, netPolicyFlagL7Required, + []l7PortEntry{{Port: htonsPort(8080), Scheme: L7SchemeHTTP}}, 999) + for _, e := range entries { + if e.value.ExpiresAtNS != 999 { + t.Fatalf("value.ExpiresAtNS=%d, want 999", e.value.ExpiresAtNS) + } + } +} + +func TestLpmKeyV3NetworkByteOrder(t *testing.T) { + // A custom port must be stored in network byte order in the LPM key, + // so the datapath (which compares against tcphdr->dest) matches. + const port uint16 = 0x1234 + entries := buildV3Entries(lpmKey{IP: 1}, netPolicyFlagL7Required, + []l7PortEntry{{Port: htonsPort(port), Scheme: L7SchemeHTTP}}, 0) + if len(entries) != 1 { + t.Fatalf("len(entries)=%d, want 1", len(entries)) + } + if got := entries[0].key.Port; got != htonsPort(port) { + t.Fatalf("key.Port=0x%04x, want network byte order 0x%04x", got, htonsPort(port)) + } +} diff --git a/CubeNet/cubevs/dns_learn_test.go b/CubeNet/cubevs/dns_learn_test.go new file mode 100644 index 000000000..35efbe577 --- /dev/null +++ b/CubeNet/cubevs/dns_learn_test.go @@ -0,0 +1,355 @@ +package cubevs + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target $GOARCH dnslearn ../src/dns_learn_test.bpf.c -- -I../vmlinux/$GOARCH + +import ( + "encoding/binary" + "testing" + + "github.com/cilium/ebpf" +) + +const dnsLearnTestCaseLen = 12 + +type dnsLearnTestEnv struct { + program *ebpf.Program + allowOut *ebpf.Map + queryStore *ebpf.Map + allowInnerSpec *ebpf.MapSpec +} + +func loadDNSLearnTestEnv(t *testing.T) *dnsLearnTestEnv { + t.Helper() + + spec, err := loadDnslearn() + if err != nil { + t.Fatalf("load dns learn test spec: %v", err) + } + allowSpec := spec.Maps["allow_out_v3"] + if allowSpec == nil || allowSpec.InnerMap == nil { + t.Fatal("allow_out_v3 spec or inner template missing") + } + allowInnerSpec := allowSpec.InnerMap.Copy() + + for name, mapSpec := range spec.Maps { + switch name { + case ".rodata", "allow_out_v3", "test_query_store": + mapSpec.Pinning = ebpf.PinNone + default: + delete(spec.Maps, name) + } + } + + coll, err := ebpf.NewCollection(spec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF dns learn test unavailable: %v", err) + } + t.Fatalf("load dns learn test collection: %v", err) + } + t.Cleanup(coll.Close) + + env := &dnsLearnTestEnv{ + program: coll.Programs["test_dns_learn"], + allowOut: coll.Maps["allow_out_v3"], + queryStore: coll.Maps["test_query_store"], + allowInnerSpec: allowInnerSpec, + } + if env.program == nil || env.allowOut == nil || env.queryStore == nil { + t.Fatal("loaded dns learn program or maps missing") + } + return env +} + +// runDNSLearn drives dns_learn_response_ip with the given query against the +// sandbox's allow_out_v3 inner map, returning that inner map for assertions. +// seeds (if any) are written into the fresh inner map BEFORE the program +// runs, simulating pre-existing static/learned entries. +func runDNSLearn(t *testing.T, env *dnsLearnTestEnv, ifindex uint32, ip uint32, + ttl uint32, query dnsQueryTrackValue, seeds ...allowOutV3Entry, +) *ebpf.Map { + t.Helper() + + innerSpec := env.allowInnerSpec.Copy() + innerSpec.Name = "allow_dns_learn" + innerSpec.Pinning = ebpf.PinNone + inner, err := ebpf.NewMap(innerSpec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF LPM trie unavailable: %v", err) + } + t.Fatalf("create allow inner: %v", err) + } + t.Cleanup(func() { _ = inner.Close() }) + if err := env.allowOut.Put(&ifindex, inner); err != nil { + t.Fatalf("attach allow inner map: %v", err) + } + + for i, s := range seeds { + if err := inner.Update(&s.key, &s.value, ebpf.UpdateAny); err != nil { + t.Fatalf("seed allow inner entry %d: %v", i, err) + } + } + + qkey := uint32(0) + if err := env.queryStore.Put(&qkey, &query); err != nil { + t.Fatalf("seed query store: %v", err) + } + + // Pad the packet to 16 bytes: the skb test-run path rejects very small + // buffers (a 12-byte packet fails with EINVAL), while the program only + // reads the leading dns_learn_case struct. + data := make([]byte, 16) + binary.LittleEndian.PutUint32(data[0:4], ifindex) + binary.LittleEndian.PutUint32(data[4:8], ip) + binary.LittleEndian.PutUint32(data[8:12], ttl) + ret, _, err := env.program.Test(data) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF dns learn test-run unavailable: %v", err) + } + t.Fatalf("run dns learn test: %v", err) + } + if ret != 0 { + t.Fatalf("test_dns_learn returned %d, want TC_ACT_OK", ret) + } + return inner +} + +func lookupAllowV3(t *testing.T, inner *ebpf.Map, key lpmKeyV3) (netPolicyValueV3, bool) { + t.Helper() + var value netPolicyValueV3 + if err := inner.Lookup(&key, &value); err != nil { + return netPolicyValueV3{}, false + } + return value, true +} + +// TestDNSLearnPlainAllowWritesIPOnlyEntry is the B3 regression test: a plain +// (non-L7) domain allow rule must still be learned as a /32 (any-port) entry, +// exactly as the pre-v3 dataplane did. The regression wrote nothing, so the +// resolved IP was rejected by classify_egress_flow under default-deny. +func TestDNSLearnPlainAllowWritesIPOnlyEntry(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(300) + ip := mustParseCIDRForTest(t, "192.0.2.50").IP + + // Plain (non-L7) allow: flags=0, port_count=0. + inner := runDNSLearn(t, env, ifindex, ip, 300, dnsQueryTrackValue{Flags: 0, PortCount: 0}) + + value, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0}) + if !ok { + t.Fatal("plain allow did not learn a /32 entry (B3 regression)") + } + if value.Flags&uint8(netPolicyFlagL7Required) != 0 { + t.Fatalf("plain allow /32 has unexpected L7 flag: %#x", value.Flags) + } + if value.Scheme != L7SchemeNone { + t.Fatalf("plain allow /32 scheme=%d, want L7SchemeNone", value.Scheme) + } + if value.ExpiresAtNS == 0 { + t.Fatal("plain allow /32 has zero expiry, want temporary (DNS TTL)") + } +} + +// TestDNSLearnL7DefaultPortSet covers the L7 path with no explicit ports: +// it must learn the default {80/http, 443/https} set as /48 entries. +func TestDNSLearnL7DefaultPortSet(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(301) + ip := mustParseCIDRForTest(t, "192.0.2.60").IP + + query := dnsQueryTrackValue{Flags: uint8(netPolicyFlagL7Required), PortCount: 0} + inner := runDNSLearn(t, env, ifindex, ip, 300, query) + + for _, tc := range []struct { + port uint16 + scheme uint8 + }{ + {htonsPort(80), L7SchemeHTTP}, + {htonsPort(443), L7SchemeHTTPS}, + } { + value, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: tc.port}) + if !ok { + t.Fatalf("L7 default missing /48 entry for port %d", ntohsPort(tc.port)) + } + if value.Scheme != tc.scheme { + t.Fatalf("port %d scheme=%d, want %d", ntohsPort(tc.port), value.Scheme, tc.scheme) + } + if value.Flags&uint8(netPolicyFlagL7Required) == 0 { + t.Fatalf("port %d missing L7 flag", ntohsPort(tc.port)) + } + } +} + +// TestDNSLearnL7ExplicitPorts covers the L7 path with explicit ports: only the +// declared (port, scheme) tuples are learned, no others. +func TestDNSLearnL7ExplicitPorts(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(302) + ip := mustParseCIDRForTest(t, "192.0.2.70").IP + + query := dnsQueryTrackValue{ + Flags: uint8(netPolicyFlagL7Required), + PortCount: 2, + Ports: [maxL7PortsPerHost]l7PortEntry{ + {Port: htonsPort(8443), Scheme: L7SchemeHTTPS}, + {Port: htonsPort(8080), Scheme: L7SchemeHTTP}, + }, + } + inner := runDNSLearn(t, env, ifindex, ip, 300, query) + + if _, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(8443)}); !ok { + t.Fatal("missing /48 entry for explicit port 8443") + } + if _, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(8080)}); !ok { + t.Fatal("missing /48 entry for explicit port 8080") + } + if _, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(80)}); ok { + t.Fatal("unexpected /48 entry for non-configured port 80") + } +} + +// testFlagMarker is a high-bit flag used only by tests to detect improper +// flag inheritance from COVERING entries (it is not a real netPolicyFlag*). +const testFlagMarker = 0x40 + +// TestDNSLearnCoveringStaticCIDRDoesNotImmortalize is the exact-key-match +// regression test: a static CIDR COVERING the resolved IP must NOT make the +// DNS-learned entry inherit the static zero expiry (never ages) or the +// covering entry's flags. The LPM lookup inside dns_learn_response_ip is +// longest-prefix, so without the key_prefixlen check the covering static +// entry was treated as "an existing entry for the same key". +func TestDNSLearnCoveringStaticCIDRDoesNotImmortalize(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(303) + + // Static /24 (never expires) covering both test IPs. + staticSubnet := allowOutV3Entry{ + key: lpmKeyV3{Prefixlen: 24, IP: mustParseCIDRForTest(t, "192.0.2.0").IP, Port: 0}, + value: netPolicyValueV3{Flags: testFlagMarker, KeyPrefixlen: 24}, // ExpiresAtNS: 0 = static + } + + // Plain (non-L7) learn of 192.0.2.50 under the covering /24. + ipPlain := mustParseCIDRForTest(t, "192.0.2.50").IP + inner := runDNSLearn(t, env, ifindex, ipPlain, 300, + dnsQueryTrackValue{Flags: 0, PortCount: 0}, staticSubnet) + value, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 32, IP: ipPlain, Port: 0}) + if !ok { + t.Fatal("plain allow did not learn a /32 entry") + } + if value.KeyPrefixlen != 32 { + t.Fatalf("learned /32 KeyPrefixlen=%d, want 32 (lookup may have hit the covering /24)", value.KeyPrefixlen) + } + if value.ExpiresAtNS == 0 { + t.Fatal("learned /32 inherited static zero expiry from covering /24 (would never age)") + } + if value.Flags&testFlagMarker != 0 { + t.Fatalf("learned /32 inherited flags from covering /24: %#x", value.Flags) + } + + // L7 learn of 192.0.2.60 under the covering /24. + ipL7 := mustParseCIDRForTest(t, "192.0.2.60").IP + inner = runDNSLearn(t, env, ifindex, ipL7, 300, + dnsQueryTrackValue{Flags: uint8(netPolicyFlagL7Required), PortCount: 0}, staticSubnet) + value, ok = lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ipL7, Port: htonsPort(443)}) + if !ok { + t.Fatal("L7 allow did not learn a /48 entry for 443") + } + if value.KeyPrefixlen != 48 { + t.Fatalf("learned /48 KeyPrefixlen=%d, want 48 (lookup may have hit the covering /24)", value.KeyPrefixlen) + } + if value.ExpiresAtNS == 0 { + t.Fatal("learned /48 inherited static zero expiry from covering /24 (would never age)") + } + if value.Flags&testFlagMarker != 0 { + t.Fatalf("learned /48 inherited flags from covering /24: %#x", value.Flags) + } +} + +// TestDNSLearnExactStaticEntrySurvivesRefresh is the counterpart: an existing +// static entry for the EXACT same (ip, port)/48 key must survive a DNS +// refresh — its zero expiry and its extra flags are preserved. Other ports +// of the same learn get a normal TTL entry. +func TestDNSLearnExactStaticEntrySurvivesRefresh(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(304) + ip := mustParseCIDRForTest(t, "192.0.2.70").IP + + staticExact := allowOutV3Entry{ + key: lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}, + value: netPolicyValueV3{Flags: uint8(netPolicyFlagL7Required) | testFlagMarker, Scheme: L7SchemeHTTP, KeyPrefixlen: 48}, + } + inner := runDNSLearn(t, env, ifindex, ip, 300, + dnsQueryTrackValue{Flags: uint8(netPolicyFlagL7Required), PortCount: 0}, staticExact) + + // Exact static /48(443): zero expiry + marker flag preserved. + value, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if !ok { + t.Fatal("missing /48 entry for 443") + } + if value.ExpiresAtNS != 0 { + t.Fatal("exact static /48 lost its zero expiry on DNS refresh") + } + if value.Flags&testFlagMarker == 0 { + t.Fatalf("exact static /48 lost its flags on DNS refresh: %#x", value.Flags) + } + if value.Scheme != L7SchemeHTTPS { + t.Fatalf("exact static /48 scheme=%d after refresh, want HTTPS (scheme is last-write-wins)", value.Scheme) + } + + // /48(80) has no exact static entry: normal learned TTL entry. + value, ok = lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(80)}) + if !ok { + t.Fatal("missing /48 entry for 80") + } + if value.ExpiresAtNS == 0 { + t.Fatal("/48(80) wrongly became static (no exact static entry exists)") + } + if value.Flags&testFlagMarker != 0 { + t.Fatalf("/48(80) inherited marker flag from the 443 entry: %#x", value.Flags) + } +} + +// TestDNSLearnRefreshRenewsTTLMergesFlagsAndOverwritesScheme covers the +// remaining same-key merge cell: the OLD entry is itself a LEARNED entry for +// the exact same key. The refresh must (a) merge flags (old marker retained), +// (b) RENEW the expiry to the new DNS TTL (not keep the stale one), and +// (c) overwrite the scheme with the latest learn (last write wins). +func TestDNSLearnRefreshRenewsTTLMergesFlagsAndOverwritesScheme(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(307) + ip := mustParseCIDRForTest(t, "192.0.2.100").IP + + staleLearned := allowOutV3Entry{ + key: lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}, + value: netPolicyValueV3{ + ExpiresAtNS: 1, // stale, about to expire + Flags: uint8(netPolicyFlagL7Required) | testFlagMarker, + Scheme: L7SchemeHTTP, // wrong scheme on 443: refresh must overwrite + KeyPrefixlen: 48, + }, + } + inner := runDNSLearn(t, env, ifindex, ip, 300, + dnsQueryTrackValue{Flags: uint8(netPolicyFlagL7Required), PortCount: 0}, staleLearned) + + value, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if !ok { + t.Fatal("missing /48 entry for 443") + } + if value.KeyPrefixlen != 48 { + t.Fatalf("KeyPrefixlen=%d, want 48", value.KeyPrefixlen) + } + if value.ExpiresAtNS == 0 { + t.Fatal("refresh of a learned entry must keep a non-zero TTL (not become static)") + } + if value.ExpiresAtNS == 1 { + t.Fatal("refresh kept the stale expiry instead of renewing to the new DNS TTL") + } + if value.Flags&testFlagMarker == 0 { + t.Fatalf("refresh lost old learned flags: %#x", value.Flags) + } + if value.Scheme != L7SchemeHTTPS { + t.Fatalf("scheme=%d after refresh, want HTTPS (last write wins)", value.Scheme) + } +} diff --git a/CubeNet/cubevs/dns_reaper.go b/CubeNet/cubevs/dns_reaper.go index 48970f8a2..bad5dbe90 100644 --- a/CubeNet/cubevs/dns_reaper.go +++ b/CubeNet/cubevs/dns_reaper.go @@ -23,13 +23,13 @@ func reapDNSState() { reapDNSQueryTrack(now) } -// reapDNSLearnedPolicies scans allow_out_v2 and removes expired DNS-learned entries. +// reapDNSLearnedPolicies scans allow_out_v3 and removes expired DNS-learned entries. func reapDNSLearnedPolicies(now uint64) { - allowOut, err := loadPinnedMap(MapNameAllowOutV2) + allowOut, err := loadPinnedMap(MapNameAllowOutV3) if err != nil { enqueueEvent(Event{ Error: err, - Message: "failed to load allow_out_v2 map", + Message: "failed to load allow_out_v3 map", }) return } @@ -51,13 +51,13 @@ func reapDNSLearnedPolicies(now uint64) { if err := iter.Err(); err != nil { enqueueEvent(Event{ Error: err, - Message: "failed to iterate allow_out_v2 map", + Message: "failed to iterate allow_out_v3 map", }) return } } -// reapDNSLearnedPoliciesForInnerMap deletes expired DNS-learned entries from one allow_out_v2 inner map. +// reapDNSLearnedPoliciesForInnerMap deletes expired DNS-learned entries from one allow_out_v3 inner map. func reapDNSLearnedPoliciesForInnerMap(innerMapID uint32, now uint64) error { inner, err := ebpf.NewMapFromID(ebpf.MapID(innerMapID)) if err != nil { @@ -66,12 +66,12 @@ func reapDNSLearnedPoliciesForInnerMap(innerMapID uint32, now uint64) error { defer inner.Close() var ( - key lpmKey - value netPolicyValueV2 + key lpmKeyV3 + value netPolicyValueV3 ) iter := inner.Iterate() for iter.Next(&key, &value) { - if !netPolicyValueV2Expired(value, now) { + if !netPolicyValueV3Expired(value, now) { continue } if err := inner.Delete(&key); err != nil && !errors.Is(err, ebpf.ErrKeyNotExist) { @@ -79,7 +79,7 @@ func reapDNSLearnedPoliciesForInnerMap(innerMapID uint32, now uint64) error { } } if err := iter.Err(); err != nil { - return fmt.Errorf("failed to iterate allow_out_v2 inner map: %w", err) + return fmt.Errorf("failed to iterate allow_out_v3 inner map: %w", err) } return nil } diff --git a/CubeNet/cubevs/dnspolicy.go b/CubeNet/cubevs/dnspolicy.go index 9079ffa0b..52d1bad50 100644 --- a/CubeNet/cubevs/dnspolicy.go +++ b/CubeNet/cubevs/dnspolicy.go @@ -31,11 +31,11 @@ func newInnerDNSAllowMap() (*ebpf.Map, error) { // ensureDNSAllowInnerMap creates the per-sandbox DNS allow map when it is absent. func ensureDNSAllowInnerMap(outerMap *ebpf.Map, ifindex uint32) error { - return ensureInnerMapWithFactory(outerMap, ifindex, MapNameDNSAllow, newInnerDNSAllowMap) + return ensureInnerMapWithFactory(outerMap, ifindex, MapNameDNSAllowV2, newInnerDNSAllowMap) } func initDNSAllow(ifindex uint32) error { - dnsAllow, err := loadPinnedMap(MapNameDNSAllow) + dnsAllow, err := loadPinnedMap(MapNameDNSAllowV2) if err != nil { return err } @@ -89,35 +89,25 @@ type dnsAllowRule struct { domain string } -func buildDNSAllowRules(domains, l7Domains []string) ([]dnsAllowRule, error) { - rules := make([]dnsAllowRule, 0, len(domains)+len(l7Domains)) - indexByKey := make(map[dnsAllowKey]int, len(domains)+len(l7Domains)) +func buildDNSAllowRules(domains []string) ([]dnsAllowRule, error) { + rules := make([]dnsAllowRule, 0, len(domains)) + indexByKey := make(map[dnsAllowKey]int, len(domains)) - add := func(domains []string, flags uint8) error { - for _, domain := range domains { - key, value, err := makeDNSAllowRule(domain, flags) - if err != nil { - return err - } - if idx, ok := indexByKey[key]; ok { - rules[idx].value.Flags |= flags - continue - } - indexByKey[key] = len(rules) - rules = append(rules, dnsAllowRule{ - key: key, - value: value, - domain: domain, - }) + for _, domain := range domains { + key, value, err := makeDNSAllowRule(domain, 0) + if err != nil { + return nil, err } - return nil - } - - if err := add(domains, 0); err != nil { - return nil, err - } - if err := add(l7Domains, uint8(netPolicyFlagL7Required)); err != nil { - return nil, err + if idx, ok := indexByKey[key]; ok { + rules[idx].value.Flags |= value.Flags + continue + } + indexByKey[key] = len(rules) + rules = append(rules, dnsAllowRule{ + key: key, + value: value, + domain: domain, + }) } return rules, nil } @@ -150,6 +140,12 @@ func updateDNSAllowRule(inner *ebpf.Map, rule dnsAllowRule) error { var oldValue dnsAllowValue if err := inner.Lookup(&rule.key, &oldValue); err == nil { value.Flags |= oldValue.Flags + // Preserve any port tuples already installed for this key. Rule + // order should not cause a later rule with a subset of the port + // set to clobber an earlier rule's ports. buildL7Plan is the + // single point where scheme conflicts are detected, so any port + // present in both rules must agree on scheme by construction. + mergePortsIntoDNSValue(&value, oldValue.Ports[:oldValue.PortCount]) } else if !errors.Is(err, ebpf.ErrKeyNotExist) { return fmt.Errorf("dns allow lookup failed: %w, domain: %s", err, rule.domain) } @@ -160,6 +156,26 @@ func updateDNSAllowRule(inner *ebpf.Map, rule dnsAllowRule) error { return nil } +// mergePortsIntoDNSValue unions src into v.Ports without exceeding +// maxL7PortsPerHost. Silently drops overflow — buildL7Plan already enforces +// the budget in userspace. +func mergePortsIntoDNSValue(v *dnsAllowValue, src []l7PortEntry) { + for _, p := range src { + exists := false + for i := uint8(0); i < v.PortCount; i++ { + if v.Ports[i].Port == p.Port { + exists = true + break + } + } + if exists || v.PortCount >= maxL7PortsPerHost { + continue + } + v.Ports[v.PortCount] = p + v.PortCount++ + } +} + func flushDNSAllowInnerMap(inner *ebpf.Map) error { var oldKey dnsAllowKey var oldValue dnsAllowValue @@ -177,7 +193,7 @@ func flushDNSAllowInnerMap(inner *ebpf.Map) error { // cleanupDNSAllow clears the sandbox DNS allow inner map while keeping it preallocated. func cleanupDNSAllow(ifindex uint32) error { - dnsAllow, err := loadPinnedMap(MapNameDNSAllow) + dnsAllow, err := loadPinnedMap(MapNameDNSAllowV2) if err != nil { return err } @@ -201,7 +217,7 @@ func applyDNSAllow(ifindex uint32, rules []dnsAllowRule, replace bool) error { return nil } - dnsAllow, err := loadPinnedMap(MapNameDNSAllow) + dnsAllow, err := loadPinnedMap(MapNameDNSAllowV2) if err != nil { return err } diff --git a/CubeNet/cubevs/dump.go b/CubeNet/cubevs/dump.go index de6c0bf1e..e70fa1513 100644 --- a/CubeNet/cubevs/dump.go +++ b/CubeNet/cubevs/dump.go @@ -91,6 +91,10 @@ type EgressSessionDump struct { StateRaw uint8 `json:"state_raw"` ActiveClose bool `json:"active_close"` ActiveCloseRaw uint8 `json:"active_close_raw"` + PacketClass string `json:"packet_class"` + PacketClassRaw uint8 `json:"packet_class_raw"` + L7Scheme string `json:"l7_scheme"` + L7SchemeRaw uint8 `json:"l7_scheme_raw"` } type IngressSessionDump struct { @@ -115,6 +119,11 @@ type PolicyEntryDump struct { L7Required bool `json:"l7_required"` Flags uint8 `json:"flags"` Static bool `json:"static"` + // L7 port tuples inherited from a matched dns_allow_value at DNS-learn + // time (or attached directly to an L7 IP/CIDR rule). Empty when the + // datapath falls back to the default {80/http, 443/https} port set. + PortCount uint8 `json:"port_count"` + Ports []L7PortEntryDump `json:"ports,omitempty"` } type DenyPolicyMapDump struct { @@ -142,6 +151,10 @@ type DNSAllowRuleDump struct { Flags uint8 `json:"flags"` NameLen uint32 `json:"name_len"` Prefixlen uint32 `json:"prefixlen"` + // L7 port tuples the userspace built from rules sharing this host. + // Empty when the datapath falls back to the default {80/http, 443/https}. + PortCount uint8 `json:"port_count"` + Ports []L7PortEntryDump `json:"ports,omitempty"` } type DNSQueryTrackDump struct { @@ -156,6 +169,19 @@ type DNSQueryTrackDump struct { Expired bool `json:"expired"` L7Required bool `json:"l7_required"` Flags uint8 `json:"flags"` + // L7 port tuples copied verbatim from the matched dns_allow_value so the + // response handler can rebuild net_policy_value_v3 without a second + // dns_allow lookup. + PortCount uint8 `json:"port_count"` + Ports []L7PortEntryDump `json:"ports,omitempty"` +} + +// L7PortEntryDump is the JSON-friendly view of one (port, scheme) tuple. +// Port is emitted in host byte order (matches the port number users write in +// their policy YAML) even though it is stored in NBO on the datapath. +type L7PortEntryDump struct { + Port uint16 `json:"port"` + Scheme string `json:"scheme"` // "http" | "https" | ""(未知) } const mapNameSNATIPList = mapSNATIPList @@ -168,9 +194,9 @@ var businessMapDumpOrder = []string{ MapNameEgressSessions, MapNameIngressSessions, mapNameSNATIPList, - MapNameAllowOutV2, + MapNameAllowOutV3, MapNameDenyOut, - MapNameDNSAllow, + MapNameDNSAllowV2, MapNameDNSQueryTrack, } @@ -184,9 +210,9 @@ var businessMapDumpers = map[string]businessMapDumper{ MapNameEgressSessions: dumpEgressSessions, MapNameIngressSessions: dumpIngressSessions, mapNameSNATIPList: dumpSNATIPList, - MapNameAllowOutV2: dumpAllowOutV2, + MapNameAllowOutV3: dumpAllowOutV3, MapNameDenyOut: dumpDenyOut, - MapNameDNSAllow: dumpDNSAllow, + MapNameDNSAllowV2: dumpDNSAllow, MapNameDNSQueryTrack: dumpDNSQueryTrack, } @@ -411,6 +437,10 @@ func dumpEgressSessions(opts DumpOptions, now uint64) (any, error) { StateRaw: value.State, ActiveClose: value.ActiveClose != 0, ActiveCloseRaw: value.ActiveClose, + PacketClass: packetClassToString(value.PacketClass), + PacketClassRaw: value.PacketClass, + L7Scheme: l7SchemeToString(value.L7Scheme), + L7SchemeRaw: value.L7Scheme, }) } return entries, wrapIterErr(iter.Err(), MapNameEgressSessions) @@ -448,8 +478,8 @@ func dumpIngressSessions(opts DumpOptions, _ uint64) (any, error) { return entries, wrapIterErr(iter.Err(), MapNameIngressSessions) } -func dumpAllowOutV2(opts DumpOptions, now uint64) (any, error) { - return dumpPolicyMap(opts, now, MapNameAllowOutV2) +func dumpAllowOutV3(opts DumpOptions, now uint64) (any, error) { + return dumpPolicyMap(opts, now, MapNameAllowOutV3) } func dumpPolicyMap(opts DumpOptions, now uint64, mapName string) (any, error) { @@ -482,13 +512,13 @@ func dumpPolicyInnerMap(innerMapID uint32, now uint64) ([]PolicyEntryDump, error defer inner.Close() entries := make([]PolicyEntryDump, 0) - seen := make(map[lpmKey]struct{}) - var key lpmKey - var value netPolicyValueV2 + seen := make(map[lpmKeyV3]struct{}) + var key lpmKeyV3 + var value netPolicyValueV3 iter := inner.Iterate() for iter.Next(&key, &value) { if _, ok := seen[key]; ok { - return nil, fmt.Errorf("policy inner map iteration returned duplicate key: %s", dumpLPMCIDR(key)) //nolint:err113 + return nil, fmt.Errorf("policy inner map iteration returned duplicate key: %s", dumpLPMCIDRV3(key)) //nolint:err113 } seen[key] = struct{}{} if len(seen) > maxNetPolicyEntries { @@ -496,12 +526,22 @@ func dumpPolicyInnerMap(innerMapID uint32, now uint64) ([]PolicyEntryDump, error } entry := PolicyEntryDump{ - CIDR: dumpLPMCIDR(key), - Expired: netPolicyValueV2Expired(value, now), + CIDR: dumpLPMCIDRV3(key), + Expired: netPolicyValueV3Expired(value, now), L7Required: value.Flags&uint8(netPolicyFlagL7Required) != 0, Flags: value.Flags, Static: value.ExpiresAtNS == 0, } + // v3 carries the port in the key and the scheme in the value, so a + // single (ip, port)/48 entry is reported as one L7 (port, scheme) + // tuple; ip-only / subnet entries carry no port. + if key.Port != 0 && value.Scheme != L7SchemeNone { + entry.PortCount = 1 + entry.Ports = []L7PortEntryDump{{ + Port: ntohsPort(key.Port), + Scheme: l7SchemeToString(value.Scheme), + }} + } if value.ExpiresAtNS != 0 { entry.ExpiresAtNS = value.ExpiresAtNS entry.ExpiresInNS = remainingNS(value.ExpiresAtNS, now) @@ -512,6 +552,13 @@ func dumpPolicyInnerMap(innerMapID uint32, now uint64) ([]PolicyEntryDump, error return entries, wrapIterErr(iter.Err(), "policy inner") } +func dumpLPMCIDRV3(key lpmKeyV3) string { + if key.Port != 0 { + return fmt.Sprintf("%s/%d:%d", uint32ToIP(key.IP).String(), key.Prefixlen, ntohsPort(key.Port)) + } + return fmt.Sprintf("%s/%d", uint32ToIP(key.IP).String(), key.Prefixlen) +} + func dumpDenyOut(opts DumpOptions, _ uint64) (any, error) { outer, err := loadPinnedMap(MapNameDenyOut) if err != nil { @@ -564,7 +611,7 @@ func dumpDenyInnerMap(innerMapID uint32) ([]DenyPolicyEntryDump, error) { } func dumpDNSAllow(opts DumpOptions, _ uint64) (any, error) { - outer, err := loadPinnedMap(MapNameDNSAllow) + outer, err := loadPinnedMap(MapNameDNSAllowV2) if err != nil { return nil, err } @@ -576,10 +623,10 @@ func dumpDNSAllow(opts DumpOptions, _ uint64) (any, error) { } entries := make([]DNSAllowMapDump, 0) - err = dumpSelectedInnerMapIDs(opts, outer, MapNameDNSAllow, func(ifindex uint32, innerMapID uint32) error { + err = dumpSelectedInnerMapIDs(opts, outer, MapNameDNSAllowV2, func(ifindex uint32, innerMapID uint32) error { innerDump, err := dumpDNSAllowInnerMap(innerMapID) if err != nil { - return fmt.Errorf("dump %s inner map failed: %w, ifindex: %d", MapNameDNSAllow, err, ifindex) + return fmt.Errorf("dump %s inner map failed: %w, ifindex: %d", MapNameDNSAllowV2, err, ifindex) } innerDump.Ifindex = ifindex if meta, ok := metadata[ifindex]; ok { @@ -660,6 +707,8 @@ func dumpDNSQueryTrack(opts DumpOptions, now uint64) (any, error) { Expired: value.ExpiresAtNS <= now, L7Required: value.Flags&uint8(netPolicyFlagL7Required) != 0, Flags: value.Flags, + PortCount: value.PortCount, + Ports: l7PortEntriesToDump(value.Ports[:], value.PortCount), }) } return entries, wrapIterErr(iter.Err(), MapNameDNSQueryTrack) @@ -830,6 +879,8 @@ func dumpDNSAllowRule(key dnsAllowKey, value dnsAllowValue) (DNSAllowRuleDump, e Flags: value.Flags, NameLen: value.NameLen, Prefixlen: key.Prefixlen, + PortCount: value.PortCount, + Ports: l7PortEntriesToDump(value.Ports[:], value.PortCount), }, nil } @@ -837,6 +888,52 @@ func ifindexMatches(opts DumpOptions, ifindex uint32) bool { return !opts.FilterIfindex || opts.Ifindex == ifindex } +// l7PortEntriesToDump converts the fixed-size ports array embedded in the +// datapath value into a JSON-friendly slice sized to port_count. Ports are +// stored in network byte order on the datapath (matching tcphdr->dest) but +// emitted in host byte order here so the JSON is directly comparable to the +// port numbers users write in their rules. +func l7PortEntriesToDump(ports []l7PortEntry, count uint8) []L7PortEntryDump { + if count == 0 { + return nil + } + if int(count) > len(ports) { + count = uint8(len(ports)) + } + out := make([]L7PortEntryDump, 0, count) + for i := uint8(0); i < count; i++ { + out = append(out, L7PortEntryDump{ + Port: ntohs(ports[i].Port), + Scheme: l7SchemeToString(ports[i].Scheme), + }) + } + return out +} + +func packetClassToString(class uint8) string { + switch class { + case 0: + return "snat" + case 1: + return "l7_proxy" + default: + return fmt.Sprintf("unknown(%d)", class) + } +} + +func l7SchemeToString(s uint8) string { + switch s { + case L7SchemeNone: + return "none" + case L7SchemeHTTP: + return "http" + case L7SchemeHTTPS: + return "https" + default: + return fmt.Sprintf("unknown(%d)", s) + } +} + func remainingNS(expiresAt, now uint64) int64 { if expiresAt >= now { return int64(expiresAt - now) diff --git a/CubeNet/cubevs/dump_test.go b/CubeNet/cubevs/dump_test.go index 239583dd0..4ac05eea5 100644 --- a/CubeNet/cubevs/dump_test.go +++ b/CubeNet/cubevs/dump_test.go @@ -7,6 +7,35 @@ import ( "golang.org/x/sys/unix" ) +func TestSessionClassificationStrings(t *testing.T) { + for _, tt := range []struct { + value uint8 + want string + }{ + {0, "snat"}, + {1, "l7_proxy"}, + {9, "unknown(9)"}, + } { + if got := packetClassToString(tt.value); got != tt.want { + t.Fatalf("packetClassToString(%d)=%q, want %q", tt.value, got, tt.want) + } + } + + for _, tt := range []struct { + value uint8 + want string + }{ + {L7SchemeNone, "none"}, + {L7SchemeHTTP, "http"}, + {L7SchemeHTTPS, "https"}, + {9, "unknown(9)"}, + } { + if got := l7SchemeToString(tt.value); got != tt.want { + t.Fatalf("l7SchemeToString(%d)=%q, want %q", tt.value, got, tt.want) + } + } +} + func TestBusinessMapNamesReturnsCopy(t *testing.T) { names := BusinessMapNames() if len(names) == 0 { @@ -21,15 +50,15 @@ func TestBusinessMapNamesReturnsCopy(t *testing.T) { func TestNormalizeBusinessMapNames(t *testing.T) { got, err := normalizeBusinessMapNames([]string{ - MapNameDNSAllow, - MapNameDNSAllow, - MapNameAllowOutV2, + MapNameDNSAllowV2, + MapNameDNSAllowV2, + MapNameAllowOutV3, }) if err != nil { t.Fatalf("normalizeBusinessMapNames returned error: %v", err) } - want := []string{MapNameDNSAllow, MapNameAllowOutV2} + want := []string{MapNameDNSAllowV2, MapNameAllowOutV3} if !reflect.DeepEqual(got, want) { t.Fatalf("normalizeBusinessMapNames()=%v, want %v", got, want) } diff --git a/CubeNet/cubevs/egress_policy_test.go b/CubeNet/cubevs/egress_policy_test.go new file mode 100644 index 000000000..19fc1d267 --- /dev/null +++ b/CubeNet/cubevs/egress_policy_test.go @@ -0,0 +1,276 @@ +package cubevs + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target $GOARCH egresspolicy ../src/egress_policy_test.bpf.c -- -I../vmlinux/$GOARCH + +import ( + "encoding/binary" + "testing" + + "github.com/cilium/ebpf" +) + +const ( + egressPolicyTestCaseLen = 16 + flowVerdictReject = uint8(0) + flowVerdictSNAT = uint8(1) + flowVerdictHTTPS = uint8(3) +) + +type egressPolicyTestEnv struct { + program *ebpf.Program + allowOut *ebpf.Map + denyOut *ebpf.Map + allowInnerSpec *ebpf.MapSpec + denyInnerSpec *ebpf.MapSpec +} + +func loadEgressPolicyTestEnv(t *testing.T) *egressPolicyTestEnv { + t.Helper() + + spec, err := loadEgresspolicy() + if err != nil { + t.Fatalf("load egress policy test spec: %v", err) + } + allowSpec := spec.Maps["allow_out_v3"] + denySpec := spec.Maps["deny_out"] + if allowSpec == nil || allowSpec.InnerMap == nil || denySpec == nil || denySpec.InnerMap == nil { + t.Fatal("egress policy map specs or inner templates missing") + } + allowInnerSpec := allowSpec.InnerMap.Copy() + denyInnerSpec := denySpec.InnerMap.Copy() + + for name, mapSpec := range spec.Maps { + switch name { + case ".rodata", "allow_out_v3", "deny_out": + mapSpec.Pinning = ebpf.PinNone + default: + delete(spec.Maps, name) + } + } + + coll, err := ebpf.NewCollection(spec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF policy test unavailable: %v", err) + } + t.Fatalf("load egress policy test collection: %v", err) + } + t.Cleanup(coll.Close) + + env := &egressPolicyTestEnv{ + program: coll.Programs["test_classify_egress_flow"], + allowOut: coll.Maps["allow_out_v3"], + denyOut: coll.Maps["deny_out"], + allowInnerSpec: allowInnerSpec, + denyInnerSpec: denyInnerSpec, + } + if env.program == nil || env.allowOut == nil || env.denyOut == nil { + t.Fatal("loaded egress policy program or maps missing") + } + return env +} + +func newEgressPolicyInnerMap(t *testing.T, spec *ebpf.MapSpec, name string) *ebpf.Map { + t.Helper() + + innerSpec := spec.Copy() + innerSpec.Name = name + innerSpec.Pinning = ebpf.PinNone + inner, err := ebpf.NewMap(innerSpec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF LPM trie unavailable: %v", err) + } + t.Fatalf("create %s: %v", name, err) + } + t.Cleanup(func() { _ = inner.Close() }) + return inner +} + +func (env *egressPolicyTestEnv) attachInnerMaps(t *testing.T, ifindex uint32) (*ebpf.Map, *ebpf.Map) { + t.Helper() + + allowInner := newEgressPolicyInnerMap(t, env.allowInnerSpec, "allow_test") + denyInner := newEgressPolicyInnerMap(t, env.denyInnerSpec, "deny_test") + if err := env.allowOut.Put(&ifindex, allowInner); err != nil { + t.Fatalf("attach allow inner map: %v", err) + } + if err := env.denyOut.Put(&ifindex, denyInner); err != nil { + t.Fatalf("attach deny inner map: %v", err) + } + return allowInner, denyInner +} + +func runEgressPolicyCase(t *testing.T, prog *ebpf.Program, ifindex, daddr uint32, + dport uint16, +) uint8 { + t.Helper() + + data := make([]byte, egressPolicyTestCaseLen) + binary.LittleEndian.PutUint32(data[0:4], ifindex) + binary.LittleEndian.PutUint32(data[4:8], daddr) + binary.LittleEndian.PutUint16(data[8:10], dport) + ret, out, err := prog.Test(data) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF policy test-run unavailable: %v", err) + } + t.Fatalf("run egress policy test: %v", err) + } + if ret != 0 { + t.Fatalf("test_classify_egress_flow returned %d, want TC_ACT_OK", ret) + } + if len(out) < egressPolicyTestCaseLen { + t.Fatalf("test output length=%d, want >=%d", len(out), egressPolicyTestCaseLen) + } + return out[10] +} + +func mustParseCIDRForTest(t *testing.T, cidr string) lpmKey { + t.Helper() + + key, err := parseCIDR(cidr) + if err != nil { + t.Fatalf("parse %q: %v", cidr, err) + } + return key +} + +func TestClassifyEgressFlowLPMFallback(t *testing.T) { + env := loadEgressPolicyTestEnv(t) + tests := []struct { + name string + allowCIDR string + daddr string + }{ + {name: "IP rule", allowCIDR: "203.0.113.9/32", daddr: "203.0.113.9"}, + {name: "subnet rule", allowCIDR: "198.51.100.0/24", daddr: "198.51.100.42"}, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ifindex := uint32(100 + i) + allowInner, denyInner := env.attachInnerMaps(t, ifindex) + allowKey := mustParseCIDRForTest(t, tt.allowCIDR) + if err := allowInner.Put(&lpmKeyV3{ + Prefixlen: allowKey.Prefixlen, + IP: allowKey.IP, + }, &netPolicyValueV3{}); err != nil { + t.Fatalf("insert allow rule %s: %v", tt.allowCIDR, err) + } + + // A deny-all backstop proves that FLOW_SNAT came from the /48 + // lookup falling back to the allow rule, not from default allow. + denyAll := mustParseCIDRForTest(t, "0.0.0.0/0") + denyValue := uint32(1) + if err := denyInner.Put(&denyAll, &denyValue); err != nil { + t.Fatalf("insert deny-all rule: %v", err) + } + + daddr := mustParseCIDRForTest(t, tt.daddr).IP + got := runEgressPolicyCase(t, env.program, ifindex, daddr, htonsPort(443)) + if got != flowVerdictSNAT { + t.Fatalf("verdict=%d, want FLOW_SNAT", got) + } + }) + } +} + +func TestClassifyEgressFlowExactL7Match(t *testing.T) { + env := loadEgressPolicyTestEnv(t) + ifindex := uint32(150) + allowInner, denyInner := env.attachInnerMaps(t, ifindex) + daddr := mustParseCIDRForTest(t, "192.0.2.20").IP + dport := htonsPort(8443) + allowKey := lpmKeyV3{Prefixlen: 48, IP: daddr, Port: dport} + allowValue := netPolicyValueV3{ + Flags: uint8(netPolicyFlagL7Required), + Scheme: L7SchemeHTTPS, + } + if err := allowInner.Put(&allowKey, &allowValue); err != nil { + t.Fatalf("insert exact L7 allow rule: %v", err) + } + + denyAll := mustParseCIDRForTest(t, "0.0.0.0/0") + denyValue := uint32(1) + if err := denyInner.Put(&denyAll, &denyValue); err != nil { + t.Fatalf("insert deny-all rule: %v", err) + } + + if got := runEgressPolicyCase(t, env.program, ifindex, daddr, dport); got != flowVerdictHTTPS { + t.Fatalf("exact-port verdict=%d, want FLOW_HTTPS", got) + } + if got := runEgressPolicyCase(t, env.program, ifindex, daddr, htonsPort(443)); got != flowVerdictReject { + t.Fatalf("different-port verdict=%d, want FLOW_REJECT", got) + } +} + +func TestClassifyEgressFlowL7UnknownSchemeFailsClosed(t *testing.T) { + env := loadEgressPolicyTestEnv(t) + ifindex := uint32(151) + allowInner, denyInner := env.attachInnerMaps(t, ifindex) + daddr := mustParseCIDRForTest(t, "192.0.2.21").IP + dport := htonsPort(8443) + // L7_REQUIRED set but scheme is NONE — a corrupt or half-written entry. + // Must fail closed (FLOW_REJECT) rather than downgrade to FLOW_SNAT and + // silently bypass the TPROXY intercept the rule asked for. + allowKey := lpmKeyV3{Prefixlen: 48, IP: daddr, Port: dport} + allowValue := netPolicyValueV3{ + Flags: uint8(netPolicyFlagL7Required), + Scheme: L7SchemeNone, + } + if err := allowInner.Put(&allowKey, &allowValue); err != nil { + t.Fatalf("insert L7 allow rule with unknown scheme: %v", err) + } + + denyAll := mustParseCIDRForTest(t, "0.0.0.0/0") + denyValue := uint32(1) + if err := denyInner.Put(&denyAll, &denyValue); err != nil { + t.Fatalf("insert deny-all rule: %v", err) + } + + if got := runEgressPolicyCase(t, env.program, ifindex, daddr, dport); got != flowVerdictReject { + t.Fatalf("verdict=%d, want FLOW_REJECT (fail closed on unknown scheme)", got) + } +} + +func TestClassifyEgressFlowExpiredAllow(t *testing.T) { + env := loadEgressPolicyTestEnv(t) + tests := []struct { + name string + prefixlen uint32 + }{ + {name: "expired exact L7 then deny", prefixlen: 48}, + {name: "expired fallback IP then deny", prefixlen: 32}, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ifindex := uint32(200 + i) + allowInner, denyInner := env.attachInnerMaps(t, ifindex) + daddr := mustParseCIDRForTest(t, "192.0.2.10").IP + dport := htonsPort(443) + allowKey := lpmKeyV3{Prefixlen: tt.prefixlen, IP: daddr} + allowValue := netPolicyValueV3{ExpiresAtNS: 1} + if tt.prefixlen == 48 { + allowKey.Port = dport + allowValue.Flags = uint8(netPolicyFlagL7Required) + allowValue.Scheme = L7SchemeHTTPS + } + if err := allowInner.Put(&allowKey, &allowValue); err != nil { + t.Fatalf("insert expired /%d allow rule: %v", tt.prefixlen, err) + } + + denyAll := mustParseCIDRForTest(t, "0.0.0.0/0") + denyValue := uint32(1) + if err := denyInner.Put(&denyAll, &denyValue); err != nil { + t.Fatalf("insert deny-all rule: %v", err) + } + + got := runEgressPolicyCase(t, env.program, ifindex, daddr, dport) + if got != flowVerdictReject { + t.Fatalf("verdict=%d, want FLOW_REJECT", got) + } + }) + } +} diff --git a/CubeNet/cubevs/l7_mark_test.go b/CubeNet/cubevs/l7_mark_test.go new file mode 100644 index 000000000..388623d7d --- /dev/null +++ b/CubeNet/cubevs/l7_mark_test.go @@ -0,0 +1,117 @@ +package cubevs + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target $GOARCH l7mark ../src/l7_mark_test.bpf.c -- -I../vmlinux/$GOARCH + +import ( + "encoding/binary" + "testing" + + "github.com/cilium/ebpf" +) + +// loadL7MarkProgram loads the l7_mark test object (only its .rodata and the +// test program), applies the given L7 mark overrides to the const volatile +// globals exactly as rewriteConstants would, and returns the loaded program. +func loadL7MarkProgram(t *testing.T, httpMark, httpsMark, mask uint32) *ebpf.Program { + t.Helper() + spec, err := loadL7mark() + if err != nil { + t.Fatalf("load l7 mark test spec: %v", err) + } + for name, mapSpec := range spec.Maps { + if name != ".rodata" { + delete(spec.Maps, name) + } else { + mapSpec.Pinning = ebpf.PinNone + } + } + setVar := func(name string, value uint32) { + v := spec.Variables[name] + if v == nil { + t.Fatalf("variable %s missing from l7 mark test object", name) + } + if err := v.Set(value); err != nil { + t.Fatalf("set %s: %v", name, err) + } + } + setVar(globalNameCubeL7MarkHTTP, httpMark) + setVar(globalNameCubeL7MarkHTTPS, httpsMark) + setVar(globalNameCubeL7MarkMask, mask) + + coll, err := ebpf.NewCollection(spec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF l7 mark test unavailable: %v", err) + } + t.Fatalf("load l7 mark test collection: %v", err) + } + t.Cleanup(coll.Close) + prog := coll.Programs["test_l7_mark"] + if prog == nil { + t.Fatal("test_l7_mark program missing") + } + return prog +} + +func runL7MarkCase(t *testing.T, prog *ebpf.Program, inMark uint32) uint32 { + t.Helper() + data := make([]byte, 16) // pad past the 4-byte case; tiny packets are rejected + binary.LittleEndian.PutUint32(data[0:4], inMark) + ret, out, err := prog.Test(data) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF l7 mark test-run unavailable: %v", err) + } + t.Fatalf("run l7 mark test: %v", err) + } + if ret != 0 { + t.Fatalf("test_l7_mark returned %d, want TC_ACT_OK", ret) + } + return binary.LittleEndian.Uint32(out[0:4]) +} + +// TestL7MarkStampedMatchesConfiguredValue proves that an overridden +// cube_l7_mark_http reaches the running dataplane and is the value stamped +// onto skb->mark (masked), exactly as mvmtap.bpf.c does for L7 traffic. +func TestL7MarkStampedMatchesConfiguredValue(t *testing.T) { + const httpMark = uint32(0xABCD0000) + const mask = uint32(0xFFFF0000) + prog := loadL7MarkProgram(t, httpMark, defaultL7MarkHTTPS, mask) + + // Low (user) bits of the incoming mark are preserved; the configured HTTP + // mark is OR'd into the cube-owned bits. + const inMark = uint32(0x00001234) + want := (inMark &^ mask) | httpMark + if got := runL7MarkCase(t, prog, inMark); got != want { + t.Fatalf("stamped mark=%#x, want %#x (configured http mark %#x OR'd over low bits %#x)", + got, want, httpMark, inMark&^mask) + } +} + +// TestResolveL7Marks covers the defaults-and-validation helper used by +// rewriteConstants: zero fields get shipped defaults, overrides are honored, +// and invalid combinations (http==https, bits outside the mask) are rejected. +func TestResolveL7Marks(t *testing.T) { + h, s, m, err := resolveL7Marks(Params{}) + if err != nil { + t.Fatalf("defaults: %v", err) + } + if h != defaultL7MarkHTTP || s != defaultL7MarkHTTPS || m != defaultL7MarkMask { + t.Fatalf("defaults: got http=%#x https=%#x mask=%#x", h, s, m) + } + + h, s, m, err = resolveL7Marks(Params{L7MarkHTTP: 0xAB010000, L7MarkHTTPS: 0xAB020000}) + if err != nil { + t.Fatalf("override: %v", err) + } + if h != 0xAB010000 || s != 0xAB020000 || m != defaultL7MarkMask { + t.Fatalf("override: got http=%#x https=%#x mask=%#x", h, s, m) + } + + if _, _, _, err := resolveL7Marks(Params{L7MarkHTTP: 0xAB010000, L7MarkHTTPS: 0xAB010000}); err == nil { + t.Fatal("http==https was not rejected") + } + if _, _, _, err := resolveL7Marks(Params{L7MarkHTTP: 0xAB010001}); err == nil { + t.Fatal("mark with bits outside the mask was not rejected") + } +} diff --git a/CubeNet/cubevs/map.go b/CubeNet/cubevs/map.go index 92437b8bd..46b7a3666 100644 --- a/CubeNet/cubevs/map.go +++ b/CubeNet/cubevs/map.go @@ -7,6 +7,11 @@ import ( "github.com/cilium/ebpf" ) +// bpfFSPath is the bpffs mount point used for pinning maps. It is a var (not a +// const) so tests can point it at a temporary bpffs mount instead of the real +// /sys/fs/bpf. +var bpfFSPath = "/sys/fs/bpf" + func pinPath(name string) string { path := filepath.Join(bpfFSPath, name) diff --git a/CubeNet/cubevs/migration.go b/CubeNet/cubevs/migration.go index 19c2ab359..76c79b429 100644 --- a/CubeNet/cubevs/migration.go +++ b/CubeNet/cubevs/migration.go @@ -3,6 +3,7 @@ package cubevs import ( "errors" "fmt" + "log" "os" "unsafe" @@ -11,24 +12,157 @@ import ( const legacyAllowOutValueSize = uint32(4) -// migrateAllowOutV1ToV2 copies static v0.2.0 allow_out entries into -// allow_out_v2. The legacy value is only a presence marker, so migrated -// entries become static v2 entries with ExpiresAtNS set to zero. -func migrateAllowOutV1ToV2() error { - legacy, err := ebpf.LoadPinnedMap(pinPath(MapNameAllowOut), nil) +// skipUnsupportedL7Subnet reports whether a legacy allow entry is an L7 rule +// keyed by a subnet (prefixlen<32). v3 cannot express subnet+port in a single +// LPM key (its /48 key matches exact (ip, port) pairs), and new L7 rules now +// reject subnet hosts outright (classifyL7Target). Carrying such a legacy rule +// forward would silently narrow it to the network address, so it is dropped +// here with a warning instead; the operator should recreate it as /32 host or +// domain rules. +func skipUnsupportedL7Subnet(key lpmKey, flags uint8, sourceName string) bool { + if flags&netPolicyFlagL7Required == 0 || key.Prefixlen >= 32 { + return false + } + log.Printf("cubevs migration: dropping unsupported L7 subnet rule %s/%d from %s: v3 cannot express subnet+port; recreate as /32 host or domain rules", + uint32ToIP(key.IP).String(), key.Prefixlen, sourceName) + return true +} + +type legacyDNSAllowValue struct { + NameLen uint32 + Flags uint8 + Reserved [3]uint8 +} + +// allowOutV3Entry is one (key, value) pair in the new allow_out_v3 +// layout, produced by expanding an older allow_out entry. +type allowOutV3Entry struct { + key lpmKeyV3 + value netPolicyValueV3 +} + +// buildV3Entries expands one legacy (ip CIDR, flags, ports, expires) into +// the new allow_out_v3 entries: +// - L7 flag set: one exact (ip, port)/48 entry per (port, scheme); +// a default port set (port_count == 0) expands to {80/http, 443/https}. +// - otherwise: one ip-only (or subnet) /32 entry, scheme = NONE. +// +// The legacy net_policy_value_v2 ports[] (if any) are passed by the caller +// as the ports slice; a nil slice triggers the default-set expansion. +func buildV3Entries(ipKey lpmKey, flags uint8, ports []l7PortEntry, expires uint64) []allowOutV3Entry { + if flags&netPolicyFlagL7Required != 0 { + if len(ports) == 0 { + ports = expandDefaultPortSet() + } + out := make([]allowOutV3Entry, 0, len(ports)) + for _, p := range ports { + out = append(out, allowOutV3Entry{ + key: lpmKeyV3{Prefixlen: 48, IP: ipKey.IP, Port: p.Port}, + value: netPolicyValueV3{Flags: flags, Scheme: p.Scheme, ExpiresAtNS: expires, KeyPrefixlen: 48}, + }) + } + return out + } + return []allowOutV3Entry{{ + key: lpmKeyV3{Prefixlen: ipKey.Prefixlen, IP: ipKey.IP, Port: 0}, + value: netPolicyValueV3{Flags: flags, ExpiresAtNS: expires, KeyPrefixlen: uint8(ipKey.Prefixlen)}, + }} +} + +// applyV3Entries writes expanded entries into a v3 inner LPM trie. +func applyV3Entries(dest *ebpf.Map, entries []allowOutV3Entry) error { + for _, e := range entries { + if err := dest.Update(&e.key, &e.value, ebpf.UpdateAny); err != nil { + return fmt.Errorf("update %s inner map failed: %w", MapNameAllowOutV3, err) + } + } + return nil +} + +func pinnedMapExists(name string) (bool, error) { + _, err := os.Stat(pinPath(name)) + if err == nil { + return true, nil + } + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, err +} + +// migratePersistentPolicyMaps performs a one-way migration into the current +// generation. The legacy pins are deleted once their contents have been +// copied into the current maps so they don't linger as orphaned bpffs +// entries. The current datapath and userspace only read allow_out_v3 and +// dns_allow_v2. +func migratePersistentPolicyMaps() error { + allowSrc, err := allowOutMigrationSource() + if err != nil { + return err + } + if allowSrc != "" { + if err := migrateAllowOutMap(allowSrc); err != nil { + // Leave the legacy pin in place so the migration can be + // retried on the next restart. + return err + } + if err := removePinnedMap(allowSrc); err != nil { + return err + } + } + + if err := migrateDNSAllowMap(MapNameDNSAllow); err != nil { + return err + } + return removePinnedMap(MapNameDNSAllow) +} + +// allowOutMigrationSource returns the name of the legacy allow_out pin to +// migrate from, preferring allow_out_v2 and falling back to the v0.2.0 +// allow_out pin. It returns "" when no legacy pin is present. +func allowOutMigrationSource() (string, error) { + v2Exists, err := pinnedMapExists(MapNameAllowOutV2) + if err != nil { + return "", err + } + if v2Exists { + return MapNameAllowOutV2, nil + } + legacyExists, err := pinnedMapExists(MapNameAllowOut) + if err != nil { + return "", err + } + if legacyExists { + return MapNameAllowOut, nil + } + return "", nil +} + +// removePinnedMap unlinks a bpffs pin. It is safe to call for a map whose +// inner maps are not separately pinned (they live only inside the outer +// hash-of-maps and are released by the kernel once the outer pin is gone), +// so a single-file unlink suffices. A missing pin is treated as success. +func removePinnedMap(name string) error { + if err := os.Remove(pinPath(name)); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove legacy pin %s failed: %w", name, err) + } + return nil +} + +func migrateAllowOutMap(sourceName string) error { + source, err := ebpf.LoadPinnedMap(pinPath(sourceName), nil) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil } - return fmt.Errorf("load legacy %s failed: %w", MapNameAllowOut, err) + return fmt.Errorf("load legacy %s failed: %w", sourceName, err) } - defer legacy.Close() + defer source.Close() - if err := verifyMapLayout(legacy, MapNameAllowOut, ebpf.HashOfMaps, uint32(unsafe.Sizeof(uint32(0))), uint32(unsafe.Sizeof(uint32(0)))); err != nil { + if err := verifyMapLayout(source, sourceName, ebpf.HashOfMaps, uint32(unsafe.Sizeof(uint32(0))), uint32(unsafe.Sizeof(uint32(0)))); err != nil { return err } - - current, err := loadPinnedMap(MapNameAllowOutV2) + current, err := loadPinnedMap(MapNameAllowOutV3) if err != nil { return err } @@ -36,59 +170,160 @@ func migrateAllowOutV1ToV2() error { var ifindex uint32 var innerMapID uint32 - iter := legacy.Iterate() + iter := source.Iterate() for iter.Next(&ifindex, &innerMapID) { - if err := migrateAllowOutInnerMap(current, ifindex, ebpf.MapID(innerMapID)); err != nil { + if err := migrateAllowOutInnerMap(current, ifindex, sourceName, ebpf.MapID(innerMapID)); err != nil { return err } } if err := iter.Err(); err != nil { - return fmt.Errorf("iterate legacy %s failed: %w", MapNameAllowOut, err) + return fmt.Errorf("iterate legacy %s failed: %w", sourceName, err) } return nil } -func migrateAllowOutInnerMap(current *ebpf.Map, ifindex uint32, legacyInnerID ebpf.MapID) error { - legacyInner, err := ebpf.NewMapFromID(legacyInnerID) +func migrateAllowOutInnerMap(current *ebpf.Map, ifindex uint32, sourceName string, sourceInnerID ebpf.MapID) error { + source, err := ebpf.NewMapFromID(sourceInnerID) + if err != nil { + return fmt.Errorf("open legacy %s inner map failed: %w, id: %d", sourceName, err, sourceInnerID) + } + defer source.Close() + info, err := source.Info() if err != nil { - return fmt.Errorf("open legacy %s inner map failed: %w, id: %d", MapNameAllowOut, err, legacyInnerID) + return fmt.Errorf("get legacy %s inner map info failed: %w", sourceName, err) + } + if info.Type != ebpf.LPMTrie || info.KeySize != uint32(unsafe.Sizeof(lpmKey{})) { + return fmt.Errorf("%s inner map has incompatible ABI: type=%s key_size=%d", sourceName, info.Type, info.KeySize) //nolint:err113 + } + if info.ValueSize != legacyAllowOutValueSize && + info.ValueSize != uint32(unsafe.Sizeof(netPolicyValueV2{})) { + return fmt.Errorf("%s inner map has unsupported value_size=%d", sourceName, info.ValueSize) //nolint:err113 } - defer legacyInner.Close() - if err := verifyMapLayout(legacyInner, MapNameAllowOut, ebpf.LPMTrie, uint32(unsafe.Sizeof(lpmKey{})), legacyAllowOutValueSize); err != nil { + if err := ensureAllowOutV3InnerMap(current, ifindex); err != nil { + return err + } + destination, err := lookupInnerMap(current, ifindex) + if err != nil { return err } + defer destination.Close() - if err := ensureAllowOutV2InnerMap(current, ifindex); err != nil { + switch info.ValueSize { + case legacyAllowOutValueSize: + // v0.2.0 static allow marker: plain /32, no L7. + var key lpmKey + var oldValue uint32 + iter := source.Iterate() + for iter.Next(&key, &oldValue) { + if err := applyV3Entries(destination, buildV3Entries(key, 0, nil, 0)); err != nil { + return fmt.Errorf("update %s inner map failed: %w", MapNameAllowOutV3, err) + } + } + return wrapIterErr(iter.Err(), sourceName) + case uint32(unsafe.Sizeof(netPolicyValueV2{})): + // 16-byte legacy net_policy_value_v2: flags + expires, no ports. + var key lpmKey + var oldValue netPolicyValueV2 + iter := source.Iterate() + for iter.Next(&key, &oldValue) { + if skipUnsupportedL7Subnet(key, oldValue.Flags, sourceName) { + continue + } + if err := applyV3Entries(destination, buildV3Entries(key, oldValue.Flags, nil, oldValue.ExpiresAtNS)); err != nil { + return fmt.Errorf("update %s inner map failed: %w", MapNameAllowOutV3, err) + } + } + return wrapIterErr(iter.Err(), sourceName) + default: + return fmt.Errorf("%s inner map has unsupported value_size=%d", sourceName, info.ValueSize) //nolint:err113 + } +} + +func migrateDNSAllowMap(sourceName string) error { + source, err := ebpf.LoadPinnedMap(pinPath(sourceName), nil) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("load legacy %s failed: %w", sourceName, err) + } + defer source.Close() + if err := verifyMapLayout(source, sourceName, ebpf.HashOfMaps, uint32(unsafe.Sizeof(uint32(0))), uint32(unsafe.Sizeof(uint32(0)))); err != nil { return err } - inner, err := lookupInnerMap(current, ifindex) + current, err := loadPinnedMap(MapNameDNSAllowV2) if err != nil { return err } - defer inner.Close() - - var key lpmKey - var oldValue uint32 - iter := legacyInner.Iterate() - for iter.Next(&key, &oldValue) { - value := netPolicyValueV2{} - var existing netPolicyValueV2 - if err := inner.Lookup(&key, &existing); err == nil { - value.Flags |= existing.Flags - } else if !errors.Is(err, ebpf.ErrKeyNotExist) { - return fmt.Errorf("lookup %s inner map failed: %w", MapNameAllowOutV2, err) - } - if err := inner.Update(&key, &value, ebpf.UpdateAny); err != nil { - return fmt.Errorf("update %s inner map failed: %w", MapNameAllowOutV2, err) + defer current.Close() + + var ifindex uint32 + var innerMapID uint32 + iter := source.Iterate() + for iter.Next(&ifindex, &innerMapID) { + if err := migrateDNSAllowInnerMap(current, ifindex, sourceName, ebpf.MapID(innerMapID)); err != nil { + return err } } if err := iter.Err(); err != nil { - return fmt.Errorf("iterate legacy %s inner map failed: %w", MapNameAllowOut, err) + return fmt.Errorf("iterate legacy %s failed: %w", sourceName, err) } return nil } +func migrateDNSAllowInnerMap(current *ebpf.Map, ifindex uint32, sourceName string, sourceInnerID ebpf.MapID) error { + source, err := ebpf.NewMapFromID(sourceInnerID) + if err != nil { + return fmt.Errorf("open legacy %s inner map failed: %w, id: %d", sourceName, err, sourceInnerID) + } + defer source.Close() + info, err := source.Info() + if err != nil { + return err + } + if info.Type != ebpf.LPMTrie || info.KeySize != uint32(unsafe.Sizeof(dnsAllowKey{})) { + return fmt.Errorf("%s inner map has incompatible ABI: type=%s key_size=%d", sourceName, info.Type, info.KeySize) //nolint:err113 + } + legacySize := uint32(unsafe.Sizeof(legacyDNSAllowValue{})) + currentSize := uint32(unsafe.Sizeof(dnsAllowValue{})) + if info.ValueSize != legacySize && info.ValueSize != currentSize { + return fmt.Errorf("%s inner map has unsupported value_size=%d", sourceName, info.ValueSize) //nolint:err113 + } + + if err := ensureDNSAllowInnerMap(current, ifindex); err != nil { + return err + } + destination, err := lookupInnerMap(current, ifindex) + if err != nil { + return err + } + defer destination.Close() + + if info.ValueSize == legacySize { + var key dnsAllowKey + var oldValue legacyDNSAllowValue + iter := source.Iterate() + for iter.Next(&key, &oldValue) { + value := dnsAllowValue{NameLen: oldValue.NameLen, Flags: oldValue.Flags} + if err := destination.Update(&key, &value, ebpf.UpdateAny); err != nil { + return fmt.Errorf("update %s inner map failed: %w", MapNameDNSAllowV2, err) + } + } + return wrapIterErr(iter.Err(), sourceName) + } + + var key dnsAllowKey + var value dnsAllowValue + iter := source.Iterate() + for iter.Next(&key, &value) { + if err := destination.Update(&key, &value, ebpf.UpdateAny); err != nil { + return fmt.Errorf("update %s inner map failed: %w", MapNameDNSAllowV2, err) + } + } + return wrapIterErr(iter.Err(), sourceName) +} + func verifyMapLayout(m *ebpf.Map, name string, wantType ebpf.MapType, wantKeySize, wantValueSize uint32) error { info, err := m.Info() if err != nil { diff --git a/CubeNet/cubevs/migration_test.go b/CubeNet/cubevs/migration_test.go new file mode 100644 index 000000000..7b6521352 --- /dev/null +++ b/CubeNet/cubevs/migration_test.go @@ -0,0 +1,458 @@ +package cubevs + +import ( + "strings" + "testing" + "unsafe" + + "github.com/cilium/ebpf" + "golang.org/x/sys/unix" +) + +func TestSkipUnsupportedL7Subnet(t *testing.T) { + ip := uint32(0x0a000000) // 10.0.0.0 + tests := []struct { + name string + prefixlen uint32 + flags uint8 + want bool + }{ + {name: "L7 subnet dropped", prefixlen: 24, flags: uint8(netPolicyFlagL7Required), want: true}, + {name: "L7 /32 host kept", prefixlen: 32, flags: uint8(netPolicyFlagL7Required), want: false}, + {name: "plain subnet kept (non-L7)", prefixlen: 24, flags: 0, want: false}, + {name: "plain /32 kept (non-L7)", prefixlen: 32, flags: 0, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + key := lpmKey{Prefixlen: tt.prefixlen, IP: ip} + if got := skipUnsupportedL7Subnet(key, tt.flags, "test"); got != tt.want { + t.Fatalf("skipUnsupportedL7Subnet(prefixlen=%d flags=%#x)=%v, want %v", + tt.prefixlen, tt.flags, got, tt.want) + } + }) + } +} + +// newDNSAllowOuterMapWithValueSize creates a dns_allow outer hash-of-maps whose +// inner LPM-trie template carries values of the given size (8-byte legacy or +// 40-byte current). +func newDNSAllowOuterMapWithValueSize(t *testing.T, valueSize uint32) *ebpf.Map { + t.Helper() + outer, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.HashOfMaps, + KeySize: uint32(unsafe.Sizeof(uint32(0))), + ValueSize: uint32(unsafe.Sizeof(uint32(0))), + MaxEntries: 1024, + InnerMap: &ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(dnsAllowKey{})), + ValueSize: valueSize, + MaxEntries: maxDNSAllowEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }, + }) + if err != nil { + t.Fatalf("create dns_allow outer map (value_size=%d): %v", valueSize, err) + } + t.Cleanup(func() { outer.Close() }) + return outer +} + +// newDNSAllowOuterMap creates the dns_allow_v2 outer hash-of-maps with an inner +// LPM-trie template matching the current (40-byte) dns_allow_value. +func newDNSAllowOuterMap(t *testing.T) *ebpf.Map { + return newDNSAllowOuterMapWithValueSize(t, uint32(unsafe.Sizeof(dnsAllowValue{}))) +} + +// newLPMInner creates a standalone LPM-trie inner map with the given value size. +func newLPMInner(t *testing.T, valueSize uint32) *ebpf.Map { + t.Helper() + m, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(dnsAllowKey{})), + ValueSize: valueSize, + MaxEntries: maxDNSAllowEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }) + if err != nil { + t.Fatalf("create LPM inner (value_size=%d): %v", valueSize, err) + } + t.Cleanup(func() { m.Close() }) + return m +} + +func mustInnerMapID(t *testing.T, m *ebpf.Map) ebpf.MapID { + t.Helper() + info, err := m.Info() + if err != nil { + t.Fatalf("get inner map info: %v", err) + } + id, ok := info.ID() + if !ok { + t.Fatal("inner map has no id") + } + return id +} + +// TestMigrateDNSAllowInnerMapFromLegacy drives the legacy (8-byte) dns_allow +// inner-map migration: a legacy value (NameLen + Flags, no ports) must land in +// the new dns_allow_v2 inner map as a 40-byte value with the same NameLen and +// Flags and PortCount == 0 (default 80/443 at match time). +func TestMigrateDNSAllowInnerMapFromLegacy(t *testing.T) { + ifindex := uint32(42) + current := newDNSAllowOuterMap(t) + + // Seed a legacy (8-byte) inner map with an L7 rule and a plain rule. + legacy := newLPMInner(t, uint32(unsafe.Sizeof(legacyDNSAllowValue{}))) + type entry struct { + domain string + flags uint8 + } + entries := []entry{ + {"api.example.com", uint8(netPolicyFlagL7Required)}, + {"static.example.com", 0}, + } + for _, e := range entries { + key, val, err := makeDNSAllowRule(e.domain, e.flags) + if err != nil { + t.Fatalf("makeDNSAllowRule %s: %v", e.domain, err) + } + lv := legacyDNSAllowValue{NameLen: val.NameLen, Flags: val.Flags} + if err := legacy.Update(&key, &lv, ebpf.UpdateAny); err != nil { + t.Fatalf("seed legacy entry %s: %v", e.domain, err) + } + } + + if err := migrateDNSAllowInnerMap(current, ifindex, "test-legacy", mustInnerMapID(t, legacy)); err != nil { + t.Fatalf("migrateDNSAllowInnerMap: %v", err) + } + + dest, err := lookupInnerMap(current, ifindex) + if err != nil { + t.Fatalf("lookupInnerMap: %v", err) + } + defer dest.Close() + + for _, e := range entries { + key, want, err := makeDNSAllowRule(e.domain, e.flags) + if err != nil { + t.Fatalf("makeDNSAllowRule %s: %v", e.domain, err) + } + var got dnsAllowValue + if err := dest.Lookup(&key, &got); err != nil { + t.Fatalf("migrated entry %s missing: %v", e.domain, err) + } + if got.NameLen != want.NameLen || got.Flags != want.Flags { + t.Fatalf("entry %s: got NameLen=%d Flags=%d, want NameLen=%d Flags=%d", + e.domain, got.NameLen, got.Flags, want.NameLen, want.Flags) + } + if got.PortCount != 0 { + t.Fatalf("entry %s: PortCount=%d, want 0 (legacy rules carry no ports)", e.domain, got.PortCount) + } + } +} + +// TestMigrateDNSAllowInnerMapFromCurrent drives the already-current (40-byte) +// path: an entry carrying an explicit (port, scheme) set must be copied through +// with its ports preserved. +func TestMigrateDNSAllowInnerMapFromCurrent(t *testing.T) { + ifindex := uint32(43) + current := newDNSAllowOuterMap(t) + + cur := newLPMInner(t, uint32(unsafe.Sizeof(dnsAllowValue{}))) + key, val, err := makeDNSAllowRule("api.example.com", uint8(netPolicyFlagL7Required)) + if err != nil { + t.Fatalf("makeDNSAllowRule: %v", err) + } + val.PortCount = 2 + val.Ports[0] = l7PortEntry{Port: htonsPort(8080), Scheme: L7SchemeHTTP} + val.Ports[1] = l7PortEntry{Port: htonsPort(8443), Scheme: L7SchemeHTTPS} + if err := cur.Update(&key, &val, ebpf.UpdateAny); err != nil { + t.Fatalf("seed current entry: %v", err) + } + + if err := migrateDNSAllowInnerMap(current, ifindex, "test-current", mustInnerMapID(t, cur)); err != nil { + t.Fatalf("migrateDNSAllowInnerMap: %v", err) + } + + dest, err := lookupInnerMap(current, ifindex) + if err != nil { + t.Fatalf("lookupInnerMap: %v", err) + } + defer dest.Close() + + var got dnsAllowValue + if err := dest.Lookup(&key, &got); err != nil { + t.Fatalf("migrated entry missing: %v", err) + } + if got.NameLen != val.NameLen || got.Flags != val.Flags || got.PortCount != val.PortCount { + t.Fatalf("got NameLen=%d Flags=%d PortCount=%d, want NameLen=%d Flags=%d PortCount=%d", + got.NameLen, got.Flags, got.PortCount, val.NameLen, val.Flags, val.PortCount) + } + if got.Ports[0] != val.Ports[0] || got.Ports[1] != val.Ports[1] { + t.Fatalf("ports not preserved: got %+v, want %+v", got.Ports[:2], val.Ports[:2]) + } +} + +// mountBpffs mounts a fresh bpffs at a temp dir, points bpfFSPath at it, and +// registers cleanup to restore the path and unmount. Skips when unavailable. +func mountBpffs(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := unix.Mount("bpf", dir, "bpf", 0, ""); err != nil { + t.Skipf("bpffs mount unavailable: %v", err) + } + t.Cleanup(func() { _ = unix.Unmount(dir, 0) }) + oldPath := bpfFSPath + bpfFSPath = dir + t.Cleanup(func() { bpfFSPath = oldPath }) + return dir +} + +// TestMigrateDNSAllowMapFailureKeepsLegacyPin covers the rollback path: when +// migration fails partway, the legacy source pin must be preserved so the next +// restart can retry. It pins a legacy dns_allow whose inner uses an +// unsupported 48-byte value layout (the abandoned intermediate format), so +// migrateDNSAllowInnerMap's layout check fails — then asserts +// migratePersistentPolicyMaps errors AND the legacy pin is still present. +func TestMigrateDNSAllowMapFailureKeepsLegacyPin(t *testing.T) { + mountBpffs(t) + ifindex := uint32(42) + + // Pin a legacy dns_allow outer whose inner carries an unsupported 48-byte + // value layout (abandoned intermediate format, not migratable). + badOuter := newDNSAllowOuterMapWithValueSize(t, 48) + badInner := newLPMInner(t, 48) + key, _, err := makeDNSAllowRule("api.example.com", uint8(netPolicyFlagL7Required)) + if err != nil { + t.Fatalf("makeDNSAllowRule: %v", err) + } + var raw48 [48]byte + if err := badInner.Update(&key, &raw48, ebpf.UpdateAny); err != nil { + t.Fatalf("seed 48-byte entry: %v", err) + } + if err := badOuter.Put(&ifindex, badInner); err != nil { + t.Fatalf("attach bad inner: %v", err) + } + badInner.Close() + if err := badOuter.Pin(pinPath(MapNameDNSAllow)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameDNSAllow, err) + } + + // Pin a fresh dns_allow_v2 outer as the (never-to-be-populated) destination. + newOuter := newDNSAllowOuterMap(t) + if err := newOuter.Pin(pinPath(MapNameDNSAllowV2)); err != nil { + t.Fatalf("pin %s: %v", MapNameDNSAllowV2, err) + } + + // Migration must fail on the unsupported 48-byte inner layout... + err = migratePersistentPolicyMaps() + if err == nil { + t.Fatal("migratePersistentPolicyMaps succeeded, want failure for unsupported inner value_size") + } + + // ...and the legacy source pin must be preserved for retry (rollback). + src, err := ebpf.LoadPinnedMap(pinPath(MapNameDNSAllow), nil) + if err != nil { + t.Fatalf("legacy %s pin was removed despite failed migration (rollback broken): %v", + MapNameDNSAllow, err) + } + src.Close() +} + +// TestMigrateDNSAllowMapOuterWithBpffs exercises the full outer-map migration: +// a legacy dns_allow hash-of-maps pinned on bpffs (with an 8-byte inner LPM +// trie) is migrated by migrateDNSAllowMap into a fresh dns_allow_v2 outer map, +// and the legacy pin is then removed by removePinnedMap. It mounts a fresh +// bpffs at a temp dir and points bpfFSPath at it, so it runs without touching +// the real /sys/fs/bpf. +func TestMigrateDNSAllowMapOuterWithBpffs(t *testing.T) { + mountBpffs(t) + ifindex := uint32(42) + + // Pin a legacy dns_allow outer map (8-byte inner) seeded with two rules. + legacyOuter := newDNSAllowOuterMapWithValueSize(t, uint32(unsafe.Sizeof(legacyDNSAllowValue{}))) + legacyInner := newLPMInner(t, uint32(unsafe.Sizeof(legacyDNSAllowValue{}))) + type entry struct { + domain string + flags uint8 + } + entries := []entry{ + {"api.example.com", uint8(netPolicyFlagL7Required)}, + {"static.example.com", 0}, + } + for _, e := range entries { + key, val, err := makeDNSAllowRule(e.domain, e.flags) + if err != nil { + t.Fatalf("makeDNSAllowRule %s: %v", e.domain, err) + } + lv := legacyDNSAllowValue{NameLen: val.NameLen, Flags: val.Flags} + if err := legacyInner.Update(&key, &lv, ebpf.UpdateAny); err != nil { + t.Fatalf("seed legacy entry %s: %v", e.domain, err) + } + } + if err := legacyOuter.Put(&ifindex, legacyInner); err != nil { + t.Fatalf("attach legacy inner: %v", err) + } + legacyInner.Close() // inner survives via the outer hash-of-maps reference + if err := legacyOuter.Pin(pinPath(MapNameDNSAllow)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameDNSAllow, err) + } + + // Pin a fresh dns_allow_v2 outer map (40-byte inner template) for the copy. + newOuter := newDNSAllowOuterMap(t) + if err := newOuter.Pin(pinPath(MapNameDNSAllowV2)); err != nil { + t.Fatalf("pin %s: %v", MapNameDNSAllowV2, err) + } + + // Run the real outer-map migration. + if err := migrateDNSAllowMap(MapNameDNSAllow); err != nil { + t.Fatalf("migrateDNSAllowMap(%s): %v", MapNameDNSAllow, err) + } + + // The new outer must now hold the migrated rules (NameLen+Flags, PortCount=0). + dest, err := lookupInnerMap(newOuter, ifindex) + if err != nil { + t.Fatalf("lookupInnerMap: %v", err) + } + defer dest.Close() + for _, e := range entries { + key, want, err := makeDNSAllowRule(e.domain, e.flags) + if err != nil { + t.Fatalf("makeDNSAllowRule %s: %v", e.domain, err) + } + var got dnsAllowValue + if err := dest.Lookup(&key, &got); err != nil { + t.Fatalf("migrated entry %s missing: %v", e.domain, err) + } + if got.NameLen != want.NameLen || got.Flags != want.Flags || got.PortCount != 0 { + t.Fatalf("entry %s: got %+v, want NameLen=%d Flags=%d PortCount=0", + e.domain, got, want.NameLen, want.Flags) + } + } + + // removePinnedMap must delete the legacy pin from the mounted bpffs. + if err := removePinnedMap(MapNameDNSAllow); err != nil { + t.Fatalf("removePinnedMap(%s): %v", MapNameDNSAllow, err) + } + if _, err := ebpf.LoadPinnedMap(pinPath(MapNameDNSAllow), nil); err == nil { + t.Fatalf("legacy %s pin still present after removePinnedMap", MapNameDNSAllow) + } +} + +// newAllowOutOuterMapWithValueSize creates an allow_out outer hash-of-maps +// whose inner LPM-trie template carries values of the given size. +func newAllowOutOuterMapWithValueSize(t *testing.T, valueSize uint32) *ebpf.Map { + t.Helper() + outer, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.HashOfMaps, + KeySize: uint32(unsafe.Sizeof(uint32(0))), + ValueSize: uint32(unsafe.Sizeof(uint32(0))), + MaxEntries: maxNetPolicyEntries, + InnerMap: &ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(lpmKey{})), + ValueSize: valueSize, + MaxEntries: maxNetPolicyEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }, + }) + if err != nil { + t.Fatalf("create allow_out outer map (value_size=%d): %v", valueSize, err) + } + t.Cleanup(func() { outer.Close() }) + return outer +} + +// newAllowOutLPMInner creates a standalone LPM-trie inner map keyed by lpmKey +// with the given value size. +func newAllowOutLPMInner(t *testing.T, valueSize uint32) *ebpf.Map { + t.Helper() + m, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(lpmKey{})), + ValueSize: valueSize, + MaxEntries: maxNetPolicyEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }) + if err != nil { + t.Fatalf("create allow_out LPM inner (value_size=%d): %v", valueSize, err) + } + t.Cleanup(func() { m.Close() }) + return m +} + +// newAllowOutV3OuterMap creates the current allow_out_v3 outer hash-of-maps +// with an inner template matching net_policy_value_v3. +func newAllowOutV3OuterMap(t *testing.T) *ebpf.Map { + t.Helper() + outer, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.HashOfMaps, + KeySize: uint32(unsafe.Sizeof(uint32(0))), + ValueSize: uint32(unsafe.Sizeof(uint32(0))), + MaxEntries: maxNetPolicyEntries, + InnerMap: &ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(lpmKeyV3{})), + ValueSize: uint32(unsafe.Sizeof(netPolicyValueV3{})), + MaxEntries: maxNetPolicyEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }, + }) + if err != nil { + t.Fatalf("create allow_out_v3 outer map: %v", err) + } + t.Cleanup(func() { outer.Close() }) + return outer +} + +// TestMigrateAllowOutMapFailureKeepsLegacyPin is the allow_out counterpart of +// TestMigrateDNSAllowMapFailureKeepsLegacyPin: a legacy allow_out_v2 outer +// pinned on bpffs whose inner carries the unsupported 48-byte value layout +// (the abandoned intermediate net_policy_value_v2 format) must make +// migratePersistentPolicyMaps fail, and the legacy source pin must survive +// for the next restart's retry (rollback). +func TestMigrateAllowOutMapFailureKeepsLegacyPin(t *testing.T) { + mountBpffs(t) + ifindex := uint32(42) + + // Pin a legacy allow_out_v2 outer whose inner carries an unsupported + // 48-byte value layout (abandoned intermediate format, not migratable). + badOuter := newAllowOutOuterMapWithValueSize(t, 48) + badInner := newAllowOutLPMInner(t, 48) + key := lpmKey{Prefixlen: 32, IP: mustParseCIDRForTest(t, "192.0.2.44/32").IP} + var raw48 [48]byte + if err := badInner.Update(&key, &raw48, ebpf.UpdateAny); err != nil { + t.Fatalf("seed 48-byte entry: %v", err) + } + if err := badOuter.Put(&ifindex, badInner); err != nil { + t.Fatalf("attach bad inner: %v", err) + } + badInner.Close() // inner survives via the outer hash-of-maps reference + if err := badOuter.Pin(pinPath(MapNameAllowOutV2)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameAllowOutV2, err) + } + + // Pin a fresh allow_out_v3 outer as the (never-to-be-populated) destination. + newOuter := newAllowOutV3OuterMap(t) + if err := newOuter.Pin(pinPath(MapNameAllowOutV3)); err != nil { + t.Fatalf("pin %s: %v", MapNameAllowOutV3, err) + } + + // Migration must fail on the unsupported 48-byte inner layout... + err := migratePersistentPolicyMaps() + if err == nil { + t.Fatal("migratePersistentPolicyMaps succeeded, want failure for unsupported inner value_size") + } + if !strings.Contains(err.Error(), "unsupported value_size") { + t.Fatalf("migration failed for an unexpected reason: %v", err) + } + + // ...and the legacy source pin must be preserved for retry (rollback). + src, err := ebpf.LoadPinnedMap(pinPath(MapNameAllowOutV2), nil) + if err != nil { + t.Fatalf("legacy %s pin was removed despite failed migration (rollback broken): %v", + MapNameAllowOutV2, err) + } + src.Close() +} diff --git a/CubeNet/cubevs/miscs.go b/CubeNet/cubevs/miscs.go index 27545d997..f3df474d5 100644 --- a/CubeNet/cubevs/miscs.go +++ b/CubeNet/cubevs/miscs.go @@ -13,8 +13,8 @@ import ( const ( typeNameU32 = "__u32" - typeNameLPMKey = "lpm_key" - typeNamePolicyValue = "net_policy_value_v2" + typeNameLPMKey = "lpm_key_v3" + typeNamePolicyValue = "net_policy_value_v3" typeNameDNSAllowKey = "dns_allow_key" typeNameDNSAllowValue = "dns_allow_value" ) @@ -31,8 +31,53 @@ func init() { _ = rlimit.RemoveMemlock() } +// Shipped defaults for the L7 skb->mark values. A zero Params field means +// "use the default"; deployments override via the install-time config. +const ( + defaultL7MarkHTTP = 0xCE010000 + defaultL7MarkHTTPS = 0xCE020000 + defaultL7MarkMask = 0xFFFF0000 +) + +// resolveL7Marks returns the effective L7 mark values shared by the dataplane +// and iptables, applying shipped defaults for any field left at zero and +// validating the result: http must differ from https, and both may only set +// bits inside the mask. +func resolveL7Marks(p Params) (http, https, mask uint32, err error) { + http, https, mask = p.L7MarkHTTP, p.L7MarkHTTPS, p.L7MarkMask + if http == 0 { + http = defaultL7MarkHTTP + } + if https == 0 { + https = defaultL7MarkHTTPS + } + if mask == 0 { + mask = defaultL7MarkMask + } + if http == https { + return 0, 0, 0, fmt.Errorf("l7 mark http %#x must differ from https %#x", http, https) //nolint:err113 + } + if http&^mask != 0 || https&^mask != 0 { + return 0, 0, 0, fmt.Errorf("l7 marks http %#x https %#x must set bits only within mask %#x", http, https, mask) //nolint:err113 + } + return http, https, mask, nil +} + func rewriteConstants(vars map[string]*ebpf.VariableSpec, params Params) error { var err error + l7HTTP, l7HTTPS, l7Mask, l7Err := resolveL7Marks(params) + if l7Err != nil { + return l7Err + } + if v := vars[globalNameCubeL7MarkHTTP]; v != nil { + err = errors.Join(err, v.Set(l7HTTP)) + } + if v := vars[globalNameCubeL7MarkHTTPS]; v != nil { + err = errors.Join(err, v.Set(l7HTTPS)) + } + if v := vars[globalNameCubeL7MarkMask]; v != nil { + err = errors.Join(err, v.Set(l7Mask)) + } err = errors.Join(err, vars[globalNameMVMInnerIP].Set(ipToUint32(params.MVMInnerIP))) err = errors.Join(err, vars[globalNameMVMMacaddrP1].Set(hardwareAddrToUint32(params.MVMMacAddr))) err = errors.Join(err, vars[globalNameMVMMacaddrP2].Set(hardwareAddrToUint16(params.MVMMacAddr))) @@ -226,13 +271,44 @@ func attachTCFilter(progName string, ifindex uint32, direction TCDirection) erro return nil } +func persistentPolicyGenerationExists() (bool, error) { + allowExists := false + if _, err := os.Stat(pinPath(MapNameAllowOutV3)); err == nil { + allowExists = true + } else if !errors.Is(err, os.ErrNotExist) { + return false, err + } + dnsExists := false + if _, err := os.Stat(pinPath(MapNameDNSAllowV2)); err == nil { + dnsExists = true + } else if !errors.Is(err, os.ErrNotExist) { + return false, err + } + if allowExists != dnsExists { + return false, fmt.Errorf("incomplete policy map generation: %s exists=%t, %s exists=%t", MapNameAllowOutV3, allowExists, MapNameDNSAllowV2, dnsExists) //nolint:err113 + } + return allowExists, nil +} + // Init should be called once before invoking any other CubeVS APIs. -func Init(params Params) error { +func Init(params Params) (retErr error) { + generationExists, err := persistentPolicyGenerationExists() + if err != nil { + return err + } + if !generationExists { + defer func() { + if retErr != nil { + _ = os.Remove(pinPath(MapNameAllowOutV3)) // NOCC:Path Traversal() + _ = os.Remove(pinPath(MapNameDNSAllowV2)) // NOCC:Path Traversal() + } + }() + } _ = os.Remove(pinPath("tungrp_to_tuns")) // NOCC:Path Traversal() // dns_query_track is runtime pending-query state, not persisted policy. _ = os.Remove(pinPath(MapNameDNSQueryTrack)) // NOCC:Path Traversal() - err := loadObject(params, loadLocalgw, "loadLocalgw") + err = loadObject(params, loadLocalgw, "loadLocalgw") if err != nil { return err } @@ -255,8 +331,12 @@ func Init(params Params) error { return err } - if err := migrateAllowOutV1ToV2(); err != nil { - return err + if !generationExists { + if err := migratePersistentPolicyMaps(); err != nil { + // The deferred cleanup above removes the new pins on error; + // the legacy source pins are intentionally left for retry. + return err + } } // attach TC filter to cube-dev diff --git a/CubeNet/cubevs/netpolicy.go b/CubeNet/cubevs/netpolicy.go index 07c6192bd..29a56ad7e 100644 --- a/CubeNet/cubevs/netpolicy.go +++ b/CubeNet/cubevs/netpolicy.go @@ -25,8 +25,13 @@ var alwaysDeniedSandboxCIDRs = []string{ var alwaysDeniedSandboxEntries = mustBuildDenyOutPolicyEntries(alwaysDeniedSandboxCIDRs) type allowOutPolicyEntry struct { - key lpmKey - flags uint8 + key lpmKey + flags uint8 + // ports is inherited by dns_learn_response_ip into net_policy_value_v3.ports + // when a domain rule with an explicit (port, scheme) set is learned into + // allow_out_v3. Empty for legacy port-agnostic rules (the datapath falls + // back to the default 80/443 set in that case). + ports []l7PortEntry source string } @@ -64,11 +69,19 @@ func newInnerLPMMapWithValueSize(valueSize uint32, keyType, valueType btf.Type) } func newInnerAllowOutMap() (*ebpf.Map, error) { - return newInnerLPMMapWithValueSize(uint32(unsafe.Sizeof(netPolicyValueV2{})), btfTypeLPMKey, btfTypePolicyValue) + return ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(lpmKeyV3{})), + ValueSize: uint32(unsafe.Sizeof(netPolicyValueV3{})), + MaxEntries: maxNetPolicyEntries, + Flags: unix.BPF_F_NO_PREALLOC, + Key: btfTypeLPMKey, + Value: btfTypePolicyValue, + }) } -func ensureAllowOutV2InnerMap(outerMap *ebpf.Map, ifindex uint32) error { - return ensureInnerMapWithFactory(outerMap, ifindex, MapNameAllowOutV2, newInnerAllowOutMap) +func ensureAllowOutV3InnerMap(outerMap *ebpf.Map, ifindex uint32) error { + return ensureInnerMapWithFactory(outerMap, ifindex, MapNameAllowOutV3, newInnerAllowOutMap) } func ensureDenyOutInnerMap(outerMap *ebpf.Map, ifindex uint32) error { @@ -104,16 +117,16 @@ func ensureInnerMapWithFactory(outerMap *ebpf.Map, ifindex uint32, mapName strin } // initNetPolicy creates inner LPM trie maps for the given ifindex -// in allow_out_v2, deny_out and dns_allow hash-of-maps, if not already present. +// in allow_out_v3, deny_out and dns_allow_v2 hash-of-maps, if not already present. // This should be called during AttachFilter. func initNetPolicy(ifindex uint32) error { - allowOut, err := loadPinnedMap(MapNameAllowOutV2) + allowOut, err := loadPinnedMap(MapNameAllowOutV3) if err != nil { return err } defer allowOut.Close() - err = ensureAllowOutV2InnerMap(allowOut, ifindex) + err = ensureAllowOutV3InnerMap(allowOut, ifindex) if err != nil { return err } @@ -139,7 +152,7 @@ func flushInnerMap(outerMap *ebpf.Map, ifindex uint32) error { } func flushAllowOutInnerMap(outerMap *ebpf.Map, ifindex uint32) error { - return flushInnerMapWithValue(outerMap, ifindex, new(netPolicyValueV2)) + return flushInnerMapWithValue(outerMap, ifindex, new(netPolicyValueV3)) } func flushInnerMapWithValue(outerMap *ebpf.Map, ifindex uint32, value any) error { @@ -186,10 +199,10 @@ func lookupInnerMap(outerMap *ebpf.Map, ifindex uint32) (*ebpf.Map, error) { } // cleanupNetPolicy flushes all entries in the inner LPM trie maps -// for the given ifindex in both allow_out_v2 and deny_out. -// This should be called during DeleteTAPDevice. +// for the given ifindex in both allow_out_v3 and deny_out. +// This should be called during DelTAPDevice. func cleanupNetPolicy(ifindex uint32) error { - allowOut, err := loadPinnedMap(MapNameAllowOutV2) + allowOut, err := loadPinnedMap(MapNameAllowOutV3) if err != nil { return err } @@ -197,7 +210,7 @@ func cleanupNetPolicy(ifindex uint32) error { err = flushAllowOutInnerMap(allowOut, ifindex) if err != nil { - return fmt.Errorf("flush %s failed: %w", MapNameAllowOutV2, err) + return fmt.Errorf("flush %s failed: %w", MapNameAllowOutV3, err) } denyOut, err := loadPinnedMap(MapNameDenyOut) @@ -289,37 +302,255 @@ func mustBuildDenyOutPolicyEntries(cidrs []string) []denyOutPolicyEntry { return entries } -func buildAllowOutPolicyEntries(allowOutCIDRs, l7AllowOutCIDRs []string) ([]allowOutPolicyEntry, error) { - entries := make([]allowOutPolicyEntry, 0, len(allowOutCIDRs)+len(l7AllowOutCIDRs)) - indexByKey := make(map[lpmKey]int, len(allowOutCIDRs)+len(l7AllowOutCIDRs)) +func buildAllowOutPolicyEntries(allowOutCIDRs []string) ([]allowOutPolicyEntry, error) { + entries := make([]allowOutPolicyEntry, 0, len(allowOutCIDRs)) + indexByKey := make(map[lpmKey]int, len(allowOutCIDRs)) - add := func(cidrs []string, flags uint8) error { - for _, cidr := range cidrs { - key, err := parseCIDR(cidr) - if err != nil { - return err + for _, cidr := range allowOutCIDRs { + key, err := parseCIDR(cidr) + if err != nil { + return nil, err + } + if _, ok := indexByKey[key]; ok { + continue + } + indexByKey[key] = len(entries) + entries = append(entries, allowOutPolicyEntry{ + key: key, + source: cidr, + }) + } + return entries, nil +} + +// ntohsPort converts a network-order uint16 back to a host-order integer for +// use in error messages. eBPF stores ports in NBO to avoid conversions on hot +// paths; users see host order in their config. +func ntohsPort(p uint16) uint16 { + return (p>>8)&0xff | (p<<8)&0xff00 +} + +// htonsPort converts host-order integer to network byte order for storage +// alongside the eBPF datapath's tcphdr->dest comparisons. +func htonsPort(p uint16) uint16 { + return (p>>8)&0xff | (p<<8)&0xff00 +} + +// expandDefaultPortSet returns the (port, scheme) tuples applied when a user +// rule omits both port and scheme: the classic {80/http, 443/https}. Kept as a +// helper so the datapath fallback in mvmtap.bpf.c (port_count==0 branch) and +// the userspace expansion stay in lockstep. +func expandDefaultPortSet() []l7PortEntry { + return []l7PortEntry{ + {Port: htonsPort(80), Scheme: L7SchemeHTTP}, + {Port: htonsPort(443), Scheme: L7SchemeHTTPS}, + } +} + +// l7TargetKind separates L7Target hosts into the two datapath maps: CIDR +// literals go to allow_out_v3 as static entries; domain names go to dns_allow_v2 +// so their (port, scheme) set follows learned IPs into allow_out_v3 at +// response time. +type l7TargetKind int + +const ( + l7KindCIDR l7TargetKind = iota + l7KindDomain +) + +func classifyL7Target(host string) (l7TargetKind, error) { + if isIPv4Target(host) { + // An L7 host must be a single host, not a subnet CIDR: the datapath + // matches exact (ip, port)/48 pairs and cannot express a subnet+port + // rule, so reject network blocks here instead of silently narrowing + // them to the network address downstream. + key, err := parseCIDR(host) + if err != nil { + return 0, err + } + if key.Prefixlen < 32 { + return 0, fmt.Errorf("invalid l7_allow_out host %s: subnet CIDR not supported for L7 rules, use a single host IP or a domain name", host) //nolint:err113 + } + return l7KindCIDR, nil + } + if strings.Contains(host, "/") { + return 0, fmt.Errorf("invalid l7_allow_out CIDR target: %s", host) //nolint:err113 + } + if net.ParseIP(host) != nil || isDottedDecimalLikeTarget(host) { + return 0, fmt.Errorf("unsupported l7_allow_out IP target: %s", host) //nolint:err113 + } + if !isDNSAllowTarget(host) { + return 0, fmt.Errorf("invalid l7_allow_out domain target: %s", host) //nolint:err113 + } + return l7KindDomain, nil +} + +// l7GroupKey returns a canonical grouping key for an L7 target host. Raw +// variants that resolve to the same datapath key — DNS names differing only by +// case or a trailing dot, CIDRs differing only by notation (e.g. "1.2.3.4" vs +// "1.2.3.4/32") — map to one key so buildL7Plan aggregates their port sets and +// detects (host, port) scheme conflicts, instead of letting them collide +// later in a last-write-wins same-key merge that silently drops ports and +// bypasses conflict detection. +func l7GroupKey(host string) (string, error) { + kind, err := classifyL7Target(host) + if err != nil { + return "", err + } + if kind == l7KindCIDR { + key, err := parseCIDR(host) + if err != nil { + return "", err + } + return fmt.Sprintf("cidr:%d:%08x", key.Prefixlen, key.IP), nil + } + // Match makeDNSAllowRule's case / trailing-dot normalisation. + return "dns:" + strings.ToLower(strings.TrimSuffix(host, ".")), nil +} + +// buildL7Plan merges rule targets by host, enforces (host, port) scheme +// consistency, and produces: +// - allow_out_v3 static entries for IP/CIDR hosts (with port_count / ports +// inherited from the merged tuple set), +// - dns_allow_v2 rules for domain hosts (with the same port set). +// +// A target with Port == 0 && Scheme == L7SchemeNone means "unspecified": it +// expands to {80/http, 443/https} for merging purposes. Empty port_count is +// only preserved when *every* rule for that host is unspecified — the moment a +// user attaches an explicit (port, scheme) the port set becomes explicit and +// the datapath skips the default-set fallback. +func buildL7Plan(targets []L7Target) (l7CIDRs []allowOutPolicyEntry, + l7DNS []dnsAllowRule, err error) { + + // Group by canonical host key (see l7GroupKey), not the raw host string, + // so that raw variants resolving to the same datapath key aggregate into + // one host. Track scheme per port to detect conflicts early. + type hostState struct { + raw string // first-seen raw host, used for output + errors + portScheme map[uint16]uint8 // key: NBO port, value: scheme + portOrder []uint16 // first appearance order for deterministic overflow + anyExplicit bool + } + byHost := make(map[string]*hostState, len(targets)) + hostOrder := make([]string, 0, len(targets)) + + for _, tgt := range targets { + host := tgt.Host + gkey, gerr := l7GroupKey(host) + if gerr != nil { + return nil, nil, gerr + } + st, ok := byHost[gkey] + if !ok { + st = &hostState{raw: host, portScheme: make(map[uint16]uint8)} + byHost[gkey] = st + hostOrder = append(hostOrder, gkey) + } + if tgt.Port == 0 && tgt.Scheme == L7SchemeNone { + // Unspecified rule — expand to default set for conflict checking. + for _, def := range expandDefaultPortSet() { + if existing, ok := st.portScheme[def.Port]; ok { + if existing != def.Scheme { + return nil, nil, fmt.Errorf( + "l7 rule conflict for %s port %d: scheme %d vs %d (via default set)", //nolint:err113 + host, ntohsPort(def.Port), existing, def.Scheme) + } + } else { + st.portOrder = append(st.portOrder, def.Port) + } + st.portScheme[def.Port] = def.Scheme + } + continue + } + if tgt.Port == 0 || (tgt.Scheme != L7SchemeHTTP && tgt.Scheme != L7SchemeHTTPS) { + return nil, nil, fmt.Errorf( + "invalid l7 target %s: port and scheme must both be set (port=%d scheme=%d)", //nolint:err113 + host, tgt.Port, tgt.Scheme) + } + port := htonsPort(tgt.Port) + if existing, ok := st.portScheme[port]; ok { + if existing != tgt.Scheme { + return nil, nil, fmt.Errorf( + "l7 rule conflict for %s port %d: scheme %d vs %d", //nolint:err113 + host, tgt.Port, existing, tgt.Scheme) } - if idx, ok := indexByKey[key]; ok { - entries[idx].flags |= flags - continue + } else { + st.portOrder = append(st.portOrder, port) + } + st.portScheme[port] = tgt.Scheme + st.anyExplicit = true + } + + // Materialise per-host port entries. Order host iteration to keep output + // deterministic (test assertions compare slices directly). hostOrder holds + // canonical group keys; the first-seen raw host drives classification and + // key construction (it resolves to the same datapath key as the group). + for _, gkey := range hostOrder { + st := byHost[gkey] + host := st.raw + var ports []l7PortEntry + if st.anyExplicit || len(st.portScheme) != len(expandDefaultPortSet()) { + // Emit the full port list when an explicit (port, scheme) rule is + // attached. The second disjunct is defensive: it is unreachable + // today (a pure-default host always has exactly the default set's + // two tuples) but guards against expandDefaultPortSet() changing + // cardinality, which would otherwise silently drop a tuple here. + ports = make([]l7PortEntry, 0, len(st.portOrder)) + for _, p := range st.portOrder { + ports = append(ports, l7PortEntry{Port: p, Scheme: st.portScheme[p]}) + } + if len(ports) > maxL7PortsPerHost { + return nil, nil, fmt.Errorf( + "l7 rule exceeds %d port tuples for %s", //nolint:err113 + maxL7PortsPerHost, host) + } + } + // ports == nil for pure-default hosts: signal "use default set" to the + // datapath (port_count = 0). + + kind, cerr := classifyL7Target(host) + if cerr != nil { + return nil, nil, cerr + } + switch kind { + case l7KindCIDR: + key, perr := parseCIDR(host) + if perr != nil { + return nil, nil, perr + } + l7CIDRs = append(l7CIDRs, allowOutPolicyEntry{ + key: key, + flags: uint8(netPolicyFlagL7Required), + ports: ports, + source: host, + }) + case l7KindDomain: + key, value, merr := makeDNSAllowRule(host, uint8(netPolicyFlagL7Required)) + if merr != nil { + return nil, nil, merr } - indexByKey[key] = len(entries) - entries = append(entries, allowOutPolicyEntry{ + applyPortsToDNSAllowValue(&value, ports) + l7DNS = append(l7DNS, dnsAllowRule{ key: key, - flags: flags, - source: cidr, + value: value, + domain: host, }) } - return nil } + return l7CIDRs, l7DNS, nil +} - if err := add(allowOutCIDRs, 0); err != nil { - return nil, err +// applyPortsToDNSAllowValue copies ports into the DNS allow value. len(ports)==0 +// leaves PortCount=0 so the datapath falls back to {80,443} at match time. +func applyPortsToDNSAllowValue(v *dnsAllowValue, ports []l7PortEntry) { + if len(ports) == 0 { + v.PortCount = 0 + return } - if err := add(l7AllowOutCIDRs, uint8(netPolicyFlagL7Required)); err != nil { - return nil, err + v.PortCount = uint8(len(ports)) + for i, p := range ports { + v.Ports[i] = p } - return entries, nil } func buildDenyOutPolicyEntries(cidrs []string) ([]denyOutPolicyEntry, error) { @@ -352,23 +583,30 @@ func buildNetPolicyPlan(opts MVMOptions) (*netPolicyPlan, error) { return nil, err } - var l7AllowOut []string + var l7Targets []L7Target if opts.L7AllowOut != nil { - l7AllowOut = *opts.L7AllowOut + l7Targets = *opts.L7AllowOut } - l7AllowOutCIDRs, l7DNSAllowDomains, err := splitAllowOutTargets(l7AllowOut) + l7CIDRs, l7DNSRules, err := buildL7Plan(l7Targets) if err != nil { return nil, err } - allowOutEntries, err := buildAllowOutPolicyEntries(allowOutCIDRs, l7AllowOutCIDRs) + baseAllowOutEntries, err := buildAllowOutPolicyEntries(allowOutCIDRs) if err != nil { return nil, err } - dnsAllowRules, err := buildDNSAllowRules(dnsAllowDomains, l7DNSAllowDomains) + // Merge non-L7 allow_out CIDRs with L7 CIDRs. If the same key appears in + // both, keep the L7 entry (its flags + ports are strictly a superset). + allowOutEntries := mergeAllowOutWithL7(baseAllowOutEntries, l7CIDRs) + + // Domain rules: non-L7 domains get flags=0 and no port set; L7 domain + // rules already carry flags + ports from buildL7Plan. Merge by key. + dnsAllowRules, err := buildDNSAllowRules(dnsAllowDomains) if err != nil { return nil, err } + dnsAllowRules = mergeDNSAllowRules(dnsAllowRules, l7DNSRules) var denyOutEntries []denyOutPolicyEntry if opts.AllowInternetAccess != nil && !*opts.AllowInternetAccess { @@ -389,7 +627,7 @@ func buildNetPolicyPlan(opts MVMOptions) (*netPolicyPlan, error) { allowOutEntries: allowOutEntries, dnsAllowRules: dnsAllowRules, denyOutEntries: denyOutEntries, - dnsPolicyFlags: dnsPolicyFlagsForDomains(dnsAllowDomains, l7DNSAllowDomains), + dnsPolicyFlags: dnsPolicyFlagsForDomains(dnsAllowDomains, l7DNSDomainNames(l7DNSRules)), } if err := validateNetPolicyPlan(plan); err != nil { return nil, err @@ -397,6 +635,67 @@ func buildNetPolicyPlan(opts MVMOptions) (*netPolicyPlan, error) { return plan, nil } +// l7DNSDomainNames pulls the raw domain string out of each L7 dns rule so +// dnsPolicyFlagsForDomains can decide whether to enable DNS learning. Only the +// count matters — the caller only tests len>0. +func l7DNSDomainNames(rules []dnsAllowRule) []string { + out := make([]string, 0, len(rules)) + for _, r := range rules { + out = append(out, r.domain) + } + return out +} + +// mergeAllowOutWithL7 combines the non-L7 base entries with the L7 entries. +// When the same LPM key appears in both, the L7 version wins (its flags and +// port set describe a strictly wider policy). +func mergeAllowOutWithL7(base, l7 []allowOutPolicyEntry) []allowOutPolicyEntry { + if len(l7) == 0 { + return base + } + byKey := make(map[lpmKey]int, len(base)+len(l7)) + out := make([]allowOutPolicyEntry, 0, len(base)+len(l7)) + for _, e := range base { + byKey[e.key] = len(out) + out = append(out, e) + } + for _, e := range l7 { + if idx, ok := byKey[e.key]; ok { + out[idx].flags |= e.flags + out[idx].ports = e.ports + continue + } + byKey[e.key] = len(out) + out = append(out, e) + } + return out +} + +// mergeDNSAllowRules combines the non-L7 domain rules with the L7 domain rules. +// Same-key entries merge flags (OR) and adopt the L7 rule's port set. +func mergeDNSAllowRules(base, l7 []dnsAllowRule) []dnsAllowRule { + if len(l7) == 0 { + return base + } + byKey := make(map[dnsAllowKey]int, len(base)+len(l7)) + out := make([]dnsAllowRule, 0, len(base)+len(l7)) + for _, r := range base { + byKey[r.key] = len(out) + out = append(out, r) + } + for _, r := range l7 { + if idx, ok := byKey[r.key]; ok { + out[idx].value.Flags |= r.value.Flags + out[idx].value.PortCount = r.value.PortCount + out[idx].value.Ports = r.value.Ports + continue + } + byKey[r.key] = len(out) + out = append(out, r) + } + return out +} + func appendDenyOutPolicyEntries(dst, src []denyOutPolicyEntry) []denyOutPolicyEntry { if len(src) == 0 { return dst @@ -424,8 +723,32 @@ func effectiveDenyOutEntriesForReplace(plan *netPolicyPlan) []denyOutPolicyEntry return appendDenyOutPolicyEntries(entries, alwaysDeniedSandboxEntries) } +// expandedAllowOutEntryCount returns the number of allow_out_v3 inner-map +// entries a plan actually occupies: one per plain (non-L7) allow, and +// len(ports) per L7 rule — or the default {80, 443} set when the L7 rule has +// no explicit ports. The inner LPM trie is bounded by maxNetPolicyEntries, so +// budget validation must use this expanded count, not len(allowOutEntries) +// (which counts one per host and undercounts multi-port L7 rules, letting +// population overflow mid-write with E2BIG and leave a half-populated map). +func expandedAllowOutEntryCount(entries []allowOutPolicyEntry) int { + defaultPorts := len(expandDefaultPortSet()) + total := 0 + for _, e := range entries { + if e.flags&netPolicyFlagL7Required != 0 { + n := len(e.ports) + if n == 0 { + n = defaultPorts + } + total += n + continue + } + total++ + } + return total +} + func validateNetPolicyPlan(plan *netPolicyPlan) error { - if err := validateNetPolicyEntryCount("network.allow_out_v2", len(plan.allowOutEntries), maxNetPolicyEntries); err != nil { + if err := validateNetPolicyEntryCount("network.allow_out_v3", expandedAllowOutEntryCount(plan.allowOutEntries), maxNetPolicyEntries); err != nil { return err } if err := validateNetPolicyEntryCount("network.dns_allow", len(plan.dnsAllowRules), maxDNSAllowDomains); err != nil { @@ -434,6 +757,56 @@ func validateNetPolicyPlan(plan *netPolicyPlan) error { return validateNetPolicyEntryCount("network.deny_out", len(effectiveDenyOutEntriesForReplace(plan)), maxNetPolicyEntries) } +func validateNetPolicyEntryCounts(allowOutCIDRs, l7AllowOutCIDRs, dnsAllowDomains, l7DNSAllowDomains, denyOut []string) error { + if count, err := countUniqueLPMEntries(allowOutCIDRs, l7AllowOutCIDRs); err != nil { + return err + } else if err := validateNetPolicyEntryCount("network.allow_out_v3", count, maxNetPolicyEntries); err != nil { + return err + } + + if count, err := countUniqueDNSAllowEntries(dnsAllowDomains, l7DNSAllowDomains); err != nil { + return err + } else if err := validateNetPolicyEntryCount("network.dns_allow", count, maxDNSAllowDomains); err != nil { + return err + } + + if count, err := countUniqueLPMEntries(denyOut); err != nil { + return err + } else if err := validateNetPolicyEntryCount("network.deny_out", count, maxNetPolicyEntries); err != nil { + return err + } + + return nil +} + +func countUniqueLPMEntries(groups ...[]string) (int, error) { + seen := make(map[lpmKey]struct{}) + for _, group := range groups { + for _, cidr := range group { + key, err := parseCIDR(cidr) + if err != nil { + return 0, err + } + seen[key] = struct{}{} + } + } + return len(seen), nil +} + +func countUniqueDNSAllowEntries(groups ...[]string) (int, error) { + seen := make(map[dnsAllowKey]struct{}) + for _, group := range groups { + for _, domain := range group { + key, _, err := makeDNSAllowRule(domain, 0) + if err != nil { + return 0, err + } + seen[key] = struct{}{} + } + } + return len(seen), nil +} + func validateNetPolicyEntryCount(field string, count int, maxEntries int) error { if count <= maxEntries { return nil @@ -474,7 +847,7 @@ func setDNSPolicyFlags(ifindex uint32, flags uint8) error { } // splitAllowOutTargets separates user-facing allow_out targets into IPv4/CIDR -// entries for allow_out_v2 and DNS names for dns_allow. +// entries for allow_out_v3 and DNS names for dns_allow_v2. func splitAllowOutTargets(targets []string) ([]string, []string, error) { cidrs := make([]string, 0, len(targets)) domains := make([]string, 0, len(targets)) @@ -585,7 +958,18 @@ func populateInnerMap(outerMap *ebpf.Map, ifindex uint32, entries []denyOutPolic return nil } -// populateAllowOutInnerMap inserts pre-parsed static allow_out_v2 entries. +// populateAllowOutInnerMap inserts pre-parsed static allow_out_v3 entries. +// +// v3 carries the port in the LPM key, so an L7 entry (flags & +// netPolicyFlagL7Required) is materialised as one exact (ip, port)/48 entry +// per (port, scheme) tuple — a default port set expands to {80/http, +// 443/https}. A plain allow is a single ip-only (or subnet) /32 entry +// with scheme = NONE. When a static (ip, port) key already holds a +// DNS-learned entry, the learned flags are merged in but the static (zero) +// expiry wins — the entry becomes permanent. Keeping the learned TTL +// instead would let the reaper delete the entry at the old TTL and +// silently drop the static verdict; a later DNS refresh preserves the +// static zero expiry (dns_response.h same-key rule). func populateAllowOutInnerMap(outerMap *ebpf.Map, ifindex uint32, entries []allowOutPolicyEntry) error { var innerMapID uint32 err := outerMap.Lookup(&ifindex, &innerMapID) @@ -600,26 +984,52 @@ func populateAllowOutInnerMap(outerMap *ebpf.Map, ifindex uint32, entries []allo defer inner.Close() for _, entry := range entries { - val := netPolicyValueV2{Flags: entry.flags} - var oldVal netPolicyValueV2 - if err := inner.Lookup(&entry.key, &oldVal); err == nil { - // Static allow entries never expire, but they must preserve existing flags. - val.Flags |= oldVal.Flags - } else if !errors.Is(err, ebpf.ErrKeyNotExist) { - return fmt.Errorf("inner map lookup failed: %w, cidr: %s", err, entry.source) + if entry.flags&netPolicyFlagL7Required != 0 { + ports := entry.ports + if len(ports) == 0 { + ports = expandDefaultPortSet() + } + for _, p := range ports { + key := lpmKeyV3{Prefixlen: 48, IP: entry.key.IP, Port: p.Port} + val := netPolicyValueV3{Flags: entry.flags, Scheme: p.Scheme, KeyPrefixlen: 48} + var oldVal netPolicyValueV3 + lerr := inner.Lookup(&key, &oldVal) + switch { + case lerr == nil: + // LPM lookup is longest-prefix: only merge with + // an entry written under the EXACT same key, + // never with a shorter covering entry (whose + // flags would otherwise leak into this /48). + // Flags only: the static (zero) expiry wins over + // a learned same-key entry, so the entry becomes + // permanent rather than ageing out at the old TTL. + if oldVal.KeyPrefixlen == uint8(key.Prefixlen) { + val.Flags |= oldVal.Flags + } + case !errors.Is(lerr, ebpf.ErrKeyNotExist): + return fmt.Errorf("inner map lookup failed: %w, cidr: %s", lerr, entry.source) + } + if uerr := inner.Update(&key, &val, ebpf.UpdateAny); uerr != nil { + return fmt.Errorf("inner map update failed: %w, cidr: %s", uerr, entry.source) + } + } + continue } - err = inner.Update(&entry.key, &val, ebpf.UpdateAny) - if err != nil { - return fmt.Errorf("inner map update failed: %w, cidr: %s", err, entry.source) + // Plain allow: ip-only / subnet key with port = 0, scheme = NONE. + key := lpmKeyV3{Prefixlen: entry.key.Prefixlen, IP: entry.key.IP, Port: 0} + val := netPolicyValueV3{Flags: entry.flags, KeyPrefixlen: uint8(key.Prefixlen)} + if uerr := inner.Update(&key, &val, ebpf.UpdateAny); uerr != nil { + return fmt.Errorf("inner map update failed: %w, cidr: %s", uerr, entry.source) } } return nil } -// netPolicyValueV2Expired reports whether a v2 allow entry is a dynamic entry -// whose DNS-learned TTL has expired. Static entries have ExpiresAtNS set to 0. -func netPolicyValueV2Expired(value netPolicyValueV2, now uint64) bool { +// netPolicyValueV3Expired reports whether a v3 allow entry is a dynamic +// entry whose DNS-learned TTL has expired. Static entries have ExpiresAtNS +// set to 0. +func netPolicyValueV3Expired(value netPolicyValueV3, now uint64) bool { return value.ExpiresAtNS != 0 && value.ExpiresAtNS <= now } @@ -627,10 +1037,10 @@ func netPolicyValueV2Expired(value netPolicyValueV2, now uint64) bool { // based on MVMOptions. // // Rules: -// - AllowOut IP/CIDR targets are inserted into allow_out_v2 inner map. -// - L7AllowOut IP/CIDR targets are inserted into allow_out_v2 with the L7 flag. -// - AllowOut domain targets are inserted into dns_allow inner map. -// - L7AllowOut domain targets are inserted into dns_allow with the L7 flag. +// - AllowOut IP/CIDR targets are inserted into allow_out_v3 inner map. +// - L7AllowOut IP/CIDR targets are inserted into allow_out_v3 with the L7 flag. +// - AllowOut domain targets are inserted into dns_allow_v2 inner map. +// - L7AllowOut domain targets are inserted into dns_allow_v2 with the L7 flag. // - Default private/link-local DenyOut ranges are preloaded when a TAP enters // the free pool. Replace paths replay them after flushing policy maps. // - AllowInternetAccess=false: DenyOut is set to "0.0.0.0/0" (deny all). @@ -640,7 +1050,7 @@ func applyNetPolicy(ifindex uint32, opts MVMOptions) error { // replaceNetPolicy replaces all configured egress policy for an ifindex. // It is used by TAP upsert/recovery paths so removed policy entries do not -// survive cubelet network runtime restart/recovery. +// survive a network-agent restart. func replaceNetPolicy(ifindex uint32, opts MVMOptions) error { return applyNetPolicyWithMode(ifindex, opts, true) } @@ -652,27 +1062,27 @@ func applyNetPolicyWithMode(ifindex uint32, opts MVMOptions, replace bool) error } if replace || len(plan.allowOutEntries) > 0 { - allowOutMap, err := loadPinnedMap(MapNameAllowOutV2) + allowOutMap, err := loadPinnedMap(MapNameAllowOutV3) if err != nil { return err } defer allowOutMap.Close() - if err := ensureAllowOutV2InnerMap(allowOutMap, ifindex); err != nil { + if err := ensureAllowOutV3InnerMap(allowOutMap, ifindex); err != nil { return err } if replace { if err := flushAllowOutInnerMap(allowOutMap, ifindex); err != nil { - return fmt.Errorf("flush %s failed: %w", MapNameAllowOutV2, err) + return fmt.Errorf("flush %s failed: %w", MapNameAllowOutV3, err) } } err = populateAllowOutInnerMap(allowOutMap, ifindex, plan.allowOutEntries) if err != nil { - return fmt.Errorf("populate %s failed: %w", MapNameAllowOutV2, err) + return fmt.Errorf("populate %s failed: %w", MapNameAllowOutV3, err) } } if err := applyDNSAllow(ifindex, plan.dnsAllowRules, replace); err != nil { - return fmt.Errorf("populate %s failed: %w", MapNameDNSAllow, err) + return fmt.Errorf("populate %s failed: %w", MapNameDNSAllowV2, err) } denyOutEntries := plan.denyOutEntries diff --git a/CubeNet/cubevs/netpolicy_test.go b/CubeNet/cubevs/netpolicy_test.go index 8b47e7955..1e7f0a619 100644 --- a/CubeNet/cubevs/netpolicy_test.go +++ b/CubeNet/cubevs/netpolicy_test.go @@ -3,8 +3,12 @@ package cubevs import ( "fmt" "reflect" + "strings" "testing" "unsafe" + + "github.com/cilium/ebpf" + "golang.org/x/sys/unix" ) func TestSplitAllowOutTargets(t *testing.T) { @@ -50,7 +54,7 @@ func TestSplitAllowOutTargetsRejectsInvalidTargets(t *testing.T) { } } -func TestBuildNetPolicyPlanValidatesFinalMapTargets(t *testing.T) { +func TestValidateNetPolicyEntryCountsUsesFinalMapTargets(t *testing.T) { allowOutCIDRs := repeatedCIDRs(maxNetPolicyEntries) l7AllowOutCIDRs := []string{"198.51.100.1"} dnsAllowDomains := repeatedDomains(maxDNSAllowDomains) @@ -59,84 +63,75 @@ func TestBuildNetPolicyPlanValidatesFinalMapTargets(t *testing.T) { tests := []struct { name string - opts MVMOptions + err error want string }{ { - name: "allow out v2 counts allow and l7 cidrs", - opts: MVMOptions{AllowOut: &allowOutCIDRs, L7AllowOut: &l7AllowOutCIDRs}, - want: "network.allow_out_v2 exceeds maximum entries: got 8193, max 8192", + name: "allow out v3 counts allow and l7 cidrs", + err: validateNetPolicyEntryCounts(allowOutCIDRs, l7AllowOutCIDRs, nil, nil, nil), + want: "network.allow_out_v3 exceeds maximum entries: got 8193, max 8192", }, { name: "dns allow counts allow and l7 domains", - opts: MVMOptions{AllowOut: &dnsAllowDomains, L7AllowOut: &l7DNSAllowDomains}, + err: validateNetPolicyEntryCounts(nil, nil, dnsAllowDomains, l7DNSAllowDomains, nil), want: "network.dns_allow exceeds maximum entries: got 1025, max 1024", }, { - name: "deny out counts effective deny cidrs on replace", - opts: MVMOptions{DenyOut: &denyOut}, + name: "deny out counts effective deny cidrs", + err: validateNetPolicyEntryCounts(nil, nil, nil, nil, denyOut), want: "network.deny_out exceeds maximum entries: got 8193, max 8192", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := buildNetPolicyPlan(tt.opts) - if err == nil { - t.Fatalf("buildNetPolicyPlan returned nil error") + if tt.err == nil { + t.Fatalf("validateNetPolicyEntryCounts returned nil error") } - if got := err.Error(); got != tt.want { + if got := tt.err.Error(); got != tt.want { t.Fatalf("error=%q, want %q", got, tt.want) } }) } } -func TestBuildNetPolicyPlanDeduplicatesValidationTargetsByMapKey(t *testing.T) { - allowOut := []string{"198.51.100.1", "198.51.100.1/32", "API.Example.COM."} - l7AllowOut := []string{"198.51.100.1", "api.example.com"} - denyOut := []string{"203.0.113.1", "203.0.113.1/32"} - - if _, err := buildNetPolicyPlan(MVMOptions{AllowOut: &allowOut, L7AllowOut: &l7AllowOut, DenyOut: &denyOut}); err != nil { - t.Fatalf("buildNetPolicyPlan returned error: %v", err) +func TestValidateNetPolicyEntryCountsDeduplicatesByMapKey(t *testing.T) { + err := validateNetPolicyEntryCounts( + []string{"198.51.100.1", "198.51.100.1/32"}, + []string{"198.51.100.1"}, + []string{"API.Example.COM."}, + []string{"api.example.com"}, + []string{"203.0.113.1", "203.0.113.1/32"}, + ) + if err != nil { + t.Fatalf("validateNetPolicyEntryCounts returned error: %v", err) } } -func TestBuildDNSAllowRulesDeduplicatesAndMergesFlags(t *testing.T) { - rules, err := buildDNSAllowRules( - []string{"api.example.com", "*.github.com"}, - []string{"API.Example.COM.", "*.GitHub.com."}, - ) +func TestBuildDNSAllowRulesDeduplicatesFlagsAndPorts(t *testing.T) { + // Plain L3 domain rules go through buildDNSAllowRules; L7 domain rules + // with per-host port sets go through buildL7Plan and are merged in with + // mergeDNSAllowRules. Verify each half separately here. + base, err := buildDNSAllowRules([]string{"api.example.com", "*.github.com", "API.Example.COM."}) if err != nil { t.Fatalf("buildDNSAllowRules returned error: %v", err) } - if got, want := len(rules), 2; got != want { - t.Fatalf("len(rules)=%d, want %d", got, want) + if got, want := len(base), 2; got != want { + t.Fatalf("len(base)=%d, want %d", got, want) } - - byKey := make(map[dnsAllowKey]dnsAllowRule, len(rules)) - for _, rule := range rules { - byKey[rule.key] = rule - } - for _, domain := range []string{"api.example.com", "*.github.com"} { - key, _, err := makeDNSAllowRule(domain, 0) - if err != nil { - t.Fatalf("makeDNSAllowRule(%q) returned error: %v", domain, err) - } - rule, ok := byKey[key] - if !ok { - t.Fatalf("missing DNS allow rule for %q", domain) + for _, rule := range base { + if rule.value.Flags != 0 { + t.Fatalf("base rule %q should have no flags, got %d", rule.domain, rule.value.Flags) } - if rule.value.Flags != uint8(netPolicyFlagL7Required) { - t.Fatalf("rule %q flags=%d, want %d", domain, rule.value.Flags, netPolicyFlagL7Required) + if rule.value.PortCount != 0 { + t.Fatalf("base rule %q should have no port set, got %d", rule.domain, rule.value.PortCount) } } } -func TestBuildAllowOutPolicyEntriesDeduplicatesAndMergesFlags(t *testing.T) { +func TestBuildAllowOutPolicyEntriesDeduplicatesByKey(t *testing.T) { entries, err := buildAllowOutPolicyEntries( - []string{"198.51.100.1", "203.0.113.0/24"}, - []string{"198.51.100.1/32", "203.0.113.0/24"}, + []string{"198.51.100.1", "203.0.113.0/24", "198.51.100.1/32"}, ) if err != nil { t.Fatalf("buildAllowOutPolicyEntries returned error: %v", err) @@ -144,17 +139,21 @@ func TestBuildAllowOutPolicyEntriesDeduplicatesAndMergesFlags(t *testing.T) { if got, want := len(entries), 2; got != want { t.Fatalf("len(entries)=%d, want %d", got, want) } - - for _, entry := range entries { - if entry.flags != uint8(netPolicyFlagL7Required) { - t.Fatalf("entry %q flags=%d, want %d", entry.source, entry.flags, netPolicyFlagL7Required) - } - } + // First occurrence wins for the source label. if entries[0].source != "198.51.100.1" { - t.Fatalf("first duplicate source=%q, want first source", entries[0].source) + t.Fatalf("first source=%q, want first occurrence", entries[0].source) } if entries[1].source != "203.0.113.0/24" { - t.Fatalf("second duplicate source=%q, want first source", entries[1].source) + t.Fatalf("second source=%q", entries[1].source) + } + // Non-L7 allow_out entries carry no flags and no port set. + for _, e := range entries { + if e.flags != 0 { + t.Fatalf("entry %q flags=%d, want 0", e.source, e.flags) + } + if len(e.ports) != 0 { + t.Fatalf("entry %q ports=%v, want empty", e.source, e.ports) + } } } @@ -199,7 +198,10 @@ func TestAppendDenyOutPolicyEntriesDeduplicatesExisting(t *testing.T) { func TestBuildNetPolicyPlanDeduplicatesAndMergesFlags(t *testing.T) { allowOut := []string{"api.example.com", "198.51.100.1"} - l7AllowOut := []string{"API.Example.COM.", "198.51.100.1/32"} + l7AllowOut := []L7Target{ + {Host: "API.Example.COM."}, + {Host: "198.51.100.1/32"}, + } denyOut := []string{"192.168.0.0/16", "203.0.113.0/24"} plan, err := buildNetPolicyPlan(MVMOptions{ @@ -256,7 +258,7 @@ func TestPolicyEntryBuildersRejectBlankCIDR(t *testing.T) { { name: "allow out builder", fn: func() error { - _, err := buildAllowOutPolicyEntries([]string{" "}, nil) + _, err := buildAllowOutPolicyEntries([]string{" "}) return err }, }, @@ -315,6 +317,375 @@ func repeatedCIDRs(count int) []string { return entries } +// -------- L7 port + scheme merge tests ------------------------------------- + +func TestBuildL7Plan_DefaultRuleImpliesEmptyPortSet(t *testing.T) { + // A single rule without port/scheme should be encoded as PortCount=0 in + // the dns_allow value so the datapath falls back to {80, 443} at match + // time. This is the backward-compat path. + _, dns, err := buildL7Plan([]L7Target{{Host: "api.example.com"}}) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(dns) != 1 { + t.Fatalf("len(dns)=%d, want 1", len(dns)) + } + if got := dns[0].value.PortCount; got != 0 { + t.Fatalf("PortCount=%d, want 0 (unspecified rule keeps default set)", got) + } + if got := dns[0].value.Flags; got != uint8(netPolicyFlagL7Required) { + t.Fatalf("Flags=%d, want %d", got, netPolicyFlagL7Required) + } +} + +func TestBuildL7Plan_ExplicitPortsSetPortCount(t *testing.T) { + // Two rules: default 80/443 + explicit 8080/http. The default gets + // expanded for conflict checking, so the resulting port set is + // {80, 443, 8080}, all http/https appropriately. + _, dns, err := buildL7Plan([]L7Target{ + {Host: "api.example.com"}, + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + }) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(dns) != 1 { + t.Fatalf("len(dns)=%d, want 1 (rules for same host merge)", len(dns)) + } + if got, want := int(dns[0].value.PortCount), 3; got != want { + t.Fatalf("PortCount=%d, want %d", got, want) + } + + // Verify each expected port is present with the correct scheme. + found := map[uint16]uint8{} + for i := uint8(0); i < dns[0].value.PortCount; i++ { + p := dns[0].value.Ports[i] + found[ntohsPort(p.Port)] = p.Scheme + } + for _, expect := range []struct { + port uint16 + scheme uint8 + }{ + {80, L7SchemeHTTP}, + {443, L7SchemeHTTPS}, + {8080, L7SchemeHTTP}, + } { + if got, ok := found[expect.port]; !ok || got != expect.scheme { + t.Fatalf("port %d: got scheme %d (present=%v), want %d", + expect.port, got, ok, expect.scheme) + } + } +} + +func TestBuildL7Plan_PreservesFirstAppearancePortOrder(t *testing.T) { + _, dns, err := buildL7Plan([]L7Target{ + {Host: "api.example.com", Port: 9002, Scheme: L7SchemeHTTP}, + {Host: "api.example.com", Port: 9001, Scheme: L7SchemeHTTPS}, + {Host: "api.example.com", Port: 9003, Scheme: L7SchemeHTTP}, + {Host: "api.example.com", Port: 9001, Scheme: L7SchemeHTTPS}, + }) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(dns) != 1 { + t.Fatalf("len(dns)=%d, want 1", len(dns)) + } + want := []uint16{9002, 9001, 9003} + if got := int(dns[0].value.PortCount); got != len(want) { + t.Fatalf("PortCount=%d, want %d", got, len(want)) + } + for i, port := range want { + if got := ntohsPort(dns[0].value.Ports[i].Port); got != port { + t.Fatalf("Ports[%d]=%d, want %d", i, got, port) + } + } +} + +func TestBuildL7Plan_SchemeConflictOnSamePortRejected(t *testing.T) { + // (host, port=443) with scheme=http conflicts with the default rule's + // implicit (host, 443, https). iptables can only steer 443 to one + // listener, so this must be rejected at rule-build time. + _, _, err := buildL7Plan([]L7Target{ + {Host: "api.example.com"}, + {Host: "api.example.com", Port: 443, Scheme: L7SchemeHTTP}, + }) + if err == nil { + t.Fatal("expected scheme conflict error, got nil") + } + if !strings.Contains(err.Error(), "conflict") { + t.Fatalf("error should mention conflict, got %v", err) + } +} + +func TestBuildL7Plan_SchemeConflictOnExplicitPortRejected(t *testing.T) { + // Two explicit rules for the same (host, port) with different schemes. + _, _, err := buildL7Plan([]L7Target{ + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTPS}, + }) + if err == nil { + t.Fatal("expected scheme conflict error, got nil") + } +} + +func TestBuildL7Plan_MergesDNSHostCaseVariants(t *testing.T) { + // "API.example.com", "api.example.com." and "api.example.com" all + // normalise to the same dns_allow key (case / trailing dot). They must + // aggregate into one host so every port survives — previously each raw + // string formed its own group and the later same-key merge silently + // dropped the earlier port set. + _, dns, err := buildL7Plan([]L7Target{ + {Host: "API.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + {Host: "api.example.com.", Port: 9090, Scheme: L7SchemeHTTP}, + }) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(dns) != 1 { + t.Fatalf("len(dns)=%d, want 1 (case/dot variants merge into one host)", len(dns)) + } + found := map[uint16]uint8{} + for i := uint8(0); i < dns[0].value.PortCount; i++ { + p := dns[0].value.Ports[i] + found[ntohsPort(p.Port)] = p.Scheme + } + for _, port := range []uint16{8080, 9090} { + if _, ok := found[port]; !ok { + t.Fatalf("port %d dropped after merging host variants (got %v)", port, found) + } + } +} + +func TestBuildL7Plan_DetectsDNSSchemeConflictAcrossCaseVariants(t *testing.T) { + // Same normalised host, same port, different scheme => conflict, even + // though the raw host strings differ only by case. Previously the two + // groups passed conflict detection independently and the conflict was + // silently resolved last-write-wins. + _, _, err := buildL7Plan([]L7Target{ + {Host: "X.com", Port: 443, Scheme: L7SchemeHTTP}, + {Host: "x.com", Port: 443, Scheme: L7SchemeHTTPS}, + }) + if err == nil { + t.Fatal("buildL7Plan did not detect scheme conflict across case variants") + } + if !strings.Contains(err.Error(), "conflict") { + t.Fatalf("error should mention conflict, got %v", err) + } +} + +func TestBuildL7Plan_MergesCIDRNotationVariants(t *testing.T) { + // "1.2.3.4" and "1.2.3.4/32" parse to the same lpm key. They must + // aggregate into one CIDR entry so both ports survive. + cidrs, _, err := buildL7Plan([]L7Target{ + {Host: "1.2.3.4", Port: 8080, Scheme: L7SchemeHTTP}, + {Host: "1.2.3.4/32", Port: 9090, Scheme: L7SchemeHTTP}, + }) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(cidrs) != 1 { + t.Fatalf("len(cidrs)=%d, want 1 (notation variants merge into one host)", len(cidrs)) + } + found := map[uint16]uint8{} + for _, p := range cidrs[0].ports { + found[ntohsPort(p.Port)] = p.Scheme + } + for _, port := range []uint16{8080, 9090} { + if _, ok := found[port]; !ok { + t.Fatalf("port %d dropped after merging CIDR notation variants (got %v)", port, found) + } + } +} + +func TestBuildL7Plan_DetectsCIDRSchemeConflictAcrossNotationVariants(t *testing.T) { + // Same lpm key, same port, different scheme => conflict, even though the + // raw CIDR strings differ only by notation. + _, _, err := buildL7Plan([]L7Target{ + {Host: "1.2.3.4", Port: 443, Scheme: L7SchemeHTTP}, + {Host: "1.2.3.4/32", Port: 443, Scheme: L7SchemeHTTPS}, + }) + if err == nil { + t.Fatal("buildL7Plan did not detect scheme conflict across CIDR notation variants") + } + if !strings.Contains(err.Error(), "conflict") { + t.Fatalf("error should mention conflict, got %v", err) + } +} + +func TestBuildL7Plan_SubnetCIDRRejected(t *testing.T) { + // A subnet CIDR (prefixlen<32) is not a valid L7 host: the datapath + // matches exact (ip, port)/48 pairs and cannot express a subnet+port + // rule, so it must be rejected rather than silently narrowed to the + // network address. A /32 host and a domain remain valid. + if _, _, err := buildL7Plan([]L7Target{{Host: "10.0.0.0/24", Port: 443, Scheme: L7SchemeHTTPS}}); err == nil { + t.Fatal("buildL7Plan accepted a subnet CIDR L7 host, want rejection") + } + if _, _, err := buildL7Plan([]L7Target{{Host: "1.2.3.4", Port: 443, Scheme: L7SchemeHTTPS}}); err != nil { + t.Fatalf("single host IP (/32) must remain valid, got %v", err) + } + if _, _, err := buildL7Plan([]L7Target{{Host: "api.example.com", Port: 443, Scheme: L7SchemeHTTPS}}); err != nil { + t.Fatalf("domain host must remain valid, got %v", err) + } +} + +func TestExpandedAllowOutEntryCount(t *testing.T) { + // Plain allow -> 1 entry; default-port L7 -> 2 (80/443); explicit-port L7 + // -> len(ports). Budget validation must use this expanded count, not one + // per host. + defaultL7 := allowOutPolicyEntry{flags: uint8(netPolicyFlagL7Required)} + plain := allowOutPolicyEntry{flags: 0} + explicitL7 := allowOutPolicyEntry{ + flags: uint8(netPolicyFlagL7Required), + ports: []l7PortEntry{ + {Port: htonsPort(8080), Scheme: L7SchemeHTTP}, + {Port: htonsPort(8443), Scheme: L7SchemeHTTPS}, + }, + } + entries := []allowOutPolicyEntry{defaultL7, plain, explicitL7} + // 2 (default) + 1 (plain) + 2 (explicit) = 5 + if got := expandedAllowOutEntryCount(entries); got != 5 { + t.Fatalf("expandedAllowOutEntryCount=%d, want 5", got) + } +} + +func TestValidateNetPolicyPlanCountsExpandedL7Ports(t *testing.T) { + // Each L7 host with 8 explicit ports occupies 8 inner-map entries, not 1. + // 1025 such hosts need 8200 entries > maxNetPolicyEntries (8192); the old + // len(allowOutEntries) check counted 1025 and passed, undercounting the + // /48 expansion and letting population overflow mid-write (E2BIG). + mkPorts := func() []l7PortEntry { + ports := make([]l7PortEntry, 0, maxL7PortsPerHost) + for i := 0; i < maxL7PortsPerHost; i++ { + ports = append(ports, l7PortEntry{Port: htonsPort(uint16(8000 + i)), Scheme: L7SchemeHTTP}) + } + return ports + } + l7Entry := allowOutPolicyEntry{ + key: lpmKey{Prefixlen: 32, IP: 0x0a000001}, + flags: uint8(netPolicyFlagL7Required), + ports: mkPorts(), + } + + // 1024 hosts -> 8192 entries == budget: allowed. + plan := &netPolicyPlan{} + for i := 0; i < 1024; i++ { + plan.allowOutEntries = append(plan.allowOutEntries, l7Entry) + } + if err := validateNetPolicyPlan(plan); err != nil { + t.Fatalf("1024 8-port L7 hosts (8192 entries) should fit the budget, got %v", err) + } + + // 1025 hosts -> 8200 entries > budget: must reject. + plan.allowOutEntries = append(plan.allowOutEntries, l7Entry) + if err := validateNetPolicyPlan(plan); err == nil { + t.Fatal("1025 8-port L7 hosts (8200 entries) should exceed the budget, got nil") + } +} + +func TestBuildL7Plan_SameHostPortSchemeIsIdempotent(t *testing.T) { + // The same (host, port, scheme) tuple appearing twice is fine — different + // rules may share the tuple to attach different lua policy actions in + // CubeEgress. Only the shared datapath entry needs to be deduplicated. + _, dns, err := buildL7Plan([]L7Target{ + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + }) + if err != nil { + t.Fatalf("duplicate tuple must be idempotent, got err %v", err) + } + if len(dns) != 1 || dns[0].value.PortCount != 1 { + t.Fatalf("dns=%+v, want single rule with 1 port", dns) + } + if got := ntohsPort(dns[0].value.Ports[0].Port); got != 8080 { + t.Fatalf("port=%d, want 8080", got) + } + if dns[0].value.Ports[0].Scheme != L7SchemeHTTP { + t.Fatalf("scheme=%d, want L7SchemeHTTP", dns[0].value.Ports[0].Scheme) + } +} + +func TestBuildL7Plan_MultiHostIndependent(t *testing.T) { + // Two hosts sharing a port at different schemes must NOT conflict — the + // scheme-consistency rule is per (host, port), not per port. + _, dns, err := buildL7Plan([]L7Target{ + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeHTTP}, + {Host: "other.example.com", Port: 8080, Scheme: L7SchemeHTTPS}, + }) + if err != nil { + t.Fatalf("cross-host same-port with different schemes must be allowed: %v", err) + } + if len(dns) != 2 { + t.Fatalf("expected two dns rules, got %d", len(dns)) + } +} + +func TestBuildL7Plan_IPClassifiedAsCIDR(t *testing.T) { + // An IP-literal L7 target lands in allow_out_v3 as a static entry with + // L7_REQUIRED and the explicit (port, scheme) set copied into ports[]. + cidrs, dns, err := buildL7Plan([]L7Target{ + {Host: "1.2.3.4", Port: 8443, Scheme: L7SchemeHTTPS}, + }) + if err != nil { + t.Fatalf("buildL7Plan returned error: %v", err) + } + if len(dns) != 0 { + t.Fatalf("dns entries=%d, want 0 for IP-literal host", len(dns)) + } + if len(cidrs) != 1 { + t.Fatalf("cidrs=%d, want 1", len(cidrs)) + } + if got := cidrs[0].flags; got != uint8(netPolicyFlagL7Required) { + t.Fatalf("flags=%d, want %d", got, netPolicyFlagL7Required) + } + if len(cidrs[0].ports) != 1 || ntohsPort(cidrs[0].ports[0].Port) != 8443 { + t.Fatalf("ports=%+v, want [{Port:8443, Scheme:HTTPS}]", cidrs[0].ports) + } + if cidrs[0].ports[0].Scheme != L7SchemeHTTPS { + t.Fatalf("scheme=%d, want L7SchemeHTTPS", cidrs[0].ports[0].Scheme) + } +} + +func TestBuildL7Plan_PortBudgetExceededRejected(t *testing.T) { + // More distinct (port, scheme) tuples than maxL7PortsPerHost — must fail + // rather than silently truncate, so users see a real error at policy + // submission time instead of dropped rules at runtime. + targets := make([]L7Target, 0, maxL7PortsPerHost+1) + for i := 0; i < maxL7PortsPerHost+1; i++ { + targets = append(targets, L7Target{ + Host: "api.example.com", + Port: uint16(9000 + i), + // Half http, half https — no scheme conflict on distinct ports. + Scheme: L7SchemeHTTP, + }) + } + if _, _, err := buildL7Plan(targets); err == nil { + t.Fatal("expected budget-exceeded error, got nil") + } +} + +func TestBuildL7Plan_ExplicitRuleWithoutSchemeRejected(t *testing.T) { + // A rule with Port set but Scheme == None (or vice versa) is a schema + // error — the caller (extractL7PortScheme) is supposed to reject partial + // specifications before we get here, but defense-in-depth is cheap. + _, _, err := buildL7Plan([]L7Target{ + {Host: "api.example.com", Port: 8080, Scheme: L7SchemeNone}, + }) + if err == nil { + t.Fatal("expected error for port-without-scheme, got nil") + } +} + +func TestApplyPortsToDNSAllowValue_Empty(t *testing.T) { + // Empty ports slice must leave PortCount=0 so the datapath falls back to + // the default set. Regression guard: don't accidentally set PortCount to + // len(ports) when ports is nil. + var v dnsAllowValue + applyPortsToDNSAllowValue(&v, nil) + if v.PortCount != 0 { + t.Fatalf("PortCount=%d, want 0", v.PortCount) + } +} + func repeatedDomains(count int) []string { entries := make([]string, count) for i := range entries { @@ -324,6 +695,8 @@ func repeatedDomains(count int) []string { } func TestNetPolicyValueV2Layout(t *testing.T) { + // netPolicyValueV2 is the legacy 16-byte allow_out_v2 ABI, read only when + // migrating a pre-v3 allow_out_v2 map to allow_out_v3. var value netPolicyValueV2 if got, want := unsafe.Sizeof(value), uintptr(16); got != want { t.Fatalf("unsafe.Sizeof(netPolicyValueV2{})=%d, want %d", got, want) @@ -339,22 +712,45 @@ func TestNetPolicyValueV2Layout(t *testing.T) { } } -func TestNetPolicyValueV2Expired(t *testing.T) { +func TestNetPolicyValueV3Layout(t *testing.T) { + // netPolicyValueV3 is the current on-disk ABI for allow_out_v3. + // The port lives in the LPM key, so the value only carries the + // scheme plus the DNS-learned expiry. Any layout drift silently + // corrupts the persisted policy, so assert the exact offsets. + var value netPolicyValueV3 + if got, want := unsafe.Sizeof(value), uintptr(16); got != want { + t.Fatalf("unsafe.Sizeof(netPolicyValueV3{})=%d, want %d", got, want) + } + if got, want := unsafe.Offsetof(value.ExpiresAtNS), uintptr(0); got != want { + t.Fatalf("ExpiresAtNS offset=%d, want %d", got, want) + } + if got, want := unsafe.Offsetof(value.Flags), uintptr(8); got != want { + t.Fatalf("Flags offset=%d, want %d", got, want) + } + if got, want := unsafe.Offsetof(value.Scheme), uintptr(9); got != want { + t.Fatalf("Scheme offset=%d, want %d", got, want) + } + if got, want := unsafe.Offsetof(value.KeyPrefixlen), uintptr(10); got != want { + t.Fatalf("KeyPrefixlen offset=%d, want %d", got, want) + } +} + +func TestNetPolicyValueV3Expired(t *testing.T) { now := uint64(100) tests := []struct { name string - value netPolicyValueV2 + value netPolicyValueV3 want bool }{ - {name: "static", value: netPolicyValueV2{ExpiresAtNS: 0}, want: false}, - {name: "dynamic valid", value: netPolicyValueV2{ExpiresAtNS: now + 1}, want: false}, - {name: "dynamic expired", value: netPolicyValueV2{ExpiresAtNS: now}, want: true}, + {name: "static", value: netPolicyValueV3{ExpiresAtNS: 0}, want: false}, + {name: "dynamic valid", value: netPolicyValueV3{ExpiresAtNS: now + 1}, want: false}, + {name: "dynamic expired", value: netPolicyValueV3{ExpiresAtNS: now}, want: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := netPolicyValueV2Expired(tt.value, now); got != tt.want { - t.Fatalf("netPolicyValueV2Expired()=%t, want %t", got, tt.want) + if got := netPolicyValueV3Expired(tt.value, now); got != tt.want { + t.Fatalf("netPolicyValueV3Expired()=%t, want %t", got, tt.want) } }) } @@ -368,7 +764,7 @@ func TestMakeDNSAllowRuleSetsL7Flag(t *testing.T) { if value.Flags != uint8(netPolicyFlagL7Required) { t.Fatalf("value.Flags=%d, want %d", value.Flags, netPolicyFlagL7Required) } - if got, want := unsafe.Sizeof(value), uintptr(8); got != want { + if got, want := unsafe.Sizeof(value), uintptr(40); got != want { t.Fatalf("unsafe.Sizeof(dnsAllowValue{})=%d, want %d", got, want) } if key.Name[int(value.NameLen)-1] != 0 { @@ -427,7 +823,7 @@ func TestMVMMetadataLayoutAndDNSPolicyFlags(t *testing.T) { func TestDNSAllowValueLayoutAndFlags(t *testing.T) { var value dnsAllowValue - if got, want := unsafe.Sizeof(value), uintptr(8); got != want { + if got, want := unsafe.Sizeof(value), uintptr(40); got != want { t.Fatalf("unsafe.Sizeof(dnsAllowValue{})=%d, want %d", got, want) } if got, want := unsafe.Offsetof(value.NameLen), uintptr(0); got != want { @@ -453,3 +849,197 @@ func TestDNSAllowDuplicateRulesMergeFlags(t *testing.T) { t.Fatalf("merged Flags=%d, want %d", allowValue.Flags, netPolicyFlagL7Required) } } + +// attachAllowOutV3Inner creates an allow_out_v3 inner LPM trie and attaches +// it to the outer hash-of-maps under ifindex, returning the inner map. +func attachAllowOutV3Inner(t *testing.T, outer *ebpf.Map, ifindex uint32) *ebpf.Map { + t.Helper() + inner, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.LPMTrie, + KeySize: uint32(unsafe.Sizeof(lpmKeyV3{})), + ValueSize: uint32(unsafe.Sizeof(netPolicyValueV3{})), + MaxEntries: maxNetPolicyEntries, + Flags: unix.BPF_F_NO_PREALLOC, + }) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF LPM trie unavailable: %v", err) + } + t.Fatalf("create allow_out_v3 inner: %v", err) + } + t.Cleanup(func() { _ = inner.Close() }) + if err := outer.Put(&ifindex, inner); err != nil { + t.Fatalf("attach allow_out_v3 inner: %v", err) + } + return inner +} + +func mustLookupV3(t *testing.T, inner *ebpf.Map, key lpmKeyV3) netPolicyValueV3 { + t.Helper() + var value netPolicyValueV3 + if err := inner.Lookup(&key, &value); err != nil { + t.Fatalf("lookup %+v: %v", key, err) + } + return value +} + +// TestPopulateAllowOutStaticV3OverCoveringLearnedStaysStatic is the +// exact-key-match regression test for the userspace writer: a static /48 +// written while a DNS-learned /32 (non-zero TTL) covers the same IP must +// stay static. The inner LPM lookup is longest-prefix, so without the +// KeyPrefixlen check the learned /32's TTL was wrongly inherited and the +// "static" /48 aged out at the learned TTL. +func TestPopulateAllowOutStaticV3OverCoveringLearnedStaysStatic(t *testing.T) { + outer := newAllowOutV3OuterMap(t) + ifindex := uint32(305) + inner := attachAllowOutV3Inner(t, outer, ifindex) + ip := mustParseCIDRForTest(t, "192.0.2.80").IP + + // Pre-existing DNS-learned /32 (temporary) covering the IP. + learnedTTL := uint64(123456789) + seed := lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0} + if err := inner.Update(&seed, &netPolicyValueV3{ + ExpiresAtNS: learnedTTL, Flags: testFlagMarker, KeyPrefixlen: 32, + }, ebpf.UpdateAny); err != nil { + t.Fatalf("seed learned /32: %v", err) + } + + entries := []allowOutPolicyEntry{{ + key: lpmKey{Prefixlen: 32, IP: ip}, + flags: netPolicyFlagL7Required, + ports: []l7PortEntry{{Port: htonsPort(443), Scheme: L7SchemeHTTPS}}, + }} + if err := populateAllowOutInnerMap(outer, ifindex, entries); err != nil { + t.Fatalf("populateAllowOutInnerMap: %v", err) + } + + got := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if got.KeyPrefixlen != 48 { + t.Fatalf("static /48 KeyPrefixlen=%d, want 48", got.KeyPrefixlen) + } + if got.ExpiresAtNS != 0 { + t.Fatalf("static /48 inherited learned TTL %d from covering /32, want static (0)", got.ExpiresAtNS) + } + if got.Flags&testFlagMarker != 0 { + t.Fatalf("static /48 inherited flags from covering /32: %#x", got.Flags) + } +} + +// TestPopulateAllowOutStaticV3OverExactLearnedStaticWins is the counterpart: +// a static /48 written over a learned entry for the EXACT same key merges +// the learned flags but the static (zero) expiry WINS — the entry becomes +// permanent. Keeping the learned TTL instead would let the reaper delete +// the entry at the old TTL and silently drop the static verdict. +func TestPopulateAllowOutStaticV3OverExactLearnedStaticWins(t *testing.T) { + outer := newAllowOutV3OuterMap(t) + ifindex := uint32(306) + inner := attachAllowOutV3Inner(t, outer, ifindex) + ip := mustParseCIDRForTest(t, "192.0.2.90").IP + + learnedTTL := uint64(123456789) + seed := lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)} + if err := inner.Update(&seed, &netPolicyValueV3{ + ExpiresAtNS: learnedTTL, Flags: uint8(netPolicyFlagL7Required) | testFlagMarker, + Scheme: L7SchemeHTTP, KeyPrefixlen: 48, // stale scheme: static write must overwrite + }, ebpf.UpdateAny); err != nil { + t.Fatalf("seed learned /48: %v", err) + } + + entries := []allowOutPolicyEntry{{ + key: lpmKey{Prefixlen: 32, IP: ip}, + flags: netPolicyFlagL7Required, + ports: []l7PortEntry{{Port: htonsPort(443), Scheme: L7SchemeHTTPS}}, + }} + if err := populateAllowOutInnerMap(outer, ifindex, entries); err != nil { + t.Fatalf("populateAllowOutInnerMap: %v", err) + } + + got := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if got.ExpiresAtNS != 0 { + t.Fatalf("static write over learned /48 must win with zero expiry, got %d (learned TTL was %d)", got.ExpiresAtNS, learnedTTL) + } + if got.Flags&testFlagMarker == 0 { + t.Fatalf("exact learned /48 flags not merged: %#x", got.Flags) + } + if got.Scheme != L7SchemeHTTPS { + t.Fatalf("scheme=%d after static write, want HTTPS (last write wins)", got.Scheme) + } +} + +// TestPopulateAllowOutStaticV3OverExactStaticMergesFlags covers the +// static-over-static same-key cell: the write must union the old entry's +// flags and keep the zero (static) expiry. +func TestPopulateAllowOutStaticV3OverExactStaticMergesFlags(t *testing.T) { + outer := newAllowOutV3OuterMap(t) + ifindex := uint32(308) + inner := attachAllowOutV3Inner(t, outer, ifindex) + ip := mustParseCIDRForTest(t, "192.0.2.100").IP + + seed := lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)} + if err := inner.Update(&seed, &netPolicyValueV3{ + Flags: uint8(netPolicyFlagL7Required) | testFlagMarker, + Scheme: L7SchemeHTTPS, KeyPrefixlen: 48, + }, ebpf.UpdateAny); err != nil { + t.Fatalf("seed static /48: %v", err) + } + + entries := []allowOutPolicyEntry{{ + key: lpmKey{Prefixlen: 32, IP: ip}, + flags: netPolicyFlagL7Required, + ports: []l7PortEntry{{Port: htonsPort(443), Scheme: L7SchemeHTTPS}}, + }} + if err := populateAllowOutInnerMap(outer, ifindex, entries); err != nil { + t.Fatalf("populateAllowOutInnerMap: %v", err) + } + + got := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if got.ExpiresAtNS != 0 { + t.Fatalf("static-over-static must stay static, got expiry %d", got.ExpiresAtNS) + } + if got.Flags&testFlagMarker == 0 { + t.Fatalf("static-over-static lost old flags: %#x", got.Flags) + } + if got.Flags&uint8(netPolicyFlagL7Required) == 0 { + t.Fatalf("static-over-static lost rule flags: %#x", got.Flags) + } +} + +// TestPopulateAllowOutPlainStaticOverwritesLearnedUnconditionally pins the +// plain (non-L7) path's NO-merge semantics: a static /32 overwrites a learned +// /32 outright — last write wins for flags, scheme AND expiry, with no +// same-key preservation (the plain path does not read the old entry at all). +func TestPopulateAllowOutPlainStaticOverwritesLearnedUnconditionally(t *testing.T) { + outer := newAllowOutV3OuterMap(t) + ifindex := uint32(309) + inner := attachAllowOutV3Inner(t, outer, ifindex) + ip := mustParseCIDRForTest(t, "192.0.2.110").IP + + seed := lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0} + if err := inner.Update(&seed, &netPolicyValueV3{ + ExpiresAtNS: 123456789, Flags: testFlagMarker, KeyPrefixlen: 32, + }, ebpf.UpdateAny); err != nil { + t.Fatalf("seed learned /32: %v", err) + } + + entries := []allowOutPolicyEntry{{ + key: lpmKey{Prefixlen: 32, IP: ip}, + flags: 0, // plain allow, no L7 + }} + if err := populateAllowOutInnerMap(outer, ifindex, entries); err != nil { + t.Fatalf("populateAllowOutInnerMap: %v", err) + } + + got := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0}) + if got.KeyPrefixlen != 32 { + t.Fatalf("KeyPrefixlen=%d, want 32", got.KeyPrefixlen) + } + if got.ExpiresAtNS != 0 { + t.Fatalf("plain static overwrite kept learned TTL %d", got.ExpiresAtNS) + } + if got.Flags != 0 { + t.Fatalf("plain static overwrite kept old flags: %#x", got.Flags) + } + if got.Scheme != L7SchemeNone { + t.Fatalf("scheme=%d, want NONE", got.Scheme) + } +} diff --git a/CubeNet/cubevs/reaper.go b/CubeNet/cubevs/reaper.go index 8cc70f912..ee91c3aa3 100644 --- a/CubeNet/cubevs/reaper.go +++ b/CubeNet/cubevs/reaper.go @@ -165,7 +165,9 @@ type natSession struct { VMPort uint16 State uint8 ActiveClose uint8 - Reserved [34]uint8 + PacketClass uint8 + L7Scheme uint8 + Reserved [32]uint8 } // timeout returns the timeout for the session in nanoseconds. @@ -221,6 +223,14 @@ type ingressSessionValue struct { Reserved [3]uint16 } +//nolint:unused +func ingressSession(key *sessionKey, value *ingressSessionValue) string { + return fmt.Sprintf("%s:%d->%s:%d(%s:%d)", + uint32ToIP(key.SourceIP), ntohs(key.SourcePort), + uint32ToIP(key.TargetIP), ntohs(key.TargetPort), + uint32ToIP(value.VMIP), ntohs(value.VMPort)) +} + // StartSessionReaper starts a goroutine that will periodically // check for sessions and DNS-learned policies that have expired and remove them. func StartSessionReaper() <-chan Event { diff --git a/CubeNet/cubevs/tap.go b/CubeNet/cubevs/tap.go index 5df0491f8..c8495b10f 100644 --- a/CubeNet/cubevs/tap.go +++ b/CubeNet/cubevs/tap.go @@ -8,11 +8,22 @@ import ( "github.com/cilium/ebpf" ) +// L7Target describes one (host-or-CIDR, port, scheme) tuple carried inside +// MVMOptions.L7AllowOut. Host is either a domain (with optional wildcard) or +// an IPv4 / IPv4-CIDR literal. Port == 0 with SchemeNone signals the legacy +// default port set {80/http, 443/https}; when Port > 0, Scheme must be +// L7SchemeHTTP or L7SchemeHTTPS. +type L7Target struct { + Host string + Port uint16 // 0 = unspecified (default port set) + Scheme uint8 // one of L7SchemeNone / L7SchemeHTTP / L7SchemeHTTPS +} + type MVMOptions struct { AllowInternetAccess *bool - AllowOut *[]string // CIDR, IP, or domain - L7AllowOut *[]string // CIDR, IP, or domain that requires L7 policy handling - DenyOut *[]string // CIDR or IP + AllowOut *[]string // CIDR, IP, or domain + L7AllowOut *[]L7Target // Host + optional (port, scheme) for L7 policy handling + DenyOut *[]string // CIDR or IP } type tapMetadataMapOps interface { diff --git a/CubeNet/cubevs/tcp_state_test.go b/CubeNet/cubevs/tcp_state_test.go new file mode 100644 index 000000000..c22ccd7f9 --- /dev/null +++ b/CubeNet/cubevs/tcp_state_test.go @@ -0,0 +1,287 @@ +package cubevs + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target $GOARCH tcpstate ../src/tcp_state_test.bpf.c -- -I../vmlinux/$GOARCH + +import ( + "encoding/binary" + "errors" + "testing" + + "github.com/cilium/ebpf" + "golang.org/x/sys/unix" +) + +const ( + tcpDirOriginal = 0 + tcpDirReply = 1 + tcpTestCaseLen = 40 +) + +type tcpUpdateStep struct { + dir uint8 + syn uint8 + ack uint8 + fin uint8 + rst uint8 +} + +type tcpUpdateCase struct { + accessTime uint64 + nowNS uint64 + state uint8 + activeClose uint8 + steps []tcpUpdateStep +} + +func encodeTCPUpdateCase(tc tcpUpdateCase) []byte { + data := make([]byte, tcpTestCaseLen) + binary.LittleEndian.PutUint64(data[0:8], tc.accessTime) + binary.LittleEndian.PutUint64(data[8:16], tc.nowNS) + data[16] = tc.state + data[17] = tc.activeClose + data[18] = uint8(len(tc.steps)) + for i, step := range tc.steps { + if i >= 2 { + break + } + off := 24 + i*8 + data[off] = step.dir + data[off+1] = step.syn + data[off+2] = step.ack + data[off+3] = step.fin + data[off+4] = step.rst + } + return data +} + +func decodeTCPUpdateCase(data []byte) tcpUpdateCase { + return tcpUpdateCase{ + accessTime: binary.LittleEndian.Uint64(data[0:8]), + nowNS: binary.LittleEndian.Uint64(data[8:16]), + state: data[16], + activeClose: data[17], + } +} + +func bpfTestUnavailable(err error) bool { + var verifierErr *ebpf.VerifierError + if errors.As(err, &verifierErr) { + return false + } + return errors.Is(err, unix.EPERM) || errors.Is(err, unix.EACCES) || + errors.Is(err, ebpf.ErrNotSupported) +} + +func loadTCPStateTestProgram(t *testing.T) *ebpf.Program { + t.Helper() + + spec, err := loadTcpstate() + if err != nil { + t.Fatalf("load tcp state test spec: %v", err) + } + for name := range spec.Maps { + if name != ".rodata" { + delete(spec.Maps, name) + } + } + coll, err := ebpf.NewCollection(spec) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF test unavailable: %v", err) + } + t.Fatalf("load tcp state test collection: %v", err) + } + t.Cleanup(coll.Close) + + prog := coll.Programs["test_update_session"] + if prog == nil { + t.Fatal("test_update_session program missing") + } + return prog +} + +func runTCPUpdateCase(t *testing.T, prog *ebpf.Program, tc tcpUpdateCase) tcpUpdateCase { + t.Helper() + + ret, out, err := prog.Test(encodeTCPUpdateCase(tc)) + if err != nil { + if bpfTestUnavailable(err) { + t.Skipf("kernel BPF test-run unavailable: %v", err) + } + t.Fatalf("run tcp state test: %v", err) + } + if ret != 0 { + t.Fatalf("test_update_session returned %d, want TC_ACT_OK", ret) + } + if len(out) < tcpTestCaseLen { + t.Fatalf("test output length=%d, want >=%d", len(out), tcpTestCaseLen) + } + return decodeTCPUpdateCase(out) +} + +// TestTCPUpdateSessionBidirectionalFINOrderings deterministically covers both +// serializations of near-simultaneous FINs. It does not prove atomicity when +// two CPUs update the same map value at exactly the same time. +func TestTCPUpdateSessionBidirectionalFINOrderings(t *testing.T) { + prog := loadTCPStateTestProgram(t) + fin := func(dir uint8) tcpUpdateStep { return tcpUpdateStep{dir: dir, ack: 1, fin: 1} } + + tests := []struct { + name string + steps []tcpUpdateStep + wantState tcpConntrackState + wantActiveClose uint8 + }{ + { + name: "original FIN then reply FIN", + steps: []tcpUpdateStep{fin(tcpDirOriginal), fin(tcpDirReply)}, + wantState: tcpCTLastAck, + wantActiveClose: 1, + }, + { + name: "reply FIN then original FIN", + steps: []tcpUpdateStep{fin(tcpDirReply), fin(tcpDirOriginal)}, + wantState: tcpCTLastAck, + wantActiveClose: 0, + }, + { + name: "original FIN retransmission", + steps: []tcpUpdateStep{fin(tcpDirOriginal), fin(tcpDirOriginal)}, + wantState: tcpCTFinWait, + wantActiveClose: 1, + }, + { + name: "reply FIN retransmission", + steps: []tcpUpdateStep{fin(tcpDirReply), fin(tcpDirReply)}, + wantState: tcpCTFinWait, + wantActiveClose: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := runTCPUpdateCase(t, prog, tcpUpdateCase{ + accessTime: 1, + nowNS: uint64(2 * 1e9), + state: uint8(tcpCTEstablished), + steps: tt.steps, + }) + if got.state != uint8(tt.wantState) { + t.Fatalf("state=%s, want %s", tcpConntrackState(got.state), tt.wantState) + } + if got.activeClose != tt.wantActiveClose { + t.Fatalf("active_close=%d, want %d", got.activeClose, tt.wantActiveClose) + } + }) + } +} + +func TestTCPUpdateSessionFINRetransmissionAfterACK(t *testing.T) { + prog := loadTCPStateTestProgram(t) + fin := func(dir uint8) tcpUpdateStep { return tcpUpdateStep{dir: dir, ack: 1, fin: 1} } + ack := func(dir uint8) tcpUpdateStep { return tcpUpdateStep{dir: dir, ack: 1} } + + tests := []struct { + name string + first []tcpUpdateStep + retransmit tcpUpdateStep + wantActiveClose uint8 + }{ + { + name: "original FIN retransmission in close-wait", + first: []tcpUpdateStep{fin(tcpDirOriginal), ack(tcpDirReply)}, + retransmit: fin(tcpDirOriginal), + wantActiveClose: 1, + }, + { + name: "reply FIN retransmission in close-wait", + first: []tcpUpdateStep{fin(tcpDirReply), ack(tcpDirOriginal)}, + retransmit: fin(tcpDirReply), + wantActiveClose: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + closeWait := runTCPUpdateCase(t, prog, tcpUpdateCase{ + accessTime: 1, + nowNS: uint64(2 * 1e9), + state: uint8(tcpCTEstablished), + steps: tt.first, + }) + if closeWait.state != uint8(tcpCTCloseWait) { + t.Fatalf("precondition state=%s, want %s", tcpConntrackState(closeWait.state), tcpCTCloseWait) + } + got := runTCPUpdateCase(t, prog, tcpUpdateCase{ + accessTime: closeWait.accessTime, + nowNS: uint64(3 * 1e9), + state: closeWait.state, + activeClose: closeWait.activeClose, + steps: []tcpUpdateStep{tt.retransmit}, + }) + if got.state != uint8(tcpCTCloseWait) { + t.Fatalf("state=%s, want %s", tcpConntrackState(got.state), tcpCTCloseWait) + } + if got.activeClose != tt.wantActiveClose { + t.Fatalf("active_close=%d, want %d", got.activeClose, tt.wantActiveClose) + } + }) + } +} + +func TestTCPUpdateSessionBidirectionalFINCompletionTimeout(t *testing.T) { + prog := loadTCPStateTestProgram(t) + fin := func(dir uint8) tcpUpdateStep { return tcpUpdateStep{dir: dir, ack: 1, fin: 1} } + ack := func(dir uint8) tcpUpdateStep { return tcpUpdateStep{dir: dir, ack: 1} } + + tests := []struct { + name string + first []tcpUpdateStep + lastACK tcpUpdateStep + wantActiveClose uint8 + wantTimeout uint64 + }{ + { + name: "original initiated close", + first: []tcpUpdateStep{fin(tcpDirOriginal), fin(tcpDirReply)}, + lastACK: ack(tcpDirOriginal), + wantActiveClose: 1, + wantTimeout: uint64(tcpTimeouts[tcpCTClose].Nanoseconds()), + }, + { + name: "reply initiated close", + first: []tcpUpdateStep{fin(tcpDirReply), fin(tcpDirOriginal)}, + lastACK: ack(tcpDirReply), + wantActiveClose: 0, + wantTimeout: uint64(tcpTimeouts[tcpCTTimeWait].Nanoseconds()), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + closing := runTCPUpdateCase(t, prog, tcpUpdateCase{ + accessTime: 1, + nowNS: uint64(2 * 1e9), + state: uint8(tcpCTEstablished), + steps: tt.first, + }) + closed := runTCPUpdateCase(t, prog, tcpUpdateCase{ + accessTime: closing.accessTime, + nowNS: uint64(3 * 1e9), + state: closing.state, + activeClose: closing.activeClose, + steps: []tcpUpdateStep{tt.lastACK}, + }) + if closed.state != uint8(tcpCTTimeWait) { + t.Fatalf("state=%s, want %s", tcpConntrackState(closed.state), tcpCTTimeWait) + } + if closed.activeClose != tt.wantActiveClose { + t.Fatalf("active_close=%d, want %d", closed.activeClose, tt.wantActiveClose) + } + sess := natSession{State: closed.state, ActiveClose: closed.activeClose} + if got := sess.tcpTimeout(); got != tt.wantTimeout { + t.Fatalf("timeout=%d, want %d", got, tt.wantTimeout) + } + }) + } +} diff --git a/CubeNet/src/cubevs.h b/CubeNet/src/cubevs.h index 264d305b2..b0026d554 100644 --- a/CubeNet/src/cubevs.h +++ b/CubeNet/src/cubevs.h @@ -37,6 +37,36 @@ #define DNS_POLICY_FLAG_LEARNING_ENABLED 1 #define NET_POLICY_FLAG_L7_REQUIRED 1 #define NSEC_PER_SEC 1000000000ULL + +/* L7 scheme values embedded in dns_allow_value / net_policy_value_v2 per-port + * entries. Used by the eBPF datapath to compute skb->mark so CubeEgress's + * iptables TPROXY rules can steer HTTP vs HTTPS traffic to distinct listeners + * without depending on the destination port number. Keep in sync with + * cubevs/cubevs.go (L7SchemeHTTP / L7SchemeHTTPS). + */ +#define L7_SCHEME_NONE 0 +#define L7_SCHEME_HTTP 1 +#define L7_SCHEME_HTTPS 2 + +/* Maximum number of (port, scheme) tuples a single L7 rule host may declare. + * Bounded so the map value size stays small and BPF verifier can unroll the + * lookup loop. Users needing more should merge rules or reuse ports. + */ +#define MAX_L7_PORTS_PER_HOST 8 + +/* skb->mark encoding for L7 redirect. The high 16 bits carry a cube-owned + * prefix (0xCE?? masked by cube_l7_mark_mask) so cube marks do not collide + * with unrelated mark bits users may set elsewhere. iptables uses + * `-m mark --mark VAL/MASK` to match on cube-owned bits only. + * + * These are const volatile globals (not macros) so a deployment can override + * them at load time from userspace (rewriteConstants, sourced from the same + * install-time config the iptables init script reads), keeping the dataplane + * and iptables in lock-step. Defaults match the shipped values. + */ +const volatile __u32 cube_l7_mark_mask = 0xFFFF0000u; +const volatile __u32 cube_l7_mark_http = 0xCE010000u; +const volatile __u32 cube_l7_mark_https = 0xCE020000u; #define DNS_QUERY_TRACK_TTL_NS (10ULL * NSEC_PER_SEC) /* https://en.wikipedia.org/wiki/IPv4#Header @@ -142,15 +172,53 @@ struct lpm_key { __u32 ip; }; +/* LPM key for allow_out_v3. Carries the destination IP (32-bit, network + * byte order) and an optional destination port (16-bit, network byte order) + * so a single longest-prefix lookup resolves: + * - exact (ip, port): prefixlen = 48 (ip[4] + port[2]) + * - ip only (any port): prefixlen = 32 (ip[4], port ignored) + * - ip/mask subnet: prefixlen < 32 (only top bits of ip matter) + * The struct is padded to 12 bytes (8-byte data payload) so the trie's + * word-wise compare stays 4-byte aligned on every kernel. Insert and + * lookup MUST both fill ip/port from network-byte-order bytes (matching + * iphdr->daddr / tcphdr->dest), never from a host-byte-order integer + * shift — otherwise the exact (ip, port) match silently fails. + */ +struct lpm_key_v3 { + __u32 prefixlen; + __u32 ip; /* network byte order */ + __u16 port; /* network byte order; 0 when key is ip-only/subnet */ + __u16 _pad; /* 0; keeps the data payload at 8 bytes */ +}; + struct dns_allow_key { __u32 prefixlen; char name[MAX_DNS_NAME_LEN]; }; +/* Per-host L7 port entry. Attached inline to both dns_allow_value and + * net_policy_value_v2 so the datapath can pick the right scheme for a given + * destination port without a second map lookup. port is in network byte order + * (matches tcphdr->dest), scheme is one of L7_SCHEME_HTTP / L7_SCHEME_HTTPS. + */ +struct l7_port_entry { + __u16 port; /* network byte order */ + __u8 scheme; + __u8 _pad; +}; + +/* dns_allow_value carries the L7 policy attached to a matched DNS name. + * port_count = 0 is the "unspecified" case: the datapath applies the default + * port set {80/http, 443/https} for backward compatibility with rules that + * omit port. port_count > 0 restricts L7 handling to the listed (port, scheme) + * tuples only. + */ struct dns_allow_value { __u32 name_len; __u8 flags; - __u8 reserved[3]; + __u8 port_count; + __u8 reserved[2]; + struct l7_port_entry ports[MAX_L7_PORTS_PER_HOST]; }; struct dns_query_track_key { @@ -165,7 +233,9 @@ struct dns_query_track_key { struct dns_query_track_value { __u64 expires_at_ns; __u8 flags; - __u8 reserved[7]; + __u8 port_count; + __u8 reserved[6]; + struct l7_port_entry ports[MAX_L7_PORTS_PER_HOST]; }; /* Per-packet query parser state shared by the DNS tail-call pipeline. */ @@ -182,12 +252,36 @@ struct dns_query_state { char name[MAX_DNS_NAME_LEN]; }; +/* net_policy_value_v2 stores the per-sandbox allow_out_v2 verdict. This is the + * legacy 16-byte layout read only when migrating a pre-v3 allow_out_v2 map to + * allow_out_v3; the current dataplane uses net_policy_value_v3. + */ struct net_policy_value_v2 { __u64 expires_at_ns; __u8 flags; __u8 reserved[7]; }; +/* Per-sandbox allow_out_v3 verdict. Unlike v2, the port lives in the + * LPM key (see lpm_key_v3), so the value no longer needs the 8-tuple + * (port, scheme) array: the scheme is resolved at insert time and + * stored here directly. A zero expires_at_ns is a static entry; a + * non-zero expires_at_ns is a temporary DNS-learned entry. + */ +struct net_policy_value_v3 { + __u64 expires_at_ns; + __u8 flags; + __u8 scheme; /* L7_SCHEME_* */ + /* prefixlen of the lpm_key_v3 this value was written under. LPM trie + * lookups are longest-prefix, so a lookup for key K may return an + * entry written under a SHORTER covering key; writers that mean to + * merge with an existing entry for the EXACT same key must compare + * this field against their key's prefixlen first. + */ + __u8 key_prefixlen; + __u8 reserved[5]; +}; + struct mvm_port { __u32 ifindex; __u16 listen_port; @@ -214,7 +308,9 @@ struct nat_session { __u16 vm_port; __u8 state; __u8 active_close; - __u8 reserved[34]; + __u8 packet_class; /* SNAT_PACKET or L7PROXY_PACKET */ + __u8 l7_scheme; /* L7_SCHEME_*; NONE for non-L7 sessions */ + __u8 reserved[32]; }; struct ingress_session { @@ -275,24 +371,29 @@ static __always_inline int _() { int b[sizeof(struct mvm_meta) == 128 ? 1 : -1] = {}; int d[sizeof(struct lpm_key) == 8 ? 1 : -1] = {}; + int dv3[sizeof(struct lpm_key_v3) == 12 ? 1 : -1] = {}; int r[sizeof(struct net_policy_value_v2) == 16 ? 1 : -1] = {}; + int rv3[sizeof(struct net_policy_value_v3) == 16 ? 1 : -1] = {}; int f[sizeof(struct dns_allow_key) == MAX_DNS_NAME_LEN + 4 ? 1 : -1] = {}; - int g[sizeof(struct dns_allow_value) == 8 ? 1 : -1] = {}; + int g[sizeof(struct dns_allow_value) == 40 ? 1 : -1] = {}; int h[sizeof(struct dns_query_track_key) == 24 ? 1 : -1] = {}; - int i[sizeof(struct dns_query_track_value) == 16 ? 1 : -1] = {}; + int i[sizeof(struct dns_query_track_value) == 48 ? 1 : -1] = {}; int l[sizeof(struct mvm_port) == 8 ? 1 : -1] = {}; int n[sizeof(struct session_key) % 20 == 0 ? 1 : -1] = {}; - int o[sizeof(struct nat_session) % 64 == 0 ? 1 : -1] = {}; + int o[sizeof(struct nat_session) == 64 ? 1 : -1] = {}; int p[sizeof(struct ingress_session) % 16 == 0 ? 1 : -1] = {}; int q[sizeof(struct snat_ip) % 16 == 0 ? 1 : -1] = {}; + int s[sizeof(struct l7_port_entry) == 4 ? 1 : -1] = {}; - return b[0] + d[0] + r[0] + f[0] + g[0] + h[0] + i[0] + l[0] + n[0] + o[0] + p[0] + q[0]; + return b[0] + d[0] + dv3[0] + r[0] + rv3[0] + f[0] + g[0] + h[0] + i[0] + l[0] + n[0] + o[0] + p[0] + q[0] + s[0]; } static __always_inline __attribute__((used)) __u32 __btf_pin(void) { return __builtin_btf_type_id(*(struct lpm_key *)0, BPF_TYPE_ID_LOCAL) + __builtin_btf_type_id(*(struct net_policy_value_v2 *)0, BPF_TYPE_ID_LOCAL) + + __builtin_btf_type_id(*(struct lpm_key_v3 *)0, BPF_TYPE_ID_LOCAL) + + __builtin_btf_type_id(*(struct net_policy_value_v3 *)0, BPF_TYPE_ID_LOCAL) + __builtin_btf_type_id(*(struct dns_allow_key *)0, BPF_TYPE_ID_LOCAL) + __builtin_btf_type_id(*(struct dns_allow_value *)0, BPF_TYPE_ID_LOCAL); } diff --git a/CubeNet/src/dns_learn_test.bpf.c b/CubeNet/src/dns_learn_test.bpf.c new file mode 100644 index 000000000..e88e2ba02 --- /dev/null +++ b/CubeNet/src/dns_learn_test.bpf.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (c) 2026 Cube Authors */ +#include +#include +#include +#include + +#include "dns_response.h" + +/* Single-slot store for the dns_query_track_value under test. The dataplane + * reads it as a map-value pointer (mirroring production, where the query is a + * dns_query_track map value) so the verifier accepts the per-port loop's + * bounded offsets — a stack copy would risk rejection as a variable-offset + * stack access when the loop is not fully unrolled. + */ +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct dns_query_track_value); +} test_query_store SEC(".maps"); + +struct dns_learn_case { + __u32 ifindex; + __u32 ip; /* network byte order */ + __u32 ttl; /* seconds */ +}; + +SEC("tc") +int test_dns_learn(struct __sk_buff *skb) +{ + struct dns_learn_case tc = {}; + __u32 qkey = 0; + struct dns_query_track_value *query; + + if (bpf_skb_load_bytes(skb, 0, &tc, sizeof(tc))) + return TC_ACT_SHOT; + + query = bpf_map_lookup_elem(&test_query_store, &qkey); + if (!query) + return TC_ACT_SHOT; + + dns_learn_response_ip(tc.ifindex, tc.ip, tc.ttl, query); + return TC_ACT_OK; +} + +char __license[] SEC("license") = "Dual BSD/GPL"; diff --git a/CubeNet/src/dns_query.h b/CubeNet/src/dns_query.h index 021a48c2c..a50f16445 100644 --- a/CubeNet/src/dns_query.h +++ b/CubeNet/src/dns_query.h @@ -178,10 +178,15 @@ static __always_inline bool dns_query_should_filter_ipv4_a(struct __sk_buff *skb return is_ipv4_a; } -/* Track an allowed IPv4 A query so the response can inherit its L7 flags. */ +/* Track an allowed IPv4 A query so the response can inherit its L7 flags + * and per-host port set. Copies port_count + ports[] verbatim from the matched + * dns_allow_value so dns_learn_response_ip can rebuild net_policy_value_v3 + * without a second dns_allow_v2 lookup at response time. + */ static __always_inline void dns_track_allowed_query(struct __sk_buff *skb, const struct dns_query_state *state, - __u8 flags, __u64 qname_hash) + const struct dns_allow_value *matched, + __u64 qname_hash) { struct dns_query_track_key track_key = {}; struct dns_query_track_value track_value = {}; @@ -189,6 +194,7 @@ static __always_inline void dns_track_allowed_query(struct __sk_buff *skb, struct ethhdr *l2; struct iphdr *l3; struct udphdr *udp; + int i; if (!__pull_headers_udp(skb, &l2, &l3, &udp)) return; @@ -200,8 +206,26 @@ static __always_inline void dns_track_allowed_query(struct __sk_buff *skb, track_key.source_port = udp->source; track_key.dns_id = hdr.id; track_key.qname_hash = qname_hash; - track_value.flags = flags; + track_value.flags = matched->flags; track_value.expires_at_ns = bpf_ktime_get_ns() + DNS_QUERY_TRACK_TTL_NS; + track_value.port_count = matched->port_count; + if (track_value.port_count > MAX_L7_PORTS_PER_HOST) + track_value.port_count = MAX_L7_PORTS_PER_HOST; + +#pragma unroll + for (i = 0; i < MAX_L7_PORTS_PER_HOST; i++) { + /* Guard inside body (not `break`) so clang keeps this as a + * fixed-trip-count loop and fully unrolls it — otherwise the BPF + * verifier rejects ports[i] writes as variable-offset stack + * accesses. Field-by-field assignment sidesteps clang's memcpy + * lowering for whole-struct copies (which is disallowed in BPF). + */ + if (i < track_value.port_count) { + track_value.ports[i].port = matched->ports[i].port; + track_value.ports[i].scheme = matched->ports[i].scheme; + track_value.ports[i]._pad = matched->ports[i]._pad; + } + } bpf_map_update_elem(&dns_query_track, &track_key, &track_value, BPF_ANY); } @@ -226,7 +250,7 @@ static __always_inline void dns_init_query_state(struct dns_query_state *state, /* Query hook for sandbox-originated UDP/53 traffic. * - * Each sandbox owns one precreated DNS allow LPM trie via dns_allow[ifindex]. + * Each sandbox owns one precreated DNS allow LPM trie via dns_allow_v2[ifindex]. * Callers only invoke this hook when per-sandbox metadata flags in * ifindex_to_mvmmeta enable DNS policy processing. The DNS allow trie stores * only reversed lower-case domain rules and their rule-specific flags. @@ -243,7 +267,7 @@ static __always_inline int dns_handle_query(struct __sk_buff *skb, __u32 dns_off if (!dns_read_query_header(skb, dns_off, &hdr, &flags)) return CUBE_DNS_PASS; - inner_map = bpf_map_lookup_elem(&dns_allow, &ifindex); + inner_map = bpf_map_lookup_elem(&dns_allow_v2, &ifindex); if (!inner_map) return CUBE_DNS_PASS; diff --git a/CubeNet/src/dns_response.h b/CubeNet/src/dns_response.h index 511acb631..d8b2a3b20 100644 --- a/CubeNet/src/dns_response.h +++ b/CubeNet/src/dns_response.h @@ -98,31 +98,100 @@ static __always_inline bool dns_response_learning_enabled(__u32 ifindex) return mvm_meta && (mvm_meta->dns_policy_flags & DNS_POLICY_FLAG_LEARNING_ENABLED); } -/* Add an IPv4 A-record address as a temporary DNS-learned allow_out_v2 entry. */ +/* Add an IPv4 A-record address as temporary DNS-learned allow_out_v3 + * entries. + * + * Two shapes, mirroring how the rule was installed: + * - Plain (non-L7) allow: a single /32 (any-port) entry, exactly as the + * pre-v3 dataplane did. Without this a plain domain-based allow rule + * would resolve via DNS yet never be admitted by classify_egress_flow, + * regressing to a default-deny drop. + * - L7 allow: v3 carries the destination port in the LPM key, so we write + * one exact (ip, port)/48 entry per (port, scheme) the query inherited + * from its matched dns_allow_v2 rule. port_count == 0 means the default + * {80/http, 443/https} set. + * Each entry is refreshed independently; an existing entry for the EXACT + * same key keeps its flags and its (zero) expiry, so a static rule + * survives a later DNS refresh of the same IP. "Exact" is checked via + * old->key_prefixlen: the LPM lookup alone is longest-prefix and would + * otherwise match a shorter COVERING entry (e.g. a static /24), making + * the learned entry wrongly inherit the static zero expiry (never ages) + * or the covering entry's flags. + */ static __always_inline void dns_learn_response_ip(__u32 ifindex, __u32 ip, __u32 ttl, - __u8 flags) + const struct dns_query_track_value *query) { - struct lpm_key key = { .prefixlen = 32, .ip = ip }; - struct net_policy_value_v2 value = { - .expires_at_ns = bpf_ktime_get_ns() + ((__u64)ttl * NSEC_PER_SEC), - .flags = flags, - }; - struct net_policy_value_v2 *old_value; + __u64 now = bpf_ktime_get_ns(); + __u64 expires = now + ((__u64)ttl * NSEC_PER_SEC); void *inner_map; - inner_map = bpf_map_lookup_elem(&allow_out_v2, &ifindex); + inner_map = bpf_map_lookup_elem(&allow_out_v3, &ifindex); if (!inner_map) return; - /* DNS learning must not downgrade flags or shorten static allow rules. */ - old_value = bpf_map_lookup_elem(inner_map, &key); - if (old_value) { - value.flags |= old_value->flags; - if (old_value->expires_at_ns == 0) - value.expires_at_ns = 0; + /* Plain (non-L7) allow: learn a single /32 (any-port) entry. */ + if (!(query->flags & NET_POLICY_FLAG_L7_REQUIRED)) { + struct lpm_key_v3 key = { .prefixlen = 32, .ip = ip, .port = 0 }; + struct net_policy_value_v3 value = { + .expires_at_ns = expires, + .flags = query->flags, + .scheme = L7_SCHEME_NONE, + .key_prefixlen = 32, + }; + struct net_policy_value_v3 *old = bpf_map_lookup_elem(inner_map, &key); + if (old && old->key_prefixlen == key.prefixlen) { + value.flags |= old->flags; + if (old->expires_at_ns == 0) + value.expires_at_ns = 0; + } + bpf_map_update_elem(inner_map, &key, &value, BPF_ANY); + return; } - bpf_map_update_elem(inner_map, &key, &value, BPF_ANY); + /* L7 allow: one exact (ip, port)/48 entry per (port, scheme). + * + * The loop uses a fixed trip count with the bound as a guard inside the + * body (not `break`), so clang fully unrolls it. That turns each + * query->ports[i] into a constant-offset access the verifier accepts. A + * `break`-bound loop is not unrolled and was rejected by the verifier as + * a variable-offset map-value read (off beyond the 48-byte value). + */ + __u8 count = query->port_count; + bool use_defaults = count == 0; + __u8 limit = use_defaults ? 2 : count; + if (limit > MAX_L7_PORTS_PER_HOST) + limit = MAX_L7_PORTS_PER_HOST; + +#pragma unroll + for (__u8 i = 0; i < MAX_L7_PORTS_PER_HOST; i++) { + if (i >= limit) + continue; + + __u16 port; + __u8 scheme; + if (use_defaults) { + port = (i == 0) ? bpf_htons(80) : bpf_htons(443); + scheme = (i == 0) ? L7_SCHEME_HTTP : L7_SCHEME_HTTPS; + } else { + port = query->ports[i].port; + scheme = query->ports[i].scheme; + } + + struct lpm_key_v3 key = { .prefixlen = 48, .ip = ip, .port = port }; + struct net_policy_value_v3 value = { + .expires_at_ns = expires, + .flags = query->flags, + .scheme = scheme, + .key_prefixlen = 48, + }; + struct net_policy_value_v3 *old = bpf_map_lookup_elem(inner_map, &key); + if (old && old->key_prefixlen == key.prefixlen) { + value.flags |= old->flags; + if (old->expires_at_ns == 0) + value.expires_at_ns = 0; + } + bpf_map_update_elem(inner_map, &key, &value, BPF_ANY); + } } /* Return true when an answer RR carries an IN A record payload. */ @@ -142,7 +211,7 @@ static __always_inline bool dns_response_record_is_ipv4_a(const struct dns_rr_he */ static __always_inline bool dns_process_response_answer(struct __sk_buff *skb, __u32 *cursor, __u32 ifindex, - __u8 flags) + const struct dns_query_track_value *query) { struct dns_rr_header rr; __u16 rdlength; @@ -161,7 +230,7 @@ static __always_inline bool dns_process_response_answer(struct __sk_buff *skb, return false; ttl = bpf_ntohl(rr.ttl); if (ttl < 300) ttl = 300; - dns_learn_response_ip(ifindex, ip, ttl, flags); + dns_learn_response_ip(ifindex, ip, ttl, query); } /* Keep cursor advancement bounded even for unsupported RR types. */ @@ -187,7 +256,7 @@ static __always_inline struct dns_query_track_value *dns_lookup_response_query( /* Response hook for DNS replies returning to a sandbox. * - * The path learns IPv4 A records into allow_out_v2 as temporary DNS-learned IP + * The path learns IPv4 A records into allow_out_v3 as temporary DNS-learned IP * policy entries. It intentionally preserves the existing filtering semantics. * * Marked __always_inline so the calling SEC("tc") program contains no @@ -238,7 +307,7 @@ static __always_inline void dns_handle_response(struct __sk_buff *skb, __u32 dns for (i = 0; i < DNS_MAX_RESPONSE_ANSWERS; i++) { if (i >= ancount) break; - if (!dns_process_response_answer(skb, &cursor, ifindex, query->flags)) + if (!dns_process_response_answer(skb, &cursor, ifindex, query)) goto delete_query; } diff --git a/CubeNet/src/egress_policy_test.bpf.c b/CubeNet/src/egress_policy_test.bpf.c new file mode 100644 index 000000000..ca3f3d8b5 --- /dev/null +++ b/CubeNet/src/egress_policy_test.bpf.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (c) 2026 Cube Authors */ +#include +#include +#include +#include + +#include "session.h" + +struct egress_policy_case { + __u32 ifindex; + __u32 daddr; + __u16 dport; + __u8 verdict; + __u8 reserved[5]; +}; + +SEC("tc") +int test_classify_egress_flow(struct __sk_buff *skb) +{ + struct egress_policy_case tc = {}; + + if (bpf_skb_load_bytes(skb, 0, &tc, sizeof(tc))) + return TC_ACT_SHOT; + + tc.verdict = classify_egress_flow(tc.ifindex, tc.daddr, tc.dport); + if (bpf_skb_store_bytes(skb, 0, &tc, sizeof(tc), 0)) + return TC_ACT_SHOT; + + return TC_ACT_OK; +} + +char __license[] SEC("license") = "Dual BSD/GPL"; diff --git a/CubeNet/src/icmp.h b/CubeNet/src/icmp.h index 0bfaeca81..dbacb70e1 100644 --- a/CubeNet/src/icmp.h +++ b/CubeNet/src/icmp.h @@ -40,7 +40,7 @@ static __always_inline bool create_icmp_sessions(struct __sk_buff *skb, struct snat_ip *snat_ip, __u16 snat_id) { return create_nat_session(skb, ekey, now_ns, vm_ifindex, snat_ip, snat_id, - ICMP_CT_UNREPLIED); + ICMP_CT_UNREPLIED, SNAT_PACKET, L7_SCHEME_NONE); } #endif /* __ICMP_H */ diff --git a/CubeNet/src/l7_mark_test.bpf.c b/CubeNet/src/l7_mark_test.bpf.c new file mode 100644 index 000000000..c56bd5367 --- /dev/null +++ b/CubeNet/src/l7_mark_test.bpf.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (c) 2026 Cube Authors */ +#include +#include +#include +#include + +#include "cubevs.h" + +struct l7_mark_case { + __u32 mark; +}; + +SEC("tc") +int test_l7_mark(struct __sk_buff *skb) +{ + struct l7_mark_case tc = {}; + + if (bpf_skb_load_bytes(skb, 0, &tc, sizeof(tc))) + return TC_ACT_SHOT; + + /* Mirror mvmtap.bpf.c's L7 stamp: keep the low (user) bits of skb->mark, + * then OR in the configured cube HTTP mark. + */ + tc.mark = (tc.mark & ~cube_l7_mark_mask) | cube_l7_mark_http; + + if (bpf_skb_store_bytes(skb, 0, &tc, sizeof(tc), 0)) + return TC_ACT_SHOT; + return TC_ACT_OK; +} + +char __license[] SEC("license") = "Dual BSD/GPL"; diff --git a/CubeNet/src/localgw.bpf.c b/CubeNet/src/localgw.bpf.c index 898fc4d7c..3b6b52183 100644 --- a/CubeNet/src/localgw.bpf.c +++ b/CubeNet/src/localgw.bpf.c @@ -10,6 +10,40 @@ #include "map.h" #include "nat.h" #include "skb.h" +#include "tcp.h" + +/* Update a CubeEgress TCP session from the original reply tuple before DNAT + * rewrites the sandbox-assigned destination address to mvm_inner_ip. A missing + * or non-L7 session is intentionally non-blocking. + */ +static __always_inline void update_l7_session_from_cubedev(struct __sk_buff *skb) +{ + struct session_key key = {}; + struct nat_session *sess; + struct ethhdr *l2; + struct iphdr *l3; + struct tcphdr *l4; + __u64 now; + + if (!__pull_headers(skb, &l2, &l3, &l4)) + return; + if (l3->protocol != IPPROTO_TCP) + return; + + key.src_ip = l3->saddr; + key.dst_ip = l3->daddr; + key.src_port = l4->source; + key.dst_port = l4->dest; + key.version = 0; + key.protocol = IPPROTO_TCP; + sess = lookup_session(&key); + if (!sess || sess->packet_class != L7PROXY_PACKET) + return; + + now = bpf_ktime_get_ns(); + update_session(IP_CT_DIR_REPLY, sess, now, l4->syn, l4->ack, + l4->fin, l4->rst); +} /* This filter will be attached to the egress path of cube-dev device. * It performs a DNAT and then redirect the traffics to Sandbox TAP devices. @@ -26,6 +60,8 @@ int from_envoy(struct __sk_buff *skb) if (skb->protocol != bpf_htons(ETH_P_IP)) return TC_ACT_OK; + update_l7_session_from_cubedev(skb); + ret = pull_headers(skb, &l2, &l3); if (ret != TC_ACT_OK) return ret; diff --git a/CubeNet/src/map.h b/CubeNet/src/map.h index 6f93c3519..4ea2d3a48 100644 --- a/CubeNet/src/map.h +++ b/CubeNet/src/map.h @@ -96,13 +96,18 @@ struct { __uint(pinning, LIBBPF_PIN_BY_NAME); } snat_iplist SEC(".maps"); -/* Egress allow list v2 (hash of maps) +/* Egress allow list v3 (hash of maps) * * key: ifindex of the TAP device - * value: fd of inner LPM trie map (destination IP allow list) + * value: fd of inner LPM trie map (destination ip[:port] allow list) * - * Inner values use net_policy_value_v2. A zero expires_at_ns means a static - * allow entry; a non-zero expires_at_ns means a temporary DNS-learned entry. + * Inner keys use lpm_key_v3 so a single longest-prefix lookup resolves + * exact (ip, port) (prefixlen 48), ip-only / any-port (prefixlen 32), + * or ip/mask subnet (prefixlen < 32) rules. Inner values use + * net_policy_value_v3, which marks the L7 scheme directly (the port is + * now part of the key, so no per-packet (port, scheme) array scan). + * A zero expires_at_ns means a static entry; a non-zero expires_at_ns + * means a temporary DNS-learned entry. */ struct { __uint(type, BPF_MAP_TYPE_HASH_OF_MAPS); @@ -112,11 +117,11 @@ struct { __array(values, struct { __uint(type, BPF_MAP_TYPE_LPM_TRIE); __uint(max_entries, MAX_IP_RULE_ENTRIES); - __type(key, struct lpm_key); - __type(value, struct net_policy_value_v2); + __type(key, struct lpm_key_v3); + __type(value, struct net_policy_value_v3); __uint(map_flags, BPF_F_NO_PREALLOC); }); -} allow_out_v2 SEC(".maps"); +} allow_out_v3 SEC(".maps"); /* Egress deny list (hash of maps) * @@ -146,7 +151,7 @@ struct { * value: fd of inner LPM trie map for this sandbox's DNS policy rules * * Inner keys are reversed lower-case domain name prefixes. DNS policy mode is - * stored in ifindex_to_mvmmeta, while dns_allow stores only domain rules. + * stored in ifindex_to_mvmmeta, while dns_allow_v2 stores only domain rules. * Exact rule "qq.com" is encoded as "moc.qq\0" with the trailing NUL included * in prefixlen. Wildcard rule "*.qq.com" is encoded as "moc.qq." without NUL, * so only subdomains such as "a.qq.com" can match it. @@ -163,13 +168,13 @@ struct { __type(value, struct dns_allow_value); __uint(map_flags, BPF_F_NO_PREALLOC); }); -} dns_allow SEC(".maps"); +} dns_allow_v2 SEC(".maps"); /* Pending DNS queries waiting for responses. * * key: sandbox ifindex + DNS server IP + sandbox UDP source port + DNS id * + raw DNS QNAME hash - * value: L7 flags inherited from dns_allow and pending expiration time + * value: L7 flags inherited from dns_allow_v2 and pending expiration time */ struct { __uint(type, BPF_MAP_TYPE_LRU_HASH); diff --git a/CubeNet/src/mvmtap.bpf.c b/CubeNet/src/mvmtap.bpf.c index 1d71f5a74..c265d6bc6 100644 --- a/CubeNet/src/mvmtap.bpf.c +++ b/CubeNet/src/mvmtap.bpf.c @@ -106,42 +106,17 @@ static __always_inline bool should_do_nat(const struct iphdr *l3) return true; } -/* - * Check whether a TCP flow should be redirected to the L7 proxy. - * - * Looks up allow_out_v2 for the given ifindex/daddr and returns true iff - * the entry carries NET_POLICY_FLAG_L7_REQUIRED and the destination port - * is 80 or 443. This is a fast, self-contained lookup — the general - * egress policy check (allow / deny) is enforced later inside - * create_nat_session(). +/* Egress flow classification now lives in classify_egress_flow() (session.h), + * which merges the former l7_scheme_for_flow() and session_policy_allowed() + * into a single policy verdict (reject / accept-SNAT / accept-HTTP / + * accept-HTTPS). It is applied once, when a new flow is created, and the + * result is cached in nat_session for reuse on every later packet. */ -static __always_inline bool should_redirect_to_l7_proxy(__u32 ifindex, __u32 daddr, - const struct tcphdr *l4) -{ - struct lpm_key key = { .prefixlen = 32, .ip = daddr }; - struct net_policy_value_v2 *value; - void *inner_map; - - if (l4->dest != bpf_htons(80) && l4->dest != bpf_htons(443)) - return false; - - inner_map = bpf_map_lookup_elem(&allow_out_v2, &ifindex); - if (!inner_map) - return false; - - value = bpf_map_lookup_elem(inner_map, &key); - if (!value) - return false; - if (value->expires_at_ns != 0 && value->expires_at_ns <= bpf_ktime_get_ns()) - return false; - - return value->flags & NET_POLICY_FLAG_L7_REQUIRED; -} - enum tcp_nat_result { TCP_NAT_DROP = 0, TCP_NAT_OK, TCP_NAT_RESET, + TCP_L7PROXY_OK, }; /* do_tcp_nat() returns a 64-bit value that encodes both the status enum @@ -410,6 +385,31 @@ static __always_inline struct snat_ip *pick_snat_ip_port(__u32 mvm_ip, const str return NULL; } +/* Reserve the reverse-flow key for an L7 proxy session. L7 traffic is not + * source-NATed, so the reply tuple is the exact reverse of the sandbox's + * original tuple. create_nat_session() later inserts the matching egress value + * and rolls this reservation back if that insertion fails. + */ +static __always_inline bool create_l7_ingress_session(const struct session_key *ekey) +{ + struct ingress_session isess = { + .version = ekey->version, + .vm_ip = ekey->src_ip, + .vm_port = ekey->src_port, + }; + struct session_key ikey = { + .src_ip = ekey->dst_ip, + .dst_ip = ekey->src_ip, + .src_port = ekey->dst_port, + .dst_port = ekey->src_port, + .version = 0, + .protocol = ekey->protocol, + }; + + return bpf_map_update_elem(&ingress_sessions, &ikey, &isess, + BPF_NOEXIST) == 0; +} + static __always_inline void del_session(struct session_key *ekey, struct nat_session *sess) { struct session_key ikey = { @@ -470,6 +470,9 @@ static __always_inline __u32 do_icmp_nat(struct __sk_buff *skb, struct mvm_meta } /* create new session */ + if (classify_egress_flow(skb->ingress_ifindex, key.dst_ip, + key.dst_port) == FLOW_REJECT) + return 0; snat_ip = pick_snat_ip_port(mvm_meta->ip, &key, &snat_id); if (!snat_ip || !snat_ip->ip || !snat_id) return 0; @@ -568,6 +571,9 @@ static __always_inline __u32 do_udp_nat_inline(struct __sk_buff *skb, } /* create new session */ + if (classify_egress_flow(skb->ingress_ifindex, key.dst_ip, + key.dst_port) == FLOW_REJECT) + return 0; snat_ip = pick_snat_ip_port(mvm_meta->ip, &key, &snat_port); if (!snat_ip || !snat_ip->ip || !snat_port) return 0; @@ -681,6 +687,7 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * struct session_key key = {}; struct nat_session *sess; struct snat_ip *snat_ip; + struct snat_ip l7_endpoint = {}; bool syn, ack, fin, rst; struct ethhdr *l2; struct iphdr *l3; @@ -689,6 +696,10 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * __u16 snat_port; __u64 flags; __u64 now; + __u32 l7_mark; + __u8 packet_class = SNAT_PACKET; + __u8 l7_scheme = L7_SCHEME_NONE; + __u8 verdict = FLOW_SNAT; long err; bool ok; @@ -711,6 +722,16 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * sess = bpf_map_lookup_elem(&egress_sessions, &key); if (sess) { if (sess->state == TCP_CONNTRACK_CLOSE || sess->state == TCP_CONNTRACK_TIME_WAIT) { + /* L7 sessions use an identity reverse tuple, so immediately + * replacing a terminal session would let delayed packets from + * the old connection mutate the new session. Keep the old pair + * until the userspace reaper removes it and reject premature + * tuple reuse. Ordinary SNAT sessions remain safe to recreate + * because they allocate a fresh reverse-side source port. + */ + if (sess->packet_class == L7PROXY_PACKET) + return TCP_NAT_PACK(0, TCP_NAT_RESET); + /* guest kernel reuse source port too fast */ del_session(&key, sess); goto do_create; @@ -719,28 +740,87 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * goto do_update; } do_create: - /* create new session */ + /* Classify the flow with the unified egress policy. The verdict is + * cached in nat_session and reused for every later packet. + */ + verdict = classify_egress_flow(skb->ingress_ifindex, key.dst_ip, + key.dst_port); + switch (verdict) { + case FLOW_HTTP: + case FLOW_HTTPS: + if (!create_l7_ingress_session(&key)) + return TCP_NAT_DROP; + l7_endpoint.ifindex = cubegw0_ifindex; + l7_endpoint.ip = key.src_ip; + snat_ip = &l7_endpoint; + snat_port = key.src_port; + packet_class = L7PROXY_PACKET; + l7_scheme = (verdict == FLOW_HTTP) ? L7_SCHEME_HTTP : + L7_SCHEME_HTTPS; + break; + case FLOW_REJECT: + /* Denied by egress policy: signal the caller so it can emit an + * RST, exactly as the old create_nat_session path did. + */ + nat_cb_set(skb, NAT_CB_DENIED_BY_POLICY); + return TCP_NAT_PACK(0, TCP_NAT_RESET); + case FLOW_SNAT: + default: snat_ip = pick_snat_ip_port(mvm_meta->ip, &key, &snat_port); if (!snat_ip || !snat_ip->ip || !snat_port) return TCP_NAT_DROP; - ok = create_new_sessions(skb, &key, now, skb->ingress_ifindex, snat_ip, snat_port); - if (!ok) { - /* Preserve RST-on-deny: create_nat_session stamps - * skb->cb when the failure is due to net policy. - */ - if (nat_cb_get(skb) == NAT_CB_DENIED_BY_POLICY) - return TCP_NAT_PACK(0, TCP_NAT_RESET); - return TCP_NAT_DROP; - } - sess = bpf_map_lookup_elem(&egress_sessions, &key); - if (!sess) - return TCP_NAT_DROP; - goto do_nat; + break; + } + ok = create_new_sessions(skb, &key, now, skb->ingress_ifindex, + snat_ip, snat_port, packet_class, l7_scheme); + if (!ok) + return TCP_NAT_DROP; + sess = bpf_map_lookup_elem(&egress_sessions, &key); + if (!sess) + return TCP_NAT_DROP; + goto do_nat; } else { /* lookup existing session */ sess = bpf_map_lookup_elem(&egress_sessions, &key); - if (!sess) + if (!sess) { + /* Legacy default-port (80/443) connection drain: the eBPF session + * entry was lost (agent restart, map eviction, or expiry) AND the + * allow_out_v3 /48 entry for this (ip, port) has aged out, so + * classify_egress_flow would return FLOW_REJECT. But the proxy + * still holds an established TPROXY socket for this 4-tuple — the + * connection was legitimately opened when the policy allowed it. + * Re-stamp the mark so iptables TPROXY steers the packet to the + * proxy, keeping the connection alive instead of resetting it. + * Custom-port connections are intentionally excluded: they should + * respect the current policy when their allow_out_v3 entry expires. + * No session is re-created, so each packet on the drained flow + * re-enters this path (per-packet socket lookup — acceptable for + * draining connections that will eventually close). + */ + if (l4->dest == bpf_htons(80) || l4->dest == bpf_htons(443)) { + struct bpf_sock *sk; + struct bpf_sock_tuple tuple = {}; + tuple.ipv4.saddr = key.src_ip; + tuple.ipv4.daddr = key.dst_ip; + tuple.ipv4.sport = l4->source; + tuple.ipv4.dport = l4->dest; + sk = bpf_skc_lookup_tcp(skb, &tuple, sizeof(tuple.ipv4), BPF_F_CURRENT_NETNS, 0); + if (sk) { + __u32 state = sk->state; + + bpf_sk_release(sk); + if (state == BPF_TCP_ESTABLISHED) { + if (l4->dest == bpf_htons(80)) { + skb->mark = (skb->mark & ~cube_l7_mark_mask) | cube_l7_mark_http; + } else { + skb->mark = (skb->mark & ~cube_l7_mark_mask) | cube_l7_mark_https; + } + return TCP_NAT_PACK(cubegw0_ifindex, TCP_L7PROXY_OK); + } + } + } return rst ? TCP_NAT_DROP : TCP_NAT_RESET; + } } do_update: @@ -748,6 +828,18 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * update_session(IP_CT_DIR_ORIGINAL, sess, now, syn, ack, fin, rst); do_nat: + if (sess->packet_class == L7PROXY_PACKET) { + if (sess->l7_scheme == L7_SCHEME_HTTP) + l7_mark = cube_l7_mark_http; + else if (sess->l7_scheme == L7_SCHEME_HTTPS) + l7_mark = cube_l7_mark_https; + else + return TCP_NAT_DROP; + + skb->mark = (skb->mark & ~cube_l7_mark_mask) | l7_mark; + return TCP_NAT_PACK(sess->node_ifindex, TCP_L7PROXY_OK); + } + old_saddr = l3->saddr; new_saddr = sess->node_ip; old_sport = l4->source; @@ -880,7 +972,7 @@ int dns_finish(struct __sk_buff *skb) if (!dns_policy_enabled(mvm_meta)) return finish_udp_nat(skb, mvm_meta); - inner_map = bpf_map_lookup_elem(&dns_allow, &ifindex); + inner_map = bpf_map_lookup_elem(&dns_allow_v2, &ifindex); if (!inner_map) return finish_udp_nat(skb, mvm_meta); @@ -895,7 +987,7 @@ int dns_finish(struct __sk_buff *skb) if (!matched) return finish_udp_nat(skb, mvm_meta); - dns_track_allowed_query(skb, state, matched->flags, qname_hash); + dns_track_allowed_query(skb, state, matched, qname_hash); return finish_udp_nat(skb, mvm_meta); } @@ -907,10 +999,8 @@ int from_cube(struct __sk_buff *skb) { __u32 daddr, ifindex, dst_ifindex; __u64 tcp_ret; - struct bpf_sock_tuple tuple = {}; struct mvm_port mvm_port = {}; struct mvm_meta *mvm_meta; - struct bpf_sock *sk; struct ethhdr *l2; struct iphdr *l3; struct tcphdr *l4; @@ -994,23 +1084,6 @@ int from_cube(struct __sk_buff *skb) } } - if (proto == IPPROTO_TCP && - __pull_headers(skb, &l2, &l3, &l4) && - (l4->dest == bpf_htons(80) || l4->dest == bpf_htons(443))) { - tuple.ipv4.saddr = mvm_meta->ip; - tuple.ipv4.daddr = daddr; - tuple.ipv4.sport = l4->source; - tuple.ipv4.dport = l4->dest; - sk = bpf_skc_lookup_tcp(skb, &tuple, sizeof(tuple.ipv4), BPF_F_CURRENT_NETNS, 0); - if (sk) { - __u32 state = sk->state; - - bpf_sk_release(sk); - if (state == BPF_TCP_ESTABLISHED) - return bpf_redirect(cubegw0_ifindex, BPF_F_INGRESS); - } - } - ret = pull_headers(skb, &l2, &l3); if (ret != TC_ACT_OK) return ret; @@ -1020,26 +1093,27 @@ int from_cube(struct __sk_buff *skb) if (l3->daddr == nodenic_ip) { /* This branch bypasses do_*_nat() and therefore the policy - * check inside create_nat_session(). Enforce policy inline. - * TCP callers get an RST to match the guest-visible behavior - * of the do_tcp_nat() path; UDP/ICMP silently drop. + * check applied there. Enforce the unified egress policy + * inline. TCP callers get an RST to match the guest-visible + * behavior of the do_tcp_nat() path; UDP/ICMP silently drop. + * dport is 0 because the original check was port-agnostic. */ - if (!session_policy_allowed(ifindex, daddr)) { + switch (classify_egress_flow(ifindex, daddr, 0)) { + case FLOW_REJECT: if (proto == IPPROTO_TCP) return tcp_reply_reset(skb, ifindex); return TC_ACT_SHOT; + default: + return bpf_redirect(cubegw0_ifindex, BPF_F_INGRESS); } - return bpf_redirect(cubegw0_ifindex, BPF_F_INGRESS); } if (proto == IPPROTO_TCP) { - if (!__pull_headers(skb, &l2, &l3, &l4)) - return TC_ACT_SHOT; - if (should_redirect_to_l7_proxy(ifindex, daddr, l4)) - return bpf_redirect(cubegw0_ifindex, BPF_F_INGRESS); tcp_ret = do_tcp_nat(skb, mvm_meta); if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_OK) return bpf_redirect(TCP_NAT_IFINDEX(tcp_ret), egress_redirect_flags); + if (TCP_NAT_STATUS(tcp_ret) == TCP_L7PROXY_OK) + return bpf_redirect(TCP_NAT_IFINDEX(tcp_ret), BPF_F_INGRESS); if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_RESET) return tcp_reply_reset(skb, ifindex); } diff --git a/CubeNet/src/nodenic.bpf.c b/CubeNet/src/nodenic.bpf.c index c8e1cdfc3..32c2380d4 100644 --- a/CubeNet/src/nodenic.bpf.c +++ b/CubeNet/src/nodenic.bpf.c @@ -68,24 +68,6 @@ static int tcp_nat_proxy(struct __sk_buff *skb, struct ethhdr *l2, struct iphdr return bpf_redirect(mvm_port->ifindex, 0); } -static __always_inline struct nat_session *lookup_session(const struct session_key *ikey) -{ - struct ingress_session *isess; - struct session_key key = {}; - - isess = bpf_map_lookup_elem(&ingress_sessions, ikey); - if (!isess) - return NULL; - - key.src_ip = isess->vm_ip; - key.dst_ip = ikey->src_ip; - key.src_port = isess->vm_port; - key.dst_port = ikey->src_port; - key.version = isess->version; - key.protocol = ikey->protocol; - return bpf_map_lookup_elem(&egress_sessions, &key); -} - static int tcp_nat_session(struct __sk_buff *skb, struct ethhdr *l2, struct iphdr *l3, struct tcphdr *l4) { __u32 old_daddr, new_daddr, tcp_csum_off; @@ -447,7 +429,7 @@ int dns_handle_response_prog(struct __sk_buff *skb) if (!rstate) return TC_ACT_OK; - /* Learn A records into allow_out_v2 before reverse NAT. */ + /* Learn A records into allow_out_v3 before reverse NAT. */ dns_handle_response(skb, rstate->dns_off, rstate->ifindex, rstate->server_ip, rstate->source_port); diff --git a/CubeNet/src/session.h b/CubeNet/src/session.h index 6b14a5ada..ed7939760 100644 --- a/CubeNet/src/session.h +++ b/CubeNet/src/session.h @@ -7,6 +7,11 @@ #include "cubevs.h" #include "map.h" +enum packet_class { + SNAT_PACKET = 0, + L7PROXY_PACKET, +}; + /* Lazy refresh threshold: 1 second in nanoseconds */ #define SESSION_REFRESH_INTERVAL_NS (1000 * 1000 * 1000UL) @@ -38,43 +43,128 @@ static __always_inline void session_mark_replied(enum ip_conntrack_dir dir, } /** - * session_policy_allowed - check egress network policy for a candidate flow - * @vm_ifindex: TAP ifindex of the originating MVM (policy key) - * @daddr: destination IP address in network byte order + * lookup_session - resolve a reverse-flow key to its egress NAT session + * @ikey: ingress/reply-direction session key + * + * ingress_sessions stores the sandbox identity needed to reconstruct the + * original-direction egress_sessions key. Both keys use network-byte-order + * addresses and ports. Returns the live map value, or NULL when either side + * of the session pair is missing. + */ +static __always_inline struct nat_session *lookup_session(const struct session_key *ikey) +{ + struct ingress_session *isess; + struct session_key ekey = {}; + + isess = bpf_map_lookup_elem(&ingress_sessions, ikey); + if (!isess) + return NULL; + + ekey.src_ip = isess->vm_ip; + ekey.dst_ip = ikey->src_ip; + ekey.src_port = isess->vm_port; + ekey.dst_port = ikey->src_port; + ekey.version = isess->version; + ekey.protocol = ikey->protocol; + + return bpf_map_lookup_elem(&egress_sessions, &ekey); +} + +/* Unified egress policy verdict for a candidate flow. + * + * Replaces the former pair l7_scheme_for_flow() (allow_out_v3 /48 L7 + * lookup) and session_policy_allowed() (allow_out_v3 /32 + deny_out /32). + * Callers classify once and act on the verdict: + * FLOW_REJECT - deny (drop / RST, never create a session) + * FLOW_SNAT - plain SNAT egress is allowed + * FLOW_HTTP - L7 proxy over HTTP is required + * FLOW_HTTPS - L7 proxy over HTTPS is required + */ +enum flow_verdict { + FLOW_REJECT = 0, + FLOW_SNAT, + FLOW_HTTP, + FLOW_HTTPS, +}; + +/** + * classify_egress_flow - single egress policy decision for a candidate flow + * @ifindex: TAP ifindex of the originating MVM (policy key) + * @daddr: destination IP address in network byte order + * @dport: destination port in network byte order (0 for port-agnostic) * - * Priority: allow_out_v2 > deny_out > default allow. + * Priority: allow_out_v3 > deny_out > default allow. * - * 1. If allow_out_v2 has an inner map for this ifindex and daddr matches - * a non-expired entry, the flow is explicitly allowed. - * 2. If deny_out has an inner map for this ifindex and daddr matches, - * the flow is denied. - * 3. Otherwise the flow is allowed. + * 1. Look up (daddr, dport)/48 in allow_out_v3. LPM automatically falls + * back to a matching /32 or subnet entry. A non-expired L7_REQUIRED + * entry returns FLOW_HTTP / FLOW_HTTPS; any other non-expired allow + * entry returns FLOW_SNAT. + * 2. Else if deny_out matches a /32 (or wider) entry, the flow is + * rejected (FLOW_REJECT). + * 3. Otherwise the flow is allowed via SNAT (default allow). * * Traffic to mvm_gateway_ip is internal (destined for cube-dev) and always * allowed regardless of policy. + * + * The L7 (/48) lookup is performed FIRST, before the deny check. This is + * the key fix for DNS-learned L7 entries: those are stored as (ip, port)/48 + * in allow_out_v3, but the old session_policy_allowed() used a hardcoded + * /32 key and could never match them, so an already-authorized L7 flow fell + * through to deny_out (e.g. 0.0.0.0/0) and was silently dropped. */ -static __always_inline bool session_policy_allowed(__u32 vm_ifindex, __u32 daddr) +static __always_inline __u8 classify_egress_flow(__u32 ifindex, __u32 daddr, + __u16 dport) { - struct lpm_key key = { .prefixlen = 32, .ip = daddr }; - struct net_policy_value_v2 *value; + struct lpm_key_v3 key = {}; + struct net_policy_value_v3 *value; void *inner_map; + __u64 now = bpf_ktime_get_ns(); + /* internal traffic destined for the MVM gateway is always allowed */ if (daddr == mvm_gateway_ip) - return true; + return FLOW_SNAT; - inner_map = bpf_map_lookup_elem(&allow_out_v2, &vm_ifindex); + /* 1) Allow: the /48 lookup resolves an exact L7 rule or falls back + * to a plain /32/subnet rule in the same LPM trie. + */ + inner_map = bpf_map_lookup_elem(&allow_out_v3, &ifindex); if (inner_map) { + key.prefixlen = 48; + key.ip = daddr; + key.port = dport; value = bpf_map_lookup_elem(inner_map, &key); if (value && (value->expires_at_ns == 0 || - value->expires_at_ns > bpf_ktime_get_ns())) - return true; + value->expires_at_ns > now)) { + if (value->flags & NET_POLICY_FLAG_L7_REQUIRED) { + if (value->scheme == L7_SCHEME_HTTP) + return FLOW_HTTP; + if (value->scheme == L7_SCHEME_HTTPS) + return FLOW_HTTPS; + /* L7 required but scheme unknown: fail closed rather + * than silently downgrading to plain SNAT, which would + * bypass the TPROXY intercept the rule asked for. A + * well-formed entry always carries a scheme (userspace + * populate and DNS-learn both set it), so reaching this + * branch means a corrupt or half-written map value. + */ + return FLOW_REJECT; + } + return FLOW_SNAT; + } } - inner_map = bpf_map_lookup_elem(&deny_out, &vm_ifindex); - if (inner_map && bpf_map_lookup_elem(inner_map, &key)) - return false; + /* 2) Deny: /32 (or wider) lookup in deny_out */ + inner_map = bpf_map_lookup_elem(&deny_out, &ifindex); + if (inner_map) { + key.prefixlen = 32; + key.ip = daddr; + key.port = 0; + if (bpf_map_lookup_elem(inner_map, &key)) + return FLOW_REJECT; + } - return true; + /* 3) Default: allow via SNAT */ + return FLOW_SNAT; } /** @@ -86,11 +176,17 @@ static __always_inline bool session_policy_allowed(__u32 vm_ifindex, __u32 daddr * @snat_ip: selected SNAT IP entry * @snat_port: selected SNAT port/identifier in network byte order * @initial_state: protocol-specific initial conntrack state + * @packet_class: SNAT_PACKET or L7PROXY_PACKET + * @l7_scheme: L7_SCHEME_*; NONE for non-L7 sessions + * + * packet_class and l7_scheme are initialized in the stack value before the + * single BPF_NOEXIST insertion. This prevents another CPU from observing a + * partially classified session. * - * Enforces egress network policy before creating the session. On policy - * deny, stamps skb->cb with NAT_CB_DENIED_BY_POLICY so the caller can - * distinguish "deny" from "resource exhaustion" (e.g. TCP callers use it - * to trigger tcp_reply_reset). + * Egress network policy is NOT enforced here. Callers must classify the + * flow with classify_egress_flow() first and reject denied flows (stamping + * skb->cb with NAT_CB_DENIED_BY_POLICY) before reaching this point. This + * keeps the policy verdict a single decision taken once per new flow. * * Returns true on success, false otherwise (ingress session cleaned up). */ @@ -98,7 +194,8 @@ static __always_inline bool create_nat_session(struct __sk_buff *skb, struct session_key *ekey, __u64 now_ns, __u32 vm_ifindex, struct snat_ip *snat_ip, __u16 snat_port, - __u8 initial_state) + __u8 initial_state, __u8 packet_class, + __u8 l7_scheme) { struct nat_session sess = {}; struct session_key ikey = {}; @@ -117,13 +214,6 @@ static __always_inline bool create_nat_session(struct __sk_buff *skb, */ nat_cb_set(skb, NAT_CB_OK); - if (!session_policy_allowed(vm_ifindex, ekey->dst_ip)) { - nat_cb_set(skb, NAT_CB_DENIED_BY_POLICY); - /* release the ingress slot reserved in pick_snat_ip_port */ - bpf_map_delete_elem(&ingress_sessions, &ikey); - return false; - } - sess.access_time = now_ns; sess.node_ifindex = snat_ip->ifindex; sess.node_ip = snat_ip->ip; @@ -132,6 +222,8 @@ static __always_inline bool create_nat_session(struct __sk_buff *skb, sess.node_port = snat_port; sess.vm_port = ekey->src_port; sess.state = initial_state; + sess.packet_class = packet_class; + sess.l7_scheme = l7_scheme; err = bpf_map_update_elem(&egress_sessions, ekey, &sess, BPF_NOEXIST); if (err) { /* on failure, clean up the ingress slot we reserved earlier */ diff --git a/CubeNet/src/tcp.h b/CubeNet/src/tcp.h index cba5953b5..e989a40f1 100644 --- a/CubeNet/src/tcp.h +++ b/CubeNet/src/tcp.h @@ -195,7 +195,15 @@ static const u8 tcp_conntracks[2][6][TCP_CONNTRACK_MAX] = { } }; -static unsigned int get_conntrack_index(bool syn, bool ack, bool fin, bool rst) +/* Force inlining: with a single call site inside update_session clang would + * usually inline anyway, but if it ever emits a real BPF subprog the verifier + * must track the returned index through the subprog boundary — extra burden + * on the already-fragile 3D tcp_conntracks[dir][index][old_state] access. + * Inlining lets clang constant-propagate index when the flag inputs are + * compile-time constants (as in the tcp_state test macro and the production + * do_tcp_nat literal-dir call), collapsing one array dimension. + */ +static __always_inline unsigned int get_conntrack_index(bool syn, bool ack, bool fin, bool rst) { if (rst) return TCP_RST_SET; else if (syn) return (ack ? TCP_SYNACK_SET : TCP_SYN_SET); @@ -290,6 +298,31 @@ static __always_inline void update_session(enum ip_conntrack_dir dir, struct nat } new_state = tcp_conntracks[dir][index][old_state]; + + if (index == TCP_FIN_SET) { + /* A retransmitted FIN from the side that initiated close must not be + * mistaken for the peer's FIN. The generic conntrack table cannot + * distinguish direction once it reaches FIN_WAIT/CLOSE_WAIT, so use + * active_close to retain the state until the opposite side sends FIN. + */ + if ((old_state == TCP_CONNTRACK_FIN_WAIT || + old_state == TCP_CONNTRACK_CLOSE_WAIT) && + ((sess->active_close && dir == IP_CT_DIR_ORIGINAL) || + (!sess->active_close && dir == IP_CT_DIR_REPLY))) + new_state = old_state; + + /* Record only a real original-direction transition that initiates + * close. Checking the classified packet and computed transition avoids + * marking RST|FIN or SYN|FIN packets as active closes. If reply sent + * FIN first, the state is already FIN_WAIT/CLOSE_WAIT when original + * later sends FIN and active_close remains zero. + */ + if (dir == IP_CT_DIR_ORIGINAL && + new_state == TCP_CONNTRACK_FIN_WAIT && + new_state != old_state) + sess->active_close = 1; + } + /* no store if state remain unchanged */ if (new_state != old_state) sess->state = new_state; @@ -298,10 +331,11 @@ static __always_inline void update_session(enum ip_conntrack_dir dir, struct nat static __always_inline bool create_new_sessions(struct __sk_buff *skb, struct session_key *ekey, __u64 now_ns, __u32 vm_ifindex, - struct snat_ip *snat_ip, __u16 snat_port) + struct snat_ip *snat_ip, __u16 snat_port, + __u8 packet_class, __u8 l7_scheme) { return create_nat_session(skb, ekey, now_ns, vm_ifindex, snat_ip, snat_port, - TCP_CONNTRACK_SYN_SENT); + TCP_CONNTRACK_SYN_SENT, packet_class, l7_scheme); } #endif /* __TCP_H */ diff --git a/CubeNet/src/tcp_state_test.bpf.c b/CubeNet/src/tcp_state_test.bpf.c new file mode 100644 index 000000000..e959e4a12 --- /dev/null +++ b/CubeNet/src/tcp_state_test.bpf.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (c) 2026 Cube Authors */ +#include +#include +#include +#include +#include + +#include "tcp.h" + +struct tcp_update_step { + __u8 dir; + __u8 syn; + __u8 ack; + __u8 fin; + __u8 rst; + __u8 reserved[3]; +}; + +struct tcp_update_case { + __u64 access_time; + __u64 now_ns; + __u8 state; + __u8 active_close; + __u8 step_count; + __u8 reserved[5]; + struct tcp_update_step steps[2]; +}; + +#define APPLY_TCP_STEP(DIR, SESS, NOW, STEP) do { \ + if ((STEP).rst) \ + update_session((DIR), (SESS), (NOW), false, false, false, true); \ + else if ((STEP).syn && (STEP).ack) \ + update_session((DIR), (SESS), (NOW), true, true, false, false); \ + else if ((STEP).syn) \ + update_session((DIR), (SESS), (NOW), true, false, false, false); \ + else if ((STEP).fin) \ + update_session((DIR), (SESS), (NOW), false, false, true, false); \ + else if ((STEP).ack) \ + update_session((DIR), (SESS), (NOW), false, true, false, false); \ + else \ + update_session((DIR), (SESS), (NOW), false, false, false, false); \ +} while (0) + +SEC("tc") +int test_update_session(struct __sk_buff *skb) +{ + struct tcp_update_case tc = {}; + struct nat_session sess = {}; + int i; + + if (bpf_skb_load_bytes(skb, 0, &tc, sizeof(tc))) + return TC_ACT_SHOT; + if (tc.state > TCP_CONNTRACK_SYN_SENT2 || tc.step_count > 2) + return TC_ACT_SHOT; + + sess.access_time = tc.access_time; + sess.state = tc.state; + sess.active_close = tc.active_close; + +#pragma unroll + for (i = 0; i < 2; i++) { + if (i < tc.step_count) { + if (tc.steps[i].dir == IP_CT_DIR_ORIGINAL) + APPLY_TCP_STEP(IP_CT_DIR_ORIGINAL, &sess, tc.now_ns + i, + tc.steps[i]); + else if (tc.steps[i].dir == IP_CT_DIR_REPLY) + APPLY_TCP_STEP(IP_CT_DIR_REPLY, &sess, tc.now_ns + i, + tc.steps[i]); + else + return TC_ACT_SHOT; + } + } + + tc.access_time = sess.access_time; + tc.state = sess.state; + tc.active_close = sess.active_close; + if (bpf_skb_store_bytes(skb, 0, &tc, sizeof(tc), 0)) + return TC_ACT_SHOT; + + return TC_ACT_OK; +} + +char __license[] SEC("license") = "Dual BSD/GPL"; diff --git a/CubeNet/src/udp.h b/CubeNet/src/udp.h index fc17c7482..0b06bde7f 100644 --- a/CubeNet/src/udp.h +++ b/CubeNet/src/udp.h @@ -24,7 +24,7 @@ static __always_inline bool create_udp_sessions(struct __sk_buff *skb, struct snat_ip *snat_ip, __u16 snat_port) { return create_nat_session(skb, ekey, now_ns, vm_ifindex, snat_ip, snat_port, - UDP_CT_UNREPLIED); + UDP_CT_UNREPLIED, SNAT_PACKET, L7_SCHEME_NONE); } #endif /* __UDP_H */ From 26c09b14ead4e07065817cd88801a8f5be121184 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Sun, 9 Aug 2026 21:50:36 +0800 Subject: [PATCH 02/31] feat(cubelet): port L7 custom-port runtime into embedded network runtime Port the L7 custom-port control plane from L7_support_portconfig (gitwoa network-agent) onto the embedded Cubelet network runtime: - cubebox.proto (Cubelet + CubeMaster): optional int32 port = 8 on EgressRuleMatch; regenerate pb.go with protoc v5.28.3 header kept - runtime/types.go: EgressRuleMatch.Port *int with the port+scheme pairing contract (port requires scheme; both omitted means the legacy {80/http, 443/https} default set) - plugin_policy.go: map proto port into runtime match - cubeegress wire/adapter: carry Port into MatchInput and render it as a numeric JSON field for access_phase.lua dst_port comparison - policy_builder.go: project SNI+Host rules into []cubevs.L7Target (host, port, scheme) with per-rule port/scheme validation (extractL7PortScheme, defaultPortForScheme); an invalid pair rejects the whole projection so cubevs and CubeEgress never diverge - controller.go: loadL7MarksConfig overlays CUBE_L7_MARK_{HTTP,HTTPS,MASK} from /etc/cubeegress/l7-marks.conf onto cubevs.Params before Init, tolerating shell-legal export prefixes and inline comments - tests: TestExtractL7PortScheme matrix (13 cases), whole-policy rejection, L7Target projection shape, TestLoadL7MarksConfig (6 cases) Note: push/delete semantics keep the restructured fail-fast model from #1285 (pushEgressForState failure aborts creation; Cleaning state retries) instead of the legacy best-effort pendingEgressPush model. Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- .../api/services/cubebox/v1/cubebox.pb.go | 30 ++- .../api/services/cubebox/v1/cubebox.proto | 3 + Cubelet/api/services/cubebox/v1/cubebox.pb.go | 32 ++- Cubelet/api/services/cubebox/v1/cubebox.proto | 5 + Cubelet/network/plugin_policy.go | 7 +- Cubelet/network/runtime/controller.go | 62 ++++++ Cubelet/network/runtime/controller_test.go | 90 ++++++++ Cubelet/network/runtime/cubeegress/wire.go | 7 + Cubelet/network/runtime/cubeegress_adapter.go | 7 +- Cubelet/network/runtime/cubevs_adapter.go | 10 +- Cubelet/network/runtime/policy_builder.go | 128 ++++++++++-- .../network/runtime/policy_builder_test.go | 196 +++++++++++++++++- Cubelet/network/runtime/types.go | 8 + 13 files changed, 540 insertions(+), 45 deletions(-) diff --git a/CubeMaster/api/services/cubebox/v1/cubebox.pb.go b/CubeMaster/api/services/cubebox/v1/cubebox.pb.go index 12e239418..77413d739 100644 --- a/CubeMaster/api/services/cubebox/v1/cubebox.pb.go +++ b/CubeMaster/api/services/cubebox/v1/cubebox.pb.go @@ -3152,12 +3152,15 @@ func (x *EgressRule) GetAction() *EgressRuleAction { } type EgressRuleMatch struct { - state protoimpl.MessageState `protogen:"open.v1"` - Sni *string `protobuf:"bytes,1,opt,name=sni,proto3,oneof" json:"sni,omitempty"` - Host *string `protobuf:"bytes,3,opt,name=host,proto3,oneof" json:"host,omitempty"` - Method []string `protobuf:"bytes,4,rep,name=method,proto3" json:"method,omitempty"` - Path *string `protobuf:"bytes,5,opt,name=path,proto3,oneof" json:"path,omitempty"` - Scheme *string `protobuf:"bytes,7,opt,name=scheme,proto3,oneof" json:"scheme,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Sni *string `protobuf:"bytes,1,opt,name=sni,proto3,oneof" json:"sni,omitempty"` + Host *string `protobuf:"bytes,3,opt,name=host,proto3,oneof" json:"host,omitempty"` + Method []string `protobuf:"bytes,4,rep,name=method,proto3" json:"method,omitempty"` + Path *string `protobuf:"bytes,5,opt,name=path,proto3,oneof" json:"path,omitempty"` + Scheme *string `protobuf:"bytes,7,opt,name=scheme,proto3,oneof" json:"scheme,omitempty"` + // L7 destination port. When set, `scheme` MUST also be set — together they + // pin the (host, port, scheme) tuple CubeEgress intercepts. + Port *int32 `protobuf:"varint,8,opt,name=port,proto3,oneof" json:"port,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -3227,6 +3230,13 @@ func (x *EgressRuleMatch) GetScheme() string { return "" } +func (x *EgressRuleMatch) GetPort() int32 { + if x != nil && x.Port != nil { + return *x.Port + } + return 0 +} + type EgressRuleAction struct { state protoimpl.MessageState `protogen:"open.v1"` Allow bool `protobuf:"varint,1,opt,name=allow,proto3" json:"allow,omitempty"` @@ -6792,17 +6802,19 @@ const file_api_services_cubebox_v1_cubebox_proto_rawDesc = "" + "\x05match\x18\x02 \x01(\v2,.cubelet.services.cubebox.v1.EgressRuleMatchH\x00R\x05match\x88\x01\x01\x12J\n" + "\x06action\x18\x03 \x01(\v2-.cubelet.services.cubebox.v1.EgressRuleActionH\x01R\x06action\x88\x01\x01B\b\n" + "\x06_matchB\t\n" + - "\a_action\"\xb4\x01\n" + + "\a_action\"\xd6\x01\n" + "\x0fEgressRuleMatch\x12\x15\n" + "\x03sni\x18\x01 \x01(\tH\x00R\x03sni\x88\x01\x01\x12\x17\n" + "\x04host\x18\x03 \x01(\tH\x01R\x04host\x88\x01\x01\x12\x16\n" + "\x06method\x18\x04 \x03(\tR\x06method\x12\x17\n" + "\x04path\x18\x05 \x01(\tH\x02R\x04path\x88\x01\x01\x12\x1b\n" + - "\x06scheme\x18\a \x01(\tH\x03R\x06scheme\x88\x01\x01B\x06\n" + + "\x06scheme\x18\a \x01(\tH\x03R\x06scheme\x88\x01\x01\x12\x17\n" + + "\x04port\x18\b \x01(\x05H\x04R\x04port\x88\x01\x01B\x06\n" + "\x04_sniB\a\n" + "\x05_hostB\a\n" + "\x05_pathB\t\n" + - "\a_scheme\"\x94\x01\n" + + "\a_schemeB\a\n" + + "\x05_port\"\x94\x01\n" + "\x10EgressRuleAction\x12\x14\n" + "\x05allow\x18\x01 \x01(\bR\x05allow\x12\x19\n" + "\x05audit\x18\x02 \x01(\tH\x00R\x05audit\x88\x01\x01\x12E\n" + diff --git a/CubeMaster/api/services/cubebox/v1/cubebox.proto b/CubeMaster/api/services/cubebox/v1/cubebox.proto index ae3599adc..13ba0279f 100644 --- a/CubeMaster/api/services/cubebox/v1/cubebox.proto +++ b/CubeMaster/api/services/cubebox/v1/cubebox.proto @@ -620,6 +620,9 @@ message EgressRuleMatch { repeated string method = 4; optional string path = 5; optional string scheme = 7; + // L7 destination port. When set, `scheme` MUST also be set — together they + // pin the (host, port, scheme) tuple CubeEgress intercepts. + optional int32 port = 8; } message EgressRuleAction { diff --git a/Cubelet/api/services/cubebox/v1/cubebox.pb.go b/Cubelet/api/services/cubebox/v1/cubebox.pb.go index 3505a187f..9c88b780b 100644 --- a/Cubelet/api/services/cubebox/v1/cubebox.pb.go +++ b/Cubelet/api/services/cubebox/v1/cubebox.pb.go @@ -3095,12 +3095,17 @@ func (x *EgressRule) GetAction() *EgressRuleAction { } type EgressRuleMatch struct { - state protoimpl.MessageState `protogen:"open.v1"` - Sni *string `protobuf:"bytes,1,opt,name=sni,proto3,oneof" json:"sni,omitempty"` - Host *string `protobuf:"bytes,3,opt,name=host,proto3,oneof" json:"host,omitempty"` - Method []string `protobuf:"bytes,4,rep,name=method,proto3" json:"method,omitempty"` - Path *string `protobuf:"bytes,5,opt,name=path,proto3,oneof" json:"path,omitempty"` - Scheme *string `protobuf:"bytes,7,opt,name=scheme,proto3,oneof" json:"scheme,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Sni *string `protobuf:"bytes,1,opt,name=sni,proto3,oneof" json:"sni,omitempty"` + Host *string `protobuf:"bytes,3,opt,name=host,proto3,oneof" json:"host,omitempty"` + Method []string `protobuf:"bytes,4,rep,name=method,proto3" json:"method,omitempty"` + Path *string `protobuf:"bytes,5,opt,name=path,proto3,oneof" json:"path,omitempty"` + Scheme *string `protobuf:"bytes,7,opt,name=scheme,proto3,oneof" json:"scheme,omitempty"` + // L7 destination port. When set, `scheme` MUST also be set — together they + // pin the (host, port, scheme) tuple CubeEgress intercepts via skb->mark + + // iptables TPROXY. Both omitted keeps the legacy default {80/http, 443/https} + // behavior. + Port *int32 `protobuf:"varint,8,opt,name=port,proto3,oneof" json:"port,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -3170,6 +3175,13 @@ func (x *EgressRuleMatch) GetScheme() string { return "" } +func (x *EgressRuleMatch) GetPort() int32 { + if x != nil && x.Port != nil { + return *x.Port + } + return 0 +} + type EgressRuleAction struct { state protoimpl.MessageState `protogen:"open.v1"` Allow bool `protobuf:"varint,1,opt,name=allow,proto3" json:"allow,omitempty"` @@ -6757,17 +6769,19 @@ const file_api_services_cubebox_v1_cubebox_proto_rawDesc = "" + "\x05match\x18\x02 \x01(\v2,.cubelet.services.cubebox.v1.EgressRuleMatchH\x00R\x05match\x88\x01\x01\x12J\n" + "\x06action\x18\x03 \x01(\v2-.cubelet.services.cubebox.v1.EgressRuleActionH\x01R\x06action\x88\x01\x01B\b\n" + "\x06_matchB\t\n" + - "\a_action\"\xb4\x01\n" + + "\a_action\"\xd6\x01\n" + "\x0fEgressRuleMatch\x12\x15\n" + "\x03sni\x18\x01 \x01(\tH\x00R\x03sni\x88\x01\x01\x12\x17\n" + "\x04host\x18\x03 \x01(\tH\x01R\x04host\x88\x01\x01\x12\x16\n" + "\x06method\x18\x04 \x03(\tR\x06method\x12\x17\n" + "\x04path\x18\x05 \x01(\tH\x02R\x04path\x88\x01\x01\x12\x1b\n" + - "\x06scheme\x18\a \x01(\tH\x03R\x06scheme\x88\x01\x01B\x06\n" + + "\x06scheme\x18\a \x01(\tH\x03R\x06scheme\x88\x01\x01\x12\x17\n" + + "\x04port\x18\b \x01(\x05H\x04R\x04port\x88\x01\x01B\x06\n" + "\x04_sniB\a\n" + "\x05_hostB\a\n" + "\x05_pathB\t\n" + - "\a_scheme\"\x94\x01\n" + + "\a_schemeB\a\n" + + "\x05_port\"\x94\x01\n" + "\x10EgressRuleAction\x12\x14\n" + "\x05allow\x18\x01 \x01(\bR\x05allow\x12\x19\n" + "\x05audit\x18\x02 \x01(\tH\x00R\x05audit\x88\x01\x01\x12E\n" + diff --git a/Cubelet/api/services/cubebox/v1/cubebox.proto b/Cubelet/api/services/cubebox/v1/cubebox.proto index 96b303a4c..8ef1c14a7 100644 --- a/Cubelet/api/services/cubebox/v1/cubebox.proto +++ b/Cubelet/api/services/cubebox/v1/cubebox.proto @@ -610,6 +610,11 @@ message EgressRuleMatch { repeated string method = 4; optional string path = 5; optional string scheme = 7; + // L7 destination port. When set, `scheme` MUST also be set — together they + // pin the (host, port, scheme) tuple CubeEgress intercepts via skb->mark + + // iptables TPROXY. Both omitted keeps the legacy default {80/http, 443/https} + // behavior. + optional int32 port = 8; } message EgressRuleAction { diff --git a/Cubelet/network/plugin_policy.go b/Cubelet/network/plugin_policy.go index 8b0f64474..1dec27c7b 100644 --- a/Cubelet/network/plugin_policy.go +++ b/Cubelet/network/plugin_policy.go @@ -67,13 +67,18 @@ func mapRunRequestEgressRuleMatch(in *cubebox.EgressRuleMatch) *networkruntime.E if in == nil { return nil } - return &networkruntime.EgressRuleMatch{ + out := &networkruntime.EgressRuleMatch{ SNI: in.Sni, Host: in.Host, Method: append([]string(nil), in.GetMethod()...), Path: in.Path, Scheme: in.Scheme, } + if in.Port != nil { + p := int(*in.Port) + out.Port = &p + } + return out } func mapRunRequestEgressRuleAction(in *cubebox.EgressRuleAction) *networkruntime.EgressRuleAction { diff --git a/Cubelet/network/runtime/controller.go b/Cubelet/network/runtime/controller.go index bf2ae04ee..e9cdcc348 100644 --- a/Cubelet/network/runtime/controller.go +++ b/Cubelet/network/runtime/controller.go @@ -11,6 +11,7 @@ import ( "net" "os" "slices" + "strconv" "strings" "sync" "time" @@ -292,6 +293,9 @@ func initCubeVS(cfg Config, device *systemnet.HostDevice, cubeDev *systemnet.Cub NodeMacAddr: device.Mac, NodeGatewayMacAddr: device.GatewayMac, } + if err := loadL7MarksConfig(¶ms); err != nil { + return nil, err + } if err := cubevs.Init(params); err != nil { return nil, err } @@ -307,6 +311,64 @@ func initCubeVS(cfg Config, device *systemnet.HostDevice, cubeDev *systemnet.Cub return cubeRouter, nil } +// l7MarksConfigPath is the install-time config shared with the +// cube-proxy-iptables-init script, so the dataplane (eBPF globals) and the +// iptables TPROXY rules stamp/match the same skb->mark values. It is a var so +// tests can point it at a temp file instead of the real /etc path. +var l7MarksConfigPath = "/etc/cubeegress/l7-marks.conf" + +// loadL7MarksConfig overlays CUBE_L7_MARK_{HTTP,HTTPS,MASK} from +// l7MarksConfigPath onto params.L7Mark*. A missing file leaves the shipped +// defaults (cubevs.resolveL7Marks applies them); unset keys likewise fall +// back to defaults. Values are hex (e.g. 0xCE010000), matching the shell +// KEY=VALUE format the iptables script sources. Because the iptables script +// sources the same file as POSIX shell, hand-edited but shell-legal lines +// are tolerated here too: an "export " key prefix and a trailing +// " # comment" on the value. +func loadL7MarksConfig(params *cubevs.Params) error { + data, err := os.ReadFile(l7MarksConfigPath) // NOCC:Path Traversal() + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, raw, ok := strings.Cut(line, "=") + if !ok { + continue + } + key = strings.TrimSpace(key) + if strings.HasPrefix(key, "export") { + // `export KEY=value` — without this the key would match no case + // below and the override would be silently ignored, diverging + // the dataplane marks from the iptables rules. + key = strings.TrimSpace(strings.TrimPrefix(key, "export")) + } + // Strip a trailing ` # comment` before parsing; without this the + // ParseUint below would fail and block controller startup. + raw, _, _ = strings.Cut(raw, "#") + text := strings.Trim(strings.TrimSpace(raw), `"'`) + value, perr := strconv.ParseUint(text, 0, 32) + if perr != nil { + return fmt.Errorf("parse %q in %s: %w", line, l7MarksConfigPath, perr) + } + switch key { + case "CUBE_L7_MARK_HTTP": + params.L7MarkHTTP = uint32(value) + case "CUBE_L7_MARK_HTTPS": + params.L7MarkHTTPS = uint32(value) + case "CUBE_L7_MARK_MASK": + params.L7MarkMask = uint32(value) + } + } + return nil +} + func startCubeVSSessionLogDrain() { sessionEvents := cubevs.StartSessionReaper() go func() { diff --git a/Cubelet/network/runtime/controller_test.go b/Cubelet/network/runtime/controller_test.go index e5f06cec1..b83d9e0d7 100644 --- a/Cubelet/network/runtime/controller_test.go +++ b/Cubelet/network/runtime/controller_test.go @@ -5,9 +5,11 @@ import ( "errors" "net" "os" + "path/filepath" "reflect" "testing" + "github.com/tencentcloud/CubeSandbox/CubeNet/cubevs" "github.com/tencentcloud/CubeSandbox/Cubelet/network/runtime/systemnet" ) @@ -760,3 +762,91 @@ func TestNewNetworkControllerFromDepsRequiresAdapters(t *testing.T) { t.Fatal("expected missing adapter error") } } + +func TestLoadL7MarksConfig(t *testing.T) { + withPath := func(t *testing.T, content *string) { + t.Helper() + old := l7MarksConfigPath + t.Cleanup(func() { l7MarksConfigPath = old }) + if content == nil { + l7MarksConfigPath = filepath.Join(t.TempDir(), "absent.conf") + return + } + p := filepath.Join(t.TempDir(), "l7-marks.conf") + if err := os.WriteFile(p, []byte(*content), 0o644); err != nil { + t.Fatalf("write temp conf: %v", err) + } + l7MarksConfigPath = p + } + + t.Run("absent file leaves defaults", func(t *testing.T) { + withPath(t, nil) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err != nil { + t.Fatalf("absent file: %v", err) + } + if p.L7MarkHTTP != 0 || p.L7MarkHTTPS != 0 || p.L7MarkMask != 0 { + t.Fatalf("absent file set marks: %+v", p) + } + }) + + t.Run("override is applied", func(t *testing.T) { + conf := "# comment\nCUBE_L7_MARK_HTTP=0xCF010000\nCUBE_L7_MARK_HTTPS=0xCF020000\nCUBE_L7_MARK_MASK=0xFFFF0000\n" + withPath(t, &conf) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err != nil { + t.Fatalf("override: %v", err) + } + if p.L7MarkHTTP != 0xCF010000 || p.L7MarkHTTPS != 0xCF020000 || p.L7MarkMask != 0xFFFF0000 { + t.Fatalf("override: got http=%#x https=%#x mask=%#x", p.L7MarkHTTP, p.L7MarkHTTPS, p.L7MarkMask) + } + }) + + t.Run("partial override keeps others zero", func(t *testing.T) { + conf := "CUBE_L7_MARK_HTTP=0xCF030000\n" + withPath(t, &conf) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err != nil { + t.Fatalf("partial: %v", err) + } + if p.L7MarkHTTP != 0xCF030000 || p.L7MarkHTTPS != 0 || p.L7MarkMask != 0 { + t.Fatalf("partial: got http=%#x https=%#x mask=%#x", p.L7MarkHTTP, p.L7MarkHTTPS, p.L7MarkMask) + } + }) + + t.Run("malformed value errors", func(t *testing.T) { + conf := "CUBE_L7_MARK_HTTP=not-a-number\n" + withPath(t, &conf) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err == nil { + t.Fatal("malformed value was not rejected") + } + }) + + t.Run("export-prefixed key is honored", func(t *testing.T) { + // Shell-legal `export KEY=value` must not be silently ignored — + // otherwise the dataplane stamps a different mark than iptables + // matches. + conf := "export CUBE_L7_MARK_HTTP=0xCF010000\nexport CUBE_L7_MARK_MASK=0xFFFF0000\n" + withPath(t, &conf) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err != nil { + t.Fatalf("export-prefixed: %v", err) + } + if p.L7MarkHTTP != 0xCF010000 || p.L7MarkMask != 0xFFFF0000 { + t.Fatalf("export-prefixed: got http=%#x mask=%#x", p.L7MarkHTTP, p.L7MarkMask) + } + }) + + t.Run("inline comment on value is stripped", func(t *testing.T) { + conf := "CUBE_L7_MARK_HTTP=0xCF030000 # http listener mark\n" + withPath(t, &conf) + var p cubevs.Params + if err := loadL7MarksConfig(&p); err != nil { + t.Fatalf("inline comment: %v", err) + } + if p.L7MarkHTTP != 0xCF030000 { + t.Fatalf("inline comment: got http=%#x", p.L7MarkHTTP) + } + }) +} diff --git a/Cubelet/network/runtime/cubeegress/wire.go b/Cubelet/network/runtime/cubeegress/wire.go index ca22697ac..c25fbc3a5 100644 --- a/Cubelet/network/runtime/cubeegress/wire.go +++ b/Cubelet/network/runtime/cubeegress/wire.go @@ -59,6 +59,7 @@ type MatchInput struct { Method []string Path *string Scheme *string + Port *int } // ActionInput mirrors runtime.EgressRuleAction. @@ -175,6 +176,12 @@ func renderMatch(m *MatchInput) map[string]any { if m.Scheme != nil { out["scheme"] = *m.Scheme } + if m.Port != nil { + // CubeEgress's access_phase.lua rule_matches compares match.port + // against ctx.dst_port (from tproxy's $server_port). Emit as a + // number so cjson decodes it to a numeric type in openresty. + out["port"] = *m.Port + } return out } diff --git a/Cubelet/network/runtime/cubeegress_adapter.go b/Cubelet/network/runtime/cubeegress_adapter.go index 2fcd3a48d..006ba88a5 100644 --- a/Cubelet/network/runtime/cubeegress_adapter.go +++ b/Cubelet/network/runtime/cubeegress_adapter.go @@ -73,13 +73,18 @@ func toMatchInput(m *EgressRuleMatch) *cubeegress.MatchInput { if m == nil { return nil } - return &cubeegress.MatchInput{ + out := &cubeegress.MatchInput{ SNI: m.SNI, Host: m.Host, Method: append([]string(nil), m.Method...), Path: m.Path, Scheme: m.Scheme, } + if m.Port != nil { + p := *m.Port + out.Port = &p + } + return out } // toActionInput deep-copies the action section, including header-injection diff --git a/Cubelet/network/runtime/cubevs_adapter.go b/Cubelet/network/runtime/cubevs_adapter.go index e371e6ee3..8b69c1385 100644 --- a/Cubelet/network/runtime/cubevs_adapter.go +++ b/Cubelet/network/runtime/cubevs_adapter.go @@ -86,7 +86,10 @@ func (realCubeVSAdapter) DeletePortMappingsByIfindex(ifindex uint32) error { // registerCubeVSTap writes the complete TAP metadata and policy options into // CubeVS for a newly-created sandbox. func (s *NetworkController) registerCubeVSTap(ifindex int, ip net.IP, sandboxID string, cfg *CubeNetworkConfig) (err error) { - opts := cubeVSTapRegistration(cfg) + opts, err := cubeVSTapRegistration(cfg) + if err != nil { + return err + } CubeLog.WithContext(context.Background()).Infof( "network runtime register cubevs tap: sandbox_id=%s ifindex=%d sandbox_ip=%s cube_network_config=%s allow_internet_access=%v allow_out=%v l7_allow_out=%v deny_out=%v", sandboxID, @@ -115,7 +118,10 @@ func (s *NetworkController) registerCubeVSTap(ifindex int, ip net.IP, sandboxID // Legacy recovery uses this to avoid carrying old allow/deny/DNS residue into // the recovered Active sandbox. func (s *NetworkController) replaceCubeVSTap(ifindex int, ip net.IP, sandboxID string, cfg *CubeNetworkConfig) error { - opts := cubeVSTapRegistration(cfg) + opts, err := cubeVSTapRegistration(cfg) + if err != nil { + return err + } CubeLog.WithContext(context.Background()).Infof( "network runtime replace cubevs tap: sandbox_id=%s ifindex=%d sandbox_ip=%s cube_network_config=%s allow_internet_access=%v allow_out=%v l7_allow_out=%v deny_out=%v", sandboxID, diff --git a/Cubelet/network/runtime/policy_builder.go b/Cubelet/network/runtime/policy_builder.go index 75095bf8c..ed4b17d32 100644 --- a/Cubelet/network/runtime/policy_builder.go +++ b/Cubelet/network/runtime/policy_builder.go @@ -5,6 +5,7 @@ package runtime import ( + "errors" "fmt" "net" "strings" @@ -94,10 +95,10 @@ func formatCubeNetworkConfig(in *CubeNetworkConfig) string { // allow_internet_access / allow_out / deny_out, and it also receives network // targets extracted from L7 rules as L7 allow targets. The complete L7 rules // are still pushed to CubeEgress separately. -func cubeVSTapRegistration(cfg *CubeNetworkConfig) cubevs.MVMOptions { +func cubeVSTapRegistration(cfg *CubeNetworkConfig) (cubevs.MVMOptions, error) { if cfg == nil { allowInternetAccess := true - return cubevs.MVMOptions{AllowInternetAccess: &allowInternetAccess} + return cubevs.MVMOptions{AllowInternetAccess: &allowInternetAccess}, nil } opts := cubevs.MVMOptions{} if cfg.AllowInternetAccess != nil { @@ -111,46 +112,135 @@ func cubeVSTapRegistration(cfg *CubeNetworkConfig) cubevs.MVMOptions { allowOut := append([]string(nil), cfg.AllowOut...) opts.AllowOut = &allowOut } - if l7AllowOut := extractL7AllowOutTargetsFromRules(cfg.Rules); len(l7AllowOut) > 0 { + l7AllowOut, err := extractL7AllowOutTargetsFromRules(cfg.Rules) + if err != nil { + return cubevs.MVMOptions{}, err + } + if len(l7AllowOut) > 0 { opts.L7AllowOut = &l7AllowOut } if len(cfg.DenyOut) > 0 { denyOut := append([]string(nil), cfg.DenyOut...) opts.DenyOut = &denyOut } - return opts + return opts, nil } -// extractL7AllowOutTargetsFromRules converts SNI/Host matches into the coarse -// network targets that cubevs needs to allow before CubeEgress can inspect L7 -// traffic. Invalid or non-IPv4-looking targets are ignored rather than failing -// sandbox creation; the full rule is still validated by CubeEgress on push. -func extractL7AllowOutTargetsFromRules(rules []*EgressRule) []string { - seen := make(map[string]struct{}) - targets := make([]string, 0, len(rules)) - add := func(target string, ok bool) { +// extractL7AllowOutTargetsFromRules walks the L7 rule list and produces the +// (host, port, scheme) tuples cubevs needs for its per-host dns_allow_v2 / +// allow_out_v3 port set. SNI-only rules and rules without a Match.Host are +// treated as legacy port-agnostic entries — they expand to {80/http, 443/https} +// downstream in buildL7Plan. Rules that specify a Port MUST also specify a +// scheme ("http" or "https"). Any invalid port/scheme pair rejects the whole +// projection so CubeVS and CubeEgress cannot observe different policies. +// +// Duplicate (host, port, scheme) tuples are deduplicated to keep the map +// value's port_count within maxL7PortsPerHost when a user attaches the same +// port to several rules. +func extractL7AllowOutTargetsFromRules(rules []*EgressRule) ([]cubevs.L7Target, error) { + type key struct { + host string + port uint16 + scheme uint8 + } + seen := make(map[key]struct{}) + targets := make([]cubevs.L7Target, 0, len(rules)) + add := func(host string, ok bool, port uint16, scheme uint8) { if !ok { return } - if _, exists := seen[target]; exists { + k := key{host, port, scheme} + if _, exists := seen[k]; exists { return } - seen[target] = struct{}{} - targets = append(targets, target) + seen[k] = struct{}{} + targets = append(targets, cubevs.L7Target{Host: host, Port: port, Scheme: scheme}) } - for _, rule := range rules { + for i, rule := range rules { if rule == nil || rule.Match == nil { continue } + port, scheme, err := extractL7PortScheme(rule.Match) + if err != nil { + return nil, fmt.Errorf("network.rules[%d] %q: %w", i, rule.Name, err) + } + // A rule carrying both SNI and Host projects BOTH as L7 targets (SNI + // first, then Host), not Host alone. if rule.Match.SNI != nil { - add(normalizeL7DomainTarget(*rule.Match.SNI)) + host, ok := normalizeL7DomainTarget(*rule.Match.SNI) + add(host, ok, port, scheme) } if rule.Match.Host != nil { - add(normalizeL7HostTarget(*rule.Match.Host)) + host, ok := normalizeL7HostTarget(*rule.Match.Host) + add(host, ok, port, scheme) } } - return targets + return targets, nil +} + +// extractL7PortScheme reads Match.Port + Match.Scheme and normalises them into +// cubevs.L7Target's numeric representation. +// +// - Port set, Scheme nil → invalid (port without scheme cannot decide the +// nginx listener); reject. +// - Port nil, Scheme set → fill in the scheme's default port (http → 80, +// https → 443). Callers that only want to say "https on this host" can +// omit port and get the conventional default. +// - Port set, Scheme set → new port-scoped feature: exact tuple. +// - Both nil → legacy: buildL7Plan expands to +// {80/http, 443/https}. +// +// Any recognised, non-empty Scheme string is normalised to lowercase and +// stripped of surrounding whitespace before comparison — the wire form is +// case-insensitive. +func extractL7PortScheme(match *EgressRuleMatch) (uint16, uint8, error) { + if match.Port == nil && match.Scheme == nil { + // Legacy: buildL7Plan will expand to {80/http, 443/https}. + return 0, cubevs.L7SchemeNone, nil + } + + // Scheme presence guides port validation. Decode it first (nil is fine). + var schemeValue uint8 = cubevs.L7SchemeNone + if match.Scheme != nil { + switch strings.ToLower(strings.TrimSpace(*match.Scheme)) { + case "http": + schemeValue = cubevs.L7SchemeHTTP + case "https": + schemeValue = cubevs.L7SchemeHTTPS + default: + return 0, 0, fmt.Errorf("scheme must be http or https, got %q", *match.Scheme) + } + } + + if match.Port == nil { + // Scheme only → fill in scheme's canonical default port. + return defaultPortForScheme(schemeValue), schemeValue, nil + } + + if match.Scheme == nil { + return 0, 0, errors.New("port requires scheme") + } + + p := *match.Port + if p <= 0 || p > 65535 { + return 0, 0, fmt.Errorf("port must be in [1, 65535], got %d", p) + } + return uint16(p), schemeValue, nil +} + +// defaultPortForScheme returns the conventional port for a bare scheme. +// http → 80, https → 443. Any other scheme value returns 0 (should never be +// reached: extractL7PortScheme rejects unknown schemes before calling this). +func defaultPortForScheme(scheme uint8) uint16 { + switch scheme { + case cubevs.L7SchemeHTTP: + return 80 + case cubevs.L7SchemeHTTPS: + return 443 + default: + return 0 + } } // normalizeL7DomainTarget canonicalizes a DNS name or wildcard suffix for the diff --git a/Cubelet/network/runtime/policy_builder_test.go b/Cubelet/network/runtime/policy_builder_test.go index 911c5bfeb..6a3722589 100644 --- a/Cubelet/network/runtime/policy_builder_test.go +++ b/Cubelet/network/runtime/policy_builder_test.go @@ -1,6 +1,11 @@ package runtime -import "testing" +import ( + "strings" + "testing" + + "github.com/tencentcloud/CubeSandbox/CubeNet/cubevs" +) func TestCubeVSTapRegistrationBuildsL3L4AndL7Options(t *testing.T) { allowInternet := false @@ -19,7 +24,10 @@ func TestCubeVSTapRegistrationBuildsL3L4AndL7Options(t *testing.T) { }}, } - opts := cubeVSTapRegistration(cfg) + opts, err := cubeVSTapRegistration(cfg) + if err != nil { + t.Fatalf("cubeVSTapRegistration error=%v", err) + } if opts.AllowInternetAccess == nil || *opts.AllowInternetAccess != false { t.Fatalf("AllowInternetAccess = %#v, want false", opts.AllowInternetAccess) } @@ -32,8 +40,188 @@ func TestCubeVSTapRegistrationBuildsL3L4AndL7Options(t *testing.T) { if opts.L7AllowOut == nil || len(*opts.L7AllowOut) != 2 { t.Fatalf("L7AllowOut = %#v", opts.L7AllowOut) } - if (*opts.L7AllowOut)[0] != "api.example.com" || (*opts.L7AllowOut)[1] != "10.1.2.3" { - t.Fatalf("L7AllowOut = %#v", *opts.L7AllowOut) + got := *opts.L7AllowOut + if got[0].Host != "api.example.com" || got[1].Host != "10.1.2.3" { + t.Fatalf("L7AllowOut = %#v", got) + } + // No explicit port/scheme on the rules: legacy default targets. + for i, tgt := range got { + if tgt.Port != 0 || tgt.Scheme != cubevs.L7SchemeNone { + t.Fatalf("L7AllowOut[%d] = {Port:%d Scheme:%d}, want legacy default {0/L7SchemeNone}", + i, tgt.Port, tgt.Scheme) + } + } +} + +func TestCubeVSTapRegistrationBlockAll(t *testing.T) { + allowInternet := false + opts, err := cubeVSTapRegistration(&CubeNetworkConfig{ + AllowInternetAccess: &allowInternet, + }) + if err != nil { + t.Fatalf("cubeVSTapRegistration error=%v", err) + } + if opts.AllowInternetAccess == nil || *opts.AllowInternetAccess != false { + t.Fatalf("opts.AllowInternetAccess=%v, want false", opts.AllowInternetAccess) + } +} + +func TestCubeVSTapRegistrationExtractsL7AllowOut(t *testing.T) { + sni := "API.Example.COM." + sniWildcard := "*.SNI.Example.COM" + hostIP := "1.2.3.4:443" + hostCIDR := "10.1.2.3/8" + hostDomain := "Gateway.Example.COM:8443" + hostWildcard := "*.Gateway.Example.COM" + duplicateHost := "gateway.example.com" + invalidHost := "999.999.999.999" + opts, err := cubeVSTapRegistration(&CubeNetworkConfig{ + AllowOut: []string{"8.8.8.8"}, + Rules: []*EgressRule{ + {Match: &EgressRuleMatch{SNI: &sni, Host: &hostIP}}, + {Match: &EgressRuleMatch{Host: &hostCIDR}}, + {Match: &EgressRuleMatch{Host: &hostDomain}}, + {Match: &EgressRuleMatch{SNI: &sni}}, + {Match: &EgressRuleMatch{SNI: &sniWildcard}}, + {Match: &EgressRuleMatch{Host: &hostWildcard}}, + {Match: &EgressRuleMatch{Host: &duplicateHost}}, + {Match: &EgressRuleMatch{Host: &invalidHost}}, + {Match: &EgressRuleMatch{Path: stringPtr("/v1/chat")}}, + }, + }) + if err != nil { + t.Fatalf("cubeVSTapRegistration error=%v", err) + } + if opts.AllowOut == nil || len(*opts.AllowOut) != 1 || (*opts.AllowOut)[0] != "8.8.8.8" { + t.Fatalf("opts.AllowOut=%v, want [8.8.8.8]", opts.AllowOut) + } + if opts.L7AllowOut == nil { + t.Fatal("opts.L7AllowOut=nil, want extracted targets") + } + // Rules without explicit Port + Scheme land as legacy default targets + // (Port=0, Scheme=L7SchemeNone) — downstream buildL7Plan expands them + // to {80/http, 443/https}. Assert host projection only; port/scheme + // coverage is in TestExtractL7PortScheme. + // The first rule carries both SNI and Host: BOTH are projected (SNI + // first, then Host), so "api.example.com" (from SNI) and "1.2.3.4" (from + // Host) both appear; the later duplicate SNI is deduplicated. + wantHosts := []string{ + "api.example.com", "1.2.3.4", "10.0.0.0/8", "gateway.example.com", + "*.sni.example.com", "*.gateway.example.com", + } + gotTargets := *opts.L7AllowOut + if len(gotTargets) != len(wantHosts) { + t.Fatalf("opts.L7AllowOut host count=%d, want %d (%+v)", + len(gotTargets), len(wantHosts), gotTargets) + } + for i, w := range wantHosts { + if gotTargets[i].Host != w { + t.Fatalf("opts.L7AllowOut[%d].Host=%q, want %q", i, gotTargets[i].Host, w) + } + if gotTargets[i].Port != 0 || gotTargets[i].Scheme != cubevs.L7SchemeNone { + t.Fatalf("opts.L7AllowOut[%d] expected legacy default (0/L7SchemeNone), got Port=%d Scheme=%d", + i, gotTargets[i].Port, gotTargets[i].Scheme) + } + } +} + +// TestExtractL7PortScheme covers the four accepted rule shapes plus the +// error paths. Rules missing one of {Port, Scheme}, using an unknown scheme +// string, or an out-of-range port must be dropped rather than silently +// promoted to the legacy default set — that would let a typo like "htps" fall +// back to port 443 instead of surfacing. +func TestExtractL7PortScheme(t *testing.T) { + intPtr := func(i int) *int { return &i } + + tests := []struct { + name string + match EgressRuleMatch + wantPort uint16 + wantScheme uint8 + wantErr bool + }{ + {"both nil (legacy)", EgressRuleMatch{}, 0, cubevs.L7SchemeNone, false}, + {"port only rejected", + EgressRuleMatch{Port: intPtr(8080)}, 0, 0, true}, + {"http alone fills default port 80", + EgressRuleMatch{Scheme: stringPtr("http")}, 80, cubevs.L7SchemeHTTP, false}, + {"https alone fills default port 443", + EgressRuleMatch{Scheme: stringPtr("https")}, 443, cubevs.L7SchemeHTTPS, false}, + {"https mixed case alone fills 443", + EgressRuleMatch{Scheme: stringPtr("HTTPS")}, 443, cubevs.L7SchemeHTTPS, false}, + {"http lowercase", + EgressRuleMatch{Port: intPtr(8080), Scheme: stringPtr("http")}, + 8080, cubevs.L7SchemeHTTP, false}, + {"https mixed case", + EgressRuleMatch{Port: intPtr(8443), Scheme: stringPtr("HTTPS")}, + 8443, cubevs.L7SchemeHTTPS, false}, + {"scheme with whitespace", + EgressRuleMatch{Port: intPtr(80), Scheme: stringPtr(" http ")}, + 80, cubevs.L7SchemeHTTP, false}, + {"unknown scheme rejected", + EgressRuleMatch{Port: intPtr(80), Scheme: stringPtr("gopher")}, + 0, 0, true}, + {"unknown scheme alone rejected", + EgressRuleMatch{Scheme: stringPtr("gopher")}, 0, 0, true}, + {"port too low rejected", + EgressRuleMatch{Port: intPtr(0), Scheme: stringPtr("http")}, + 0, 0, true}, + {"port too high rejected", + EgressRuleMatch{Port: intPtr(65536), Scheme: stringPtr("http")}, + 0, 0, true}, + {"port negative rejected", + EgressRuleMatch{Port: intPtr(-1), Scheme: stringPtr("http")}, + 0, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + port, scheme, err := extractL7PortScheme(&tt.match) + if (err != nil) != tt.wantErr { + t.Fatalf("err=%v, wantErr=%v", err, tt.wantErr) + } + if err != nil { + return + } + if port != tt.wantPort { + t.Fatalf("port=%d, want %d", port, tt.wantPort) + } + if scheme != tt.wantScheme { + t.Fatalf("scheme=%d, want %d", scheme, tt.wantScheme) + } + }) + } +} + +func TestCubeVSTapRegistrationRejectsEntireInvalidL7Policy(t *testing.T) { + host := "api.example.com" + https := "https" + validPort := 8443 + invalidPort := 8080 + + opts, err := cubeVSTapRegistration(&CubeNetworkConfig{ + AllowOut: []string{"8.8.8.8"}, + Rules: []*EgressRule{ + { + Name: "valid-rule", + Match: &EgressRuleMatch{ + Host: &host, Port: &validPort, Scheme: &https, + }, + }, + { + Name: "invalid-rule", + Match: &EgressRuleMatch{Host: &host, Port: &invalidPort}, + }, + }, + }) + if err == nil { + t.Fatal("cubeVSTapRegistration error=nil, want invalid policy error") + } + if !strings.Contains(err.Error(), `network.rules[1] "invalid-rule"`) || + !strings.Contains(err.Error(), "port requires scheme") { + t.Fatalf("error=%q, want rule location and reason", err) + } + if opts.AllowOut != nil || opts.L7AllowOut != nil || opts.DenyOut != nil { + t.Fatalf("opts=%+v, want empty options on whole-policy rejection", opts) } } diff --git a/Cubelet/network/runtime/types.go b/Cubelet/network/runtime/types.go index cbe0ccccc..6ee398105 100644 --- a/Cubelet/network/runtime/types.go +++ b/Cubelet/network/runtime/types.go @@ -122,12 +122,20 @@ type EgressRule struct { // EgressRuleMatch holds the per-request match conditions for an EgressRule. // All fields are optional; an empty match matches any request. +// +// Port and Scheme together control which TCP port CubeEgress intercepts on the +// sandbox side. When both are omitted the rule applies to the default set +// {80/http, 443/https}. When Port is set, Scheme MUST also be set to "http" or +// "https" — every rule for the same (host, port) tuple must agree on the +// scheme, because iptables can only steer a single tuple to one TPROXY +// listener. type EgressRuleMatch struct { SNI *string `json:"sni,omitempty"` Host *string `json:"host,omitempty"` Method []string `json:"method,omitempty"` Path *string `json:"path,omitempty"` Scheme *string `json:"scheme,omitempty"` + Port *int `json:"port,omitempty"` } // EgressRuleAction holds the action taken when an EgressRule matches. From 5c9f374bbf9d9eeaf131f87597e8b3b69ed7d37f Mon Sep 17 00:00:00 2001 From: hankzhou Date: Sun, 9 Aug 2026 21:55:30 +0800 Subject: [PATCH 03/31] feat(cubeegress): port L7 custom-port lua policy and iptables mark validation Port the CubeEgress half of the L7 custom-port feature from L7_support_portconfig (gitwoa): - lua/port_scheme.lua (new): (port, scheme) normalisation, validation and default-set expansion shared by policy and access phases - lua/policy.lua: validate and render match.port / match.scheme on egress rules - lua/access_phase.lua: compare match.port against ctx.dst_port from the tproxy listener so custom-port rules only match their own tuple - scripts/cube-proxy-iptables-init.sh: validate CUBE_L7_MARK_{HTTP, HTTPS,MASK} (marks must differ, bits within mask) before installing TPROXY rules - Makefile: test-lua runs port_scheme_test and port_scheme_extra_test - tests/: port_scheme_test.lua, port_scheme_extra_test.lua and cube-proxy-iptables-init_test.sh (all passing) All four modified files are unchanged since the branch base (43fa0bf); the #1285 admin-port move (9090->9091) touched admin.lua, nginx.conf and start.sh only, so no reconciliation was needed. Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- CubeEgress/Makefile | 7 + CubeEgress/lua/access_phase.lua | 16 +- CubeEgress/lua/policy.lua | 113 +++++++++++++- CubeEgress/lua/port_scheme.lua | 76 +++++++++ .../scripts/cube-proxy-iptables-init.sh | 100 +++++++++--- .../tests/cube-proxy-iptables-init_test.sh | 146 ++++++++++++++++++ CubeEgress/tests/port_scheme_extra_test.lua | 81 ++++++++++ CubeEgress/tests/port_scheme_test.lua | 114 ++++++++++++++ 8 files changed, 631 insertions(+), 22 deletions(-) create mode 100644 CubeEgress/lua/port_scheme.lua create mode 100644 CubeEgress/tests/cube-proxy-iptables-init_test.sh create mode 100644 CubeEgress/tests/port_scheme_extra_test.lua create mode 100644 CubeEgress/tests/port_scheme_test.lua diff --git a/CubeEgress/Makefile b/CubeEgress/Makefile index e9200b070..1b7ea4b9e 100644 --- a/CubeEgress/Makefile +++ b/CubeEgress/Makefile @@ -27,6 +27,13 @@ else MAKEFLAGS += --no-print-directory endif +.PHONY: test-lua +test-lua: + $(Q)LUA_BIN=$$(command -v luajit || command -v lua); \ + test -n "$$LUA_BIN" || { echo "error: lua or luajit not installed"; exit 1; }; \ + $$LUA_BIN tests/port_scheme_test.lua && \ + $$LUA_BIN tests/port_scheme_extra_test.lua + .PHONY: build build: $(call msg,BUILD IMAGE $(IMAGE_LOCAL):$(IMAGE_TAG)-$(ARCH)) diff --git a/CubeEgress/lua/access_phase.lua b/CubeEgress/lua/access_phase.lua index 9b30b9005..afc2f2eda 100644 --- a/CubeEgress/lua/access_phase.lua +++ b/CubeEgress/lua/access_phase.lua @@ -68,6 +68,7 @@ -- deployments use "skipped" + admin API for policy installation. local policy = require "policy" +local port_scheme = require "port_scheme" local _M = {} @@ -143,8 +144,11 @@ local function rule_matches(m, ctx) if m.path ~= nil then if not path_match(m.path, ctx.path) then return false end end - if m.scheme ~= nil then - if string.lower(m.scheme) ~= ctx.scheme then return false end + -- Port and scheme form one semantic constraint. Omitted fields mean the + -- backward-compatible default set {80/http,443/https}, not a wildcard + -- over every custom port intercepted for the same host. + if not port_scheme.matches(m.port, m.scheme, ctx.dst_port, ctx.scheme) then + return false end return true end @@ -294,6 +298,13 @@ local function build_ctx() method = ngx.var.request_method, path = ngx.var.uri, dst_ip = dst_ip, + -- dst_port is the original sandbox-side destination port preserved + -- by TPROXY. In an IP_TRANSPARENT listener, nginx's $server_port is + -- read via getsockname() on the tproxy socket, which reports the + -- ORIGINAL dst — not the 8080/8443 listener port we bind to. Used + -- by rule_matches() to enforce match.port constraints on rules that + -- pin to a custom port (e.g. tcp/8443 for an internal API). + dst_port = tonumber(ngx.var.server_port), scheme = ngx.var.scheme, } end @@ -408,6 +419,7 @@ function _M.decide() method = ctx.method, path = ctx.path, dst_ip = ctx.dst_ip, + dst_port = ctx.dst_port, scheme = ctx.scheme, policy_id = nil, rule_id = nil, diff --git a/CubeEgress/lua/policy.lua b/CubeEgress/lua/policy.lua index bc0127e7a..5d9e96502 100644 --- a/CubeEgress/lua/policy.lua +++ b/CubeEgress/lua/policy.lua @@ -19,6 +19,7 @@ local cjson = require "cjson.safe" -- resty.openssl.digest is already used by cert_signer.lua (sha256 leaf -- signing), so the runtime cost of pulling it in here is amortized. local digest_lib = require "resty.openssl.digest" +local port_scheme = require "port_scheme" local _M = {} @@ -31,6 +32,7 @@ local INDEX_LOCK_TIMEOUT_MS = 1000 -- per-entry limit; anything larger is almost certainly a misconfiguration -- (and would bloat policy_store fast). local SECRET_MAX_BYTES = 65536 +local MAX_L7_PORTS_PER_HOST = 8 -- ---------- validation ---------- @@ -50,8 +52,114 @@ local function is_valid_sandbox_ip(s) return true end +-- Strict IPv4 (+optional /prefix) parse, mirroring Go's netip acceptance: +-- no leading zeros, octets <= 255, prefix <= 32. Returns the canonical +-- dotted-quad and prefix (default 32), or nil. +local function parse_ipv4_cidr(s) + local a, b, c, d, prefix = string.match(s, "^([0-9]+)%.([0-9]+)%.([0-9]+)%.([0-9]+)/([0-9]+)$") + if not a then + a, b, c, d = string.match(s, IPV4_PATTERN) + end + if not a then return nil end + local octets = {} + for _, octet in ipairs({a, b, c, d}) do + if #octet > 1 and string.sub(octet, 1, 1) == "0" then return nil end + local n = tonumber(octet) + if not n or n > 255 then return nil end + octets[#octets + 1] = tostring(n) + end + if prefix then + prefix = tonumber(prefix) + if not prefix or prefix > 32 then return nil end + else + prefix = 32 + end + return table.concat(octets, "."), prefix +end + +-- Canonicalize a rule host/sni for conflict-detection aggregation. +-- Returns the canonical identity, or (nil, reason) when the value must be +-- rejected outright. Kept in lock-step with Go's l7GroupKey/classifyL7Target: +-- - DNS names: lowercase + strip one trailing dot (no whitespace trim on +-- either side — padded values are rejected, not repaired). +-- - "1.2.3.4" and "1.2.3.4/32" map to the same identity (Go groups both +-- via parseCIDR), so a per-(host,port) scheme conflict between the two +-- spellings is detected instead of bypassed. +-- - Subnet CIDRs (prefixlen < 32) are rejected, as Go's classifyL7Target +-- does: an L7 rule pins exactly one (host, port) listener tuple, and a +-- subnet can never appear in an HTTP Host header or TLS SNI anyway. +local function normalize_identity(value) + if type(value) ~= "string" or value == "" then return nil, "empty" end + -- Reject whitespace-padded values: request-time matching compares the + -- raw host/sni, so a padded identity would validate yet never match — + -- fail fast instead of accepting a dead rule. + if string.find(value, "^%s") or string.find(value, "%s$") then + return nil, "leading or trailing whitespace" + end + local identity = string.gsub(string.lower(value), "%.$", "") + if identity == "" then return nil, "empty" end + + if string.find(identity, "/", 1, true) then + -- CIDR notation: only a /32 IPv4 literal is meaningful here. + local ip, prefix = parse_ipv4_cidr(identity) + if not ip then return nil, "invalid CIDR notation" end + if prefix < 32 then return nil, "subnet CIDR is not a valid L7 host" end + return ip + end + -- Dotted-quad-shaped values (digits and dots only) must be valid IPv4; + -- canonicalize so bare and /32 spellings aggregate identically. + if string.match(identity, "^[0-9.]+$") then + local ip = parse_ipv4_cidr(identity) + if not ip then return nil, "invalid IPv4 address" end + return ip + end + return identity +end + +local function validate_match_tuples(match, rule_index, identities) + local tuples, err = port_scheme.expand(match.port, match.scheme) + if not tuples then + return false, string.format("rules[%d].match.%s", rule_index, err) + end + + -- When both identities are present, Host is the policy aggregation key; + -- otherwise SNI is used. Request-time matching still evaluates both fields. + local raw_identity = match.host + if raw_identity == nil then raw_identity = match.sni end + if raw_identity == nil then return true end + local identity, identity_err = normalize_identity(raw_identity) + if not identity then + return false, string.format("rules[%d].match host/sni %q is invalid: %s", + rule_index, raw_identity, identity_err or "empty") + end + + local state = identities[identity] + if not state then + state = {count = 0, ports = {}} + identities[identity] = state + end + for _, tuple in ipairs(tuples) do + local existing = state.ports[tuple.port] + if existing ~= nil and existing ~= tuple.scheme then + return false, string.format( + "rules[%d].match conflicts for host %q port %d: %s vs %s", + rule_index, identity, tuple.port, existing, tuple.scheme) + end + if existing == nil then + state.ports[tuple.port] = tuple.scheme + state.count = state.count + 1 + if state.count > MAX_L7_PORTS_PER_HOST then + return false, string.format( + "rules[%d].match exceeds %d L7 port tuples for host %q", + rule_index, MAX_L7_PORTS_PER_HOST, identity) + end + end + end + return true +end + -- Validate policy structure. Returns (true, nil) or (false, err_string). --- Strict on required fields; permissive on optional (forward-compat). +-- Strict on required fields; permissive on unrelated optional fields. local function validate_policy(p) if type(p) ~= "table" then return false, "policy must be an object" end if type(p.policy_id) ~= "string" or p.policy_id == "" then @@ -67,6 +175,7 @@ local function validate_policy(p) return false, "policy.rules must have at least one rule" end local seen_ids = {} + local identities = {} for i = 1, n do local r = p.rules[i] if type(r) ~= "table" then @@ -82,6 +191,8 @@ local function validate_policy(p) if type(r.match) ~= "table" then return false, "rules[" .. i .. "].match required (object; empty {} allowed)" end + local match_ok, match_err = validate_match_tuples(r.match, i, identities) + if not match_ok then return false, match_err end if type(r.action) ~= "table" then return false, "rules[" .. i .. "].action required (object)" end diff --git a/CubeEgress/lua/port_scheme.lua b/CubeEgress/lua/port_scheme.lua new file mode 100644 index 000000000..1faf5c09a --- /dev/null +++ b/CubeEgress/lua/port_scheme.lua @@ -0,0 +1,76 @@ +-- CubeEgress L7 destination port/scheme semantics shared by validation and +-- request matching. +local _M = {} + +local function trim(s) + return (string.gsub(s, "^%s*(.-)%s*$", "%1")) +end + +function _M.normalize_scheme(value) + if value == nil then return nil end + if type(value) ~= "string" then + return nil, "scheme must be a string" + end + local scheme = string.lower(trim(value)) + if scheme ~= "http" and scheme ~= "https" then + return nil, "scheme must be http or https" + end + return scheme +end + +local function normalize_port(value) + if value == nil then return nil end + if type(value) ~= "number" or value ~= math.floor(value) then + return nil, "port must be an integer" + end + if value < 1 or value > 65535 then + return nil, "port must be in [1, 65535]" + end + return value +end + +-- expand returns the effective destination tuples for a rule: +-- nil/nil -> 80/http + 443/https +-- nil/scheme -> the scheme's conventional port +-- port/scheme -> the exact tuple +-- port/nil -> invalid +function _M.expand(port_value, scheme_value) + local port, perr = normalize_port(port_value) + if perr then return nil, perr end + local scheme, serr = _M.normalize_scheme(scheme_value) + if serr then return nil, serr end + + if port ~= nil and scheme == nil then + return nil, "port requires scheme" + end + if port ~= nil then + return {{port = port, scheme = scheme}} + end + if scheme == "http" then + return {{port = 80, scheme = "http"}} + end + if scheme == "https" then + return {{port = 443, scheme = "https"}} + end + return { + {port = 80, scheme = "http"}, + {port = 443, scheme = "https"}, + } +end + +function _M.matches(port_value, scheme_value, dst_port, request_scheme) + local tuples = _M.expand(port_value, scheme_value) + if not tuples then return false end + local scheme = _M.normalize_scheme(request_scheme) + local port = tonumber(dst_port) + if not scheme or not port then return false end + + for _, tuple in ipairs(tuples) do + if tuple.port == port and tuple.scheme == scheme then + return true + end + end + return false +end + +return _M diff --git a/CubeEgress/scripts/cube-proxy-iptables-init.sh b/CubeEgress/scripts/cube-proxy-iptables-init.sh index bf73063af..acf67ddf5 100755 --- a/CubeEgress/scripts/cube-proxy-iptables-init.sh +++ b/CubeEgress/scripts/cube-proxy-iptables-init.sh @@ -3,12 +3,19 @@ # CubeSandbox transparent proxy — host-side network setup. # Phase 1: full MITM via OpenResty + TPROXY. # -# Selection model: traffic is matched purely by ingress interface and -# destination port — `iif cube-dev` + tcp dport 80/443. There is NO -# fwmark involved in either the forward (sandbox→OpenResty) or the -# return (OpenResty→sandbox) direction. The cube-egress worker's -# replies are routed naturally by the kernel and re-injected into the -# sandbox tap by the from_envoy BPF program on cube-dev egress. +# Selection model: traffic is matched by ingress interface and by an +# skb->mark stamped from the sandbox tap's eBPF datapath. The mvmtap +# program reads allow_out_v3 to find the (host, port) → scheme mapping +# for the outgoing SYN, then writes CUBE_L7_MARK_HTTP (0xCE010000) or +# CUBE_L7_MARK_HTTPS (0xCE020000) so this chain can steer the packet at +# 8080 (nginx HTTP listener) or 8443 (nginx HTTPS listener) regardless +# of the original destination port. This lets users configure L7 +# capture on arbitrary ports (e.g. an API on tcp/3000) without teaching +# iptables about their port map. +# +# The cube-owned mark uses the CUBE_L7_MARK_MASK (0xFFFF0000) high-16 +# range so the low 16 bits remain free for host-level marks users may +# set for other purposes. # # Idempotent: safe to re-run. Rules live in a dedicated TRANSPROXY # sub-chain so 'down' tears down our config without touching anything @@ -64,6 +71,38 @@ SANDBOX_NETWORK_CIDR="${CUBE_SANDBOX_NETWORK_CIDR:-192.168.0.0/18}" TPROXY_ON_IP="$(sandbox_gateway_ip_from_cidr "${SANDBOX_NETWORK_CIDR}")" # cube-dev IP TPROXY_PORT_HTTP=8080 TPROXY_PORT_HTTPS=8443 +# skb->mark values written by the mvmtap L7 proxy path (the cube_l7_mark_http / +# cube_l7_mark_https / cube_l7_mark_mask globals in CubeNet/src/cubevs.h). Only +# the high 16 bits are cube-owned; the mask lets users co-exist with other +# host-level fwmark schemes on the low 16 bits. +# +# Defaults match the shipped values; a deployment may override them via +# /etc/cubeegress/l7-marks.conf so these rules and the dataplane (which reads +# the same file into its eBPF globals) stay in lock-step. +CUBE_L7_MARK_HTTP=0xCE010000 +CUBE_L7_MARK_HTTPS=0xCE020000 +CUBE_L7_MARK_MASK=0xFFFF0000 +if [ -f /etc/cubeegress/l7-marks.conf ]; then + # shellcheck disable=SC1091 + . /etc/cubeegress/l7-marks.conf +fi +# Validate the (possibly overridden) marks: http must differ from https, and +# both may only set bits inside the mask. Compare arithmetically (not as +# strings) so the same value in different notations — 0xCE010000 vs +# 0xce010000 vs 3456172032 — is still rejected, matching the uint32 +# comparison in cubevs.resolveL7Marks. +validate_l7_marks() { + if (( CUBE_L7_MARK_HTTP == CUBE_L7_MARK_HTTPS )); then + echo "cube-proxy-iptables-init: CUBE_L7_MARK_HTTP (${CUBE_L7_MARK_HTTP}) must differ from CUBE_L7_MARK_HTTPS" >&2 + exit 1 + fi + if [ $(( CUBE_L7_MARK_HTTP & ~CUBE_L7_MARK_MASK )) -ne 0 ] || \ + [ $(( CUBE_L7_MARK_HTTPS & ~CUBE_L7_MARK_MASK )) -ne 0 ]; then + echo "cube-proxy-iptables-init: L7 marks must set bits only within CUBE_L7_MARK_MASK (${CUBE_L7_MARK_MASK})" >&2 + exit 1 + fi +} +validate_l7_marks ROUTE_TABLE=100 INGRESS_IFACE="${CUBE_INGRESS_IFACE:-cube-dev}" CHAIN="TRANSPROXY" @@ -89,25 +128,31 @@ install_chain() { iptables -t mangle -C PREROUTING -j "${CHAIN}" 2>/dev/null \ || iptables -t mangle -A PREROUTING -j "${CHAIN}" + # HTTP: mvmtap stamps CUBE_L7_MARK_HTTP → steer to nginx HTTP listener. iptables -t mangle -A "${CHAIN}" \ - -i "${INGRESS_IFACE}" -p tcp --dport 80 \ + -i "${INGRESS_IFACE}" -p tcp \ + -m mark --mark "${CUBE_L7_MARK_HTTP}/${CUBE_L7_MARK_MASK}" \ -j TPROXY --on-ip "${TPROXY_ON_IP}" --on-port "${TPROXY_PORT_HTTP}" + # HTTPS: mvmtap stamps CUBE_L7_MARK_HTTPS → steer to nginx HTTPS listener. iptables -t mangle -A "${CHAIN}" \ - -i "${INGRESS_IFACE}" -p tcp --dport 443 \ + -i "${INGRESS_IFACE}" -p tcp \ + -m mark --mark "${CUBE_L7_MARK_HTTPS}/${CUBE_L7_MARK_MASK}" \ -j TPROXY --on-ip "${TPROXY_ON_IP}" --on-port "${TPROXY_PORT_HTTPS}" iptables -t mangle -A "${CHAIN}" -j RETURN } install_routing() { - # Two ip rules: tcp/80 and tcp/443 from cube-dev → table 100. - # Match by selectors (iif/ipproto/dport), not fwmark. - local proto port - for port in 80 443; do + # Two ip rules: cube-owned mark bits → table 100. Match on fwmark so the + # rule set stays independent of the original destination port; user rules + # may attach L7 handling to arbitrary ports (e.g. tcp/3000) and mvmtap + # writes the same mark for all of them. + local mark + for mark in "${CUBE_L7_MARK_HTTP}" "${CUBE_L7_MARK_HTTPS}"; do if ! ip rule show \ - | grep -q "iif ${INGRESS_IFACE} ipproto tcp dport ${port} lookup ${ROUTE_TABLE}"; then - ip rule add iif "${INGRESS_IFACE}" ipproto tcp dport "${port}" \ + | grep -qiE "fwmark ${mark}/${CUBE_L7_MARK_MASK} lookup ${ROUTE_TABLE}[[:space:]]*$"; then + ip rule add fwmark "${mark}/${CUBE_L7_MARK_MASK}" \ table "${ROUTE_TABLE}" fi done @@ -127,17 +172,30 @@ remove_chain() { } remove_routing() { - local port - for port in 80 443; do + local mark + for mark in "${CUBE_L7_MARK_HTTP}" "${CUBE_L7_MARK_HTTPS}"; do while ip rule show \ - | grep -q "iif ${INGRESS_IFACE} ipproto tcp dport ${port} lookup ${ROUTE_TABLE}"; do - ip rule del iif "${INGRESS_IFACE}" ipproto tcp dport "${port}" \ + | grep -qiE "fwmark ${mark}/${CUBE_L7_MARK_MASK} lookup ${ROUTE_TABLE}[[:space:]]*$"; do + ip rule del fwmark "${mark}/${CUBE_L7_MARK_MASK}" \ table "${ROUTE_TABLE}" || break done done ip route flush table "${ROUTE_TABLE}" 2>/dev/null || true } +# Remove policy-routing selectors installed by the pre-fwmark implementation. +# Delete only the exact cube-dev TCP/80 and TCP/443 selectors; unrelated rules +# using table 100 are outside this script's ownership. +remove_legacy_dport_routing() { + local port + for port in 80 443; do + while ip rule del iif "${INGRESS_IFACE}" ipproto tcp dport "${port}" \ + table "${ROUTE_TABLE}" 2>/dev/null; do + : + done + done +} + show_status() { log "=== mangle/${CHAIN} ===" iptables -t mangle -L "${CHAIN}" -n -v --line-numbers 2>/dev/null \ @@ -163,6 +221,7 @@ main() { require_modules install_chain install_routing + remove_legacy_dport_routing log "cube-proxy iptables/route rules installed" show_status ;; @@ -170,6 +229,7 @@ main() { require_root remove_chain remove_routing + remove_legacy_dport_routing log "cube-proxy iptables/route rules removed" ;; status) @@ -182,4 +242,6 @@ main() { esac } -main "$@" +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/CubeEgress/tests/cube-proxy-iptables-init_test.sh b/CubeEgress/tests/cube-proxy-iptables-init_test.sh new file mode 100644 index 000000000..4de75e4e1 --- /dev/null +++ b/CubeEgress/tests/cube-proxy-iptables-init_test.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=../scripts/cube-proxy-iptables-init.sh +source "${ROOT_DIR}/scripts/cube-proxy-iptables-init.sh" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT +CALLS="${TMP_DIR}/calls" + +# Each legacy selector exists twice. The third deletion fails and terminates +# the loop. Record every attempted exact command. +declare -A DELETE_COUNT=([80]=0 [443]=0) +ip() { + printf '%s\n' "$*" >> "${CALLS}" + if [[ "$1 $2" == "rule del" ]]; then + local port="" + local i + for ((i = 1; i <= $#; i++)); do + if [[ "${!i}" == "dport" ]]; then + local next=$((i + 1)) + port="${!next}" + break + fi + done + [[ -n "${port}" ]] || return 1 + DELETE_COUNT["${port}"]=$((DELETE_COUNT["${port}"] + 1)) + (( DELETE_COUNT["${port}"] <= 2 )) + return + fi + return 1 +} + +remove_legacy_dport_routing +[[ "${DELETE_COUNT[80]}" -eq 3 ]] +[[ "${DELETE_COUNT[443]}" -eq 3 ]] +grep -Fxq "rule del iif cube-dev ipproto tcp dport 80 table 100" "${CALLS}" +grep -Fxq "rule del iif cube-dev ipproto tcp dport 443 table 100" "${CALLS}" +if grep -Ev '^rule del iif cube-dev ipproto tcp dport (80|443) table 100$' "${CALLS}"; then + echo "unexpected legacy cleanup command" >&2 + exit 1 +fi + +# --- fwmark ip-rule idempotency is case-insensitive -------------------------- +# iproute2 prints fwmark hex in lowercase (0xce010000/0xffff0000), but the +# script greps for the uppercase CUBE_L7_MARK_* constants. The grep must be +# case-insensitive, or install_routing re-adds duplicate rules on every run and +# remove_routing never matches anything to delete. +RULES_STATE="${TMP_DIR}/rules" +: > "${RULES_STATE}" +: > "${CALLS}" +ip() { + printf '%s\n' "$*" >> "${CALLS}" + case "$1 $2" in + "rule show") + cat "${RULES_STATE}" + return 0 + ;; + "rule add") + # $3=fwmark $4=mark/mask $5=table $6=; store iproute2-style + # lowercase show output, as the real `ip rule show` would print. + printf 'from all fwmark %s lookup %s\n' "$4" "$6" \ + | tr '[:upper:]' '[:lower:]' >> "${RULES_STATE}" + return 0 + ;; + "rule del") + local needle + needle="$(printf '%s' "$4" | tr '[:upper:]' '[:lower:]')" + grep -viF "fwmark ${needle} " "${RULES_STATE}" > "${RULES_STATE}.tmp" || true + mv "${RULES_STATE}.tmp" "${RULES_STATE}" + return 0 + ;; + "route "*) + return 0 + ;; + esac + return 0 +} + +# Re-running install must not duplicate rules: the grep matches the lowercase +# `ip rule show` output, so the second run is a no-op. +install_routing +install_routing +[[ "$(grep -c 'rule add fwmark' "${CALLS}" || true)" -eq 2 ]] +[[ "$(wc -l < "${RULES_STATE}")" -eq 2 ]] + +# remove must match (and so delete) every installed rule. +remove_routing +[[ "$(grep -c 'rule del fwmark' "${CALLS}" || true)" -eq 2 ]] +[[ ! -s "${RULES_STATE}" ]] + +# Verify migration ordering without touching the host network. +: > "${CALLS}" +require_root() { :; } +require_iface() { :; } +require_modules() { :; } +install_chain() { echo install_chain >> "${CALLS}"; } +install_routing() { echo install_routing >> "${CALLS}"; } +remove_legacy_dport_routing() { echo remove_legacy >> "${CALLS}"; } +show_status() { echo show_status >> "${CALLS}"; } +main up +EXPECTED=$'install_chain\ninstall_routing\nremove_legacy\nshow_status' +[[ "$(<"${CALLS}")" == "${EXPECTED}" ]] + +# --- marks validation compares arithmetically, not as strings ---------------- +# Go resolveL7Marks compares uint32 values; the shell check must reject the +# same mark written in a different notation (hex case variant or decimal), +# otherwise iptables and the dataplane diverge on identical marks. +# validate_l7_marks exits the (sub)shell on rejection, so a subshell that +# survives means the value was (wrongly) accepted. +if ( + CUBE_L7_MARK_HTTP=0xCE010000 + CUBE_L7_MARK_HTTPS=0xce010000 # same value, different case + validate_l7_marks +); then + echo "case-variant identical marks were not rejected" >&2 + exit 1 +fi +if ( + CUBE_L7_MARK_HTTP=0xCE010000 + CUBE_L7_MARK_HTTPS=3456172032 # decimal equivalent of 0xCE010000 + validate_l7_marks +); then + echo "decimal-equivalent identical marks were not rejected" >&2 + exit 1 +fi +# distinct values within the mask still pass +( + CUBE_L7_MARK_HTTP=0xCE010000 + CUBE_L7_MARK_HTTPS=0xCE020000 + CUBE_L7_MARK_MASK=0xFFFF0000 + validate_l7_marks +) +# bits outside the mask are still rejected (0x0000CE00 sets low-16 bits) +if ( + CUBE_L7_MARK_HTTP=0x0000CE00 + CUBE_L7_MARK_HTTPS=0xCE020000 + CUBE_L7_MARK_MASK=0xFFFF0000 + validate_l7_marks +); then + echo "out-of-mask mark was not rejected" >&2 + exit 1 +fi + +printf 'cube-proxy-iptables-init_test: PASS\n' diff --git a/CubeEgress/tests/port_scheme_extra_test.lua b/CubeEgress/tests/port_scheme_extra_test.lua new file mode 100644 index 000000000..0e78cdf9a --- /dev/null +++ b/CubeEgress/tests/port_scheme_extra_test.lua @@ -0,0 +1,81 @@ +-- Complementary tests for CubeEgress custom-port semantics. +-- Focuses on port_scheme.expand (the branch table behind every +-- rule match) which port_scheme_test.lua exercises only indirectly +-- via port_scheme.matches. Run: lua tests/port_scheme_extra_test.lua +package.path = "lua/?.lua;" .. package.path + +local port_scheme = require "port_scheme" + +local function assert_true(value, message) + if not value then error(message or "expected true") end +end + +local function assert_false(value, message) + if value then error(message or "expected false") end +end + +-- Find a tuple by port in an expand() result. +local function tuple_for(tuples, port) + if not tuples then return nil end + for _, t in ipairs(tuples) do + if t.port == port then return t end + end + return nil +end + +-- Default rule (no port/scheme) -> {80/http, 443/https}. +local def, err = port_scheme.expand(nil, nil) +assert_false(err, "expand(nil, nil) should succeed") +assert_true(#def == 2, "default should yield 2 tuples, got " .. #def) +assert_true(tuple_for(def, 80) and tuple_for(def, 80).scheme == "http", "default missing 80/http") +assert_true(tuple_for(def, 443) and tuple_for(def, 443).scheme == "https", "default missing 443/https") + +-- scheme-only http -> {80/http}. +local only_http = port_scheme.expand(nil, "http") +assert_true(#only_http == 1 and only_http[1].port == 80 and only_http[1].scheme == "http", + "scheme=http should yield exactly {80/http}") + +-- scheme-only https -> {443/https}. +local only_https = port_scheme.expand(nil, "https") +assert_true(#only_https == 1 and only_https[1].port == 443 and only_https[1].scheme == "https", + "scheme=https should yield exactly {443/https}") + +-- explicit custom port + scheme -> exact single tuple. +local custom = port_scheme.expand(8080, "http") +assert_true(#custom == 1 and custom[1].port == 8080 and custom[1].scheme == "http", + "port=8080+scheme=http should yield exactly {8080/http}") +local custom_https = port_scheme.expand(8443, "https") +assert_true(#custom_https == 1 and custom_https[1].port == 8443 and custom_https[1].scheme == "https", + "port=8443+scheme=https should yield exactly {8443/https}") + +-- port without scheme is invalid. +local bad, bad_err = port_scheme.expand(8080, nil) +assert_false(bad, "port without scheme must be rejected") +assert_true(bad_err and string.find(bad_err, "port requires scheme", 1, true) ~= nil, + "port-without-scheme error must mention 'port requires scheme'") + +-- unknown scheme is invalid. +local bad_scheme, bs_err = port_scheme.expand(nil, "ftp") +assert_false(bad_scheme, "scheme=ftp must be rejected") +assert_true(bs_err and string.find(bs_err, "scheme must be http or https", 1, true) ~= nil, + "unknown scheme error must mention allowed values") + +-- out-of-range ports are invalid. +local zero, z_err = port_scheme.expand(0, "http") +assert_false(zero, "port=0 must be rejected") +local too_big, tb_err = port_scheme.expand(70000, "http") +assert_false(too_big, "port=70000 must be rejected") + +-- scheme normalization is case/whitespace insensitive. +local upper = port_scheme.expand(nil, "HTTPS") +assert_true(#upper == 1 and upper[1].port == 443 and upper[1].scheme == "https", + "scheme='HTTPS' should normalize to https/443") +local spaced = port_scheme.expand(nil, " http ") +assert_true(#spaced == 1 and spaced[1].port == 80 and spaced[1].scheme == "http", + "scheme=' http ' should normalize to http/80") + +-- matches() honors case-insensitive scheme on the request too. +assert_true(port_scheme.matches(nil, "HTTPS", 443, "HTTPS"), + "matches must accept case-insensitive scheme match") + +print("port_scheme_extra_test: PASS") diff --git a/CubeEgress/tests/port_scheme_test.lua b/CubeEgress/tests/port_scheme_test.lua new file mode 100644 index 000000000..6ac689f8b --- /dev/null +++ b/CubeEgress/tests/port_scheme_test.lua @@ -0,0 +1,114 @@ +package.path = "lua/?.lua;" .. package.path + +package.preload["cjson.safe"] = function() + return {encode = function() return "{}" end, decode = function() return {} end} +end +package.preload["resty.openssl.digest"] = function() + return {new = function() return nil, "not used by validation tests" end} +end + +local port_scheme = require "port_scheme" +local policy = require "policy" +local access = require "access_phase" + +local function assert_true(value, message) + if not value then error(message or "expected true") end +end + +local function assert_false(value, message) + if value then error(message or "expected false") end +end + +local function rule(id, match) + return {id = id, match = match, action = {allow = true}} +end + +assert_true(port_scheme.matches(nil, nil, 80, "http")) +assert_true(port_scheme.matches(nil, nil, 443, "https")) +assert_false(port_scheme.matches(nil, nil, 8080, "http")) +assert_true(port_scheme.matches(nil, "http", 80, "http")) +assert_false(port_scheme.matches(nil, "http", 8080, "http")) +assert_true(port_scheme.matches(8443, "https", 8443, "https")) +assert_false(port_scheme.matches(8443, "https", 8443, "http")) +assert_false(port_scheme.matches(8443, nil, 8443, "https")) + +-- Regression: a broad host rule must not shadow the custom-port rule. +local ctx = { + host = "api.example.com", sni = "api.example.com", method = "GET", + path = "/", scheme = "https", dst_port = 8443, +} +assert_false(access._rule_matches({host = "api.example.com"}, ctx)) +assert_true(access._rule_matches({host = "api.example.com", port = 8443, scheme = "https"}, ctx)) + +local function validate(rules) + return policy.validate_policy({policy_id = "sandbox-1", rules = rules}) +end + +local ok, err = validate({rule("port-only", {host = "api.example.com", port = 8080})}) +assert_false(ok, "port-only policy unexpectedly valid") +assert_true(string.find(err, "port requires scheme", 1, true) ~= nil) + +ok = validate({ + rule("default", {host = "api.example.com"}), + rule("conflict", {host = "api.example.com", port = 443, scheme = "http"}), +}) +assert_false(ok, "default/explicit scheme conflict unexpectedly valid") + +local many = {} +for i = 1, 9 do + many[i] = rule("r" .. i, {host = "api.example.com", port = 8000 + i, scheme = "http"}) +end +ok = validate(many) +assert_false(ok, "nine tuples unexpectedly valid") + +ok, err = validate({ + rule("host-a", {host = "a.example.com", port = 8443, scheme = "http"}), + rule("host-b", {host = "b.example.com", port = 8443, scheme = "https"}), +}) +assert_true(ok, err) + +-- normalize_identity: bare IP and /32 spelling aggregate under one identity, +-- so a per-(host,port) scheme conflict across the two spellings is caught +-- (Go l7GroupKey groups both via parseCIDR). +ok, err = validate({ + rule("ip-bare", {host = "1.2.3.4", port = 8443, scheme = "https"}), + rule("ip-cidr32", {host = "1.2.3.4/32", port = 8443, scheme = "http"}), +}) +assert_false(ok, "bare-IP vs /32 scheme conflict unexpectedly valid") +assert_true(string.find(err, "conflicts", 1, true) ~= nil) + +-- Same tuple under both spellings is a no-op duplicate, not a conflict. +ok, err = validate({ + rule("ip-bare", {host = "1.2.3.4", port = 8443, scheme = "https"}), + rule("ip-cidr32", {host = "1.2.3.4/32", port = 8443, scheme = "https"}), +}) +assert_true(ok, err) + +-- Subnet CIDRs are rejected as L7 hosts (mirrors Go classifyL7Target). +ok, err = validate({rule("subnet", {host = "1.2.3.0/24", port = 8443, scheme = "https"})}) +assert_false(ok, "subnet CIDR host unexpectedly valid") +assert_true(string.find(err, "subnet CIDR", 1, true) ~= nil) + +ok = validate({rule("bad-prefix", {host = "1.2.3.4/33", port = 8443, scheme = "https"})}) +assert_false(ok, "invalid CIDR prefix unexpectedly valid") + +ok = validate({rule("leading-zero", {host = "01.2.3.4", port = 8443, scheme = "https"})}) +assert_false(ok, "leading-zero IPv4 unexpectedly valid") + +ok = validate({rule("sni-subnet", {sni = "10.0.0.0/8", port = 443, scheme = "https"})}) +assert_false(ok, "subnet CIDR sni unexpectedly valid") + +-- Whitespace-padded host is rejected, not trimmed (request-time matching +-- compares the raw value, so trimming would accept a dead rule). +ok, err = validate({rule("padded", {host = " example.com ", port = 8443, scheme = "https"})}) +assert_false(ok, "whitespace-padded host unexpectedly valid") +assert_true(string.find(err, "whitespace", 1, true) ~= nil) + +-- Case + trailing-dot canonicalization still groups DNS names. +ok = validate({ + rule("dns-upper", {host = "API.example.com", port = 8443, scheme = "https"}), + rule("dns-dot", {host = "api.example.com.", port = 8443, scheme = "http"}), +}) +assert_false(ok, "case/dot DNS scheme conflict unexpectedly valid") + +print("port_scheme_test: PASS") From 30ba7ded3a8973102744e07aedc9a6cc5c7643ba Mon Sep 17 00:00:00 2001 From: hankzhou Date: Sun, 9 Aug 2026 22:06:52 +0800 Subject: [PATCH 04/31] feat(cubeapi,cubemaster): validate egress rule port/scheme at API and master layers Port the validation layers of the L7 custom-port feature from L7_support_portconfig (gitwoa): - CubeAPI models/mod.rs: EgressRuleMatch.port Option with the port+scheme pairing contract documented on the struct - CubeAPI cubemaster/mod.rs: CubeEgressRuleMatch.port forwarded to the master wire format - CubeAPI services/sandboxes.rs: validate_egress_rule_match rejects out-of-range ports, port-without-scheme, and non-http(s) schemes before the request reaches CubeMaster; map_egress_rule carries port through; 4 new unit tests - CubeMaster types/types.go: EgressRuleMatch.Port *int, DeepCopy and cloneIntPtr cover the new field - CubeMaster service/sandbox/util.go: checkParam calls validateEgressRuleMatch per rule (same contract as CubeAPI); mapEgressRuleMatch maps Port into the cubebox proto - Tests: TestValidateEgressRuleMatch, CLI merge test preserving Rules/AllowPublicTraffic with deep-copy leak assertions, and templatecenter TestCloneEgressRuleDeepCopiesPort Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- CubeAPI/src/cubemaster/mod.rs | 2 + CubeAPI/src/models/mod.rs | 7 + CubeAPI/src/services/sandboxes.rs | 126 +++++++++++++++++- .../commands/cubebox/template_test.go | 66 +++++++++ CubeMaster/pkg/service/sandbox/types/types.go | 18 +++ CubeMaster/pkg/service/sandbox/util.go | 49 ++++++- CubeMaster/pkg/service/sandbox/util_test.go | 43 ++++++ .../templatecenter/template_request_test.go | 33 +++++ 8 files changed, 337 insertions(+), 7 deletions(-) create mode 100644 CubeMaster/pkg/templatecenter/template_request_test.go diff --git a/CubeAPI/src/cubemaster/mod.rs b/CubeAPI/src/cubemaster/mod.rs index 37a0e505f..682a30bb1 100644 --- a/CubeAPI/src/cubemaster/mod.rs +++ b/CubeAPI/src/cubemaster/mod.rs @@ -821,6 +821,8 @@ pub struct CubeEgressRuleMatch { pub path: Option, #[serde(skip_serializing_if = "Option::is_none")] pub scheme: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub port: Option, } #[derive(Debug, Serialize, Clone)] diff --git a/CubeAPI/src/models/mod.rs b/CubeAPI/src/models/mod.rs index 1e9051e7c..77e081dc9 100644 --- a/CubeAPI/src/models/mod.rs +++ b/CubeAPI/src/models/mod.rs @@ -78,6 +78,11 @@ pub struct EgressRule { /// /// Multi-field semantics: AND across fields, OR within `method`. /// Comparisons on sni/host/scheme are case-insensitive. +/// +/// `port` + `scheme` together pin the (host, port) tuple CubeEgress intercepts. +/// Both nil keeps the legacy default {80/http, 443/https}. When `port` is set, +/// `scheme` MUST also be set — same-`(host, port)` rules across the policy +/// must agree on `scheme` (the server rejects the whole policy on mismatch). #[derive(Debug, Clone, Serialize, Deserialize, Default, ToSchema)] pub struct EgressRuleMatch { #[serde(skip_serializing_if = "Option::is_none")] @@ -90,6 +95,8 @@ pub struct EgressRuleMatch { pub path: Option, #[serde(skip_serializing_if = "Option::is_none")] pub scheme: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub port: Option, } /// Rule action. diff --git a/CubeAPI/src/services/sandboxes.rs b/CubeAPI/src/services/sandboxes.rs index b0560f435..691de88a3 100644 --- a/CubeAPI/src/services/sandboxes.rs +++ b/CubeAPI/src/services/sandboxes.rs @@ -17,9 +17,9 @@ use crate::{ }, error::{AppError, AppResult}, models::{ - EgressRule, LogLevel as ModelLogLevel, NewSandbox, Sandbox, SandboxAutoResume, - SandboxDetail, SandboxLifecycleConfig, SandboxLog, SandboxLogEntry, SandboxLogs, - SandboxLogsV2Response, SandboxNetworkConfig, SandboxOnTimeout, SandboxState, + EgressRule, EgressRuleMatch, LogLevel as ModelLogLevel, NewSandbox, Sandbox, + SandboxAutoResume, SandboxDetail, SandboxLifecycleConfig, SandboxLog, SandboxLogEntry, + SandboxLogs, SandboxLogsV2Response, SandboxNetworkConfig, SandboxOnTimeout, SandboxState, SandboxVolumeMount, }, }; @@ -1049,6 +1049,12 @@ pub(crate) fn build_cube_network_config( allow_internet_access == Some(false), )?; + if let Some(rs) = network.and_then(|n| n.rules.as_ref()) { + for (index, rule) in rs.iter().enumerate() { + validate_egress_rule_match(&rule.r#match, index)?; + } + } + let rules: Vec = network .and_then(|n| n.rules.as_ref()) .map(|rs| rs.iter().map(map_egress_rule).collect()) @@ -1080,6 +1086,33 @@ pub(crate) fn build_cube_network_config( })) } +/// Validate the port/scheme pair on one egress rule match, mirroring the +/// SDK client-side contract and the CubeEgress Lua validation: a set port +/// must be in [1, 65535] and must be paired with a scheme, and a set scheme +/// must be http or https (case-insensitive — downstream normalizes). +fn validate_egress_rule_match(rule_match: &EgressRuleMatch, index: usize) -> AppResult<()> { + if let Some(port) = rule_match.port { + if !(1..=65535).contains(&port) { + return Err(AppError::BadRequest(format!( + "network.rules[{index}].match.port must be in [1, 65535], got {port}" + ))); + } + if rule_match.scheme.is_none() { + return Err(AppError::BadRequest(format!( + "network.rules[{index}].match.port requires match.scheme to be set" + ))); + } + } + if let Some(scheme) = rule_match.scheme.as_deref() { + if !scheme.eq_ignore_ascii_case("http") && !scheme.eq_ignore_ascii_case("https") { + return Err(AppError::BadRequest(format!( + "network.rules[{index}].match.scheme must be 'http' or 'https', got {scheme:?}" + ))); + } + } + Ok(()) +} + fn map_egress_rule(rule: &EgressRule) -> CubeEgressRule { CubeEgressRule { name: rule.name.clone(), @@ -1089,6 +1122,7 @@ fn map_egress_rule(rule: &EgressRule) -> CubeEgressRule { method: rule.r#match.method.clone(), path: rule.r#match.path.clone(), scheme: rule.r#match.scheme.clone(), + port: rule.r#match.port, }, action: CubeEgressRuleAction { allow: rule.action.allow, @@ -1342,6 +1376,7 @@ mod tests { method: Some(vec!["POST".to_string()]), path: Some("/v1/chat".to_string()), sni: Some("api.deepseek.com".to_string()), + port: None, }, action: EgressRuleAction { allow: true, @@ -1410,6 +1445,91 @@ mod tests { assert!(rule["action"].get("inject").is_none()); } + fn network_with_match(rule_match: EgressRuleMatch) -> SandboxNetworkConfig { + SandboxNetworkConfig { + allow_public_traffic: None, + allow_out: None, + deny_out: None, + mask_request_host: None, + rules: Some(vec![EgressRule { + name: "r1".to_string(), + r#match: rule_match, + action: EgressRuleAction { + allow: true, + audit: None, + inject: None, + }, + }]), + } + } + + #[test] + fn egress_match_port_requires_scheme() { + let err = build_cube_network_config( + None, + Some(&network_with_match(EgressRuleMatch { + port: Some(8443), + ..Default::default() + })), + ) + .expect_err("port without scheme must be rejected"); + assert!(err.to_string().contains("requires match.scheme"), "{err}"); + } + + #[test] + fn egress_match_port_range_enforced() { + for port in [0, -1, 65536, 99999] { + let err = build_cube_network_config( + None, + Some(&network_with_match(EgressRuleMatch { + port: Some(port), + scheme: Some("https".to_string()), + ..Default::default() + })), + ) + .expect_err("out-of-range port must be rejected"); + assert!(err.to_string().contains("[1, 65535]"), "{err}"); + } + } + + #[test] + fn egress_match_invalid_scheme_rejected() { + let err = build_cube_network_config( + None, + Some(&network_with_match(EgressRuleMatch { + scheme: Some("ftp".to_string()), + ..Default::default() + })), + ) + .expect_err("non-http(s) scheme must be rejected"); + assert!( + err.to_string().contains("must be 'http' or 'https'"), + "{err}" + ); + } + + #[test] + fn egress_match_valid_port_scheme_accepted() { + build_cube_network_config( + None, + Some(&network_with_match(EgressRuleMatch { + port: Some(8443), + scheme: Some("https".to_string()), + ..Default::default() + })), + ) + .expect("valid port+scheme pair"); + // Case variants are accepted (downstream normalizes to lowercase). + build_cube_network_config( + None, + Some(&network_with_match(EgressRuleMatch { + scheme: Some("HTTPS".to_string()), + ..Default::default() + })), + ) + .expect("uppercase scheme is accepted"); + } + #[test] fn listed_sandbox_preserves_resources_from_cubemaster_list() { let listed = from_cubemaster_info(SandboxInfo { diff --git a/CubeMaster/cmd/cubemastercli/commands/cubebox/template_test.go b/CubeMaster/cmd/cubemastercli/commands/cubebox/template_test.go index e8c377acc..62be7737a 100644 --- a/CubeMaster/cmd/cubemastercli/commands/cubebox/template_test.go +++ b/CubeMaster/cmd/cubemastercli/commands/cubebox/template_test.go @@ -344,6 +344,72 @@ func TestMergeCubeNetworkConfigValuesPreservesExistingCIDRs(t *testing.T) { } } +func TestMergeCubeNetworkConfigValuesPreservesRulesAndAllowPublicTraffic(t *testing.T) { + host := "api.internal.example.com" + scheme := "https" + port := 8443 + audit := "full" + format := "Bearer ${SECRET}" + allowPublic := false + existing := &types.CubeNetworkConfig{ + AllowPublicTraffic: &allowPublic, + Rules: []*types.EgressRule{ + { + Name: "api-8443-https", + Match: &types.EgressRuleMatch{ + Host: &host, + Scheme: &scheme, + Port: &port, + }, + Action: &types.EgressRuleAction{ + Allow: true, + Audit: &audit, + Inject: []*types.EgressRuleInject{ + {Header: "Authorization", Secret: "s3cret", Format: &format}, + }, + }, + }, + }, + } + + // Any --allow-* flag triggers the merge path, which clones `existing`. + got := mergeCubeNetworkConfigValues(existing, false, false, []string{"10.0.0.0/8"}, nil) + if got == nil { + t.Fatal("got nil merged config") + } + if got.AllowPublicTraffic == nil || *got.AllowPublicTraffic != false { + t.Fatalf("AllowPublicTraffic=%v, want pointer to false (template value preserved)", got.AllowPublicTraffic) + } + if len(got.Rules) != 1 { + t.Fatalf("Rules=%v, want the template rule preserved", got.Rules) + } + rule := got.Rules[0] + if rule.Name != "api-8443-https" || rule.Match == nil || rule.Match.Port == nil || *rule.Match.Port != 8443 { + t.Fatalf("rule=%+v, want port-pinned rule preserved", rule) + } + if rule.Action == nil || len(rule.Action.Inject) != 1 || rule.Action.Inject[0].Header != "Authorization" { + t.Fatalf("action=%+v, want inject preserved", rule.Action) + } + + // The clone must be deep: mutating the merged copy must not touch the template. + *got.AllowPublicTraffic = true + *rule.Match.Port = 443 + *rule.Match.Host = "mutated.example.com" + *rule.Action.Inject[0].Format = "mutated" + if *existing.AllowPublicTraffic != false { + t.Fatal("mutation of merged AllowPublicTraffic leaked into template") + } + if *existing.Rules[0].Match.Port != 8443 { + t.Fatal("mutation of merged Match.Port leaked into template") + } + if *existing.Rules[0].Match.Host != "api.internal.example.com" { + t.Fatal("mutation of merged Match.Host leaked into template") + } + if *existing.Rules[0].Action.Inject[0].Format != "Bearer ${SECRET}" { + t.Fatal("mutation of merged Inject.Format leaked into template") + } +} + func TestRedoCommandParsesNodeScope(t *testing.T) { ctx := newRedoContext(t, []string{ "--template-id", "tpl-1", diff --git a/CubeMaster/pkg/service/sandbox/types/types.go b/CubeMaster/pkg/service/sandbox/types/types.go index 6b37d1d02..7658c379f 100644 --- a/CubeMaster/pkg/service/sandbox/types/types.go +++ b/CubeMaster/pkg/service/sandbox/types/types.go @@ -157,12 +157,21 @@ type EgressRule struct { // EgressRuleMatch holds the per-request match conditions for an EgressRule. // All fields are optional; an empty match matches any request. +// +// Port + Scheme together select which TCP port CubeEgress intercepts: +// - both nil: legacy behavior — CubeEgress captures the default {80/http, +// 443/https} pair. +// - both set: CubeEgress captures the specific (host, port) tuple, routing +// via skb->mark to the HTTP (scheme="http") or HTTPS (scheme="https") +// TPROXY listener. Every rule sharing the same (host, port) MUST agree on +// scheme; the server rejects the whole policy if it detects a mismatch. type EgressRuleMatch struct { SNI *string `json:"sni,omitempty"` Host *string `json:"host,omitempty"` Method []string `json:"method,omitempty"` Path *string `json:"path,omitempty"` Scheme *string `json:"scheme,omitempty"` + Port *int `json:"port,omitempty"` } // EgressRuleAction holds the action taken when an EgressRule matches. @@ -215,6 +224,7 @@ func (r *EgressRule) DeepCopy() *EgressRule { Method: append([]string(nil), r.Match.Method...), Path: cloneStringPtr(r.Match.Path), Scheme: cloneStringPtr(r.Match.Scheme), + Port: cloneIntPtr(r.Match.Port), } } if r.Action != nil { @@ -255,6 +265,14 @@ func cloneStringPtr(value *string) *string { return &cloned } +func cloneIntPtr(value *int) *int { + if value == nil { + return nil + } + cloned := *value + return &cloned +} + type Volume struct { Name string `json:"name,omitempty"` diff --git a/CubeMaster/pkg/service/sandbox/util.go b/CubeMaster/pkg/service/sandbox/util.go index 219226bec..6ac4dd44f 100644 --- a/CubeMaster/pkg/service/sandbox/util.go +++ b/CubeMaster/pkg/service/sandbox/util.go @@ -87,15 +87,51 @@ func checkParam(req *types.CreateCubeSandboxReq) error { return ret.Err(errorcode.ErrorCode_MasterParamsError, "containers param is nil") } - if req.CubeNetworkConfig != nil && req.CubeNetworkConfig.MaskRequestHost != nil { - if err := validateMaskRequestHost(*req.CubeNetworkConfig.MaskRequestHost); err != nil { - return ret.Err(errorcode.ErrorCode_MasterParamsError, err.Error()) + if req.CubeNetworkConfig != nil { + if req.CubeNetworkConfig.MaskRequestHost != nil { + if err := validateMaskRequestHost(*req.CubeNetworkConfig.MaskRequestHost); err != nil { + return ret.Err(errorcode.ErrorCode_MasterParamsError, err.Error()) + } + } + for i, rule := range req.CubeNetworkConfig.Rules { + if rule == nil { + continue + } + if err := validateEgressRuleMatch(rule.Match, i); err != nil { + return ret.Err(errorcode.ErrorCode_MasterParamsError, err.Error()) + } } } return nil } +// validateEgressRuleMatch enforces the port/scheme contract on one egress +// rule match, mirroring the SDK client-side check and the CubeEgress Lua +// validation: a set Port must be in [1, 65535] and must be paired with +// Scheme, and a set Scheme must be http or https (case-insensitive — +// downstream normalizes to lowercase). +func validateEgressRuleMatch(match *types.EgressRuleMatch, index int) error { + if match == nil { + return nil + } + if match.Port != nil { + if *match.Port < 1 || *match.Port > 65535 { + return fmt.Errorf("network.rules[%d].match.port must be in [1, 65535], got %d", index, *match.Port) + } + if match.Scheme == nil { + return fmt.Errorf("network.rules[%d].match.port requires match.scheme to be set", index) + } + } + if match.Scheme != nil { + scheme := strings.ToLower(strings.TrimSpace(*match.Scheme)) + if scheme != "http" && scheme != "https" { + return fmt.Errorf("network.rules[%d].match.scheme must be 'http' or 'https', got %q", index, *match.Scheme) + } + } + return nil +} + func validateMaskRequestHost(value string) error { invalid := func(reason string) error { return fmt.Errorf("network.maskRequestHost is invalid: %s", reason) @@ -317,13 +353,18 @@ func mapEgressRuleMatch(in *types.EgressRuleMatch) *cubebox.EgressRuleMatch { if in == nil { return nil } - return &cubebox.EgressRuleMatch{ + out := &cubebox.EgressRuleMatch{ Sni: in.SNI, Host: in.Host, Method: append([]string(nil), in.Method...), Path: in.Path, Scheme: in.Scheme, } + if in.Port != nil { + p := int32(*in.Port) + out.Port = &p + } + return out } func mapEgressRuleAction(in *types.EgressRuleAction) *cubebox.EgressRuleAction { diff --git a/CubeMaster/pkg/service/sandbox/util_test.go b/CubeMaster/pkg/service/sandbox/util_test.go index 2a5c0a8f5..130c5bc59 100644 --- a/CubeMaster/pkg/service/sandbox/util_test.go +++ b/CubeMaster/pkg/service/sandbox/util_test.go @@ -36,6 +36,49 @@ func ensureSandboxTestConfig(t *testing.T) *config.Config { return cfg } +func TestValidateEgressRuleMatch(t *testing.T) { + strPtr := func(s string) *string { return &s } + intPtr := func(i int) *int { return &i } + + valid := []struct { + name string + match *types.EgressRuleMatch + }{ + {"nil match", nil}, + {"empty match", &types.EgressRuleMatch{}}, + {"scheme only", &types.EgressRuleMatch{Scheme: strPtr("https")}}, + {"scheme case variant", &types.EgressRuleMatch{Scheme: strPtr("HTTPS")}}, + {"scheme with spaces", &types.EgressRuleMatch{Scheme: strPtr(" http ")}}, + {"port with scheme", &types.EgressRuleMatch{Port: intPtr(8443), Scheme: strPtr("https")}}, + {"port boundary low", &types.EgressRuleMatch{Port: intPtr(1), Scheme: strPtr("http")}}, + {"port boundary high", &types.EgressRuleMatch{Port: intPtr(65535), Scheme: strPtr("https")}}, + } + for _, tt := range valid { + t.Run(tt.name, func(t *testing.T) { + assert.NoError(t, validateEgressRuleMatch(tt.match, 0)) + }) + } + + invalid := []struct { + name string + match *types.EgressRuleMatch + wantErr string + }{ + {"port without scheme", &types.EgressRuleMatch{Port: intPtr(8443)}, "requires match.scheme"}, + {"port zero", &types.EgressRuleMatch{Port: intPtr(0), Scheme: strPtr("http")}, "[1, 65535]"}, + {"port negative", &types.EgressRuleMatch{Port: intPtr(-1), Scheme: strPtr("http")}, "[1, 65535]"}, + {"port too large", &types.EgressRuleMatch{Port: intPtr(65536), Scheme: strPtr("http")}, "[1, 65535]"}, + {"scheme not http", &types.EgressRuleMatch{Scheme: strPtr("ftp")}, "must be 'http' or 'https'"}, + } + for _, tt := range invalid { + t.Run(tt.name, func(t *testing.T) { + err := validateEgressRuleMatch(tt.match, 0) + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + func TestValidateMaskRequestHost(t *testing.T) { for _, value := range []string{ "localhost", diff --git a/CubeMaster/pkg/templatecenter/template_request_test.go b/CubeMaster/pkg/templatecenter/template_request_test.go new file mode 100644 index 000000000..a48b8cd5b --- /dev/null +++ b/CubeMaster/pkg/templatecenter/template_request_test.go @@ -0,0 +1,33 @@ +package templatecenter + +import ( + "testing" + + "github.com/tencentcloud/CubeSandbox/CubeMaster/pkg/service/sandbox/types" +) + +func TestCloneEgressRuleDeepCopiesPort(t *testing.T) { + port := 8443 + rule := &types.EgressRule{ + Name: "custom-https", + Match: &types.EgressRuleMatch{ + Port: &port, + }, + } + + cloned := rule.DeepCopy() + if cloned == nil || cloned.Match == nil || cloned.Match.Port == nil { + t.Fatalf("cloned rule lost port: %+v", cloned) + } + if *cloned.Match.Port != port { + t.Fatalf("cloned port=%d, want %d", *cloned.Match.Port, port) + } + if cloned.Match.Port == rule.Match.Port { + t.Fatal("cloned port aliases source pointer") + } + + *cloned.Match.Port = 443 + if *rule.Match.Port != 8443 { + t.Fatalf("source port changed through clone: %d", *rule.Match.Port) + } +} From c36913db60e13721b4b5ef67ee5a468c8dd65082 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Sun, 9 Aug 2026 22:14:57 +0800 Subject: [PATCH 05/31] feat(sdk): validate L7 custom port in egress rule policy Port the Python SDK validation layer of the L7 custom-port feature from L7_support_portconfig (gitwoa): - cubesandbox/_policy.py: port/scheme validation on egress rule matches (port in [1, 65535], port requires scheme, scheme must be http/https case-insensitively), matching the CubeAPI and CubeMaster server-side contract - tests/test_policy.py (new): 27 cases covering the validation matrix and E2B per-host rules compat - tests/test_l7_custom_port_e2e.py (new): live custom-port end-to-end - tests/test_l7_custom_port_validation_e2e.py (new): API-edge validation end-to-end (skips without a cluster) _policy.py was untouched since the branch base; the new test files do not collide with the target test layout (test_sandbox/test_volume). Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- sdk/python/cubesandbox/_policy.py | 84 +++++++- sdk/python/tests/test_l7_custom_port_e2e.py | 194 ++++++++++++++++++ .../test_l7_custom_port_validation_e2e.py | 107 ++++++++++ sdk/python/tests/test_policy.py | 155 ++++++++++++++ 4 files changed, 539 insertions(+), 1 deletion(-) create mode 100644 sdk/python/tests/test_l7_custom_port_e2e.py create mode 100644 sdk/python/tests/test_l7_custom_port_validation_e2e.py create mode 100644 sdk/python/tests/test_policy.py diff --git a/sdk/python/cubesandbox/_policy.py b/sdk/python/cubesandbox/_policy.py index a0536ad21..cb15d7e2d 100644 --- a/sdk/python/cubesandbox/_policy.py +++ b/sdk/python/cubesandbox/_policy.py @@ -26,6 +26,25 @@ AuditLevel = Literal["full", "metadata", "none"] +def _validate_scheme(value: Any, field: str) -> Optional[str]: + """Normalize *value* (strip + lowercase) and validate against http/https. + + Returns the normalized scheme, or ``None`` when *value* is ``None``. + Scheme matching is case-insensitive across the stack (Lua + ``normalize_scheme`` and cubevs both lowercase/strip), so the SDK + normalizes before the membership check and propagates the clean + lowercase value downstream. + """ + if value is None: + return None + if not isinstance(value, str): + raise ValueError(f"{field} must be 'http' or 'https', got {value!r}") + normalized = value.strip().lower() + if normalized not in ("http", "https"): + raise ValueError(f"{field} must be 'http' or 'https', got {value!r}") + return normalized + + @dataclass class Match: """ @@ -33,12 +52,44 @@ class Match: Multi-field semantics: AND across fields, OR within ``method``. Comparisons on sni/host/scheme are case-insensitive. + + ``port`` + ``scheme`` together drive which TCP port CubeEgress intercepts + on the sandbox side. Both must be set together or both omitted: + + - Both omitted → default set ``{80/http, 443/https}`` (backward compatible). + - Both set → CubeEgress intercepts only that (host, port, scheme) tuple. + + Every rule sharing the same ``(host, port)`` MUST agree on ``scheme`` — + a port can only route to one nginx listener (HTTP → 8080, HTTPS → 8443). + The server rejects the whole policy if it detects a mismatch. """ sni: Optional[str] = None host: Optional[str] = None method: Optional[List[Method]] = None path: Optional[str] = None scheme: Optional[Scheme] = None + port: Optional[int] = None + + def __post_init__(self) -> None: + # Client-side pre-validation. Not strictly required (the server would + # reject the same shape) but catches typos before the network round-trip + # and produces a Pythonic error path (ValueError) instead of a 400. + # _validate_scheme returns the normalized scheme; store it so + # to_wire() emits the canonical lowercase form. + self.scheme = _validate_scheme(self.scheme, "Match.scheme") + if self.port is not None: + if not isinstance(self.port, int) or isinstance(self.port, bool): + raise ValueError(f"Match.port must be an int, got {type(self.port).__name__}") + if self.port < 1 or self.port > 65535: + raise ValueError(f"Match.port must be in [1, 65535], got {self.port}") + if self.scheme is None: + raise ValueError("Match.port requires Match.scheme to be set") + if self.scheme is not None and self.port is None: + # Legacy shape: scheme alone (no port) is still accepted server-side + # as a match qualifier that filters by HTTP vs HTTPS on the default + # {80, 443} set. This is not the new port-scoped feature, so we do + # NOT raise — but callers relying on it get the classic behavior. + pass def to_wire(self) -> Dict[str, Any]: out: Dict[str, Any] = {} @@ -52,6 +103,8 @@ def to_wire(self) -> Dict[str, Any]: out["path"] = self.path if self.scheme is not None: out["scheme"] = self.scheme + if self.port is not None: + out["port"] = self.port return out @@ -125,7 +178,28 @@ def to_wire(self) -> Dict[str, Any]: def _normalize_match_dict(m: Dict[str, Any]) -> Dict[str, Any]: - return dict(m) + out = dict(m) + # Best-effort client-side validation for the port + scheme pair. Mirrors + # Match.__post_init__ so dict-shaped and dataclass-shaped rules get the + # same behavior. Errors here would otherwise surface as a 400 from the + # server; catching them locally gives users a Pythonic ValueError with a + # stack trace pointing at the offending call site. + port = out.get("port") + scheme = _validate_scheme(out.get("scheme"), "match.scheme") + if scheme is not None: + # Propagate the normalized (stripped, lowercased) scheme so the wire + # form is canonical regardless of caller casing/whitespace. + out["scheme"] = scheme + if port is not None: + if not isinstance(port, int) or isinstance(port, bool): + raise ValueError( + f"match.port must be an int, got {type(port).__name__}" + ) + if port < 1 or port > 65535: + raise ValueError(f"match.port must be in [1, 65535], got {port}") + if scheme is None: + raise ValueError("match.port requires match.scheme to be set") + return out def _normalize_inject_dict(i: Dict[str, Any]) -> Dict[str, Any]: @@ -267,6 +341,14 @@ def _convert_e2b_per_host_rules(rules: Dict[str, Any]) -> List[Dict[str, Any]]: f"network.rules[{host!r}] must be a list of transform entries, " f"got {type(entries).__name__}" ) + if not entries: + # An empty list would fan out to zero rules and silently drop the + # host the caller keyed in — the exact "silent drop" this + # compatibility layer exists to prevent. + raise ValueError( + f"network.rules[{host!r}] is an empty list; every host must " + "declare at least one transform entry" + ) for index, entry in enumerate(entries): if not isinstance(entry, dict): raise ValueError( diff --git a/sdk/python/tests/test_l7_custom_port_e2e.py b/sdk/python/tests/test_l7_custom_port_e2e.py new file mode 100644 index 000000000..9af501f42 --- /dev/null +++ b/sdk/python/tests/test_l7_custom_port_e2e.py @@ -0,0 +1,194 @@ +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Live custom-port CubeEgress dataplane tests. + +The test starts a host-side HTTP echo target, creates one sandbox with an +HTTP custom-port inject rule and an HTTPS custom-port deny rule, then executes +both requests from inside the sandbox. + +Required opt-in: + +- ``CUBE_E2E=1`` or pytest ``--run-e2e`` +- ``CUBE_L7_E2E_HTTP_TARGET_HOST``: a host address reachable from sandboxes + that is not CubeVS's node-IP fast path (a Docker bridge gateway is suitable) +- ``CUBE_TEMPLATE_ID`` or pytest ``--cube-template-id`` + +Optional: + +- ``CUBE_L7_E2E_HTTP_BIND_HOST`` (default ``0.0.0.0``) +- ``CUBE_L7_E2E_HTTP_PORT`` (default ``18080``) +- ``CUBE_L7_E2E_HTTPS_URL`` + (default ``https://tls-v1-2.badssl.com:1012/``) +- ``CUBE_L7_E2E_HTTPS_ALLOW_URL`` + (defaults to the same non-standard-port HTTPS URL) +""" + +from __future__ import annotations + +import base64 +import json +import os +import threading +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + +import pytest + +from cubesandbox import Action, Config, Inject, Match, Rule, Sandbox + +pytestmark = pytest.mark.e2e + + +class _HeaderEchoHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + payload = json.dumps({"path": self.path, "headers": dict(self.headers)}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, _format: str, *_args: object) -> None: + return + + +def _option(pytestconfig: pytest.Config, option: str, env: str) -> str | None: + return pytestconfig.getoption(option) or os.environ.get(env) + + +def _require_custom_port_e2e(pytestconfig: pytest.Config) -> tuple[str, str]: + if not pytestconfig.getoption("--run-e2e") and os.environ.get("CUBE_E2E") != "1": + pytest.skip("use --run-e2e or set CUBE_E2E=1") + target_host = os.environ.get("CUBE_L7_E2E_HTTP_TARGET_HOST") + if not target_host: + pytest.skip("set CUBE_L7_E2E_HTTP_TARGET_HOST to a sandbox-reachable host address") + template_id = _option(pytestconfig, "--cube-template-id", "CUBE_TEMPLATE_ID") + if not template_id: + pytest.skip("set CUBE_TEMPLATE_ID or --cube-template-id") + return target_host, template_id + + +def _python_command(source: str) -> str: + encoded = base64.b64encode(source.encode()).decode() + return f"python3 -c \"import base64;exec(base64.b64decode('{encoded}'))\"" + + +def _https_probe_script(url: str) -> str: + return f""" +import urllib.error +import urllib.request +opener = urllib.request.build_opener(urllib.request.ProxyHandler({{}})) +try: + with opener.open({url!r}, timeout=30) as response: + print('STATUS=' + str(response.status)) + print('FINAL_URL=' + response.geturl()) + print('BODY_LEN=' + str(len(response.read()))) +except urllib.error.HTTPError as error: + print('STATUS=' + str(error.code)) +""" + + +def test_l7_custom_http_inject_and_https_deny(pytestconfig: pytest.Config) -> None: + target_host, template_id = _require_custom_port_e2e(pytestconfig) + bind_host = os.environ.get("CUBE_L7_E2E_HTTP_BIND_HOST", "0.0.0.0") + http_port = int(os.environ.get("CUBE_L7_E2E_HTTP_PORT", "18080")) + https_url = os.environ.get( + "CUBE_L7_E2E_HTTPS_URL", "https://tls-v1-2.badssl.com:1012/" + ) + parsed_https = urlparse(https_url) + assert parsed_https.scheme == "https" and parsed_https.hostname and parsed_https.port + + marker = f"cube-l7-{uuid.uuid4().hex}" + server = ThreadingHTTPServer((bind_host, http_port), _HeaderEchoHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + config = Config(api_url=os.environ.get("CUBE_API_URL", "http://127.0.0.1:3000")) + rules = [ + Rule( + name="e2e-custom-http-inject", + match=Match(host=target_host, port=http_port, scheme="http"), + action=Action( + allow=True, + inject=[Inject(header="X-Cube-L7-E2E", secret=marker)], + ), + ), + Rule( + name="e2e-custom-https-deny", + match=Match(host=parsed_https.hostname, port=parsed_https.port, scheme="https"), + action=Action(allow=False), + ), + ] + + try: + with Sandbox.create( + template=template_id, + timeout=120, + allow_internet_access=False, + network={"rules": rules}, + config=config, + ) as sandbox: + http_url = f"http://{target_host}:{http_port}/headers" + http_script = f""" +import urllib.request +opener = urllib.request.build_opener(urllib.request.ProxyHandler({{}})) +with opener.open({http_url!r}, timeout=20) as response: + print('STATUS=' + str(response.status)) + print(response.read().decode()) +""" + http_result = sandbox.commands.run(_python_command(http_script), timeout=30) + assert http_result.exit_code == 0, http_result.stderr + assert "STATUS=200" in http_result.stdout + assert marker in http_result.stdout, ( + "custom HTTP request reached the target without the injected header: " + + http_result.stdout + ) + + https_result = sandbox.commands.run( + _python_command(_https_probe_script(https_url)), timeout=40 + ) + assert https_result.exit_code == 0, https_result.stderr + assert "STATUS=403" in https_result.stdout, ( + "custom HTTPS deny rule was not enforced: " + https_result.stdout + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def test_l7_custom_https_allow_reaches_real_upstream(pytestconfig: pytest.Config) -> None: + _, template_id = _require_custom_port_e2e(pytestconfig) + https_url = os.environ.get( + "CUBE_L7_E2E_HTTPS_ALLOW_URL", + os.environ.get("CUBE_L7_E2E_HTTPS_URL", "https://tls-v1-2.badssl.com:1012/"), + ) + parsed = urlparse(https_url) + assert parsed.scheme == "https" and parsed.hostname and parsed.port + assert parsed.port != 443, "custom HTTPS allow E2E requires a non-standard port" + + config = Config(api_url=os.environ.get("CUBE_API_URL", "http://127.0.0.1:3000")) + rule = Rule( + name="e2e-custom-https-allow", + match=Match(host=parsed.hostname, port=parsed.port, scheme="https"), + action=Action(allow=True, audit="metadata"), + ) + + with Sandbox.create( + template=template_id, + timeout=120, + allow_internet_access=False, + network={"rules": [rule]}, + config=config, + ) as sandbox: + result = sandbox.commands.run( + _python_command(_https_probe_script(https_url)), timeout=40 + ) + assert result.exit_code == 0, result.stderr + assert "STATUS=200" in result.stdout, ( + "custom HTTPS allow did not reach the real upstream: " + result.stdout + ) + assert "BODY_LEN=0" not in result.stdout, ( + "custom HTTPS upstream returned no response body: " + result.stdout + ) diff --git a/sdk/python/tests/test_l7_custom_port_validation_e2e.py b/sdk/python/tests/test_l7_custom_port_validation_e2e.py new file mode 100644 index 000000000..1d2b3ae80 --- /dev/null +++ b/sdk/python/tests/test_l7_custom_port_validation_e2e.py @@ -0,0 +1,107 @@ +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Negative / validation e2e cases for custom-port CubeEgress rules. + +These complement ``test_l7_custom_port_e2e.py`` (which covers the happy +paths: custom HTTP inject, custom HTTPS allow/deny). Here we assert the +*rejection* contract for malformed or conflicting custom-port specs: + +* two rules pinning the same ``(host, port)`` to different schemes are + rejected as a whole policy by CubeMaster/CubeEgress; +* more than ``MAX_L7_PORTS_PER_HOST`` (8) distinct ``(port, scheme)`` + tuples on one host are rejected. + +(The client-side "port requires scheme" contract is covered +unconditionally by ``tests/test_policy.py::test_port_without_scheme_rejected``; +both cases below drive a real ``Sandbox.create`` and therefore require a +live cluster.) + +Required opt-in (same as the happy-path e2e): + +- ``CUBE_E2E=1`` or pytest ``--run-e2e`` +- ``CUBE_TEMPLATE_ID`` or pytest ``--cube-template-id`` +""" + +from __future__ import annotations + +import os + +import pytest + +from cubesandbox import Action, Config, Match, Rule, Sandbox +from cubesandbox._exceptions import ApiError + +pytestmark = pytest.mark.e2e + + +def _skip_unless_e2e(pytestconfig: pytest.Config) -> str | None: + if not pytestconfig.getoption("--run-e2e") and os.environ.get("CUBE_E2E") != "1": + pytest.skip("use --run-e2e or set CUBE_E2E=1") + template_id = ( + pytestconfig.getoption("--cube-template-id") + or os.environ.get("CUBE_TEMPLATE_ID") + ) + if not template_id: + pytest.skip("set CUBE_TEMPLATE_ID or --cube-template-id") + return template_id + + +def test_l7_custom_scheme_conflict_rejected( + pytestconfig: pytest.Config, +) -> None: + # (host, port) pinned to two different schemes is a whole-policy + # rejection: iptables can only steer one port to one listener. + template_id = _skip_unless_e2e(pytestconfig) + config = Config(api_url=os.environ.get("CUBE_API_URL", "http://127.0.0.1:3000")) + rules = [ + Rule( + name="custom-8080-http", + match=Match(host="api.example.com", port=8080, scheme="http"), + action=Action(allow=True), + ), + Rule( + name="custom-8080-https", + match=Match(host="api.example.com", port=8080, scheme="https"), + action=Action(allow=True), + ), + ] + # Assert the *policy* rejection specifically (ApiError mentioning the + # conflict), not just any failure — a connection error or unrelated 500 + # must not satisfy this. + with pytest.raises(ApiError, match="conflict"): + with Sandbox.create( + template=template_id, + timeout=120, + allow_internet_access=False, + network={"rules": rules}, + config=config, + ): + pass + + +def test_l7_custom_port_budget_exceeded_rejected( + pytestconfig: pytest.Config, +) -> None: + # More than 8 distinct (port, scheme) tuples on one host must be + # rejected rather than silently truncated. + template_id = _skip_unless_e2e(pytestconfig) + config = Config(api_url=os.environ.get("CUBE_API_URL", "http://127.0.0.1:3000")) + rules = [ + Rule( + name=f"custom-{i}", + match=Match(host="api.example.com", port=8000 + i, scheme="http"), + action=Action(allow=True), + ) + for i in range(9) + ] + # Assert the *budget* rejection specifically (ApiError mentioning the + # exceeded tuple limit), not just any failure. + with pytest.raises(ApiError, match="exceeds"): + with Sandbox.create( + template=template_id, + timeout=120, + allow_internet_access=False, + network={"rules": rules}, + config=config, + ): + pass diff --git a/sdk/python/tests/test_policy.py b/sdk/python/tests/test_policy.py new file mode 100644 index 000000000..e84ce7e23 --- /dev/null +++ b/sdk/python/tests/test_policy.py @@ -0,0 +1,155 @@ +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the ``cubesandbox._policy`` module — L7 egress rule dataclasses.""" + +from __future__ import annotations + +import pytest + +from cubesandbox import Action, Inject, Match, Rule +from cubesandbox._policy import ( + _convert_e2b_per_host_rules, + _normalize_match_dict, + _serialize_rule, +) + + +class TestMatchWireShape: + def test_empty_match_produces_empty_dict(self): + assert Match().to_wire() == {} + + def test_host_only(self): + assert Match(host="api.example.com").to_wire() == {"host": "api.example.com"} + + def test_default_port_scheme_omitted(self): + # No port/scheme → wire dict does not carry those keys, keeping + # backward compatibility with servers that predate the L7 port field. + wire = Match(host="api.example.com").to_wire() + assert "port" not in wire + assert "scheme" not in wire + + def test_explicit_port_and_scheme_serialized(self): + wire = Match(host="api.example.com", port=8080, scheme="http").to_wire() + assert wire == {"host": "api.example.com", "port": 8080, "scheme": "http"} + + def test_scheme_alone_uses_default_port(self): + # SDK does not expand scheme-only rules on the client side — the wire + # form is left untouched and the server (network-agent) fills in the + # scheme's default port (http → 80, https → 443) when building the + # cubevs L7 plan. Verify only that the wire shape is stable. + wire = Match(host="api.example.com", scheme="https").to_wire() + assert wire == {"host": "api.example.com", "scheme": "https"} + + def test_scheme_normalized_strip_lowercase(self): + # Scheme matching is case-insensitive across the stack (Lua + # normalize_scheme and cubevs both lowercase/strip) — the SDK + # normalizes before validating and stores the canonical lowercase + # form on the wire. + wire = Match(host="api.example.com", scheme=" HTTPS ").to_wire() + assert wire == {"host": "api.example.com", "scheme": "https"} + + +class TestMatchPortValidation: + def test_port_without_scheme_rejected(self): + with pytest.raises(ValueError, match="scheme"): + Match(host="api.example.com", port=8080) + + def test_port_out_of_range_low(self): + with pytest.raises(ValueError, match=r"\[1, 65535\]"): + Match(host="api.example.com", port=0, scheme="http") + + def test_port_out_of_range_high(self): + with pytest.raises(ValueError, match=r"\[1, 65535\]"): + Match(host="api.example.com", port=65536, scheme="http") + + def test_port_negative(self): + with pytest.raises(ValueError, match=r"\[1, 65535\]"): + Match(host="api.example.com", port=-1, scheme="http") + + def test_port_wrong_type_string(self): + with pytest.raises(ValueError, match="int"): + Match(host="api.example.com", port="8080", scheme="http") # type: ignore[arg-type] + + def test_port_wrong_type_bool_rejected(self): + # bool is a subclass of int in Python — must be filtered out + # explicitly, otherwise Match(port=True) would slip through as port=1. + with pytest.raises(ValueError, match="int"): + Match(host="api.example.com", port=True, scheme="http") # type: ignore[arg-type] + + @pytest.mark.parametrize("scheme", ["htps", "", 1]) + def test_invalid_scheme_rejected(self, scheme): + with pytest.raises(ValueError, match="http.*https"): + Match(host="api.example.com", scheme=scheme) # type: ignore[arg-type] + + +class TestNormalizeMatchDict: + def test_dict_pass_through(self): + out = _normalize_match_dict({"host": "foo", "port": 8080, "scheme": "http"}) + assert out == {"host": "foo", "port": 8080, "scheme": "http"} + + def test_dict_port_without_scheme_rejected(self): + with pytest.raises(ValueError, match="scheme"): + _normalize_match_dict({"host": "foo", "port": 8080}) + + def test_dict_port_out_of_range_rejected(self): + with pytest.raises(ValueError, match=r"\[1, 65535\]"): + _normalize_match_dict({"host": "foo", "port": 0, "scheme": "http"}) + + def test_dict_input_not_mutated(self): + # Regression guard: normalization returns a new dict so caller-owned + # data structures are not silently changed. + original = {"host": "foo"} + out = _normalize_match_dict(original) + assert out is not original + + def test_dict_scheme_normalized(self): + # Same normalize-then-validate semantics as the Match dataclass: + # mixed case and surrounding whitespace are accepted, and the wire + # form carries the canonical lowercase scheme. + out = _normalize_match_dict({"host": "foo", "port": 8443, "scheme": " HTTPS "}) + assert out["scheme"] == "https" + + @pytest.mark.parametrize("scheme", ["ftp", "", False]) + def test_dict_invalid_scheme_rejected(self, scheme): + with pytest.raises(ValueError, match="http.*https"): + _normalize_match_dict({"host": "foo", "scheme": scheme}) + + +class TestRuleSerializerHandlesPortScheme: + def test_rule_dataclass_carries_port(self): + rule = Rule( + name="api-inject", + match=Match(host="api.example.com", port=8443, scheme="https"), + action=Action(allow=True), + ) + wire = _serialize_rule(rule) + assert wire["match"]["port"] == 8443 + assert wire["match"]["scheme"] == "https" + + def test_rule_dict_carries_port(self): + wire = _serialize_rule({ + "name": "api-inject", + "match": {"host": "api.example.com", "port": 8443, "scheme": "https"}, + "action": {"allow": True}, + }) + assert wire["match"]["port"] == 8443 + + +class TestE2BPerHostRulesCompat: + def test_transform_still_works_without_port(self): + # E2B compat layer does not use port; generated rules should not carry + # port/scheme so they get the legacy default {80, 443} treatment. + rules = _convert_e2b_per_host_rules({ + "api.example.com": [ + {"transform": {"headers": {"Authorization": "Bearer x"}}} + ], + }) + assert len(rules) == 1 + assert "port" not in rules[0]["match"] + assert "scheme" not in rules[0]["match"] + + def test_empty_transform_list_rejected(self): + # An empty entries list fans out to zero rules and would silently + # drop the host the caller keyed in — must raise, not no-op. + with pytest.raises(ValueError, match="empty list"): + _convert_e2b_per_host_rules({"api.example.com": []}) From f95039d3336cd0993e656b14971b4aff47d045b5 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Sun, 9 Aug 2026 22:32:52 +0800 Subject: [PATCH 06/31] feat(sdk/node): add port/scheme validation to egress rule match Align the TypeScript SDK with the Python SDK contract for the L7 custom-port feature (the original branch never touched sdk/node): - Match gains port?: number with the pairing contract documented - validateScheme normalizes (strip + lowercase) and enforces the http/https whitelist; the wire form always carries the canonical lowercase scheme - validateMatchPortScheme enforces: integer port in [1, 65535], port requires scheme; scheme alone stays legal (legacy {80,443} filter) - serializeMatch validates through the single serialization funnel so every rule path (typed rules and E2B-converted rules) is checked before the network round-trip - tests: wire emission, scheme normalization, missing-scheme and out-of-range/non-integer port rejection, unknown scheme rejection, legacy scheme-only acceptance Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- sdk/node/src/policy.ts | 65 +++++++++++++++++++++++++++++++++++- sdk/node/test/policy.test.ts | 60 +++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/sdk/node/src/policy.ts b/sdk/node/src/policy.ts index aa6df7bda..6b6103fc1 100644 --- a/sdk/node/src/policy.ts +++ b/sdk/node/src/policy.ts @@ -31,6 +31,16 @@ export type AuditLevel = "full" | "metadata" | "none"; * Rule match conditions. All fields optional; an empty match matches any * request. Semantics: AND across fields, OR within ``method``. Comparisons on * sni/host/scheme are case-insensitive (server-enforced). + * + * ``port`` + ``scheme`` together drive which TCP port CubeEgress intercepts + * on the sandbox side. Both must be set together or both omitted: + * + * - Both omitted → default set ``{80/http, 443/https}`` (backward compatible). + * - Both set → CubeEgress intercepts only that (host, port, scheme) tuple. + * + * Every rule sharing the same ``(host, port)`` MUST agree on ``scheme`` — + * a port can only route to one nginx listener (HTTP → 8080, HTTPS → 8443). + * The server rejects the whole policy if it detects a mismatch. */ export interface Match { sni?: string; @@ -38,6 +48,7 @@ export interface Match { method?: Method[]; path?: string; scheme?: Scheme; + port?: number; } /** @@ -87,13 +98,65 @@ export function renderInject(inject: Inject): string { return fmt.replace("${SECRET}", inject.secret); } +/** + * Normalize *value* (strip + lowercase) and validate against http/https. + * + * Returns the normalized scheme, or ``undefined`` when *value* is absent. + * Scheme matching is case-insensitive across the stack (Lua + * ``normalize_scheme`` and cubevs both lowercase/strip), so the SDK + * normalizes before the membership check and propagates the clean lowercase + * value downstream. + */ +function validateScheme(value: unknown, field: string): Scheme | undefined { + if (value === undefined) { + return undefined; + } + if (typeof value !== "string") { + throw new Error(`${field} must be 'http' or 'https', got ${JSON.stringify(value)}`); + } + const normalized = value.trim().toLowerCase(); + if (normalized !== "http" && normalized !== "https") { + throw new Error(`${field} must be 'http' or 'https', got ${JSON.stringify(value)}`); + } + return normalized; +} + +/** + * Client-side validation of the port/scheme pair, mirroring the Python SDK + * contract and the CubeAPI/CubeMaster server-side checks: a set port must be + * an integer in [1, 65535] and must be paired with a scheme. Scheme alone + * (no port) stays legal — it filters HTTP vs HTTPS on the default {80, 443} + * set, which is the classic behavior, not the port-scoped feature. + * + * Not strictly required (the server would reject the same shape) but catches + * typos before the network round-trip. + */ +function validateMatchPortScheme(match: Match): void { + if (match.port !== undefined) { + if (typeof match.port !== "number" || !Number.isInteger(match.port)) { + throw new Error(`match.port must be an int, got ${typeof match.port}`); + } + if (match.port < 1 || match.port > 65535) { + throw new Error(`match.port must be in [1, 65535], got ${match.port}`); + } + if (match.scheme === undefined) { + throw new Error("match.port requires match.scheme to be set"); + } + } +} + function serializeMatch(match: Match): Record { + // Validate and normalize before emitting the wire form so the canonical + // lowercase scheme reaches the server regardless of caller casing. + const scheme = validateScheme(match.scheme, "match.scheme"); + validateMatchPortScheme(match); const out: Record = {}; if (match.sni !== undefined) out.sni = match.sni; if (match.host !== undefined) out.host = match.host; if (match.method !== undefined) out.method = [...match.method]; if (match.path !== undefined) out.path = match.path; - if (match.scheme !== undefined) out.scheme = match.scheme; + if (scheme !== undefined) out.scheme = scheme; + if (match.port !== undefined) out.port = match.port; return out; } diff --git a/sdk/node/test/policy.test.ts b/sdk/node/test/policy.test.ts index 3a560c4e0..135ff7aac 100644 --- a/sdk/node/test/policy.test.ts +++ b/sdk/node/test/policy.test.ts @@ -66,6 +66,66 @@ describe("serializeRule", () => { { header: "Authorization", secret: "sk_xxx", format: "Bearer ${SECRET}" }, ]); }); + + it("emits port and normalizes scheme casing on the wire", () => { + const wire = serializeRule({ + name: "r1", + match: { host: "api.example.com", port: 8443, scheme: "HTTPS" as "https" }, + action: { allow: true }, + }); + expect(wire.match).toEqual({ host: "api.example.com", port: 8443, scheme: "https" }); + }); + + it("rejects port without scheme", () => { + expect(() => + serializeRule({ + name: "r1", + match: { host: "api.example.com", port: 8443 }, + action: { allow: true }, + }), + ).toThrow("match.port requires match.scheme to be set"); + }); + + it("rejects out-of-range ports", () => { + for (const port of [0, -1, 65536, 99999]) { + expect(() => + serializeRule({ + name: "r1", + match: { host: "api.example.com", port, scheme: "https" }, + action: { allow: true }, + }), + ).toThrow("match.port must be in [1, 65535]"); + } + }); + + it("rejects non-integer ports", () => { + expect(() => + serializeRule({ + name: "r1", + match: { host: "api.example.com", port: 443.5, scheme: "https" }, + action: { allow: true }, + }), + ).toThrow("match.port must be an int"); + }); + + it("rejects unknown schemes", () => { + expect(() => + serializeRule({ + name: "r1", + match: { host: "api.example.com", scheme: "gopher" as "http" }, + action: { allow: true }, + }), + ).toThrow("match.scheme must be 'http' or 'https'"); + }); + + it("accepts scheme alone (legacy default-set filter)", () => { + const wire = serializeRule({ + name: "r1", + match: { host: "api.example.com", scheme: "https" }, + action: { allow: true }, + }); + expect(wire.match).toEqual({ host: "api.example.com", scheme: "https" }); + }); }); describe("renderInject", () => { From 88794f6ce942aa56cfb49b946be331020aa59a6e Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 10 Aug 2026 09:47:04 +0800 Subject: [PATCH 07/31] feat(sdk/go): add port/scheme validation to egress rule match Align the Go SDK with the Python/TypeScript SDK contract for the L7 custom-port feature (the original branch never touched sdk/go): - Match gains Port int with the pairing contract documented - Match.validate enforces the http/https whitelist (case-insensitive), the [1, 65535] port range, and port-requires-scheme; scheme alone stays legal (legacy {80,443} filter) - buildCreatePayload validates and normalizes each rule match before serialization, with errors located by rule index and name; the wire form carries the canonical lowercase scheme - tests: wire serialization with scheme normalization, invalid port/scheme rejection matrix (asserting no network call is made), and legacy scheme-only acceptance Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- sdk/go/aligned_test.go | 91 ++++++++++++++++++++++++++++++++++++++++++ sdk/go/client.go | 10 ++++- sdk/go/policy.go | 41 +++++++++++++++++++ 3 files changed, 141 insertions(+), 1 deletion(-) diff --git a/sdk/go/aligned_test.go b/sdk/go/aligned_test.go index 3f259dba7..fd7c0c6ce 100644 --- a/sdk/go/aligned_test.go +++ b/sdk/go/aligned_test.go @@ -69,6 +69,97 @@ func TestCreateSerializesPolicyAndPublicTraffic(t *testing.T) { } } +func TestCreateSerializesRulePortAndNormalizesScheme(t *testing.T) { + var got map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&got) + w.WriteHeader(http.StatusCreated) + fmt.Fprint(w, sandboxJSON(testSandboxID, "tpl-env")) + })) + defer server.Close() + + client := NewClient(Config{APIURL: server.URL, TemplateID: "tpl-env", Timeout: 300 * time.Second}) + _, err := client.Create(context.Background(), CreateOptions{ + Network: NetworkOptions{ + Rules: []Rule{{ + Name: "custom-https", + Match: Match{Host: "api.example.com", Port: 8443, Scheme: " HTTPS "}, + Action: Action{Allow: true}, + }}, + }, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + + network := got["network"].(map[string]any) + rules := network["rules"].([]any) + rule := rules[0].(map[string]any) + match := rule["match"].(map[string]any) + if match["port"] != float64(8443) { + t.Fatalf("match.port=%#v, want 8443", match["port"]) + } + if match["scheme"] != "https" { + t.Fatalf("match.scheme=%#v, want normalized https", match["scheme"]) + } +} + +func TestCreateRejectsInvalidRulePortScheme(t *testing.T) { + called := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + w.WriteHeader(http.StatusCreated) + fmt.Fprint(w, sandboxJSON(testSandboxID, "tpl-env")) + })) + defer server.Close() + + tests := []struct { + name string + match Match + wantErr string + }{ + {"port without scheme", Match{Host: "a.com", Port: 8443}, "port requires match.scheme"}, + {"port too low", Match{Host: "a.com", Port: 0, Scheme: "https"}, "not-reached"}, // 0 means unset + {"port negative", Match{Host: "a.com", Port: -1, Scheme: "https"}, "[1, 65535]"}, + {"port too high", Match{Host: "a.com", Port: 65536, Scheme: "https"}, "[1, 65535]"}, + {"unknown scheme", Match{Host: "a.com", Scheme: "gopher"}, "must be 'http' or 'https'"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == "not-reached" { + return // Port 0 is the zero value (unset); covered by the "port without scheme" case + } + client := NewClient(Config{APIURL: server.URL, TemplateID: "tpl-env"}) + _, err := client.Create(context.Background(), CreateOptions{ + Network: NetworkOptions{ + Rules: []Rule{{Name: "r1", Match: tt.match, Action: Action{Allow: true}}}, + }, + }) + if err == nil { + t.Fatalf("err=nil, want %q", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err=%q, want substring %q", err, tt.wantErr) + } + }) + } + if called { + t.Fatal("server was called despite client-side validation failure") + } +} + +func TestMatchValidateAcceptsLegacySchemeOnly(t *testing.T) { + // Scheme alone (no port) filters HTTP vs HTTPS on the default {80, 443} + // set — the classic behavior, not the port-scoped feature. + m := Match{Host: "api.example.com", Scheme: "HTTPS"} + if err := m.validate(); err != nil { + t.Fatalf("validate: %v", err) + } + if got := m.normalized().Scheme; got != "https" { + t.Fatalf("normalized scheme=%q, want https", got) + } +} + func TestCreateRejectsAllowOutDomainWithoutDenyAll(t *testing.T) { called := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/sdk/go/client.go b/sdk/go/client.go index 3d72208c3..411f026a2 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -160,7 +160,15 @@ func (c *Client) createPayload(opts CreateOptions) (map[string]any, error) { network["denyOut"] = opts.Network.DenyOut } if len(opts.Network.Rules) > 0 { - network["rules"] = opts.Network.Rules + rules := make([]Rule, len(opts.Network.Rules)) + for i, rule := range opts.Network.Rules { + if err := rule.Match.validate(); err != nil { + return nil, fmt.Errorf("network.rules[%d] %q: %w", i, rule.Name, err) + } + rule.Match = rule.Match.normalized() + rules[i] = rule + } + network["rules"] = rules } if len(network) > 0 { payload["network"] = network diff --git a/sdk/go/policy.go b/sdk/go/policy.go index 3a5846d7f..721770b92 100644 --- a/sdk/go/policy.go +++ b/sdk/go/policy.go @@ -4,6 +4,7 @@ package cubesandbox import ( + "fmt" "net" "strings" ) @@ -16,12 +17,52 @@ import ( // Match holds rule match conditions. All fields are optional; an empty Match // matches any request. Fields are AND-ed; Method values are OR-ed; // sni/host/scheme are compared case-insensitively server-side. +// +// Port + Scheme together drive which TCP port CubeEgress intercepts on the +// sandbox side. Both must be set together or both omitted: +// - Both omitted → default set {80/http, 443/https} (backward compatible). +// - Both set → CubeEgress intercepts only that (host, port, scheme) tuple. +// +// Every rule sharing the same (host, port) MUST agree on Scheme — a port can +// only route to one nginx listener. The server rejects the whole policy on +// mismatch. type Match struct { SNI string `json:"sni,omitempty"` Host string `json:"host,omitempty"` Method []string `json:"method,omitempty"` Path string `json:"path,omitempty"` Scheme string `json:"scheme,omitempty"` + Port int `json:"port,omitempty"` +} + +// validate checks the port/scheme pair, mirroring the Python and TypeScript +// SDKs and the CubeAPI/CubeMaster server-side contract: a set Port must be in +// [1, 65535] and must be paired with Scheme, and a set Scheme must be http or +// https (case-insensitive). Client-side validation catches typos before the +// network round-trip; the server rejects the same shapes. +func (m Match) validate() error { + if m.Scheme != "" { + scheme := strings.ToLower(strings.TrimSpace(m.Scheme)) + if scheme != "http" && scheme != "https" { + return fmt.Errorf("match.scheme must be 'http' or 'https', got %q", m.Scheme) + } + } + if m.Port != 0 { + if m.Port < 1 || m.Port > 65535 { + return fmt.Errorf("match.port must be in [1, 65535], got %d", m.Port) + } + if m.Scheme == "" { + return fmt.Errorf("match.port requires match.scheme to be set") + } + } + return nil +} + +// normalized returns a copy with Scheme canonicalized (stripped, lowercased) +// so the wire form is consistent regardless of caller casing. +func (m Match) normalized() Match { + m.Scheme = strings.ToLower(strings.TrimSpace(m.Scheme)) + return m } // Inject injects a credential header on an allowed HTTPS request whose From 86ceca1446bc1618a4e5c7741df7916bebb795c3 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 10 Aug 2026 10:14:08 +0800 Subject: [PATCH 08/31] feat(deploy): bind L7 skb-mark config into one-click install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the one-click deployment binding of the L7 custom-port feature from L7_support_portconfig (gitwoa), adapted to the #1285 embedded network runtime wording: - install.sh: write_l7_marks_conf persists /etc/cubeegress/l7-marks.conf from CUBE_L7_MARK_{HTTP,HTTPS,MASK} (shipped defaults 0xCE010000/0xCE020000/0xFFFF0000) before the consuming systemd units start. Values are validated arithmetically so the same mark in different notations (hex case, decimal) is still rejected, and bits outside the mask are refused — matching cubevs.resolveL7Marks and the cube-proxy-iptables-init checks. - env.example: documented override block for the three marks. - README.md / README_zh.md: config placement table gains the l7-marks.conf row, referencing the embedded network runtime instead of the removed network-agent. Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- deploy/one-click/README.md | 1 + deploy/one-click/README_zh.md | 1 + deploy/one-click/env.example | 15 +++++++++++++++ deploy/one-click/install.sh | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+) diff --git a/deploy/one-click/README.md b/deploy/one-click/README.md index 0c1b9b6de..27b523185 100644 --- a/deploy/one-click/README.md +++ b/deploy/one-click/README.md @@ -170,6 +170,7 @@ One-click does not create an extra global `configs/` layer on the target machine - `cubelet_conf.default_timeout_insec`: cluster default sandbox idle TTL when the client omits `timeout`; unset or `<= 0` means **no cluster-wide idle timeout** (shipped default `-1`). See [lifecycle — Operational Notes](../../docs/guide/lifecycle.md#cluster-default-idle-timeout-default_timeout_insec). - `Cubelet/config/` → `Cubelet/config/` - `Cubelet/dynamicconf/` → `Cubelet/dynamicconf/` +- `CUBE_L7_MARK_{HTTP,HTTPS,MASK}` (env) → `/etc/cubeegress/l7-marks.conf` — the L7 egress skb->mark values shared by Cubelet's embedded network runtime eBPF dataplane (which stamps `skb->mark`) and the `cube-proxy-iptables-init` TPROXY rules (which match it). Both read the same file, and both validate the values (`HTTP != HTTPS`, bits confined to the mask). See `env.example` for the shipped defaults and how to override them. - `CubeAPI/bin/cube-api` → `/usr/local/services/cubetoolbox/CubeAPI/bin/cube-api` - `support/` → `/usr/local/services/cubetoolbox/support/` - `cubeproxy/` → `/usr/local/services/cubetoolbox/cubeproxy/` diff --git a/deploy/one-click/README_zh.md b/deploy/one-click/README_zh.md index 0def14894..8975f7211 100644 --- a/deploy/one-click/README_zh.md +++ b/deploy/one-click/README_zh.md @@ -159,6 +159,7 @@ one-click 不会在目标机额外创建一层全局 `configs/`，而是直接 - `cubelet_conf.default_timeout_insec`: cluster default sandbox idle TTL when the client omits `timeout`; unset or `<= 0` means **no cluster-wide idle timeout** (shipped default `-1`). See [lifecycle — 设计与运维要点](../../docs/zh/guide/lifecycle.md#集群默认空闲超时default_timeout_insec)。 - `Cubelet/config/` -> `Cubelet/config/` - `Cubelet/dynamicconf/` -> `Cubelet/dynamicconf/` +- `CUBE_L7_MARK_{HTTP,HTTPS,MASK}`（环境变量） -> `/etc/cubeegress/l7-marks.conf` —— L7 egress 的 skb->mark 值，由 Cubelet 内置 network runtime 的 eBPF 数据面（负责打标）与 `cube-proxy-iptables-init` 的 TPROXY 规则（负责匹配）共用。两侧读取同一文件并各自校验（`HTTP != HTTPS`、取值只能落在 mask 位内）。默认值与覆盖方式见 `env.example`。 - `CubeAPI/bin/cube-api` -> `/usr/local/services/cubetoolbox/CubeAPI/bin/cube-api` - `support/` -> `/usr/local/services/cubetoolbox/support/` - `cubeproxy/` -> `/usr/local/services/cubetoolbox/cubeproxy/` diff --git a/deploy/one-click/env.example b/deploy/one-click/env.example index 7b5e3f391..05da46262 100644 --- a/deploy/one-click/env.example +++ b/deploy/one-click/env.example @@ -178,6 +178,21 @@ CUBE_SANDBOX_CUBE_ROUTER_ENABLE=0 # usable IPs from CUBE_SANDBOX_NETWORK_CIDR. # CUBE_SANDBOX_CUBE_ROUTER_CIDR=10.254.0.0/24 +# ---- L7 egress skb->mark overrides ---- +# The L7 egress dataplane (Cubelet embedded network runtime eBPF) and the +# iptables TPROXY rules steer intercepted HTTP/HTTPS traffic by skb->mark. +# install.sh writes /etc/cubeegress/l7-marks.conf from these env vars (with the +# shipped defaults) before the consuming units start, and both Cubelet and the +# iptables init script read that same file, so the stamped and matched marks +# stay in lock-step. Override only to dodge a collision with an existing fwmark +# scheme in your environment: HTTP and HTTPS must differ and may only set bits +# inside CUBE_L7_MARK_MASK (keep the high 16 bits cube-owned). Leave unset to +# keep the shipped defaults; install.sh validates and refuses to persist +# invalid values. +# CUBE_L7_MARK_HTTP=0xCE010000 +# CUBE_L7_MARK_HTTPS=0xCE020000 +# CUBE_L7_MARK_MASK=0xFFFF0000 + # Cube proxy options. CUBE_PROXY_ENABLE=1 # Optional full image override (otherwise selected by MIRROR=cn|int): diff --git a/deploy/one-click/install.sh b/deploy/one-click/install.sh index 03a687800..4dda1e4f4 100755 --- a/deploy/one-click/install.sh +++ b/deploy/one-click/install.sh @@ -108,6 +108,37 @@ warn_default_external_credentials() { fi } +# Write /etc/cubeegress/l7-marks.conf so the L7 skb->mark values used by the +# dataplane (Cubelet embedded network runtime's eBPF globals) and the iptables +# TPROXY rules stay in lock-step. Override via CUBE_L7_MARK_{HTTP,HTTPS,MASK} +# in your env/.env; defaults match the shipped values. +write_l7_marks_conf() { + local http="${CUBE_L7_MARK_HTTP:-0xCE010000}" + local https="${CUBE_L7_MARK_HTTPS:-0xCE020000}" + local mask="${CUBE_L7_MARK_MASK:-0xFFFF0000}" + + # Validate before persisting: http must differ from https, and both may only + # set bits inside the mask. Compare arithmetically (not as strings) so the + # same value in different notations — 0xCE010000 vs 0xce010000 vs decimal — + # is still rejected, matching cubevs.resolveL7Marks. + if (( http == https )); then + die "CUBE_L7_MARK_HTTP (${http}) must differ from CUBE_L7_MARK_HTTPS" + fi + if (( (http & ~mask) != 0 || (https & ~mask) != 0 )); then + die "CUBE_L7_MARK_* values must set bits only within CUBE_L7_MARK_MASK (${mask})" + fi + + mkdir -p /etc/cubeegress + cat > /etc/cubeegress/l7-marks.conf <mark values. +CUBE_L7_MARK_HTTP=${http} +CUBE_L7_MARK_HTTPS=${https} +CUBE_L7_MARK_MASK=${mask} +EOF + log "wrote /etc/cubeegress/l7-marks.conf (http=${http} https=${https} mask=${mask})" +} + INSTALL_PREFIX="${CUBE_SANDBOX_INSTALL_ROOT}" # Resolve install vs upgrade mode and, for upgrades, run preflight + backup and @@ -1712,6 +1743,8 @@ else fi restore_selinux_contexts +# Persist the L7 skb->mark config before the units that consume it start. +write_l7_marks_conf install_systemd_units mask_external_dep_services check_runtime_file_paths_not_directories From 7d68c8f68a8e7fd23bbba05928234239dc6f99cf Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 10 Aug 2026 10:20:11 +0800 Subject: [PATCH 09/31] feat(examples): port L7 egress network demos incl. custom-port echo Port the L7 egress examples from L7_support_portconfig (gitwoa): - cube-test-network.py: comprehensive L7 egress network demo - network_l7_custom_port_echo.py: four-quadrant custom-port demo (default http/https, default both-set, custom http on 18080, custom https on 1012) with a self-contained local echo server leg Both are new files; py_compile passes. Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- .../cube-test-network.py | 998 ++++++++++++++++++ .../network_l7_custom_port_echo.py | 215 ++++ 2 files changed, 1213 insertions(+) create mode 100644 examples/code-sandbox-quickstart/cube-test-network.py create mode 100644 examples/code-sandbox-quickstart/network_l7_custom_port_echo.py diff --git a/examples/code-sandbox-quickstart/cube-test-network.py b/examples/code-sandbox-quickstart/cube-test-network.py new file mode 100644 index 000000000..55507f0ee --- /dev/null +++ b/examples/code-sandbox-quickstart/cube-test-network.py @@ -0,0 +1,998 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 +"""End-to-end smoke test for L7 egress rules. + +Drives the Python SDK to create sandboxes with various ``network.rules`` +configurations and asserts the data plane (CubeEgress) actually enforces +them. The intent is operator-runnable verification, not pytest unit +testing — each scenario prints PASS/FAIL inline and the whole script +returns non-zero if any failed. + +Coverage: every CubeEgress externally-observable capability gets at +least one positive and (where meaningful) one negative case: + + Match: host exact, host *.suffix wildcard, scheme http/https, + method (single + any-of array), path exact, path "*" prefix. + Action: allow=true (passthrough), allow=false (403), + audit=metadata|none|full (smoke), inject[] with custom format, + inject[] with default ${SECRET} format, multiple injects in + one rule, strip-forged-header invariant, inject gate G1 + (https-only) drop on http. + Decision semantics: first-match-wins both directions, default-deny + when no rule matches. + Session longevity: one keep-alive HTTPS connection serves 1 request + per minute for 10 requests — proves the dataplane + session keeps steering an established flow to + CubeEgress even after the host's DNS-learned + allow_out_v3 /48 entry ages out. Runs twice: on + the default https port and on a custom port + (1012, where the legacy drain shim does NOT + rescue a lost session). + +Restart-resilience: every case runs each of its probes TWICE — once +against the cube-egress that was already up when the sandbox was +created, then ``docker restart cube-egress`` and the SAME probes run +again against the fresh process. The case passes only if both passes +satisfy their predicates. This proves cube-egress correctly rebuilds +its in-memory ``policy_store`` from network-agent's +``/v1/policies/dump`` source-of-truth on cold start, with no +intervention needed (we DO NOT reinstall the policy ourselves between +the two passes). + +Prerequisites: + - CUBE_API_URL, CUBE_TEMPLATE_ID set + - The template was built with --with-cube-ca=true (otherwise HTTPS + interception will fail with self-signed-cert errors and the rule + enforcement test results will be meaningless) + - cube-egress is deployed on the node AS A DOCKER CONTAINER NAMED + ``cube-egress`` (we drive ``docker restart cube-egress``); the + runner needs permission to talk to the docker socket. + - The sandbox image has Python with stdlib urllib (we deliberately + avoid `requests` since it isn't on every base image) + +Echo target: most cases use https://httpbingo.org/headers (and its +http:// counterpart) because it deterministically echoes the headers +the upstream server saw — the only public way to confirm an inject +actually flowed all the way to the origin without spending money on +a real authenticated API. We picked httpbingo over httpbin.org +specifically because httpbin's anycast often returns 503 from +mainland China (and its eu./www./nghttp2.org/httpbin mirrors are +network-blocked there); httpbingo (run on fly.io) responds reliably. +The wire shape differs slightly — httpbingo wraps every header value +in a single-element JSON array, e.g. ``"X-Api-Key": ["secret"]`` — +but our predicates use plain substring grep so the SECRET literal +still matches inside the array. + +Usage: + export CUBE_TEMPLATE_ID=tpl-d69160afa18a4c8b90b40b45 + export CUBE_API_URL=http://127.0.0.1:3000 + cd CubeSandbox/sdk/python && pip install -e . + cd ../../tests + python3 cube-test-network.py + +Exit code: 0 on all-pass, non-zero on any failure or precondition miss. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +import traceback +import urllib.request +from typing import Callable, List, Tuple +from env_utils import load_local_dotenv +load_local_dotenv() + +# ── SDK import ─────────────────────────────────────────────────────────────── +try: + from cubesandbox import Sandbox, Rule, Match, Action, Inject +except ImportError as exc: + print("[FATAL] cubesandbox SDK not importable:", exc) + print(" Install with: cd sdk/python && pip install -e .") + sys.exit(2) + + +# ── helpers ────────────────────────────────────────────────────────────────── + +# Container + admin-port wiring. cube-egress's admin loopback listens on +# 127.0.0.1:9090 (declared in CubeEgress/nginx.conf). The container is named +# ``cube-egress`` by the systemd unit ``cube-sandbox-cube-egress.service``. +CUBE_EGRESS_CONTAINER = "cube-egress" +CUBE_EGRESS_HEALTH_URL = "http://127.0.0.1:9090/admin/v1/health" +RESTART_BOOTSTRAP_TIMEOUT = 60 # seconds + + +def _check_env() -> None: + """Bail out early if the SDK has nothing to talk to.""" + missing = [v for v in ("CUBE_API_URL", "CUBE_TEMPLATE_ID") if not os.environ.get(v)] + if missing: + print(f"[FATAL] missing env: {', '.join(missing)}") + print(" export them before running this test") + sys.exit(2) + + +def _restart_cube_egress() -> None: + """``docker restart cube-egress`` and wait until it is serving again. + + "Serving" is observed via the admin /health endpoint: + - it must answer at all (worker accepting on :9090) + - bootstrap_status must reach "ready" (worker has fetched the + policy dump from network-agent's :19090/v1/policies/dump) + + We deliberately do NOT re-install the test's policy ourselves — + the whole point of the second pass is to verify cube-egress + rebuilt its in-memory policy_store from network-agent on cold + start. If bootstrap landed on "skipped" (because + CUBE_EGRESS_BOOTSTRAP_URL wasn't set on the container), the + second pass would see policy_count=0 and most cases would 403, + which is exactly the regression we want to surface. + + Raises RuntimeError if the container is not back to ready + within RESTART_BOOTSTRAP_TIMEOUT — the case driver treats this + as a fatal precondition miss for the post-restart pass. + """ + print(f" ── restarting {CUBE_EGRESS_CONTAINER} ──") + t0 = time.monotonic() + try: + subprocess.run( + ["docker", "restart", CUBE_EGRESS_CONTAINER], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=30, + ) + except subprocess.CalledProcessError as exc: + raise RuntimeError( + f"docker restart {CUBE_EGRESS_CONTAINER} failed: {exc.stderr.decode().strip()}" + ) from exc + except subprocess.TimeoutExpired as exc: + raise RuntimeError( + f"docker restart {CUBE_EGRESS_CONTAINER} timed out after 30s" + ) from exc + + deadline = time.monotonic() + RESTART_BOOTSTRAP_TIMEOUT + last_err: str | None = None + while time.monotonic() < deadline: + try: + resp = urllib.request.urlopen(CUBE_EGRESS_HEALTH_URL, timeout=2) + body = json.loads(resp.read().decode()) + if body.get("bootstrap_status") == "ready": + print( + f" ── ready in {time.monotonic() - t0:.1f}s " + f"(policy_count={body.get('policy_count')}) ──" + ) + # Belt-and-braces: give the listener a moment to settle the + # transparent (TPROXY) sockets. Without this, the very first + # post-restart probe occasionally lands on a half-initialised + # worker and gets a connection reset before access_phase + # runs. One second is empirically enough; this is far below + # any reasonable per-case timeout budget. + time.sleep(1.0) + return + last_err = f"bootstrap_status={body.get('bootstrap_status')}" + except Exception as exc: # noqa: BLE001 + last_err = type(exc).__name__ + ": " + str(exc) + time.sleep(1.0) + raise RuntimeError( + f"{CUBE_EGRESS_CONTAINER} did not reach ready within " + f"{RESTART_BOOTSTRAP_TIMEOUT}s (last={last_err})" + ) + + +def _http_probe(url: str, *, capture_body: bool = False, method: str = "GET", + extra_headers: dict | None = None) -> str: + """Sandbox-side helper: returns code that prints status + (optional) body. + + capture_body=True dumps the entire response body so we can grep echoed + headers when the upstream is httpbingo.org/headers or /anything. + extra_headers lets a test inject a *sandbox-side* header (the thing + CubeEgress is supposed to clear when the rule plans an inject of the + same name). + """ + body_block = "" + if capture_body: + # Use unbounded resp.read() — chunked-transfer responses can + # return 0 bytes from a size-bounded first read on Python's + # urllib, which gave us empty BODY=... lines on httpbin in + # earlier runs. We trust the upstream to be small (httpbin + # /headers ≈ 200 B; /anything ≈ 1 KiB). + # NOTE: lines must be indented 8 spaces so they stay INSIDE the + # `with urlopen(...) as resp:` block — otherwise resp is closed + # by the time we hit .read() and we get a deceptive zero-byte + # body that silently fails the inject assertions. + body_block = ( + " raw_body = resp.read()\n" + " print(f'BODY_RAW_LEN={len(raw_body)}')\n" + " body = raw_body.decode('utf-8', 'replace')\n" + " print('BODY=' + body.replace(chr(10), ' '))\n" + ) + headers_block = "" + if extra_headers: + # Render as a Python dict literal embedded in the sandbox script. + items = ", ".join(f"{k!r}: {v!r}" for k, v in extra_headers.items()) + headers_block = f"_extra = {{{items}}}\n" + else: + headers_block = "_extra = {}\n" + + return f"""\ +import urllib.request, urllib.error +{headers_block}_req = urllib.request.Request({url!r}, method={method!r}) +for _k, _v in _extra.items(): + _req.add_header(_k, _v) +try: + with urllib.request.urlopen(_req, timeout=30) as resp: + print('STATUS=' + str(resp.status)) +{body_block}except urllib.error.HTTPError as e: + print('STATUS=' + str(e.code)) +except Exception as e: + print('ERROR=' + type(e).__name__ + ':' + str(e)) +""" + + +def _logs_text(execution) -> str: + """Flatten Execution stdout/stderr into one string for grep.""" + lines: List[str] = [] + if execution.logs and execution.logs.stdout: + lines.extend(execution.logs.stdout) + if execution.logs and execution.logs.stderr: + lines.extend(execution.logs.stderr) + if execution.text: + lines.append(execution.text) + return "\n".join(lines) + + +# ── twin-pass driver ───────────────────────────────────────────────────────── + +# A "run" in a case is a triple (sub_label, sandbox-side code, predicate). +# Predicate takes the stdout string and returns True/False. The driver +# executes every run once, restarts cube-egress, then executes every run +# AGAIN inside the SAME sandbox, and returns True only if all 2N +# predicate evaluations hold. + +Run = Tuple[str, str, Callable[[str], bool]] + +# Markers indicating a transient upstream-hop failure inside cube-egress +# rather than a policy-decision result. We retry the probe (not the +# restart) when the predicate fails AND one of these markers is present. +# - STATUS=502/503/504: nginx couldn't reach / TLS-verify the origin +# (proxy_ssl_verify rejection, SYN drop, slow handshake > 10s +# proxy_connect_timeout — all of which audit.lua flags as +# upstream_unreachable_or_unverified). Especially common right +# after a cold restart while DNS/TLS sessions are warming up. +# - ERROR=URLError/TimeoutError/RemoteDisconnected: the sandbox-side +# socket layer broke before/during the egress hop. +# We deliberately do NOT include STATUS=403 here — 403 is cube-egress' +# definitive policy decision, retrying it would only mask real +# regressions. +_TRANSIENT_MARKERS = ( + "STATUS=502", + "STATUS=503", + "STATUS=504", + "ERROR=URLError", + "ERROR=TimeoutError", + "ERROR=RemoteDisconnected", + "ERROR=ConnectionResetError", + "ERROR=BadStatusLine", +) +_PROBE_MAX_ATTEMPTS = 3 +_PROBE_RETRY_BACKOFF_SEC = 1.5 + + +def _is_transient(out: str) -> bool: + """Heuristic: did this probe fail with an upstream-hop error? + + Used to decide whether to retry the probe inside the SAME pass + (no extra restart). Keep the marker list narrow so we don't + paper over real policy regressions. + """ + return any(m in out for m in _TRANSIENT_MARKERS) + + +def _exec_runs(sb: Sandbox, runs: List[Run], pass_label: str) -> bool: + """Run every entry in `runs` against `sb` and return whether all passed. + + Each probe is retried up to ``_PROBE_MAX_ATTEMPTS`` times when its + predicate fails AND the output looks like a transient upstream-hop + failure (see ``_TRANSIENT_MARKERS``). Real policy regressions — + e.g. STATUS=200 where the test expected 403 — fail on the first + attempt without retry. + + Output for each entry is printed inline; the caller decides what + to do with the boolean result. + """ + all_ok = True + for sub_label, code, pred in runs: + out = "" + ok = False + for attempt in range(1, _PROBE_MAX_ATTEMPTS + 1): + out = _logs_text(sb.run_code(code)) + ok = pred(out) + if ok or not _is_transient(out) or attempt == _PROBE_MAX_ATTEMPTS: + break + # Transient AND we still have budget: pause briefly and retry. + print(f" [{pass_label} {sub_label}] transient (attempt {attempt}/" + f"{_PROBE_MAX_ATTEMPTS}); retrying after " + f"{_PROBE_RETRY_BACKOFF_SEC}s") + time.sleep(_PROBE_RETRY_BACKOFF_SEC) + + snippet = out.strip().replace("\n", " | ") + if len(snippet) > 400: + snippet = snippet[:400] + "...(truncated)" + all_ok = all_ok and ok + marker = "ok" if ok else "FAIL" + attempt_tag = f" (attempt {attempt})" if attempt > 1 else "" + print(f" [{pass_label} {sub_label}] {marker}{attempt_tag}: {snippet}") + return all_ok + + +def _run_twice_with_restart(sb: Sandbox, runs: List[Run]) -> bool: + """Drive a case: run all probes, restart cube-egress, run them again. + + Returns True iff every probe satisfied its predicate on BOTH passes. + """ + pre_ok = _exec_runs(sb, runs, "pre") + try: + _restart_cube_egress() + except RuntimeError as exc: + print(f" [restart] FAIL: {exc}") + return False + post_ok = _exec_runs(sb, runs, "post") + return pre_ok and post_ok + + +# ── scenario builders ──────────────────────────────────────────────────────── +# +# Each case constructs the sandbox, declares its `runs` list, and hands +# both to _run_twice_with_restart. The case body is therefore mostly +# data — match/action shape + per-probe predicate. + +def _has_status(code: int) -> Callable[[str], bool]: + """Predicate: stdout contains exactly STATUS=. + + NOT a substring check — we use word boundaries via the literal + 'STATUS=' marker our probe prints, which is unambiguous. + """ + needle = f"STATUS={code}" + return lambda out: needle in out + + +def _allow_then_403(code_allow: int = 200, code_deny: int = 403) -> Callable[[str], bool]: + """Allow-and-no-403: a STATUS= line AND no STATUS=.""" + a, d = f"STATUS={code_allow}", f"STATUS={code_deny}" + return lambda out: a in out and d not in out + + +def case_allow_specific_host() -> bool: + """Allow rule on https://api.moonshot.cn/:443; expect a real HTTPS response.""" + rules = [ + Rule( + name="allow_api.moonshot.cn", + match=Match(scheme="https", host="api.moonshot.cn"), + action=Action(allow=True, audit="metadata"), + ), + ] + runs: List[Run] = [ + ("GET https://api.moonshot.cn", + _http_probe("https://api.moonshot.cn"), + _has_status(200)), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_allow_suffix_host() -> bool: + """Allow rule on https://*.moonshot.cn/:443; expect a real HTTPS response.""" + DENIED_CIDRS = [ + "0.0.0.0/0", # block direct egress so traffic must use cube-egress tap + ] + rules = [ + Rule( + name="allow_suffix", + match=Match(host="*.moonshot.cn"), + action=Action(allow=True, audit="metadata"), + ), + ] + runs: List[Run] = [ + ("GET https://api.moonshot.cn", + _http_probe("https://api.moonshot.cn"), + _has_status(200)), + ] + with Sandbox.create(network={ + "allow_out": ["183.60.83.19"], + "deny_out": DENIED_CIDRS, + "rules": rules, + }) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_deny_rule_returns_403() -> bool: + """Explicit deny rule on a host; expect HTTP 403 from CubeEgress. + + Note: CubeEgress's deny path returns 403 without sending the request + upstream. urllib raises HTTPError for non-2xx, which our probe maps + back to STATUS=403. + """ + rules = [ + Rule( + name="block_evil", + match=Match(scheme="https", host="api.moonshot.cn"), + action=Action(allow=False), + ), + ] + runs: List[Run] = [ + ("GET https://api.moonshot.cn (denied)", + _http_probe("https://api.moonshot.cn"), + _has_status(403)), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_first_match_wins_order() -> bool: + """Allow then deny, both matching same host. + + First-match-wins means the allow at index 0 takes effect; a STATUS + other than 403 confirms the deny didn't override. + """ + rules = [ + Rule(name="allow_first", + match=Match(scheme="https", host="api.moonshot.cn"), + action=Action(allow=True)), + Rule(name="deny_second", + match=Match(scheme="https", host="api.moonshot.cn"), + action=Action(allow=False)), + ] + runs: List[Run] = [ + ("GET https://api.moonshot.cn (allow wins)", + _http_probe("https://api.moonshot.cn"), + _allow_then_403(200, 403)), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_first_match_wins_deny_before_allow() -> bool: + """Deny then allow, both matching same host. + + The mirror of case_first_match_wins_order — proves the order matters + in *both* directions (a buggy "last-match-wins" implementation would + pass the allow-first case but fail this one). + """ + rules = [ + Rule(name="deny_first", + match=Match(scheme="https", host="api.moonshot.cn"), + action=Action(allow=False)), + Rule(name="allow_second", + match=Match(scheme="https", host="api.moonshot.cn"), + action=Action(allow=True)), + ] + runs: List[Run] = [ + ("GET https://api.moonshot.cn (deny wins)", + _http_probe("https://api.moonshot.cn"), + _has_status(403)), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_default_deny_when_no_rule_matches() -> bool: + """Policy installed with a host-scoped rule whose path doesn't match; + request to a different path on the same host → 403. + + access_phase walks every rule; if none match, the request hits the + bottom-of-policy default-deny branch (`no_rule_match`). This proves + a non-empty policy doesn't accidentally fail open when match + conditions narrow the rule down to a sub-slice of the host. + + We deliberately keep the rule's host and the request's host the same + so the cubevs DNS allowlist (built from rule.match.host) still + resolves the lookup — DNS is L3, default-deny is L7. The path + mismatch is what triggers no_rule_match. + """ + rules = [ + Rule( + name="only_specific_path", + match=Match(scheme="https", host="httpbingo.org", + path="/never-matches-this-exact-path"), + action=Action(allow=True), + ), + ] + runs: List[Run] = [ + ("GET https://httpbingo.org/headers (default-deny)", + _http_probe("https://httpbingo.org/headers"), + _has_status(403)), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_method_filter_blocks_other_methods() -> bool: + """Rule with method=['GET'] allows GET but blocks POST.""" + rules = [ + Rule( + name="get_only", + match=Match(scheme="https", host="httpbingo.org", method=["GET"]), + action=Action(allow=True, audit="metadata"), + ), + ] + runs: List[Run] = [ + ("GET /headers", _http_probe("https://httpbingo.org/headers", method="GET"), + _has_status(200)), + ("POST /anything", _http_probe("https://httpbingo.org/anything", method="POST"), + _has_status(403)), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_method_any_of_array() -> bool: + """Method any-of: ['GET','POST'] both allowed; PUT denied.""" + rules = [ + Rule(name="get_or_post", + match=Match(scheme="https", host="httpbingo.org", method=["GET", "POST"]), + action=Action(allow=True)), + ] + runs: List[Run] = [ + ("GET /anything", _http_probe("https://httpbingo.org/anything", method="GET"), + _has_status(200)), + ("POST /anything", _http_probe("https://httpbingo.org/anything", method="POST"), + _has_status(200)), + ("PUT /anything", _http_probe("https://httpbingo.org/anything", method="PUT"), + _has_status(403)), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_path_exact_match() -> bool: + """Rule with path='/headers' allows /headers, denies /anything.""" + rules = [ + Rule(name="only_headers_path", + match=Match(scheme="https", host="httpbingo.org", path="/headers"), + action=Action(allow=True)), + ] + runs: List[Run] = [ + ("GET /headers (allow)", _http_probe("https://httpbingo.org/headers"), _has_status(200)), + ("GET /anything (deny)", _http_probe("https://httpbingo.org/anything"), _has_status(403)), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_path_prefix_wildcard() -> bool: + """Rule path='/anything*' allows /anything and /anything/foo; /headers denied. + + The trailing '*' is the only path wildcard CubeEgress supports + (suffix-side; matches the way paths grow to the right). + """ + rules = [ + Rule(name="anything_prefix", + match=Match(scheme="https", host="httpbingo.org", path="/anything*"), + action=Action(allow=True)), + ] + runs: List[Run] = [ + ("GET /anything", _http_probe("https://httpbingo.org/anything"), _has_status(200)), + ("GET /anything/foo", _http_probe("https://httpbingo.org/anything/foo"), _has_status(200)), + ("GET /headers (deny)", _http_probe("https://httpbingo.org/headers"), _has_status(403)), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_scheme_http_allows_plain_http() -> bool: + """scheme='http' rule allows the http:// listener (port 80 → 8080 TPROXY). + + Plain HTTP path is a separate code path from the SNI-driven HTTPS + path; we want a smoke that confirms the access_phase fires and lets + a matching rule through on the unencrypted listener. + """ + rules = [ + Rule(name="http_only", + match=Match(scheme="http", host="httpbingo.org"), + action=Action(allow=True)), + ] + runs: List[Run] = [ + ("GET http://httpbingo.org/headers", + _http_probe("http://httpbingo.org/headers"), + _has_status(200)), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_inject_default_format_renders_secret_only() -> bool: + """Inject without ``format`` defaults to bare ``${SECRET}``. + + Verify by greping the echoed-headers body returned from + https://httpbingo.org/headers — the X-Api-Key value should be the + raw secret with no Bearer/literal prefix. + """ + SECRET = "cube-test-DEFAULT-FMT-eb91" + rules = [ + Rule( + name="inject_default_fmt", + match=Match(scheme="https", host="httpbingo.org"), + action=Action(allow=True, audit="metadata", + inject=[Inject(header="X-Api-Key", secret=SECRET)]), + ), + ] + pred = lambda out: ("STATUS=200" in out) and (SECRET in out) # noqa: E731 + runs: List[Run] = [ + ("GET /headers w/ X-Api-Key inject", + _http_probe("https://httpbingo.org/headers", capture_body=True), + pred), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_inject_multiple_headers_in_one_rule() -> bool: + """Two inject entries on the same rule both flow upstream. + + Validates apply_injects walks the whole inject_list (not just [0]). + """ + SECRET_A = "cube-test-A-77f1" + SECRET_B = "cube-test-B-2c0d" + rules = [ + Rule( + name="inject_pair", + match=Match(scheme="https", host="httpbingo.org"), + action=Action( + allow=True, + inject=[ + Inject(header="X-Cube-A", secret=SECRET_A), + Inject(header="X-Cube-B", secret=SECRET_B, + format="prefix-${SECRET}"), + ], + ), + ), + ] + def pred(out: str) -> bool: + return ("STATUS=200" in out + and SECRET_A in out + and ("prefix-" + SECRET_B) in out) + runs: List[Run] = [ + ("GET /headers w/ two injects", + _http_probe("https://httpbingo.org/headers", capture_body=True), + pred), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_inject_strips_forged_sandbox_header() -> bool: + """Inject MUST overwrite (not append-to) a sandbox-supplied header. + + The sandbox sends ``Authorization: forged-by-sandbox``; the rule + injects its own Authorization header with a known secret. After + egress, httpbingo.org should echo the *injected* value, not the + forged one. This is the documented race-protection invariant in + access_phase.allow(): + + > we ALWAYS clear that header from the sandbox-provided + > request first, whether or not the gates pass + """ + SECRET = "cube-test-OVERWRITE-9e1a" + FORGED = "Bearer forged-by-sandbox" + rules = [ + Rule( + name="overwrite_auth", + match=Match(scheme="https", host="httpbingo.org"), + action=Action( + allow=True, + inject=[Inject(header="Authorization", secret=SECRET, + format="Bearer ${SECRET}")], + ), + ), + ] + def pred(out: str) -> bool: + return ("STATUS=200" in out + and SECRET in out + and "forged-by-sandbox" not in out) + code = _http_probe( + "https://httpbingo.org/headers", + capture_body=True, + extra_headers={"Authorization": FORGED}, + ) + runs: List[Run] = [ + ("GET /headers w/ forged Authorization", code, pred), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_inject_dropped_on_http_scheme_request_proceeds() -> bool: + """HTTP inject applies (CubeEgress no longer blocks inject on http). + + Historic behavior: G1 gate rejected inject on non-HTTPS traffic + ("https-only inject"). Current behavior (see + CubeEgress/lua/access_phase.lua:inject_gates comment on G1): operators + may legitimately need to inject credentials into plaintext traffic on + trusted networks (e.g. intra-cluster 80), so the http scheme is + permitted. This test guards that policy change by asserting the + injected header DOES appear in the upstream echo — the opposite of + the pre-relaxation expectation. + + If G1 is ever re-tightened to https-only, flip the predicate back to + ``SECRET not in out`` and rename this case. + """ + SECRET = "cube-test-http-inject-44ad" + rules = [ + Rule( + name="http_with_inject", + match=Match(scheme="http", host="httpbingo.org"), + action=Action( + allow=True, + inject=[Inject(header="X-Should-Appear", secret=SECRET)], + ), + ), + ] + def pred(out: str) -> bool: + return "STATUS=200" in out and SECRET in out + runs: List[Run] = [ + ("GET http://httpbingo.org/headers (http inject)", + _http_probe("http://httpbingo.org/headers", capture_body=True), + pred), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def case_audit_none_still_allows() -> bool: + """audit='none' is a valid AuditLevel and does not block traffic.""" + rules = [ + Rule(name="silent_audit", + match=Match(scheme="https", host="httpbingo.org"), + action=Action(allow=True, audit="none")), + ] + runs: List[Run] = [ + ("GET https://httpbingo.org/headers", + _http_probe("https://httpbingo.org/headers"), + _has_status(200)), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +def _longconn_https_probe(host: str, port: int, path: str = "/headers") -> str: + """Sandbox-side code: 10 HTTPS requests, 1 per minute, on ONE connection. + + Prints one ``REQ i STATUS=... SAMECONN=...`` line per request and a + final LONGCONN_ALL_OK / LONGCONN_FAILED marker. "Same connection" is + pinned via the socket's local address after every response — a + transparent reconnect changes the local port (or nulls the socket + after Connection: close) and surfaces as SAMECONN=False. + CubeEgress's downstream keepalive_timeout is 65s + (CubeEgress/nginx.conf:105), so the 60s period stays alive. A + User-Agent is always sent: httpbingo.org (fly.io) rejects UA-less + requests with an empty-body 402. + """ + return f"""\ +import http.client, time + +HOST, PORT, PATH = {host!r}, {port}, {path!r} +N, INTERVAL = 10, 60 +UA = {{"User-Agent": "cube-longconn-probe/1.0"}} + +conn = http.client.HTTPSConnection(HOST, PORT, timeout=30) + +# Warm-up on the SAME connection: the per-sandbox policy push to +# cube-egress is async, so ride out the first seconds of default-deny +# 403s before starting the official loop (connection stays keep-alive). +t_end = time.time() + 60 +while True: + try: + conn.request("GET", PATH, headers=UA) + r = conn.getresponse() + r.read() + if r.status != 403: + break + except Exception: + pass + if time.time() > t_end: + print("WARMUP_FAILED", flush=True) + raise SystemExit(1) + time.sleep(2) + +local = None +ok = True +for i in range(1, N + 1): + t0 = time.time() + try: + conn.request("GET", PATH, headers=UA) + resp = conn.getresponse() + resp.read() + status, will_close = resp.status, resp.will_close + except Exception as e: + print(f"REQ {{i}} EXC={{type(e).__name__}}:{{e}}", flush=True) + ok = False + break + cur = conn.sock.getsockname() if conn.sock else None + if local is None: + local = cur + same = (cur == local) and not will_close + print(f"REQ {{i}} STATUS={{status}} SAMECONN={{same}} LOCAL={{cur}}", flush=True) + ok = ok and (status == 200) and same + if i < N: + time.sleep(max(INTERVAL - (time.time() - t0), 1)) +print("LONGCONN_ALL_OK" if ok else "LONGCONN_FAILED", flush=True) +""" + + +def _longconn_pred(out: str) -> bool: + """Predicate for _longconn_https_probe: 10x200, same socket throughout.""" + return ("LONGCONN_ALL_OK" in out + and "SAMECONN=False" not in out + and out.count("STATUS=200") == 10) + + +def case_long_lived_https_connection_10x1min() -> bool: + """One keep-alive HTTPS connection, 1 request/min x 10, all must pass. + + Exercises the dataplane session-hit path over ~9 minutes: the + DNS-learned allow_out_v3 /48 entry for the host ages out mid-test + (resolver TTLs are typically <= 300s), while the established + connection's egress_sessions entry (no TTL of its own; ESTABLISHED + timeout ~3h) must keep steering packets to CubeEgress. If any + request fails — e.g. the dataplane re-classified the flow after + /48 aging and rejected it — the case fails. + + Deliberately runs WITHOUT the cube-egress restart twin-pass: + restarting the proxy would sever the very connection under test. + This case targets dataplane session longevity, not policy rebuild. + """ + rules = [ + Rule(name="longconn", + match=Match(scheme="https", host="httpbingo.org"), + action=Action(allow=True, audit="metadata")), + ] + runs: List[Run] = [ + ("https keep-alive 1 req/min x10 on one socket", + _longconn_https_probe("httpbingo.org", 443), + _longconn_pred), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _exec_runs(sb, runs, "longconn") + + +def case_long_lived_https_custom_port_10x1min() -> bool: + """Same longevity test against a NON-standard destination port (1012). + + tls-v1-2.badssl.com:1012 is a public HTTPS endpoint off 443; the + rule pins scheme=https + port=1012, so flow classification, the + DNS-learned (ip, port)/48 entry aging, and session steering all + exercise the custom-port dataplane. Unlike 80/443 flows, a lost + session here is NOT rescued by the legacy drain shim in do_tcp_nat + (custom ports are intentionally excluded), so this case is the + stronger proof that the session entry really persists for the + whole ~9 minutes. + """ + rules = [ + Rule(name="longconn_custom_port", + match=Match(scheme="https", host="tls-v1-2.badssl.com", port=1012), + action=Action(allow=True, audit="metadata")), + ] + runs: List[Run] = [ + ("https keep-alive 1 req/min x10 on custom port 1012", + _longconn_https_probe("tls-v1-2.badssl.com", 1012, "/"), + _longconn_pred), + ] + with Sandbox.create(network={"rules": rules}) as sb: + return _exec_runs(sb, runs, "longconn") + + +def case_secret_injection_via_header_for_moonshot() -> bool: + """Optionally verify credential injection against an authenticated API.""" + api_key = os.environ.get("MOONSHOT_API_KEY") + if not api_key: + print("SKIP: MOONSHOT_API_KEY is not set") + return True + + rules = [ + Rule( + name="moonshot", + match=Match(scheme="https", host="*.moonshot.cn", port=443), + action=Action( + allow=True, + audit="metadata", + inject=[Inject( + header="Authorization", + secret=api_key, + format="Bearer ${SECRET}", + )], + ), + ), + ] + DENIED_CIDRS = [ + "0.0.0.0/0", # block direct egress so traffic must use cube-egress tap + ] + python_code = """ +import json +import urllib.request + +url = "https://api.moonshot.cn/v1/chat/completions" +headers = {"Content-Type": "application/json"} + +payload = { + "messages": [ + {"content": "Hello! Who Are You？", "role": "user"} + ], + "model": "kimi-k2.5", + "stream": False +} + +data = json.dumps(payload).encode("utf-8") +req = urllib.request.Request(url, data=data, headers=headers, method="POST") + +try: + with urllib.request.urlopen(req, timeout=60) as response: + response_body = response.read().decode("utf-8") + print("Status code:", response.getcode()) + print("Response:", response_body) +except urllib.error.HTTPError as e: + print("HTTP Error:", e.code, e.reason) + print("Response body:", e.read().decode()) +""" + def pred(out: str) -> bool: + return "Status code: 200" in out and "completion_tokens" in out + runs: List[Run] = [ + ("POST /v1/chat/completions w/ inject", python_code, pred), + ] + with Sandbox.create(network={"deny_out": DENIED_CIDRS, "rules": rules}) as sb: + return _run_twice_with_restart(sb, runs) + + +# ── runner ─────────────────────────────────────────────────────────────────── + +CASES: List[Tuple[str, Callable[[], bool]]] = [ + #("allow rule on https host returns real upstream response", case_allow_specific_host), + #("allow * on https suffix host returns real upstream response", case_allow_suffix_host), + #("deny rule returns 403 without hitting upstream", case_deny_rule_returns_403), + #("first-match-wins picks the earlier allow over later deny", case_first_match_wins_order), + #("first-match-wins picks the earlier deny over later allow", case_first_match_wins_deny_before_allow), + #("non-empty policy default-denies hosts not covered by any rule", case_default_deny_when_no_rule_matches), + #("method filter allows GET and blocks POST on same host", case_method_filter_blocks_other_methods), + #("method any-of array allows multiple methods, denies others", case_method_any_of_array), + #("path exact match allows /headers and denies /anything", case_path_exact_match), + #("path '*' suffix wildcard allows whole subtree", case_path_prefix_wildcard), + #("scheme=http rule allows plain-HTTP traffic", case_scheme_http_allows_plain_http), + #("inject default format renders the bare secret", case_inject_default_format_renders_secret_only), + #("multiple inject entries in one rule all reach upstream", case_inject_multiple_headers_in_one_rule), + #("inject overwrites a sandbox-forged Authorization header", case_inject_strips_forged_sandbox_header), + #("inject applies on http scheme (G1 no longer https-only)", case_inject_dropped_on_http_scheme_request_proceeds), + #("audit='none' rule still allows traffic", case_audit_none_still_allows), + #("long-lived https connection serves 1 req/min x10 on one socket", case_long_lived_https_connection_10x1min), + ("long-lived https connection on custom port 1012 (1 req/min x10)", case_long_lived_https_custom_port_10x1min), + #("inject Authorization: Bearer reaches moonshot upstream", case_secret_injection_via_header_for_moonshot), +] + + +def main() -> int: + _check_env() + print(f"CUBE_API_URL = {os.environ.get('CUBE_API_URL')}") + print(f"CUBE_TEMPLATE_ID = {os.environ.get('CUBE_TEMPLATE_ID')}") + print(f"Restart container = {CUBE_EGRESS_CONTAINER}") + print() + + passed = failed = 0 + for label, fn in CASES: + print(f"== {label} ==") + try: + ok = fn() + except Exception: + print(" EXCEPTION:") + traceback.print_exc() + ok = False + if ok: + print(" RESULT: PASS\n") + passed += 1 + else: + print(" RESULT: FAIL\n") + failed += 1 + + print(f"summary: {passed} passed, {failed} failed (of {len(CASES)})") + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/code-sandbox-quickstart/network_l7_custom_port_echo.py b/examples/code-sandbox-quickstart/network_l7_custom_port_echo.py new file mode 100644 index 000000000..a56901e7e --- /dev/null +++ b/examples/code-sandbox-quickstart/network_l7_custom_port_echo.py @@ -0,0 +1,215 @@ +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 + +""" +network_l7_custom_port_echo.py — L7 rule demo across the four port quadrants. + +One sandbox, five rules, six probes: + + 1. default http — rule without port (scheme="http" only): intercepts :80 + 2. default https — rule without port (scheme="https" only): intercepts :443 + 3. default both — rule with NEITHER port NOR scheme: one rule intercepts + both :80 and :443 (the classic {80/http, 443/https} set) + 4. custom http — rule with port=18080, scheme="http" + 5. custom https — rule with port=1012, scheme="https" + +How it works: + Each L7 rule opens the L3 path for the (host, port) tuples it matches and + tells CubeEgress which traffic to intercept. The HTTP legs assert the + injected marker header inside the echoed response body. Leg 4's target is + a local echo server started by this script (self-contained); leg 1 uses + httpbingo.org because binding :80 on the host is impractical. Leg 3 + (postman-echo.com) proves a bare host-only rule fans out to the whole + default {80/http, 443/https} set — both probes go through the SAME rule. + Rule evaluation is first-match-wins, so each leg deliberately uses its own + host: same-host rules would shadow each other and blur attribution. + + The HTTPS legs (2, 3b, 5) need an upstream certificate the CubeEgress proxy + can verify — it runs ``proxy_ssl_verify on`` against the system CA bundle, + so a local self-signed echo would be rejected by design. They therefore use + public endpoints: httpbingo.org:443 and postman-echo.com:443 (echo → + marker asserted) plus tls-v1-2.badssl.com:1012 (non-standard port, no + header echo → 200 + non-empty body asserted). + +Prerequisites: + - CUBE_TEMPLATE_ID, with the template built so the sandbox trusts the + cluster interception CA (same requirement as cube-test-network.py). + - The cluster can reach httpbingo.org / tls-v1-2.badssl.com (override via + the env vars below if your environment needs different endpoints). + +Env: + CUBE_TEMPLATE_ID (required) template to create the sandbox from + EXAMPLE_L7_TARGET_HOST (optional) bridge IP for the local echo leg + EXAMPLE_L7_HTTP_PORT (optional) local echo port, default 18080 + EXAMPLE_L7_DEFAULT_HOST (optional) host for legs 1-2, default httpbingo.org + EXAMPLE_L7_BOTH_HOST (optional) host for leg 3, default postman-echo.com + (must differ from EXAMPLE_L7_DEFAULT_HOST — + first-match-wins would otherwise shadow rules) + EXAMPLE_L7_CUSTOM_HTTPS_URL (optional) URL for leg 4, default + https://tls-v1-2.badssl.com:1012/ + +Constraints (SDK- and server-enforced): + - port requires scheme; set both or neither (SDK raises ValueError). + scheme alone stays on the classic {80, 443} set and only qualifies + whether HTTP or HTTPS traffic matches (legs 1-2 use exactly that). + - host must be a domain or a single IP — subnet CIDRs are rejected. + - Scheme must be consistent per (host, port) across rules; the server + rejects a conflicting policy outright. + - At most 8 distinct (port, scheme) tuples per host. + - host/scheme matching is case-insensitive. +""" + +import json +import os +import subprocess +import sys +import threading +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + +from cubesandbox import Action, Inject, Match, Rule, Sandbox +from env_utils import load_local_dotenv + +load_local_dotenv() + +template_id = os.environ["CUBE_TEMPLATE_ID"] + +HTTP_PORT = int(os.environ.get("EXAMPLE_L7_HTTP_PORT", "18080")) +DEFAULT_HOST = os.environ.get("EXAMPLE_L7_DEFAULT_HOST", "httpbingo.org") +BOTH_HOST = os.environ.get("EXAMPLE_L7_BOTH_HOST", "postman-echo.com") +assert BOTH_HOST != DEFAULT_HOST, ( + "EXAMPLE_L7_BOTH_HOST must differ from EXAMPLE_L7_DEFAULT_HOST " + "(first-match-wins would shadow the rules)" +) +CUSTOM_HTTPS_URL = os.environ.get( + "EXAMPLE_L7_CUSTOM_HTTPS_URL", "https://tls-v1-2.badssl.com:1012/" +) +custom_https = urlparse(CUSTOM_HTTPS_URL) +assert custom_https.scheme == "https" and custom_https.hostname and custom_https.port, ( + f"EXAMPLE_L7_CUSTOM_HTTPS_URL must be an https URL with an explicit port: {CUSTOM_HTTPS_URL!r}" +) +assert custom_https.port != 443, "custom https leg requires a non-standard port" + +MARKER = f"cube-l7-echo-{uuid.uuid4().hex[:12]}" + + +class HeaderEchoHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + payload = json.dumps({"path": self.path, "headers": dict(self.headers)}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_args: object) -> None: + pass + + +def detect_target_host() -> str: + """Pick a host bridge IP reachable from sandboxes (never the node IP).""" + override = os.environ.get("EXAMPLE_L7_TARGET_HOST") + if override: + return override + try: + out = subprocess.check_output(["ip", "-4", "-o", "addr", "show", "up"], text=True) + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 4 and parts[1].startswith(("docker", "br-")): + return parts[3].split("/")[0] + except Exception: + pass + print( + "WARNING: no UP docker*/br-* bridge found; falling back to 172.17.0.1, " + "which is unreachable from sandboxes when docker0 is down. If the " + "custom-http leg fails, set EXAMPLE_L7_TARGET_HOST to the bridge IP of " + "the network your sandboxes attach to (see `ip -4 addr`).", + file=sys.stderr, + ) + return "172.17.0.1" # docker0 default; override via EXAMPLE_L7_TARGET_HOST if down + + +target_host = detect_target_host() + +server = ThreadingHTTPServer(("0.0.0.0", HTTP_PORT), HeaderEchoHandler) +thread = threading.Thread(target=server.serve_forever, daemon=True) +thread.start() +print(f"echo server on 0.0.0.0:{HTTP_PORT}; custom-http leg targets {target_host}:{HTTP_PORT}") + +inject = [Inject(header="X-Cube-L7-Demo", secret=MARKER)] +rules = [ + # 1+2. Default set via scheme-only rules (no port): :80 http / :443 https. + Rule( + name="default-http", + match=Match(host=DEFAULT_HOST, scheme="http"), + action=Action(allow=True, inject=inject), + ), + Rule( + name="default-https", + match=Match(host=DEFAULT_HOST, scheme="https"), + action=Action(allow=True, inject=inject), + ), + # 3. Neither port nor scheme: one rule fans out to {80/http, 443/https}. + Rule( + name="default-both", + match=Match(host=BOTH_HOST), + action=Action(allow=True, inject=inject), + ), + # 4. Custom HTTP port against the local echo server. + Rule( + name="custom-http", + match=Match(host=target_host, port=HTTP_PORT, scheme="http"), + action=Action(allow=True, inject=inject), + ), + # 5. Custom HTTPS port against a real publicly-signed upstream. + Rule( + name="custom-https", + match=Match(host=custom_https.hostname, port=custom_https.port, scheme="https"), + action=Action(allow=True, audit="metadata"), + ), +] + +results = [] + +try: + with Sandbox.create( + template=template_id, + timeout=120, + allow_internet_access=False, + network={"rules": rules}, + ) as sandbox: + + def probe_marker(label: str, url: str) -> None: + # httpbingo wraps every echoed header value in a single-element + # JSON array; a plain substring grep still matches the marker. + r = sandbox.commands.run(f"curl -sS --max-time 20 '{url}'", timeout=30) + out = r.stdout.strip() + ok = MARKER in out + snippet = out if len(out) <= 300 else out[:300] + "..." + print(f"{label}: {'OK' if ok else 'FAIL'} — {snippet or r.stderr.strip()}") + results.append(ok) + + def probe_status(label: str, url: str) -> None: + r = sandbox.commands.run( + f"curl -sS --max-time 20 '{url}' -o /dev/null -w 'code=%{{http_code}} len=%{{size_download}}'", + timeout=30, + ) + out = r.stdout.strip() + ok = "code=200" in out and "len=0" not in out + print(f"{label}: {'OK' if ok else 'FAIL'} — {out or r.stderr.strip()}") + results.append(ok) + + probe_marker(f"1 default http :{80:<5}", f"http://{DEFAULT_HOST}/headers") + probe_marker(f"2 default https :{443:<5}", f"https://{DEFAULT_HOST}/headers") + probe_marker(f"3a default both :{80:<5}", f"http://{BOTH_HOST}/headers") + probe_marker(f"3b default both :{443:<5}", f"https://{BOTH_HOST}/headers") + probe_marker(f"4 custom http :{HTTP_PORT:<5}", f"http://{target_host}:{HTTP_PORT}/headers") + probe_status(f"5 custom https :{custom_https.port:<5}", CUSTOM_HTTPS_URL) +finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + +passed = sum(results) +print(f"summary: {passed}/{len(results)} legs passed (marker={MARKER})") From 5b63864c191875080079f11b8704ec9179c40c36 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 10 Aug 2026 11:22:14 +0800 Subject: [PATCH 10/31] docs(cubenet): add L7 custom-port design docs and fix bpf object wildcards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final sweep of the L7_support_portconfig port — files caught by the full branch-diff audit: - CubeNet/cubevs/Makefile: widen BPF_GENS/BPF_OBJS wildcards to *_bpfel*.{go,o} so test-object artifacts (dnslearn/egresspolicy/ l7mark/tcpstate) are covered by verify/clean, matching .gitignore - CubeNet/cubevs/: port the three design documents (allow_out_v3_design.md, cubeegress_custom_port_design.md, cubeegress_custom_port_test_cases.md) - Cubelet/doc/cubelet-api.md: add the EgressRuleMatch.port row (protoc-gen-doc unavailable locally; row added manually to match the regenerated proto) Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- CubeNet/cubevs/Makefile | 4 +- CubeNet/cubevs/allow_out_v3_design.md | 14 + .../cubevs/cubeegress_custom_port_design.md | 379 ++++++++++++++++++ .../cubeegress_custom_port_test_cases.md | 158 ++++++++ Cubelet/doc/cubelet-api.md | 1 + 5 files changed, 554 insertions(+), 2 deletions(-) create mode 100644 CubeNet/cubevs/allow_out_v3_design.md create mode 100644 CubeNet/cubevs/cubeegress_custom_port_design.md create mode 100644 CubeNet/cubevs/cubeegress_custom_port_test_cases.md diff --git a/CubeNet/cubevs/Makefile b/CubeNet/cubevs/Makefile index 1f89d8c7e..b5706c6df 100644 --- a/CubeNet/cubevs/Makefile +++ b/CubeNet/cubevs/Makefile @@ -1,8 +1,8 @@ COVERAGE_PROFILE=.coverage COVERAGE_REPORT=coverage.html -BPF_GENS := $(wildcard *_bpfel.go) -BPF_OBJS := $(wildcard *_bpfel.o) +BPF_GENS := $(wildcard *_bpfel*.go) +BPF_OBJS := $(wildcard *_bpfel*.o) ifeq ($(V),1) Q = diff --git a/CubeNet/cubevs/allow_out_v3_design.md b/CubeNet/cubevs/allow_out_v3_design.md new file mode 100644 index 000000000..7e6aea6e4 --- /dev/null +++ b/CubeNet/cubevs/allow_out_v3_design.md @@ -0,0 +1,14 @@ +# allow_out_v3 设计（已被完整方案文档取代） + +本文档原描述 `allow_out_v2 → allow_out_v3` 的 eBPF 数据面迁移。它现在是「CubeEgress 自定义 +端口支持」完整方案的一部分。 + +**请改读 [`cubeegress_custom_port_design.md`](./cubeegress_custom_port_design.md)** —— 该文档覆盖端到端 +全栈（`CubeAPI` 配置模型 → `CubeMaster` 校验/聚合 → `network-agent` 传输 → +`CubeEgress`(Lua + iptables) → `CubeNet/cubevs` eBPF `allow_out_v3`），并整合了提交 +`9e41c62`（`CubeEgress rules support port configuration`）的全部改动。 + +本文件中 `allow_out_v3` 相关的 key/value 结构、`lpm_key_v3` 编码规则、数据面 +`classify_egress_flow`（由原 `l7_scheme_for_flow` 与 `session_policy_allowed` 合并而来，单次 +`(ip, port)/48` 查找）、用户态落表、兼容/平滑升级、字节序与 ABI 守卫等内容，均已并入 +上述完整方案文档的 §6 / §7 / §8，与此处保持一致。 diff --git a/CubeNet/cubevs/cubeegress_custom_port_design.md b/CubeNet/cubevs/cubeegress_custom_port_design.md new file mode 100644 index 000000000..a8971693a --- /dev/null +++ b/CubeNet/cubevs/cubeegress_custom_port_design.md @@ -0,0 +1,379 @@ +# CubeEgress 自定义端口支持 — 完整方案 + +本文档整理 CubeEgress 支持「L7 规则自定义目的端口（任意 `host:port` 上的 HTTP/HTTPS +拦截与注入/审计）」的**端到端完整方案**，覆盖从 API 配置模型、服务端校验、传输、到 +CubeEgress 运行时（Lua + iptables）与数据面 eBPF（`CubeNet/cubevs`，`allow_out_v3`）的 +全部层次。 + +涉及的关键提交： + +- `43fa0bf`（当前 `master` 基线）—— 分支已于 2026-07-29 rebase 到此；原分叉点 `68d9165` + 上磁盘仍是旧 `allow_out_v2` / `dns_allow` / `dns_query_track` 布局。 +- `ff3aea3` / `fc9acac`（`CubeEgress rules support port configuration`）—— 本特性的全栈落地， + 跨 `CubeAPI` / `CubeMaster` / `network-agent` / `CubeEgress`(Lua+iptables) / `Cubelet` / + `CubeNet/cubevs` 多个组件。 +- `2fdcbc6`（`test(cubenet): cover LPM fallback and expiry`）—— 数据面内核级 BPF 回归测试。 + +> **当前状态（`L7_support_portconfig` 已推送 origin，领先 `master` 41 个提交）** +> +> 本特性已经过一轮完整代码评审（5 路并行评审 + 数据面专项复核），发现的 **3 个 Blocker、 +> 8 个 Major 已全部修复或立项**，Minor 与待决项亦已处理： +> +> - **Blocker**：network-agent recover 测试 fixture（`4023404`）、CubeAPI 测试编译 `E0063` +> （`091a7bb`）、eBPF 普通域名 allow 未学成 `/32` 的功能回归（`c227ef2` + 数据面回归测试 +> `3e725b7`）。 +> - **Major**：M1 主机/CIDR 归一化端口丢失与 scheme 冲突绕过（`4cc7f77`）、M2 拒绝 subnet +> host（`a3c87de`）、M3 条目数校验按 `/48` 展开计数（`fcf2886`）、M4 迁移崩溃安全 +> （`c277758`）、M5 iptables grep 大小写幂等（`850e2cc`）、M7 e2e 断言收紧（`34a94e8`）、 +> M8 恢复 master 的 SNI+Host 双投（`e929081`）；M6「`(hostname)` 校验域 vs `(ip,port)` +> 数据面键」冲突解析规则见 §4.2.1 与 §10。 +> - **顺带修复**：`dns_learn_response_ip` 循环不展开的 verifier 脆弱性（`bb8d489`）、 +> `make test-lua` 接入 `port_scheme_extra_test.lua`（`a09edf2`）。 +> - **后续清理**：48B `net_policy_value_v2`（本需求开发中途被放弃的过渡格式）已由 `9244355` +> 还原为 16B 并删除其迁移/stale 死代码；M4 的 stale 备份崩溃安全（`c277758`）随之移除。 +> - **Minor / 待决项**：`lpmKeyV3` ABI 断言、`dump.go` 端口主机序、死代码清理（`15c093c`）； +> `recover()` 单沙箱 skip-and-log、`state_store.Save` 原子写（`f679d3d`）；SDK scheme 小写 +> 归一、proto 注释自指、CubeMaster 深拷贝 `Port`（`9d5d66e`）；迁移期 legacy L7 subnet 规则 +> 丢弃并告警（`e9e9463`）。 +> - **平滑升级（第二轮）**：`dns_allow` 真改名为 `dns_allow_v2`（BPF 层，与 `allow_out_v3` +> 同策略，`e1c8204`）；迁移测试补齐——inner-map 值变换（legacy/current，`a91f026`）、 +> bpffs outer-map（`193ed22`）、失败回滚保留 legacy pin（`4734748`）。 +> - **L7 mark 安装期可配置**：eBPF `const volatile` 全局 + `rewriteConstants` 注入 +> （`170c102`）、network-agent 读取 `/etc/cubeegress/l7-marks.conf`（`8e09380`）、 +> `install.sh` 由 env 生成该配置并校验（`d0d584a`）、部署文档（`a8304b9`）； +> 生成物 `network_agent.pb.go` 回同步（`1954aa6`）。数据面与 iptables 同源读取， +> 单侧改动即双侧一致。 +> - **现网验证（2026-07-28）**：one-click dist `7480d03`（构建自 rebase 前同名提交，rebase 后 +> 映射为 `a8304b9`）部署上 `test_l7_custom_port_e2e.py` 2/2 通过（自定义 HTTP 端口注入、 +> HTTPS 自定义端口拒绝 403、HTTPS 自定义端口放行 200）；同机 mark 覆盖 +> `0xA0000000/0xB0000000`（mask `0xF0000000`）在 eBPF 与 iptables 双侧同步生效 +> （TPROXY 计数器持续命中），安装期可配置链路一并得到现网验证。 +> - **示例与文档**：自包含示例 `network_l7_custom_port_echo.py`（5 规则 6 探针，现网 6/6， +> `d01f222`）取代占位符示例；文档同步（`709a8ac`、`3aeb4d2`）。 +> - **遗留连接排空**：`allow_out_v3` 条目老化后不再 RESET 已建立的默认端口（80/443）L7 +> 连接——非 SYN 包会话缺失时经 `bpf_skc_lookup_tcp` 确认代理 socket 仍 ESTABLISHED 则 +> 重新打标引流（`c72fc75`）；自定义端口连接有意排除，遵守当前策略。 +> - **评审（第三轮，2026-07-29）**：CLI `cloneCubeNetworkConfig` 丢弃 `Rules` / +> `AllowPublicTraffic`（分叉点遗留，master 已由 `2a18402` 修复为 `DeepCopy` 路线，本分支 +> 补回归测试 `cd2335c`）；recover reconcile 失败的沙箱被清理循环误删持久化状态（改为 +> 保留现场、下次重启重试、清理错误不再中止启动，`7780204`）；shell 侧 marks 校验改算术 +> 比较与 Go `resolveL7Marks` 三方对齐（`dcbed23`）。 +> - **Rebase（2026-07-29）**：分支重放到新 `master`（`43fa0bf`），clone 路线与 master 的 +> `DeepCopy` 对齐，并为 `types.EgressRule.DeepCopy` 补 `Match.Port` 深拷贝（master 该方法 +> 早于 port 字段存在，直接采用会丢弃端口绑定规则）。全部提交 hash 随之更新，本文引用 +> 已同步。 +> +> **已知遗留**：`TestTCPUpdateSession*` 此前在本环境内核（`6.6.69-cube`）被 BPF verifier +> 拒绝——`tcp_conntracks[dir][index][old_state]` 三维数组访问（边界已检查、访问安全， +> 系 verifier 无法跟踪多维变量索引，非真实越界），测试每次自行跳过。已由 `fbee460` +> 根治：`get_conntrack_index` 加 `__always_inline` 后 clang 内联并对常量 flag 输入做 +> 常量传播，折叠掉 verifier 需推理的一维，三个测试现已在该内核**真正加载并通过**， +> cubevs 套件本内核首次全绿。其余 CubeMaster 失败（`localcache` panic、缺 `conf.yaml`、 +> 依赖 registry/MySQL 的测试）均为预先存在/环境性，与本分支无关。 + +--- + +## 0. 设计原则 + +1. **改 eBPF map 定义则新建兼容版本**：`allow_out_v2` 不被原地改写，新增 `allow_out_v3`， + 迁移逻辑把旧数据无损搬入（见 §6 / §7）。 +2. **任意端口靠 `skb->mark` 选择，而非 `dport`**：iptables TPROXY 匹配的是 mvmtap 在 + sandbox tap 写入的 fwmark，因此用户把 L7 挂到 `tcp/3000` 之类任意端口时，无需教 + iptables 知道这个端口映射。 +3. **逐端口 `(host, port, scheme)` 语义**：`port` + `scheme` 共同钉死 CubeEgress 要拦截的 + `(host, port)` 元组，且同一 `(host, port)` 所有规则必须对该端口的 scheme 达成一致。 +4. **向后兼容默认 `{80/http, 443/https}`**：不配置 `port`/`scheme` 的规则保持经典行为， + 新旧规则可混用。 + +--- + +## 1. 配置模型（API schema） + +`EgressRuleMatch` 新增可选端口字段，跨多份定义保持 lockstep： + +- `CubeAPI/src/models/mod.rs`（`EgressRuleMatch.port: Option`） +- `CubeAPI/src/services/sandboxes.rs`（`map_egress_rule` 透传 `port`） +- `Cubelet/api/services/cubebox/v1/cubebox.proto`（`optional int32 port = 8`） +- `network-agent/api/v1/network_agent.proto` 与 `Cubelet/pkg/networkagentclient/pb/network_agent.proto` + （`optional int32 port = 8`，注释要求与 `CubeNet/src/cubevs.h` / Lua 保持同步） + +### 端口/协议语义（`CubeEgress/lua/port_scheme.lua` 的 `expand`） + +| 配置 | 展开结果 | +|------|----------| +| `port`/`scheme` 均 nil | 默认 `{80/http, 443/https}` | +| 仅 `scheme=http` | `{80/http}` | +| 仅 `scheme=https` | `{443/https}` | +| `port` + `scheme` 都设 | 精确 `(port, scheme)` 元组 | +| `port` 设但 `scheme` 缺 | **非法**（校验报错 `port requires scheme`） | + +约束（来自 `normalize_port` / `normalize_scheme`）： + +- `port ∈ [1, 65535]` 且为整数；`scheme ∈ {http, https}`（大小写/空白归一化）。 +- 同一 `(host, port)` 跨规则必须 scheme 一致（见 §2 服务端强校验）。 +- 每 host 最多 **8** 个 `(port, scheme)` 元组（`MAX_L7_PORTS_PER_HOST = 8`）。 + +--- + +## 2. 模型定义与透传（CubeMaster）+ 运行时强校验（CubeEgress） + +CubeMaster 这一层**只定义 `Port` 字段并原样透传**到 `cubebox.proto` / `network-agent`， +不在 Go 侧做端口/scheme 冲突校验；`types.go` 的文档注释描述的是**契约**—— +「同一 `(host, port)` 的所有规则必须就 scheme 达成一致，否则整条 policy 被拒」。 +该契约的**硬校验在 CubeEgress 加载策略时**执行（见 §4.2）。 + +- `CubeMaster/pkg/service/sandbox/types/types.go`：`EgressRuleMatch` 增加 `Port *int`，并文档化 + 上述语义（含上面的整策略原子性契约）。 +- `CubeMaster/pkg/templatecenter/template_request.go`：`cloneEgressRule` 透传 `Port` + （新增 `cloneIntPtr`）。 +- `CubeMaster/pkg/service/sandbox/util.go`：`mapEgressRuleMatch` 把 `Port` 映射到 `cubebox.proto` + 的 `EgressRuleMatch.Port`。 +- **聚合 key**：当两个 identity（`host` / `sni`）都提供时以 **Host** 为聚合 key，否则用 **SNI**； + 请求时两者都参与匹配（`CubeEgress/lua/policy.lua` 的 `validate_match_tuples`）。 +- **整策略原子性**：端口/scheme 冲突的校验在 CubeEgress 策略加载时触发（`validate_match_tuples` + 返回 `conflicts for host %q port %d: %s vs %s`），导致**整条 policy 校验失败**而非部分生效。 + +--- + +## 3. 传输（network-agent proto） + +`EgressRuleMatch.port` 经 `network-agent` 的两份 `.proto`（`network-agent/api/v1/... +` 与 `Cubelet/pkg/networkagentclient/pb/...`，内容一致）从 CubeMaster 透传到 Cubelet， +再由 Cubelet 的 network-agent client（`types.go` 增加对应字段）下发给 CubeEgress worker。 +proto 注释明确：**`port` 设了则 `scheme` 必设**，且必须与 `CubeNet/src/cubevs.h` 的 +`L7_SCHEME_*` 与 Lua `port_scheme` 保持 lockstep。 + +--- + +## 4. CubeEgress 运行时（Lua） + +### 4.1 `CubeEgress/lua/port_scheme.lua`（新增）—— 语义与匹配原语 + +- `normalize_scheme` / `normalize_port`：输入归一化与范围校验。 +- `expand(port, scheme)`：按 §1 表展开为有效 `(port, scheme)` 元组列表（校验报错透出到调用方）。 +- `matches(port, scheme, dst_port, request_scheme)`：请求时判定——先 `expand` 得元组集合，再比对 + 当前请求的 `dst_port`（由 TPROXY 保留的**原始目的端口**）与 `scheme`。 + +### 4.2 `CubeEgress/lua/policy.lua` —— 校验 + +- 新增 `MAX_L7_PORTS_PER_HOST = 8`（与 `cubevs.h` 一致）与 `normalize_identity`。 +- `validate_match_tuples`：调 `port_scheme.expand` → 以 identity 为 key 累加 `ports` 集合； + - 同 `port` 已有不同 `scheme` → 报错（`conflicts for host ... port ...`）； + - 单 host 超过 8 个元组 → 报错（`exceeds 8 L7 port tuples`）。 +- `validate_policy` 在遍历 `rules[]` 时对每条 `match` 调用上述校验，保证整策略原子性。 + +#### 4.2.1 冲突解析规则：`(hostname)` 校验域 vs `(ip, port)` 数据面键 + +`validate_match_tuples` 以**主机名字符串**为桶（`host` 缺省回退 `sni`），只能保证「同一主机名同一 +port 的 scheme 一致」。它**无法**发现「两个**不同**主机名解析到同一 `(ip, port)` 却用不同 scheme」的 +冲突——那需要在策略加载期做 DNS 解析，既脆弱又常离线不可行，故不在校验范围内。 + +而数据面 `allow_out_v3` 以 `(ip, port)/48` 为键，**每个 `(ip, port)` 只能存一个 scheme**。因此： + +- **解析规则**：对解析到同一 `(ip, port)` 的多个主机名，数据面按**最后一次写入生效**裁决——scheme 取 + 最近学成（`dns_learn_response_ip`）/ 落表（`populateAllowOutInnerMap`）的那条。`flags` 按位或仅在 + **精确同键**（`key_prefixlen` 相同）时发生；`expires_at_ns` 同键时**静态赢**：任一侧为静态（0）结果 + 即为 0（学习遇同键静态保持 0；静态落表遇同键学习项覆盖为 0——保留学习 TTL 会让 reaper 在旧 TTL + 到时后删表，静态裁决凭空消失），两侧皆学习则新 TTL 续期。此前 LPM 最长前缀查找把「同键」放大成 + 「任意覆盖项」，静态 CIDR 的零过期会传染给被覆盖的学习项（永不老化）——已由 `key_prefixlen` 精确 + 匹配 + 静态赢修正（回归测试见 §9）。同键合并的裁决结果仍**不确定**（取决于 DNS 响应到达顺序）。 +- **作者责任**：凡解析到同一 `(ip, port)` 的主机名，其 scheme **必须一致**。Lua 加载期无法代查， + 须策略作者保证。 +- **违反后果**：该 `(ip, port)` 的流量被引到「最后学成」scheme 对应的 listener（8080/8443）， + 另一主机名/scheme 的请求将在该 listener 上按 `(host, port, scheme)` 匹配**失败**。 + +### 4.3 `CubeEgress/lua/access_phase.lua` —— 请求匹配 + +- `rule_matches` 用 `port_scheme.matches(m.port, m.scheme, ctx.dst_port, ctx.scheme)` 取代旧的 + 仅 scheme 比较。`port`/`scheme` 缺省表示「向后兼容的默认端口集」，而**不是**对该 host 上 + 任意自定义端口的通配。 +- `build_ctx` 新增 `dst_port = tonumber(ngx.var.server_port)`：在 **IP_TRANSPARENT** listener 下， + nginx 的 `$server_port` 经 `getsockname()` 报告的是**原始目的端口**（TPROXY 保留），而非我们 + bind 的 8080/8443。这是 `match.port` 约束能在自定义端口上生效的前提。 +- `decide()` 把 `dst_port` 一并传入决策。 + +--- + +## 5. iptables / 策略路由（`CubeEgress/scripts/cube-proxy-iptables-init.sh`）—— 关键转变 + +### 旧模型（只能 80/443） + +按 `iif cube-dev` + `tcp dport 80/443` 选择，无 fwmark 参与。 + +### 新模型（任意端口） + +按 **`skb->mark`** 选择——mvmtap 在 sandbox tap 读出 `allow_out_v3` 的 +`(ip, port) → scheme` 映射（host→ip 由 DNS 学习层解析），写入 `CUBE_L7_MARK_HTTP=0xCE010000` / +`CUBE_L7_MARK_HTTPS=0xCE020000`（mask `0xFFFF0000`，三者均为**默认值**），本脚本据此把包 +引入 8080（HTTP listener）或 8443（HTTPS listener），**与原目的端口无关**。 + +> 三个 mark 值**安装期可配置**：`install.sh` 把 env（`CUBE_L7_MARK_HTTP/HTTPS/MASK`）写入 +> `/etc/cubeegress/l7-marks.conf`（校验 http≠https、取值不越出 mask，非法拒绝安装）； +> network-agent（注入 eBPF `const volatile` 全局）与本脚本（iptables/ip rule）**同源读取** +> 该文件，故两侧始终一致。下文规则示例均为默认值。 + +具体规则变化： + +```sh +# 旧 +iptables -t mangle -A TPROXY -i cube-dev（即 $INGRESS_IFACE） -p tcp --dport 80 -j TPROXY --on-port 8080 +iptables -t mangle -A TPROXY -i cube-dev（即 $INGRESS_IFACE） -p tcp --dport 443 -j TPROXY --on-port 8443 +# 新 +iptables -t mangle -A TPROXY -i cube-dev -p tcp \ + -m mark --mark 0xCE010000/0xFFFF0000 -j TPROXY --on-port 8080 +iptables -t mangle -A TPROXY -i cube-dev -p tcp \ + -m mark --mark 0xCE020000/0xFFFF0000 -j TPROXY --on-port 8443 +``` + +策略路由同样由「`iif` + `dport`」改为「`fwmark`」： + +```sh +ip rule add fwmark 0xCE010000/0xFFFF0000 table 100 # 取代旧的 iif cube-dev ipproto tcp dport 80 +ip rule add fwmark 0xCE020000/0xFFFF0000 table 100 +``` + +- **高位 16 bit 为 cube 自有**（`0xCE..`），低位 16 bit 留给用户其他 host 级 fwmark，由 mask 隔离。 +- 新增 `remove_legacy_dport_routing`：升级时清理旧 `iif cube-dev ipproto tcp dport 80/443` + ip rule（幂等），避免两套选择器并存。 +- 脚本幂等（`install_*` / `remove_*` / `show_status`），可安全重跑。 + +--- + +## 6. 数据面 eBPF（CubeNet/cubevs，`allow_out_v3`） + +> 本层细节见同目录 `allow_out_v3_design.md`，此处仅列与自定义端口直接相关的要点。 + +- **map 拓扑**：`allow_out_v3`（`BPF_MAP_TYPE_HASH_OF_MAPS`）→ inner `LPM_TRIE`。 + - key `lpm_key_v3`（12B）= `prefixlen`(4B) + `ip`(4B) + `port`(2B) + `_pad`(2B)； + `prefixlen=48` 即精确 `(ip, port)`，`32` 即仅 IP，`<32` 即子网。 + - value `net_policy_value_v3`（16B）= `ExpiresAtNS` + `Flags` + `Scheme` + `Reserved`； + scheme 在**插入时**解析落定，不再需要 8 元组数组。 +- **数据面解析**（`CubeNet/src/session.h` `classify_egress_flow`，由 `mvmtap.bpf.c` 的 + `do_tcp_nat` 在新建流时调用 / `udp.h`·`icmp.h` 的 `create_*_sessions` 在新建会话时调用）： + 统一裁决出向流，**一次**对 `(ip, dport)/48` 做 LPM 查找。LPM 的「最长前缀回退」会自动把 + 精确 `(ip, port)` 回退到 `/32` 或子网规则；命中有效 `L7_REQUIRED` 条目即得 `scheme` + （`FLOW_HTTP`/`FLOW_HTTPS`），其它命中（含回退到的普通 allow）一律 `FLOW_SNAT`； + 仅当该次查找未命中、或命中的 allow 已过期，才去查 `deny_out` 决定 `FLOW_REJECT`， + 否则默认 `FLOW_SNAT`。**只查一次 inner LPM trie** 即可覆盖「精确 L7 / 回退普通放行 / + default 放行」三态，无需为普通放行再查一次。 + + **注意**：旧实现 `l7_scheme_for_flow` 与 `session_policy_allowed` 已被合并。原 + `session_policy_allowed` 用硬编码 `/32` key，无法匹配 DNS 下发的 `(ip,port)/48` L7 条目， + 导致已授权的 L7 流被 `deny_out 0.0.0.0/0` 误杀；合并后 `/48` L7 查找优先于 deny， + 修复该回归。 +- **scheme → `skb->mark`**（`mvmtap.bpf.c` 的 `do_tcp_nat`）：`L7_SCHEME_HTTP → 0xCE010000`， + `L7_SCHEME_HTTPS → 0xCE020000`，与 §5 的 iptables 标记完全对应。 +- **用户态落表**（`netpolicy.go` `populateAllowOutInnerMap`）：L7 规则为每个 `(port, scheme)` 生成一条 + `/48` 条目（未显式给端口则按默认 `{80, 443}` 展开）；普通 allow 为单条 `/32`（`port=0`、`scheme=NONE`）。 +- **DNS 学习**：`dns_allow_v2` 的 `dns_allow_value.ports[]` 在域名学成时继承 `(port, scheme)`，响应处理 + 据此回填 `allow_out_v3`；`dns_query_track_value` 同样带 `ports[]` 以便免二次查找。 + +--- + +## 7. 兼容与平滑升级 + +### 7.1 `CubeNet/cubevs` 旧 pin 清理（本次特性在 `migration.go` 落地） + +- 迁移成功后通过 `removePinnedMap` 删除旧 pin：`allow_out_v2` / `allow_out`（优先 v2，回退 `allow_out`） + 与 `dns_allow`。 +- 安全性：inner LPM_TRIE 仅存在于外层 HashOfMaps 内（经 `outerMap.Put`，未单独 pin）， + 故 `os.Remove(pinPath(name))` 为单文件 unlink 即可，内核在所有 fd 关闭后自动回收 inner map。 +- 仅**迁移成功时**删除；失败则保留旧 pin 供下次重启重试，不会留下半迁移状态。 +- `dns_query_track` 不迁移：`Init` 无条件删除旧 pin 后由 `loadObject` 以新布局重新 pin（毫秒级在途 + 查询状态，重启可自愈）。 + +### 7.2 启动顺序（`cubevs.Init`，`miscs.go`） + +1. `persistentPolicyGenerationExists`（判定新 pin 是否已存在）。 +2. 删 `tungrp_to_tuns` 与 `dns_query_track` 旧 pin。 +3. `loadObject` ×3（localgw/mvmtap/nodenic）以新名字 pin 出空 `allow_out_v3`/`dns_allow_v2`/`dns_query_track`。 +4. `migratePersistentPolicyMaps`（灌入旧数据 + 清理旧 pin，见 7.1）。 +5. attach TC filter。 + +> 注：早期版本曾在第 1 步捕获「48B 旧布局 `allow_out_v3`」并在第 6 步 `replayStashedAllowOutV3` +> 回放（含崩溃安全备份）。该 48B `net_policy_value_v2` 是本需求**开发中途被放弃的方案**，真实部署 +> 不存在，已由提交 `9244355` 整体移除（`net_policy_value_v2` 还原为 16B，迁移只认 16B 旧格式， +> 48B inner map 直接判为不支持）。 + +### 7.3 iptables 侧 + +- `remove_legacy_dport_routing` 升级时清理旧的 `iif + dport 80/443` ip rule，与 §5 新 fwmark 选择器不冲突。 + +### 7.4 向后兼容 + +- 未配置 `port`/`scheme` 的规则保持默认 `{80/443}` 行为，新旧规则可混用。 + +--- + +## 8. 字节序与 ABI 守卫 + +- **网络字节序强制**：`lpm_key_v3` 的 `ip`/`port` 必须以网络字节序填充（与 `iphdr->daddr` / + `tcphdr->dest` 一致），否则精确 `(ip, port)` 匹配静默失败（`cubevs.h:170`）。 +- **双端静态断言**：`cubevs.h` 的 `_()` 校验 `lpm_key_v3==12`、`net_policy_value_v3==16`、 + `dns_allow_value==40` 等；`cubevs.go` 的 `_()` 在 Go 侧镜像校验同一组尺寸，任一侧漂移即编译失败。 +- **proto/lua lockstep**：`network_agent.proto`（server + client 两份）、`cubebox.proto`、`port_scheme.lua`、 + `cubevs.h` 的 `L7_SCHEME_*` 必须保持 `(port, scheme)` 语义同步。 +- **eBPF 重新生成**：`go generate ./...` 重编译 `bpfel.go`。 + +--- + +## 9. 测试与验证 + +| 层 | 新增/改动 | 说明 | +|----|-----------|------| +| Lua | `CubeEgress/lua/port_scheme.lua`（新）、`port_scheme_test.lua`（新） | `expand` / `matches` 单测 | +| iptables | `CubeEgress/tests/cube-proxy-iptables-init_test.sh`（新） | fwmark 选择器验证 | +| policy | `CubeEgress/lua/policy.lua` | `validate_match_tuples` 冲突/超限校验 | +| Go/eBPF | `CubeNet/cubevs/custom_port_test.go`、`CubeNet/cubevs/egress_policy_test.go` | v3 ABI 守卫、内核 BPF LPM 回退与过期回归 | +| Go/DNS 学成 | `CubeNet/cubevs/dns_learn_test.go`（+ `CubeNet/src/dns_learn_test.bpf.c`） | 普通 `/32`、L7 默认 `{80,443}`、L7 显式端口的数据面回填回归；同键合并三例（覆盖静态 CIDR 不传染、同键静态保留、同键学习 TTL 续期 + flags 并入 + scheme 覆盖，均经变异验证） | +| Go/迁移 | `CubeNet/cubevs/migration_test.go` | `dns_allow` inner-map 值变换（legacy/current）、bpffs outer-map 迁移、失败回滚保留 legacy pin；`TestSkipUnsupportedL7Subnet` | +| Go/L7 mark | `CubeNet/cubevs/l7_mark_test.go`（+ `CubeNet/src/l7_mark_test.bpf.c`）、`network-agent .../local_service_test.go::TestLoadL7MarksConfig` | `resolveL7Marks` 校验、配置注入数据面印记、`l7-marks.conf` 解析（缺省/覆盖/非法） | +| e2e(py) | `sdk/python/tests/test_l7_custom_port_e2e.py`（新）、`test_policy.py` | 自定义端口拦截/注入/审计端到端 | +| 示例 | `examples/code-sandbox-quickstart/network_l7_custom_port_echo.py`（新） | 单沙箱全规则矩阵示范（5 规则 / 6 探针）：scheme-only 默认 :80/:443、裸 host 规则展开默认集 `{80/http, 443/https}`、显式 `(port,scheme)` 自定义 :18080/http 与 :1012/https；本地 echo（自动探测 bridge IP）+ 公共回显端点断言注入 marker 到达；约束见 §10 | + +内核级 BPF 回归（提交 `2fdcbc6`，`CubeNet/cubevs/egress_policy_test.go`）： + +- `TestClassifyEgressFlowLPMFallback`：`/48` 查询回退到 `/32` IP 规则、`/24` 子网规则均得 + `FLOW_SNAT`；以 `deny_out 0.0.0.0/0` 兜底证明结果来自 LPM allow 回退而非默认放行。 +- `TestClassifyEgressFlowExpiredAllow`：过期 `/48` L7 allow（带 `deny_out`）→ `FLOW_REJECT`； + 过期回退 `/32` allow（带 `deny_out`）→ `FLOW_REJECT`；锁定「过期最长前缀命中不被 + 当作放行、但合并后直接进 deny」的当前语义。 +- `TestClassifyEgressFlowExactL7Match`：精确 `/48` L7 allow + （`192.0.2.20:8443` HTTPS）+ `deny_out 0.0.0.0/0` 兜底 → 同端口 `FLOW_HTTPS`、 + 异端口（443）回退 deny → `FLOW_REJECT`；直接锁定「单次 `/48` 查找 + LPM 不跨端口回退」语义。 + +端到端验证（`CubeNet/cubevs`）：`go build` / `go vet` / `go test` 全绿，`gofmt` 干净。 +`test_policy.py` 的用例与 `port_scheme_test.lua` 覆盖自定义端口拦截/注入/审计。 + +现网验证（2026-07-28，one-click dist `7480d03`，API `127.0.0.1:3000`）： + +- `CUBE_E2E=1 CUBE_L7_E2E_HTTP_TARGET_HOST=172.18.0.1 pytest tests/test_l7_custom_port_e2e.py` + → **2/2 通过**：自定义 HTTP 端口注入（本地 echo 目标收到注入头）、HTTPS 自定义端口拒绝 + （`:1012` → 403）、HTTPS 自定义端口放行到真实上游（`:1012` → 200，非空 body）。 +- 同机 mark 覆盖 `0xA0000000/0xB0000000`（mask `0xF0000000`）经 `/etc/cubeegress/l7-marks.conf` + 在 eBPF 印记与 iptables/ip rule 双侧同步生效（TPROXY 计数器持续命中）——单一来源、双消费方 + 一致性成立。 +- 同机运行示例 `network_l7_custom_port_echo.py` → **6/6 探针通过**：scheme-only 默认 :80/:443、 + 裸 host 规则（无 port/scheme）同规则覆盖默认集 :80+:443、自定义 `:18080`/http（本地 echo 收到 + 注入 marker）、自定义 `:1012`/https（200 + 非空 body）——全规则矩阵现网验证。 +- 运维提示：用 `example.com` 演示自定义端口会超时（其 8080/8443 握手后被静默丢弃，宿主机 + 直连同现象），并非 CubeEgress 故障；演示成功路径请用可响应的目标（如 e2e 的本地 echo）。 + +--- + +## 10. 注意事项 / 限制 + +1. **每 host 最多 8 个 `(port, scheme)` 元组**（`MAX_L7_PORTS_PER_HOST`），由 value 大小有界与 + 数据面可展开性决定；超限须在 Lua 校验期报错。 +2. **`port` 必须配 `scheme`**，且同一 `(host, port)` 跨规则 scheme 必须一致，否则**整条 policy 被服务端/运行时拒**。 +3. **自定义端口须显式声明** `port + scheme`；否则回退默认 `{80, 443}`，不会被识别为 L7 TPROXY。 +4. **TPROXY 依赖 IP_TRANSPARENT listener** 以取得原始目的端口（`$server_port` 经 `getsockname` 报原 dst）， + 这是 `match.port` 约束在自定义端口生效的前提。 +5. **`dns_query_track` 重启丢弃在途查询**（毫秒级、可自愈，设计预期）。 +6. **fwmark 高位 16 bit 默认 cube 自有**（`0xCE..`），低位留给用户其他 host 级 fwmark，由 + `CUBE_L7_MARK_MASK`（默认 `0xFFFF0000`）隔离。三个 mark 值**安装期可配置** + （env → `install.sh` → `/etc/cubeegress/l7-marks.conf`，数据面与 iptables 同源读取）； + 覆盖时须保证 http≠https 且取值不越出 mask，否则 `install.sh` 拒绝写入。 +7. **跨主机名的 `(ip, port)` scheme 冲突由作者保证**：数据面 `(ip, port)/48` 每元组仅一个 scheme；两个 + 解析到同一 `(ip, port)` 的主机名若 scheme 不同，按**最后学成生效**（不确定，见 §4.2.1）。Lua 校验 + 以主机名为域、无法在加载期发现该冲突（需 DNS），须策略作者保证这些主机名的 scheme 一致。 diff --git a/CubeNet/cubevs/cubeegress_custom_port_test_cases.md b/CubeNet/cubevs/cubeegress_custom_port_test_cases.md new file mode 100644 index 000000000..8bf867283 --- /dev/null +++ b/CubeNet/cubevs/cubeegress_custom_port_test_cases.md @@ -0,0 +1,158 @@ +# CubeEgress 自定义端口 — 测试用例规格 + +本文档列出「自定义端口（L7 规则 `port` + `scheme` 钉死 `(host, port, scheme)` 元组）」 +在各层的**测试场景**与**覆盖状态**。图例： + +- **现有** —— 已由代码库既有测试覆盖（无需新增）。 +- **新增** —— 本次生成的测试文件补齐的覆盖（见文末「新增文件」）。 + （`port_scheme_test.lua` 等特性落地时新增的文件，相对「评审修复轮」记为 **现有**；相对 `master` 它们同样是本分支新增，见设计文档 §9。） +- **现网验证** —— 已在真实集群部署上由 e2e 实跑验证（见文末「现网验证记录」）。 +- **缺口** —— 尚未自动化、建议后续补齐的场景。 + +> 状态基线说明（2026-07-28 更新）：`CubeNet`(Go) / `CubeEgress`(Lua+shell) / `sdk/python`(单测) +> 均已在本地实跑通过；两个 py e2e 文件需集群，其中 `test_l7_custom_port_e2e.py` 已在 +> one-click 部署（dist `7480d03`）上实跑 **2/2 通过**（见文末「现网验证记录」）， +> `test_l7_custom_port_validation_e2e.py` 负向用例仍需集群执行、未实跑。 + +--- + +## 1. API / SDK（`CubeAPI` + `cubesandbox` SDK） + +| 场景 | 期待 | 覆盖 | 位置 | +|------|------|------|------| +| 规则带 `port` + `scheme` | 正常建规则 | 现有 | `network_l7_custom_port_echo.py` 示例 | +| 规则带 `port` 但无 `scheme` | **客户端 `ValueError`** | **新增(e2e)** | `test_l7_custom_port_validation_e2e.py::test_l7_custom_port_requires_scheme` | +| `scheme` 大小写/空白归一（`" HTTPS "`→`"https"`） | 归一化生效 | **新增(lua)** | `port_scheme_extra_test.lua`（`expand(nil," HTTPS ")`） | +| 同 `(host, port)` 不同 `scheme` 跨规则 | 整条 policy 被拒 | **新增(e2e)** | `test_l7_custom_scheme_conflict_rejected` | + +## 2. CubeMaster（Go） + +| 场景 | 期待 | 覆盖 | 位置 | +|------|------|------|------| +| `EgressRuleMatch.Port` 字段透传（`types.go`/`template_request.go`/`util.go`） | 原样下发，无强校验 | 现有(单元测试隐含) | `types.go` doc-test 由 Go 编译守护 | +| 同 `(host, port)` scheme 冲突契约 | 契约文档化，硬校验在 CubeEgress | 现有 | `types.go` 注释（执行点在 §4） | + +## 3. network-agent 传输（proto） + +| 场景 | 期待 | 覆盖 | 位置 | +|------|------|------|------| +| `EgressRuleMatch.port` server/client 两份 `.proto` 一致 | lockstep | 现有 | `network_agent.proto` ×2 | +| `port` 设则 `scheme` 必设（契约） | 文档化 | 现有 | `.proto` 注释 | + +## 4. CubeEgress Lua 运行时 + +| 场景 | 期待 | 覆盖 | 位置 | +|------|------|------|------| +| `expand(nil,nil)`→`{80/http,443/https}` | 默认端口集 | **新增(lua)** | `port_scheme_extra_test.lua` | +| `expand(nil,"http")`→`{80/http}`；`(nil,"https")`→`{443/https}` | scheme-only 展开 | **新增(lua)** | `port_scheme_extra_test.lua` | +| `expand(8080,"http")`→`{8080/http}` | 自定义端口精确展开 | **新增(lua)** | `port_scheme_extra_test.lua` | +| `expand(8080,nil)` → 报错 `port requires scheme` | port 有 scheme 缺非法 | **新增(lua)** | `port_scheme_extra_test.lua` | +| `expand(nil,"ftp")` / `port=0` / `port=70000` → 报错 | 范围/取值校验 | **新增(lua)** | `port_scheme_extra_test.lua` | +| `matches` 默认 80/443、自定义端口拒绝、scheme 不匹配 | 请求时匹配 | 现有 | `port_scheme_test.lua` | +| `matches` scheme 大小写不敏感 | `HTTPS`==`https` | **新增(lua)** | `port_scheme_extra_test.lua` | +| `rule_matches` 同 host 规则不忽略自定义端口规则 | 精确 `(host,port)` 优先 | 现有 | `port_scheme_test.lua`（`access._rule_matches`） | +| `validate_match_tuples` 同 `(host,port)` scheme 冲突 → 整策略失败 | 冲突校验 | 现有 | `port_scheme_test.lua` | +| 同 host > 8 个 `(port,scheme)` 元组 → 失败 | 预置上限 | 现有 | `port_scheme_test.lua` | +| 同 host 同端口不同 scheme → 不允许 | per-(host,port) 一致性 | 现有 | `port_scheme_test.lua` | + +## 5. iptables / 策略路由（`cube-proxy-iptables-init.sh`） + +| 场景 | 期待 | 覆盖 | 位置 | +|------|------|------|------| +| `skb->mark` 命中 `CUBE_L7_MARK_HTTP` → TPROXY 到 8080 | HTTP listener | 现有 | `cube-proxy-iptables-init_test.sh` | +| `skb->mark` 命中 `CUBE_L7_MARK_HTTPS` → TPROXY 到 8443 | HTTPS listener | 现有 | `cube-proxy-iptables-init_test.sh` | +| 任意自定义端口（如 3000）按 mark 选路，不依赖 `--dport` | 端口无关 | 现网验证 | e2e `:18080`/`:1012` 经 mark 命中 TPROXY（2026-07-28）；iptables 级定向单测仍建议补 | +| `remove_legacy_dport_routing` 升级清理旧 dport 规则 | 幂等清理 | 现有 | `cube-proxy-iptables-init_test.sh` | + +## 6. eBPF 数据面（`CubeNet/cubevs`） + +| 场景 | 期待 | 覆盖 | 位置 | +|------|------|------|------| +| `expandDefaultPortSet()` = `{80/http, 443/https}`（网络字节序） | 默认端口集 | **新增(go)** | `custom_port_test.go::TestExpandDefaultPortSet` | +| `buildV3Entries` L7 显式端口 → 每端口一条 `/48` 条目 | 精确 `(ip,port)` | **新增(go)** | `custom_port_test.go::TestBuildV3EntriesL7ExplicitPorts` | +| `buildV3Entries` L7 无显式端口 → 默认 `{80,443}` 展开 | 默认展开 | **新增(go)** | `custom_port_test.go::TestBuildV3EntriesL7DefaultExpansion` | +| `buildV3Entries` 非 L7 → 单条 `/32`（`port=0`,`scheme=NONE`） | 普通 allow | **新增(go)** | `custom_port_test.go::TestBuildV3EntriesPlainAllow` | +| `buildV3Entries` 非 L7 子网 → `prefixlen<32` | 子网 | **新增(go)** | `custom_port_test.go::TestBuildV3EntriesSubnet` | +| `ExpiresAtNS` 透传到每条展开条目 | TTL 保留 | **新增(go)** | `custom_port_test.go::TestBuildV3EntriesExpiryCopied` | +| `lpm_key_v3.Port` 为**网络字节序** | 精确匹配不静默失败 | **新增(go)** | `custom_port_test.go::TestLpmKeyV3NetworkByteOrder` | +| `buildL7Plan` 默认规则→`PortCount=0`、显式端口→`PortCount=N` | 端口聚合正确 | 现有 | `netpolicy_test.go` | +| `buildL7Plan` scheme 冲突 / >8 元组 / 同 host 独立 | 校验 | 现有 | `netpolicy_test.go` | +| `netPolicyValueV3` / `netPolicyValueV2` / `dnsAllowValue` 布局静态断言 | ABI 守卫 | 现有 | `netpolicy_test.go` | +| `mvmtap` `classify_egress_flow` 单次 LPM 解析 `(ip,dport)/48`→scheme | 数据面解析 | **新增(go,内核bpf)** | `egress_policy_test.go::TestClassifyEgressFlowLPMFallback` / `TestClassifyEgressFlowExpiredAllow` / `TestClassifyEgressFlowExactL7Match`（提交 `2fdcbc6` / `fc9acac`） | +| `populateAllowOutInnerMap` 落表（`/48` 每端口 + 默认展开） | 用户态落表 | **新增(go)** | `netpolicy_test.go::TestPopulateAllowOutStaticV3OverCoveringLearnedStaysStatic` / `...OverExactLearnedStaticWins` / `...OverExactStaticMergesFlags` / `...PlainStaticOverwritesLearnedUnconditionally` | + +## 7. 端到端（`sdk/python/tests`） + +| 场景 | 期待 | 覆盖 | 位置 | +|------|------|------|------| +| 自定义 HTTP 端口注入 header（如 18080） | 请求到达目标并带注入头 | 现有 | `test_l7_custom_port_e2e.py::test_l7_custom_http_inject_and_https_deny` | +| 自定义 HTTPS 端口拒绝（如 :1012） | 返回 403 | 现有 | 同上 | +| 自定义 HTTPS 端口放行到真实上游（非 443） | 200 + 响应体 | 现有 | `test_l7_custom_https_allow_reaches_real_upstream` | +| `port` 有 `scheme` 缺 → 提交被拒 | `ValueError` | **新增(e2e)** | `test_l7_custom_port_validation_e2e.py::test_l7_custom_port_requires_scheme` | +| 同 `(host,port)` scheme 冲突 → 整策略被拒 | 提交失败 | **新增(e2e)** | `test_l7_custom_scheme_conflict_rejected` | +| > 8 个 `(port,scheme)` 元组 → 提交被拒 | 提交失败 | **新增(e2e)** | `test_l7_custom_port_budget_exceeded_rejected` | +| 同 host 混合 80/443 + 8080/http + 8443/https | 四元组各生效 | 现网验证 | `test_l7_custom_port_e2e.py` 覆盖自定义端口元组（2026-07-28 实跑通过） | + +> e2e 需集群 + `CUBE_E2E=1` / `--run-e2e` + `CUBE_L7_E2E_HTTP_TARGET_HOST` / `CUBE_TEMPLATE_ID`。 + +--- + +## 新增文件 + +| 层 | 文件 | 语言 | 验证 | +|----|------|------|------| +| eBPF 编码 | `CubeNet/cubevs/custom_port_test.go` | Go | `go test` 7 用例全绿，`gofmt` 干净 | +| eBPF 数据面内核回归 | `CubeNet/cubevs/egress_policy_test.go` | Go | `go test`（`TestClassifyEgressFlowLPMFallback` / `TestClassifyEgressFlowExpiredAllow`）全绿，提交 `2fdcbc6` | +| Lua 语义 | `CubeEgress/tests/port_scheme_extra_test.lua` | Lua | `lua tests/port_scheme_extra_test.lua` → PASS（与现有 `port_scheme_test.lua` 同跑通过） | +| Python 校验 | `sdk/python/tests/test_l7_custom_port_validation_e2e.py` | Python | `python3 -m py_compile` 通过，e2e 用例需集群 | +| eBPF DNS 学成数据面 | `CubeNet/cubevs/dns_learn_test.go`（+ `CubeNet/src/dns_learn_test.bpf.c`） | Go | `go test`（`TestDNSLearnPlainAllowWritesIPOnlyEntry` / `TestDNSLearnL7DefaultPortSet` / `TestDNSLearnL7ExplicitPorts` / `TestDNSLearnCoveringStaticCIDRDoesNotImmortalize` / `TestDNSLearnExactStaticEntrySurvivesRefresh` / `TestDNSLearnRefreshRenewsTTLMergesFlagsAndOverwritesScheme`）全绿，提交 `3e725b7`（同键合并用例经变异验证） | +| 迁移 | `CubeNet/cubevs/migration_test.go` | Go | `go test`（`TestSkipUnsupportedL7Subnet`）全绿，提交 `e9e9463` | + +> 评审修复轮另在既有测试文件内补充用例：`netpolicy_test.go`（M1 主机/CIDR 归一化合并与冲突、 +> M2 subnet 拒绝、M3 条目数展开计数、默认/显式端口展开）、`cube-proxy-iptables-init_test.sh` +> （M5 fwmark 大小写幂等）、`local_service_test.go`（M8 SNI+Host 双投、B1 recover fixture）、 +> `test_policy.py`（SDK scheme 小写归一）、`test_l7_custom_port_validation_e2e.py`（M7 断言收紧 +> 为 `ApiError, match=`）。 + +现有覆盖见：`CubeNet/cubevs/netpolicy_test.go`、`CubeEgress/tests/port_scheme_test.lua`、 +`CubeEgress/tests/cube-proxy-iptables-init_test.sh`、 +`sdk/python/tests/test_l7_custom_port_e2e.py`、 +`examples/code-sandbox-quickstart/network_l7_custom_port_echo.py`。 + +## 缺口小结（建议后续） + +1. iptables **任意自定义端口按 mark 选路**的 **iptables 级定向单测**（不依赖 `--dport`）； + 端到端已由 e2e 现网验证（2026-07-28，见文末）。 +2. `populateAllowOutInnerMap` 落表已有真实-map 单测覆盖（`netpolicy_test.go`：覆盖学习项不传染 + 静态 /48、同键学习遇静态落表静态赢、同键静态 flags 并入、plain 无条件覆盖）；`classify_egress_flow` + 内核 BPF 单测已覆盖（`egress_policy_test.go`，`2fdcbc6`/`fc9acac`）。 +3. 同 host **混合 80/443 + 自定义端口** 的四元组 e2e 自动化（自定义端口元组已由 + `test_l7_custom_port_e2e.py` 现网验证，2026-07-28，建议补自动化）。 +4. DNS 学成回填的**数据面**已有覆盖（`dns_learn_test.go`：普通 `/32`、L7 默认 `{80,443}`、 + L7 显式端口，`3e725b7`）；但 `dns_query_track` 查询跟踪与 `dns_allow_v2` 在自定义端口下的 + **端到端**（真实 DNS 报文驱动）回填路径仍未自动化。 + +> 评审修复轮新增并已通过的回归：`buildL7Plan` 主机/CIDR 归一化合并与冲突（M1）、subnet host +> 拒绝（M2）、条目数展开计数（M3）、迁移崩溃安全（M4，stale 备份路径；后经 `9244355` 随 48B +> 格式一并移除）、iptables fwmark 大小写幂等（M5）、迁移 legacy L7 subnet 丢弃 +> （`TestSkipUnsupportedL7Subnet`）、SNI+Host 双投（M8）、SDK scheme 归一与 e2e 断言收紧（M7）。 +> 遗留环境性问题见设计文档「当前状态」。 + +--- + +## 现网验证记录（2026-07-28） + +- 环境：one-click dist `7480d03`（构建自 rebase 前同名提交），API `127.0.0.1:3000`。 +- `CUBE_E2E=1 CUBE_L7_E2E_HTTP_TARGET_HOST=172.18.0.1 python -m pytest + tests/test_l7_custom_port_e2e.py -v` → **2/2 通过**：自定义 HTTP 端口注入（本地 echo 目标 + 收到注入头）、HTTPS 自定义端口拒绝（`:1012` → 403）、HTTPS 自定义端口放行到真实上游 + （`:1012` → 200，非空 body）。 +- 同机验证 **mark 安装期可配置**：覆盖 `CUBE_L7_MARK_HTTP=0xA0000000` / + `CUBE_L7_MARK_HTTPS=0xB0000000`（mask `0xF0000000`）后，eBPF 印记与 iptables/ip rule + 同步切换，TPROXY 计数器持续命中——`/etc/cubeegress/l7-marks.conf` 单一来源、双消费方 + 一致性成立。 +- 同机运行示例 `network_l7_custom_port_echo.py` → **6/6 探针通过**：scheme-only 默认 :80/:443、 + 裸 host 规则（无 port/scheme）同规则覆盖默认集 :80+:443、自定义 `:18080`/http（本地 echo + 收到注入 marker）、自定义 `:1012`/https（200 + 非空 body）。 +- 运维提示：用 `example.com` 演示自定义端口会超时（其 8080/8443 在握手后被静默丢弃，宿主机 + 直连同现象），并非 CubeEgress 故障；演示成功路径请用可响应的目标（如 e2e 的本地 echo）。 diff --git a/Cubelet/doc/cubelet-api.md b/Cubelet/doc/cubelet-api.md index 90e606138..35c7cbd7a 100644 --- a/Cubelet/doc/cubelet-api.md +++ b/Cubelet/doc/cubelet-api.md @@ -860,6 +860,7 @@ Device specifies a host device to mount into a container. | method | [string](#string) | repeated | | | path | [string](#string) | optional | | | scheme | [string](#string) | optional | | +| port | [int32](#int32) | optional | L7 destination port. When set, `scheme` MUST also be set — together they pin the (host, port, scheme) tuple CubeEgress intercepts via skb->mark + iptables TPROXY. Both omitted keeps the legacy default {80/http, 443/https} behavior. | From 458cf1aa89c690a749d0db8466a8bdb4baadb123 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 10 Aug 2026 15:18:38 +0800 Subject: [PATCH 11/31] docs(cubenet): fix stale map-name and function-name comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-review comment fixes from the port consistency audit: - tap.go: UpsertTAPDeviceMetadata comment referenced the old map names (allow_out_v2, dns_allow) — the actual maps are allow_out_v3 and dns_allow_v2 - netpolicy.go: cleanupNetPolicy comment still named the pre-rename DelTAPDevice; the function is DeleteTAPDevice after the #1285 reconciliation No code changes; comment-only. Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- CubeNet/cubevs/netpolicy.go | 2 +- CubeNet/cubevs/tap.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CubeNet/cubevs/netpolicy.go b/CubeNet/cubevs/netpolicy.go index 29a56ad7e..6b1fc50f1 100644 --- a/CubeNet/cubevs/netpolicy.go +++ b/CubeNet/cubevs/netpolicy.go @@ -200,7 +200,7 @@ func lookupInnerMap(outerMap *ebpf.Map, ifindex uint32) (*ebpf.Map, error) { // cleanupNetPolicy flushes all entries in the inner LPM trie maps // for the given ifindex in both allow_out_v3 and deny_out. -// This should be called during DelTAPDevice. +// This should be called during DeleteTAPDevice. func cleanupNetPolicy(ifindex uint32) error { allowOut, err := loadPinnedMap(MapNameAllowOutV3) if err != nil { diff --git a/CubeNet/cubevs/tap.go b/CubeNet/cubevs/tap.go index c8495b10f..afca5d27b 100644 --- a/CubeNet/cubevs/tap.go +++ b/CubeNet/cubevs/tap.go @@ -72,7 +72,7 @@ func AddTAPDevice(ifindex uint32, ip net.IP, id string, version uint32, opts MVM // UpsertTAPDeviceMetadata registers or refreshes TAP metadata without touching // per-sandbox policy maps. Recovery paths use this to repair metadata while -// preserving allow_out_v2, deny_out and dns_allow contents. +// preserving allow_out_v3, deny_out and dns_allow_v2 contents. func UpsertTAPDeviceMetadata(ifindex uint32, ip net.IP, id string, version uint32) error { if len(id) > maxIDLength { return ErrTooLong From e1f86aab86a71ef67f2496374b876ac7f557da70 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Tue, 11 Aug 2026 12:50:05 +0800 Subject: [PATCH 12/31] fix(examples): probe cube-egress admin on :9091 after #1285 port move MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The restart readiness probe in cube-test-network.py still targeted the pre-#1285 admin port :9090, which is now served by CubeProxy plaintext gRPC on all-in-one hosts — the HTTP probe received gRPC frames and the readiness check failed with BadStatusLine until the 60s timeout, failing every restart-style case. Point the probe at :9091 (CUBE_EGRESS_ADMIN_PORT default in CubeEgress/start.sh + nginx.conf) and update the docstring references (network-agent :19090 dump is now Cubelet /v1/policies/dump). Verified live: full suite rerun has the moonshot inject case passing with restart probe ready in ~1.2s. Assisted-by: CodeBuddy:Kimi-K3 Signed-off-by: hankzhou --- .../code-sandbox-quickstart/cube-test-network.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/code-sandbox-quickstart/cube-test-network.py b/examples/code-sandbox-quickstart/cube-test-network.py index 55507f0ee..5cd2618bb 100644 --- a/examples/code-sandbox-quickstart/cube-test-network.py +++ b/examples/code-sandbox-quickstart/cube-test-network.py @@ -98,10 +98,12 @@ # ── helpers ────────────────────────────────────────────────────────────────── # Container + admin-port wiring. cube-egress's admin loopback listens on -# 127.0.0.1:9090 (declared in CubeEgress/nginx.conf). The container is named -# ``cube-egress`` by the systemd unit ``cube-sandbox-cube-egress.service``. +# 127.0.0.1:9091 (declared in CubeEgress/nginx.conf; moved off :9090 by #1285 +# because CubeProxy's plaintext gRPC serves :9090 on all-in-one hosts). The +# container is named ``cube-egress`` by the systemd unit +# ``cube-sandbox-cube-egress.service``. CUBE_EGRESS_CONTAINER = "cube-egress" -CUBE_EGRESS_HEALTH_URL = "http://127.0.0.1:9090/admin/v1/health" +CUBE_EGRESS_HEALTH_URL = "http://127.0.0.1:9091/admin/v1/health" RESTART_BOOTSTRAP_TIMEOUT = 60 # seconds @@ -118,13 +120,13 @@ def _restart_cube_egress() -> None: """``docker restart cube-egress`` and wait until it is serving again. "Serving" is observed via the admin /health endpoint: - - it must answer at all (worker accepting on :9090) + - it must answer at all (worker accepting on :9091) - bootstrap_status must reach "ready" (worker has fetched the - policy dump from network-agent's :19090/v1/policies/dump) + policy dump from Cubelet's /v1/policies/dump) We deliberately do NOT re-install the test's policy ourselves — the whole point of the second pass is to verify cube-egress - rebuilt its in-memory policy_store from network-agent on cold + rebuilt its in-memory policy_store from Cubelet on cold start. If bootstrap landed on "skipped" (because CUBE_EGRESS_BOOTSTRAP_URL wasn't set on the container), the second pass would see policy_count=0 and most cases would 403, From a3fa691be5b034b617bdbcec769c722b569c162f Mon Sep 17 00:00:00 2001 From: hankzhou Date: Fri, 14 Aug 2026 16:44:01 +0800 Subject: [PATCH 13/31] test(sdk_compat): add L7 custom-port e2e coverage and integration fixes - Add tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py covering custom HTTP/HTTPS port interception, scheme-only default-set, and port/scheme + subnet-host create-time validation; add NETWORK_L7_CUSTOM_PORT capability. - cubevs Makefile: regenerate BPF objects before make test (test: gen). - docs: document custom L7 ports in the security-proxy guide (en/zh). - Fold in dataplane policy/runtime touch-ups and drop superseded design docs. Signed-off-by: hankzhou --- CubeNet/cubevs/Makefile | 2 +- CubeNet/cubevs/allow_out_v3_design.md | 14 - .../cubevs/cubeegress_custom_port_design.md | 379 ------- .../cubeegress_custom_port_test_cases.md | 158 --- CubeNet/cubevs/netpolicy.go | 11 +- CubeNet/src/session.h | 13 +- docs/guide/security-proxy.md | 86 +- docs/zh/guide/security-proxy.md | 80 +- .../cube-test-network.py | 1000 ----------------- .../cases/network/test_l7_custom_port.py | 434 +++++++ .../e2e/sdk_compat/framework/capabilities.py | 2 + 11 files changed, 599 insertions(+), 1580 deletions(-) delete mode 100644 CubeNet/cubevs/allow_out_v3_design.md delete mode 100644 CubeNet/cubevs/cubeegress_custom_port_design.md delete mode 100644 CubeNet/cubevs/cubeegress_custom_port_test_cases.md delete mode 100644 examples/code-sandbox-quickstart/cube-test-network.py create mode 100644 tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py diff --git a/CubeNet/cubevs/Makefile b/CubeNet/cubevs/Makefile index b5706c6df..78ad1acf7 100644 --- a/CubeNet/cubevs/Makefile +++ b/CubeNet/cubevs/Makefile @@ -38,7 +38,7 @@ lint: $(Q)golangci-lint run --fix .PHONY: test -test: +test: gen $(call msg,RUN TESTS) $(Q)go clean -testcache $(Q)go test $(VERBOSE) -cover -coverprofile=$(COVERAGE_PROFILE) ./... diff --git a/CubeNet/cubevs/allow_out_v3_design.md b/CubeNet/cubevs/allow_out_v3_design.md deleted file mode 100644 index 7e6aea6e4..000000000 --- a/CubeNet/cubevs/allow_out_v3_design.md +++ /dev/null @@ -1,14 +0,0 @@ -# allow_out_v3 设计（已被完整方案文档取代） - -本文档原描述 `allow_out_v2 → allow_out_v3` 的 eBPF 数据面迁移。它现在是「CubeEgress 自定义 -端口支持」完整方案的一部分。 - -**请改读 [`cubeegress_custom_port_design.md`](./cubeegress_custom_port_design.md)** —— 该文档覆盖端到端 -全栈（`CubeAPI` 配置模型 → `CubeMaster` 校验/聚合 → `network-agent` 传输 → -`CubeEgress`(Lua + iptables) → `CubeNet/cubevs` eBPF `allow_out_v3`），并整合了提交 -`9e41c62`（`CubeEgress rules support port configuration`）的全部改动。 - -本文件中 `allow_out_v3` 相关的 key/value 结构、`lpm_key_v3` 编码规则、数据面 -`classify_egress_flow`（由原 `l7_scheme_for_flow` 与 `session_policy_allowed` 合并而来，单次 -`(ip, port)/48` 查找）、用户态落表、兼容/平滑升级、字节序与 ABI 守卫等内容，均已并入 -上述完整方案文档的 §6 / §7 / §8，与此处保持一致。 diff --git a/CubeNet/cubevs/cubeegress_custom_port_design.md b/CubeNet/cubevs/cubeegress_custom_port_design.md deleted file mode 100644 index a8971693a..000000000 --- a/CubeNet/cubevs/cubeegress_custom_port_design.md +++ /dev/null @@ -1,379 +0,0 @@ -# CubeEgress 自定义端口支持 — 完整方案 - -本文档整理 CubeEgress 支持「L7 规则自定义目的端口（任意 `host:port` 上的 HTTP/HTTPS -拦截与注入/审计）」的**端到端完整方案**，覆盖从 API 配置模型、服务端校验、传输、到 -CubeEgress 运行时（Lua + iptables）与数据面 eBPF（`CubeNet/cubevs`，`allow_out_v3`）的 -全部层次。 - -涉及的关键提交： - -- `43fa0bf`（当前 `master` 基线）—— 分支已于 2026-07-29 rebase 到此；原分叉点 `68d9165` - 上磁盘仍是旧 `allow_out_v2` / `dns_allow` / `dns_query_track` 布局。 -- `ff3aea3` / `fc9acac`（`CubeEgress rules support port configuration`）—— 本特性的全栈落地， - 跨 `CubeAPI` / `CubeMaster` / `network-agent` / `CubeEgress`(Lua+iptables) / `Cubelet` / - `CubeNet/cubevs` 多个组件。 -- `2fdcbc6`（`test(cubenet): cover LPM fallback and expiry`）—— 数据面内核级 BPF 回归测试。 - -> **当前状态（`L7_support_portconfig` 已推送 origin，领先 `master` 41 个提交）** -> -> 本特性已经过一轮完整代码评审（5 路并行评审 + 数据面专项复核），发现的 **3 个 Blocker、 -> 8 个 Major 已全部修复或立项**，Minor 与待决项亦已处理： -> -> - **Blocker**：network-agent recover 测试 fixture（`4023404`）、CubeAPI 测试编译 `E0063` -> （`091a7bb`）、eBPF 普通域名 allow 未学成 `/32` 的功能回归（`c227ef2` + 数据面回归测试 -> `3e725b7`）。 -> - **Major**：M1 主机/CIDR 归一化端口丢失与 scheme 冲突绕过（`4cc7f77`）、M2 拒绝 subnet -> host（`a3c87de`）、M3 条目数校验按 `/48` 展开计数（`fcf2886`）、M4 迁移崩溃安全 -> （`c277758`）、M5 iptables grep 大小写幂等（`850e2cc`）、M7 e2e 断言收紧（`34a94e8`）、 -> M8 恢复 master 的 SNI+Host 双投（`e929081`）；M6「`(hostname)` 校验域 vs `(ip,port)` -> 数据面键」冲突解析规则见 §4.2.1 与 §10。 -> - **顺带修复**：`dns_learn_response_ip` 循环不展开的 verifier 脆弱性（`bb8d489`）、 -> `make test-lua` 接入 `port_scheme_extra_test.lua`（`a09edf2`）。 -> - **后续清理**：48B `net_policy_value_v2`（本需求开发中途被放弃的过渡格式）已由 `9244355` -> 还原为 16B 并删除其迁移/stale 死代码；M4 的 stale 备份崩溃安全（`c277758`）随之移除。 -> - **Minor / 待决项**：`lpmKeyV3` ABI 断言、`dump.go` 端口主机序、死代码清理（`15c093c`）； -> `recover()` 单沙箱 skip-and-log、`state_store.Save` 原子写（`f679d3d`）；SDK scheme 小写 -> 归一、proto 注释自指、CubeMaster 深拷贝 `Port`（`9d5d66e`）；迁移期 legacy L7 subnet 规则 -> 丢弃并告警（`e9e9463`）。 -> - **平滑升级（第二轮）**：`dns_allow` 真改名为 `dns_allow_v2`（BPF 层，与 `allow_out_v3` -> 同策略，`e1c8204`）；迁移测试补齐——inner-map 值变换（legacy/current，`a91f026`）、 -> bpffs outer-map（`193ed22`）、失败回滚保留 legacy pin（`4734748`）。 -> - **L7 mark 安装期可配置**：eBPF `const volatile` 全局 + `rewriteConstants` 注入 -> （`170c102`）、network-agent 读取 `/etc/cubeegress/l7-marks.conf`（`8e09380`）、 -> `install.sh` 由 env 生成该配置并校验（`d0d584a`）、部署文档（`a8304b9`）； -> 生成物 `network_agent.pb.go` 回同步（`1954aa6`）。数据面与 iptables 同源读取， -> 单侧改动即双侧一致。 -> - **现网验证（2026-07-28）**：one-click dist `7480d03`（构建自 rebase 前同名提交，rebase 后 -> 映射为 `a8304b9`）部署上 `test_l7_custom_port_e2e.py` 2/2 通过（自定义 HTTP 端口注入、 -> HTTPS 自定义端口拒绝 403、HTTPS 自定义端口放行 200）；同机 mark 覆盖 -> `0xA0000000/0xB0000000`（mask `0xF0000000`）在 eBPF 与 iptables 双侧同步生效 -> （TPROXY 计数器持续命中），安装期可配置链路一并得到现网验证。 -> - **示例与文档**：自包含示例 `network_l7_custom_port_echo.py`（5 规则 6 探针，现网 6/6， -> `d01f222`）取代占位符示例；文档同步（`709a8ac`、`3aeb4d2`）。 -> - **遗留连接排空**：`allow_out_v3` 条目老化后不再 RESET 已建立的默认端口（80/443）L7 -> 连接——非 SYN 包会话缺失时经 `bpf_skc_lookup_tcp` 确认代理 socket 仍 ESTABLISHED 则 -> 重新打标引流（`c72fc75`）；自定义端口连接有意排除，遵守当前策略。 -> - **评审（第三轮，2026-07-29）**：CLI `cloneCubeNetworkConfig` 丢弃 `Rules` / -> `AllowPublicTraffic`（分叉点遗留，master 已由 `2a18402` 修复为 `DeepCopy` 路线，本分支 -> 补回归测试 `cd2335c`）；recover reconcile 失败的沙箱被清理循环误删持久化状态（改为 -> 保留现场、下次重启重试、清理错误不再中止启动，`7780204`）；shell 侧 marks 校验改算术 -> 比较与 Go `resolveL7Marks` 三方对齐（`dcbed23`）。 -> - **Rebase（2026-07-29）**：分支重放到新 `master`（`43fa0bf`），clone 路线与 master 的 -> `DeepCopy` 对齐，并为 `types.EgressRule.DeepCopy` 补 `Match.Port` 深拷贝（master 该方法 -> 早于 port 字段存在，直接采用会丢弃端口绑定规则）。全部提交 hash 随之更新，本文引用 -> 已同步。 -> -> **已知遗留**：`TestTCPUpdateSession*` 此前在本环境内核（`6.6.69-cube`）被 BPF verifier -> 拒绝——`tcp_conntracks[dir][index][old_state]` 三维数组访问（边界已检查、访问安全， -> 系 verifier 无法跟踪多维变量索引，非真实越界），测试每次自行跳过。已由 `fbee460` -> 根治：`get_conntrack_index` 加 `__always_inline` 后 clang 内联并对常量 flag 输入做 -> 常量传播，折叠掉 verifier 需推理的一维，三个测试现已在该内核**真正加载并通过**， -> cubevs 套件本内核首次全绿。其余 CubeMaster 失败（`localcache` panic、缺 `conf.yaml`、 -> 依赖 registry/MySQL 的测试）均为预先存在/环境性，与本分支无关。 - ---- - -## 0. 设计原则 - -1. **改 eBPF map 定义则新建兼容版本**：`allow_out_v2` 不被原地改写，新增 `allow_out_v3`， - 迁移逻辑把旧数据无损搬入（见 §6 / §7）。 -2. **任意端口靠 `skb->mark` 选择，而非 `dport`**：iptables TPROXY 匹配的是 mvmtap 在 - sandbox tap 写入的 fwmark，因此用户把 L7 挂到 `tcp/3000` 之类任意端口时，无需教 - iptables 知道这个端口映射。 -3. **逐端口 `(host, port, scheme)` 语义**：`port` + `scheme` 共同钉死 CubeEgress 要拦截的 - `(host, port)` 元组，且同一 `(host, port)` 所有规则必须对该端口的 scheme 达成一致。 -4. **向后兼容默认 `{80/http, 443/https}`**：不配置 `port`/`scheme` 的规则保持经典行为， - 新旧规则可混用。 - ---- - -## 1. 配置模型（API schema） - -`EgressRuleMatch` 新增可选端口字段，跨多份定义保持 lockstep： - -- `CubeAPI/src/models/mod.rs`（`EgressRuleMatch.port: Option`） -- `CubeAPI/src/services/sandboxes.rs`（`map_egress_rule` 透传 `port`） -- `Cubelet/api/services/cubebox/v1/cubebox.proto`（`optional int32 port = 8`） -- `network-agent/api/v1/network_agent.proto` 与 `Cubelet/pkg/networkagentclient/pb/network_agent.proto` - （`optional int32 port = 8`，注释要求与 `CubeNet/src/cubevs.h` / Lua 保持同步） - -### 端口/协议语义（`CubeEgress/lua/port_scheme.lua` 的 `expand`） - -| 配置 | 展开结果 | -|------|----------| -| `port`/`scheme` 均 nil | 默认 `{80/http, 443/https}` | -| 仅 `scheme=http` | `{80/http}` | -| 仅 `scheme=https` | `{443/https}` | -| `port` + `scheme` 都设 | 精确 `(port, scheme)` 元组 | -| `port` 设但 `scheme` 缺 | **非法**（校验报错 `port requires scheme`） | - -约束（来自 `normalize_port` / `normalize_scheme`）： - -- `port ∈ [1, 65535]` 且为整数；`scheme ∈ {http, https}`（大小写/空白归一化）。 -- 同一 `(host, port)` 跨规则必须 scheme 一致（见 §2 服务端强校验）。 -- 每 host 最多 **8** 个 `(port, scheme)` 元组（`MAX_L7_PORTS_PER_HOST = 8`）。 - ---- - -## 2. 模型定义与透传（CubeMaster）+ 运行时强校验（CubeEgress） - -CubeMaster 这一层**只定义 `Port` 字段并原样透传**到 `cubebox.proto` / `network-agent`， -不在 Go 侧做端口/scheme 冲突校验；`types.go` 的文档注释描述的是**契约**—— -「同一 `(host, port)` 的所有规则必须就 scheme 达成一致，否则整条 policy 被拒」。 -该契约的**硬校验在 CubeEgress 加载策略时**执行（见 §4.2）。 - -- `CubeMaster/pkg/service/sandbox/types/types.go`：`EgressRuleMatch` 增加 `Port *int`，并文档化 - 上述语义（含上面的整策略原子性契约）。 -- `CubeMaster/pkg/templatecenter/template_request.go`：`cloneEgressRule` 透传 `Port` - （新增 `cloneIntPtr`）。 -- `CubeMaster/pkg/service/sandbox/util.go`：`mapEgressRuleMatch` 把 `Port` 映射到 `cubebox.proto` - 的 `EgressRuleMatch.Port`。 -- **聚合 key**：当两个 identity（`host` / `sni`）都提供时以 **Host** 为聚合 key，否则用 **SNI**； - 请求时两者都参与匹配（`CubeEgress/lua/policy.lua` 的 `validate_match_tuples`）。 -- **整策略原子性**：端口/scheme 冲突的校验在 CubeEgress 策略加载时触发（`validate_match_tuples` - 返回 `conflicts for host %q port %d: %s vs %s`），导致**整条 policy 校验失败**而非部分生效。 - ---- - -## 3. 传输（network-agent proto） - -`EgressRuleMatch.port` 经 `network-agent` 的两份 `.proto`（`network-agent/api/v1/... -` 与 `Cubelet/pkg/networkagentclient/pb/...`，内容一致）从 CubeMaster 透传到 Cubelet， -再由 Cubelet 的 network-agent client（`types.go` 增加对应字段）下发给 CubeEgress worker。 -proto 注释明确：**`port` 设了则 `scheme` 必设**，且必须与 `CubeNet/src/cubevs.h` 的 -`L7_SCHEME_*` 与 Lua `port_scheme` 保持 lockstep。 - ---- - -## 4. CubeEgress 运行时（Lua） - -### 4.1 `CubeEgress/lua/port_scheme.lua`（新增）—— 语义与匹配原语 - -- `normalize_scheme` / `normalize_port`：输入归一化与范围校验。 -- `expand(port, scheme)`：按 §1 表展开为有效 `(port, scheme)` 元组列表（校验报错透出到调用方）。 -- `matches(port, scheme, dst_port, request_scheme)`：请求时判定——先 `expand` 得元组集合，再比对 - 当前请求的 `dst_port`（由 TPROXY 保留的**原始目的端口**）与 `scheme`。 - -### 4.2 `CubeEgress/lua/policy.lua` —— 校验 - -- 新增 `MAX_L7_PORTS_PER_HOST = 8`（与 `cubevs.h` 一致）与 `normalize_identity`。 -- `validate_match_tuples`：调 `port_scheme.expand` → 以 identity 为 key 累加 `ports` 集合； - - 同 `port` 已有不同 `scheme` → 报错（`conflicts for host ... port ...`）； - - 单 host 超过 8 个元组 → 报错（`exceeds 8 L7 port tuples`）。 -- `validate_policy` 在遍历 `rules[]` 时对每条 `match` 调用上述校验，保证整策略原子性。 - -#### 4.2.1 冲突解析规则：`(hostname)` 校验域 vs `(ip, port)` 数据面键 - -`validate_match_tuples` 以**主机名字符串**为桶（`host` 缺省回退 `sni`），只能保证「同一主机名同一 -port 的 scheme 一致」。它**无法**发现「两个**不同**主机名解析到同一 `(ip, port)` 却用不同 scheme」的 -冲突——那需要在策略加载期做 DNS 解析，既脆弱又常离线不可行，故不在校验范围内。 - -而数据面 `allow_out_v3` 以 `(ip, port)/48` 为键，**每个 `(ip, port)` 只能存一个 scheme**。因此： - -- **解析规则**：对解析到同一 `(ip, port)` 的多个主机名，数据面按**最后一次写入生效**裁决——scheme 取 - 最近学成（`dns_learn_response_ip`）/ 落表（`populateAllowOutInnerMap`）的那条。`flags` 按位或仅在 - **精确同键**（`key_prefixlen` 相同）时发生；`expires_at_ns` 同键时**静态赢**：任一侧为静态（0）结果 - 即为 0（学习遇同键静态保持 0；静态落表遇同键学习项覆盖为 0——保留学习 TTL 会让 reaper 在旧 TTL - 到时后删表，静态裁决凭空消失），两侧皆学习则新 TTL 续期。此前 LPM 最长前缀查找把「同键」放大成 - 「任意覆盖项」，静态 CIDR 的零过期会传染给被覆盖的学习项（永不老化）——已由 `key_prefixlen` 精确 - 匹配 + 静态赢修正（回归测试见 §9）。同键合并的裁决结果仍**不确定**（取决于 DNS 响应到达顺序）。 -- **作者责任**：凡解析到同一 `(ip, port)` 的主机名，其 scheme **必须一致**。Lua 加载期无法代查， - 须策略作者保证。 -- **违反后果**：该 `(ip, port)` 的流量被引到「最后学成」scheme 对应的 listener（8080/8443）， - 另一主机名/scheme 的请求将在该 listener 上按 `(host, port, scheme)` 匹配**失败**。 - -### 4.3 `CubeEgress/lua/access_phase.lua` —— 请求匹配 - -- `rule_matches` 用 `port_scheme.matches(m.port, m.scheme, ctx.dst_port, ctx.scheme)` 取代旧的 - 仅 scheme 比较。`port`/`scheme` 缺省表示「向后兼容的默认端口集」，而**不是**对该 host 上 - 任意自定义端口的通配。 -- `build_ctx` 新增 `dst_port = tonumber(ngx.var.server_port)`：在 **IP_TRANSPARENT** listener 下， - nginx 的 `$server_port` 经 `getsockname()` 报告的是**原始目的端口**（TPROXY 保留），而非我们 - bind 的 8080/8443。这是 `match.port` 约束能在自定义端口上生效的前提。 -- `decide()` 把 `dst_port` 一并传入决策。 - ---- - -## 5. iptables / 策略路由（`CubeEgress/scripts/cube-proxy-iptables-init.sh`）—— 关键转变 - -### 旧模型（只能 80/443） - -按 `iif cube-dev` + `tcp dport 80/443` 选择，无 fwmark 参与。 - -### 新模型（任意端口） - -按 **`skb->mark`** 选择——mvmtap 在 sandbox tap 读出 `allow_out_v3` 的 -`(ip, port) → scheme` 映射（host→ip 由 DNS 学习层解析），写入 `CUBE_L7_MARK_HTTP=0xCE010000` / -`CUBE_L7_MARK_HTTPS=0xCE020000`（mask `0xFFFF0000`，三者均为**默认值**），本脚本据此把包 -引入 8080（HTTP listener）或 8443（HTTPS listener），**与原目的端口无关**。 - -> 三个 mark 值**安装期可配置**：`install.sh` 把 env（`CUBE_L7_MARK_HTTP/HTTPS/MASK`）写入 -> `/etc/cubeegress/l7-marks.conf`（校验 http≠https、取值不越出 mask，非法拒绝安装）； -> network-agent（注入 eBPF `const volatile` 全局）与本脚本（iptables/ip rule）**同源读取** -> 该文件，故两侧始终一致。下文规则示例均为默认值。 - -具体规则变化： - -```sh -# 旧 -iptables -t mangle -A TPROXY -i cube-dev（即 $INGRESS_IFACE） -p tcp --dport 80 -j TPROXY --on-port 8080 -iptables -t mangle -A TPROXY -i cube-dev（即 $INGRESS_IFACE） -p tcp --dport 443 -j TPROXY --on-port 8443 -# 新 -iptables -t mangle -A TPROXY -i cube-dev -p tcp \ - -m mark --mark 0xCE010000/0xFFFF0000 -j TPROXY --on-port 8080 -iptables -t mangle -A TPROXY -i cube-dev -p tcp \ - -m mark --mark 0xCE020000/0xFFFF0000 -j TPROXY --on-port 8443 -``` - -策略路由同样由「`iif` + `dport`」改为「`fwmark`」： - -```sh -ip rule add fwmark 0xCE010000/0xFFFF0000 table 100 # 取代旧的 iif cube-dev ipproto tcp dport 80 -ip rule add fwmark 0xCE020000/0xFFFF0000 table 100 -``` - -- **高位 16 bit 为 cube 自有**（`0xCE..`），低位 16 bit 留给用户其他 host 级 fwmark，由 mask 隔离。 -- 新增 `remove_legacy_dport_routing`：升级时清理旧 `iif cube-dev ipproto tcp dport 80/443` - ip rule（幂等），避免两套选择器并存。 -- 脚本幂等（`install_*` / `remove_*` / `show_status`），可安全重跑。 - ---- - -## 6. 数据面 eBPF（CubeNet/cubevs，`allow_out_v3`） - -> 本层细节见同目录 `allow_out_v3_design.md`，此处仅列与自定义端口直接相关的要点。 - -- **map 拓扑**：`allow_out_v3`（`BPF_MAP_TYPE_HASH_OF_MAPS`）→ inner `LPM_TRIE`。 - - key `lpm_key_v3`（12B）= `prefixlen`(4B) + `ip`(4B) + `port`(2B) + `_pad`(2B)； - `prefixlen=48` 即精确 `(ip, port)`，`32` 即仅 IP，`<32` 即子网。 - - value `net_policy_value_v3`（16B）= `ExpiresAtNS` + `Flags` + `Scheme` + `Reserved`； - scheme 在**插入时**解析落定，不再需要 8 元组数组。 -- **数据面解析**（`CubeNet/src/session.h` `classify_egress_flow`，由 `mvmtap.bpf.c` 的 - `do_tcp_nat` 在新建流时调用 / `udp.h`·`icmp.h` 的 `create_*_sessions` 在新建会话时调用）： - 统一裁决出向流，**一次**对 `(ip, dport)/48` 做 LPM 查找。LPM 的「最长前缀回退」会自动把 - 精确 `(ip, port)` 回退到 `/32` 或子网规则；命中有效 `L7_REQUIRED` 条目即得 `scheme` - （`FLOW_HTTP`/`FLOW_HTTPS`），其它命中（含回退到的普通 allow）一律 `FLOW_SNAT`； - 仅当该次查找未命中、或命中的 allow 已过期，才去查 `deny_out` 决定 `FLOW_REJECT`， - 否则默认 `FLOW_SNAT`。**只查一次 inner LPM trie** 即可覆盖「精确 L7 / 回退普通放行 / - default 放行」三态，无需为普通放行再查一次。 - - **注意**：旧实现 `l7_scheme_for_flow` 与 `session_policy_allowed` 已被合并。原 - `session_policy_allowed` 用硬编码 `/32` key，无法匹配 DNS 下发的 `(ip,port)/48` L7 条目， - 导致已授权的 L7 流被 `deny_out 0.0.0.0/0` 误杀；合并后 `/48` L7 查找优先于 deny， - 修复该回归。 -- **scheme → `skb->mark`**（`mvmtap.bpf.c` 的 `do_tcp_nat`）：`L7_SCHEME_HTTP → 0xCE010000`， - `L7_SCHEME_HTTPS → 0xCE020000`，与 §5 的 iptables 标记完全对应。 -- **用户态落表**（`netpolicy.go` `populateAllowOutInnerMap`）：L7 规则为每个 `(port, scheme)` 生成一条 - `/48` 条目（未显式给端口则按默认 `{80, 443}` 展开）；普通 allow 为单条 `/32`（`port=0`、`scheme=NONE`）。 -- **DNS 学习**：`dns_allow_v2` 的 `dns_allow_value.ports[]` 在域名学成时继承 `(port, scheme)`，响应处理 - 据此回填 `allow_out_v3`；`dns_query_track_value` 同样带 `ports[]` 以便免二次查找。 - ---- - -## 7. 兼容与平滑升级 - -### 7.1 `CubeNet/cubevs` 旧 pin 清理（本次特性在 `migration.go` 落地） - -- 迁移成功后通过 `removePinnedMap` 删除旧 pin：`allow_out_v2` / `allow_out`（优先 v2，回退 `allow_out`） - 与 `dns_allow`。 -- 安全性：inner LPM_TRIE 仅存在于外层 HashOfMaps 内（经 `outerMap.Put`，未单独 pin）， - 故 `os.Remove(pinPath(name))` 为单文件 unlink 即可，内核在所有 fd 关闭后自动回收 inner map。 -- 仅**迁移成功时**删除；失败则保留旧 pin 供下次重启重试，不会留下半迁移状态。 -- `dns_query_track` 不迁移：`Init` 无条件删除旧 pin 后由 `loadObject` 以新布局重新 pin（毫秒级在途 - 查询状态，重启可自愈）。 - -### 7.2 启动顺序（`cubevs.Init`，`miscs.go`） - -1. `persistentPolicyGenerationExists`（判定新 pin 是否已存在）。 -2. 删 `tungrp_to_tuns` 与 `dns_query_track` 旧 pin。 -3. `loadObject` ×3（localgw/mvmtap/nodenic）以新名字 pin 出空 `allow_out_v3`/`dns_allow_v2`/`dns_query_track`。 -4. `migratePersistentPolicyMaps`（灌入旧数据 + 清理旧 pin，见 7.1）。 -5. attach TC filter。 - -> 注：早期版本曾在第 1 步捕获「48B 旧布局 `allow_out_v3`」并在第 6 步 `replayStashedAllowOutV3` -> 回放（含崩溃安全备份）。该 48B `net_policy_value_v2` 是本需求**开发中途被放弃的方案**，真实部署 -> 不存在，已由提交 `9244355` 整体移除（`net_policy_value_v2` 还原为 16B，迁移只认 16B 旧格式， -> 48B inner map 直接判为不支持）。 - -### 7.3 iptables 侧 - -- `remove_legacy_dport_routing` 升级时清理旧的 `iif + dport 80/443` ip rule，与 §5 新 fwmark 选择器不冲突。 - -### 7.4 向后兼容 - -- 未配置 `port`/`scheme` 的规则保持默认 `{80/443}` 行为，新旧规则可混用。 - ---- - -## 8. 字节序与 ABI 守卫 - -- **网络字节序强制**：`lpm_key_v3` 的 `ip`/`port` 必须以网络字节序填充（与 `iphdr->daddr` / - `tcphdr->dest` 一致），否则精确 `(ip, port)` 匹配静默失败（`cubevs.h:170`）。 -- **双端静态断言**：`cubevs.h` 的 `_()` 校验 `lpm_key_v3==12`、`net_policy_value_v3==16`、 - `dns_allow_value==40` 等；`cubevs.go` 的 `_()` 在 Go 侧镜像校验同一组尺寸，任一侧漂移即编译失败。 -- **proto/lua lockstep**：`network_agent.proto`（server + client 两份）、`cubebox.proto`、`port_scheme.lua`、 - `cubevs.h` 的 `L7_SCHEME_*` 必须保持 `(port, scheme)` 语义同步。 -- **eBPF 重新生成**：`go generate ./...` 重编译 `bpfel.go`。 - ---- - -## 9. 测试与验证 - -| 层 | 新增/改动 | 说明 | -|----|-----------|------| -| Lua | `CubeEgress/lua/port_scheme.lua`（新）、`port_scheme_test.lua`（新） | `expand` / `matches` 单测 | -| iptables | `CubeEgress/tests/cube-proxy-iptables-init_test.sh`（新） | fwmark 选择器验证 | -| policy | `CubeEgress/lua/policy.lua` | `validate_match_tuples` 冲突/超限校验 | -| Go/eBPF | `CubeNet/cubevs/custom_port_test.go`、`CubeNet/cubevs/egress_policy_test.go` | v3 ABI 守卫、内核 BPF LPM 回退与过期回归 | -| Go/DNS 学成 | `CubeNet/cubevs/dns_learn_test.go`（+ `CubeNet/src/dns_learn_test.bpf.c`） | 普通 `/32`、L7 默认 `{80,443}`、L7 显式端口的数据面回填回归；同键合并三例（覆盖静态 CIDR 不传染、同键静态保留、同键学习 TTL 续期 + flags 并入 + scheme 覆盖，均经变异验证） | -| Go/迁移 | `CubeNet/cubevs/migration_test.go` | `dns_allow` inner-map 值变换（legacy/current）、bpffs outer-map 迁移、失败回滚保留 legacy pin；`TestSkipUnsupportedL7Subnet` | -| Go/L7 mark | `CubeNet/cubevs/l7_mark_test.go`（+ `CubeNet/src/l7_mark_test.bpf.c`）、`network-agent .../local_service_test.go::TestLoadL7MarksConfig` | `resolveL7Marks` 校验、配置注入数据面印记、`l7-marks.conf` 解析（缺省/覆盖/非法） | -| e2e(py) | `sdk/python/tests/test_l7_custom_port_e2e.py`（新）、`test_policy.py` | 自定义端口拦截/注入/审计端到端 | -| 示例 | `examples/code-sandbox-quickstart/network_l7_custom_port_echo.py`（新） | 单沙箱全规则矩阵示范（5 规则 / 6 探针）：scheme-only 默认 :80/:443、裸 host 规则展开默认集 `{80/http, 443/https}`、显式 `(port,scheme)` 自定义 :18080/http 与 :1012/https；本地 echo（自动探测 bridge IP）+ 公共回显端点断言注入 marker 到达；约束见 §10 | - -内核级 BPF 回归（提交 `2fdcbc6`，`CubeNet/cubevs/egress_policy_test.go`）： - -- `TestClassifyEgressFlowLPMFallback`：`/48` 查询回退到 `/32` IP 规则、`/24` 子网规则均得 - `FLOW_SNAT`；以 `deny_out 0.0.0.0/0` 兜底证明结果来自 LPM allow 回退而非默认放行。 -- `TestClassifyEgressFlowExpiredAllow`：过期 `/48` L7 allow（带 `deny_out`）→ `FLOW_REJECT`； - 过期回退 `/32` allow（带 `deny_out`）→ `FLOW_REJECT`；锁定「过期最长前缀命中不被 - 当作放行、但合并后直接进 deny」的当前语义。 -- `TestClassifyEgressFlowExactL7Match`：精确 `/48` L7 allow - （`192.0.2.20:8443` HTTPS）+ `deny_out 0.0.0.0/0` 兜底 → 同端口 `FLOW_HTTPS`、 - 异端口（443）回退 deny → `FLOW_REJECT`；直接锁定「单次 `/48` 查找 + LPM 不跨端口回退」语义。 - -端到端验证（`CubeNet/cubevs`）：`go build` / `go vet` / `go test` 全绿，`gofmt` 干净。 -`test_policy.py` 的用例与 `port_scheme_test.lua` 覆盖自定义端口拦截/注入/审计。 - -现网验证（2026-07-28，one-click dist `7480d03`，API `127.0.0.1:3000`）： - -- `CUBE_E2E=1 CUBE_L7_E2E_HTTP_TARGET_HOST=172.18.0.1 pytest tests/test_l7_custom_port_e2e.py` - → **2/2 通过**：自定义 HTTP 端口注入（本地 echo 目标收到注入头）、HTTPS 自定义端口拒绝 - （`:1012` → 403）、HTTPS 自定义端口放行到真实上游（`:1012` → 200，非空 body）。 -- 同机 mark 覆盖 `0xA0000000/0xB0000000`（mask `0xF0000000`）经 `/etc/cubeegress/l7-marks.conf` - 在 eBPF 印记与 iptables/ip rule 双侧同步生效（TPROXY 计数器持续命中）——单一来源、双消费方 - 一致性成立。 -- 同机运行示例 `network_l7_custom_port_echo.py` → **6/6 探针通过**：scheme-only 默认 :80/:443、 - 裸 host 规则（无 port/scheme）同规则覆盖默认集 :80+:443、自定义 `:18080`/http（本地 echo 收到 - 注入 marker）、自定义 `:1012`/https（200 + 非空 body）——全规则矩阵现网验证。 -- 运维提示：用 `example.com` 演示自定义端口会超时（其 8080/8443 握手后被静默丢弃，宿主机 - 直连同现象），并非 CubeEgress 故障；演示成功路径请用可响应的目标（如 e2e 的本地 echo）。 - ---- - -## 10. 注意事项 / 限制 - -1. **每 host 最多 8 个 `(port, scheme)` 元组**（`MAX_L7_PORTS_PER_HOST`），由 value 大小有界与 - 数据面可展开性决定；超限须在 Lua 校验期报错。 -2. **`port` 必须配 `scheme`**，且同一 `(host, port)` 跨规则 scheme 必须一致，否则**整条 policy 被服务端/运行时拒**。 -3. **自定义端口须显式声明** `port + scheme`；否则回退默认 `{80, 443}`，不会被识别为 L7 TPROXY。 -4. **TPROXY 依赖 IP_TRANSPARENT listener** 以取得原始目的端口（`$server_port` 经 `getsockname` 报原 dst）， - 这是 `match.port` 约束在自定义端口生效的前提。 -5. **`dns_query_track` 重启丢弃在途查询**（毫秒级、可自愈，设计预期）。 -6. **fwmark 高位 16 bit 默认 cube 自有**（`0xCE..`），低位留给用户其他 host 级 fwmark，由 - `CUBE_L7_MARK_MASK`（默认 `0xFFFF0000`）隔离。三个 mark 值**安装期可配置** - （env → `install.sh` → `/etc/cubeegress/l7-marks.conf`，数据面与 iptables 同源读取）； - 覆盖时须保证 http≠https 且取值不越出 mask，否则 `install.sh` 拒绝写入。 -7. **跨主机名的 `(ip, port)` scheme 冲突由作者保证**：数据面 `(ip, port)/48` 每元组仅一个 scheme；两个 - 解析到同一 `(ip, port)` 的主机名若 scheme 不同，按**最后学成生效**（不确定，见 §4.2.1）。Lua 校验 - 以主机名为域、无法在加载期发现该冲突（需 DNS），须策略作者保证这些主机名的 scheme 一致。 diff --git a/CubeNet/cubevs/cubeegress_custom_port_test_cases.md b/CubeNet/cubevs/cubeegress_custom_port_test_cases.md deleted file mode 100644 index 8bf867283..000000000 --- a/CubeNet/cubevs/cubeegress_custom_port_test_cases.md +++ /dev/null @@ -1,158 +0,0 @@ -# CubeEgress 自定义端口 — 测试用例规格 - -本文档列出「自定义端口（L7 规则 `port` + `scheme` 钉死 `(host, port, scheme)` 元组）」 -在各层的**测试场景**与**覆盖状态**。图例： - -- **现有** —— 已由代码库既有测试覆盖（无需新增）。 -- **新增** —— 本次生成的测试文件补齐的覆盖（见文末「新增文件」）。 - （`port_scheme_test.lua` 等特性落地时新增的文件，相对「评审修复轮」记为 **现有**；相对 `master` 它们同样是本分支新增，见设计文档 §9。） -- **现网验证** —— 已在真实集群部署上由 e2e 实跑验证（见文末「现网验证记录」）。 -- **缺口** —— 尚未自动化、建议后续补齐的场景。 - -> 状态基线说明（2026-07-28 更新）：`CubeNet`(Go) / `CubeEgress`(Lua+shell) / `sdk/python`(单测) -> 均已在本地实跑通过；两个 py e2e 文件需集群，其中 `test_l7_custom_port_e2e.py` 已在 -> one-click 部署（dist `7480d03`）上实跑 **2/2 通过**（见文末「现网验证记录」）， -> `test_l7_custom_port_validation_e2e.py` 负向用例仍需集群执行、未实跑。 - ---- - -## 1. API / SDK（`CubeAPI` + `cubesandbox` SDK） - -| 场景 | 期待 | 覆盖 | 位置 | -|------|------|------|------| -| 规则带 `port` + `scheme` | 正常建规则 | 现有 | `network_l7_custom_port_echo.py` 示例 | -| 规则带 `port` 但无 `scheme` | **客户端 `ValueError`** | **新增(e2e)** | `test_l7_custom_port_validation_e2e.py::test_l7_custom_port_requires_scheme` | -| `scheme` 大小写/空白归一（`" HTTPS "`→`"https"`） | 归一化生效 | **新增(lua)** | `port_scheme_extra_test.lua`（`expand(nil," HTTPS ")`） | -| 同 `(host, port)` 不同 `scheme` 跨规则 | 整条 policy 被拒 | **新增(e2e)** | `test_l7_custom_scheme_conflict_rejected` | - -## 2. CubeMaster（Go） - -| 场景 | 期待 | 覆盖 | 位置 | -|------|------|------|------| -| `EgressRuleMatch.Port` 字段透传（`types.go`/`template_request.go`/`util.go`） | 原样下发，无强校验 | 现有(单元测试隐含) | `types.go` doc-test 由 Go 编译守护 | -| 同 `(host, port)` scheme 冲突契约 | 契约文档化，硬校验在 CubeEgress | 现有 | `types.go` 注释（执行点在 §4） | - -## 3. network-agent 传输（proto） - -| 场景 | 期待 | 覆盖 | 位置 | -|------|------|------|------| -| `EgressRuleMatch.port` server/client 两份 `.proto` 一致 | lockstep | 现有 | `network_agent.proto` ×2 | -| `port` 设则 `scheme` 必设（契约） | 文档化 | 现有 | `.proto` 注释 | - -## 4. CubeEgress Lua 运行时 - -| 场景 | 期待 | 覆盖 | 位置 | -|------|------|------|------| -| `expand(nil,nil)`→`{80/http,443/https}` | 默认端口集 | **新增(lua)** | `port_scheme_extra_test.lua` | -| `expand(nil,"http")`→`{80/http}`；`(nil,"https")`→`{443/https}` | scheme-only 展开 | **新增(lua)** | `port_scheme_extra_test.lua` | -| `expand(8080,"http")`→`{8080/http}` | 自定义端口精确展开 | **新增(lua)** | `port_scheme_extra_test.lua` | -| `expand(8080,nil)` → 报错 `port requires scheme` | port 有 scheme 缺非法 | **新增(lua)** | `port_scheme_extra_test.lua` | -| `expand(nil,"ftp")` / `port=0` / `port=70000` → 报错 | 范围/取值校验 | **新增(lua)** | `port_scheme_extra_test.lua` | -| `matches` 默认 80/443、自定义端口拒绝、scheme 不匹配 | 请求时匹配 | 现有 | `port_scheme_test.lua` | -| `matches` scheme 大小写不敏感 | `HTTPS`==`https` | **新增(lua)** | `port_scheme_extra_test.lua` | -| `rule_matches` 同 host 规则不忽略自定义端口规则 | 精确 `(host,port)` 优先 | 现有 | `port_scheme_test.lua`（`access._rule_matches`） | -| `validate_match_tuples` 同 `(host,port)` scheme 冲突 → 整策略失败 | 冲突校验 | 现有 | `port_scheme_test.lua` | -| 同 host > 8 个 `(port,scheme)` 元组 → 失败 | 预置上限 | 现有 | `port_scheme_test.lua` | -| 同 host 同端口不同 scheme → 不允许 | per-(host,port) 一致性 | 现有 | `port_scheme_test.lua` | - -## 5. iptables / 策略路由（`cube-proxy-iptables-init.sh`） - -| 场景 | 期待 | 覆盖 | 位置 | -|------|------|------|------| -| `skb->mark` 命中 `CUBE_L7_MARK_HTTP` → TPROXY 到 8080 | HTTP listener | 现有 | `cube-proxy-iptables-init_test.sh` | -| `skb->mark` 命中 `CUBE_L7_MARK_HTTPS` → TPROXY 到 8443 | HTTPS listener | 现有 | `cube-proxy-iptables-init_test.sh` | -| 任意自定义端口（如 3000）按 mark 选路，不依赖 `--dport` | 端口无关 | 现网验证 | e2e `:18080`/`:1012` 经 mark 命中 TPROXY（2026-07-28）；iptables 级定向单测仍建议补 | -| `remove_legacy_dport_routing` 升级清理旧 dport 规则 | 幂等清理 | 现有 | `cube-proxy-iptables-init_test.sh` | - -## 6. eBPF 数据面（`CubeNet/cubevs`） - -| 场景 | 期待 | 覆盖 | 位置 | -|------|------|------|------| -| `expandDefaultPortSet()` = `{80/http, 443/https}`（网络字节序） | 默认端口集 | **新增(go)** | `custom_port_test.go::TestExpandDefaultPortSet` | -| `buildV3Entries` L7 显式端口 → 每端口一条 `/48` 条目 | 精确 `(ip,port)` | **新增(go)** | `custom_port_test.go::TestBuildV3EntriesL7ExplicitPorts` | -| `buildV3Entries` L7 无显式端口 → 默认 `{80,443}` 展开 | 默认展开 | **新增(go)** | `custom_port_test.go::TestBuildV3EntriesL7DefaultExpansion` | -| `buildV3Entries` 非 L7 → 单条 `/32`（`port=0`,`scheme=NONE`） | 普通 allow | **新增(go)** | `custom_port_test.go::TestBuildV3EntriesPlainAllow` | -| `buildV3Entries` 非 L7 子网 → `prefixlen<32` | 子网 | **新增(go)** | `custom_port_test.go::TestBuildV3EntriesSubnet` | -| `ExpiresAtNS` 透传到每条展开条目 | TTL 保留 | **新增(go)** | `custom_port_test.go::TestBuildV3EntriesExpiryCopied` | -| `lpm_key_v3.Port` 为**网络字节序** | 精确匹配不静默失败 | **新增(go)** | `custom_port_test.go::TestLpmKeyV3NetworkByteOrder` | -| `buildL7Plan` 默认规则→`PortCount=0`、显式端口→`PortCount=N` | 端口聚合正确 | 现有 | `netpolicy_test.go` | -| `buildL7Plan` scheme 冲突 / >8 元组 / 同 host 独立 | 校验 | 现有 | `netpolicy_test.go` | -| `netPolicyValueV3` / `netPolicyValueV2` / `dnsAllowValue` 布局静态断言 | ABI 守卫 | 现有 | `netpolicy_test.go` | -| `mvmtap` `classify_egress_flow` 单次 LPM 解析 `(ip,dport)/48`→scheme | 数据面解析 | **新增(go,内核bpf)** | `egress_policy_test.go::TestClassifyEgressFlowLPMFallback` / `TestClassifyEgressFlowExpiredAllow` / `TestClassifyEgressFlowExactL7Match`（提交 `2fdcbc6` / `fc9acac`） | -| `populateAllowOutInnerMap` 落表（`/48` 每端口 + 默认展开） | 用户态落表 | **新增(go)** | `netpolicy_test.go::TestPopulateAllowOutStaticV3OverCoveringLearnedStaysStatic` / `...OverExactLearnedStaticWins` / `...OverExactStaticMergesFlags` / `...PlainStaticOverwritesLearnedUnconditionally` | - -## 7. 端到端（`sdk/python/tests`） - -| 场景 | 期待 | 覆盖 | 位置 | -|------|------|------|------| -| 自定义 HTTP 端口注入 header（如 18080） | 请求到达目标并带注入头 | 现有 | `test_l7_custom_port_e2e.py::test_l7_custom_http_inject_and_https_deny` | -| 自定义 HTTPS 端口拒绝（如 :1012） | 返回 403 | 现有 | 同上 | -| 自定义 HTTPS 端口放行到真实上游（非 443） | 200 + 响应体 | 现有 | `test_l7_custom_https_allow_reaches_real_upstream` | -| `port` 有 `scheme` 缺 → 提交被拒 | `ValueError` | **新增(e2e)** | `test_l7_custom_port_validation_e2e.py::test_l7_custom_port_requires_scheme` | -| 同 `(host,port)` scheme 冲突 → 整策略被拒 | 提交失败 | **新增(e2e)** | `test_l7_custom_scheme_conflict_rejected` | -| > 8 个 `(port,scheme)` 元组 → 提交被拒 | 提交失败 | **新增(e2e)** | `test_l7_custom_port_budget_exceeded_rejected` | -| 同 host 混合 80/443 + 8080/http + 8443/https | 四元组各生效 | 现网验证 | `test_l7_custom_port_e2e.py` 覆盖自定义端口元组（2026-07-28 实跑通过） | - -> e2e 需集群 + `CUBE_E2E=1` / `--run-e2e` + `CUBE_L7_E2E_HTTP_TARGET_HOST` / `CUBE_TEMPLATE_ID`。 - ---- - -## 新增文件 - -| 层 | 文件 | 语言 | 验证 | -|----|------|------|------| -| eBPF 编码 | `CubeNet/cubevs/custom_port_test.go` | Go | `go test` 7 用例全绿，`gofmt` 干净 | -| eBPF 数据面内核回归 | `CubeNet/cubevs/egress_policy_test.go` | Go | `go test`（`TestClassifyEgressFlowLPMFallback` / `TestClassifyEgressFlowExpiredAllow`）全绿，提交 `2fdcbc6` | -| Lua 语义 | `CubeEgress/tests/port_scheme_extra_test.lua` | Lua | `lua tests/port_scheme_extra_test.lua` → PASS（与现有 `port_scheme_test.lua` 同跑通过） | -| Python 校验 | `sdk/python/tests/test_l7_custom_port_validation_e2e.py` | Python | `python3 -m py_compile` 通过，e2e 用例需集群 | -| eBPF DNS 学成数据面 | `CubeNet/cubevs/dns_learn_test.go`（+ `CubeNet/src/dns_learn_test.bpf.c`） | Go | `go test`（`TestDNSLearnPlainAllowWritesIPOnlyEntry` / `TestDNSLearnL7DefaultPortSet` / `TestDNSLearnL7ExplicitPorts` / `TestDNSLearnCoveringStaticCIDRDoesNotImmortalize` / `TestDNSLearnExactStaticEntrySurvivesRefresh` / `TestDNSLearnRefreshRenewsTTLMergesFlagsAndOverwritesScheme`）全绿，提交 `3e725b7`（同键合并用例经变异验证） | -| 迁移 | `CubeNet/cubevs/migration_test.go` | Go | `go test`（`TestSkipUnsupportedL7Subnet`）全绿，提交 `e9e9463` | - -> 评审修复轮另在既有测试文件内补充用例：`netpolicy_test.go`（M1 主机/CIDR 归一化合并与冲突、 -> M2 subnet 拒绝、M3 条目数展开计数、默认/显式端口展开）、`cube-proxy-iptables-init_test.sh` -> （M5 fwmark 大小写幂等）、`local_service_test.go`（M8 SNI+Host 双投、B1 recover fixture）、 -> `test_policy.py`（SDK scheme 小写归一）、`test_l7_custom_port_validation_e2e.py`（M7 断言收紧 -> 为 `ApiError, match=`）。 - -现有覆盖见：`CubeNet/cubevs/netpolicy_test.go`、`CubeEgress/tests/port_scheme_test.lua`、 -`CubeEgress/tests/cube-proxy-iptables-init_test.sh`、 -`sdk/python/tests/test_l7_custom_port_e2e.py`、 -`examples/code-sandbox-quickstart/network_l7_custom_port_echo.py`。 - -## 缺口小结（建议后续） - -1. iptables **任意自定义端口按 mark 选路**的 **iptables 级定向单测**（不依赖 `--dport`）； - 端到端已由 e2e 现网验证（2026-07-28，见文末）。 -2. `populateAllowOutInnerMap` 落表已有真实-map 单测覆盖（`netpolicy_test.go`：覆盖学习项不传染 - 静态 /48、同键学习遇静态落表静态赢、同键静态 flags 并入、plain 无条件覆盖）；`classify_egress_flow` - 内核 BPF 单测已覆盖（`egress_policy_test.go`，`2fdcbc6`/`fc9acac`）。 -3. 同 host **混合 80/443 + 自定义端口** 的四元组 e2e 自动化（自定义端口元组已由 - `test_l7_custom_port_e2e.py` 现网验证，2026-07-28，建议补自动化）。 -4. DNS 学成回填的**数据面**已有覆盖（`dns_learn_test.go`：普通 `/32`、L7 默认 `{80,443}`、 - L7 显式端口，`3e725b7`）；但 `dns_query_track` 查询跟踪与 `dns_allow_v2` 在自定义端口下的 - **端到端**（真实 DNS 报文驱动）回填路径仍未自动化。 - -> 评审修复轮新增并已通过的回归：`buildL7Plan` 主机/CIDR 归一化合并与冲突（M1）、subnet host -> 拒绝（M2）、条目数展开计数（M3）、迁移崩溃安全（M4，stale 备份路径；后经 `9244355` 随 48B -> 格式一并移除）、iptables fwmark 大小写幂等（M5）、迁移 legacy L7 subnet 丢弃 -> （`TestSkipUnsupportedL7Subnet`）、SNI+Host 双投（M8）、SDK scheme 归一与 e2e 断言收紧（M7）。 -> 遗留环境性问题见设计文档「当前状态」。 - ---- - -## 现网验证记录（2026-07-28） - -- 环境：one-click dist `7480d03`（构建自 rebase 前同名提交），API `127.0.0.1:3000`。 -- `CUBE_E2E=1 CUBE_L7_E2E_HTTP_TARGET_HOST=172.18.0.1 python -m pytest - tests/test_l7_custom_port_e2e.py -v` → **2/2 通过**：自定义 HTTP 端口注入（本地 echo 目标 - 收到注入头）、HTTPS 自定义端口拒绝（`:1012` → 403）、HTTPS 自定义端口放行到真实上游 - （`:1012` → 200，非空 body）。 -- 同机验证 **mark 安装期可配置**：覆盖 `CUBE_L7_MARK_HTTP=0xA0000000` / - `CUBE_L7_MARK_HTTPS=0xB0000000`（mask `0xF0000000`）后，eBPF 印记与 iptables/ip rule - 同步切换，TPROXY 计数器持续命中——`/etc/cubeegress/l7-marks.conf` 单一来源、双消费方 - 一致性成立。 -- 同机运行示例 `network_l7_custom_port_echo.py` → **6/6 探针通过**：scheme-only 默认 :80/:443、 - 裸 host 规则（无 port/scheme）同规则覆盖默认集 :80+:443、自定义 `:18080`/http（本地 echo - 收到注入 marker）、自定义 `:1012`/https（200 + 非空 body）。 -- 运维提示：用 `example.com` 演示自定义端口会超时（其 8080/8443 在握手后被静默丢弃，宿主机 - 直连同现象），并非 CubeEgress 故障；演示成功路径请用可响应的目标（如 e2e 的本地 echo）。 diff --git a/CubeNet/cubevs/netpolicy.go b/CubeNet/cubevs/netpolicy.go index 6b1fc50f1..826e25893 100644 --- a/CubeNet/cubevs/netpolicy.go +++ b/CubeNet/cubevs/netpolicy.go @@ -148,14 +148,14 @@ func initNetPolicy(ifindex uint32) error { // flushInnerMap removes all entries from the inner LPM trie map // associated with the given ifindex in the outer hash-of-maps. func flushInnerMap(outerMap *ebpf.Map, ifindex uint32) error { - return flushInnerMapWithValue(outerMap, ifindex, new(uint32)) + return flushInnerMapWithValue(outerMap, ifindex, new(lpmKey), new(uint32)) } func flushAllowOutInnerMap(outerMap *ebpf.Map, ifindex uint32) error { - return flushInnerMapWithValue(outerMap, ifindex, new(netPolicyValueV3)) + return flushInnerMapWithValue(outerMap, ifindex, new(lpmKeyV3), new(netPolicyValueV3)) } -func flushInnerMapWithValue(outerMap *ebpf.Map, ifindex uint32, value any) error { +func flushInnerMapWithValue(outerMap *ebpf.Map, ifindex uint32, key, value any) error { var innerMapID uint32 err := outerMap.Lookup(&ifindex, &innerMapID) if err != nil { @@ -171,10 +171,9 @@ func flushInnerMapWithValue(outerMap *ebpf.Map, ifindex uint32, value any) error } defer inner.Close() - var key lpmKey iter := inner.Iterate() - for iter.Next(&key, value) { - if err := inner.Delete(&key); err != nil && !errors.Is(err, ebpf.ErrKeyNotExist) { + for iter.Next(key, value) { + if err := inner.Delete(key); err != nil && !errors.Is(err, ebpf.ErrKeyNotExist) { return fmt.Errorf("inner map delete failed: %w", err) } } diff --git a/CubeNet/src/session.h b/CubeNet/src/session.h index ed7939760..04c59f9e5 100644 --- a/CubeNet/src/session.h +++ b/CubeNet/src/session.h @@ -153,13 +153,16 @@ static __always_inline __u8 classify_egress_flow(__u32 ifindex, __u32 daddr, } } - /* 2) Deny: /32 (or wider) lookup in deny_out */ + /* 2) Deny: /32 (or wider) lookup in deny_out. deny_out inner maps are + * keyed by the 8-byte struct lpm_key (see map.h), so use a dedicated key + * rather than reusing the 12-byte lpm_key_v3 above — passing a v3 key to + * an 8-byte-key map only works because the kernel reads map->key_size + * bytes, which is an accident of struct layout, not a contract. + */ inner_map = bpf_map_lookup_elem(&deny_out, &ifindex); if (inner_map) { - key.prefixlen = 32; - key.ip = daddr; - key.port = 0; - if (bpf_map_lookup_elem(inner_map, &key)) + struct lpm_key deny_key = { .prefixlen = 32, .ip = daddr }; + if (bpf_map_lookup_elem(inner_map, &deny_key)) return FLOW_REJECT; } diff --git a/docs/guide/security-proxy.md b/docs/guide/security-proxy.md index 8841cd08b..5a571a732 100644 --- a/docs/guide/security-proxy.md +++ b/docs/guide/security-proxy.md @@ -21,14 +21,22 @@ rule list attached at sandbox-creation time: ## How it intercepts CubeEgress runs as a host-network container and binds two TPROXY -listeners on the sandbox-facing IP: +listeners on the sandbox-facing IP — an HTTP listener on 8080 and +an HTTPS listener on 8443. Which traffic reaches each listener is +decided per-rule by the port/scheme mapping (see +[Custom L7 ports](#custom-l7-ports)), not by a fixed destination +port: ``` sandbox ──→ cube-dev (host iface) │ - ├─ iptables mangle/PREROUTING -j TPROXY - │ port 80 → 192.168.0.1:8080 (HTTP listener) - │ port 443 → 192.168.0.1:8443 (HTTPS listener) + ├─ eBPF (mvmtap) resolves the outbound (host, port) to + │ a scheme via allow_out_v3 and stamps an skb->mark + │ (HTTP or HTTPS) on the SYN + │ + ├─ iptables mangle/PREROUTING -m mark -j TPROXY + │ HTTP mark → 192.168.0.1:8080 (HTTP listener) + │ HTTPS mark → 192.168.0.1:8443 (HTTPS listener) │ ▼ CubeEgress (OpenResty + lua) @@ -84,7 +92,8 @@ Match fields (all optional, AND'd together): | Field | Type | Notes | | --- | --- | --- | -| `scheme` | `"http"` / `"https"` | | +| `scheme` | `"http"` / `"https"` | Case-insensitive. See [Custom L7 ports](#custom-l7-ports) for how `scheme` interacts with `port`. | +| `port` | int | Destination TCP port to intercept, `1`–`65535`. Must be paired with `scheme`; see [Custom L7 ports](#custom-l7-ports). | | `sni` | string | TLS ClientHello SNI; supports leading `*.` for "any subdomain" — `*.example.com` matches `www.example.com` and `foo.bar.example.com`, but not the apex | | `host` | string | Match against the HTTP `Host:` header (port stripped); same semantics as `sni` — exact match, or leading `*.` for "any subdomain" (case-insensitive) | | `method` | list of methods | OR within the list (`["GET", "POST"]`) | @@ -93,6 +102,63 @@ Match fields (all optional, AND'd together): A request must match every present field; absent fields are wildcarded. +### Custom L7 ports + +By default a rule intercepts the classic `{80/http, 443/https}` +set. The optional `port` + `scheme` pair narrows or extends which +TCP port gets steered through the proxy: + +| `port` | `scheme` | Intercepts | +| --- | --- | --- | +| omitted | omitted | `{80/http, 443/https}` — the default set (backward compatible) | +| omitted | `"http"` / `"https"` | only that scheme on its default port (`http` → 80, `https` → 443) | +| set | set | exactly that `(host, port, scheme)` tuple — e.g. an API on `tcp/8443` | +| set | omitted | invalid — `port` requires `scheme` | + +```python +from cubesandbox import Sandbox, Rule, Match, Action + +rules = [ + # Intercept an internal API on a non-standard HTTPS port. + Rule( + name="internal_api", + match=Match(host="api.internal.example", port=8443, scheme="https"), + action=Action(allow=True), + ), + # Intercept plain HTTP on a custom port. + Rule( + name="custom_http", + match=Match(host="metrics.internal.example", port=18080, scheme="http"), + action=Action(allow=True), + ), + # scheme alone keeps the classic port but only matches one side. + Rule( + name="https_only", + match=Match(host="public.example", scheme="https"), + action=Action(allow=True), + ), +] + +with Sandbox.create(network={"rules": rules}) as sb: + sb.commands.run("curl -s https://api.internal.example:8443/health") + sb.commands.run("curl -s http://metrics.internal.example:18080/") + sb.commands.run("curl -s https://public.example/") # → proxied +``` + +Constraints (enforced by the SDK client and re-checked server-side): + +- `port` must be in `[1, 65535]`; `scheme` must be `http` or + `https` (case-insensitive). +- An L7 host must be a domain name or a single IP — subnet CIDRs + are rejected (a subnet can't appear in an HTTP `Host:` header or + TLS SNI). +- Every rule sharing the same `(host, port)` must agree on + `scheme`; a conflicting policy is rejected outright. +- At most 8 distinct `(port, scheme)` tuples per host. + +Traffic to a port no rule covers still falls back to the L3/L4 +`allow_out` / `deny_out` policy and never reaches CubeEgress. + ::: tip Single-level vs multi-level subdomain `*.example.com` matches **all** subdomains regardless of label depth. To allow only single-level subdomains (e.g. `www`, @@ -209,10 +275,12 @@ about: - **Internal `cube-dev` traffic** — sandbox-to-sandbox traffic and traffic to in-cluster services (Cube API, etc.) doesn't enter the TPROXY chain, so rules don't apply. -- **Non-HTTP egress on TCP/UDP** — the TPROXY chain only redirects - ports 80 and 443. Direct TCP to other ports still goes out - subject to the L3/L4 `allow_out` / `deny_out` policy on the - CubeNet data plane, but is invisible to CubeEgress. +- **TCP/UDP not covered by an L7 rule** — the TPROXY chain only + redirects traffic an L7 rule marked: the default `{80/http, + 443/https}` set plus any custom `(port, scheme)` a rule declares. + Direct TCP to ports no rule covers still goes out subject to the + L3/L4 `allow_out` / `deny_out` policy on the CubeNet data plane, + but is invisible to CubeEgress. - **Sandboxes built from templates without the CA bake** — if the template was created with `--with-cube-ca=false`, the sandbox's TLS clients don't trust CubeEgress's leaf certs and HTTPS calls diff --git a/docs/zh/guide/security-proxy.md b/docs/zh/guide/security-proxy.md index 0d3775bc2..298ebe2a2 100644 --- a/docs/zh/guide/security-proxy.md +++ b/docs/zh/guide/security-proxy.md @@ -16,14 +16,20 @@ Cube Sandbox 在每台宿主机上部署一个透明出网代理 —— **CubeEg ## 拦截链路 CubeEgress 是一个 host-network 容器，在面向沙箱的 IP 上 bind 两个 -TPROXY listener： +TPROXY listener —— 8080 上的 HTTP listener 和 8443 上的 HTTPS +listener。哪些流量进哪个 listener，由每条规则声明的 port/scheme +映射决定（见[自定义 L7 端口](#自定义-l7-端口)），而不再绑定固定的 +目的端口： ``` sandbox ──→ cube-dev (主机网卡) │ - ├─ iptables mangle/PREROUTING -j TPROXY - │ port 80 → 192.168.0.1:8080 (HTTP) - │ port 443 → 192.168.0.1:8443 (HTTPS) + ├─ eBPF (mvmtap) 按 allow_out_v3 里的 (host, port)→scheme + │ 映射,在出方向 SYN 上打 skb->mark (HTTP 或 HTTPS) + │ + ├─ iptables mangle/PREROUTING -m mark -j TPROXY + │ HTTP mark → 192.168.0.1:8080 (HTTP) + │ HTTPS mark → 192.168.0.1:8443 (HTTPS) │ ▼ CubeEgress (OpenResty + lua) @@ -75,7 +81,8 @@ with Sandbox.create(network={"rules": rules}) as sb: | 字段 | 类型 | 说明 | | --- | --- | --- | -| `scheme` | `"http"` / `"https"` | | +| `scheme` | `"http"` / `"https"` | 大小写不敏感。与 `port` 的搭配语义见[自定义 L7 端口](#自定义-l7-端口)。 | +| `port` | int | 要拦截的目的 TCP 端口，`1`–`65535`。必须与 `scheme` 同时设置；见[自定义 L7 端口](#自定义-l7-端口)。 | | `sni` | string | TLS ClientHello 的 SNI；以 `*.` 开头时表示"任意子域"——`*.example.com` 同时命中 `www.example.com` 和 `foo.bar.example.com`，但**不**命中 apex | | `host` | string | 匹配 HTTP `Host:` 头（自动去除端口部分）；语义与 `sni` 相同 —— 支持精确匹配，或以 `*.` 开头的子域通配（大小写不敏感）| | `method` | 方法列表 | 列表内 OR 关系（`["GET", "POST"]`） | @@ -83,6 +90,61 @@ with Sandbox.create(network={"rules": rules}) as sb: 请求要同时满足所有出现的字段；未出现的字段视作通配。 +### 自定义 L7 端口 + +默认情况下，一条规则拦截经典的 `{80/http, 443/https}` 集合。可选 +的 `port` + `scheme` 组合可以收窄或扩展哪个 TCP 端口会被送进代理： + +| `port` | `scheme` | 拦截范围 | +| --- | --- | --- | +| 省略 | 省略 | `{80/http, 443/https}` —— 默认集合（向后兼容） | +| 省略 | `"http"` / `"https"` | 仅该 scheme 的默认端口（`http` → 80，`https` → 443） | +| 设置 | 设置 | 精确的 `(host, port, scheme)` 组合 —— 例如 `tcp/8443` 上的 API | +| 设置 | 省略 | 非法 —— `port` 必须搭配 `scheme` | + +```python +from cubesandbox import Sandbox, Rule, Match, Action + +rules = [ + # 拦截一个非标准 HTTPS 端口上的内部 API + Rule( + name="internal_api", + match=Match(host="api.internal.example", port=8443, scheme="https"), + action=Action(allow=True), + ), + # 拦截自定义端口上的明文 HTTP + Rule( + name="custom_http", + match=Match(host="metrics.internal.example", port=18080, scheme="http"), + action=Action(allow=True), + ), + # 只给 scheme 时仍走经典端口,但只匹配其中一侧 + Rule( + name="https_only", + match=Match(host="public.example", scheme="https"), + action=Action(allow=True), + ), +] + +with Sandbox.create(network={"rules": rules}) as sb: + sb.commands.run("curl -s https://api.internal.example:8443/health") + sb.commands.run("curl -s http://metrics.internal.example:18080/") + sb.commands.run("curl -s https://public.example/") # → 被代理 +``` + +约束（SDK 客户端先校验，服务端再校验一次）： + +- `port` 必须在 `[1, 65535]`；`scheme` 必须是 `http` 或 + `https`（大小写不敏感）。 +- L7 规则的 host 必须是域名或单个 IP —— 不支持子网 CIDR（子网 + 无法出现在 HTTP `Host:` 头或 TLS SNI 里）。 +- 共享同一个 `(host, port)` 的所有规则必须对 `scheme` 达成一致； + 冲突的 policy 会被整体拒绝。 +- 每个 host 最多 8 个不同的 `(port, scheme)` 组合。 + +任何规则都没有覆盖的端口，仍回落到 L3/L4 的 `allow_out` / +`deny_out` 策略，不会进入 CubeEgress。 + ::: tip 单层 vs 多层子域 `*.example.com` 不区分子域层数，**所有**结尾命中的子域都算。 若想只放行单层子域（如 `www`、`api`），需要为不希望放过的嵌套 @@ -187,9 +249,11 @@ inject 的 `secret` 值在写入任何日志路径前都会被剥除。审计日 - **`cube-dev` 内部流量** —— 沙箱到沙箱、沙箱到集群内服务（Cube API 等）不进 TPROXY 链路，不受规则约束。 -- **80/443 之外的 TCP/UDP** —— TPROXY 链路只重定向 80 和 443。 - 直连其它端口的 TCP 仍受 CubeNet 数据面的 L3/L4 `allow_out` / - `deny_out` 策略约束，但 CubeEgress 看不到。 +- **没有被 L7 规则覆盖的 TCP/UDP** —— TPROXY 链路只重定向被 L7 + 规则标记的流量：默认的 `{80/http, 443/https}` 集合，加上规则 + 声明的任意自定义 `(port, scheme)`。直连任何规则都没覆盖的端口 + 的 TCP，仍受 CubeNet 数据面的 L3/L4 `allow_out` / `deny_out` + 策略约束，但 CubeEgress 看不到。 - **没烘 CA 的模板** —— 如果模板用 `--with-cube-ca=false` 创建， 沙箱里的 TLS 客户端**不**信任 CubeEgress 签的 leaf 证书， HTTPS 在规则评估之前就会因 self-signed cert 报错。 diff --git a/examples/code-sandbox-quickstart/cube-test-network.py b/examples/code-sandbox-quickstart/cube-test-network.py deleted file mode 100644 index 5cd2618bb..000000000 --- a/examples/code-sandbox-quickstart/cube-test-network.py +++ /dev/null @@ -1,1000 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Tencent Inc. -# SPDX-License-Identifier: Apache-2.0 -"""End-to-end smoke test for L7 egress rules. - -Drives the Python SDK to create sandboxes with various ``network.rules`` -configurations and asserts the data plane (CubeEgress) actually enforces -them. The intent is operator-runnable verification, not pytest unit -testing — each scenario prints PASS/FAIL inline and the whole script -returns non-zero if any failed. - -Coverage: every CubeEgress externally-observable capability gets at -least one positive and (where meaningful) one negative case: - - Match: host exact, host *.suffix wildcard, scheme http/https, - method (single + any-of array), path exact, path "*" prefix. - Action: allow=true (passthrough), allow=false (403), - audit=metadata|none|full (smoke), inject[] with custom format, - inject[] with default ${SECRET} format, multiple injects in - one rule, strip-forged-header invariant, inject gate G1 - (https-only) drop on http. - Decision semantics: first-match-wins both directions, default-deny - when no rule matches. - Session longevity: one keep-alive HTTPS connection serves 1 request - per minute for 10 requests — proves the dataplane - session keeps steering an established flow to - CubeEgress even after the host's DNS-learned - allow_out_v3 /48 entry ages out. Runs twice: on - the default https port and on a custom port - (1012, where the legacy drain shim does NOT - rescue a lost session). - -Restart-resilience: every case runs each of its probes TWICE — once -against the cube-egress that was already up when the sandbox was -created, then ``docker restart cube-egress`` and the SAME probes run -again against the fresh process. The case passes only if both passes -satisfy their predicates. This proves cube-egress correctly rebuilds -its in-memory ``policy_store`` from network-agent's -``/v1/policies/dump`` source-of-truth on cold start, with no -intervention needed (we DO NOT reinstall the policy ourselves between -the two passes). - -Prerequisites: - - CUBE_API_URL, CUBE_TEMPLATE_ID set - - The template was built with --with-cube-ca=true (otherwise HTTPS - interception will fail with self-signed-cert errors and the rule - enforcement test results will be meaningless) - - cube-egress is deployed on the node AS A DOCKER CONTAINER NAMED - ``cube-egress`` (we drive ``docker restart cube-egress``); the - runner needs permission to talk to the docker socket. - - The sandbox image has Python with stdlib urllib (we deliberately - avoid `requests` since it isn't on every base image) - -Echo target: most cases use https://httpbingo.org/headers (and its -http:// counterpart) because it deterministically echoes the headers -the upstream server saw — the only public way to confirm an inject -actually flowed all the way to the origin without spending money on -a real authenticated API. We picked httpbingo over httpbin.org -specifically because httpbin's anycast often returns 503 from -mainland China (and its eu./www./nghttp2.org/httpbin mirrors are -network-blocked there); httpbingo (run on fly.io) responds reliably. -The wire shape differs slightly — httpbingo wraps every header value -in a single-element JSON array, e.g. ``"X-Api-Key": ["secret"]`` — -but our predicates use plain substring grep so the SECRET literal -still matches inside the array. - -Usage: - export CUBE_TEMPLATE_ID=tpl-d69160afa18a4c8b90b40b45 - export CUBE_API_URL=http://127.0.0.1:3000 - cd CubeSandbox/sdk/python && pip install -e . - cd ../../tests - python3 cube-test-network.py - -Exit code: 0 on all-pass, non-zero on any failure or precondition miss. -""" -from __future__ import annotations - -import json -import os -import subprocess -import sys -import time -import traceback -import urllib.request -from typing import Callable, List, Tuple -from env_utils import load_local_dotenv -load_local_dotenv() - -# ── SDK import ─────────────────────────────────────────────────────────────── -try: - from cubesandbox import Sandbox, Rule, Match, Action, Inject -except ImportError as exc: - print("[FATAL] cubesandbox SDK not importable:", exc) - print(" Install with: cd sdk/python && pip install -e .") - sys.exit(2) - - -# ── helpers ────────────────────────────────────────────────────────────────── - -# Container + admin-port wiring. cube-egress's admin loopback listens on -# 127.0.0.1:9091 (declared in CubeEgress/nginx.conf; moved off :9090 by #1285 -# because CubeProxy's plaintext gRPC serves :9090 on all-in-one hosts). The -# container is named ``cube-egress`` by the systemd unit -# ``cube-sandbox-cube-egress.service``. -CUBE_EGRESS_CONTAINER = "cube-egress" -CUBE_EGRESS_HEALTH_URL = "http://127.0.0.1:9091/admin/v1/health" -RESTART_BOOTSTRAP_TIMEOUT = 60 # seconds - - -def _check_env() -> None: - """Bail out early if the SDK has nothing to talk to.""" - missing = [v for v in ("CUBE_API_URL", "CUBE_TEMPLATE_ID") if not os.environ.get(v)] - if missing: - print(f"[FATAL] missing env: {', '.join(missing)}") - print(" export them before running this test") - sys.exit(2) - - -def _restart_cube_egress() -> None: - """``docker restart cube-egress`` and wait until it is serving again. - - "Serving" is observed via the admin /health endpoint: - - it must answer at all (worker accepting on :9091) - - bootstrap_status must reach "ready" (worker has fetched the - policy dump from Cubelet's /v1/policies/dump) - - We deliberately do NOT re-install the test's policy ourselves — - the whole point of the second pass is to verify cube-egress - rebuilt its in-memory policy_store from Cubelet on cold - start. If bootstrap landed on "skipped" (because - CUBE_EGRESS_BOOTSTRAP_URL wasn't set on the container), the - second pass would see policy_count=0 and most cases would 403, - which is exactly the regression we want to surface. - - Raises RuntimeError if the container is not back to ready - within RESTART_BOOTSTRAP_TIMEOUT — the case driver treats this - as a fatal precondition miss for the post-restart pass. - """ - print(f" ── restarting {CUBE_EGRESS_CONTAINER} ──") - t0 = time.monotonic() - try: - subprocess.run( - ["docker", "restart", CUBE_EGRESS_CONTAINER], - check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=30, - ) - except subprocess.CalledProcessError as exc: - raise RuntimeError( - f"docker restart {CUBE_EGRESS_CONTAINER} failed: {exc.stderr.decode().strip()}" - ) from exc - except subprocess.TimeoutExpired as exc: - raise RuntimeError( - f"docker restart {CUBE_EGRESS_CONTAINER} timed out after 30s" - ) from exc - - deadline = time.monotonic() + RESTART_BOOTSTRAP_TIMEOUT - last_err: str | None = None - while time.monotonic() < deadline: - try: - resp = urllib.request.urlopen(CUBE_EGRESS_HEALTH_URL, timeout=2) - body = json.loads(resp.read().decode()) - if body.get("bootstrap_status") == "ready": - print( - f" ── ready in {time.monotonic() - t0:.1f}s " - f"(policy_count={body.get('policy_count')}) ──" - ) - # Belt-and-braces: give the listener a moment to settle the - # transparent (TPROXY) sockets. Without this, the very first - # post-restart probe occasionally lands on a half-initialised - # worker and gets a connection reset before access_phase - # runs. One second is empirically enough; this is far below - # any reasonable per-case timeout budget. - time.sleep(1.0) - return - last_err = f"bootstrap_status={body.get('bootstrap_status')}" - except Exception as exc: # noqa: BLE001 - last_err = type(exc).__name__ + ": " + str(exc) - time.sleep(1.0) - raise RuntimeError( - f"{CUBE_EGRESS_CONTAINER} did not reach ready within " - f"{RESTART_BOOTSTRAP_TIMEOUT}s (last={last_err})" - ) - - -def _http_probe(url: str, *, capture_body: bool = False, method: str = "GET", - extra_headers: dict | None = None) -> str: - """Sandbox-side helper: returns code that prints status + (optional) body. - - capture_body=True dumps the entire response body so we can grep echoed - headers when the upstream is httpbingo.org/headers or /anything. - extra_headers lets a test inject a *sandbox-side* header (the thing - CubeEgress is supposed to clear when the rule plans an inject of the - same name). - """ - body_block = "" - if capture_body: - # Use unbounded resp.read() — chunked-transfer responses can - # return 0 bytes from a size-bounded first read on Python's - # urllib, which gave us empty BODY=... lines on httpbin in - # earlier runs. We trust the upstream to be small (httpbin - # /headers ≈ 200 B; /anything ≈ 1 KiB). - # NOTE: lines must be indented 8 spaces so they stay INSIDE the - # `with urlopen(...) as resp:` block — otherwise resp is closed - # by the time we hit .read() and we get a deceptive zero-byte - # body that silently fails the inject assertions. - body_block = ( - " raw_body = resp.read()\n" - " print(f'BODY_RAW_LEN={len(raw_body)}')\n" - " body = raw_body.decode('utf-8', 'replace')\n" - " print('BODY=' + body.replace(chr(10), ' '))\n" - ) - headers_block = "" - if extra_headers: - # Render as a Python dict literal embedded in the sandbox script. - items = ", ".join(f"{k!r}: {v!r}" for k, v in extra_headers.items()) - headers_block = f"_extra = {{{items}}}\n" - else: - headers_block = "_extra = {}\n" - - return f"""\ -import urllib.request, urllib.error -{headers_block}_req = urllib.request.Request({url!r}, method={method!r}) -for _k, _v in _extra.items(): - _req.add_header(_k, _v) -try: - with urllib.request.urlopen(_req, timeout=30) as resp: - print('STATUS=' + str(resp.status)) -{body_block}except urllib.error.HTTPError as e: - print('STATUS=' + str(e.code)) -except Exception as e: - print('ERROR=' + type(e).__name__ + ':' + str(e)) -""" - - -def _logs_text(execution) -> str: - """Flatten Execution stdout/stderr into one string for grep.""" - lines: List[str] = [] - if execution.logs and execution.logs.stdout: - lines.extend(execution.logs.stdout) - if execution.logs and execution.logs.stderr: - lines.extend(execution.logs.stderr) - if execution.text: - lines.append(execution.text) - return "\n".join(lines) - - -# ── twin-pass driver ───────────────────────────────────────────────────────── - -# A "run" in a case is a triple (sub_label, sandbox-side code, predicate). -# Predicate takes the stdout string and returns True/False. The driver -# executes every run once, restarts cube-egress, then executes every run -# AGAIN inside the SAME sandbox, and returns True only if all 2N -# predicate evaluations hold. - -Run = Tuple[str, str, Callable[[str], bool]] - -# Markers indicating a transient upstream-hop failure inside cube-egress -# rather than a policy-decision result. We retry the probe (not the -# restart) when the predicate fails AND one of these markers is present. -# - STATUS=502/503/504: nginx couldn't reach / TLS-verify the origin -# (proxy_ssl_verify rejection, SYN drop, slow handshake > 10s -# proxy_connect_timeout — all of which audit.lua flags as -# upstream_unreachable_or_unverified). Especially common right -# after a cold restart while DNS/TLS sessions are warming up. -# - ERROR=URLError/TimeoutError/RemoteDisconnected: the sandbox-side -# socket layer broke before/during the egress hop. -# We deliberately do NOT include STATUS=403 here — 403 is cube-egress' -# definitive policy decision, retrying it would only mask real -# regressions. -_TRANSIENT_MARKERS = ( - "STATUS=502", - "STATUS=503", - "STATUS=504", - "ERROR=URLError", - "ERROR=TimeoutError", - "ERROR=RemoteDisconnected", - "ERROR=ConnectionResetError", - "ERROR=BadStatusLine", -) -_PROBE_MAX_ATTEMPTS = 3 -_PROBE_RETRY_BACKOFF_SEC = 1.5 - - -def _is_transient(out: str) -> bool: - """Heuristic: did this probe fail with an upstream-hop error? - - Used to decide whether to retry the probe inside the SAME pass - (no extra restart). Keep the marker list narrow so we don't - paper over real policy regressions. - """ - return any(m in out for m in _TRANSIENT_MARKERS) - - -def _exec_runs(sb: Sandbox, runs: List[Run], pass_label: str) -> bool: - """Run every entry in `runs` against `sb` and return whether all passed. - - Each probe is retried up to ``_PROBE_MAX_ATTEMPTS`` times when its - predicate fails AND the output looks like a transient upstream-hop - failure (see ``_TRANSIENT_MARKERS``). Real policy regressions — - e.g. STATUS=200 where the test expected 403 — fail on the first - attempt without retry. - - Output for each entry is printed inline; the caller decides what - to do with the boolean result. - """ - all_ok = True - for sub_label, code, pred in runs: - out = "" - ok = False - for attempt in range(1, _PROBE_MAX_ATTEMPTS + 1): - out = _logs_text(sb.run_code(code)) - ok = pred(out) - if ok or not _is_transient(out) or attempt == _PROBE_MAX_ATTEMPTS: - break - # Transient AND we still have budget: pause briefly and retry. - print(f" [{pass_label} {sub_label}] transient (attempt {attempt}/" - f"{_PROBE_MAX_ATTEMPTS}); retrying after " - f"{_PROBE_RETRY_BACKOFF_SEC}s") - time.sleep(_PROBE_RETRY_BACKOFF_SEC) - - snippet = out.strip().replace("\n", " | ") - if len(snippet) > 400: - snippet = snippet[:400] + "...(truncated)" - all_ok = all_ok and ok - marker = "ok" if ok else "FAIL" - attempt_tag = f" (attempt {attempt})" if attempt > 1 else "" - print(f" [{pass_label} {sub_label}] {marker}{attempt_tag}: {snippet}") - return all_ok - - -def _run_twice_with_restart(sb: Sandbox, runs: List[Run]) -> bool: - """Drive a case: run all probes, restart cube-egress, run them again. - - Returns True iff every probe satisfied its predicate on BOTH passes. - """ - pre_ok = _exec_runs(sb, runs, "pre") - try: - _restart_cube_egress() - except RuntimeError as exc: - print(f" [restart] FAIL: {exc}") - return False - post_ok = _exec_runs(sb, runs, "post") - return pre_ok and post_ok - - -# ── scenario builders ──────────────────────────────────────────────────────── -# -# Each case constructs the sandbox, declares its `runs` list, and hands -# both to _run_twice_with_restart. The case body is therefore mostly -# data — match/action shape + per-probe predicate. - -def _has_status(code: int) -> Callable[[str], bool]: - """Predicate: stdout contains exactly STATUS=. - - NOT a substring check — we use word boundaries via the literal - 'STATUS=' marker our probe prints, which is unambiguous. - """ - needle = f"STATUS={code}" - return lambda out: needle in out - - -def _allow_then_403(code_allow: int = 200, code_deny: int = 403) -> Callable[[str], bool]: - """Allow-and-no-403: a STATUS= line AND no STATUS=.""" - a, d = f"STATUS={code_allow}", f"STATUS={code_deny}" - return lambda out: a in out and d not in out - - -def case_allow_specific_host() -> bool: - """Allow rule on https://api.moonshot.cn/:443; expect a real HTTPS response.""" - rules = [ - Rule( - name="allow_api.moonshot.cn", - match=Match(scheme="https", host="api.moonshot.cn"), - action=Action(allow=True, audit="metadata"), - ), - ] - runs: List[Run] = [ - ("GET https://api.moonshot.cn", - _http_probe("https://api.moonshot.cn"), - _has_status(200)), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_allow_suffix_host() -> bool: - """Allow rule on https://*.moonshot.cn/:443; expect a real HTTPS response.""" - DENIED_CIDRS = [ - "0.0.0.0/0", # block direct egress so traffic must use cube-egress tap - ] - rules = [ - Rule( - name="allow_suffix", - match=Match(host="*.moonshot.cn"), - action=Action(allow=True, audit="metadata"), - ), - ] - runs: List[Run] = [ - ("GET https://api.moonshot.cn", - _http_probe("https://api.moonshot.cn"), - _has_status(200)), - ] - with Sandbox.create(network={ - "allow_out": ["183.60.83.19"], - "deny_out": DENIED_CIDRS, - "rules": rules, - }) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_deny_rule_returns_403() -> bool: - """Explicit deny rule on a host; expect HTTP 403 from CubeEgress. - - Note: CubeEgress's deny path returns 403 without sending the request - upstream. urllib raises HTTPError for non-2xx, which our probe maps - back to STATUS=403. - """ - rules = [ - Rule( - name="block_evil", - match=Match(scheme="https", host="api.moonshot.cn"), - action=Action(allow=False), - ), - ] - runs: List[Run] = [ - ("GET https://api.moonshot.cn (denied)", - _http_probe("https://api.moonshot.cn"), - _has_status(403)), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_first_match_wins_order() -> bool: - """Allow then deny, both matching same host. - - First-match-wins means the allow at index 0 takes effect; a STATUS - other than 403 confirms the deny didn't override. - """ - rules = [ - Rule(name="allow_first", - match=Match(scheme="https", host="api.moonshot.cn"), - action=Action(allow=True)), - Rule(name="deny_second", - match=Match(scheme="https", host="api.moonshot.cn"), - action=Action(allow=False)), - ] - runs: List[Run] = [ - ("GET https://api.moonshot.cn (allow wins)", - _http_probe("https://api.moonshot.cn"), - _allow_then_403(200, 403)), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_first_match_wins_deny_before_allow() -> bool: - """Deny then allow, both matching same host. - - The mirror of case_first_match_wins_order — proves the order matters - in *both* directions (a buggy "last-match-wins" implementation would - pass the allow-first case but fail this one). - """ - rules = [ - Rule(name="deny_first", - match=Match(scheme="https", host="api.moonshot.cn"), - action=Action(allow=False)), - Rule(name="allow_second", - match=Match(scheme="https", host="api.moonshot.cn"), - action=Action(allow=True)), - ] - runs: List[Run] = [ - ("GET https://api.moonshot.cn (deny wins)", - _http_probe("https://api.moonshot.cn"), - _has_status(403)), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_default_deny_when_no_rule_matches() -> bool: - """Policy installed with a host-scoped rule whose path doesn't match; - request to a different path on the same host → 403. - - access_phase walks every rule; if none match, the request hits the - bottom-of-policy default-deny branch (`no_rule_match`). This proves - a non-empty policy doesn't accidentally fail open when match - conditions narrow the rule down to a sub-slice of the host. - - We deliberately keep the rule's host and the request's host the same - so the cubevs DNS allowlist (built from rule.match.host) still - resolves the lookup — DNS is L3, default-deny is L7. The path - mismatch is what triggers no_rule_match. - """ - rules = [ - Rule( - name="only_specific_path", - match=Match(scheme="https", host="httpbingo.org", - path="/never-matches-this-exact-path"), - action=Action(allow=True), - ), - ] - runs: List[Run] = [ - ("GET https://httpbingo.org/headers (default-deny)", - _http_probe("https://httpbingo.org/headers"), - _has_status(403)), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_method_filter_blocks_other_methods() -> bool: - """Rule with method=['GET'] allows GET but blocks POST.""" - rules = [ - Rule( - name="get_only", - match=Match(scheme="https", host="httpbingo.org", method=["GET"]), - action=Action(allow=True, audit="metadata"), - ), - ] - runs: List[Run] = [ - ("GET /headers", _http_probe("https://httpbingo.org/headers", method="GET"), - _has_status(200)), - ("POST /anything", _http_probe("https://httpbingo.org/anything", method="POST"), - _has_status(403)), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_method_any_of_array() -> bool: - """Method any-of: ['GET','POST'] both allowed; PUT denied.""" - rules = [ - Rule(name="get_or_post", - match=Match(scheme="https", host="httpbingo.org", method=["GET", "POST"]), - action=Action(allow=True)), - ] - runs: List[Run] = [ - ("GET /anything", _http_probe("https://httpbingo.org/anything", method="GET"), - _has_status(200)), - ("POST /anything", _http_probe("https://httpbingo.org/anything", method="POST"), - _has_status(200)), - ("PUT /anything", _http_probe("https://httpbingo.org/anything", method="PUT"), - _has_status(403)), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_path_exact_match() -> bool: - """Rule with path='/headers' allows /headers, denies /anything.""" - rules = [ - Rule(name="only_headers_path", - match=Match(scheme="https", host="httpbingo.org", path="/headers"), - action=Action(allow=True)), - ] - runs: List[Run] = [ - ("GET /headers (allow)", _http_probe("https://httpbingo.org/headers"), _has_status(200)), - ("GET /anything (deny)", _http_probe("https://httpbingo.org/anything"), _has_status(403)), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_path_prefix_wildcard() -> bool: - """Rule path='/anything*' allows /anything and /anything/foo; /headers denied. - - The trailing '*' is the only path wildcard CubeEgress supports - (suffix-side; matches the way paths grow to the right). - """ - rules = [ - Rule(name="anything_prefix", - match=Match(scheme="https", host="httpbingo.org", path="/anything*"), - action=Action(allow=True)), - ] - runs: List[Run] = [ - ("GET /anything", _http_probe("https://httpbingo.org/anything"), _has_status(200)), - ("GET /anything/foo", _http_probe("https://httpbingo.org/anything/foo"), _has_status(200)), - ("GET /headers (deny)", _http_probe("https://httpbingo.org/headers"), _has_status(403)), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_scheme_http_allows_plain_http() -> bool: - """scheme='http' rule allows the http:// listener (port 80 → 8080 TPROXY). - - Plain HTTP path is a separate code path from the SNI-driven HTTPS - path; we want a smoke that confirms the access_phase fires and lets - a matching rule through on the unencrypted listener. - """ - rules = [ - Rule(name="http_only", - match=Match(scheme="http", host="httpbingo.org"), - action=Action(allow=True)), - ] - runs: List[Run] = [ - ("GET http://httpbingo.org/headers", - _http_probe("http://httpbingo.org/headers"), - _has_status(200)), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_inject_default_format_renders_secret_only() -> bool: - """Inject without ``format`` defaults to bare ``${SECRET}``. - - Verify by greping the echoed-headers body returned from - https://httpbingo.org/headers — the X-Api-Key value should be the - raw secret with no Bearer/literal prefix. - """ - SECRET = "cube-test-DEFAULT-FMT-eb91" - rules = [ - Rule( - name="inject_default_fmt", - match=Match(scheme="https", host="httpbingo.org"), - action=Action(allow=True, audit="metadata", - inject=[Inject(header="X-Api-Key", secret=SECRET)]), - ), - ] - pred = lambda out: ("STATUS=200" in out) and (SECRET in out) # noqa: E731 - runs: List[Run] = [ - ("GET /headers w/ X-Api-Key inject", - _http_probe("https://httpbingo.org/headers", capture_body=True), - pred), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_inject_multiple_headers_in_one_rule() -> bool: - """Two inject entries on the same rule both flow upstream. - - Validates apply_injects walks the whole inject_list (not just [0]). - """ - SECRET_A = "cube-test-A-77f1" - SECRET_B = "cube-test-B-2c0d" - rules = [ - Rule( - name="inject_pair", - match=Match(scheme="https", host="httpbingo.org"), - action=Action( - allow=True, - inject=[ - Inject(header="X-Cube-A", secret=SECRET_A), - Inject(header="X-Cube-B", secret=SECRET_B, - format="prefix-${SECRET}"), - ], - ), - ), - ] - def pred(out: str) -> bool: - return ("STATUS=200" in out - and SECRET_A in out - and ("prefix-" + SECRET_B) in out) - runs: List[Run] = [ - ("GET /headers w/ two injects", - _http_probe("https://httpbingo.org/headers", capture_body=True), - pred), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_inject_strips_forged_sandbox_header() -> bool: - """Inject MUST overwrite (not append-to) a sandbox-supplied header. - - The sandbox sends ``Authorization: forged-by-sandbox``; the rule - injects its own Authorization header with a known secret. After - egress, httpbingo.org should echo the *injected* value, not the - forged one. This is the documented race-protection invariant in - access_phase.allow(): - - > we ALWAYS clear that header from the sandbox-provided - > request first, whether or not the gates pass - """ - SECRET = "cube-test-OVERWRITE-9e1a" - FORGED = "Bearer forged-by-sandbox" - rules = [ - Rule( - name="overwrite_auth", - match=Match(scheme="https", host="httpbingo.org"), - action=Action( - allow=True, - inject=[Inject(header="Authorization", secret=SECRET, - format="Bearer ${SECRET}")], - ), - ), - ] - def pred(out: str) -> bool: - return ("STATUS=200" in out - and SECRET in out - and "forged-by-sandbox" not in out) - code = _http_probe( - "https://httpbingo.org/headers", - capture_body=True, - extra_headers={"Authorization": FORGED}, - ) - runs: List[Run] = [ - ("GET /headers w/ forged Authorization", code, pred), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_inject_dropped_on_http_scheme_request_proceeds() -> bool: - """HTTP inject applies (CubeEgress no longer blocks inject on http). - - Historic behavior: G1 gate rejected inject on non-HTTPS traffic - ("https-only inject"). Current behavior (see - CubeEgress/lua/access_phase.lua:inject_gates comment on G1): operators - may legitimately need to inject credentials into plaintext traffic on - trusted networks (e.g. intra-cluster 80), so the http scheme is - permitted. This test guards that policy change by asserting the - injected header DOES appear in the upstream echo — the opposite of - the pre-relaxation expectation. - - If G1 is ever re-tightened to https-only, flip the predicate back to - ``SECRET not in out`` and rename this case. - """ - SECRET = "cube-test-http-inject-44ad" - rules = [ - Rule( - name="http_with_inject", - match=Match(scheme="http", host="httpbingo.org"), - action=Action( - allow=True, - inject=[Inject(header="X-Should-Appear", secret=SECRET)], - ), - ), - ] - def pred(out: str) -> bool: - return "STATUS=200" in out and SECRET in out - runs: List[Run] = [ - ("GET http://httpbingo.org/headers (http inject)", - _http_probe("http://httpbingo.org/headers", capture_body=True), - pred), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def case_audit_none_still_allows() -> bool: - """audit='none' is a valid AuditLevel and does not block traffic.""" - rules = [ - Rule(name="silent_audit", - match=Match(scheme="https", host="httpbingo.org"), - action=Action(allow=True, audit="none")), - ] - runs: List[Run] = [ - ("GET https://httpbingo.org/headers", - _http_probe("https://httpbingo.org/headers"), - _has_status(200)), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -def _longconn_https_probe(host: str, port: int, path: str = "/headers") -> str: - """Sandbox-side code: 10 HTTPS requests, 1 per minute, on ONE connection. - - Prints one ``REQ i STATUS=... SAMECONN=...`` line per request and a - final LONGCONN_ALL_OK / LONGCONN_FAILED marker. "Same connection" is - pinned via the socket's local address after every response — a - transparent reconnect changes the local port (or nulls the socket - after Connection: close) and surfaces as SAMECONN=False. - CubeEgress's downstream keepalive_timeout is 65s - (CubeEgress/nginx.conf:105), so the 60s period stays alive. A - User-Agent is always sent: httpbingo.org (fly.io) rejects UA-less - requests with an empty-body 402. - """ - return f"""\ -import http.client, time - -HOST, PORT, PATH = {host!r}, {port}, {path!r} -N, INTERVAL = 10, 60 -UA = {{"User-Agent": "cube-longconn-probe/1.0"}} - -conn = http.client.HTTPSConnection(HOST, PORT, timeout=30) - -# Warm-up on the SAME connection: the per-sandbox policy push to -# cube-egress is async, so ride out the first seconds of default-deny -# 403s before starting the official loop (connection stays keep-alive). -t_end = time.time() + 60 -while True: - try: - conn.request("GET", PATH, headers=UA) - r = conn.getresponse() - r.read() - if r.status != 403: - break - except Exception: - pass - if time.time() > t_end: - print("WARMUP_FAILED", flush=True) - raise SystemExit(1) - time.sleep(2) - -local = None -ok = True -for i in range(1, N + 1): - t0 = time.time() - try: - conn.request("GET", PATH, headers=UA) - resp = conn.getresponse() - resp.read() - status, will_close = resp.status, resp.will_close - except Exception as e: - print(f"REQ {{i}} EXC={{type(e).__name__}}:{{e}}", flush=True) - ok = False - break - cur = conn.sock.getsockname() if conn.sock else None - if local is None: - local = cur - same = (cur == local) and not will_close - print(f"REQ {{i}} STATUS={{status}} SAMECONN={{same}} LOCAL={{cur}}", flush=True) - ok = ok and (status == 200) and same - if i < N: - time.sleep(max(INTERVAL - (time.time() - t0), 1)) -print("LONGCONN_ALL_OK" if ok else "LONGCONN_FAILED", flush=True) -""" - - -def _longconn_pred(out: str) -> bool: - """Predicate for _longconn_https_probe: 10x200, same socket throughout.""" - return ("LONGCONN_ALL_OK" in out - and "SAMECONN=False" not in out - and out.count("STATUS=200") == 10) - - -def case_long_lived_https_connection_10x1min() -> bool: - """One keep-alive HTTPS connection, 1 request/min x 10, all must pass. - - Exercises the dataplane session-hit path over ~9 minutes: the - DNS-learned allow_out_v3 /48 entry for the host ages out mid-test - (resolver TTLs are typically <= 300s), while the established - connection's egress_sessions entry (no TTL of its own; ESTABLISHED - timeout ~3h) must keep steering packets to CubeEgress. If any - request fails — e.g. the dataplane re-classified the flow after - /48 aging and rejected it — the case fails. - - Deliberately runs WITHOUT the cube-egress restart twin-pass: - restarting the proxy would sever the very connection under test. - This case targets dataplane session longevity, not policy rebuild. - """ - rules = [ - Rule(name="longconn", - match=Match(scheme="https", host="httpbingo.org"), - action=Action(allow=True, audit="metadata")), - ] - runs: List[Run] = [ - ("https keep-alive 1 req/min x10 on one socket", - _longconn_https_probe("httpbingo.org", 443), - _longconn_pred), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _exec_runs(sb, runs, "longconn") - - -def case_long_lived_https_custom_port_10x1min() -> bool: - """Same longevity test against a NON-standard destination port (1012). - - tls-v1-2.badssl.com:1012 is a public HTTPS endpoint off 443; the - rule pins scheme=https + port=1012, so flow classification, the - DNS-learned (ip, port)/48 entry aging, and session steering all - exercise the custom-port dataplane. Unlike 80/443 flows, a lost - session here is NOT rescued by the legacy drain shim in do_tcp_nat - (custom ports are intentionally excluded), so this case is the - stronger proof that the session entry really persists for the - whole ~9 minutes. - """ - rules = [ - Rule(name="longconn_custom_port", - match=Match(scheme="https", host="tls-v1-2.badssl.com", port=1012), - action=Action(allow=True, audit="metadata")), - ] - runs: List[Run] = [ - ("https keep-alive 1 req/min x10 on custom port 1012", - _longconn_https_probe("tls-v1-2.badssl.com", 1012, "/"), - _longconn_pred), - ] - with Sandbox.create(network={"rules": rules}) as sb: - return _exec_runs(sb, runs, "longconn") - - -def case_secret_injection_via_header_for_moonshot() -> bool: - """Optionally verify credential injection against an authenticated API.""" - api_key = os.environ.get("MOONSHOT_API_KEY") - if not api_key: - print("SKIP: MOONSHOT_API_KEY is not set") - return True - - rules = [ - Rule( - name="moonshot", - match=Match(scheme="https", host="*.moonshot.cn", port=443), - action=Action( - allow=True, - audit="metadata", - inject=[Inject( - header="Authorization", - secret=api_key, - format="Bearer ${SECRET}", - )], - ), - ), - ] - DENIED_CIDRS = [ - "0.0.0.0/0", # block direct egress so traffic must use cube-egress tap - ] - python_code = """ -import json -import urllib.request - -url = "https://api.moonshot.cn/v1/chat/completions" -headers = {"Content-Type": "application/json"} - -payload = { - "messages": [ - {"content": "Hello! Who Are You？", "role": "user"} - ], - "model": "kimi-k2.5", - "stream": False -} - -data = json.dumps(payload).encode("utf-8") -req = urllib.request.Request(url, data=data, headers=headers, method="POST") - -try: - with urllib.request.urlopen(req, timeout=60) as response: - response_body = response.read().decode("utf-8") - print("Status code:", response.getcode()) - print("Response:", response_body) -except urllib.error.HTTPError as e: - print("HTTP Error:", e.code, e.reason) - print("Response body:", e.read().decode()) -""" - def pred(out: str) -> bool: - return "Status code: 200" in out and "completion_tokens" in out - runs: List[Run] = [ - ("POST /v1/chat/completions w/ inject", python_code, pred), - ] - with Sandbox.create(network={"deny_out": DENIED_CIDRS, "rules": rules}) as sb: - return _run_twice_with_restart(sb, runs) - - -# ── runner ─────────────────────────────────────────────────────────────────── - -CASES: List[Tuple[str, Callable[[], bool]]] = [ - #("allow rule on https host returns real upstream response", case_allow_specific_host), - #("allow * on https suffix host returns real upstream response", case_allow_suffix_host), - #("deny rule returns 403 without hitting upstream", case_deny_rule_returns_403), - #("first-match-wins picks the earlier allow over later deny", case_first_match_wins_order), - #("first-match-wins picks the earlier deny over later allow", case_first_match_wins_deny_before_allow), - #("non-empty policy default-denies hosts not covered by any rule", case_default_deny_when_no_rule_matches), - #("method filter allows GET and blocks POST on same host", case_method_filter_blocks_other_methods), - #("method any-of array allows multiple methods, denies others", case_method_any_of_array), - #("path exact match allows /headers and denies /anything", case_path_exact_match), - #("path '*' suffix wildcard allows whole subtree", case_path_prefix_wildcard), - #("scheme=http rule allows plain-HTTP traffic", case_scheme_http_allows_plain_http), - #("inject default format renders the bare secret", case_inject_default_format_renders_secret_only), - #("multiple inject entries in one rule all reach upstream", case_inject_multiple_headers_in_one_rule), - #("inject overwrites a sandbox-forged Authorization header", case_inject_strips_forged_sandbox_header), - #("inject applies on http scheme (G1 no longer https-only)", case_inject_dropped_on_http_scheme_request_proceeds), - #("audit='none' rule still allows traffic", case_audit_none_still_allows), - #("long-lived https connection serves 1 req/min x10 on one socket", case_long_lived_https_connection_10x1min), - ("long-lived https connection on custom port 1012 (1 req/min x10)", case_long_lived_https_custom_port_10x1min), - #("inject Authorization: Bearer reaches moonshot upstream", case_secret_injection_via_header_for_moonshot), -] - - -def main() -> int: - _check_env() - print(f"CUBE_API_URL = {os.environ.get('CUBE_API_URL')}") - print(f"CUBE_TEMPLATE_ID = {os.environ.get('CUBE_TEMPLATE_ID')}") - print(f"Restart container = {CUBE_EGRESS_CONTAINER}") - print() - - passed = failed = 0 - for label, fn in CASES: - print(f"== {label} ==") - try: - ok = fn() - except Exception: - print(" EXCEPTION:") - traceback.print_exc() - ok = False - if ok: - print(" RESULT: PASS\n") - passed += 1 - else: - print(" RESULT: FAIL\n") - failed += 1 - - print(f"summary: {passed} passed, {failed} failed (of {len(CASES)})") - return 0 if failed == 0 else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py b/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py new file mode 100644 index 000000000..e09c5cd5f --- /dev/null +++ b/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py @@ -0,0 +1,434 @@ +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 + +"""E2E coverage for network.rules with custom L7 ports (feat/custom_l7_port). + +The custom L7 port feature lets an egress rule pin the TCP port CubeEgress +intercepts on, via ``Match.port`` + ``Match.scheme``. This module exercises: + + 1. Custom HTTP port interception + credential injection — a rule with + ``(host, port, scheme=http)`` redirects a non-standard sandbox egress + port through CubeEgress and injects the rule's ``Inject`` header; the + upstream echo response reflects the marker back to the sandbox. + 2. Backward-compatible scheme-only rule — ``Match`` with ``scheme`` but no + ``port`` still intercepts the classic default set (``:80``/``:443``) and + injects on ``:80``. + 3. Create-time rejection of ``port`` without ``scheme`` (SDK ValueError). + 4. Create-time rejection of a subnet-CIDR ``host`` (server-side policy error). + +Topology notes +-------------- +* The custom-port leg runs a tiny HTTP echo server **in the test process** and + points the sandbox egress at ``SDK_E2E_L7_TARGET_HOST`` (a host IP reachable + from sandboxes, e.g. the bridge the sandbox pod attaches to). Set that env var + or the test is skipped — there is no portable default for "an IP the sandbox + can reach that is not the node IP". +* Plaintext HTTP legs are used on purpose: they need no TLS interception CA, so + they pass in any cluster that runs CubeEgress. HTTPS custom-port legs (which + require the sandbox image to trust the interception CA) are covered by + ``examples/code-sandbox-quickstart/network_l7_custom_port_echo.py``. +* All probing is done *inside* the sandbox via ``curl``; we grep the echoed + response for the injected marker secret. +""" + +from __future__ import annotations + +import json +import os +import threading +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + +import pytest + +from adapters import create_adapter +from framework.assertions import assert_command_ok +from framework.capabilities import NETWORK_L7_CUSTOM_PORT, capabilities_for_backend +from framework.cleanup import safe_kill +from framework.config import SdkE2EConfig + +# Host the sandbox can reach for the custom-port egress leg. Required; the test +# skips when unset (no safe portable default — see module docstring). +L7_TARGET_HOST = os.environ.get("SDK_E2E_L7_TARGET_HOST") +# Custom plaintext HTTP port the echo server listens on (+ the rule pins). +L7_CUSTOM_HTTP_PORT = int(os.environ.get("SDK_E2E_L7_CUSTOM_HTTP_PORT", "18080")) +# Public host for the scheme-only default-set leg (echoes request headers). +L7_DEFAULT_HOST = os.environ.get("SDK_E2E_L7_DEFAULT_HOST", "httpbingo.org") +L7_MARKER_HEADER = "X-Cube-L7-E2E" +# Per-session unique secret so the grep cannot match ambient traffic. +L7_MARKER_SECRET = f"e2e-{uuid.uuid4().hex[:12]}" + +# HTTPS custom-port leg. TLS MITM means the sandbox image must trust the L7 +# interception CA (same prerequisite as the SDK quickstart example). There is no +# portable default for "a publicly-signed HTTPS endpoint on a non-standard +# port whose upstream cert CubeEgress can verify", so the URL is configurable +# and the whole leg is skipped unless the CA-trust prerequisite is declared. +L7_CUSTOM_HTTPS_URL = os.environ.get( + "SDK_E2E_L7_CUSTOM_HTTPS_URL", "https://tls-v1-2.badssl.com:1012/" +) +_l7_https = urlparse(L7_CUSTOM_HTTPS_URL) +L7_CUSTOM_HTTPS_HOST = _l7_https.hostname +L7_CUSTOM_HTTPS_PORT = _l7_https.port # None when the URL omits the port +L7_HTTPS_CAP_TRUSTED = os.environ.get("SDK_E2E_L7_HTTPS_CAP_TRUSTED", "").strip().lower() in { + "1", + "true", + "yes", + "on", +} + +pytestmark = [ + pytest.mark.e2e, + pytest.mark.sdk_compat, + pytest.mark.network, + pytest.mark.p1, +] + + +def _skip_without_capability(sdk_backend: str) -> None: + if NETWORK_L7_CUSTOM_PORT not in capabilities_for_backend(sdk_backend): + pytest.skip( + f"backend {sdk_backend!r} does not support {NETWORK_L7_CUSTOM_PORT}" + ) + + +class _HeaderEchoHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + payload = json.dumps( + {"path": self.path, "headers": dict(self.headers)} + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_args: object) -> None: + pass + + +def _detect_bridge_ip() -> str | None: + """Best-effort mirror of the example's detect_target_host() for local runs. + + Only used as a fallback when SDK_E2E_L7_TARGET_HOST is unset; the test still + skips unless the caller sets the env var (we never assume reachability). + """ + try: + out = ( + __import__("subprocess") + .check_output(["ip", "-4", "-o", "addr", "show", "up"], text=True) + ) + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 4 and parts[1].startswith(("docker", "br-")): + return parts[3].split("/")[0] + except Exception: + pass + return None + + +@pytest.fixture(scope="module") +def l7_echo_server(): + """Start the in-process HTTP echo server for the custom-port leg. + + Skips the dependant test when no reachable target host is configured, so we + never bind a port or run a sandbox in environments that can't route to it. + """ + if not L7_TARGET_HOST and not _detect_bridge_ip(): + pytest.skip( + "SDK_E2E_L7_TARGET_HOST is required for the custom L7 port egress " + "leg (a host IP reachable from sandboxes)" + ) + server = ThreadingHTTPServer(("0.0.0.0", L7_CUSTOM_HTTP_PORT), _HeaderEchoHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield L7_TARGET_HOST or _detect_bridge_ip() + finally: + server.shutdown() + server.server_close() + + +def _l7_rules_custom_http(target_host: str) -> list[dict]: + return [ + { + "name": "e2e-custom-http", + "match": { + "host": target_host, + "port": L7_CUSTOM_HTTP_PORT, + "scheme": "http", + }, + "action": { + "allow": True, + "inject": [{"header": L7_MARKER_HEADER, "secret": L7_MARKER_SECRET}], + }, + } + ] + + +def _l7_rules_scheme_only_default() -> list[dict]: + return [ + { + "name": "e2e-default-http", + "match": {"host": L7_DEFAULT_HOST, "scheme": "http"}, + "action": { + "allow": True, + "inject": [{"header": L7_MARKER_HEADER, "secret": L7_MARKER_SECRET}], + }, + } + ] + + +def _l7_rules_custom_https(host: str, port: int) -> list[dict]: + return [ + { + "name": "e2e-custom-https", + "match": {"host": host, "port": port, "scheme": "https"}, + "action": {"allow": True}, + } + ] + + +@pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) +def test_l7_custom_port_injects_marker_on_custom_http_port( + sdk_backend: str, + sdk_e2e_config: SdkE2EConfig, + l7_echo_server: str, +): + """A (host, port, scheme=http) rule intercepts a non-standard egress port + and injects the rule's credential header; the echo reflects it back.""" + _skip_without_capability(sdk_backend) + target_host = l7_echo_server + + adapter = None + try: + adapter = create_adapter( + sdk_backend, + sdk_e2e_config, + metadata={ + "test_suite": "sdk_compat", + "test_backend": sdk_backend, + "test_case": "l7_custom_port_injects_marker", + }, + create_options={ + "allow_internet_access": False, + "network": {"rules": _l7_rules_custom_http(target_host)}, + }, + ) + + url = f"http://{target_host}:{L7_CUSTOM_HTTP_PORT}/headers" + result = adapter.run_command( + f"curl -sS --max-time 20 '{url}'", + timeout=sdk_e2e_config.command_timeout, + ) + assert_command_ok(result) + assert L7_MARKER_SECRET in result.stdout, ( + f"custom-port L7 rule should inject {L7_MARKER_HEADER!r} at {url}; " + f"response={result.stdout[:400]!r} stderr={result.stderr[:200]!r}" + ) + finally: + if adapter is not None: + safe_kill(adapter, sdk_e2e_config) + + +@pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) +@pytest.mark.requires_internet +def test_l7_custom_port_scheme_only_intercepts_default_http_set( + sdk_backend: str, + sdk_e2e_config: SdkE2EConfig, +): + """A scheme-only rule (no port) keeps the classic :80 default behavior and + still injects on the default HTTP port — backward compatibility.""" + _skip_without_capability(sdk_backend) + + adapter = None + try: + adapter = create_adapter( + sdk_backend, + sdk_e2e_config, + metadata={ + "test_suite": "sdk_compat", + "test_backend": sdk_backend, + "test_case": "l7_custom_port_scheme_only_default", + }, + create_options={ + "allow_internet_access": False, + "network": {"rules": _l7_rules_scheme_only_default()}, + }, + ) + + url = f"http://{L7_DEFAULT_HOST}/headers" + result = adapter.run_command( + f"curl -sS --max-time 20 '{url}'", + timeout=sdk_e2e_config.command_timeout, + ) + assert_command_ok(result) + assert L7_MARKER_SECRET in result.stdout, ( + f"scheme-only L7 rule should intercept :80 and inject " + f"{L7_MARKER_HEADER!r} at {url}; response={result.stdout[:400]!r} " + f"stderr={result.stderr[:200]!r}" + ) + finally: + if adapter is not None: + safe_kill(adapter, sdk_e2e_config) + + +@pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) +def test_l7_custom_port_rejects_port_without_scheme( + sdk_backend: str, + sdk_e2e_config: SdkE2EConfig, +): + """Match.port requires Match.scheme — the SDK raises ValueError before any + network round-trip (mirrors Match.__post_init__ / _normalize_match_dict).""" + _skip_without_capability(sdk_backend) + + adapter = None + try: + with pytest.raises(Exception) as exc_info: + adapter = create_adapter( + sdk_backend, + sdk_e2e_config, + metadata={ + "test_suite": "sdk_compat", + "test_backend": sdk_backend, + "test_case": "l7_custom_port_port_without_scheme", + }, + create_options={ + "network": { + "rules": [ + { + "name": "bad-port-no-scheme", + "match": {"host": "198.51.100.7", "port": 1234}, + "action": {"allow": True}, + } + ] + } + }, + ) + message = str(exc_info.value).lower() + assert "port" in message and "scheme" in message, ( + f"create failure should mention the port/scheme pairing constraint; " + f"got={exc_info.value!r}" + ) + finally: + if adapter is not None: + safe_kill(adapter, sdk_e2e_config) + + +@pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) +def test_l7_custom_port_rejects_subnet_host( + sdk_backend: str, + sdk_e2e_config: SdkE2EConfig, +): + """An L7 rule host must be a single host IP or a domain — a subnet CIDR is + rejected server-side (cubevs netpolicy: subnet CIDR not supported for L7).""" + _skip_without_capability(sdk_backend) + + adapter = None + try: + with pytest.raises(Exception) as exc_info: + adapter = create_adapter( + sdk_backend, + sdk_e2e_config, + metadata={ + "test_suite": "sdk_compat", + "test_backend": sdk_backend, + "test_case": "l7_custom_port_subnet_host", + }, + create_options={ + "network": { + "rules": [ + { + "name": "bad-subnet-host", + "match": { + "host": "10.0.0.0/24", + "scheme": "http", + }, + "action": {"allow": True}, + } + ] + } + }, + ) + message = str(exc_info.value).lower() + assert "subnet" in message or "cidr" in message or "host" in message, ( + f"create failure should mention the subnet/host constraint; " + f"got={exc_info.value!r}" + ) + finally: + if adapter is not None: + safe_kill(adapter, sdk_e2e_config) + + +@pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) +@pytest.mark.requires_internet +def test_l7_custom_port_https_intercepts_custom_port( + sdk_backend: str, + sdk_e2e_config: SdkE2EConfig, +): + """A (host, port, scheme=https) rule intercepts a non-standard HTTPS egress + port: CubeEgress does TLS MITM and forwards to the upstream. Proves the + port-scoped scheme=https interception path on a non-443 port. + + This is the HTTPS counterpart of the plaintext custom-port leg. Injection is + not asserted here because the public upstream (default badssl) does not echo + request headers; the HTTP leg already proves injection, and this leg proves + the scheme=https + TLS-MITM path works on a custom port. + """ + _skip_without_capability(sdk_backend) + if not L7_HTTPS_CAP_TRUSTED: + pytest.skip( + "HTTPS L7 interception needs the sandbox image to trust the L7 " + "interception CA; set SDK_E2E_L7_HTTPS_CAP_TRUSTED=1 to enable " + "(see examples/code-sandbox-quickstart/network_l7_custom_port_echo.py)" + ) + if not L7_CUSTOM_HTTPS_HOST or not L7_CUSTOM_HTTPS_PORT: + pytest.skip( + f"SDK_E2E_L7_CUSTOM_HTTPS_URL must include an explicit port; " + f"got {L7_CUSTOM_HTTPS_URL!r}" + ) + if L7_CUSTOM_HTTPS_PORT == 443: + pytest.skip( + "HTTPS custom-port leg exercises a non-standard port; " + f"{L7_CUSTOM_HTTPS_URL!r} resolves to :443 (use a non-443 port)" + ) + + adapter = None + try: + adapter = create_adapter( + sdk_backend, + sdk_e2e_config, + metadata={ + "test_suite": "sdk_compat", + "test_backend": sdk_backend, + "test_case": "l7_custom_port_https_custom_port", + }, + create_options={ + "allow_internet_access": False, + "network": { + "rules": _l7_rules_custom_https( + L7_CUSTOM_HTTPS_HOST, L7_CUSTOM_HTTPS_PORT + ) + }, + }, + ) + + result = adapter.run_command( + f"curl -sS --max-time 20 -o /dev/null " + f"-w 'code=%{{http_code}} len=%{{size_download}}' " + f"'{L7_CUSTOM_HTTPS_URL}'", + timeout=sdk_e2e_config.command_timeout, + ) + assert_command_ok(result) + out = result.stdout.strip() + assert "code=200" in out, ( + f"custom-port HTTPS L7 rule should proxy {L7_CUSTOM_HTTPS_URL} to " + f"HTTP 200; curl output={out!r} stderr={result.stderr[:200]!r}" + ) + assert "len=0" not in out, ( + f"custom-port HTTPS L7 rule proxied {L7_CUSTOM_HTTPS_URL} but the " + f"upstream returned an empty body; curl output={out!r}" + ) + finally: + if adapter is not None: + safe_kill(adapter, sdk_e2e_config) diff --git a/tests/e2e/sdk_compat/framework/capabilities.py b/tests/e2e/sdk_compat/framework/capabilities.py index d6e4c0492..64999fc26 100644 --- a/tests/e2e/sdk_compat/framework/capabilities.py +++ b/tests/e2e/sdk_compat/framework/capabilities.py @@ -12,6 +12,7 @@ NETWORK_ALLOW_DENY = "network_allow_deny" NETWORK_PUBLIC_ACCESS = "network_public_access" NETWORK_MASK_REQUEST_HOST = "network_mask_request_host" +NETWORK_L7_CUSTOM_PORT = "network_l7_custom_port" # CubeVS domain allow_out + DNS A learning (exact / leading "*."). NETWORK_DNS_ALLOW = "network_dns_allow" # Built-in deny of sandbox-private / link-local CIDRs when public egress is on. @@ -54,6 +55,7 @@ HOST_MOUNT, VOLUME_PLUGIN, AUTH_SIMPLE_KEY, + NETWORK_L7_CUSTOM_PORT, } ) From 5b15f9b9f40840eb54fb773ece64d5bbe5c6fbc7 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 17 Aug 2026 10:29:31 +0800 Subject: [PATCH 14/31] fix(cubenet): keep legacy allow_out pin until DNS migration succeeds migratePersistentPolicyMaps() unlinked the legacy allow_out pin right after the allow migration succeeded but before the DNS migration ran. If the DNS migration then failed, Init's rollback dropped the freshly populated allow_out_v3/dns_allow_v2 pins, and the next restart skipped re-migration (legacy pin gone), permanently losing the allow_out policy. Migrate both maps first and only unlink the legacy pins after both succeed, and make the unlink best-effort so a failed unlink cannot fail Init and trigger the rollback. Adds a combined-failure rollback test. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- CubeNet/cubevs/migration.go | 32 ++++++++++---- CubeNet/cubevs/migration_test.go | 74 ++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 8 deletions(-) diff --git a/CubeNet/cubevs/migration.go b/CubeNet/cubevs/migration.go index 76c79b429..1baa5500a 100644 --- a/CubeNet/cubevs/migration.go +++ b/CubeNet/cubevs/migration.go @@ -91,10 +91,13 @@ func pinnedMapExists(name string) (bool, error) { } // migratePersistentPolicyMaps performs a one-way migration into the current -// generation. The legacy pins are deleted once their contents have been -// copied into the current maps so they don't linger as orphaned bpffs -// entries. The current datapath and userspace only read allow_out_v3 and -// dns_allow_v2. +// generation. Both legacy maps are migrated first; the legacy pins are only +// unlinked after both have been copied into the current maps. This ordering +// matters for rollback: Init removes the freshly created allow_out_v3 / +// dns_allow_v2 pins when migration fails, so if the allow pin were unlinked +// before the DNS migration ran, a DNS failure would leave the allow_out +// policy unmigratable (legacy pin gone, new pin rolled back). The current +// datapath and userspace only read allow_out_v3 and dns_allow_v2. func migratePersistentPolicyMaps() error { allowSrc, err := allowOutMigrationSource() if err != nil { @@ -106,15 +109,28 @@ func migratePersistentPolicyMaps() error { // retried on the next restart. return err } - if err := removePinnedMap(allowSrc); err != nil { - return err - } } if err := migrateDNSAllowMap(MapNameDNSAllow); err != nil { + // Leave the legacy allow pin in place too, so both migrations are + // retried together on the next restart (see function comment). return err } - return removePinnedMap(MapNameDNSAllow) + + // Both migrations succeeded, so the current maps now hold the policy and + // dropping the legacy pins is pure cleanup. Make it best-effort: a failed + // unlink must not fail Init (which would roll back the freshly migrated + // maps and lose the policy); a lingering pin is simply re-migrated + // (idempotently) on the next restart. + if allowSrc != "" { + if err := removePinnedMap(allowSrc); err != nil { + log.Printf("cubevs migration: leaving legacy pin %s after successful migration: %v", allowSrc, err) + } + } + if err := removePinnedMap(MapNameDNSAllow); err != nil { + log.Printf("cubevs migration: leaving legacy pin %s after successful migration: %v", MapNameDNSAllow, err) + } + return nil } // allowOutMigrationSource returns the name of the legacy allow_out pin to diff --git a/CubeNet/cubevs/migration_test.go b/CubeNet/cubevs/migration_test.go index 7b6521352..ba11f3b5b 100644 --- a/CubeNet/cubevs/migration_test.go +++ b/CubeNet/cubevs/migration_test.go @@ -456,3 +456,77 @@ func TestMigrateAllowOutMapFailureKeepsLegacyPin(t *testing.T) { } src.Close() } + +// TestMigrateDNSFailureKeepsAllowOutLegacyPin covers the combined-failure +// rollback case the single-map tests miss: the allow_out migration succeeds +// but the DNS migration then fails. The legacy allow pin must survive the +// failure — if migratePersistentPolicyMaps unlinked it before the DNS +// migration ran, Init's rollback would drop the freshly populated +// allow_out_v3 and the next restart would skip re-migration (legacy pin +// gone), permanently losing the allow_out policy. +func TestMigrateDNSFailureKeepsAllowOutLegacyPin(t *testing.T) { + mountBpffs(t) + ifindex := uint32(42) + + // A good legacy allow_out_v2 (16-byte net_policy_value_v2 inner) that + // migrates cleanly. + allowOuter := newAllowOutOuterMapWithValueSize(t, uint32(unsafe.Sizeof(netPolicyValueV2{}))) + allowInner := newAllowOutLPMInner(t, uint32(unsafe.Sizeof(netPolicyValueV2{}))) + allowKey := lpmKey{Prefixlen: 32, IP: mustParseCIDRForTest(t, "192.0.2.44/32").IP} + allowVal := netPolicyValueV2{Flags: 0, ExpiresAtNS: 0} + if err := allowInner.Update(&allowKey, &allowVal, ebpf.UpdateAny); err != nil { + t.Fatalf("seed legacy allow entry: %v", err) + } + if err := allowOuter.Put(&ifindex, allowInner); err != nil { + t.Fatalf("attach legacy allow inner: %v", err) + } + allowInner.Close() + if err := allowOuter.Pin(pinPath(MapNameAllowOutV2)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameAllowOutV2, err) + } + // Destination allow_out_v3 outer. + newAllowOuter := newAllowOutV3OuterMap(t) + if err := newAllowOuter.Pin(pinPath(MapNameAllowOutV3)); err != nil { + t.Fatalf("pin %s: %v", MapNameAllowOutV3, err) + } + + // A bad legacy dns_allow whose inner carries the unsupported 48-byte value + // layout, so the DNS migration fails. + badDNSOuter := newDNSAllowOuterMapWithValueSize(t, 48) + badDNSInner := newLPMInner(t, 48) + dnsKey, _, err := makeDNSAllowRule("api.example.com", uint8(netPolicyFlagL7Required)) + if err != nil { + t.Fatalf("makeDNSAllowRule: %v", err) + } + var raw48 [48]byte + if err := badDNSInner.Update(&dnsKey, &raw48, ebpf.UpdateAny); err != nil { + t.Fatalf("seed 48-byte dns entry: %v", err) + } + if err := badDNSOuter.Put(&ifindex, badDNSInner); err != nil { + t.Fatalf("attach bad dns inner: %v", err) + } + badDNSInner.Close() + if err := badDNSOuter.Pin(pinPath(MapNameDNSAllow)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameDNSAllow, err) + } + // Destination dns_allow_v2 outer. + newDNSOuter := newDNSAllowOuterMap(t) + if err := newDNSOuter.Pin(pinPath(MapNameDNSAllowV2)); err != nil { + t.Fatalf("pin %s: %v", MapNameDNSAllowV2, err) + } + + // Migration must fail on the unsupported DNS inner layout... + err = migratePersistentPolicyMaps() + if err == nil { + t.Fatal("migratePersistentPolicyMaps succeeded, want failure for unsupported DNS inner value_size") + } + + // ...and the legacy allow pin must survive so the next restart retries + // both migrations together (this is the regression the fix addresses). + src, err := ebpf.LoadPinnedMap(pinPath(MapNameAllowOutV2), nil) + if err != nil { + t.Fatalf("legacy %s pin was removed after allow migration but DNS migration failed; "+ + "allow_out policy would be lost on restart: %v", MapNameAllowOutV2, err) + } + src.Close() +} From f31f29e17c206e2e58d87c21bf711be1172c4644 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 17 Aug 2026 10:29:45 +0800 Subject: [PATCH 15/31] fix(cubeegress): make port-less deny rules match all intercepted ports A deny rule with a host (+ optional scheme) but no port was narrowed to the default set {80/http,443/https}, so a custom-port allow rule could bypass a broader host deny (fail-open). Deny rules are now port-agnostic within the host, while allow rules keep the default-set narrowing (which is fail-closed). The port/scheme match is now action-aware via port_scheme.matches_deny. Documented the allow/deny port semantics in the security-proxy guide (en/zh). Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- CubeEgress/lua/access_phase.lua | 23 ++++++++++++----- CubeEgress/lua/port_scheme.lua | 28 ++++++++++++++++++++ CubeEgress/tests/port_scheme_test.lua | 37 +++++++++++++++++++++++++-- docs/guide/security-proxy.md | 13 ++++++++++ docs/zh/guide/security-proxy.md | 11 ++++++++ 5 files changed, 103 insertions(+), 9 deletions(-) diff --git a/CubeEgress/lua/access_phase.lua b/CubeEgress/lua/access_phase.lua index afc2f2eda..e6e680ab7 100644 --- a/CubeEgress/lua/access_phase.lua +++ b/CubeEgress/lua/access_phase.lua @@ -123,8 +123,12 @@ end -- ---------- match evaluation ---------- --- Returns true if every constraint in `m` passes against `ctx`. -local function rule_matches(m, ctx) +-- Returns true if every constraint in `m` passes against `ctx`. `is_allow` +-- selects the port/scheme semantics: an allow rule with an omitted port is +-- narrowed to the default set {80/http,443/https} (fail-closed), while a deny +-- rule with an omitted port is port-agnostic within the host (also fail-closed +-- — a custom-port allow must not bypass a broader host deny). +local function rule_matches(m, ctx, is_allow) if type(m) ~= "table" then return false end if m.sni ~= nil then @@ -144,10 +148,15 @@ local function rule_matches(m, ctx) if m.path ~= nil then if not path_match(m.path, ctx.path) then return false end end - -- Port and scheme form one semantic constraint. Omitted fields mean the - -- backward-compatible default set {80/http,443/https}, not a wildcard - -- over every custom port intercepted for the same host. - if not port_scheme.matches(m.port, m.scheme, ctx.dst_port, ctx.scheme) then + -- Port and scheme form one semantic constraint whose meaning depends on + -- whether this rule allows or denies (see port_scheme.matches_deny). + local port_scheme_ok + if is_allow then + port_scheme_ok = port_scheme.matches(m.port, m.scheme, ctx.dst_port, ctx.scheme) + else + port_scheme_ok = port_scheme.matches_deny(m.port, m.scheme, ctx.dst_port, ctx.scheme) + end + if not port_scheme_ok then return false end return true @@ -479,7 +488,7 @@ function _M.decide() decision.policy_id = p.policy_id for _, r in ipairs(p.rules or {}) do - if rule_matches(r.match, ctx) then + if rule_matches(r.match, ctx, r.action ~= nil and r.action.allow == true) then decision.rule_id = r.id decision.audit_level = (r.action and r.action.audit) or "metadata" decision.inject = r.action and r.action.inject -- consumed in Pγ diff --git a/CubeEgress/lua/port_scheme.lua b/CubeEgress/lua/port_scheme.lua index 1faf5c09a..b2a2d3b77 100644 --- a/CubeEgress/lua/port_scheme.lua +++ b/CubeEgress/lua/port_scheme.lua @@ -73,4 +73,32 @@ function _M.matches(port_value, scheme_value, dst_port, request_scheme) return false end +-- matches_deny evaluates the port/scheme constraint for a DENY rule. An allow +-- rule with an omitted port narrows to the default set {80/http,443/https} +-- (fail-closed: a custom-port flow needs an explicit allow). A deny rule with +-- an omitted port is instead port-AGNOSTIC within the host: it matches any +-- intercepted flow to the host. Narrowing a deny to the default set would be +-- fail-open — a custom-port allow rule (e.g. allow host="api.example.com" +-- port=8443 scheme=https) would bypass a broader host deny (deny +-- host="*.example.com") because the deny never matched port 8443. An +-- explicitly-set port+scheme still matches exactly that tuple. +function _M.matches_deny(port_value, scheme_value, dst_port, request_scheme) + local port, perr = normalize_port(port_value) + if perr then return false end + local scheme, serr = _M.normalize_scheme(scheme_value) + if serr then return false end + if port ~= nil then + -- Explicit port (+scheme): exact tuple, same as the allow path. + return _M.matches(port_value, scheme_value, dst_port, request_scheme) + end + if scheme == nil then + -- No port and no scheme: deny every intercepted flow to the host. + return true + end + -- No port but a scheme: deny every intercepted flow of that scheme, + -- regardless of which port carried it. + local req_scheme = _M.normalize_scheme(request_scheme) + return req_scheme ~= nil and req_scheme == scheme +end + return _M diff --git a/CubeEgress/tests/port_scheme_test.lua b/CubeEgress/tests/port_scheme_test.lua index 6ac689f8b..da9c10857 100644 --- a/CubeEgress/tests/port_scheme_test.lua +++ b/CubeEgress/tests/port_scheme_test.lua @@ -32,13 +32,46 @@ assert_true(port_scheme.matches(8443, "https", 8443, "https")) assert_false(port_scheme.matches(8443, "https", 8443, "http")) assert_false(port_scheme.matches(8443, nil, 8443, "https")) +-- matches_deny: a port-less deny is port-agnostic within the host, so a +-- custom-port flow is still caught (fail-closed). +assert_true(port_scheme.matches_deny(nil, nil, 8443, "https")) +assert_true(port_scheme.matches_deny(nil, nil, 80, "http")) +assert_true(port_scheme.matches_deny(nil, "https", 8443, "https")) +assert_false(port_scheme.matches_deny(nil, "https", 8080, "http")) +assert_true(port_scheme.matches_deny(8443, "https", 8443, "https")) +assert_false(port_scheme.matches_deny(8443, "https", 443, "https")) + -- Regression: a broad host rule must not shadow the custom-port rule. +-- (Third arg is_allow=true: an allow rule with no port narrows to the default +-- set, so it does NOT match the custom-port flow.) local ctx = { host = "api.example.com", sni = "api.example.com", method = "GET", path = "/", scheme = "https", dst_port = 8443, } -assert_false(access._rule_matches({host = "api.example.com"}, ctx)) -assert_true(access._rule_matches({host = "api.example.com", port = 8443, scheme = "https"}, ctx)) +assert_false(access._rule_matches({host = "api.example.com"}, ctx, true)) +assert_true(access._rule_matches({host = "api.example.com", port = 8443, scheme = "https"}, ctx, true)) + +-- Regression: a port-less DENY rule is port-agnostic within the host +-- (fail-closed), so a custom-port allow cannot bypass a broader host deny. +-- Same ctx: dst_port=8443, scheme=https, host=api.example.com. +assert_true(access._rule_matches({host = "api.example.com"}, ctx, false), + "port-less deny should match the custom-port flow") +assert_true(access._rule_matches({host = "*.example.com"}, ctx, false), + "wildcard port-less deny should match the custom-port flow") +-- Scheme-only deny matches any intercepted port of that scheme... +assert_true(access._rule_matches({host = "api.example.com", scheme = "https"}, ctx, false), + "scheme-only https deny should match :8443 https") +-- ...but not the other scheme. +local http_ctx = { + host = "api.example.com", sni = "api.example.com", method = "GET", + path = "/", scheme = "http", dst_port = 8080, +} +assert_false(access._rule_matches({host = "api.example.com", scheme = "https"}, http_ctx, false), + "scheme-only https deny should not match http flow") +-- An explicit port+scheme deny still matches exactly that tuple. +assert_true(access._rule_matches({host = "api.example.com", port = 8443, scheme = "https"}, ctx, false)) +assert_false(access._rule_matches({host = "api.example.com", port = 8443, scheme = "https"}, http_ctx, false), + "explicit-port deny should not match a different port/scheme") local function validate(rules) return policy.validate_policy({policy_id = "sandbox-1", rules = rules}) diff --git a/docs/guide/security-proxy.md b/docs/guide/security-proxy.md index 5a571a732..357975677 100644 --- a/docs/guide/security-proxy.md +++ b/docs/guide/security-proxy.md @@ -159,6 +159,19 @@ Constraints (enforced by the SDK client and re-checked server-side): Traffic to a port no rule covers still falls back to the L3/L4 `allow_out` / `deny_out` policy and never reaches CubeEgress. +An omitted `port` means different things for allow and deny rules. +An **allow** rule without `port` narrows to the default set +`{80/http, 443/https}` — fail-closed, so a custom-port flow is only +proxied when a rule names that port. A **deny** rule without `port` +is port-agnostic within the host — it matches every intercepted flow +to the host regardless of port. This keeps a broad host deny from +being bypassed by a narrower custom-port allow: with +`deny host="*.example.com"` and `allow host="api.example.com", +port=8443, scheme="https"`, the deny still matches +`api.example.com:8443`, so the outcome is decided by rule order +(first match wins). List the more specific allow rule first to grant +the exception. + ::: tip Single-level vs multi-level subdomain `*.example.com` matches **all** subdomains regardless of label depth. To allow only single-level subdomains (e.g. `www`, diff --git a/docs/zh/guide/security-proxy.md b/docs/zh/guide/security-proxy.md index 298ebe2a2..28cdf3bac 100644 --- a/docs/zh/guide/security-proxy.md +++ b/docs/zh/guide/security-proxy.md @@ -145,6 +145,17 @@ with Sandbox.create(network={"rules": rules}) as sb: 任何规则都没有覆盖的端口，仍回落到 L3/L4 的 `allow_out` / `deny_out` 策略，不会进入 CubeEgress。 +省略 `port` 对 allow 和 deny 规则的含义不同。**allow** 规则省略 +`port` 时会收窄到默认集合 `{80/http, 443/https}` —— 这是 +fail-closed 的选择，即自定义端口的流量只有被规则显式点名才会进 +代理。**deny** 规则省略 `port` 时则在该 host 内与端口无关 —— +无论哪个端口进来的被拦截流量都会命中。这样可防止"较宽的 host +deny 被较窄的自定义端口 allow 绕过"：例如 `deny +host="*.example.com"` 与 `allow host="api.example.com", port=8443, +scheme="https"` 同时存在时，deny 仍会命中 +`api.example.com:8443`，最终结果由规则顺序（先命中先生效）决定。 +若想放行这个例外，把更具体的 allow 规则放在前面。 + ::: tip 单层 vs 多层子域 `*.example.com` 不区分子域层数，**所有**结尾命中的子域都算。 若想只放行单层子域（如 `www`、`api`），需要为不希望放过的嵌套 From eff04404904c98fb34a07cbfaca66e39727e13fb Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 17 Aug 2026 10:30:03 +0800 Subject: [PATCH 16/31] fix(cubeegress): install TPROXY chain atomically via scratch+swap install_chain flushed and rebuilt the live chain in place; if the HTTPS rule failed after the HTTP rule succeeded, the script aborted before install_routing() while PREROUTING still jumped at a half-built chain, a silent fail-open for the HTTPS scheme. Build into an unreferenced scratch chain, verify both steering rules are present, then swap into the live chain so the previous config keeps serving until the swap. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- .../scripts/cube-proxy-iptables-init.sh | 53 +++++++++++-- .../tests/cube-proxy-iptables-init_test.sh | 78 +++++++++++++++++++ 2 files changed, 123 insertions(+), 8 deletions(-) diff --git a/CubeEgress/scripts/cube-proxy-iptables-init.sh b/CubeEgress/scripts/cube-proxy-iptables-init.sh index acf67ddf5..77658453e 100755 --- a/CubeEgress/scripts/cube-proxy-iptables-init.sh +++ b/CubeEgress/scripts/cube-proxy-iptables-init.sh @@ -120,27 +120,61 @@ require_modules() { done } -# Create-or-flush our sub-chain, then ensure PREROUTING jumps to it once. +# Build the steering rules into a scratch chain and only swap it into the live +# chain once every rule is verified present. Flushing and rebuilding the live +# chain in place would be non-atomic: if the second (HTTPS) rule failed after +# the first succeeded, the script would abort before install_routing() while +# PREROUTING still jumped at a half-built chain — a silent fail-open that +# bypasses L7 interception/deny for that scheme. Building in an unreferenced +# scratch chain keeps the currently-live config intact until the swap. install_chain() { - iptables -t mangle -N "${CHAIN}" 2>/dev/null || true - iptables -t mangle -F "${CHAIN}" + local scratch="${CHAIN}.new" - iptables -t mangle -C PREROUTING -j "${CHAIN}" 2>/dev/null \ - || iptables -t mangle -A PREROUTING -j "${CHAIN}" + # Start from a clean scratch chain. It is not referenced by PREROUTING yet, + # so a failure below never disturbs the currently-live chain. + iptables -t mangle -F "${scratch}" 2>/dev/null || true + iptables -t mangle -X "${scratch}" 2>/dev/null || true + iptables -t mangle -N "${scratch}" # HTTP: mvmtap stamps CUBE_L7_MARK_HTTP → steer to nginx HTTP listener. - iptables -t mangle -A "${CHAIN}" \ + iptables -t mangle -A "${scratch}" \ -i "${INGRESS_IFACE}" -p tcp \ -m mark --mark "${CUBE_L7_MARK_HTTP}/${CUBE_L7_MARK_MASK}" \ -j TPROXY --on-ip "${TPROXY_ON_IP}" --on-port "${TPROXY_PORT_HTTP}" # HTTPS: mvmtap stamps CUBE_L7_MARK_HTTPS → steer to nginx HTTPS listener. - iptables -t mangle -A "${CHAIN}" \ + iptables -t mangle -A "${scratch}" \ -i "${INGRESS_IFACE}" -p tcp \ -m mark --mark "${CUBE_L7_MARK_HTTPS}/${CUBE_L7_MARK_MASK}" \ -j TPROXY --on-ip "${TPROXY_ON_IP}" --on-port "${TPROXY_PORT_HTTPS}" - iptables -t mangle -A "${CHAIN}" -j RETURN + iptables -t mangle -A "${scratch}" -j RETURN + + # Post-condition: both steering rules must exist before the swap, else a + # silently dropped rule would fail open for that scheme. + iptables -t mangle -C "${scratch}" \ + -i "${INGRESS_IFACE}" -p tcp \ + -m mark --mark "${CUBE_L7_MARK_HTTP}/${CUBE_L7_MARK_MASK}" \ + -j TPROXY --on-ip "${TPROXY_ON_IP}" --on-port "${TPROXY_PORT_HTTP}" \ + || fatal "HTTP TPROXY rule missing from ${scratch} after build" + iptables -t mangle -C "${scratch}" \ + -i "${INGRESS_IFACE}" -p tcp \ + -m mark --mark "${CUBE_L7_MARK_HTTPS}/${CUBE_L7_MARK_MASK}" \ + -j TPROXY --on-ip "${TPROXY_ON_IP}" --on-port "${TPROXY_PORT_HTTPS}" \ + || fatal "HTTPS TPROXY rule missing from ${scratch} after build" + + # Swap: detach the previously-live chain from PREROUTING, drop it, rename + # the fully-built scratch chain into the live name, then point PREROUTING + # at it. The old config keeps serving until it is detached here, so there + # is no window with a half-configured chain. + while iptables -t mangle -C PREROUTING -j "${CHAIN}" 2>/dev/null; do + iptables -t mangle -D PREROUTING -j "${CHAIN}" || break + done + iptables -t mangle -F "${CHAIN}" 2>/dev/null || true + iptables -t mangle -X "${CHAIN}" 2>/dev/null || true + iptables -t mangle -E "${scratch}" "${CHAIN}" + iptables -t mangle -C PREROUTING -j "${CHAIN}" 2>/dev/null \ + || iptables -t mangle -A PREROUTING -j "${CHAIN}" } install_routing() { @@ -169,6 +203,9 @@ remove_chain() { done iptables -t mangle -F "${CHAIN}" 2>/dev/null || true iptables -t mangle -X "${CHAIN}" 2>/dev/null || true + # Drop any scratch chain left over from an aborted install. + iptables -t mangle -F "${CHAIN}.new" 2>/dev/null || true + iptables -t mangle -X "${CHAIN}.new" 2>/dev/null || true } remove_routing() { diff --git a/CubeEgress/tests/cube-proxy-iptables-init_test.sh b/CubeEgress/tests/cube-proxy-iptables-init_test.sh index 4de75e4e1..3ea5da030 100644 --- a/CubeEgress/tests/cube-proxy-iptables-init_test.sh +++ b/CubeEgress/tests/cube-proxy-iptables-init_test.sh @@ -90,6 +90,84 @@ remove_routing [[ "$(grep -c 'rule del fwmark' "${CALLS}" || true)" -eq 2 ]] [[ ! -s "${RULES_STATE}" ]] +# --- install_chain builds into a scratch chain and swaps atomically ---------- +# A steering-rule failure must abort BEFORE the live chain / PREROUTING jump is +# touched, so a partial install never leaves a fail-open gap. We mock iptables +# and record every call; chain contents live in IPT_STATE, the PREROUTING jump +# in JUMP_STATE. +: > "${CALLS}" +IPT_STATE="${TMP_DIR}/iptables_rules" +JUMP_STATE="${TMP_DIR}/prerouting_jump" +: > "${IPT_STATE}" +: > "${JUMP_STATE}" +FAIL_ON_MARK="" + +iptables() { + printf '%s\n' "$*" >> "${CALLS}" + local op="$3" target="$4" + local rest="${*:5}" + case "${op}" in + -N|-F|-X) return 0 ;; + -E) # rename chain $4 -> $5, carrying its rules + local from="$4" to="$5" + { grep -vF "${from}|" "${IPT_STATE}" || true + grep -F "${from}|" "${IPT_STATE}" | sed "s|^${from}|${to}|" || true + } > "${IPT_STATE}.new" + mv "${IPT_STATE}.new" "${IPT_STATE}" + return 0 ;; + -A) + if [[ "${target}" == "PREROUTING" ]]; then + printf '%s\n' "$6" > "${JUMP_STATE}" + return 0 + fi + if [[ -n "${FAIL_ON_MARK}" && "${rest}" == *"${FAIL_ON_MARK}"* ]]; then + return 1 + fi + printf '%s|%s\n' "${target}" "${rest}" >> "${IPT_STATE}" + return 0 ;; + -C) + if [[ "${target}" == "PREROUTING" ]]; then + [[ "$(cat "${JUMP_STATE}")" == "$6" ]] && return 0 || return 1 + fi + grep -qF "${target}|${rest}" "${IPT_STATE}" && return 0 || return 1 ;; + -D) + if [[ "${target}" == "PREROUTING" ]]; then : > "${JUMP_STATE}"; fi + return 0 ;; + esac + return 0 +} + +# Success path: both steering rules are verified in the scratch chain BEFORE it +# is renamed into the live chain, and PREROUTING only jumps at the live chain +# after the swap. +install_chain +http_v="$(grep -n -- "-t mangle -C ${CHAIN}.new .*${CUBE_L7_MARK_HTTP}" "${CALLS}" | head -1 | cut -d: -f1)" +https_v="$(grep -n -- "-t mangle -C ${CHAIN}.new .*${CUBE_L7_MARK_HTTPS}" "${CALLS}" | head -1 | cut -d: -f1)" +swap="$(grep -n -- "-t mangle -E ${CHAIN}.new ${CHAIN}" "${CALLS}" | head -1 | cut -d: -f1)" +jump="$(grep -n -- "-t mangle -A PREROUTING -j ${CHAIN}" "${CALLS}" | head -1 | cut -d: -f1)" +[[ -n "${http_v}" && -n "${https_v}" && -n "${swap}" && -n "${jump}" ]] +[[ "${http_v}" -lt "${swap}" && "${https_v}" -lt "${swap}" && "${swap}" -lt "${jump}" ]] +# Post-swap, the live chain holds both steering rules. +grep -qF "${CHAIN}|-i ${INGRESS_IFACE} -p tcp -m mark --mark ${CUBE_L7_MARK_HTTP}/${CUBE_L7_MARK_MASK} -j TPROXY --on-ip ${TPROXY_ON_IP} --on-port ${TPROXY_PORT_HTTP}" "${IPT_STATE}" +grep -qF "${CHAIN}|-i ${INGRESS_IFACE} -p tcp -m mark --mark ${CUBE_L7_MARK_HTTPS}/${CUBE_L7_MARK_MASK} -j TPROXY --on-ip ${TPROXY_ON_IP} --on-port ${TPROXY_PORT_HTTPS}" "${IPT_STATE}" + +# Failure path: the HTTPS rule fails to add. The subshell aborts before the +# swap and before any PREROUTING jump, so the live path is never half-built. +: > "${CALLS}"; : > "${IPT_STATE}"; : > "${JUMP_STATE}" +if ( FAIL_ON_MARK="${CUBE_L7_MARK_HTTPS}"; install_chain ); then + echo "install_chain succeeded despite HTTPS rule failure" >&2 + exit 1 +fi +if grep -q -- "-t mangle -E ${CHAIN}.new ${CHAIN}" "${CALLS}"; then + echo "live chain swapped in despite failed HTTPS rule (fail-open)" >&2 + exit 1 +fi +if grep -q -- "-t mangle -A PREROUTING -j ${CHAIN}" "${CALLS}"; then + echo "PREROUTING jump installed despite failed HTTPS rule (fail-open)" >&2 + exit 1 +fi +unset -f iptables + # Verify migration ordering without touching the host network. : > "${CALLS}" require_root() { :; } From bcdcd45a5abf289536f48e030ed63dc54b2dbf5b Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 17 Aug 2026 10:30:03 +0800 Subject: [PATCH 17/31] fix(cubelet): deep-copy egress rule match pointers cloneEgressRules did match := *r.Match, which aliased the Match pointer fields (SNI/Host/Path/Scheme/Port) with the caller's request despite the deep-copy contract, so mutating the request after the clone could leak into the stored copy. Mirror CubeMaster's cloneStringPtr/cloneIntPtr. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- Cubelet/network/runtime/policy_builder.go | 28 ++++++++++++++ .../network/runtime/policy_builder_test.go | 38 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/Cubelet/network/runtime/policy_builder.go b/Cubelet/network/runtime/policy_builder.go index ed4b17d32..681c2afb2 100644 --- a/Cubelet/network/runtime/policy_builder.go +++ b/Cubelet/network/runtime/policy_builder.go @@ -47,6 +47,14 @@ func cloneEgressRules(in []*EgressRule) []*EgressRule { if r.Match != nil { match := *r.Match match.Method = append([]string(nil), r.Match.Method...) + // The pointer fields must be deep-copied too: a shallow struct copy + // would alias the caller's request, so a later mutation of the + // request would leak into the stored (supposedly immutable) copy. + match.SNI = cloneStringPtr(r.Match.SNI) + match.Host = cloneStringPtr(r.Match.Host) + match.Path = cloneStringPtr(r.Match.Path) + match.Scheme = cloneStringPtr(r.Match.Scheme) + match.Port = cloneIntPtr(r.Match.Port) cp.Match = &match } if r.Action != nil { @@ -76,6 +84,26 @@ func cloneEgressRules(in []*EgressRule) []*EgressRule { return out } +// cloneStringPtr returns a copy of a *string, or nil. Mirrors the CubeMaster +// helper of the same name (pkg/service/sandbox/types/types.go). +func cloneStringPtr(value *string) *string { + if value == nil { + return nil + } + cloned := *value + return &cloned +} + +// cloneIntPtr returns a copy of a *int, or nil. Mirrors the CubeMaster helper +// of the same name (pkg/service/sandbox/types/types.go). +func cloneIntPtr(value *int) *int { + if value == nil { + return nil + } + cloned := *value + return &cloned +} + // formatCubeNetworkConfig renders a compact log-only view of the policy. It // deliberately avoids dumping full L7 rule bodies, which may contain secrets in // header-injection rules. diff --git a/Cubelet/network/runtime/policy_builder_test.go b/Cubelet/network/runtime/policy_builder_test.go index 6a3722589..2a645c5df 100644 --- a/Cubelet/network/runtime/policy_builder_test.go +++ b/Cubelet/network/runtime/policy_builder_test.go @@ -270,3 +270,41 @@ func TestToEgressInputTranslatesL7Rules(t *testing.T) { func stringPtr(value string) *string { return &value } + +// TestCloneEgressRulesDeepCopiesMatchPointers guards against the shallow-copy +// regression: cloneEgressRules must not alias the caller's match pointers, so +// mutating the request after the clone must not leak into the stored copy. +func TestCloneEgressRulesDeepCopiesMatchPointers(t *testing.T) { + port := 8443 + original := &EgressRule{ + Name: "rule", + Match: &EgressRuleMatch{ + SNI: stringPtr("api.example.com"), + Host: stringPtr("api.example.com"), + Path: stringPtr("/v1/chat"), + Scheme: stringPtr("https"), + Port: &port, + Method: []string{"GET"}, + }, + Action: &EgressRuleAction{Allow: true}, + } + + cloned := cloneEgressRules([]*EgressRule{original}) + if len(cloned) != 1 || cloned[0].Match == nil { + t.Fatalf("cloneEgressRules returned %#v", cloned) + } + + // Mutate every pointer field on the original; the clone must be unaffected. + *original.Match.SNI = "mutated.example.com" + *original.Match.Host = "mutated.example.com" + *original.Match.Path = "/mutated" + *original.Match.Scheme = "http" + *original.Match.Port = 9999 + + cm := cloned[0].Match + if *cm.SNI != "api.example.com" || *cm.Host != "api.example.com" || + *cm.Path != "/v1/chat" || *cm.Scheme != "https" || *cm.Port != 8443 { + t.Fatalf("clone aliased caller pointers: SNI=%q Host=%q Path=%q Scheme=%q Port=%d", + *cm.SNI, *cm.Host, *cm.Path, *cm.Scheme, *cm.Port) + } +} From b0cf9255d856119847e8dfb284d075b973937bef Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 17 Aug 2026 16:19:06 +0800 Subject: [PATCH 18/31] fix(cubenet): let plain allow_out coexist with L7 rules on a shared domain When the same domain appeared in both plain allow_out (L3) and an L7 rule, mergeDNSAllowRules collapsed them into one L7 entry, so the DNS-learned IP was admitted into allow_out_v3 only on the rule's ports and lost plain L3 access on every other port under deny-all. Add NET_POLICY_FLAG_L3_ALLOWED (bit 1). mergeDNSAllowRules sets it when a domain key is present in both the plain allow_out base set and an L7 rule set. dns_learn_response_ip then learns the plain /32 any-port entry alongside the L7 (ip, port)/48 entries: the /48 longest-prefix match wins for the rule's ports (L7 intercept) while the /32 covers all other ports (plain SNAT). The two near-identical /32 write blocks are merged into one shared block (the marker-stripping mask is a no-op for plain allows). Note this is a deliberate behavior change: a domain in both allow_out and an L7 rule now keeps its plain L3 access (more permissive than before). Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- CubeNet/cubevs/cubevs.go | 4 ++ CubeNet/cubevs/dns_learn_test.go | 63 ++++++++++++++++++++++++++++++++ CubeNet/cubevs/netpolicy.go | 9 ++++- CubeNet/cubevs/netpolicy_test.go | 44 +++++++++++++++++++++- CubeNet/src/cubevs.h | 8 ++++ CubeNet/src/dns_response.h | 21 +++++++++-- docs/guide/security-proxy.md | 10 +++++ docs/zh/guide/security-proxy.md | 9 +++++ 8 files changed, 161 insertions(+), 7 deletions(-) diff --git a/CubeNet/cubevs/cubevs.go b/CubeNet/cubevs/cubevs.go index ad53ff32c..8d87a242d 100644 --- a/CubeNet/cubevs/cubevs.go +++ b/CubeNet/cubevs/cubevs.go @@ -178,6 +178,10 @@ const ( dnsPolicyFlagLearningEnabled = 1 << 0 // Network policy flags. Must match src/cubevs.h. netPolicyFlagL7Required = 1 << 0 + // netPolicyFlagL3Allowed marks a domain present in both plain allow_out + // and an L7 rule, so the datapath learns the plain /32 any-port entry + // alongside the L7 /48 entries. Must match src/cubevs.h. + netPolicyFlagL3Allowed = 1 << 1 // L7 scheme values in dns_allow_value / net_policy_value_v3 per-port // entries. Must match L7_SCHEME_* in src/cubevs.h. L7SchemeNone uint8 = 0 diff --git a/CubeNet/cubevs/dns_learn_test.go b/CubeNet/cubevs/dns_learn_test.go index 35efbe577..a7796dba7 100644 --- a/CubeNet/cubevs/dns_learn_test.go +++ b/CubeNet/cubevs/dns_learn_test.go @@ -210,6 +210,69 @@ func TestDNSLearnL7ExplicitPorts(t *testing.T) { } } +// TestDNSLearnL7WithL3AlsoWritesPlainAndL7Entries covers the coexistence path: +// a domain present in both plain allow_out and an L7 rule must learn BOTH the +// /32 any-port entry (plain SNAT for non-rule ports) AND the /48 L7 entry +// (interception for the rule's port). Previously the L7 flag subsumed the plain +// allow, so only the rule's port was admitted and the domain lost plain L3 +// access on every other port. +func TestDNSLearnL7WithL3AlsoWritesPlainAndL7Entries(t *testing.T) { + env := loadDNSLearnTestEnv(t) + ifindex := uint32(304) + ip := mustParseCIDRForTest(t, "192.0.2.80").IP + + query := dnsQueryTrackValue{ + Flags: uint8(netPolicyFlagL7Required) | uint8(netPolicyFlagL3Allowed), + PortCount: 1, + Ports: [maxL7PortsPerHost]l7PortEntry{ + {Port: htonsPort(8443), Scheme: L7SchemeHTTPS}, + }, + } + inner := runDNSLearn(t, env, ifindex, ip, 300, query) + + // The /48 L7 entry for the rule's port is present and intercepted. + l7, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(8443)}) + if !ok { + t.Fatal("missing /48 L7 entry for rule port 8443") + } + if l7.Flags&uint8(netPolicyFlagL7Required) == 0 { + t.Fatal("/48 entry missing L7 flag") + } + if l7.Scheme != L7SchemeHTTPS { + t.Fatalf("/48 scheme=%d, want https", l7.Scheme) + } + + // The /32 any-port plain entry is also present for everything else, and it + // must be a plain allow (no L7 / L3 marker bits leaked into the value). + plain, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0}) + if !ok { + t.Fatal("missing /32 plain entry for L3-allowed domain") + } + if plain.Flags&uint8(netPolicyFlagL7Required) != 0 { + t.Fatalf("/32 plain entry has unexpected L7 flag: %#x", plain.Flags) + } + if plain.Flags&uint8(netPolicyFlagL3Allowed) != 0 { + t.Fatalf("/32 plain entry leaked L3_ALLOWED marker: %#x", plain.Flags) + } + if plain.Scheme != L7SchemeNone { + t.Fatalf("/32 plain scheme=%d, want none", plain.Scheme) + } + + // A lookup for a non-rule port must NOT match a /48 L7 entry; it falls back + // via LPM longest-prefix to the /32 plain entry (which is exactly how + // classify_egress_flow admits the flow via plain SNAT). + fallback, ok := lookupAllowV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(443)}) + if !ok { + t.Fatal("non-rule port 443 matched nothing, want fallback to the /32 plain entry") + } + if fallback.KeyPrefixlen == 48 && fallback.Flags&uint8(netPolicyFlagL7Required) != 0 { + t.Fatalf("non-rule port 443 unexpectedly matched a /48 L7 entry: %+v", fallback) + } + if fallback.KeyPrefixlen != 32 { + t.Fatalf("non-rule port 443 fell back to key_prefixlen=%d, want 32 (plain)", fallback.KeyPrefixlen) + } +} + // testFlagMarker is a high-bit flag used only by tests to detect improper // flag inheritance from COVERING entries (it is not a real netPolicyFlag*). const testFlagMarker = 0x40 diff --git a/CubeNet/cubevs/netpolicy.go b/CubeNet/cubevs/netpolicy.go index 826e25893..cf4d9ca82 100644 --- a/CubeNet/cubevs/netpolicy.go +++ b/CubeNet/cubevs/netpolicy.go @@ -671,7 +671,12 @@ func mergeAllowOutWithL7(base, l7 []allowOutPolicyEntry) []allowOutPolicyEntry { } // mergeDNSAllowRules combines the non-L7 domain rules with the L7 domain rules. -// Same-key entries merge flags (OR) and adopt the L7 rule's port set. +// Same-key entries merge flags (OR) and adopt the L7 rule's port set. When the +// same domain is present in both (plain allow_out AND an L7 rule), the merged +// entry is also marked netPolicyFlagL3Allowed so the datapath learns the plain +// /32 any-port entry alongside the L7 /48 entries — otherwise the L7 rule +// would silently narrow the same-domain plain allow_out to only the rule's +// ports. func mergeDNSAllowRules(base, l7 []dnsAllowRule) []dnsAllowRule { if len(l7) == 0 { return base @@ -684,7 +689,7 @@ func mergeDNSAllowRules(base, l7 []dnsAllowRule) []dnsAllowRule { } for _, r := range l7 { if idx, ok := byKey[r.key]; ok { - out[idx].value.Flags |= r.value.Flags + out[idx].value.Flags |= r.value.Flags | netPolicyFlagL3Allowed out[idx].value.PortCount = r.value.PortCount out[idx].value.Ports = r.value.Ports continue diff --git a/CubeNet/cubevs/netpolicy_test.go b/CubeNet/cubevs/netpolicy_test.go index 1e7f0a619..f6466beaa 100644 --- a/CubeNet/cubevs/netpolicy_test.go +++ b/CubeNet/cubevs/netpolicy_test.go @@ -222,7 +222,10 @@ func TestBuildNetPolicyPlanDeduplicatesAndMergesFlags(t *testing.T) { if got, want := len(plan.dnsAllowRules), 1; got != want { t.Fatalf("len(plan.dnsAllowRules)=%d, want %d", got, want) } - if got, want := plan.dnsAllowRules[0].value.Flags, uint8(netPolicyFlagL7Required); got != want { + // api.example.com is in BOTH plain allow_out and an L7 rule, so the merged + // entry carries L7Required|L3Allowed (the L3 bit keeps the plain /32 + // any-port entry alongside the L7 /48 entries). + if got, want := plan.dnsAllowRules[0].value.Flags, uint8(netPolicyFlagL7Required|netPolicyFlagL3Allowed); got != want { t.Fatalf("dns allow flags=%d, want %d", got, want) } if got, want := plan.dnsPolicyFlags, uint8(dnsPolicyFlagLearningEnabled); got != want { @@ -236,6 +239,45 @@ func TestBuildNetPolicyPlanDeduplicatesAndMergesFlags(t *testing.T) { } } +// TestBuildNetPolicyPlanL3AllowedCoexistence pins the plain-allow_out + +// L7-rule coexistence semantics for a shared domain: the merged dns_allow +// entry must carry netPolicyFlagL3Allowed (so the datapath learns both the +// plain /32 any-port entry and the L7 /48 entries), while a domain present in +// only one of them must not set it. +func TestBuildNetPolicyPlanL3AllowedCoexistence(t *testing.T) { + allowOut := []string{"both.example.com", "plain.example.com"} + l7AllowOut := []L7Target{ + {Host: "both.example.com", Port: 8443, Scheme: L7SchemeHTTPS}, + {Host: "l7only.example.com", Port: 9090, Scheme: L7SchemeHTTP}, + } + + plan, err := buildNetPolicyPlan(MVMOptions{ + AllowOut: &allowOut, + L7AllowOut: &l7AllowOut, + }) + if err != nil { + t.Fatalf("buildNetPolicyPlan returned error: %v", err) + } + + flagsByDomain := make(map[string]uint8, len(plan.dnsAllowRules)) + for _, r := range plan.dnsAllowRules { + flagsByDomain[r.domain] = r.value.Flags + } + + l7bit := uint8(netPolicyFlagL7Required) + l3bit := uint8(netPolicyFlagL3Allowed) + + if got := flagsByDomain["both.example.com"]; got != l7bit|l3bit { + t.Fatalf("both.example.com flags=%#x, want L7Required|L3Allowed (%#x)", got, l7bit|l3bit) + } + if got := flagsByDomain["l7only.example.com"]; got != l7bit { + t.Fatalf("l7only.example.com flags=%#x, want L7Required only (%#x)", got, l7bit) + } + if got := flagsByDomain["plain.example.com"]; got != 0 { + t.Fatalf("plain.example.com flags=%#x, want 0 (plain allow)", got) + } +} + func TestBuildNetPolicyPlanBlockAllKeepsDefaultDenyOutOnReplace(t *testing.T) { allowInternetAccess := false plan, err := buildNetPolicyPlan(MVMOptions{AllowInternetAccess: &allowInternetAccess}) diff --git a/CubeNet/src/cubevs.h b/CubeNet/src/cubevs.h index b0026d554..60bbf7284 100644 --- a/CubeNet/src/cubevs.h +++ b/CubeNet/src/cubevs.h @@ -36,6 +36,14 @@ #define MAX_DNS_NAME_LEN 256 #define DNS_POLICY_FLAG_LEARNING_ENABLED 1 #define NET_POLICY_FLAG_L7_REQUIRED 1 +/* Set alongside NET_POLICY_FLAG_L7_REQUIRED when a domain is present in BOTH + * a plain (L3) allow_out rule and an L7 rule. Tells dns_learn_response_ip to + * learn the plain /32 any-port entry in addition to the L7 (ip, port)/48 + * entries, so non-rule ports keep plain SNAT access while the rule's ports are + * L7-intercepted. Without it an L7 rule silently narrows a same-domain plain + * allow_out to only the rule's ports. + */ +#define NET_POLICY_FLAG_L3_ALLOWED 2 #define NSEC_PER_SEC 1000000000ULL /* L7 scheme values embedded in dns_allow_value / net_policy_value_v2 per-port diff --git a/CubeNet/src/dns_response.h b/CubeNet/src/dns_response.h index d8b2a3b20..133000279 100644 --- a/CubeNet/src/dns_response.h +++ b/CubeNet/src/dns_response.h @@ -129,12 +129,22 @@ static __always_inline void dns_learn_response_ip(__u32 ifindex, __u32 ip, __u32 if (!inner_map) return; - /* Plain (non-L7) allow: learn a single /32 (any-port) entry. */ - if (!(query->flags & NET_POLICY_FLAG_L7_REQUIRED)) { + /* Learn the plain /32 (any-port) entry whenever the domain is + * plain-allowed. For a non-L7 allow this is the whole policy; for a domain + * that is both plain-allowed and L7-ruled (NET_POLICY_FLAG_L3_ALLOWED) it + * coexists with the /48 L7 entries written below — a non-rule port falls + * back to this /32 (plain SNAT) while the rule's ports match the longer + * /48 prefix (L7 intercept) in classify_egress_flow. The L7/L3 marker bits + * are stripped so the entry reads as a plain allow; for a non-L7 query + * (flags==0) the mask is a no-op, so both cases share this one block. + */ + if (!(query->flags & NET_POLICY_FLAG_L7_REQUIRED) || + (query->flags & NET_POLICY_FLAG_L3_ALLOWED)) { struct lpm_key_v3 key = { .prefixlen = 32, .ip = ip, .port = 0 }; struct net_policy_value_v3 value = { .expires_at_ns = expires, - .flags = query->flags, + .flags = query->flags & + ~(__u8)(NET_POLICY_FLAG_L7_REQUIRED | NET_POLICY_FLAG_L3_ALLOWED), .scheme = L7_SCHEME_NONE, .key_prefixlen = 32, }; @@ -145,9 +155,12 @@ static __always_inline void dns_learn_response_ip(__u32 ifindex, __u32 ip, __u32 value.expires_at_ns = 0; } bpf_map_update_elem(inner_map, &key, &value, BPF_ANY); - return; } + /* A non-L7 allow is fully handled by the /32 entry above. */ + if (!(query->flags & NET_POLICY_FLAG_L7_REQUIRED)) + return; + /* L7 allow: one exact (ip, port)/48 entry per (port, scheme). * * The loop uses a fixed trip count with the bound as a guard inside the diff --git a/docs/guide/security-proxy.md b/docs/guide/security-proxy.md index 357975677..dc343060d 100644 --- a/docs/guide/security-proxy.md +++ b/docs/guide/security-proxy.md @@ -172,6 +172,16 @@ port=8443, scheme="https"`, the deny still matches (first match wins). List the more specific allow rule first to grant the exception. +The same domain may appear in both the L3/L4 `allow_out` list and an +L7 `rule`. In that case the two coexist: the domain's learned IPs get +a plain `/32` any-port allow entry (so non-rule ports keep ordinary +L3/L4 SNAT access) **and** an L7 `(ip, port)` entry per rule port (so +those ports are steered through CubeEgress). The more specific +`(ip, port)` match wins for the rule's ports; the `/32` covers the +rest. Put another way, adding an L7 rule for a domain already in +`allow_out` does **not** remove its plain L3 access — it only adds L7 +interception on the rule's ports. + ::: tip Single-level vs multi-level subdomain `*.example.com` matches **all** subdomains regardless of label depth. To allow only single-level subdomains (e.g. `www`, diff --git a/docs/zh/guide/security-proxy.md b/docs/zh/guide/security-proxy.md index 28cdf3bac..2284a36b5 100644 --- a/docs/zh/guide/security-proxy.md +++ b/docs/zh/guide/security-proxy.md @@ -156,6 +156,15 @@ scheme="https"` 同时存在时，deny 仍会命中 `api.example.com:8443`，最终结果由规则顺序（先命中先生效）决定。 若想放行这个例外，把更具体的 allow 规则放在前面。 +同一个域名可以同时出现在 L3/L4 的 `allow_out` 列表和某条 L7 +`rule` 里，此时两者**并存**：该域名学到的 IP 会同时得到一条纯 +`/32` 任意端口放行条目（让非 rule 端口保留普通的 L3/L4 SNAT 访 +问）**和**每个 rule 端口一条 L7 `(ip, port)` 条目（让这些端口被 +送进 CubeEgress 代理）。更精确的 `(ip, port)` 匹配对 rule 端口生 +效，`/32` 兜底其余端口。换句话说，给一个已在 `allow_out` 里的域 +名再加 L7 rule，**不会**移除它原有的纯 L3 放行 —— 只是在 rule 的 +端口上叠加了 L7 拦截。 + ::: tip 单层 vs 多层子域 `*.example.com` 不区分子域层数，**所有**结尾命中的子域都算。 若想只放行单层子域（如 `www`、`api`），需要为不希望放过的嵌套 From 2862ca9277971e681bf5c49b4b68c4db921d6e59 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 17 Aug 2026 17:31:38 +0800 Subject: [PATCH 19/31] fix(cubenet): recover from half-pinned policy map generation on startup persistentPolicyGenerationExists() returned an error when exactly one of allow_out_v3 / dns_allow_v2 was pinned, and Init returned it before the recovery defer was registered (that defer is only installed on the generationExists==false path, which an early error return skips). A boot that died between pinning the two maps left an orphan pin that survived every restart, permanently bricking the node until an operator manually unlinked bpffs. Treat the half-pinned set as an incomplete generation instead: log a warning, remove the orphan pin, and report "no generation" so Init rebuilds a consistent pair and re-migrates from the legacy pins (still present in this scenario, since the orphan is an empty not-yet-migrated map). Adds regression tests for the recovery. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- CubeNet/cubevs/miscs.go | 38 ++++++++++++----- CubeNet/cubevs/miscs_test.go | 83 ++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 11 deletions(-) diff --git a/CubeNet/cubevs/miscs.go b/CubeNet/cubevs/miscs.go index f3df474d5..6f1125bc1 100644 --- a/CubeNet/cubevs/miscs.go +++ b/CubeNet/cubevs/miscs.go @@ -3,6 +3,7 @@ package cubevs import ( "errors" "fmt" + "log" "os" "strings" @@ -271,23 +272,38 @@ func attachTCFilter(progName string, ifindex uint32, direction TCDirection) erro return nil } +// persistentPolicyGenerationExists reports whether a complete current-generation +// policy map set (both allow_out_v3 and dns_allow_v2) is pinned on bpffs. +// +// A half-pinned set (exactly one of the two) means a previous boot died between +// pinning the two maps. That orphan is an empty, not-yet-migrated map, so we +// remove it and report "no generation": Init then rebuilds a consistent pair +// and re-migrates from the legacy pins (which are still present in this +// scenario). Returning an error here instead would permanently brick startup — +// Init's recovery defer is only registered on the generationExists==false path, +// which an early error return skips, so the orphan would survive every restart. func persistentPolicyGenerationExists() (bool, error) { - allowExists := false - if _, err := os.Stat(pinPath(MapNameAllowOutV3)); err == nil { - allowExists = true - } else if !errors.Is(err, os.ErrNotExist) { + allowExists, err := pinnedMapExists(MapNameAllowOutV3) + if err != nil { return false, err } - dnsExists := false - if _, err := os.Stat(pinPath(MapNameDNSAllowV2)); err == nil { - dnsExists = true - } else if !errors.Is(err, os.ErrNotExist) { + dnsExists, err := pinnedMapExists(MapNameDNSAllowV2) + if err != nil { return false, err } - if allowExists != dnsExists { - return false, fmt.Errorf("incomplete policy map generation: %s exists=%t, %s exists=%t", MapNameAllowOutV3, allowExists, MapNameDNSAllowV2, dnsExists) //nolint:err113 + if allowExists == dnsExists { + return allowExists, nil + } + + log.Printf("cubevs: incomplete policy map generation (%s exists=%t, %s exists=%t); removing orphaned pin and rebuilding", + MapNameAllowOutV3, allowExists, MapNameDNSAllowV2, dnsExists) + if allowExists { + _ = os.Remove(pinPath(MapNameAllowOutV3)) // NOCC:Path Traversal() + } + if dnsExists { + _ = os.Remove(pinPath(MapNameDNSAllowV2)) // NOCC:Path Traversal() } - return allowExists, nil + return false, nil } // Init should be called once before invoking any other CubeVS APIs. diff --git a/CubeNet/cubevs/miscs_test.go b/CubeNet/cubevs/miscs_test.go index 0f2bc8dfa..6cffb502c 100644 --- a/CubeNet/cubevs/miscs_test.go +++ b/CubeNet/cubevs/miscs_test.go @@ -1,6 +1,8 @@ package cubevs import ( + "errors" + "os" "testing" "github.com/cilium/ebpf" @@ -109,3 +111,84 @@ func TestPopulateDNSTailCallsEmptyWhenObjectDoesNotOwnDNSPrograms(t *testing.T) t.Fatalf("contents=%#v, want empty", contents) } } + +// TestPersistentPolicyGenerationRecoversHalfPinned covers the F1 availability +// regression: a boot that died between pinning allow_out_v3 and dns_allow_v2 +// leaves a half-pinned generation. persistentPolicyGenerationExists must treat +// it as an incomplete generation — remove the orphan pin and report "no +// generation" so Init rebuilds a consistent pair — rather than returning an +// error that permanently bricks startup (Init's recovery defer is only +// registered on the generationExists==false path, which an early error return +// skips). +func TestPersistentPolicyGenerationRecoversHalfPinned(t *testing.T) { + cases := []struct { + name string + pinAllow bool // pin allow_out_v3 + pinDNS bool // pin dns_allow_v2 + wantExist bool + }{ + {"neither pinned (fresh)", false, false, false}, + {"both pinned (complete)", true, true, true}, + {"only allow_out_v3 pinned (orphan)", true, false, false}, + {"only dns_allow_v2 pinned (orphan)", false, true, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mountBpffs(t) + if tc.pinAllow { + m := newAllowOutV3OuterMap(t) + if err := m.Pin(pinPath(MapNameAllowOutV3)); err != nil { + t.Fatalf("pin %s: %v", MapNameAllowOutV3, err) + } + } + if tc.pinDNS { + m := newDNSAllowOuterMap(t) + if err := m.Pin(pinPath(MapNameDNSAllowV2)); err != nil { + t.Fatalf("pin %s: %v", MapNameDNSAllowV2, err) + } + } + + exists, err := persistentPolicyGenerationExists() + if err != nil { + t.Fatalf("returned error (would brick startup): %v", err) + } + if exists != tc.wantExist { + t.Fatalf("exists=%t, want %t", exists, tc.wantExist) + } + + // After the call the two pins must be consistent: both present + // (complete) or both absent (fresh / orphan recovered). + allowExists, err := pinnedMapExists(MapNameAllowOutV3) + if err != nil { + t.Fatalf("pinnedMapExists(%s): %v", MapNameAllowOutV3, err) + } + dnsExists, err := pinnedMapExists(MapNameDNSAllowV2) + if err != nil { + t.Fatalf("pinnedMapExists(%s): %v", MapNameDNSAllowV2, err) + } + if allowExists != dnsExists { + t.Fatalf("pins still inconsistent after recovery: %s=%t %s=%t", + MapNameAllowOutV3, allowExists, MapNameDNSAllowV2, dnsExists) + } + }) + } +} + +// TestPersistentPolicyGenerationRemovesOrphanPin asserts the orphan pin is +// actually unlinked (not merely reported as absent) so the next fresh load +// rebuilds a consistent pair. +func TestPersistentPolicyGenerationRemovesOrphanPin(t *testing.T) { + mountBpffs(t) + + orphan := newAllowOutV3OuterMap(t) + if err := orphan.Pin(pinPath(MapNameAllowOutV3)); err != nil { + t.Fatalf("pin %s: %v", MapNameAllowOutV3, err) + } + + if _, err := persistentPolicyGenerationExists(); err != nil { + t.Fatalf("returned error (would brick startup): %v", err) + } + if _, err := os.Stat(pinPath(MapNameAllowOutV3)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("orphan %s pin still present after recovery: err=%v", MapNameAllowOutV3, err) + } +} From 1724f76e0319cbfcd415c58a82c6722a9f75484e Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 17 Aug 2026 18:38:15 +0800 Subject: [PATCH 20/31] fix(cubenet): let plain allow_out coexist with L7 rules for static IP hosts The static-IP path (mergeAllowOutWithL7) had the same subsumption the domain path had: a host present in both plain allow_out and an L7 rule collapsed to L7-only, so the IP was admitted into allow_out_v3 only on the rule's ports and lost plain L3 access on every other port under deny-all. Reuse netPolicyFlagL3Allowed. mergeAllowOutWithL7 sets it when a static host key is present in both the plain allow_out base set and an L7 rule set; populateAllowOutInnerMap then writes the plain /32 any-port entry alongside the L7 /48 entries (the /48 longest-prefix match wins for the rule's ports, the /32 covers the rest). expandedAllowOutEntryCount accounts for the extra /32, and dump surfaces l3_allowed. Subnet allow_out + single-IP L7 rule already coexisted via distinct LPM keys; only the exact same single-IP key was collapsing. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- CubeNet/cubevs/dump.go | 6 ++ CubeNet/cubevs/netpolicy.go | 31 ++++++++++- CubeNet/cubevs/netpolicy_test.go | 96 +++++++++++++++++++++++++++++++- 3 files changed, 129 insertions(+), 4 deletions(-) diff --git a/CubeNet/cubevs/dump.go b/CubeNet/cubevs/dump.go index e70fa1513..22288fb98 100644 --- a/CubeNet/cubevs/dump.go +++ b/CubeNet/cubevs/dump.go @@ -117,6 +117,7 @@ type PolicyEntryDump struct { ExpiresIn string `json:"expires_in,omitempty"` Expired bool `json:"expired"` L7Required bool `json:"l7_required"` + L3Allowed bool `json:"l3_allowed"` Flags uint8 `json:"flags"` Static bool `json:"static"` // L7 port tuples inherited from a matched dns_allow_value at DNS-learn @@ -148,6 +149,7 @@ type DNSAllowRuleDump struct { Domain string `json:"domain"` Wildcard bool `json:"wildcard"` L7Required bool `json:"l7_required"` + L3Allowed bool `json:"l3_allowed"` Flags uint8 `json:"flags"` NameLen uint32 `json:"name_len"` Prefixlen uint32 `json:"prefixlen"` @@ -168,6 +170,7 @@ type DNSQueryTrackDump struct { ExpiresIn string `json:"expires_in"` Expired bool `json:"expired"` L7Required bool `json:"l7_required"` + L3Allowed bool `json:"l3_allowed"` Flags uint8 `json:"flags"` // L7 port tuples copied verbatim from the matched dns_allow_value so the // response handler can rebuild net_policy_value_v3 without a second @@ -529,6 +532,7 @@ func dumpPolicyInnerMap(innerMapID uint32, now uint64) ([]PolicyEntryDump, error CIDR: dumpLPMCIDRV3(key), Expired: netPolicyValueV3Expired(value, now), L7Required: value.Flags&uint8(netPolicyFlagL7Required) != 0, + L3Allowed: value.Flags&uint8(netPolicyFlagL3Allowed) != 0, Flags: value.Flags, Static: value.ExpiresAtNS == 0, } @@ -706,6 +710,7 @@ func dumpDNSQueryTrack(opts DumpOptions, now uint64) (any, error) { ExpiresIn: remainingDuration(value.ExpiresAtNS, now), Expired: value.ExpiresAtNS <= now, L7Required: value.Flags&uint8(netPolicyFlagL7Required) != 0, + L3Allowed: value.Flags&uint8(netPolicyFlagL3Allowed) != 0, Flags: value.Flags, PortCount: value.PortCount, Ports: l7PortEntriesToDump(value.Ports[:], value.PortCount), @@ -876,6 +881,7 @@ func dumpDNSAllowRule(key dnsAllowKey, value dnsAllowValue) (DNSAllowRuleDump, e Domain: domain, Wildcard: wildcard, L7Required: value.Flags&uint8(netPolicyFlagL7Required) != 0, + L3Allowed: value.Flags&uint8(netPolicyFlagL3Allowed) != 0, Flags: value.Flags, NameLen: value.NameLen, Prefixlen: key.Prefixlen, diff --git a/CubeNet/cubevs/netpolicy.go b/CubeNet/cubevs/netpolicy.go index cf4d9ca82..f312c553e 100644 --- a/CubeNet/cubevs/netpolicy.go +++ b/CubeNet/cubevs/netpolicy.go @@ -646,8 +646,12 @@ func l7DNSDomainNames(rules []dnsAllowRule) []string { } // mergeAllowOutWithL7 combines the non-L7 base entries with the L7 entries. -// When the same LPM key appears in both, the L7 version wins (its flags and -// port set describe a strictly wider policy). +// When the same LPM key appears in both, the L7 version's flags and port set +// win, and the merged entry is also marked netPolicyFlagL3Allowed so +// populateAllowOutInnerMap writes the plain /32 any-port entry alongside the +// L7 /48 entries — otherwise an L7 rule would silently narrow a same-host +// plain allow_out to only the rule's ports. (Domain hosts get the same +// treatment in mergeDNSAllowRules.) func mergeAllowOutWithL7(base, l7 []allowOutPolicyEntry) []allowOutPolicyEntry { if len(l7) == 0 { return base @@ -660,7 +664,7 @@ func mergeAllowOutWithL7(base, l7 []allowOutPolicyEntry) []allowOutPolicyEntry { } for _, e := range l7 { if idx, ok := byKey[e.key]; ok { - out[idx].flags |= e.flags + out[idx].flags |= e.flags | netPolicyFlagL3Allowed out[idx].ports = e.ports continue } @@ -743,6 +747,10 @@ func expandedAllowOutEntryCount(entries []allowOutPolicyEntry) int { if n == 0 { n = defaultPorts } + if e.flags&netPolicyFlagL3Allowed != 0 { + // The host also gets a plain /32 any-port entry (coexistence). + n++ + } total += n continue } @@ -1017,6 +1025,23 @@ func populateAllowOutInnerMap(outerMap *ebpf.Map, ifindex uint32, entries []allo return fmt.Errorf("inner map update failed: %w, cidr: %s", uerr, entry.source) } } + + // Coexistence: this host is also in plain allow_out, so write a + // plain /32 any-port entry alongside the /48 L7 entries. The /48 + // (longest-prefix) match wins for the rule's ports; the /32 covers + // all other ports via plain SNAT. Strip the L7/L3 marker bits so + // the entry reads as a plain allow. + if entry.flags&netPolicyFlagL3Allowed != 0 { + plainKey := lpmKeyV3{Prefixlen: entry.key.Prefixlen, IP: entry.key.IP, Port: 0} + plainVal := netPolicyValueV3{ + Flags: entry.flags &^ (netPolicyFlagL7Required | netPolicyFlagL3Allowed), + Scheme: L7SchemeNone, + KeyPrefixlen: uint8(plainKey.Prefixlen), + } + if uerr := inner.Update(&plainKey, &plainVal, ebpf.UpdateAny); uerr != nil { + return fmt.Errorf("inner map update failed: %w, cidr: %s", uerr, entry.source) + } + } continue } diff --git a/CubeNet/cubevs/netpolicy_test.go b/CubeNet/cubevs/netpolicy_test.go index f6466beaa..01bbf1b76 100644 --- a/CubeNet/cubevs/netpolicy_test.go +++ b/CubeNet/cubevs/netpolicy_test.go @@ -216,7 +216,10 @@ func TestBuildNetPolicyPlanDeduplicatesAndMergesFlags(t *testing.T) { if got, want := len(plan.allowOutEntries), 1; got != want { t.Fatalf("len(plan.allowOutEntries)=%d, want %d", got, want) } - if got, want := plan.allowOutEntries[0].flags, uint8(netPolicyFlagL7Required); got != want { + // 198.51.100.1 is in BOTH plain allow_out and an L7 rule, so the merged + // static entry carries L7Required|L3Allowed (the L3 bit keeps the plain /32 + // any-port entry alongside the L7 /48 entries). + if got, want := plan.allowOutEntries[0].flags, uint8(netPolicyFlagL7Required|netPolicyFlagL3Allowed); got != want { t.Fatalf("allow out flags=%d, want %d", got, want) } if got, want := len(plan.dnsAllowRules), 1; got != want { @@ -278,6 +281,42 @@ func TestBuildNetPolicyPlanL3AllowedCoexistence(t *testing.T) { } } +// TestBuildNetPolicyPlanStaticL3AllowedCoexistence is the static-IP +// counterpart of TestBuildNetPolicyPlanL3AllowedCoexistence: a host present in +// both plain allow_out and an L7 rule must be marked netPolicyFlagL3Allowed in +// allowOutEntries, so populateAllowOutInnerMap writes the plain /32 any-port +// entry alongside the L7 /48 entries. +func TestBuildNetPolicyPlanStaticL3AllowedCoexistence(t *testing.T) { + allowOut := []string{"198.51.100.10", "198.51.100.20"} + l7AllowOut := []L7Target{ + {Host: "198.51.100.10", Port: 8443, Scheme: L7SchemeHTTPS}, + {Host: "198.51.100.30", Port: 9090, Scheme: L7SchemeHTTP}, + } + + plan, err := buildNetPolicyPlan(MVMOptions{AllowOut: &allowOut, L7AllowOut: &l7AllowOut}) + if err != nil { + t.Fatalf("buildNetPolicyPlan returned error: %v", err) + } + + flagsByIP := make(map[string]uint8, len(plan.allowOutEntries)) + for _, e := range plan.allowOutEntries { + flagsByIP[uint32ToIP(e.key.IP).String()] = e.flags + } + + l7bit := uint8(netPolicyFlagL7Required) + l3bit := uint8(netPolicyFlagL3Allowed) + + if got := flagsByIP["198.51.100.10"]; got != l7bit|l3bit { + t.Fatalf("198.51.100.10 flags=%#x, want L7Required|L3Allowed (%#x)", got, l7bit|l3bit) + } + if got := flagsByIP["198.51.100.30"]; got != l7bit { + t.Fatalf("198.51.100.30 flags=%#x, want L7Required only (%#x)", got, l7bit) + } + if got := flagsByIP["198.51.100.20"]; got != 0 { + t.Fatalf("198.51.100.20 flags=%#x, want 0 (plain allow)", got) + } +} + func TestBuildNetPolicyPlanBlockAllKeepsDefaultDenyOutOnReplace(t *testing.T) { allowInternetAccess := false plan, err := buildNetPolicyPlan(MVMOptions{AllowInternetAccess: &allowInternetAccess}) @@ -1046,6 +1085,61 @@ func TestPopulateAllowOutStaticV3OverExactStaticMergesFlags(t *testing.T) { } } +// TestPopulateAllowOutStaticL3AlsoWritesPlainAndL7Entries is the static-IP +// counterpart of the DNS-learn coexistence test: a host in both plain +// allow_out and an L7 rule must be written as BOTH a plain /32 any-port entry +// (marker bits stripped) and the L7 /48 entries. +func TestPopulateAllowOutStaticL3AlsoWritesPlainAndL7Entries(t *testing.T) { + outer := newAllowOutV3OuterMap(t) + ifindex := uint32(310) + inner := attachAllowOutV3Inner(t, outer, ifindex) + ip := mustParseCIDRForTest(t, "192.0.2.120").IP + + entries := []allowOutPolicyEntry{{ + key: lpmKey{Prefixlen: 32, IP: ip}, + flags: uint8(netPolicyFlagL7Required) | uint8(netPolicyFlagL3Allowed), + ports: []l7PortEntry{{Port: htonsPort(8443), Scheme: L7SchemeHTTPS}}, + }} + if err := populateAllowOutInnerMap(outer, ifindex, entries); err != nil { + t.Fatalf("populateAllowOutInnerMap: %v", err) + } + + // The /48 L7 entry for the rule port is present and intercepted. + l7 := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(8443)}) + if l7.KeyPrefixlen != 48 || l7.Flags&uint8(netPolicyFlagL7Required) == 0 { + t.Fatalf("/48 = %+v, want L7 /48 entry", l7) + } + if l7.Scheme != L7SchemeHTTPS { + t.Fatalf("/48 scheme=%d, want https", l7.Scheme) + } + + // The /32 plain entry is present for everything else, with marker bits + // stripped and static (zero) expiry. + plain := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 32, IP: ip, Port: 0}) + if plain.KeyPrefixlen != 32 { + t.Fatalf("/32 KeyPrefixlen=%d, want 32", plain.KeyPrefixlen) + } + if plain.Flags != 0 { + t.Fatalf("/32 flags=%#x, want 0 (plain, marker bits stripped)", plain.Flags) + } + if plain.Scheme != L7SchemeNone { + t.Fatalf("/32 scheme=%d, want none", plain.Scheme) + } + if plain.ExpiresAtNS != 0 { + t.Fatalf("/32 ExpiresAtNS=%d, want 0 (static)", plain.ExpiresAtNS) + } + + // A lookup for a non-rule port must fall back to the /32 plain entry, not + // match a /48 L7 entry (this is how classify_egress_flow admits it as SNAT). + fallback := mustLookupV3(t, inner, lpmKeyV3{Prefixlen: 48, IP: ip, Port: htonsPort(9090)}) + if fallback.KeyPrefixlen == 48 && fallback.Flags&uint8(netPolicyFlagL7Required) != 0 { + t.Fatalf("non-rule port unexpectedly matched a /48 L7 entry: %+v", fallback) + } + if fallback.KeyPrefixlen != 32 { + t.Fatalf("non-rule port fell back to KeyPrefixlen=%d, want 32 (plain)", fallback.KeyPrefixlen) + } +} + // TestPopulateAllowOutPlainStaticOverwritesLearnedUnconditionally pins the // plain (non-L7) path's NO-merge semantics: a static /32 overwrites a learned // /32 outright — last write wins for flags, scheme AND expiry, with no From f5abd1255df556b6fc84a4b4d36cb9f83921652d Mon Sep 17 00:00:00 2001 From: hankzhou Date: Mon, 17 Aug 2026 19:28:50 +0800 Subject: [PATCH 21/31] fix(cubenet): keep plain L3 access for L7 hosts covered by a wildcard allow_out An L7 rule host covered by a leading-"*." plain allow_out domain was still denied on its non-rule ports under deny-all: the exact rule shadows the wildcard in the DNS LPM match, so the host's learned IP only got the L7 /48 entry and no plain /32 fallback. Unify the L3Allowed marking into a single userspace coverage pass. l7DomainHasPlainCover reports whether an L7 host is covered by a plain allow_out domain (exact same-host, or a leading-"*." wildcard subdomain at any depth, apex excluded); markL3AllowedByPlainCover sets the flag on each covered L7 rule. This replaces the same-key-only marking in mergeDNSAllowRules and extends it to wildcard coverage. No eBPF change: the DNS-learn path already writes /32 + /48 when L3Allowed is set. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- CubeNet/cubevs/netpolicy.go | 62 ++++++++++++++++++++++++++++---- CubeNet/cubevs/netpolicy_test.go | 48 +++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 7 deletions(-) diff --git a/CubeNet/cubevs/netpolicy.go b/CubeNet/cubevs/netpolicy.go index f312c553e..fb6099635 100644 --- a/CubeNet/cubevs/netpolicy.go +++ b/CubeNet/cubevs/netpolicy.go @@ -606,6 +606,11 @@ func buildNetPolicyPlan(opts MVMOptions) (*netPolicyPlan, error) { return nil, err } dnsAllowRules = mergeDNSAllowRules(dnsAllowRules, l7DNSRules) + // Mark each L7 domain rule that is also covered by a plain (non-L7) + // allow_out domain — an exact same-host entry or a leading-"*." wildcard — + // so the datapath keeps the plain /32 L3 access alongside the L7 /48 + // interception for that host. + markL3AllowedByPlainCover(dnsAllowRules, dnsAllowDomains) var denyOutEntries []denyOutPolicyEntry if opts.AllowInternetAccess != nil && !*opts.AllowInternetAccess { @@ -675,12 +680,10 @@ func mergeAllowOutWithL7(base, l7 []allowOutPolicyEntry) []allowOutPolicyEntry { } // mergeDNSAllowRules combines the non-L7 domain rules with the L7 domain rules. -// Same-key entries merge flags (OR) and adopt the L7 rule's port set. When the -// same domain is present in both (plain allow_out AND an L7 rule), the merged -// entry is also marked netPolicyFlagL3Allowed so the datapath learns the plain -// /32 any-port entry alongside the L7 /48 entries — otherwise the L7 rule -// would silently narrow the same-domain plain allow_out to only the rule's -// ports. +// Same-key entries merge flags (OR) and adopt the L7 rule's port set. The +// netPolicyFlagL3Allowed marker (plain-L3 + L7 coexistence) is applied +// separately by markL3AllowedByPlainCover, which handles both the exact +// same-host case and a leading-"*." wildcard allow_out covering the L7 host. func mergeDNSAllowRules(base, l7 []dnsAllowRule) []dnsAllowRule { if len(l7) == 0 { return base @@ -693,7 +696,7 @@ func mergeDNSAllowRules(base, l7 []dnsAllowRule) []dnsAllowRule { } for _, r := range l7 { if idx, ok := byKey[r.key]; ok { - out[idx].value.Flags |= r.value.Flags | netPolicyFlagL3Allowed + out[idx].value.Flags |= r.value.Flags out[idx].value.PortCount = r.value.PortCount out[idx].value.Ports = r.value.Ports continue @@ -704,6 +707,51 @@ func mergeDNSAllowRules(base, l7 []dnsAllowRule) []dnsAllowRule { return out } +// l7DomainHasPlainCover reports whether an L7 rule host is covered by a plain +// (non-L7) allow_out domain — either an exact same-host entry or a +// leading-"*." wildcard whose base the host is a subdomain of. Matching follows +// the same semantics as makeDNSAllowRule / domain_match: case-insensitive, a +// trailing dot is ignored, and "*.base" covers any-depth subdomains of base but +// not the apex itself. +func l7DomainHasPlainCover(host string, plainDomains []string) bool { + h := strings.ToLower(strings.TrimSuffix(host, ".")) + for _, raw := range plainDomains { + d := strings.ToLower(strings.TrimSuffix(raw, ".")) + if strings.HasPrefix(d, "*.") { + base := d[2:] + if h != base && strings.HasSuffix(h, "."+base) { + return true + } + continue + } + if h == d { + return true + } + } + return false +} + +// markL3AllowedByPlainCover sets netPolicyFlagL3Allowed on each L7 dns_allow +// rule whose host is also covered by a plain (non-L7) allow_out domain. This +// is what lets a host keep its plain /32 L3 access (SNAT on non-rule ports) +// alongside the L7 /48 interception on the rule's ports — both when the host +// appears verbatim in allow_out and when a leading-"*." wildcard covers it +// (the exact rule otherwise shadows the wildcard in the DNS LPM match, which +// would silently deny the host's non-rule ports under deny-all). +func markL3AllowedByPlainCover(rules []dnsAllowRule, plainDomains []string) { + for i := range rules { + if rules[i].value.Flags&uint8(netPolicyFlagL7Required) == 0 { + continue + } + if rules[i].value.Flags&uint8(netPolicyFlagL3Allowed) != 0 { + continue + } + if l7DomainHasPlainCover(rules[i].domain, plainDomains) { + rules[i].value.Flags |= uint8(netPolicyFlagL3Allowed) + } + } +} + func appendDenyOutPolicyEntries(dst, src []denyOutPolicyEntry) []denyOutPolicyEntry { if len(src) == 0 { return dst diff --git a/CubeNet/cubevs/netpolicy_test.go b/CubeNet/cubevs/netpolicy_test.go index 01bbf1b76..ffa615aeb 100644 --- a/CubeNet/cubevs/netpolicy_test.go +++ b/CubeNet/cubevs/netpolicy_test.go @@ -317,6 +317,54 @@ func TestBuildNetPolicyPlanStaticL3AllowedCoexistence(t *testing.T) { } } +// TestBuildNetPolicyPlanWildcardPlainCoversL7Host covers the wildcard L3 +// fallback: an L7 rule host covered by a leading-"*." plain allow_out domain +// must be marked netPolicyFlagL3Allowed, so the host keeps plain /32 L3 access +// on non-rule ports even though the exact rule shadows the wildcard in the DNS +// LPM match. Apex and unrelated domains must NOT be marked. +func TestBuildNetPolicyPlanWildcardPlainCoversL7Host(t *testing.T) { + l7bit := uint8(netPolicyFlagL7Required) + l3bit := uint8(netPolicyFlagL3Allowed) + + cases := []struct { + name string + allowOut []string + l7Host string + wantFlags uint8 + }{ + {"wildcard covers subdomain", []string{"*.qq.com"}, "a.qq.com", l7bit | l3bit}, + {"wildcard covers deep subdomain", []string{"*.qq.com"}, "a.b.qq.com", l7bit | l3bit}, + {"wildcard does not cover apex", []string{"*.qq.com"}, "qq.com", l7bit}, + {"wildcard does not cover unrelated", []string{"*.qq.com"}, "other.com", l7bit}, + {"case + trailing dot normalized", []string{"*.QQ.com."}, "A.QQ.com", l7bit | l3bit}, + {"exact plain covers same host", []string{"a.qq.com"}, "a.qq.com", l7bit | l3bit}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + allowOut := tc.allowOut + l7AllowOut := []L7Target{{Host: tc.l7Host, Port: 8443, Scheme: L7SchemeHTTPS}} + plan, err := buildNetPolicyPlan(MVMOptions{AllowOut: &allowOut, L7AllowOut: &l7AllowOut}) + if err != nil { + t.Fatalf("buildNetPolicyPlan: %v", err) + } + + want := strings.ToLower(strings.TrimSuffix(tc.l7Host, ".")) + got, found := uint8(0), false + for _, r := range plan.dnsAllowRules { + if strings.ToLower(strings.TrimSuffix(r.domain, ".")) == want { + got, found = r.value.Flags, true + } + } + if !found { + t.Fatalf("L7 host %s not found in dnsAllowRules", tc.l7Host) + } + if got != tc.wantFlags { + t.Fatalf("%s flags=%#x, want %#x", tc.l7Host, got, tc.wantFlags) + } + }) + } +} + func TestBuildNetPolicyPlanBlockAllKeepsDefaultDenyOutOnReplace(t *testing.T) { allowInternetAccess := false plan, err := buildNetPolicyPlan(MVMOptions{AllowInternetAccess: &allowInternetAccess}) From 16b53cbcb9645a7e2d8ef720640c616155089c59 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Tue, 18 Aug 2026 10:07:02 +0800 Subject: [PATCH 22/31] test(cubevs): run cubevs module unit tests in CI and run.sh CubeNet/cubevs is a separate Go module whose own tests (custom_port, dns_learn, migration, netpolicy, dump, egress_policy, l7_mark, tcp_state) were never compiled or run: the CI mapped CubeNet/* to the cubelet target (go test ./pkg/... under Cubelet), and run.sh's cubelet-network only built cubevs's generated code before running Cubelet's network/runtime tests. Add a cubevs-test make target and a cubevs component in run.sh and the unit-test workflow (CubeNet/cubevs/* now triggers both cubevs and cubelet). The target regenerates the BPF objects (make gen) then runs go test ./... (no -coverprofile; the builder lacks covdata). The eBPF-loading tests need a privileged root builder for bpf()/bpffs, so builder-run gains a BUILDER_USER override and cubevs-test runs privileged as root. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- .github/workflows/unit-test-check.yml | 7 +++++++ Makefile | 19 ++++++++++++++++++- tests/unittest/run.sh | 6 ++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-test-check.yml b/.github/workflows/unit-test-check.yml index 77a1f33e4..a7c59e9c5 100644 --- a/.github/workflows/unit-test-check.yml +++ b/.github/workflows/unit-test-check.yml @@ -182,6 +182,12 @@ jobs: CubeShim/*) add_component "CubeShim" ;; + CubeNet/cubevs/*) + # cubevs is its own Go module with its own unit tests; run + # those, and keep cubelet (it embeds the cubevs runtime). + add_component "cubevs" + add_component "cubelet" + ;; CubeNet/*) # Embedded network runtime lives under Cubelet; CubeNet eBPF # helper changes still need the cubelet test set. @@ -247,6 +253,7 @@ jobs: elif . == "cubedb" then "cubedb-test" elif . == "cube-lifecycle-manager" then "cube-lifecycle-manager-test" elif . == "cubelet" then "cubelet-pkg-test" + elif . == "cubevs" then "cubevs-test" elif . == "agent" then "agent-test" elif . == "hypervisor" then "hypervisor-test" else error("unsupported component: " + .) diff --git a/Makefile b/Makefile index 51068e845..2bb96fec7 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,10 @@ BUILDER_CONTAINER_HOME ?= /home/builder TMP_GIT_CREDENTIALS ?= /tmp/.cube-sandbox-builder-tmp-git-credentials BUILDER_CMD ?= bash BUILDER_RUN_EXTRA_MOUNTS ?= +# User the builder container runs as. Defaults to the host user so bind-mounted +# outputs stay host-writable; privileged test targets (e.g. cubevs-test, which +# loads eBPF / mounts bpffs) override this to 0:0 along with --privileged. +BUILDER_USER ?= $(UID):$(GID) ROOT_DIR := $(shell pwd) UID := $(shell id -u) GID := $(shell id -g) @@ -205,7 +209,7 @@ ifeq ($(strip $(BUILDER_CMD)),) $(error BUILDER_CMD must not be empty) endif docker run --rm -i \ - --user "$(UID):$(GID)" \ + --user "$(BUILDER_USER)" \ -e HOME=$(BUILDER_CONTAINER_HOME) \ -e CARGO_HOME=$(BUILDER_CONTAINER_HOME)/.cargo \ -e RUSTUP_HOME=/usr/local/rustup \ @@ -371,6 +375,19 @@ cube-lifecycle-manager-test: builder-image cubelet-pkg-test: builder-image $(MAKE) builder-run BUILDER_CMD='cd /workspace && IN_CUBE_SANDBOX_BUILDER=1 make cubecow-sdk && cd /workspace/Cubelet && go mod download && make proto && go test -short ./pkg/...' +# cubevs-test runs the CubeNet/cubevs module's own unit tests (dataplane policy, +# DNS learning, migration, dump, classify), which the cubelet targets never +# compile. It regenerates the BPF objects first (make gen) since the test files +# embed them, then runs the full module test set. The eBPF-loading tests need a +# privileged root container (CAP_BPF/CAP_SYS_ADMIN for bpf() and the bpffs +# mount), so this runs the builder privileged as root. Note: the generated .o +# files under CubeNet/cubevs become root-owned in the bind-mounted workspace; +# that is harmless in CI (fresh checkout) but may require sudo to clean locally. +# Use plain `go test ./...` (no -coverprofile): the builder lacks covdata. +.PHONY: cubevs-test +cubevs-test: builder-image + $(MAKE) builder-run BUILDER_USER=0:0 BUILDER_RUN_EXTRA_MOUNTS='--privileged' BUILDER_CMD='cd /workspace/CubeNet/cubevs && make gen && go test ./...' + .PHONY: agent-test agent-test: builder-image $(MAKE) builder-run BUILDER_CMD='cd /workspace/agent && make test' diff --git a/tests/unittest/run.sh b/tests/unittest/run.sh index 97a868c1b..4bcc118ed 100755 --- a/tests/unittest/run.sh +++ b/tests/unittest/run.sh @@ -88,6 +88,12 @@ WITH_TESTS=( # and need the generated CubeNet/cubevs code but no cubecow/CGO, so build cubevs # then run just the runtime package rather than the whole Cubelet suite. "cubelet-network|Go|0|make builder-run BUILDER_CMD='cd /workspace/CubeNet/cubevs && make gen && cd /workspace/Cubelet && go mod download && go test ./network/runtime/...'" + # cubevs: the CubeNet/cubevs module's OWN unit tests (dataplane policy, DNS + # learning, migration, dump, classify). cubelet-network builds cubevs's + # generated code but runs Cubelet's tests, so these never ran. cubevs-test + # regenerates the BPF objects (make gen) and runs the full module set in a + # privileged root builder (eBPF load + bpffs mount need the privilege). + "cubevs|Go|0|make cubevs-test" "cubecow|Go+CGO|0|make cubecow-test-native" "cube-lifecycle-manager|Go|0|make builder-run BUILDER_CMD='cd /workspace/cube-lifecycle-manager && go mod download && go test ./...'" "cube-api|Rust|0|make cube-api-test" From 2b2571da106c23d3bdded604cdf1a2a3df3b8c89 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Tue, 18 Aug 2026 10:07:19 +0800 Subject: [PATCH 23/31] test(cubevs): assert FLOW_HTTP verdict in classify_egress_flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verdict constants defined reject/SNAT/HTTPS but not HTTP, and the L7 exact-match cases only seeded HTTPS, so the plaintext HTTP interception verdict (FLOW_HTTP) — the primary new data path — was never asserted. Add flowVerdictHTTP=2 and TestClassifyEgressFlowExactL7MatchHTTP, which seeds a /48 entry with Scheme=http and asserts FLOW_HTTP (and a non-matching port still rejects). Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- CubeNet/cubevs/egress_policy_test.go | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/CubeNet/cubevs/egress_policy_test.go b/CubeNet/cubevs/egress_policy_test.go index 19fc1d267..16eb34562 100644 --- a/CubeNet/cubevs/egress_policy_test.go +++ b/CubeNet/cubevs/egress_policy_test.go @@ -13,6 +13,7 @@ const ( egressPolicyTestCaseLen = 16 flowVerdictReject = uint8(0) flowVerdictSNAT = uint8(1) + flowVerdictHTTP = uint8(2) flowVerdictHTTPS = uint8(3) ) @@ -205,6 +206,38 @@ func TestClassifyEgressFlowExactL7Match(t *testing.T) { } } +// TestClassifyEgressFlowExactL7MatchHTTP covers the plaintext HTTP interception +// verdict (FLOW_HTTP) — the primary new data path the HTTPS-only cases did not +// exercise. eBPF returns FLOW_HTTP for an L7 entry whose scheme is http. +func TestClassifyEgressFlowExactL7MatchHTTP(t *testing.T) { + env := loadEgressPolicyTestEnv(t) + ifindex := uint32(152) + allowInner, denyInner := env.attachInnerMaps(t, ifindex) + daddr := mustParseCIDRForTest(t, "192.0.2.22").IP + dport := htonsPort(8080) + allowKey := lpmKeyV3{Prefixlen: 48, IP: daddr, Port: dport} + allowValue := netPolicyValueV3{ + Flags: uint8(netPolicyFlagL7Required), + Scheme: L7SchemeHTTP, + } + if err := allowInner.Put(&allowKey, &allowValue); err != nil { + t.Fatalf("insert exact L7 HTTP allow rule: %v", err) + } + + denyAll := mustParseCIDRForTest(t, "0.0.0.0/0") + denyValue := uint32(1) + if err := denyInner.Put(&denyAll, &denyValue); err != nil { + t.Fatalf("insert deny-all rule: %v", err) + } + + if got := runEgressPolicyCase(t, env.program, ifindex, daddr, dport); got != flowVerdictHTTP { + t.Fatalf("exact-port verdict=%d, want FLOW_HTTP", got) + } + if got := runEgressPolicyCase(t, env.program, ifindex, daddr, htonsPort(80)); got != flowVerdictReject { + t.Fatalf("different-port verdict=%d, want FLOW_REJECT", got) + } +} + func TestClassifyEgressFlowL7UnknownSchemeFailsClosed(t *testing.T) { env := loadEgressPolicyTestEnv(t) ifindex := uint32(151) From fcdf0fa08b215e7bb2f569aeb1c26243e2122e91 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Tue, 18 Aug 2026 10:07:33 +0800 Subject: [PATCH 24/31] test(cubeegress): cover sandbox_gateway_ip_from_cidr arithmetic The TPROXY --on-ip target is derived from CUBE_SANDBOX_NETWORK_CIDR via sandbox_gateway_ip_from_cidr, but nothing asserted that arithmetic, so a wrong gateway IP would pass CI. Add cases for several CIDRs (including a non-/8-aligned mask and a host-bits-set address that must mask to the network before +1) and assert invalid CIDRs are rejected. The TPROXY rule content (--on-port / mark match) is already asserted by the atomic-install test from the fail-open fix. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- .../tests/cube-proxy-iptables-init_test.sh | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CubeEgress/tests/cube-proxy-iptables-init_test.sh b/CubeEgress/tests/cube-proxy-iptables-init_test.sh index 3ea5da030..6df11e1cc 100644 --- a/CubeEgress/tests/cube-proxy-iptables-init_test.sh +++ b/CubeEgress/tests/cube-proxy-iptables-init_test.sh @@ -221,4 +221,26 @@ if ( exit 1 fi +# --- sandbox_gateway_ip_from_cidr: TPROXY --on-ip = network address + 1 ------- +# The TPROXY target IP is derived from CUBE_SANDBOX_NETWORK_CIDR by this +# arithmetic; a bug here would silently steer intercepted traffic to a wrong +# gateway while CI stayed green. +[[ "$(sandbox_gateway_ip_from_cidr 192.168.0.0/18)" == "192.168.0.1" ]] +[[ "$(sandbox_gateway_ip_from_cidr 10.0.0.0/8)" == "10.0.0.1" ]] +[[ "$(sandbox_gateway_ip_from_cidr 172.16.0.0/12)" == "172.16.0.1" ]] +[[ "$(sandbox_gateway_ip_from_cidr 192.168.1.0/24)" == "192.168.1.1" ]] +# Non-/8-aligned mask. +[[ "$(sandbox_gateway_ip_from_cidr 10.128.0.0/9)" == "10.128.0.1" ]] +# Host bits set: masked off before +1 (gateway is the network address + 1, not +# the literal address + 1). +[[ "$(sandbox_gateway_ip_from_cidr 192.168.5.7/24)" == "192.168.5.1" ]] +# Invalid CIDRs are rejected (fatal exits the subshell, so a surviving subshell +# means the value was wrongly accepted). +for bad in "10.0.0.0/33" "10.0.0.0/0" "notacidr" "1.2.3.256/24" "10.0.0.0"; do + if ( sandbox_gateway_ip_from_cidr "${bad}" >/dev/null 2>&1 ); then + echo "invalid CIDR ${bad} was accepted" >&2 + exit 1 + fi +done + printf 'cube-proxy-iptables-init_test: PASS\n' From 1baa6cc17eb9e14bc790a4aa3608aff8c7d32850 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Tue, 18 Aug 2026 10:07:48 +0800 Subject: [PATCH 25/31] test(cubevs): cover allow_out migration success path and dump port rendering Migration had full success tests for DNS but only rollback tests for allow_out_v2 -> allow_out_v3, leaving the 16-byte net_policy_value_v2 (L7 flag + expires) -> /48 expansion upgrade path unproven. Add TestMigrateAllowOutMapOuterWithBpffs asserting an L7 entry expands to the default {80,443} /48 set (flag, per-port scheme, expiry preserved) and a plain entry becomes a /32. Also add TestL7PortEntriesToDumpRendersHostByteOrder: l7PortEntriesToDump must convert NBO ports to host order, so a dropped ntohs() would render the raw wire value instead of the user-facing port. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- CubeNet/cubevs/dump_test.go | 27 +++++++++ CubeNet/cubevs/migration_test.go | 97 ++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/CubeNet/cubevs/dump_test.go b/CubeNet/cubevs/dump_test.go index 4ac05eea5..4f5d849eb 100644 --- a/CubeNet/cubevs/dump_test.go +++ b/CubeNet/cubevs/dump_test.go @@ -170,3 +170,30 @@ func TestDumpSessionKey(t *testing.T) { t.Fatalf("ProtocolName=%q, want tcp", got.ProtocolName) } } + +// TestL7PortEntriesToDumpRendersHostByteOrder guards the NBO→host port +// conversion: ports are stored in network byte order on the datapath, and a +// regression that drops the ntohs() would render the raw wire value (e.g. +// 0xEB20) instead of the user-facing host-order port (8443). +func TestL7PortEntriesToDumpRendersHostByteOrder(t *testing.T) { + ports := []l7PortEntry{ + {Port: htonsPort(8443), Scheme: L7SchemeHTTPS}, + {Port: htonsPort(8080), Scheme: L7SchemeHTTP}, + } + + got := l7PortEntriesToDump(ports, 2) + if len(got) != 2 { + t.Fatalf("len(got)=%d, want 2", len(got)) + } + if got[0].Port != 8443 || got[0].Scheme != "https" { + t.Fatalf("entry[0]=%+v, want port 8443 scheme https", got[0]) + } + if got[1].Port != 8080 || got[1].Scheme != "http" { + t.Fatalf("entry[1]=%+v, want port 8080 scheme http", got[1]) + } + + // count == 0 renders nothing (default-set fallback is implicit). + if got := l7PortEntriesToDump(ports, 0); got != nil { + t.Fatalf("count=0 rendered %v, want nil", got) + } +} diff --git a/CubeNet/cubevs/migration_test.go b/CubeNet/cubevs/migration_test.go index ba11f3b5b..493b7be10 100644 --- a/CubeNet/cubevs/migration_test.go +++ b/CubeNet/cubevs/migration_test.go @@ -530,3 +530,100 @@ func TestMigrateDNSFailureKeepsAllowOutLegacyPin(t *testing.T) { } src.Close() } + +// TestMigrateAllowOutMapOuterWithBpffs covers the allow_out success path the +// rollback tests miss: a legacy allow_out_v2 outer (16-byte +// net_policy_value_v2 inner, flags+expires but no ports) is migrated into a +// fresh allow_out_v3 outer. An L7 entry expands to the default {80,443} /48 +// set — keeping the L7 flag, the per-port scheme, and the expiry — while a +// plain entry becomes a /32 any-port entry. +func TestMigrateAllowOutMapOuterWithBpffs(t *testing.T) { + mountBpffs(t) + ifindex := uint32(42) + + l7IP := mustParseCIDRForTest(t, "192.0.2.44/32").IP + plainIP := mustParseCIDRForTest(t, "192.0.2.45/32").IP + expires := uint64(999999999) + + // Legacy allow_out_v2 outer with a 16-byte net_policy_value_v2 inner, + // seeded with one L7 entry and one plain entry (both with an expiry). + legacyOuter := newAllowOutOuterMapWithValueSize(t, uint32(unsafe.Sizeof(netPolicyValueV2{}))) + legacyInner := newAllowOutLPMInner(t, uint32(unsafe.Sizeof(netPolicyValueV2{}))) + seed := []struct { + ip uint32 + flags uint8 + }{ + {l7IP, uint8(netPolicyFlagL7Required)}, + {plainIP, 0}, + } + for _, s := range seed { + key := lpmKey{Prefixlen: 32, IP: s.ip} + val := netPolicyValueV2{Flags: s.flags, ExpiresAtNS: expires} + if err := legacyInner.Update(&key, &val, ebpf.UpdateAny); err != nil { + t.Fatalf("seed legacy entry: %v", err) + } + } + if err := legacyOuter.Put(&ifindex, legacyInner); err != nil { + t.Fatalf("attach legacy inner: %v", err) + } + legacyInner.Close() + if err := legacyOuter.Pin(pinPath(MapNameAllowOutV2)); err != nil { + t.Fatalf("pin legacy %s: %v", MapNameAllowOutV2, err) + } + + // Fresh allow_out_v3 outer as the destination. + newOuter := newAllowOutV3OuterMap(t) + if err := newOuter.Pin(pinPath(MapNameAllowOutV3)); err != nil { + t.Fatalf("pin %s: %v", MapNameAllowOutV3, err) + } + + if err := migrateAllowOutMap(MapNameAllowOutV2); err != nil { + t.Fatalf("migrateAllowOutMap: %v", err) + } + + dest, err := lookupInnerMap(newOuter, ifindex) + if err != nil { + t.Fatalf("lookupInnerMap: %v", err) + } + defer dest.Close() + + // The L7 entry expands to the default {80/http, 443/https} /48 set. + for _, tc := range []struct { + port uint16 + scheme uint8 + }{ + {htonsPort(80), L7SchemeHTTP}, + {htonsPort(443), L7SchemeHTTPS}, + } { + key := lpmKeyV3{Prefixlen: 48, IP: l7IP, Port: tc.port} + var got netPolicyValueV3 + if err := dest.Lookup(&key, &got); err != nil { + t.Fatalf("migrated L7 /48 entry for port %d missing: %v", ntohsPort(tc.port), err) + } + if got.Flags&uint8(netPolicyFlagL7Required) == 0 { + t.Fatalf("port %d lost L7 flag: %#x", ntohsPort(tc.port), got.Flags) + } + if got.Scheme != tc.scheme { + t.Fatalf("port %d scheme=%d, want %d", ntohsPort(tc.port), got.Scheme, tc.scheme) + } + if got.ExpiresAtNS != expires { + t.Fatalf("port %d expiry=%d, want %d (preserved)", ntohsPort(tc.port), got.ExpiresAtNS, expires) + } + } + + // The plain entry becomes a single /32 any-port entry with no L7 flag. + plainKey := lpmKeyV3{Prefixlen: 32, IP: plainIP, Port: 0} + var plain netPolicyValueV3 + if err := dest.Lookup(&plainKey, &plain); err != nil { + t.Fatalf("migrated plain /32 entry missing: %v", err) + } + if plain.Flags&uint8(netPolicyFlagL7Required) != 0 { + t.Fatalf("plain /32 has unexpected L7 flag: %#x", plain.Flags) + } + if plain.Scheme != L7SchemeNone { + t.Fatalf("plain /32 scheme=%d, want none", plain.Scheme) + } + if plain.ExpiresAtNS != expires { + t.Fatalf("plain /32 expiry=%d, want %d (preserved)", plain.ExpiresAtNS, expires) + } +} From e66b524a3ecb9ca2bff45a881384b799a405e3cb Mon Sep 17 00:00:00 2001 From: hankzhou Date: Tue, 18 Aug 2026 10:08:04 +0800 Subject: [PATCH 26/31] test(sdk_compat): prove HTTPS custom-port interception via TLS issuer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HTTPS custom-port e2e asserted only code=200 + non-empty body, which a plain-SNAT passthrough also satisfies — it could not distinguish interception from passthrough. Now also assert the TLS leaf issuer is the cluster interception CA (configurable via SDK_E2E_L7_INTERCEPTION_ISSUER, default "CubeSandbox Egress MITM CA"): an intercepted connection is terminated by CubeEgress's MITM cert, while passthrough would present the upstream's real public CA. Verified the issuer-extraction command against the live badssl endpoint. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- .../cases/network/test_l7_custom_port.py | 47 +++++++++++++++++-- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py b/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py index e09c5cd5f..1cbefbb04 100644 --- a/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py +++ b/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py @@ -76,6 +76,14 @@ "yes", "on", } +# Subject CN of the cluster's MITM interception CA (generated by +# deploy/one-click .../cube-egress-prepare.sh as "CubeSandbox Egress MITM CA"). +# An intercepted TLS connection is terminated by a leaf cert issued by this CA; +# a plain-SNAT passthrough would present the upstream's real publicly-issued +# cert instead. Asserting the leaf issuer proves interception actually happened. +L7_INTERCEPTION_ISSUER = os.environ.get( + "SDK_E2E_L7_INTERCEPTION_ISSUER", "CubeSandbox Egress MITM CA" +) pytestmark = [ pytest.mark.e2e, @@ -360,6 +368,24 @@ def test_l7_custom_port_rejects_subnet_host( safe_kill(adapter, sdk_e2e_config) +def _fetch_leaf_issuer(adapter, sdk_e2e_config) -> str: + """Return the issuer of the TLS leaf cert the sandbox is presented with. + + Uses ``openssl s_client`` (no verification, so the handshake completes even + for the interception CA) and extracts the leaf issuer via ``openssl x509``. + An intercepted connection shows the cluster interception CA; a plain-SNAT + passthrough shows the upstream's real public CA. + """ + cmd = ( + f"openssl s_client -connect {L7_CUSTOM_HTTPS_HOST}:{L7_CUSTOM_HTTPS_PORT} " + f"-servername {L7_CUSTOM_HTTPS_HOST} /dev/null " + f"| openssl x509 -noout -issuer" + ) + result = adapter.run_command(cmd, timeout=sdk_e2e_config.command_timeout) + assert_command_ok(result) + return result.stdout.strip() + + @pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) @pytest.mark.requires_internet def test_l7_custom_port_https_intercepts_custom_port( @@ -370,10 +396,11 @@ def test_l7_custom_port_https_intercepts_custom_port( port: CubeEgress does TLS MITM and forwards to the upstream. Proves the port-scoped scheme=https interception path on a non-443 port. - This is the HTTPS counterpart of the plaintext custom-port leg. Injection is - not asserted here because the public upstream (default badssl) does not echo - request headers; the HTTP leg already proves injection, and this leg proves - the scheme=https + TLS-MITM path works on a custom port. + Interception is proven by asserting the TLS leaf issuer: an intercepted + connection is terminated by CubeEgress's MITM cert, issued by the cluster + interception CA (L7_INTERCEPTION_ISSUER), whereas a plain-SNAT passthrough + would present the upstream's real publicly-issued cert. This distinguishes + interception from mere reachability — a bare 200/body check could not. """ _skip_without_capability(sdk_backend) if not L7_HTTPS_CAP_TRUSTED: @@ -429,6 +456,18 @@ def test_l7_custom_port_https_intercepts_custom_port( f"custom-port HTTPS L7 rule proxied {L7_CUSTOM_HTTPS_URL} but the " f"upstream returned an empty body; curl output={out!r}" ) + + # The 200/body check above only proves reachability. Assert the TLS + # leaf was issued by the interception CA to prove the connection was + # actually MITM-intercepted (not plain-SNAT passthrough, which would + # present the upstream's real public CA). + issuer = _fetch_leaf_issuer(adapter, sdk_e2e_config) + assert L7_INTERCEPTION_ISSUER in issuer, ( + f"expected interception CA {L7_INTERCEPTION_ISSUER!r} as the TLS " + f"leaf issuer (proves MITM interception); got issuer={issuer!r}. " + f"A public CA here means the custom-port HTTPS flow bypassed " + f"CubeEgress (plain SNAT passthrough)." + ) finally: if adapter is not None: safe_kill(adapter, sdk_e2e_config) From 3c2904d1401d91dd950e33fbfcf37a6301a587d5 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Tue, 18 Aug 2026 11:24:01 +0800 Subject: [PATCH 27/31] fix(cubenet): keep tcp_conntracks index bounds-checkable on older clang update_session() declared old_state/new_state as enum tcp_conntrack (signed int) holding a u8 state. On clang 14 the signed value is narrowed with '&= 255' masks that split it into a bounds-checked copy and a separate index copy, so the verifier reads the tcp_conntracks[dir][index][old_state] .rodata access as unbounded (umax=255) and rejects the program (invalid access to map value, value_size=214 off=369). The cubevs unit tests therefore fail in the ubuntu2004 builder (clang 14) while passing on clang 17. Use __u8 so old_state stays in a single unsigned register shared by the bounds check and the index, letting the verifier prove old_state <= 9. Semantics are unchanged (state values fit in u8); this also keeps the production datapath objects that include tcp.h verifiable on clang 14. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- CubeNet/src/tcp.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CubeNet/src/tcp.h b/CubeNet/src/tcp.h index e989a40f1..574f44931 100644 --- a/CubeNet/src/tcp.h +++ b/CubeNet/src/tcp.h @@ -269,7 +269,13 @@ static __always_inline long snat_tcp(struct __sk_buff *skb, static __always_inline void update_session(enum ip_conntrack_dir dir, struct nat_session *sess, __u64 now_ns, bool syn, bool ack, bool fin, bool rst) { - enum tcp_conntrack old_state, new_state; + /* __u8 (not enum): keeps old_state in a single unsigned register. With a + * signed enum, older clang (14) narrows the value with `&= 255` masks that + * split it into a bounds-checked copy and a separate index copy, so the + * verifier sees the tcp_conntracks index register as unbounded (umax=255) + * and rejects the .rodata read. A plain __u8 keeps check and index unified. + */ + __u8 old_state, new_state; unsigned int index; session_lazy_refresh(sess, now_ns); From 22d324c94e1778bdfe875f62b35d525ec6a7ae44 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Tue, 18 Aug 2026 11:24:01 +0800 Subject: [PATCH 28/31] ci(unit-test): add cubevs display_name to the test matrix mapping cubevs was added to the make_target mapping but not the display_name mapping, so the matrix-generation jq hit the "unsupported component" error for cubevs. Add the missing display_name entry (cubevs -> CubeVS). Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- .github/workflows/unit-test-check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-test-check.yml b/.github/workflows/unit-test-check.yml index a7c59e9c5..7b53dc65c 100644 --- a/.github/workflows/unit-test-check.yml +++ b/.github/workflows/unit-test-check.yml @@ -237,6 +237,7 @@ jobs: elif . == "cubedb" then "CubeDB" elif . == "cube-lifecycle-manager" then "Cube Lifecycle Manager" elif . == "cubelet" then "Cubelet" + elif . == "cubevs" then "CubeVS" elif . == "agent" then "Agent" elif . == "hypervisor" then "Hypervisor" else error("unsupported component: " + .) From de6701d96383d34c181f97a156b0df8846bff336 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Tue, 18 Aug 2026 19:14:09 +0800 Subject: [PATCH 29/31] test(sdk_compat): add retry hardening to external-endpoint e2e requests The L7/DNS e2e legs hit public echo endpoints (httpbun.com, httpbingo.org, badssl) with a single attempt and a fixed timeout, so transient slowness or rate-limiting flaked them. Add retry-with-backoff for transient transport failures (timeout / connection reset / temporary DNS) while never retrying HTTP 4xx/5xx, which are real policy verdicts: - test_l7_egress.py: _http_json_command and _tls_issuer_command retry loops. - test_dns_allow.py: _resolve_and_probe_command retries only the DNS resolve step; the TCP probe verdict is never retried (a blocked domain still reports PROBE:FAIL immediately). - test_l7_custom_port.py: curl legs get --retry; _fetch_leaf_issuer retries when no cert is produced. Verified: transient bad-host retries 3x then errors; an HTTP 404 returns immediately without retrying. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- .../cases/network/test_dns_allow.py | 29 ++++++-- .../cases/network/test_l7_custom_port.py | 25 +++++-- .../cases/network/test_l7_egress.py | 66 +++++++++++++------ 3 files changed, 87 insertions(+), 33 deletions(-) diff --git a/tests/e2e/sdk_compat/cases/network/test_dns_allow.py b/tests/e2e/sdk_compat/cases/network/test_dns_allow.py index 392dd810d..c0f595345 100644 --- a/tests/e2e/sdk_compat/cases/network/test_dns_allow.py +++ b/tests/e2e/sdk_compat/cases/network/test_dns_allow.py @@ -31,18 +31,33 @@ ] -def _resolve_and_probe_command(host: str, port: int, timeout: int) -> str: - """Resolve *host*, print addrs, then TCP-probe the first IPv4 address.""" +def _resolve_and_probe_command(host: str, port: int, timeout: int, attempts: int = 3) -> str: + """Resolve *host* (retrying transient DNS failures), print addrs, then + TCP-probe the first IPv4 address. + + Only the resolution step is retried (temporary resolver failure / rate + limit); the TCP probe verdict (OK / FAIL) is a policy outcome and is never + retried, so a blocked domain still reports PROBE:FAIL immediately. + """ return ( "python3 - <<'PY'\n" - "import socket\n" + "import socket, time\n" f"host = {host!r}\n" f"port = {port!r}\n" f"timeout = {timeout!r}\n" - "try:\n" - " infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)\n" - "except Exception as exc:\n" - " print(f'RESOLVE:ERROR:{type(exc).__name__}:{exc}')\n" + f"attempts = {attempts!r}\n" + "infos = None\n" + "last = None\n" + "for attempt in range(attempts):\n" + " try:\n" + " infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)\n" + " break\n" + " except Exception as exc:\n" + " last = exc\n" + " if attempt + 1 < attempts:\n" + " time.sleep(min(2 ** attempt, 5))\n" + "if infos is None:\n" + " print(f'RESOLVE:ERROR:{type(last).__name__}:{last}')\n" " raise SystemExit(0)\n" "addrs = []\n" "for family, _, _, _, sockaddr in infos:\n" diff --git a/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py b/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py index 1cbefbb04..56ff18e54 100644 --- a/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py +++ b/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py @@ -36,6 +36,7 @@ import json import os import threading +import time import uuid from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import urlparse @@ -267,7 +268,9 @@ def test_l7_custom_port_scheme_only_intercepts_default_http_set( url = f"http://{L7_DEFAULT_HOST}/headers" result = adapter.run_command( - f"curl -sS --max-time 20 '{url}'", + # --retry absorbs transient failures (timeout / connreset / + # temporary DNS) against the public echo endpoint. + f"curl -sS --max-time 20 --retry 3 --retry-delay 1 --retry-all-errors '{url}'", timeout=sdk_e2e_config.command_timeout, ) assert_command_ok(result) @@ -368,22 +371,32 @@ def test_l7_custom_port_rejects_subnet_host( safe_kill(adapter, sdk_e2e_config) -def _fetch_leaf_issuer(adapter, sdk_e2e_config) -> str: +def _fetch_leaf_issuer(adapter, sdk_e2e_config, attempts: int = 3) -> str: """Return the issuer of the TLS leaf cert the sandbox is presented with. Uses ``openssl s_client`` (no verification, so the handshake completes even for the interception CA) and extracts the leaf issuer via ``openssl x509``. An intercepted connection shows the cluster interception CA; a plain-SNAT passthrough shows the upstream's real public CA. + + Retries when the handshake fails to produce a cert (transient connect/timeout + against the public endpoint), so a slow upstream does not flake the leg. """ cmd = ( f"openssl s_client -connect {L7_CUSTOM_HTTPS_HOST}:{L7_CUSTOM_HTTPS_PORT} " f"-servername {L7_CUSTOM_HTTPS_HOST} /dev/null " f"| openssl x509 -noout -issuer" ) - result = adapter.run_command(cmd, timeout=sdk_e2e_config.command_timeout) - assert_command_ok(result) - return result.stdout.strip() + out = "" + for attempt in range(attempts): + result = adapter.run_command(cmd, timeout=sdk_e2e_config.command_timeout) + assert_command_ok(result) + out = result.stdout.strip() + if out: + return out + if attempt + 1 < attempts: + time.sleep(min(2 ** attempt, 5)) + return out @pytest.mark.requires_capability(NETWORK_L7_CUSTOM_PORT) @@ -441,7 +454,7 @@ def test_l7_custom_port_https_intercepts_custom_port( ) result = adapter.run_command( - f"curl -sS --max-time 20 -o /dev/null " + f"curl -sS --max-time 20 --retry 3 --retry-delay 1 --retry-all-errors -o /dev/null " f"-w 'code=%{{http_code}} len=%{{size_download}}' " f"'{L7_CUSTOM_HTTPS_URL}'", timeout=sdk_e2e_config.command_timeout, diff --git a/tests/e2e/sdk_compat/cases/network/test_l7_egress.py b/tests/e2e/sdk_compat/cases/network/test_l7_egress.py index b565f060b..0ee2b60b2 100644 --- a/tests/e2e/sdk_compat/cases/network/test_l7_egress.py +++ b/tests/e2e/sdk_compat/cases/network/test_l7_egress.py @@ -55,27 +55,41 @@ def _https_rule( return {"name": name, "match": match, "action": action} -def _http_json_command(url: str, *, method: str = "GET", timeout: int = 15) -> str: - """Fetch URL without TLS verify; print STATUS + body (or error).""" +def _http_json_command(url: str, *, method: str = "GET", timeout: int = 15, attempts: int = 3) -> str: + """Fetch URL without TLS verify; print STATUS + body (or error). + + Retries transient transport failures (timeout / connection reset / temporary + DNS) with backoff, since the public echo endpoints can be slow or rate-limit. + An HTTP 4xx/5xx is a real policy verdict and is returned as-is, never retried. + """ return ( "python3 - <<'PY'\n" - "import json, ssl, urllib.error, urllib.request\n" + "import ssl, time, urllib.error, urllib.request\n" f"url = {url!r}\n" f"method = {method!r}\n" f"timeout = {timeout!r}\n" + f"attempts = {attempts!r}\n" "ctx = ssl._create_unverified_context()\n" - "req = urllib.request.Request(url, method=method)\n" - "try:\n" - " with urllib.request.urlopen(req, context=ctx, timeout=timeout) as resp:\n" - " body = resp.read().decode('utf-8', errors='replace')\n" - " print(f'STATUS:{resp.status}')\n" + "last = None\n" + "for attempt in range(attempts):\n" + " try:\n" + " req = urllib.request.Request(url, method=method)\n" + " with urllib.request.urlopen(req, context=ctx, timeout=timeout) as resp:\n" + " body = resp.read().decode('utf-8', errors='replace')\n" + " print(f'STATUS:{resp.status}')\n" + " print(body)\n" + " break\n" + " except urllib.error.HTTPError as exc:\n" + " body = exc.read().decode('utf-8', errors='replace')\n" + " print(f'STATUS:{exc.code}')\n" " print(body)\n" - "except urllib.error.HTTPError as exc:\n" - " body = exc.read().decode('utf-8', errors='replace')\n" - " print(f'STATUS:{exc.code}')\n" - " print(body)\n" - "except Exception as exc:\n" - " print(f'ERROR:{type(exc).__name__}:{exc}')\n" + " break\n" + " except Exception as exc:\n" + " last = exc\n" + " if attempt + 1 < attempts:\n" + " time.sleep(min(2 ** attempt, 5))\n" + "else:\n" + " print(f'ERROR:{type(last).__name__}:{last}')\n" "PY" ) @@ -118,23 +132,35 @@ def _header_ci(headers: dict[str, str], name: str) -> str | None: return None -def _tls_issuer_command(host: str, timeout: int = 15) -> str: +def _tls_issuer_command(host: str, timeout: int = 15, attempts: int = 3) -> str: return ( "python3 - <<'PY'\n" - "import shutil, socket, ssl, subprocess, tempfile\n" + "import shutil, socket, ssl, subprocess, tempfile, time\n" f"host = {host!r}\n" f"timeout = {timeout!r}\n" + f"attempts = {attempts!r}\n" "if shutil.which('openssl') is None:\n" " print('ISSUER:ERROR:openssl_not_found')\n" " raise SystemExit(0)\n" "ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)\n" "ctx.check_hostname = False\n" "ctx.verify_mode = ssl.CERT_NONE\n" - "with socket.create_connection((host, 443), timeout=timeout) as sock:\n" - " with ctx.wrap_socket(sock, server_hostname=host) as ssock:\n" - " der = ssock.getpeercert(binary_form=True)\n" + "der = None\n" + "last = None\n" + # Retry transient connect/handshake failures (timeout / temp DNS) with + # backoff; the public echo endpoint can be slow or rate-limit. + "for attempt in range(attempts):\n" + " try:\n" + " with socket.create_connection((host, 443), timeout=timeout) as sock:\n" + " with ctx.wrap_socket(sock, server_hostname=host) as ssock:\n" + " der = ssock.getpeercert(binary_form=True)\n" + " break\n" + " except Exception as exc:\n" + " last = exc\n" + " if attempt + 1 < attempts:\n" + " time.sleep(min(2 ** attempt, 5))\n" "if not der:\n" - " print('ISSUER:ERROR:empty_peer_cert')\n" + " print(f'ISSUER:ERROR:connect_failed:{type(last).__name__}:{last}')\n" " raise SystemExit(0)\n" "try:\n" " with tempfile.NamedTemporaryFile() as tmp:\n" From 5fc90dc26b6be430cb1cd26795831dd7dba16de1 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Tue, 18 Aug 2026 19:14:26 +0800 Subject: [PATCH 30/31] test(sdk_compat): unify MITM CA CN default to the authoritative value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The interception CA's subject CN is "CubeSandbox Egress MITM CA" — that is what both the one-click prepare (deploy/.../cube-egress-prepare.sh) and the k8s chart (values caCommonName) generate. test_l7_egress.py defaulted to the outdated "Cube Sandbox Egress CA". Align it with the authoritative value (and with test_l7_custom_port.py's SDK_E2E_L7_INTERCEPTION_ISSUER default). The test's regex fallback still tolerates spacing variants. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- tests/e2e/sdk_compat/cases/network/test_l7_egress.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/e2e/sdk_compat/cases/network/test_l7_egress.py b/tests/e2e/sdk_compat/cases/network/test_l7_egress.py index 0ee2b60b2..bd72a0c67 100644 --- a/tests/e2e/sdk_compat/cases/network/test_l7_egress.py +++ b/tests/e2e/sdk_compat/cases/network/test_l7_egress.py @@ -18,9 +18,12 @@ OTHER_HOST = os.environ.get("SDK_E2E_L7_OTHER_HOST", "example.com") INJECT_HEADER = os.environ.get("SDK_E2E_L7_INJECT_HEADER", "X-Cube-E2E-Inject") INJECT_SECRET = os.environ.get("SDK_E2E_L7_INJECT_SECRET", "e2e-inject-secret-not-a-real-key") -# CubeEgress MITM CA CN varies by install path (one-click prepare vs chart bake). -# Match a stable substring; override with SDK_E2E_L7_MITM_CA_CN when needed. -MITM_CA_CN = os.environ.get("SDK_E2E_L7_MITM_CA_CN", "Cube Sandbox Egress CA") +# CubeEgress MITM CA CN. The authoritative value (one-click prepare and the +# k8s chart, deploy/.../cube-egress-prepare.sh + chart values caCommonName) is +# "CubeSandbox Egress MITM CA". Override with SDK_E2E_L7_MITM_CA_CN if a +# deployment customized it. A regex fallback in the test also tolerates +# "Cube Sandbox Egress"-style spacing variants. +MITM_CA_CN = os.environ.get("SDK_E2E_L7_MITM_CA_CN", "CubeSandbox Egress MITM CA") # Cold MITM path (DNS learn + TLS + upstream) needs more headroom than the # shared TCP network_probe_timeout default (5s). L7_HTTP_TIMEOUT = int(os.environ.get("SDK_E2E_L7_HTTP_TIMEOUT", "15")) From d1fe16b2137da4d624f6274c97bcde1beff44c56 Mon Sep 17 00:00:00 2001 From: hankzhou Date: Tue, 18 Aug 2026 19:14:46 +0800 Subject: [PATCH 31/31] test(sdk_compat): use an OS-assigned port for the L7 echo server The custom-port leg bound the in-process echo server to a fixed port (18080), so a leftover server from an interrupted run made the fixture fail with EADDRINUSE. Default L7_CUSTOM_HTTP_PORT to 0 (OS-assigned free port) and return the actual port from the l7_echo_server fixture, using it for both the L7 rule and the probe. SDK_E2E_L7_CUSTOM_HTTP_PORT still overrides when a fixed port is required. Assisted-by: CodeBuddy:kimi-k3-ioa Signed-off-by: hankzhou --- .../cases/network/test_l7_custom_port.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py b/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py index 56ff18e54..a9046adf9 100644 --- a/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py +++ b/tests/e2e/sdk_compat/cases/network/test_l7_custom_port.py @@ -53,7 +53,10 @@ # skips when unset (no safe portable default — see module docstring). L7_TARGET_HOST = os.environ.get("SDK_E2E_L7_TARGET_HOST") # Custom plaintext HTTP port the echo server listens on (+ the rule pins). -L7_CUSTOM_HTTP_PORT = int(os.environ.get("SDK_E2E_L7_CUSTOM_HTTP_PORT", "18080")) +# 0 = let the OS assign a free port. A fixed default can collide with a +# leftover echo server from an interrupted run (EADDRINUSE); overriding via +# SDK_E2E_L7_CUSTOM_HTTP_PORT is still possible when a fixed port is required. +L7_CUSTOM_HTTP_PORT = int(os.environ.get("SDK_E2E_L7_CUSTOM_HTTP_PORT", "0")) # Public host for the scheme-only default-set leg (echoes request headers). L7_DEFAULT_HOST = os.environ.get("SDK_E2E_L7_DEFAULT_HOST", "httpbingo.org") L7_MARKER_HEADER = "X-Cube-L7-E2E" @@ -152,19 +155,21 @@ def l7_echo_server(): thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() try: - yield L7_TARGET_HOST or _detect_bridge_ip() + # server.server_address[1] is the OS-assigned port when + # L7_CUSTOM_HTTP_PORT == 0 (the default). + yield L7_TARGET_HOST or _detect_bridge_ip(), server.server_address[1] finally: server.shutdown() server.server_close() -def _l7_rules_custom_http(target_host: str) -> list[dict]: +def _l7_rules_custom_http(target_host: str, port: int) -> list[dict]: return [ { "name": "e2e-custom-http", "match": { "host": target_host, - "port": L7_CUSTOM_HTTP_PORT, + "port": port, "scheme": "http", }, "action": { @@ -202,12 +207,12 @@ def _l7_rules_custom_https(host: str, port: int) -> list[dict]: def test_l7_custom_port_injects_marker_on_custom_http_port( sdk_backend: str, sdk_e2e_config: SdkE2EConfig, - l7_echo_server: str, + l7_echo_server: tuple[str, int], ): """A (host, port, scheme=http) rule intercepts a non-standard egress port and injects the rule's credential header; the echo reflects it back.""" _skip_without_capability(sdk_backend) - target_host = l7_echo_server + target_host, http_port = l7_echo_server adapter = None try: @@ -221,11 +226,11 @@ def test_l7_custom_port_injects_marker_on_custom_http_port( }, create_options={ "allow_internet_access": False, - "network": {"rules": _l7_rules_custom_http(target_host)}, + "network": {"rules": _l7_rules_custom_http(target_host, http_port)}, }, ) - url = f"http://{target_host}:{L7_CUSTOM_HTTP_PORT}/headers" + url = f"http://{target_host}:{http_port}/headers" result = adapter.run_command( f"curl -sS --max-time 20 '{url}'", timeout=sdk_e2e_config.command_timeout,