From b88b328ca6c01a57ab8b7b5283cad85d0ddd4ec1 Mon Sep 17 00:00:00 2001
From: jay3cx <137191587@qq.com>
Date: Wed, 12 Aug 2026 16:25:26 +0800
Subject: [PATCH 1/5] fix(cubenet): resolve direct on-link neighbors via ARP

In direct egress mode, on-link destinations no longer depend on the
gateway hairpin path. Propagate the primary node IPv4 mask into CubeVS,
cache neighbor MACs in an LRU map, and convert unresolved packets into
ARP requests learned on the node interface.

Best-effort rate-limit pending ARP probes, update probe deadlines in
place so concurrent learns cannot be clobbered, and zero ARP frame
padding so original packet contents are not broadcast. Keep the
gateway-MAC fast path for off-link and route-aware traffic.

Signed-off-by: jay3cx <137191587@qq.com>
---
 CubeNet/cubevs/cubevs.go                    |  13 +-
 CubeNet/cubevs/miscs.go                     |  15 +-
 CubeNet/cubevs/miscs_test.go                |  53 ++++++
 CubeNet/cubevs/util.go                      |  14 ++
 CubeNet/cubevs/util_test.go                 |  40 +++++
 CubeNet/src/cubevs.h                        |  15 ++
 CubeNet/src/map.h                           |  14 ++
 CubeNet/src/mvmtap.bpf.c                    | 181 ++++++++++++++++++--
 CubeNet/src/nodenic.bpf.c                   |  57 +++++-
 Cubelet/network/runtime/controller.go       |   1 +
 Cubelet/network/runtime/systemnet/device.go |  14 +-
 11 files changed, 393 insertions(+), 24 deletions(-)
 create mode 100644 CubeNet/cubevs/util_test.go

diff --git a/CubeNet/cubevs/cubevs.go b/CubeNet/cubevs/cubevs.go
index b131af6ae..cb6acdb7d 100644
--- a/CubeNet/cubevs/cubevs.go
+++ b/CubeNet/cubevs/cubevs.go
@@ -34,6 +34,7 @@ type Params struct {
 	// Ifindex, IP and MAC address of Node itself
 	NodeIfindex uint32
 	NodeIP      net.IP
+	NodeIPMask  net.IPMask
 	NodeMacAddr net.HardwareAddr
 	// MAC address of the Node gateway (next hop)
 	NodeGatewayMacAddr net.HardwareAddr
@@ -154,11 +155,12 @@ const (
 	MapNameRemotePortMapping    = "remote_port_mapping"
 	MapNameLocalPortMapping     = "local_port_mapping"
 	// MapNameAllowOut is the cube-v0.2.0 legacy migration source.
-	MapNameAllowOut      = "allow_out"
-	MapNameAllowOutV2    = "allow_out_v2"
-	MapNameDenyOut       = "deny_out"
-	MapNameDNSAllow      = "dns_allow"
-	MapNameDNSQueryTrack = "dns_query_track"
+	MapNameAllowOut        = "allow_out"
+	MapNameAllowOutV2      = "allow_out_v2"
+	MapNameDenyOut         = "deny_out"
+	MapNameDNSAllow        = "dns_allow"
+	MapNameDNSQueryTrack   = "dns_query_track"
+	MapNameDirectNeighbors = "direct_neigh"
 	// constants referenced by BPF programs.
 	globalNameMVMInnerIP           = "mvm_inner_ip"
 	globalNameMVMMacaddrP1         = "mvm_macaddr_p1"
@@ -174,6 +176,7 @@ const (
 	globalNameEgressDMacaddrP2     = "egress_dmacaddr_p2"
 	globalNameEgressRedirectFlags  = "egress_redirect_flags"
 	globalNameNodeIP               = "nodenic_ip"
+	globalNameNodeNetmask          = "nodenic_netmask"
 	globalNameNodeIfindex          = "nodenic_ifindex"
 	globalNameNodeMacaddrP1        = "nodenic_macaddr_p1"
 	globalNameNodeMacaddrP2        = "nodenic_macaddr_p2"
diff --git a/CubeNet/cubevs/miscs.go b/CubeNet/cubevs/miscs.go
index 27545d997..71da591e8 100644
--- a/CubeNet/cubevs/miscs.go
+++ b/CubeNet/cubevs/miscs.go
@@ -31,7 +31,8 @@ func init() {
 	_ = rlimit.RemoveMemlock()
 }
 
-func rewriteConstants(vars map[string]*ebpf.VariableSpec, params Params) error {
+func rewriteConstants(spec *ebpf.CollectionSpec, params Params) error {
+	vars := spec.Variables
 	var err error
 	err = errors.Join(err, vars[globalNameMVMInnerIP].Set(ipToUint32(params.MVMInnerIP)))
 	err = errors.Join(err, vars[globalNameMVMMacaddrP1].Set(hardwareAddrToUint32(params.MVMMacAddr)))
@@ -57,6 +58,13 @@ func rewriteConstants(vars map[string]*ebpf.VariableSpec, params Params) error {
 		err = errors.Join(err, v.Set(params.EgressRedirectFlags))
 	}
 	err = errors.Join(err, vars[globalNameNodeIP].Set(ipToUint32(params.NodeIP)))
+	if spec.Programs[programNameFromCube] != nil {
+		nodeNetmask, netmaskErr := ipMaskToUint32(params.NodeIPMask)
+		err = errors.Join(err, netmaskErr)
+		if netmaskErr == nil {
+			err = errors.Join(err, vars[globalNameNodeNetmask].Set(nodeNetmask))
+		}
+	}
 	err = errors.Join(err, vars[globalNameNodeIfindex].Set(params.NodeIfindex))
 	err = errors.Join(err, vars[globalNameNodeMacaddrP1].Set(hardwareAddrToUint32(params.NodeMacAddr)))
 	err = errors.Join(err, vars[globalNameNodeMacaddrP2].Set(hardwareAddrToUint16(params.NodeMacAddr)))
@@ -193,7 +201,7 @@ func loadObject(params Params, loader func() (*ebpf.CollectionSpec, error), name
 		return fmt.Errorf("%s populateDNSTailCalls failed: %w", name, err)
 	}
 
-	err = rewriteConstants(spec.Variables, params)
+	err = rewriteConstants(spec, params)
 	if err != nil {
 		return fmt.Errorf("%s rewriteConstants failed: %w", name, err)
 	}
@@ -231,6 +239,9 @@ func Init(params Params) error {
 	_ = os.Remove(pinPath("tungrp_to_tuns")) // NOCC:Path Traversal()
 	// dns_query_track is runtime pending-query state, not persisted policy.
 	_ = os.Remove(pinPath(MapNameDNSQueryTrack)) // NOCC:Path Traversal()
+	// Direct-neighbor state is tied to the current host interface and must not
+	// survive a CubeVS restart with stale MAC addresses.
+	_ = os.Remove(pinPath(MapNameDirectNeighbors)) // NOCC:Path Traversal()
 
 	err := loadObject(params, loadLocalgw, "loadLocalgw")
 	if err != nil {
diff --git a/CubeNet/cubevs/miscs_test.go b/CubeNet/cubevs/miscs_test.go
index 0f2bc8dfa..9330071a5 100644
--- a/CubeNet/cubevs/miscs_test.go
+++ b/CubeNet/cubevs/miscs_test.go
@@ -1,11 +1,64 @@
 package cubevs
 
 import (
+	"net"
 	"testing"
 
 	"github.com/cilium/ebpf"
 )
 
+func testRewriteConstantsParams() Params {
+	return Params{
+		MVMInnerIP:          net.IPv4(169, 254, 68, 6),
+		MVMMacAddr:          net.HardwareAddr{0x20, 0x90, 0x6f, 0xfc, 0xfc, 0xfc},
+		MVMGatewayIP:        net.IPv4(169, 254, 68, 5),
+		Cubegw0Ifindex:      9,
+		Cubegw0IP:           net.IPv4(192, 168, 0, 1),
+		Cubegw0MacAddr:      net.HardwareAddr{0x20, 0x90, 0x6f, 0xcf, 0xcf, 0xcf},
+		EgressSrcMacAddr:    net.HardwareAddr{0x52, 0x54, 0x00, 0x68, 0xdd, 0x16},
+		EgressDstMacAddr:    net.HardwareAddr{0xfe, 0xee, 0x32, 0x47, 0x6b, 0x93},
+		EgressRedirectFlags: 0,
+		NodeIfindex:         2,
+		NodeIP:              net.IPv4(10, 2, 3, 4),
+		NodeIPMask:          net.CIDRMask(24, 32),
+		NodeMacAddr:         net.HardwareAddr{0x52, 0x54, 0x00, 0x68, 0xdd, 0x16},
+		NodeGatewayMacAddr:  net.HardwareAddr{0xfe, 0xee, 0x32, 0x47, 0x6b, 0x93},
+	}
+}
+
+func TestRewriteConstantsSetsNodeNetmask(t *testing.T) {
+	spec, err := loadMvmtap()
+	if err != nil {
+		t.Fatal(err)
+	}
+	params := testRewriteConstantsParams()
+	if err := rewriteConstants(spec, params); err != nil {
+		t.Fatal(err)
+	}
+
+	variable, ok := spec.Variables[globalNameNodeNetmask]
+	if !ok {
+		t.Fatalf("BPF variable %q not found", globalNameNodeNetmask)
+	}
+	var got uint32
+	if err := variable.Get(&got); err != nil {
+		t.Fatal(err)
+	}
+	if got != 0x00ffffff {
+		t.Fatalf("BPF node netmask = %#08x, want %#08x", got, uint32(0x00ffffff))
+	}
+
+	// Objects without from_cube do not use the netmask constant.
+	params.NodeIPMask = nil
+	localgwSpec, err := loadLocalgw()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := rewriteConstants(localgwSpec, params); err != nil {
+		t.Fatalf("rewrite unused node netmask: %v", err)
+	}
+}
+
 func TestPopulateDNSTailCallsBindsQueryPipelinePrograms(t *testing.T) {
 	spec := &ebpf.CollectionSpec{
 		Maps: map[string]*ebpf.MapSpec{
diff --git a/CubeNet/cubevs/util.go b/CubeNet/cubevs/util.go
index e9b466440..9c52764d6 100644
--- a/CubeNet/cubevs/util.go
+++ b/CubeNet/cubevs/util.go
@@ -3,6 +3,7 @@ package cubevs
 import (
 	"bytes"
 	"encoding/binary"
+	"fmt"
 	"net"
 )
 
@@ -26,6 +27,19 @@ func ipToUint32(ip net.IP) uint32 {
 	return uint32(ip[0]) | uint32(ip[1])<<8 | uint32(ip[2])<<16 | uint32(ip[3])<<24
 }
 
+// ipMaskToUint32 converts an IPv4 mask to the byte layout used by IPv4
+// addresses in BPF packet headers on little-endian hosts.
+func ipMaskToUint32(mask net.IPMask) (uint32, error) {
+	if len(mask) != net.IPv4len {
+		return 0, fmt.Errorf("invalid IPv4 mask length: %d", len(mask))
+	}
+	if _, bits := mask.Size(); bits != 32 {
+		return 0, fmt.Errorf("invalid IPv4 mask: %v", mask)
+	}
+
+	return uint32(mask[0]) | uint32(mask[1])<<8 | uint32(mask[2])<<16 | uint32(mask[3])<<24, nil
+}
+
 // hardwareAddrToUint32 converts the first 4 bytes of MAC address to a uint32.
 func hardwareAddrToUint32(addr net.HardwareAddr) uint32 {
 	return uint32(addr[0]) | uint32(addr[1])<<8 | uint32(addr[2])<<16 | uint32(addr[3])<<24
diff --git a/CubeNet/cubevs/util_test.go b/CubeNet/cubevs/util_test.go
new file mode 100644
index 000000000..ca4f6d6de
--- /dev/null
+++ b/CubeNet/cubevs/util_test.go
@@ -0,0 +1,40 @@
+package cubevs
+
+import (
+	"net"
+	"testing"
+)
+
+func TestIPMaskToUint32(t *testing.T) {
+	tests := []struct {
+		name string
+		mask net.IPMask
+		want uint32
+	}{
+		{name: "slash zero", mask: net.CIDRMask(0, 32), want: 0x00000000},
+		{name: "slash sixteen", mask: net.CIDRMask(16, 32), want: 0x0000ffff},
+		{name: "slash twenty four", mask: net.CIDRMask(24, 32), want: 0x00ffffff},
+		{name: "slash thirty two", mask: net.CIDRMask(32, 32), want: 0xffffffff},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ipMaskToUint32(tt.mask)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != tt.want {
+				t.Fatalf("ipMaskToUint32(%v) = %#08x, want %#08x", tt.mask, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIPMaskToUint32RejectsNonIPv4Mask(t *testing.T) {
+	if _, err := ipMaskToUint32(net.CIDRMask(64, 128)); err == nil {
+		t.Fatal("ipMaskToUint32 accepted an IPv6 mask")
+	}
+	if _, err := ipMaskToUint32(net.IPMask{255, 0, 255, 0}); err == nil {
+		t.Fatal("ipMaskToUint32 accepted a non-contiguous IPv4 mask")
+	}
+}
diff --git a/CubeNet/src/cubevs.h b/CubeNet/src/cubevs.h
index 264d305b2..48a31a42b 100644
--- a/CubeNet/src/cubevs.h
+++ b/CubeNet/src/cubevs.h
@@ -38,6 +38,8 @@
 #define NET_POLICY_FLAG_L7_REQUIRED	1
 #define NSEC_PER_SEC			1000000000ULL
 #define DNS_QUERY_TRACK_TTL_NS		(10ULL * NSEC_PER_SEC)
+/* Min interval between ARP probes for the same unresolved on-link neighbor. */
+#define DIRECT_NEIGH_PROBE_INTERVAL_NS	(1ULL * NSEC_PER_SEC)
 
 /* https://en.wikipedia.org/wiki/IPv4#Header
  *
@@ -99,6 +101,7 @@ const volatile __u64 egress_redirect_flags  = BPF_F_INGRESS;
 
 /* Ifindex, IP and MAC address of Node itself */
 const volatile __u32 nodenic_ip         = 0x020a8709;	/* 9.135.10.2, network byte order */
+const volatile __u32 nodenic_netmask    = 0x00ffffff;	/* 255.255.255.0, packet-byte layout */
 const volatile __u32 nodenic_ifindex    = 2;
 const volatile __u32 nodenic_macaddr_p1 = 0x68005452;	/* 52:54:00:68:dd:16 */
 const volatile __u16 nodenic_macaddr_p2 = 0x16dd;
@@ -129,6 +132,18 @@ struct arphdr_eth {
 	__be32 ar_tip;			/* target IP address */
 } __attribute__((packed));
 
+struct arp_packet {
+	struct ethhdr eth;
+	struct arphdr_eth arp;
+} __attribute__((packed));
+
+struct direct_neighbor {
+	unsigned char addr[ETH_ALEN];
+	__u16 reserved;
+	/* Suppresses repeated ARP probes while resolution is pending. */
+	__u64 next_probe_at_ns;
+};
+
 union macaddr {
 	struct {
 		__u32 p1;
diff --git a/CubeNet/src/map.h b/CubeNet/src/map.h
index 6f93c3519..bde52d218 100644
--- a/CubeNet/src/map.h
+++ b/CubeNet/src/map.h
@@ -96,6 +96,20 @@ struct {
 	__uint(pinning, LIBBPF_PIN_BY_NAME);
 } snat_iplist SEC(".maps");
 
+/* Direct-egress on-link neighbor cache.
+ *
+ * key:   destination IPv4 address in packet-byte layout
+ * value: destination MAC plus probe deadline; an all-zero MAC means initial
+ *        resolution is pending
+ */
+struct {
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(max_entries, MAX_ENTRIES);
+	__type(key, __u32);
+	__type(value, struct direct_neighbor);
+	__uint(pinning, LIBBPF_PIN_BY_NAME);
+} direct_neigh SEC(".maps");
+
 /* Egress allow list v2 (hash of maps)
  *
  * key:   ifindex of the TAP device
diff --git a/CubeNet/src/mvmtap.bpf.c b/CubeNet/src/mvmtap.bpf.c
index 1d71f5a74..eb985e0db 100644
--- a/CubeNet/src/mvmtap.bpf.c
+++ b/CubeNet/src/mvmtap.bpf.c
@@ -106,6 +106,156 @@ static __always_inline bool should_do_nat(const struct iphdr *l3)
 	return true;
 }
 
+/* Direct egress normally bypasses the host network stack and sends packets to
+ * the node gateway MAC. On-link destinations need the real neighbor MAC from
+ * the ARP-learned cache instead.
+ * "On-link" deliberately means the primary node IPv4 prefix here, not every
+ * directly connected route that may exist on a multi-homed host.
+ */
+static __always_inline bool direct_egress_is_onlink(__u32 daddr)
+{
+	return egress_redirect_flags == 0 &&
+	       (daddr & nodenic_netmask) == (nodenic_ip & nodenic_netmask);
+}
+
+static __always_inline bool direct_neighbor_is_zero(const struct direct_neighbor *neighbor)
+{
+	const union macaddr *macaddr = (const union macaddr *)neighbor->addr;
+
+	return macaddr->p1 == 0 && macaddr->p2 == 0;
+}
+
+#define DIRECT_ARP_PRESERVED_LEN	96
+#define DIRECT_ARP_HEADROOM	32
+#define DIRECT_ARP_FRAME_LEN		(DIRECT_ARP_PRESERVED_LEN + DIRECT_ARP_HEADROOM)
+#define DIRECT_ARP_ZERO_CHUNK_LEN	32
+
+static __always_inline long direct_egress_clear_arp_padding(struct __sk_buff *skb)
+{
+	unsigned char zeroes[DIRECT_ARP_ZERO_CHUNK_LEN] = {};
+	long err;
+
+	err = bpf_skb_store_bytes(skb, sizeof(struct arp_packet),
+				  zeroes, sizeof(zeroes), 0);
+	if (err)
+		return err;
+	err = bpf_skb_store_bytes(skb,
+				  sizeof(struct arp_packet) + DIRECT_ARP_ZERO_CHUNK_LEN,
+				  zeroes, sizeof(zeroes), 0);
+	if (err)
+		return err;
+	return bpf_skb_store_bytes(skb,
+				   sizeof(struct arp_packet) + 2 * DIRECT_ARP_ZERO_CHUNK_LEN,
+				   zeroes,
+				   DIRECT_ARP_FRAME_LEN - sizeof(struct arp_packet) -
+				   2 * DIRECT_ARP_ZERO_CHUNK_LEN, 0);
+}
+
+static __always_inline int direct_egress_arp_request(struct __sk_buff *skb,
+						      __u32 dst_ifindex, __u32 daddr)
+{
+	struct arp_packet packet = {};
+	union macaddr *macaddr;
+	long err;
+
+	__builtin_memset(packet.eth.h_dest, 0xff, ETH_ALEN);
+	macaddr = (union macaddr *)packet.eth.h_source;
+	macaddr->p1 = nodenic_macaddr_p1;
+	macaddr->p2 = nodenic_macaddr_p2;
+	packet.eth.h_proto = bpf_htons(ETH_P_ARP);
+
+	packet.arp.ar_hrd = bpf_htons(ARPHRD_ETHER);
+	packet.arp.ar_pro = bpf_htons(ETH_P_IP);
+	packet.arp.ar_hln = ETH_ALEN;
+	packet.arp.ar_pln = sizeof(__be32);
+	packet.arp.ar_op = bpf_htons(ARPOP_REQUEST);
+	macaddr = (union macaddr *)packet.arp.ar_sha;
+	macaddr->p1 = nodenic_macaddr_p1;
+	macaddr->p2 = nodenic_macaddr_p2;
+	packet.arp.ar_sip = nodenic_ip;
+	packet.arp.ar_tip = daddr;
+
+	/* TAP packets may carry CHECKSUM_PARTIAL metadata. Preserve enough space for
+	 * the delayed checksum write, move it beyond the ARP header, then clear the
+	 * entire padding so no original packet data is broadcast. change_tail also
+	 * clears any GSO state.
+	 */
+	err = bpf_skb_change_tail(skb, DIRECT_ARP_PRESERVED_LEN, 0);
+	if (err)
+		return TC_ACT_SHOT;
+	err = bpf_skb_change_head(skb, DIRECT_ARP_HEADROOM, 0);
+	if (err)
+		return TC_ACT_SHOT;
+	err = bpf_skb_store_bytes(skb, 0, &packet, sizeof(packet), 0);
+	if (err)
+		return TC_ACT_SHOT;
+	err = direct_egress_clear_arp_padding(skb);
+	if (err)
+		return TC_ACT_SHOT;
+
+	return bpf_redirect(dst_ifindex, 0);
+}
+
+static __always_inline int redirect_egress(struct __sk_buff *skb, __u32 dst_ifindex,
+					   __u32 daddr)
+{
+	struct direct_neighbor pending = {};
+	struct direct_neighbor *neighbor;
+	union macaddr *neighbor_mac;
+	struct ethhdr *l2;
+	void *data, *data_end;
+	__u64 now;
+	long err;
+
+	if (!direct_egress_is_onlink(daddr))
+		return bpf_redirect(dst_ifindex, egress_redirect_flags);
+
+	neighbor = bpf_map_lookup_elem(&direct_neigh, &daddr);
+	if (neighbor && !direct_neighbor_is_zero(neighbor))
+		goto redirect_neighbor;
+
+	/* Keep a pending entry so from_world only learns neighbors requested by
+	 * this path. Rate-limit ARP retries per destination while waiting for a
+	 * reply so unresolved floods do not emit one broadcast per packet.
+	 */
+	now = bpf_ktime_get_ns();
+	if (neighbor) {
+		if (neighbor->next_probe_at_ns > now)
+			return TC_ACT_SHOT;
+
+		/* Do not replace the whole value here: learn_direct_neighbor may have
+		 * installed a MAC after the lookup above. Updating only the deadline
+		 * cannot overwrite that learned state. Refresh the lookup because an LRU
+		 * hash update replaces the entry rather than updating it in place.
+		 */
+		neighbor->next_probe_at_ns = now + DIRECT_NEIGH_PROBE_INTERVAL_NS;
+		neighbor = bpf_map_lookup_elem(&direct_neigh, &daddr);
+		if (neighbor && !direct_neighbor_is_zero(neighbor))
+			goto redirect_neighbor;
+	} else {
+		pending.next_probe_at_ns = now + DIRECT_NEIGH_PROBE_INTERVAL_NS;
+		err = bpf_map_update_elem(&direct_neigh, &daddr, &pending, BPF_NOEXIST);
+		if (err)
+			return TC_ACT_SHOT;
+	}
+
+	return direct_egress_arp_request(skb, dst_ifindex, daddr);
+
+redirect_neighbor:
+	err = bpf_skb_pull_data(skb, sizeof(struct ethhdr));
+	if (err)
+		return TC_ACT_SHOT;
+	data = (void *)(__u64)skb->data;
+	data_end = (void *)(__u64)skb->data_end;
+	if (data + sizeof(struct ethhdr) > data_end)
+		return TC_ACT_SHOT;
+	l2 = data;
+	neighbor_mac = (union macaddr *)neighbor->addr;
+	set_mac_pair(l2, nodenic_macaddr_p1, nodenic_macaddr_p2,
+		     neighbor_mac->p1, neighbor_mac->p2);
+	return bpf_redirect(dst_ifindex, 0);
+}
+
 /*
  * Check whether a TCP flow should be redirected to the L7 proxy.
  *
@@ -648,12 +798,13 @@ static __noinline __attribute__((noinline)) __u32 do_udp_nat(struct __sk_buff *s
  * cannot make bpf-to-bpf calls (see do_udp_nat_inline()'s comment).
  */
 static __always_inline int finish_udp_nat_inline(struct __sk_buff *skb,
-						 struct mvm_meta *mvm_meta)
+						 struct mvm_meta *mvm_meta,
+						 __u32 daddr)
 {
 	__u32 dst_ifindex = do_udp_nat_inline(skb, mvm_meta);
 
 	if (dst_ifindex)
-		return bpf_redirect(dst_ifindex, egress_redirect_flags);
+		return redirect_egress(skb, dst_ifindex, daddr);
 
 	return TC_ACT_SHOT;
 }
@@ -661,10 +812,14 @@ static __always_inline int finish_udp_nat_inline(struct __sk_buff *skb,
 /* Subprog-based version used by dns_finish. */
 static __always_inline int finish_udp_nat(struct __sk_buff *skb, struct mvm_meta *mvm_meta)
 {
+	__u32 daddr;
 	__u32 dst_ifindex = do_udp_nat(skb, mvm_meta);
 
-	if (dst_ifindex)
-		return bpf_redirect(dst_ifindex, egress_redirect_flags);
+	if (dst_ifindex) {
+		if (bpf_skb_load_bytes(skb, IP_DADDR_OFF, &daddr, sizeof(daddr)))
+			return TC_ACT_SHOT;
+		return redirect_egress(skb, dst_ifindex, daddr);
+	}
 
 	return TC_ACT_SHOT;
 }
@@ -878,24 +1033,26 @@ int dns_finish(struct __sk_buff *skb)
 	if (!mvm_meta)
 		return TC_ACT_SHOT;
 	if (!dns_policy_enabled(mvm_meta))
-		return finish_udp_nat(skb, mvm_meta);
+		goto do_nat;
 
 	inner_map = bpf_map_lookup_elem(&dns_allow, &ifindex);
 	if (!inner_map)
-		return finish_udp_nat(skb, mvm_meta);
+		goto do_nat;
 
 	question_cursor = state->dns_off + DNS_HDR_LEN;
 	if (state->failed)
-		return finish_udp_nat(skb, mvm_meta);
+		goto do_nat;
 	if (!dns_hash_qname(skb, &question_cursor, &question_footer,
 					&qname_hash))
-		return finish_udp_nat(skb, mvm_meta);
+		goto do_nat;
 
 	matched = dns_allow_match_value(inner_map, question);
 	if (!matched)
-		return finish_udp_nat(skb, mvm_meta);
+		goto do_nat;
 
 	dns_track_allowed_query(skb, state, matched->flags, qname_hash);
+
+do_nat:
 	return finish_udp_nat(skb, mvm_meta);
 }
 
@@ -1039,7 +1196,7 @@ int from_cube(struct __sk_buff *skb)
 			return bpf_redirect(cubegw0_ifindex, BPF_F_INGRESS);
 		tcp_ret = do_tcp_nat(skb, mvm_meta);
 		if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_OK)
-			return bpf_redirect(TCP_NAT_IFINDEX(tcp_ret), egress_redirect_flags);
+			return redirect_egress(skb, TCP_NAT_IFINDEX(tcp_ret), daddr);
 		if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_RESET)
 			return tcp_reply_reset(skb, ifindex);
 	}
@@ -1055,13 +1212,13 @@ int from_cube(struct __sk_buff *skb)
 				return ret;
 		}
 
-		return finish_udp_nat_inline(skb, mvm_meta);
+		return finish_udp_nat_inline(skb, mvm_meta, daddr);
 	}
 
 	if (proto == IPPROTO_ICMP) {
 		dst_ifindex = do_icmp_nat(skb, mvm_meta);
 		if (dst_ifindex)
-			return bpf_redirect(dst_ifindex, egress_redirect_flags);
+			return redirect_egress(skb, dst_ifindex, daddr);
 	}
 
 	return TC_ACT_SHOT;
diff --git a/CubeNet/src/nodenic.bpf.c b/CubeNet/src/nodenic.bpf.c
index c8e1cdfc3..131f84e2f 100644
--- a/CubeNet/src/nodenic.bpf.c
+++ b/CubeNet/src/nodenic.bpf.c
@@ -17,6 +17,50 @@
 #include "dns_query.h"
 #include "dns_response.h"
 
+/* Learn the sender of an ARP packet only when direct egress already created a
+ * pending cache entry for that IP. This also refreshes an existing entry when
+ * the peer announces a MAC change.
+ */
+static __always_inline void learn_direct_neighbor(struct __sk_buff *skb)
+{
+	struct direct_neighbor neighbor = {};
+	struct arp_packet *packet;
+	union macaddr *eth_src, *arp_src, *mac;
+	void *data, *data_end;
+	__u32 ip;
+
+	if (skb->ifindex != nodenic_ifindex ||
+	    bpf_skb_pull_data(skb, sizeof(struct arp_packet)))
+		return;
+
+	data = (void *)(__u64)skb->data;
+	data_end = (void *)(__u64)skb->data_end;
+	if (data + sizeof(struct arp_packet) > data_end)
+		return;
+
+	packet = data;
+	if (packet->eth.h_proto != bpf_htons(ETH_P_ARP) ||
+	    packet->arp.ar_hrd != bpf_htons(ARPHRD_ETHER) ||
+	    packet->arp.ar_pro != bpf_htons(ETH_P_IP) ||
+	    packet->arp.ar_hln != ETH_ALEN ||
+	    packet->arp.ar_pln != sizeof(__be32) ||
+	    (packet->arp.ar_op != bpf_htons(ARPOP_REQUEST) &&
+	     packet->arp.ar_op != bpf_htons(ARPOP_REPLY)))
+		return;
+
+	eth_src = (union macaddr *)packet->eth.h_source;
+	arp_src = (union macaddr *)packet->arp.ar_sha;
+	if (eth_src->p1 != arp_src->p1 || eth_src->p2 != arp_src->p2 ||
+	    (arp_src->addr[0] & 1) || (arp_src->p1 == 0 && arp_src->p2 == 0))
+		return;
+
+	mac = (union macaddr *)neighbor.addr;
+	mac->p1 = arp_src->p1;
+	mac->p2 = arp_src->p2;
+	ip = packet->arp.ar_sip;
+	bpf_map_update_elem(&direct_neigh, &ip, &neighbor, BPF_EXIST);
+}
+
 static int tcp_nat_proxy(struct __sk_buff *skb, struct ethhdr *l2, struct iphdr *l3, struct tcphdr *l4,
 			 struct mvm_port *mvm_port)
 {
@@ -350,7 +394,10 @@ static int icmp_nat_session(struct __sk_buff *skb, struct ethhdr *l2, struct iph
 	return bpf_redirect(sess->vm_ifindex, 0);
 }
 
-static int do_icmp_nat(struct __sk_buff *skb)
+/* from_world performs a DNS tail call, so these dispatch helpers must remain
+ * inline for kernels that reject tail calls from programs with BPF subcalls.
+ */
+static __always_inline int do_icmp_nat(struct __sk_buff *skb)
 {
 	struct ethhdr *l2;
 	struct iphdr *l3;
@@ -362,7 +409,7 @@ static int do_icmp_nat(struct __sk_buff *skb)
 	return icmp_nat_session(skb, l2, l3, l4);
 }
 
-static int do_udp_nat(struct __sk_buff *skb)
+static __always_inline int do_udp_nat(struct __sk_buff *skb)
 {
 	struct ethhdr *l2;
 	struct iphdr *l3;
@@ -374,7 +421,7 @@ static int do_udp_nat(struct __sk_buff *skb)
 	return udp_nat_session(skb, l2, l3, l4);
 }
 
-static int do_tcp_nat(struct __sk_buff *skb)
+static __always_inline int do_tcp_nat(struct __sk_buff *skb)
 {
 	struct mvm_port *mvm_port;
 	struct ethhdr *l2;
@@ -409,6 +456,10 @@ int from_world(struct __sk_buff *skb)
 	struct iphdr *l3;
 	int ret;
 
+	if (skb->protocol == bpf_htons(ETH_P_ARP)) {
+		learn_direct_neighbor(skb);
+		return TC_ACT_OK;
+	}
 	if (skb->protocol != bpf_htons(ETH_P_IP))
 		return TC_ACT_OK;
 
diff --git a/Cubelet/network/runtime/controller.go b/Cubelet/network/runtime/controller.go
index bf2ae04ee..87000b21a 100644
--- a/Cubelet/network/runtime/controller.go
+++ b/Cubelet/network/runtime/controller.go
@@ -289,6 +289,7 @@ func initCubeVS(cfg Config, device *systemnet.HostDevice, cubeDev *systemnet.Cub
 		CubeRouterIfindex:   cubeRouterIfindex,
 		NodeIfindex:         uint32(device.Index),
 		NodeIP:              device.IP,
+		NodeIPMask:          device.IPMask,
 		NodeMacAddr:         device.Mac,
 		NodeGatewayMacAddr:  device.GatewayMac,
 	}
diff --git a/Cubelet/network/runtime/systemnet/device.go b/Cubelet/network/runtime/systemnet/device.go
index c9e42af39..057e296ce 100644
--- a/Cubelet/network/runtime/systemnet/device.go
+++ b/Cubelet/network/runtime/systemnet/device.go
@@ -30,6 +30,7 @@ type HostDevice struct {
 	Index      int
 	Name       string
 	IP         net.IP
+	IPMask     net.IPMask
 	Mac        net.HardwareAddr
 	GatewayMac net.HardwareAddr
 }
@@ -41,13 +42,21 @@ func GetHostDevice(ifName string) (*HostDevice, error) {
 	if err != nil {
 		return nil, err
 	}
-	addrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
+	addrs, err := netlinkAddrList(link, netlink.FAMILY_V4)
 	if err != nil {
 		return nil, err
 	}
 	if len(addrs) != 1 {
 		return nil, fmt.Errorf("ipv4 address on %s is not unique", ifName)
 	}
+	if addrs[0].IPNet == nil {
+		return nil, fmt.Errorf("invalid ipv4 address on %s", ifName)
+	}
+	ip := addrs[0].IP.To4()
+	_, bits := addrs[0].Mask.Size()
+	if ip == nil || bits != 32 {
+		return nil, fmt.Errorf("invalid ipv4 address on %s", ifName)
+	}
 	gwMac, err := GetGatewayMacAddr(ifName)
 	if err != nil {
 		return nil, err
@@ -59,7 +68,8 @@ func GetHostDevice(ifName string) (*HostDevice, error) {
 	return &HostDevice{
 		Index:      link.Attrs().Index,
 		Name:       link.Attrs().Name,
-		IP:         addrs[0].IP,
+		IP:         append(net.IP(nil), ip...),
+		IPMask:     append(net.IPMask(nil), addrs[0].Mask...),
 		Mac:        link.Attrs().HardwareAddr,
 		GatewayMac: gatewayMac,
 	}, nil

From f5267bbaedec141abde2e30a2b5728608e959e2f Mon Sep 17 00:00:00 2001
From: jay3cx <137191587@qq.com>
Date: Thu, 13 Aug 2026 19:01:50 +0800
Subject: [PATCH 2/5] fix(cubenet): simplify userspace and refresh learned
 neighbors

Drop extra mask validation, copies, and Init pin removals.
Learn ARP replies only, revalidate cached MACs, and keep from_world
inlined on Linux 5.4 so the DNS tail call can load.

Signed-off-by: jay3cx <137191587@qq.com>
---
 CubeNet/cubevs/cubevs.go                    |  11 +-
 CubeNet/cubevs/miscs.go                     |  20 +--
 CubeNet/cubevs/miscs_test.go                |  53 --------
 CubeNet/cubevs/util.go                      |  15 +-
 CubeNet/cubevs/util_test.go                 |  40 ------
 CubeNet/src/cubevs.h                        |   5 +-
 CubeNet/src/mvmtap.bpf.c                    | 143 ++++++++++----------
 CubeNet/src/nodenic.bpf.c                   |  12 +-
 Cubelet/network/runtime/systemnet/device.go |  14 +-
 9 files changed, 91 insertions(+), 222 deletions(-)
 delete mode 100644 CubeNet/cubevs/util_test.go

diff --git a/CubeNet/cubevs/cubevs.go b/CubeNet/cubevs/cubevs.go
index cb6acdb7d..e5f3e85a7 100644
--- a/CubeNet/cubevs/cubevs.go
+++ b/CubeNet/cubevs/cubevs.go
@@ -155,12 +155,11 @@ const (
 	MapNameRemotePortMapping    = "remote_port_mapping"
 	MapNameLocalPortMapping     = "local_port_mapping"
 	// MapNameAllowOut is the cube-v0.2.0 legacy migration source.
-	MapNameAllowOut        = "allow_out"
-	MapNameAllowOutV2      = "allow_out_v2"
-	MapNameDenyOut         = "deny_out"
-	MapNameDNSAllow        = "dns_allow"
-	MapNameDNSQueryTrack   = "dns_query_track"
-	MapNameDirectNeighbors = "direct_neigh"
+	MapNameAllowOut      = "allow_out"
+	MapNameAllowOutV2    = "allow_out_v2"
+	MapNameDenyOut       = "deny_out"
+	MapNameDNSAllow      = "dns_allow"
+	MapNameDNSQueryTrack = "dns_query_track"
 	// constants referenced by BPF programs.
 	globalNameMVMInnerIP           = "mvm_inner_ip"
 	globalNameMVMMacaddrP1         = "mvm_macaddr_p1"
diff --git a/CubeNet/cubevs/miscs.go b/CubeNet/cubevs/miscs.go
index 71da591e8..c8fa3a95a 100644
--- a/CubeNet/cubevs/miscs.go
+++ b/CubeNet/cubevs/miscs.go
@@ -31,8 +31,7 @@ func init() {
 	_ = rlimit.RemoveMemlock()
 }
 
-func rewriteConstants(spec *ebpf.CollectionSpec, params Params) error {
-	vars := spec.Variables
+func rewriteConstants(vars map[string]*ebpf.VariableSpec, params Params) error {
 	var err error
 	err = errors.Join(err, vars[globalNameMVMInnerIP].Set(ipToUint32(params.MVMInnerIP)))
 	err = errors.Join(err, vars[globalNameMVMMacaddrP1].Set(hardwareAddrToUint32(params.MVMMacAddr)))
@@ -58,13 +57,7 @@ func rewriteConstants(spec *ebpf.CollectionSpec, params Params) error {
 		err = errors.Join(err, v.Set(params.EgressRedirectFlags))
 	}
 	err = errors.Join(err, vars[globalNameNodeIP].Set(ipToUint32(params.NodeIP)))
-	if spec.Programs[programNameFromCube] != nil {
-		nodeNetmask, netmaskErr := ipMaskToUint32(params.NodeIPMask)
-		err = errors.Join(err, netmaskErr)
-		if netmaskErr == nil {
-			err = errors.Join(err, vars[globalNameNodeNetmask].Set(nodeNetmask))
-		}
-	}
+	err = errors.Join(err, vars[globalNameNodeNetmask].Set(ipMaskToUint32(params.NodeIPMask)))
 	err = errors.Join(err, vars[globalNameNodeIfindex].Set(params.NodeIfindex))
 	err = errors.Join(err, vars[globalNameNodeMacaddrP1].Set(hardwareAddrToUint32(params.NodeMacAddr)))
 	err = errors.Join(err, vars[globalNameNodeMacaddrP2].Set(hardwareAddrToUint16(params.NodeMacAddr)))
@@ -201,7 +194,7 @@ func loadObject(params Params, loader func() (*ebpf.CollectionSpec, error), name
 		return fmt.Errorf("%s populateDNSTailCalls failed: %w", name, err)
 	}
 
-	err = rewriteConstants(spec, params)
+	err = rewriteConstants(spec.Variables, params)
 	if err != nil {
 		return fmt.Errorf("%s rewriteConstants failed: %w", name, err)
 	}
@@ -236,13 +229,6 @@ func attachTCFilter(progName string, ifindex uint32, direction TCDirection) erro
 
 // Init should be called once before invoking any other CubeVS APIs.
 func Init(params Params) error {
-	_ = os.Remove(pinPath("tungrp_to_tuns")) // NOCC:Path Traversal()
-	// dns_query_track is runtime pending-query state, not persisted policy.
-	_ = os.Remove(pinPath(MapNameDNSQueryTrack)) // NOCC:Path Traversal()
-	// Direct-neighbor state is tied to the current host interface and must not
-	// survive a CubeVS restart with stale MAC addresses.
-	_ = os.Remove(pinPath(MapNameDirectNeighbors)) // NOCC:Path Traversal()
-
 	err := loadObject(params, loadLocalgw, "loadLocalgw")
 	if err != nil {
 		return err
diff --git a/CubeNet/cubevs/miscs_test.go b/CubeNet/cubevs/miscs_test.go
index 9330071a5..0f2bc8dfa 100644
--- a/CubeNet/cubevs/miscs_test.go
+++ b/CubeNet/cubevs/miscs_test.go
@@ -1,64 +1,11 @@
 package cubevs
 
 import (
-	"net"
 	"testing"
 
 	"github.com/cilium/ebpf"
 )
 
-func testRewriteConstantsParams() Params {
-	return Params{
-		MVMInnerIP:          net.IPv4(169, 254, 68, 6),
-		MVMMacAddr:          net.HardwareAddr{0x20, 0x90, 0x6f, 0xfc, 0xfc, 0xfc},
-		MVMGatewayIP:        net.IPv4(169, 254, 68, 5),
-		Cubegw0Ifindex:      9,
-		Cubegw0IP:           net.IPv4(192, 168, 0, 1),
-		Cubegw0MacAddr:      net.HardwareAddr{0x20, 0x90, 0x6f, 0xcf, 0xcf, 0xcf},
-		EgressSrcMacAddr:    net.HardwareAddr{0x52, 0x54, 0x00, 0x68, 0xdd, 0x16},
-		EgressDstMacAddr:    net.HardwareAddr{0xfe, 0xee, 0x32, 0x47, 0x6b, 0x93},
-		EgressRedirectFlags: 0,
-		NodeIfindex:         2,
-		NodeIP:              net.IPv4(10, 2, 3, 4),
-		NodeIPMask:          net.CIDRMask(24, 32),
-		NodeMacAddr:         net.HardwareAddr{0x52, 0x54, 0x00, 0x68, 0xdd, 0x16},
-		NodeGatewayMacAddr:  net.HardwareAddr{0xfe, 0xee, 0x32, 0x47, 0x6b, 0x93},
-	}
-}
-
-func TestRewriteConstantsSetsNodeNetmask(t *testing.T) {
-	spec, err := loadMvmtap()
-	if err != nil {
-		t.Fatal(err)
-	}
-	params := testRewriteConstantsParams()
-	if err := rewriteConstants(spec, params); err != nil {
-		t.Fatal(err)
-	}
-
-	variable, ok := spec.Variables[globalNameNodeNetmask]
-	if !ok {
-		t.Fatalf("BPF variable %q not found", globalNameNodeNetmask)
-	}
-	var got uint32
-	if err := variable.Get(&got); err != nil {
-		t.Fatal(err)
-	}
-	if got != 0x00ffffff {
-		t.Fatalf("BPF node netmask = %#08x, want %#08x", got, uint32(0x00ffffff))
-	}
-
-	// Objects without from_cube do not use the netmask constant.
-	params.NodeIPMask = nil
-	localgwSpec, err := loadLocalgw()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := rewriteConstants(localgwSpec, params); err != nil {
-		t.Fatalf("rewrite unused node netmask: %v", err)
-	}
-}
-
 func TestPopulateDNSTailCallsBindsQueryPipelinePrograms(t *testing.T) {
 	spec := &ebpf.CollectionSpec{
 		Maps: map[string]*ebpf.MapSpec{
diff --git a/CubeNet/cubevs/util.go b/CubeNet/cubevs/util.go
index 9c52764d6..aa351f592 100644
--- a/CubeNet/cubevs/util.go
+++ b/CubeNet/cubevs/util.go
@@ -3,7 +3,6 @@ package cubevs
 import (
 	"bytes"
 	"encoding/binary"
-	"fmt"
 	"net"
 )
 
@@ -27,17 +26,9 @@ func ipToUint32(ip net.IP) uint32 {
 	return uint32(ip[0]) | uint32(ip[1])<<8 | uint32(ip[2])<<16 | uint32(ip[3])<<24
 }
 
-// ipMaskToUint32 converts an IPv4 mask to the byte layout used by IPv4
-// addresses in BPF packet headers on little-endian hosts.
-func ipMaskToUint32(mask net.IPMask) (uint32, error) {
-	if len(mask) != net.IPv4len {
-		return 0, fmt.Errorf("invalid IPv4 mask length: %d", len(mask))
-	}
-	if _, bits := mask.Size(); bits != 32 {
-		return 0, fmt.Errorf("invalid IPv4 mask: %v", mask)
-	}
-
-	return uint32(mask[0]) | uint32(mask[1])<<8 | uint32(mask[2])<<16 | uint32(mask[3])<<24, nil
+// ipMaskToUint32 converts an IPv4 mask to the same byte layout as ipToUint32.
+func ipMaskToUint32(mask net.IPMask) uint32 {
+	return uint32(mask[0]) | uint32(mask[1])<<8 | uint32(mask[2])<<16 | uint32(mask[3])<<24
 }
 
 // hardwareAddrToUint32 converts the first 4 bytes of MAC address to a uint32.
diff --git a/CubeNet/cubevs/util_test.go b/CubeNet/cubevs/util_test.go
deleted file mode 100644
index ca4f6d6de..000000000
--- a/CubeNet/cubevs/util_test.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package cubevs
-
-import (
-	"net"
-	"testing"
-)
-
-func TestIPMaskToUint32(t *testing.T) {
-	tests := []struct {
-		name string
-		mask net.IPMask
-		want uint32
-	}{
-		{name: "slash zero", mask: net.CIDRMask(0, 32), want: 0x00000000},
-		{name: "slash sixteen", mask: net.CIDRMask(16, 32), want: 0x0000ffff},
-		{name: "slash twenty four", mask: net.CIDRMask(24, 32), want: 0x00ffffff},
-		{name: "slash thirty two", mask: net.CIDRMask(32, 32), want: 0xffffffff},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := ipMaskToUint32(tt.mask)
-			if err != nil {
-				t.Fatal(err)
-			}
-			if got != tt.want {
-				t.Fatalf("ipMaskToUint32(%v) = %#08x, want %#08x", tt.mask, got, tt.want)
-			}
-		})
-	}
-}
-
-func TestIPMaskToUint32RejectsNonIPv4Mask(t *testing.T) {
-	if _, err := ipMaskToUint32(net.CIDRMask(64, 128)); err == nil {
-		t.Fatal("ipMaskToUint32 accepted an IPv6 mask")
-	}
-	if _, err := ipMaskToUint32(net.IPMask{255, 0, 255, 0}); err == nil {
-		t.Fatal("ipMaskToUint32 accepted a non-contiguous IPv4 mask")
-	}
-}
diff --git a/CubeNet/src/cubevs.h b/CubeNet/src/cubevs.h
index 48a31a42b..89317096f 100644
--- a/CubeNet/src/cubevs.h
+++ b/CubeNet/src/cubevs.h
@@ -38,8 +38,8 @@
 #define NET_POLICY_FLAG_L7_REQUIRED	1
 #define NSEC_PER_SEC			1000000000ULL
 #define DNS_QUERY_TRACK_TTL_NS		(10ULL * NSEC_PER_SEC)
-/* Min interval between ARP probes for the same unresolved on-link neighbor. */
-#define DIRECT_NEIGH_PROBE_INTERVAL_NS	(1ULL * NSEC_PER_SEC)
+#define DIRECT_NEIGH_PROBE_INTERVAL_NS		(1ULL * NSEC_PER_SEC)
+#define DIRECT_NEIGH_REVALIDATE_INTERVAL_NS	(5ULL * 60 * NSEC_PER_SEC)
 
 /* https://en.wikipedia.org/wiki/IPv4#Header
  *
@@ -140,7 +140,6 @@ struct arp_packet {
 struct direct_neighbor {
 	unsigned char addr[ETH_ALEN];
 	__u16 reserved;
-	/* Suppresses repeated ARP probes while resolution is pending. */
 	__u64 next_probe_at_ns;
 };
 
diff --git a/CubeNet/src/mvmtap.bpf.c b/CubeNet/src/mvmtap.bpf.c
index eb985e0db..86ac1d13d 100644
--- a/CubeNet/src/mvmtap.bpf.c
+++ b/CubeNet/src/mvmtap.bpf.c
@@ -106,12 +106,7 @@ static __always_inline bool should_do_nat(const struct iphdr *l3)
 	return true;
 }
 
-/* Direct egress normally bypasses the host network stack and sends packets to
- * the node gateway MAC. On-link destinations need the real neighbor MAC from
- * the ARP-learned cache instead.
- * "On-link" deliberately means the primary node IPv4 prefix here, not every
- * directly connected route that may exist on a multi-homed host.
- */
+/* Primary IPv4 prefix of the node NIC. Direct mode only. */
 static __always_inline bool direct_egress_is_onlink(__u32 daddr)
 {
 	return egress_redirect_flags == 0 &&
@@ -151,8 +146,7 @@ static __always_inline long direct_egress_clear_arp_padding(struct __sk_buff *sk
 				   2 * DIRECT_ARP_ZERO_CHUNK_LEN, 0);
 }
 
-static __always_inline int direct_egress_arp_request(struct __sk_buff *skb,
-						      __u32 dst_ifindex, __u32 daddr)
+static __always_inline long direct_egress_arp_request(struct __sk_buff *skb, __u32 daddr)
 {
 	struct arp_packet packet = {};
 	union macaddr *macaddr;
@@ -175,10 +169,8 @@ static __always_inline int direct_egress_arp_request(struct __sk_buff *skb,
 	packet.arp.ar_sip = nodenic_ip;
 	packet.arp.ar_tip = daddr;
 
-	/* TAP packets may carry CHECKSUM_PARTIAL metadata. Preserve enough space for
-	 * the delayed checksum write, move it beyond the ARP header, then clear the
-	 * entire padding so no original packet data is broadcast. change_tail also
-	 * clears any GSO state.
+	/* Leave padding after the ARP header so a delayed CHECKSUM_PARTIAL
+	 * write cannot clobber ar_tip, then zero it so we do not leak payload.
 	 */
 	err = bpf_skb_change_tail(skb, DIRECT_ARP_PRESERVED_LEN, 0);
 	if (err)
@@ -193,67 +185,65 @@ static __always_inline int direct_egress_arp_request(struct __sk_buff *skb,
 	if (err)
 		return TC_ACT_SHOT;
 
-	return bpf_redirect(dst_ifindex, 0);
+	return 0;
 }
 
-static __always_inline int redirect_egress(struct __sk_buff *skb, __u32 dst_ifindex,
-					   __u32 daddr)
+#define EGRESS_MAC_DROP		(-1)
+#define EGRESS_MAC_READY	0
+#define EGRESS_MAC_PROBE	1
+
+/* READY: L2 rewritten. PROBE: skb is now an ARP request. DROP: wait. */
+static __always_inline int prepare_egress_l2(struct __sk_buff *skb,
+					     struct ethhdr *l2, __u32 daddr)
 {
 	struct direct_neighbor pending = {};
 	struct direct_neighbor *neighbor;
 	union macaddr *neighbor_mac;
-	struct ethhdr *l2;
-	void *data, *data_end;
+	__u64 retry_at;
 	__u64 now;
 	long err;
 
-	if (!direct_egress_is_onlink(daddr))
-		return bpf_redirect(dst_ifindex, egress_redirect_flags);
-
-	neighbor = bpf_map_lookup_elem(&direct_neigh, &daddr);
-	if (neighbor && !direct_neighbor_is_zero(neighbor))
-		goto redirect_neighbor;
+	if (!direct_egress_is_onlink(daddr)) {
+		set_mac_pair(l2, egress_smacaddr_p1, egress_smacaddr_p2,
+			     egress_dmacaddr_p1, egress_dmacaddr_p2);
+		return EGRESS_MAC_READY;
+	}
 
-	/* Keep a pending entry so from_world only learns neighbors requested by
-	 * this path. Rate-limit ARP retries per destination while waiting for a
-	 * reply so unresolved floods do not emit one broadcast per packet.
-	 */
 	now = bpf_ktime_get_ns();
+	neighbor = bpf_map_lookup_elem(&direct_neigh, &daddr);
 	if (neighbor) {
-		if (neighbor->next_probe_at_ns > now)
-			return TC_ACT_SHOT;
+		if (neighbor->next_probe_at_ns > now) {
+			if (!direct_neighbor_is_zero(neighbor))
+				goto set_neighbor;
+			return EGRESS_MAC_DROP;
+		}
 
-		/* Do not replace the whole value here: learn_direct_neighbor may have
-		 * installed a MAC after the lookup above. Updating only the deadline
-		 * cannot overwrite that learned state. Refresh the lookup because an LRU
-		 * hash update replaces the entry rather than updating it in place.
-		 */
-		neighbor->next_probe_at_ns = now + DIRECT_NEIGH_PROBE_INTERVAL_NS;
+		/* Touch only the deadline so a concurrent learn cannot lose its MAC. */
+		retry_at = now + DIRECT_NEIGH_PROBE_INTERVAL_NS;
+		neighbor->next_probe_at_ns = retry_at;
 		neighbor = bpf_map_lookup_elem(&direct_neigh, &daddr);
-		if (neighbor && !direct_neighbor_is_zero(neighbor))
-			goto redirect_neighbor;
+		if (!neighbor)
+			return EGRESS_MAC_DROP;
+		/* Later deadline: another CPU already refreshed this entry. */
+		if (!direct_neighbor_is_zero(neighbor) &&
+		    neighbor->next_probe_at_ns > retry_at)
+			goto set_neighbor;
 	} else {
 		pending.next_probe_at_ns = now + DIRECT_NEIGH_PROBE_INTERVAL_NS;
 		err = bpf_map_update_elem(&direct_neigh, &daddr, &pending, BPF_NOEXIST);
 		if (err)
-			return TC_ACT_SHOT;
+			return EGRESS_MAC_DROP;
 	}
 
-	return direct_egress_arp_request(skb, dst_ifindex, daddr);
+	if (direct_egress_arp_request(skb, daddr))
+		return EGRESS_MAC_DROP;
+	return EGRESS_MAC_PROBE;
 
-redirect_neighbor:
-	err = bpf_skb_pull_data(skb, sizeof(struct ethhdr));
-	if (err)
-		return TC_ACT_SHOT;
-	data = (void *)(__u64)skb->data;
-	data_end = (void *)(__u64)skb->data_end;
-	if (data + sizeof(struct ethhdr) > data_end)
-		return TC_ACT_SHOT;
-	l2 = data;
+set_neighbor:
 	neighbor_mac = (union macaddr *)neighbor->addr;
 	set_mac_pair(l2, nodenic_macaddr_p1, nodenic_macaddr_p2,
 		     neighbor_mac->p1, neighbor_mac->p2);
-	return bpf_redirect(dst_ifindex, 0);
+	return EGRESS_MAC_READY;
 }
 
 /*
@@ -594,6 +584,7 @@ static __always_inline __u32 do_icmp_nat(struct __sk_buff *skb, struct mvm_meta
 	__u16 snat_id;
 	__u64 flags;
 	__u64 now;
+	int mac_result;
 	long err;
 	bool ok;
 
@@ -641,8 +632,11 @@ static __always_inline __u32 do_icmp_nat(struct __sk_buff *skb, struct mvm_meta
 	icmp_csum_off = ICMP_CSUM_OFF(ip_hlen);
 
 	/* update L2 first: csum/store helpers may invalidate packet pointers */
-	set_mac_pair(l2, egress_smacaddr_p1, egress_smacaddr_p2,
-		     egress_dmacaddr_p1, egress_dmacaddr_p2);
+	mac_result = prepare_egress_l2(skb, l2, key.dst_ip);
+	if (mac_result == EGRESS_MAC_DROP)
+		return 0;
+	if (mac_result == EGRESS_MAC_PROBE)
+		return sess->node_ifindex;
 
 	/* update ICMP csum: ICMP has no pseudo-header, so no BPF_F_PSEUDO_HDR.
 	 * Only the echo identifier change affects the csum (IP saddr is not
@@ -697,6 +691,7 @@ static __always_inline __u32 do_udp_nat_inline(struct __sk_buff *skb,
 	__u16 snat_port;
 	__u64 flags;
 	__u64 now;
+	int mac_result;
 	long err;
 	bool ok;
 
@@ -740,8 +735,11 @@ static __always_inline __u32 do_udp_nat_inline(struct __sk_buff *skb,
 	udp_csum_off = UDP_CSUM_OFF(ip_hlen);
 
 	/* update L2 first: csum/store helpers may invalidate packet pointers */
-	set_mac_pair(l2, egress_smacaddr_p1, egress_smacaddr_p2,
-		     egress_dmacaddr_p1, egress_dmacaddr_p2);
+	mac_result = prepare_egress_l2(skb, l2, key.dst_ip);
+	if (mac_result == EGRESS_MAC_DROP)
+		return 0;
+	if (mac_result == EGRESS_MAC_PROBE)
+		return sess->node_ifindex;
 
 	/* update UDP csum only if it was non-zero (UDP csum is optional over IPv4).
 	 * BPF_F_MARK_MANGLED_0 keeps a 0 csum (= disabled) intact in case the
@@ -798,13 +796,12 @@ static __noinline __attribute__((noinline)) __u32 do_udp_nat(struct __sk_buff *s
  * cannot make bpf-to-bpf calls (see do_udp_nat_inline()'s comment).
  */
 static __always_inline int finish_udp_nat_inline(struct __sk_buff *skb,
-						 struct mvm_meta *mvm_meta,
-						 __u32 daddr)
+						 struct mvm_meta *mvm_meta)
 {
 	__u32 dst_ifindex = do_udp_nat_inline(skb, mvm_meta);
 
 	if (dst_ifindex)
-		return redirect_egress(skb, dst_ifindex, daddr);
+		return bpf_redirect(dst_ifindex, egress_redirect_flags);
 
 	return TC_ACT_SHOT;
 }
@@ -812,14 +809,10 @@ static __always_inline int finish_udp_nat_inline(struct __sk_buff *skb,
 /* Subprog-based version used by dns_finish. */
 static __always_inline int finish_udp_nat(struct __sk_buff *skb, struct mvm_meta *mvm_meta)
 {
-	__u32 daddr;
 	__u32 dst_ifindex = do_udp_nat(skb, mvm_meta);
 
-	if (dst_ifindex) {
-		if (bpf_skb_load_bytes(skb, IP_DADDR_OFF, &daddr, sizeof(daddr)))
-			return TC_ACT_SHOT;
-		return redirect_egress(skb, dst_ifindex, daddr);
-	}
+	if (dst_ifindex)
+		return bpf_redirect(dst_ifindex, egress_redirect_flags);
 
 	return TC_ACT_SHOT;
 }
@@ -844,6 +837,7 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta *
 	__u16 snat_port;
 	__u64 flags;
 	__u64 now;
+	int mac_result;
 	long err;
 	bool ok;
 
@@ -913,8 +907,11 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta *
 	tcp_csum_off = TCP_CSUM_OFF(ip_hlen);
 
 	/* update L2 first: csum/store helpers may invalidate packet pointers */
-	set_mac_pair(l2, egress_smacaddr_p1, egress_smacaddr_p2,
-		     egress_dmacaddr_p1, egress_dmacaddr_p2);
+	mac_result = prepare_egress_l2(skb, l2, key.dst_ip);
+	if (mac_result == EGRESS_MAC_DROP)
+		return TCP_NAT_DROP;
+	if (mac_result == EGRESS_MAC_PROBE)
+		return TCP_NAT_PACK(sess->node_ifindex, TCP_NAT_OK);
 
 	/* update TCP csum: IP saddr is part of pseudo-header, so BPF_F_PSEUDO_HDR */
 	flags = BPF_F_PSEUDO_HDR | sizeof(old_saddr);
@@ -1033,26 +1030,24 @@ int dns_finish(struct __sk_buff *skb)
 	if (!mvm_meta)
 		return TC_ACT_SHOT;
 	if (!dns_policy_enabled(mvm_meta))
-		goto do_nat;
+		return finish_udp_nat(skb, mvm_meta);
 
 	inner_map = bpf_map_lookup_elem(&dns_allow, &ifindex);
 	if (!inner_map)
-		goto do_nat;
+		return finish_udp_nat(skb, mvm_meta);
 
 	question_cursor = state->dns_off + DNS_HDR_LEN;
 	if (state->failed)
-		goto do_nat;
+		return finish_udp_nat(skb, mvm_meta);
 	if (!dns_hash_qname(skb, &question_cursor, &question_footer,
 					&qname_hash))
-		goto do_nat;
+		return finish_udp_nat(skb, mvm_meta);
 
 	matched = dns_allow_match_value(inner_map, question);
 	if (!matched)
-		goto do_nat;
+		return finish_udp_nat(skb, mvm_meta);
 
 	dns_track_allowed_query(skb, state, matched->flags, qname_hash);
-
-do_nat:
 	return finish_udp_nat(skb, mvm_meta);
 }
 
@@ -1196,7 +1191,7 @@ int from_cube(struct __sk_buff *skb)
 			return bpf_redirect(cubegw0_ifindex, BPF_F_INGRESS);
 		tcp_ret = do_tcp_nat(skb, mvm_meta);
 		if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_OK)
-			return redirect_egress(skb, TCP_NAT_IFINDEX(tcp_ret), daddr);
+			return bpf_redirect(TCP_NAT_IFINDEX(tcp_ret), egress_redirect_flags);
 		if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_RESET)
 			return tcp_reply_reset(skb, ifindex);
 	}
@@ -1212,13 +1207,13 @@ int from_cube(struct __sk_buff *skb)
 				return ret;
 		}
 
-		return finish_udp_nat_inline(skb, mvm_meta, daddr);
+		return finish_udp_nat_inline(skb, mvm_meta);
 	}
 
 	if (proto == IPPROTO_ICMP) {
 		dst_ifindex = do_icmp_nat(skb, mvm_meta);
 		if (dst_ifindex)
-			return redirect_egress(skb, dst_ifindex, daddr);
+			return bpf_redirect(dst_ifindex, egress_redirect_flags);
 	}
 
 	return TC_ACT_SHOT;
diff --git a/CubeNet/src/nodenic.bpf.c b/CubeNet/src/nodenic.bpf.c
index 131f84e2f..131e89b36 100644
--- a/CubeNet/src/nodenic.bpf.c
+++ b/CubeNet/src/nodenic.bpf.c
@@ -17,7 +17,7 @@
 #include "dns_query.h"
 #include "dns_response.h"
 
-/* Learn the sender of an ARP packet only when direct egress already created a
+/* Learn the sender of an ARP reply only when direct egress already created a
  * pending cache entry for that IP. This also refreshes an existing entry when
  * the peer announces a MAC change.
  */
@@ -44,12 +44,12 @@ static __always_inline void learn_direct_neighbor(struct __sk_buff *skb)
 	    packet->arp.ar_pro != bpf_htons(ETH_P_IP) ||
 	    packet->arp.ar_hln != ETH_ALEN ||
 	    packet->arp.ar_pln != sizeof(__be32) ||
-	    (packet->arp.ar_op != bpf_htons(ARPOP_REQUEST) &&
-	     packet->arp.ar_op != bpf_htons(ARPOP_REPLY)))
+	    packet->arp.ar_op != bpf_htons(ARPOP_REPLY))
 		return;
 
 	eth_src = (union macaddr *)packet->eth.h_source;
 	arp_src = (union macaddr *)packet->arp.ar_sha;
+	/* Only trust a nonzero unicast sender MAC that matches the Ethernet header. */
 	if (eth_src->p1 != arp_src->p1 || eth_src->p2 != arp_src->p2 ||
 	    (arp_src->addr[0] & 1) || (arp_src->p1 == 0 && arp_src->p2 == 0))
 		return;
@@ -57,6 +57,8 @@ static __always_inline void learn_direct_neighbor(struct __sk_buff *skb)
 	mac = (union macaddr *)neighbor.addr;
 	mac->p1 = arp_src->p1;
 	mac->p2 = arp_src->p2;
+	neighbor.next_probe_at_ns = bpf_ktime_get_ns() +
+				    DIRECT_NEIGH_REVALIDATE_INTERVAL_NS;
 	ip = packet->arp.ar_sip;
 	bpf_map_update_elem(&direct_neigh, &ip, &neighbor, BPF_EXIST);
 }
@@ -394,9 +396,7 @@ static int icmp_nat_session(struct __sk_buff *skb, struct ethhdr *l2, struct iph
 	return bpf_redirect(sess->vm_ifindex, 0);
 }
 
-/* from_world performs a DNS tail call, so these dispatch helpers must remain
- * inline for kernels that reject tail calls from programs with BPF subcalls.
- */
+/* Linux 5.4 rejects tail calls from programs with BPF subcalls. */
 static __always_inline int do_icmp_nat(struct __sk_buff *skb)
 {
 	struct ethhdr *l2;
diff --git a/Cubelet/network/runtime/systemnet/device.go b/Cubelet/network/runtime/systemnet/device.go
index 057e296ce..37b6d8683 100644
--- a/Cubelet/network/runtime/systemnet/device.go
+++ b/Cubelet/network/runtime/systemnet/device.go
@@ -42,21 +42,13 @@ func GetHostDevice(ifName string) (*HostDevice, error) {
 	if err != nil {
 		return nil, err
 	}
-	addrs, err := netlinkAddrList(link, netlink.FAMILY_V4)
+	addrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
 	if err != nil {
 		return nil, err
 	}
 	if len(addrs) != 1 {
 		return nil, fmt.Errorf("ipv4 address on %s is not unique", ifName)
 	}
-	if addrs[0].IPNet == nil {
-		return nil, fmt.Errorf("invalid ipv4 address on %s", ifName)
-	}
-	ip := addrs[0].IP.To4()
-	_, bits := addrs[0].Mask.Size()
-	if ip == nil || bits != 32 {
-		return nil, fmt.Errorf("invalid ipv4 address on %s", ifName)
-	}
 	gwMac, err := GetGatewayMacAddr(ifName)
 	if err != nil {
 		return nil, err
@@ -68,8 +60,8 @@ func GetHostDevice(ifName string) (*HostDevice, error) {
 	return &HostDevice{
 		Index:      link.Attrs().Index,
 		Name:       link.Attrs().Name,
-		IP:         append(net.IP(nil), ip...),
-		IPMask:     append(net.IPMask(nil), addrs[0].Mask...),
+		IP:         addrs[0].IP,
+		IPMask:     addrs[0].Mask,
 		Mac:        link.Attrs().HardwareAddr,
 		GatewayMac: gatewayMac,
 	}, nil

From 8afbd055cf85f7418798d2a9586a42bdfcd4d5fe Mon Sep 17 00:00:00 2001
From: jay3cx <137191587@qq.com>
Date: Fri, 14 Aug 2026 15:27:40 +0800
Subject: [PATCH 3/5] fix(cubenet): try fib lookup before the neighbor cache

Use bpf_fib_lookup on the node NIC first. If the kernel already has a
neighbor, reuse that MAC; otherwise keep the existing map and ARP path.

Signed-off-by: jay3cx <137191587@qq.com>
---
 CubeNet/src/cubevs.h      |  3 +++
 CubeNet/src/mvmtap.bpf.c  | 28 +++++++++++++++++++++++-----
 CubeNet/src/nodenic.bpf.c |  5 +----
 3 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/CubeNet/src/cubevs.h b/CubeNet/src/cubevs.h
index 89317096f..60e242d15 100644
--- a/CubeNet/src/cubevs.h
+++ b/CubeNet/src/cubevs.h
@@ -25,6 +25,9 @@
 /* ARP hardware types */
 #define ARPHRD_ETHER			1	/* Ethernet */
 
+/* https://elixir.bootlin.com/linux/v5.4.217/source/include/linux/socket.h#L172 */
+#define AF_INET				2
+
 #define MAX_ENTRIES			8192
 #define MAX_IP_RULE_ENTRIES		8192
 #define MAX_DOMAIN_RULE_ENTRIES		1024
diff --git a/CubeNet/src/mvmtap.bpf.c b/CubeNet/src/mvmtap.bpf.c
index 86ac1d13d..61404e7a2 100644
--- a/CubeNet/src/mvmtap.bpf.c
+++ b/CubeNet/src/mvmtap.bpf.c
@@ -169,8 +169,9 @@ static __always_inline long direct_egress_arp_request(struct __sk_buff *skb, __u
 	packet.arp.ar_sip = nodenic_ip;
 	packet.arp.ar_tip = daddr;
 
-	/* Leave padding after the ARP header so a delayed CHECKSUM_PARTIAL
-	 * write cannot clobber ar_tip, then zero it so we do not leak payload.
+	/* CHECKSUM_PARTIAL can write L4 csum after we return.
+	 * change_tail(96) keeps that slot. change_head(32) moves it past ARP.
+	 * Then store ARP and zero the rest (broadcast).
 	 */
 	err = bpf_skb_change_tail(skb, DIRECT_ARP_PRESERVED_LEN, 0);
 	if (err)
@@ -196,9 +197,17 @@ static __always_inline long direct_egress_arp_request(struct __sk_buff *skb, __u
 static __always_inline int prepare_egress_l2(struct __sk_buff *skb,
 					     struct ethhdr *l2, __u32 daddr)
 {
+	struct bpf_fib_lookup fib = {
+		.family = AF_INET,
+		.ifindex = nodenic_ifindex,
+		.ipv4_src = nodenic_ip,
+		.ipv4_dst = daddr,
+	};
 	struct direct_neighbor pending = {};
 	struct direct_neighbor *neighbor;
 	union macaddr *neighbor_mac;
+	const union macaddr *dmac;
+	const union macaddr *smac;
 	__u64 retry_at;
 	__u64 now;
 	long err;
@@ -209,6 +218,15 @@ static __always_inline int prepare_egress_l2(struct __sk_buff *skb,
 		return EGRESS_MAC_READY;
 	}
 
+	err = bpf_fib_lookup(skb, &fib, sizeof(fib),
+			     BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT);
+	if (err == BPF_FIB_LKUP_RET_SUCCESS && fib.ifindex == nodenic_ifindex) {
+		smac = (const union macaddr *)fib.smac;
+		dmac = (const union macaddr *)fib.dmac;
+		set_mac_pair(l2, smac->p1, smac->p2, dmac->p1, dmac->p2);
+		return EGRESS_MAC_READY;
+	}
+
 	now = bpf_ktime_get_ns();
 	neighbor = bpf_map_lookup_elem(&direct_neigh, &daddr);
 	if (neighbor) {
@@ -580,11 +598,11 @@ static __always_inline __u32 do_icmp_nat(struct __sk_buff *skb, struct mvm_meta
 	struct ethhdr *l2;
 	struct iphdr *l3;
 	struct icmphdr *l4;
+	int mac_result;
 	__u16 ip_hlen;
 	__u16 snat_id;
 	__u64 flags;
 	__u64 now;
-	int mac_result;
 	long err;
 	bool ok;
 
@@ -687,11 +705,11 @@ static __always_inline __u32 do_udp_nat_inline(struct __sk_buff *skb,
 	struct ethhdr *l2;
 	struct iphdr *l3;
 	struct udphdr *l4;
+	int mac_result;
 	__u16 ip_hlen;
 	__u16 snat_port;
 	__u64 flags;
 	__u64 now;
-	int mac_result;
 	long err;
 	bool ok;
 
@@ -833,11 +851,11 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta *
 	struct ethhdr *l2;
 	struct iphdr *l3;
 	struct tcphdr *l4;
+	int mac_result;
 	__u16 ip_hlen;
 	__u16 snat_port;
 	__u64 flags;
 	__u64 now;
-	int mac_result;
 	long err;
 	bool ok;
 
diff --git a/CubeNet/src/nodenic.bpf.c b/CubeNet/src/nodenic.bpf.c
index 131e89b36..586d4076b 100644
--- a/CubeNet/src/nodenic.bpf.c
+++ b/CubeNet/src/nodenic.bpf.c
@@ -17,10 +17,7 @@
 #include "dns_query.h"
 #include "dns_response.h"
 
-/* Learn the sender of an ARP reply only when direct egress already created a
- * pending cache entry for that IP. This also refreshes an existing entry when
- * the peer announces a MAC change.
- */
+/* Learn ARP replies only for IPs already present in the map. */
 static __always_inline void learn_direct_neighbor(struct __sk_buff *skb)
 {
 	struct direct_neighbor neighbor = {};

From a5538f810f138022136b8e9525ed530b8e921c52 Mon Sep 17 00:00:00 2001
From: jay3cx <137191587@qq.com>
Date: Mon, 17 Aug 2026 18:53:37 +0800
Subject: [PATCH 4/5] fix(cubenet): prepare on-link L2 once in from_cube

Move neighbor lookup and ARP conversion out of the per-protocol NAT
helpers so from_cube rewrites L2 in one place before those helpers run.

Signed-off-by: jay3cx <137191587@qq.com>
---
 CubeNet/src/mvmtap.bpf.c | 34 ++++++++++------------------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/CubeNet/src/mvmtap.bpf.c b/CubeNet/src/mvmtap.bpf.c
index 61404e7a2..d3a58820b 100644
--- a/CubeNet/src/mvmtap.bpf.c
+++ b/CubeNet/src/mvmtap.bpf.c
@@ -598,7 +598,6 @@ static __always_inline __u32 do_icmp_nat(struct __sk_buff *skb, struct mvm_meta
 	struct ethhdr *l2;
 	struct iphdr *l3;
 	struct icmphdr *l4;
-	int mac_result;
 	__u16 ip_hlen;
 	__u16 snat_id;
 	__u64 flags;
@@ -649,13 +648,6 @@ static __always_inline __u32 do_icmp_nat(struct __sk_buff *skb, struct mvm_meta
 	ip_hlen <<= 2;
 	icmp_csum_off = ICMP_CSUM_OFF(ip_hlen);
 
-	/* update L2 first: csum/store helpers may invalidate packet pointers */
-	mac_result = prepare_egress_l2(skb, l2, key.dst_ip);
-	if (mac_result == EGRESS_MAC_DROP)
-		return 0;
-	if (mac_result == EGRESS_MAC_PROBE)
-		return sess->node_ifindex;
-
 	/* update ICMP csum: ICMP has no pseudo-header, so no BPF_F_PSEUDO_HDR.
 	 * Only the echo identifier change affects the csum (IP saddr is not
 	 * covered by ICMP checksum).
@@ -705,7 +697,6 @@ static __always_inline __u32 do_udp_nat_inline(struct __sk_buff *skb,
 	struct ethhdr *l2;
 	struct iphdr *l3;
 	struct udphdr *l4;
-	int mac_result;
 	__u16 ip_hlen;
 	__u16 snat_port;
 	__u64 flags;
@@ -752,13 +743,6 @@ static __always_inline __u32 do_udp_nat_inline(struct __sk_buff *skb,
 	ip_hlen <<= 2;
 	udp_csum_off = UDP_CSUM_OFF(ip_hlen);
 
-	/* update L2 first: csum/store helpers may invalidate packet pointers */
-	mac_result = prepare_egress_l2(skb, l2, key.dst_ip);
-	if (mac_result == EGRESS_MAC_DROP)
-		return 0;
-	if (mac_result == EGRESS_MAC_PROBE)
-		return sess->node_ifindex;
-
 	/* update UDP csum only if it was non-zero (UDP csum is optional over IPv4).
 	 * BPF_F_MARK_MANGLED_0 keeps a 0 csum (= disabled) intact in case the
 	 * incremental update would yield 0; the helper rewrites it to 0xffff.
@@ -851,7 +835,6 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta *
 	struct ethhdr *l2;
 	struct iphdr *l3;
 	struct tcphdr *l4;
-	int mac_result;
 	__u16 ip_hlen;
 	__u16 snat_port;
 	__u64 flags;
@@ -924,13 +907,6 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta *
 	ip_hlen <<= 2;
 	tcp_csum_off = TCP_CSUM_OFF(ip_hlen);
 
-	/* update L2 first: csum/store helpers may invalidate packet pointers */
-	mac_result = prepare_egress_l2(skb, l2, key.dst_ip);
-	if (mac_result == EGRESS_MAC_DROP)
-		return TCP_NAT_DROP;
-	if (mac_result == EGRESS_MAC_PROBE)
-		return TCP_NAT_PACK(sess->node_ifindex, TCP_NAT_OK);
-
 	/* update TCP csum: IP saddr is part of pseudo-header, so BPF_F_PSEUDO_HDR */
 	flags = BPF_F_PSEUDO_HDR | sizeof(old_saddr);
 	err = bpf_l4_csum_replace(skb, tcp_csum_off, old_saddr, new_saddr, flags);
@@ -1085,6 +1061,7 @@ int from_cube(struct __sk_buff *skb)
 	struct iphdr *l3;
 	struct tcphdr *l4;
 	struct udphdr *udp;
+	int mac_result;
 	__u16 *host_port;
 	__u32 dns_off;
 	__u8 proto;
@@ -1207,6 +1184,15 @@ int from_cube(struct __sk_buff *skb)
 			return TC_ACT_SHOT;
 		if (should_redirect_to_l7_proxy(ifindex, daddr, l4))
 			return bpf_redirect(cubegw0_ifindex, BPF_F_INGRESS);
+	}
+
+	mac_result = prepare_egress_l2(skb, l2, daddr);
+	if (mac_result == EGRESS_MAC_DROP)
+		return TC_ACT_SHOT;
+	if (mac_result == EGRESS_MAC_PROBE)
+		return bpf_redirect(nodenic_ifindex, 0);
+
+	if (proto == IPPROTO_TCP) {
 		tcp_ret = do_tcp_nat(skb, mvm_meta);
 		if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_OK)
 			return bpf_redirect(TCP_NAT_IFINDEX(tcp_ret), egress_redirect_flags);

From 1699a2db2930f911315df31431c878f5d914c296 Mon Sep 17 00:00:00 2001
From: jay3cx <137191587@qq.com>
Date: Thu, 20 Aug 2026 11:38:47 +0800
Subject: [PATCH 5/5] fix(cubenet): clarify CHECKSUM_PARTIAL ARP comment

change_tail cannot shrink below the late L4 write (min_len).
change_head still moves that write past the ARP header.

Signed-off-by: jay3cx <137191587@qq.com>
---
 CubeNet/src/mvmtap.bpf.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CubeNet/src/mvmtap.bpf.c b/CubeNet/src/mvmtap.bpf.c
index 29734b7f0..defd13f51 100644
--- a/CubeNet/src/mvmtap.bpf.c
+++ b/CubeNet/src/mvmtap.bpf.c
@@ -170,8 +170,8 @@ static __always_inline long direct_egress_arp_request(struct __sk_buff *skb, __u
 	packet.arp.ar_tip = daddr;
 
 	/* CHECKSUM_PARTIAL can write L4 csum after we return.
-	 * change_tail(96) keeps that slot. change_head(32) moves it past ARP.
-	 * Then store ARP and zero the rest (broadcast).
+	 * change_tail will not shrink below that write (min_len).
+	 * change_head(32) moves it past ARP. Then store ARP and zero the rest.
 	 */
 	err = bpf_skb_change_tail(skb, DIRECT_ARP_PRESERVED_LEN, 0);
 	if (err)