diff --git a/CubeNet/cubevs/cubevs.go b/CubeNet/cubevs/cubevs.go index 8d87a242d..b9165af9e 100644 --- a/CubeNet/cubevs/cubevs.go +++ b/CubeNet/cubevs/cubevs.go @@ -34,6 +34,7 @@ type Params struct { // Ifindex, IP and MAC address of Node itself NodeIfindex uint32 NodeIP net.IP + NodeIPMask net.IPMask NodeMacAddr net.HardwareAddr // MAC address of the Node gateway (next hop) NodeGatewayMacAddr net.HardwareAddr @@ -238,6 +239,7 @@ const ( globalNameEgressDMacaddrP2 = "egress_dmacaddr_p2" globalNameEgressRedirectFlags = "egress_redirect_flags" globalNameNodeIP = "nodenic_ip" + globalNameNodeNetmask = "nodenic_netmask" globalNameNodeIfindex = "nodenic_ifindex" globalNameNodeMacaddrP1 = "nodenic_macaddr_p1" globalNameNodeMacaddrP2 = "nodenic_macaddr_p2" diff --git a/CubeNet/cubevs/miscs.go b/CubeNet/cubevs/miscs.go index 6f1125bc1..bd098b337 100644 --- a/CubeNet/cubevs/miscs.go +++ b/CubeNet/cubevs/miscs.go @@ -103,6 +103,7 @@ func rewriteConstants(vars map[string]*ebpf.VariableSpec, params Params) error { err = errors.Join(err, v.Set(params.EgressRedirectFlags)) } err = errors.Join(err, vars[globalNameNodeIP].Set(ipToUint32(params.NodeIP))) + err = errors.Join(err, vars[globalNameNodeNetmask].Set(ipMaskToUint32(params.NodeIPMask))) err = errors.Join(err, vars[globalNameNodeIfindex].Set(params.NodeIfindex)) err = errors.Join(err, vars[globalNameNodeMacaddrP1].Set(hardwareAddrToUint32(params.NodeMacAddr))) err = errors.Join(err, vars[globalNameNodeMacaddrP2].Set(hardwareAddrToUint16(params.NodeMacAddr))) diff --git a/CubeNet/cubevs/util.go b/CubeNet/cubevs/util.go index e9b466440..aa351f592 100644 --- a/CubeNet/cubevs/util.go +++ b/CubeNet/cubevs/util.go @@ -26,6 +26,11 @@ func ipToUint32(ip net.IP) uint32 { return uint32(ip[0]) | uint32(ip[1])<<8 | uint32(ip[2])<<16 | uint32(ip[3])<<24 } +// ipMaskToUint32 converts an IPv4 mask to the same byte layout as ipToUint32. +func ipMaskToUint32(mask net.IPMask) uint32 { + return uint32(mask[0]) | uint32(mask[1])<<8 | uint32(mask[2])<<16 | uint32(mask[3])<<24 +} + // hardwareAddrToUint32 converts the first 4 bytes of MAC address to a uint32. func hardwareAddrToUint32(addr net.HardwareAddr) uint32 { return uint32(addr[0]) | uint32(addr[1])<<8 | uint32(addr[2])<<16 | uint32(addr[3])<<24 diff --git a/CubeNet/src/cubevs.h b/CubeNet/src/cubevs.h index 60bbf7284..ea9753c8b 100644 --- a/CubeNet/src/cubevs.h +++ b/CubeNet/src/cubevs.h @@ -25,6 +25,9 @@ /* ARP hardware types */ #define ARPHRD_ETHER 1 /* Ethernet */ +/* https://elixir.bootlin.com/linux/v5.4.217/source/include/linux/socket.h#L172 */ +#define AF_INET 2 + #define MAX_ENTRIES 8192 #define MAX_IP_RULE_ENTRIES 8192 #define MAX_DOMAIN_RULE_ENTRIES 1024 @@ -76,6 +79,8 @@ const volatile __u32 cube_l7_mark_mask = 0xFFFF0000u; const volatile __u32 cube_l7_mark_http = 0xCE010000u; const volatile __u32 cube_l7_mark_https = 0xCE020000u; #define DNS_QUERY_TRACK_TTL_NS (10ULL * NSEC_PER_SEC) +#define DIRECT_NEIGH_PROBE_INTERVAL_NS (1ULL * NSEC_PER_SEC) +#define DIRECT_NEIGH_REVALIDATE_INTERVAL_NS (5ULL * 60 * NSEC_PER_SEC) /* https://en.wikipedia.org/wiki/IPv4#Header * @@ -137,6 +142,7 @@ const volatile __u64 egress_redirect_flags = BPF_F_INGRESS; /* Ifindex, IP and MAC address of Node itself */ const volatile __u32 nodenic_ip = 0x020a8709; /* 9.135.10.2, network byte order */ +const volatile __u32 nodenic_netmask = 0x00ffffff; /* 255.255.255.0, packet-byte layout */ const volatile __u32 nodenic_ifindex = 2; const volatile __u32 nodenic_macaddr_p1 = 0x68005452; /* 52:54:00:68:dd:16 */ const volatile __u16 nodenic_macaddr_p2 = 0x16dd; @@ -167,6 +173,17 @@ struct arphdr_eth { __be32 ar_tip; /* target IP address */ } __attribute__((packed)); +struct arp_packet { + struct ethhdr eth; + struct arphdr_eth arp; +} __attribute__((packed)); + +struct direct_neighbor { + unsigned char addr[ETH_ALEN]; + __u16 reserved; + __u64 next_probe_at_ns; +}; + union macaddr { struct { __u32 p1; diff --git a/CubeNet/src/map.h b/CubeNet/src/map.h index 4ea2d3a48..5ba7953c4 100644 --- a/CubeNet/src/map.h +++ b/CubeNet/src/map.h @@ -96,6 +96,20 @@ struct { __uint(pinning, LIBBPF_PIN_BY_NAME); } snat_iplist SEC(".maps"); +/* Direct-egress on-link neighbor cache. + * + * key: destination IPv4 address in packet-byte layout + * value: destination MAC plus probe deadline; an all-zero MAC means initial + * resolution is pending + */ +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, MAX_ENTRIES); + __type(key, __u32); + __type(value, struct direct_neighbor); + __uint(pinning, LIBBPF_PIN_BY_NAME); +} direct_neigh SEC(".maps"); + /* Egress allow list v3 (hash of maps) * * key: ifindex of the TAP device diff --git a/CubeNet/src/mvmtap.bpf.c b/CubeNet/src/mvmtap.bpf.c index c265d6bc6..defd13f51 100644 --- a/CubeNet/src/mvmtap.bpf.c +++ b/CubeNet/src/mvmtap.bpf.c @@ -106,6 +106,155 @@ static __always_inline bool should_do_nat(const struct iphdr *l3) return true; } +/* Primary IPv4 prefix of the node NIC. Direct mode only. */ +static __always_inline bool direct_egress_is_onlink(__u32 daddr) +{ + return egress_redirect_flags == 0 && + (daddr & nodenic_netmask) == (nodenic_ip & nodenic_netmask); +} + +static __always_inline bool direct_neighbor_is_zero(const struct direct_neighbor *neighbor) +{ + const union macaddr *macaddr = (const union macaddr *)neighbor->addr; + + return macaddr->p1 == 0 && macaddr->p2 == 0; +} + +#define DIRECT_ARP_PRESERVED_LEN 96 +#define DIRECT_ARP_HEADROOM 32 +#define DIRECT_ARP_FRAME_LEN (DIRECT_ARP_PRESERVED_LEN + DIRECT_ARP_HEADROOM) +#define DIRECT_ARP_ZERO_CHUNK_LEN 32 + +static __always_inline long direct_egress_clear_arp_padding(struct __sk_buff *skb) +{ + unsigned char zeroes[DIRECT_ARP_ZERO_CHUNK_LEN] = {}; + long err; + + err = bpf_skb_store_bytes(skb, sizeof(struct arp_packet), + zeroes, sizeof(zeroes), 0); + if (err) + return err; + err = bpf_skb_store_bytes(skb, + sizeof(struct arp_packet) + DIRECT_ARP_ZERO_CHUNK_LEN, + zeroes, sizeof(zeroes), 0); + if (err) + return err; + return bpf_skb_store_bytes(skb, + sizeof(struct arp_packet) + 2 * DIRECT_ARP_ZERO_CHUNK_LEN, + zeroes, + DIRECT_ARP_FRAME_LEN - sizeof(struct arp_packet) - + 2 * DIRECT_ARP_ZERO_CHUNK_LEN, 0); +} + +static __always_inline long direct_egress_arp_request(struct __sk_buff *skb, __u32 daddr) +{ + struct arp_packet packet = {}; + union macaddr *macaddr; + long err; + + __builtin_memset(packet.eth.h_dest, 0xff, ETH_ALEN); + macaddr = (union macaddr *)packet.eth.h_source; + macaddr->p1 = nodenic_macaddr_p1; + macaddr->p2 = nodenic_macaddr_p2; + packet.eth.h_proto = bpf_htons(ETH_P_ARP); + + packet.arp.ar_hrd = bpf_htons(ARPHRD_ETHER); + packet.arp.ar_pro = bpf_htons(ETH_P_IP); + packet.arp.ar_hln = ETH_ALEN; + packet.arp.ar_pln = sizeof(__be32); + packet.arp.ar_op = bpf_htons(ARPOP_REQUEST); + macaddr = (union macaddr *)packet.arp.ar_sha; + macaddr->p1 = nodenic_macaddr_p1; + macaddr->p2 = nodenic_macaddr_p2; + packet.arp.ar_sip = nodenic_ip; + packet.arp.ar_tip = daddr; + + /* CHECKSUM_PARTIAL can write L4 csum after we return. + * change_tail will not shrink below that write (min_len). + * change_head(32) moves it past ARP. Then store ARP and zero the rest. + */ + err = bpf_skb_change_tail(skb, DIRECT_ARP_PRESERVED_LEN, 0); + if (err) + return TC_ACT_SHOT; + err = bpf_skb_change_head(skb, DIRECT_ARP_HEADROOM, 0); + if (err) + return TC_ACT_SHOT; + err = bpf_skb_store_bytes(skb, 0, &packet, sizeof(packet), 0); + if (err) + return TC_ACT_SHOT; + err = direct_egress_clear_arp_padding(skb); + if (err) + return TC_ACT_SHOT; + + return 0; +} + +#define EGRESS_MAC_DROP (-1) +#define EGRESS_MAC_READY 0 +#define EGRESS_MAC_PROBE 1 + +/* READY: L2 rewritten. PROBE: skb is now an ARP request. DROP: wait. */ +static __always_inline int prepare_egress_l2(struct __sk_buff *skb, + struct ethhdr *l2, __u32 daddr) +{ + struct bpf_fib_lookup fib = { + .family = AF_INET, + .ifindex = nodenic_ifindex, + .ipv4_src = nodenic_ip, + .ipv4_dst = daddr, + }; + struct direct_neighbor pending = {}; + struct direct_neighbor *neighbor; + union macaddr *neighbor_mac; + const union macaddr *dmac; + const union macaddr *smac; + __u64 now; + long err; + + if (!direct_egress_is_onlink(daddr)) { + set_mac_pair(l2, egress_smacaddr_p1, egress_smacaddr_p2, + egress_dmacaddr_p1, egress_dmacaddr_p2); + return EGRESS_MAC_READY; + } + + err = bpf_fib_lookup(skb, &fib, sizeof(fib), + BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT); + if (err == BPF_FIB_LKUP_RET_SUCCESS && fib.ifindex == nodenic_ifindex) { + smac = (const union macaddr *)fib.smac; + dmac = (const union macaddr *)fib.dmac; + set_mac_pair(l2, smac->p1, smac->p2, dmac->p1, dmac->p2); + return EGRESS_MAC_READY; + } + + now = bpf_ktime_get_ns(); + neighbor = bpf_map_lookup_elem(&direct_neigh, &daddr); + if (neighbor) { + if (neighbor->next_probe_at_ns > now) { + if (!direct_neighbor_is_zero(neighbor)) + goto set_neighbor; + return EGRESS_MAC_DROP; + } + + /* A concurrent learn may make this probe redundant, which is harmless. */ + neighbor->next_probe_at_ns = now + DIRECT_NEIGH_PROBE_INTERVAL_NS; + } else { + pending.next_probe_at_ns = now + DIRECT_NEIGH_PROBE_INTERVAL_NS; + err = bpf_map_update_elem(&direct_neigh, &daddr, &pending, BPF_NOEXIST); + if (err) + return EGRESS_MAC_DROP; + } + + if (direct_egress_arp_request(skb, daddr)) + return EGRESS_MAC_DROP; + return EGRESS_MAC_PROBE; + +set_neighbor: + neighbor_mac = (union macaddr *)neighbor->addr; + set_mac_pair(l2, nodenic_macaddr_p1, nodenic_macaddr_p2, + neighbor_mac->p1, neighbor_mac->p2); + return EGRESS_MAC_READY; +} + /* Egress flow classification now lives in classify_egress_flow() (session.h), * which merges the former l7_scheme_for_flow() and session_policy_allowed() * into a single policy verdict (reject / accept-SNAT / accept-HTTP / @@ -115,6 +264,7 @@ static __always_inline bool should_do_nat(const struct iphdr *l3) enum tcp_nat_result { TCP_NAT_DROP = 0, TCP_NAT_OK, + TCP_NAT_PROBE, TCP_NAT_RESET, TCP_L7PROXY_OK, }; @@ -493,10 +643,6 @@ static __always_inline __u32 do_icmp_nat(struct __sk_buff *skb, struct mvm_meta ip_hlen <<= 2; icmp_csum_off = ICMP_CSUM_OFF(ip_hlen); - /* update L2 first: csum/store helpers may invalidate packet pointers */ - set_mac_pair(l2, egress_smacaddr_p1, egress_smacaddr_p2, - egress_dmacaddr_p1, egress_dmacaddr_p2); - /* update ICMP csum: ICMP has no pseudo-header, so no BPF_F_PSEUDO_HDR. * Only the echo identifier change affects the csum (IP saddr is not * covered by ICMP checksum). @@ -595,10 +741,6 @@ static __always_inline __u32 do_udp_nat_inline(struct __sk_buff *skb, ip_hlen <<= 2; udp_csum_off = UDP_CSUM_OFF(ip_hlen); - /* update L2 first: csum/store helpers may invalidate packet pointers */ - set_mac_pair(l2, egress_smacaddr_p1, egress_smacaddr_p2, - egress_dmacaddr_p1, egress_dmacaddr_p2); - /* update UDP csum only if it was non-zero (UDP csum is optional over IPv4). * BPF_F_MARK_MANGLED_0 keeps a 0 csum (= disabled) intact in case the * incremental update would yield 0; the helper rewrites it to 0xffff. @@ -700,6 +842,8 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * __u8 packet_class = SNAT_PACKET; __u8 l7_scheme = L7_SCHEME_NONE; __u8 verdict = FLOW_SNAT; + bool create_snat = false; + int mac_result; long err; bool ok; @@ -766,11 +910,10 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * return TCP_NAT_PACK(0, TCP_NAT_RESET); case FLOW_SNAT: default: - snat_ip = pick_snat_ip_port(mvm_meta->ip, &key, &snat_port); - if (!snat_ip || !snat_ip->ip || !snat_port) - return TCP_NAT_DROP; - break; + create_snat = true; + goto prepare_snat; } +create_session: ok = create_new_sessions(skb, &key, now, skb->ingress_ifindex, snat_ip, snat_port, packet_class, l7_scheme); if (!ok) @@ -824,6 +967,35 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * } do_update: + if (sess->packet_class == L7PROXY_PACKET) + goto update_existing; + +prepare_snat: + /* Resolve external L2 before allocating or mutating TCP session state. + * A cold miss consumes this packet as an ARP probe, so the original TCP + * packet must remain untouched and a new session must not be installed. + */ + mac_result = prepare_egress_l2(skb, l2, key.dst_ip); + if (mac_result == EGRESS_MAC_DROP) + return TCP_NAT_DROP; + if (mac_result == EGRESS_MAC_PROBE) + return TCP_NAT_PACK(nodenic_ifindex, TCP_NAT_PROBE); + + if (create_snat) { + snat_ip = pick_snat_ip_port(mvm_meta->ip, &key, &snat_port); + if (!snat_ip || !snat_ip->ip || !snat_port) + return TCP_NAT_DROP; + goto create_session; + } + + /* prepare_egress_l2() may update the neighbor map. Reacquire the session + * value before updating it or using it for the NAT rewrite. + */ + sess = bpf_map_lookup_elem(&egress_sessions, &key); + if (!sess || sess->packet_class == L7PROXY_PACKET) + return TCP_NAT_DROP; + +update_existing: /* update session */ update_session(IP_CT_DIR_ORIGINAL, sess, now, syn, ack, fin, rst); @@ -849,10 +1021,6 @@ static __always_inline __u64 do_tcp_nat(struct __sk_buff *skb, struct mvm_meta * ip_hlen <<= 2; tcp_csum_off = TCP_CSUM_OFF(ip_hlen); - /* update L2 first: csum/store helpers may invalidate packet pointers */ - set_mac_pair(l2, egress_smacaddr_p1, egress_smacaddr_p2, - egress_dmacaddr_p1, egress_dmacaddr_p2); - /* update TCP csum: IP saddr is part of pseudo-header, so BPF_F_PSEUDO_HDR */ flags = BPF_F_PSEUDO_HDR | sizeof(old_saddr); err = bpf_l4_csum_replace(skb, tcp_csum_off, old_saddr, new_saddr, flags); @@ -1005,6 +1173,7 @@ int from_cube(struct __sk_buff *skb) struct iphdr *l3; struct tcphdr *l4; struct udphdr *udp; + int mac_result; __u16 *host_port; __u32 dns_off; __u8 proto; @@ -1112,12 +1281,21 @@ int from_cube(struct __sk_buff *skb) tcp_ret = do_tcp_nat(skb, mvm_meta); if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_OK) return bpf_redirect(TCP_NAT_IFINDEX(tcp_ret), egress_redirect_flags); + if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_PROBE) + return bpf_redirect(TCP_NAT_IFINDEX(tcp_ret), 0); if (TCP_NAT_STATUS(tcp_ret) == TCP_L7PROXY_OK) return bpf_redirect(TCP_NAT_IFINDEX(tcp_ret), BPF_F_INGRESS); if (TCP_NAT_STATUS(tcp_ret) == TCP_NAT_RESET) return tcp_reply_reset(skb, ifindex); + return TC_ACT_SHOT; } + mac_result = prepare_egress_l2(skb, l2, daddr); + if (mac_result == EGRESS_MAC_DROP) + return TC_ACT_SHOT; + if (mac_result == EGRESS_MAC_PROBE) + return bpf_redirect(nodenic_ifindex, 0); + if (proto == IPPROTO_UDP) { if (!__pull_headers_udp(skb, &l2, &l3, &udp)) return TC_ACT_SHOT; diff --git a/CubeNet/src/nodenic.bpf.c b/CubeNet/src/nodenic.bpf.c index 32c2380d4..5ada5f0bc 100644 --- a/CubeNet/src/nodenic.bpf.c +++ b/CubeNet/src/nodenic.bpf.c @@ -17,6 +17,49 @@ #include "dns_query.h" #include "dns_response.h" +/* Learn ARP replies only for IPs already present in the map. */ +static __always_inline void learn_direct_neighbor(struct __sk_buff *skb) +{ + struct direct_neighbor neighbor = {}; + struct arp_packet *packet; + union macaddr *eth_src, *arp_src, *mac; + void *data, *data_end; + __u32 ip; + + if (skb->ifindex != nodenic_ifindex || + bpf_skb_pull_data(skb, sizeof(struct arp_packet))) + return; + + data = (void *)(__u64)skb->data; + data_end = (void *)(__u64)skb->data_end; + if (data + sizeof(struct arp_packet) > data_end) + return; + + packet = data; + if (packet->eth.h_proto != bpf_htons(ETH_P_ARP) || + packet->arp.ar_hrd != bpf_htons(ARPHRD_ETHER) || + packet->arp.ar_pro != bpf_htons(ETH_P_IP) || + packet->arp.ar_hln != ETH_ALEN || + packet->arp.ar_pln != sizeof(__be32) || + packet->arp.ar_op != bpf_htons(ARPOP_REPLY)) + return; + + eth_src = (union macaddr *)packet->eth.h_source; + arp_src = (union macaddr *)packet->arp.ar_sha; + /* Only trust a nonzero unicast sender MAC that matches the Ethernet header. */ + if (eth_src->p1 != arp_src->p1 || eth_src->p2 != arp_src->p2 || + (arp_src->addr[0] & 1) || (arp_src->p1 == 0 && arp_src->p2 == 0)) + return; + + mac = (union macaddr *)neighbor.addr; + mac->p1 = arp_src->p1; + mac->p2 = arp_src->p2; + neighbor.next_probe_at_ns = bpf_ktime_get_ns() + + DIRECT_NEIGH_REVALIDATE_INTERVAL_NS; + ip = packet->arp.ar_sip; + bpf_map_update_elem(&direct_neigh, &ip, &neighbor, BPF_EXIST); +} + static int tcp_nat_proxy(struct __sk_buff *skb, struct ethhdr *l2, struct iphdr *l3, struct tcphdr *l4, struct mvm_port *mvm_port) { @@ -332,7 +375,8 @@ static int icmp_nat_session(struct __sk_buff *skb, struct ethhdr *l2, struct iph return bpf_redirect(sess->vm_ifindex, 0); } -static int do_icmp_nat(struct __sk_buff *skb) +/* Linux 5.4 rejects tail calls from programs with BPF subcalls. */ +static __always_inline int do_icmp_nat(struct __sk_buff *skb) { struct ethhdr *l2; struct iphdr *l3; @@ -344,7 +388,7 @@ static int do_icmp_nat(struct __sk_buff *skb) return icmp_nat_session(skb, l2, l3, l4); } -static int do_udp_nat(struct __sk_buff *skb) +static __always_inline int do_udp_nat(struct __sk_buff *skb) { struct ethhdr *l2; struct iphdr *l3; @@ -356,7 +400,7 @@ static int do_udp_nat(struct __sk_buff *skb) return udp_nat_session(skb, l2, l3, l4); } -static int do_tcp_nat(struct __sk_buff *skb) +static __always_inline int do_tcp_nat(struct __sk_buff *skb) { struct mvm_port *mvm_port; struct ethhdr *l2; @@ -391,6 +435,10 @@ int from_world(struct __sk_buff *skb) struct iphdr *l3; int ret; + if (skb->protocol == bpf_htons(ETH_P_ARP)) { + learn_direct_neighbor(skb); + return TC_ACT_OK; + } if (skb->protocol != bpf_htons(ETH_P_IP)) return TC_ACT_OK; diff --git a/Cubelet/network/runtime/controller.go b/Cubelet/network/runtime/controller.go index 5670728f0..753326cb5 100644 --- a/Cubelet/network/runtime/controller.go +++ b/Cubelet/network/runtime/controller.go @@ -289,6 +289,7 @@ func initCubeVS(cfg Config, device *systemnet.HostDevice, cubeDev *systemnet.Cub CubeRouterIfindex: cubeRouterIfindex, NodeIfindex: uint32(device.Index), NodeIP: device.IP, + NodeIPMask: device.IPMask, NodeMacAddr: device.Mac, NodeGatewayMacAddr: device.GatewayMac, } diff --git a/Cubelet/network/runtime/systemnet/device.go b/Cubelet/network/runtime/systemnet/device.go index bcf8a6ed3..57689cce6 100644 --- a/Cubelet/network/runtime/systemnet/device.go +++ b/Cubelet/network/runtime/systemnet/device.go @@ -65,6 +65,7 @@ type HostDevice struct { Index int Name string IP net.IP + IPMask net.IPMask Mac net.HardwareAddr GatewayMac net.HardwareAddr } @@ -95,6 +96,7 @@ func GetHostDevice(ifName string) (*HostDevice, error) { Index: link.Attrs().Index, Name: link.Attrs().Name, IP: addrs[0].IP, + IPMask: addrs[0].Mask, Mac: link.Attrs().HardwareAddr, GatewayMac: gatewayMac, }, nil