Merge pull request #8790 from tomastigera/tomas-bpf-vxlan-tunnel-src-port-fix

[BPF] Variable source port for VXLAN NodePort tunnel

Author: tomastigera

felix/bpf-gpl/nat4.h

@@ -19,7 +19,7 @@
 #define VXLAN_ENCAP_SIZE	(sizeof(struct ethhdr) + sizeof(struct iphdr) + \
 				sizeof(struct udphdr) + sizeof(struct vxlanhdr))
 
-static CALI_BPF_INLINE int vxlan_encap(struct cali_tc_ctx *ctx,  __be32 *ip_src, __be32 *ip_dst)
+static CALI_BPF_INLINE int vxlan_encap(struct cali_tc_ctx *ctx,  __be32 *ip_src, __be32 *ip_dst, __u16 src_port)
 {
 	int ret;
 	__wsum csum;
@@ -65,7 +65,8 @@ static CALI_BPF_INLINE int vxlan_encap(struct cali_tc_ctx *ctx,  __be32 *ip_src,
 	ip_hdr(ctx)->check = 0;
 	ip_hdr(ctx)->protocol = IPPROTO_UDP;
 
-	udp->source = udp->dest = bpf_htons(VXLAN_PORT);
+	udp->source = bpf_htons(src_port);
+	udp->dest = bpf_htons(VXLAN_PORT);
 	udp->len = bpf_htons(bpf_ntohs(ip_hdr(ctx)->tot_len) - sizeof(struct iphdr));
 
 	*((__u8*)&vxlan->flags) = 1 << 3; /* set the I flag to make the VNI valid */

felix/bpf-gpl/nat6.h

@@ -19,7 +19,7 @@
 #define VXLAN_ENCAP_SIZE	(sizeof(struct ethhdr) + sizeof(struct iphdr) + \
 				sizeof(struct udphdr) + sizeof(struct vxlanhdr))
 
-static CALI_BPF_INLINE int vxlan_encap(struct cali_tc_ctx *ctx, ipv6_addr_t *ip_src, ipv6_addr_t *ip_dst)
+static CALI_BPF_INLINE int vxlan_encap(struct cali_tc_ctx *ctx, ipv6_addr_t *ip_src, ipv6_addr_t *ip_dst, __u16 src_port)
 {
 	__u32 new_hdrsz = sizeof(struct ethhdr) + sizeof(struct ipv6hdr) +
 			sizeof(struct udphdr) + sizeof(struct vxlanhdr);
@@ -56,7 +56,8 @@ static CALI_BPF_INLINE int vxlan_encap(struct cali_tc_ctx *ctx, ipv6_addr_t *ip_
 	ip_hdr(ctx)->payload_len = bpf_htons(bpf_ntohs(ip_hdr(ctx)->payload_len) + new_hdrsz);
 	ip_hdr(ctx)->nexthdr = IPPROTO_UDP;
 
-	udp->source = udp->dest = bpf_htons(VXLAN_PORT);
+	udp->source = bpf_htons(src_port);
+	udp->dest = bpf_htons(VXLAN_PORT);
 	udp->len = bpf_htons(bpf_ntohs(ip_hdr(ctx)->payload_len) - sizeof(struct iphdr));
 	/* XXX we leave udp->check == 0 which is not legal in IPv6, but we are
 	 * the only ones parsing that packet!

felix/bpf-gpl/tc.c

@@ -1053,12 +1053,22 @@ static CALI_BPF_INLINE enum do_nat_res do_nat(struct cali_tc_ctx *ctx,
 		}
 	}
 
-	if (vxlan_encap(ctx, &STATE->ip_src, &STATE->ip_dst)) {
+	/* Trivial hash to use multiple vxlan flows. Note that any value will do
+	 * as long as it is a constant for this direction of the flow. Does not
+	 * even need to be the same for both directions.
+	 *
+	 * ICMP does not have ports, but there is little to no worries about a
+	 * possible out-of-order processing in relation to the related flow.
+	 */
+	__u16 vxlan_src_port = STATE->sport ^ STATE->dport;
+
+	if (vxlan_encap(ctx, &STATE->ip_src, &STATE->ip_dst, vxlan_src_port)) {
 		deny_reason(ctx, CALI_REASON_ENCAP_FAIL);
 		goto  deny;
 	}
 
-	STATE->sport = STATE->dport = VXLAN_PORT;
+	STATE->sport = vxlan_src_port;
+	STATE->dport = VXLAN_PORT;
 	STATE->ip_proto = IPPROTO_UDP;
 
 	CALI_DEBUG("vxlan return %d ifindex_fwd %d\n",

felix/bpf-gpl/ut/ipv4_opts_test.c

@@ -47,7 +47,7 @@ static CALI_BPF_INLINE int calico_unittest_entry (struct __sk_buff *skb)
 	__u32 a = 0x06060606;
 	__u32 b = 0x10101010;
 
-	if (vxlan_encap(ctx, &a, &b)) {
+	if (vxlan_encap(ctx, &a, &b, 0xdead)) {
 		CALI_DEBUG("vxlan: encap failed!\n");
 		deny_reason(ctx, CALI_REASON_ENCAP_FAIL);
 		goto  deny;

felix/bpf-gpl/ut/nat_encap_test.c

@@ -35,5 +35,5 @@ static CALI_BPF_INLINE int calico_unittest_entry (struct __sk_buff *skb)
 	__u32 a = HOST_IP;
 	__u32 b = 0x02020202;
 
-	return vxlan_encap(ctx, &a, &b);
+	return vxlan_encap(ctx, &a, &b, 0xdead);
 }

felix/bpf/ut/nat_encap_test.go

@@ -106,7 +106,6 @@ func checkVxlan(pktR gopacket.Packet) gopacket.Packet {
 	udpL := pktR.Layer(layers.LayerTypeUDP)
 	Expect(udpL).NotTo(BeNil())
 	udpR := udpL.(*layers.UDP)
-	Expect(udpR.SrcPort).To(Equal(layers.UDPPort(testVxlanPort)))
 	Expect(udpR.DstPort).To(Equal(layers.UDPPort(testVxlanPort)))
 	Expect(udpR.Checksum).To(Equal(uint16(0)))
 
@@ -175,7 +174,6 @@ func getVxlanVNI(pktR gopacket.Packet) uint32 {
 	udpL := pktR.Layer(layers.LayerTypeUDP)
 	Expect(udpL).NotTo(BeNil())
 	udpR := udpL.(*layers.UDP)
-	Expect(udpR.SrcPort).To(Equal(layers.UDPPort(testVxlanPort)))
 	Expect(udpR.DstPort).To(Equal(layers.UDPPort(testVxlanPort)))
 	Expect(udpR.Checksum).To(Equal(uint16(0)))
 

felix/bpf/ut/nat_test.go

@@ -767,6 +767,8 @@ func TestNATNodePort(t *testing.T) {
 
 	dumpCTMap(ctMap)
 
+	var vxlanSrcPort layers.UDPPort
+
 	skbMark = 0
 	// Another pkt arriving at node 1 - uses existing CT entries
 	runBpfTest(t, "calico_from_host_ep", nil, func(bpfrun bpfProgRunFn) {
@@ -783,9 +785,44 @@ func TestNATNodePort(t *testing.T) {
 		Expect(ipv4R.SrcIP.String()).To(Equal(hostIP.String()))
 		Expect(ipv4R.DstIP.String()).To(Equal(node2ip.String()))
 
+		udpL := pktR.Layer(layers.LayerTypeUDP)
+		Expect(udpL).NotTo(BeNil())
+		udpR := udpL.(*layers.UDP)
+
+		Expect(udpR.SrcPort).To(Equal(udp.SrcPort ^ udp.DstPort))
+		vxlanSrcPort = udpR.SrcPort
+
 		checkVxlanEncap(pktR, false, ipv4, udp, payload)
 	})
 
+	skbMark = 0
+	// Another pkt iwith a different source port arriving at node 1
+	runBpfTest(t, "calico_from_host_ep", nil, func(bpfrun bpfProgRunFn) {
+
+		// Change the source port
+		pktBytes[14+(20+8)] = 0xde
+		pktBytes[14+(20+8)+1] = 0xad
+
+		res, err := bpfrun(pktBytes)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Retval).To(Equal(resTC_ACT_UNSPEC))
+
+		pktR := gopacket.NewPacket(res.dataOut, layers.LayerTypeEthernet, gopacket.Default)
+		fmt.Printf("pktR = %+v\n", pktR)
+
+		ipv4L := pktR.Layer(layers.LayerTypeIPv4)
+		Expect(ipv4L).NotTo(BeNil())
+		ipv4R := ipv4L.(*layers.IPv4)
+		Expect(ipv4R.SrcIP.String()).To(Equal(hostIP.String()))
+		Expect(ipv4R.DstIP.String()).To(Equal(node2ip.String()))
+
+		udpL := pktR.Layer(layers.LayerTypeUDP)
+		Expect(udpL).NotTo(BeNil())
+		udpR := udpL.(*layers.UDP)
+
+		Expect(udpR.SrcPort).NotTo(Equal(vxlanSrcPort))
+	})
+
 	expectMark(tcdefs.MarkSeenBypassForward)
 
 	/*