fix: repair the missing ip rules and routes

Signed-off-by: l1b0k <[email protected]>

cmd/terway/main.go

@@ -57,6 +57,7 @@ func main() {
 	log.Info(version.Version)
 
 	ctx := ctrl.SetupSignalHandler()
+	ctx = ctrl.LoggerInto(ctx, ctrl.Log)
 	err = daemon.Run(ctx, utils.NormalizePath(defaultSocketPath), readonlyListen, utils.NormalizePath(configFilePath), daemonMode)
 
 	if err != nil {

daemon/daemon.go

@@ -563,6 +563,10 @@ func (n *networkService) gcPods(ctx context.Context) error {
 
 		podID := utils.PodInfoKey(podRes.PodInfo.Namespace, podRes.PodInfo.Name)
 		if _, ok := exist[podID]; ok {
+			err = ruleSync(ctx, podRes)
+			if err != nil {
+				serviceLog.Error(err, "error sync pod rule")
+			}
 			continue
 		}
 

daemon/rule_linux.go

@@ -0,0 +1,120 @@
+package daemon
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/samber/lo"
+	"github.com/vishvananda/netlink"
+
+	"github.com/AliyunContainerService/terway/pkg/link"
+	"github.com/AliyunContainerService/terway/pkg/utils/nodecap"
+	"github.com/AliyunContainerService/terway/plugin/datapath"
+	"github.com/AliyunContainerService/terway/plugin/driver/types"
+	"github.com/AliyunContainerService/terway/plugin/driver/utils"
+	"github.com/AliyunContainerService/terway/rpc"
+	terwayTypes "github.com/AliyunContainerService/terway/types"
+	"github.com/AliyunContainerService/terway/types/daemon"
+)
+
+func ruleSync(ctx context.Context, res daemon.PodResources) error {
+	if res.PodInfo == nil {
+		return nil
+	}
+
+	if res.PodInfo.PodNetworkType != daemon.PodNetworkTypeENIMultiIP {
+		return nil
+	}
+
+	switch nodecap.GetNodeCapabilities(nodecap.NodeCapabilityDataPath) {
+	case "datapathv2", "veth", "":
+	default:
+		return nil
+	}
+
+	netConf := make([]*rpc.NetConf, 0)
+	err := json.Unmarshal([]byte(res.NetConf), &netConf)
+	if err != nil {
+		return nil
+	}
+
+	links, err := netlink.LinkList()
+	if err != nil {
+		return err
+	}
+
+	for _, conf := range netConf {
+		if conf.BasicInfo == nil || conf.ENIInfo == nil ||
+			conf.BasicInfo.PodIP == nil {
+			continue
+		}
+		ifName := "eth0"
+		if conf.IfName != "" {
+			ifName = conf.IfName
+		}
+
+		hostVethName, _ := link.VethNameForPod(res.PodInfo.Name, res.PodInfo.Namespace, ifName, "cali")
+
+		// check host veth ,make sure pod is present
+		hostVeth, ok := lo.Find(links, func(item netlink.Link) bool {
+			return hostVethName == item.Attrs().Name
+		})
+		if !ok {
+			continue
+		}
+
+		eni, ok := lo.Find(links, func(item netlink.Link) bool {
+			if _, ok := item.(*netlink.Device); !ok {
+				return false
+			}
+			return item.Attrs().HardwareAddr.String() == conf.ENIInfo.MAC
+		})
+		if !ok {
+			continue
+		}
+
+		setUp := &types.SetupConfig{
+			ContainerIPNet: &terwayTypes.IPNetSet{},
+			GatewayIP:      &terwayTypes.IPSet{},
+			ENIIndex:       eni.Attrs().Index,
+		}
+		if conf.BasicInfo.PodIP.IPv4 != "" {
+			setUp.ContainerIPNet.SetIPNet(conf.BasicInfo.PodIP.IPv4 + "/32")
+		}
+		if conf.BasicInfo.PodIP.IPv6 != "" {
+			setUp.ContainerIPNet.SetIPNet(conf.BasicInfo.PodIP.IPv6 + "/128")
+		}
+		setUp.GatewayIP.SetIP(conf.BasicInfo.GatewayIP.IPv4)
+		setUp.GatewayIP.SetIP(conf.BasicInfo.GatewayIP.IPv6)
+
+		// 1. route point to hostVeth
+		table := utils.GetRouteTableID(eni.Attrs().Index)
+
+		eniConf := datapath.GenerateENICfgForPolicy(setUp, eni, table)
+		hostVethConf := datapath.GenerateHostPeerCfgForPolicy(setUp, hostVeth, table)
+
+		// default via 10.xx.xx.253 dev eth1 onlink table 1003
+		for _, route := range eniConf.Routes {
+			_, err = utils.EnsureRoute(ctx, route)
+			if err != nil {
+				return err
+			}
+		}
+		// 10.xx.xx.xx dev calixx scope link
+		for _, route := range hostVethConf.Routes {
+			_, err = utils.EnsureRoute(ctx, route)
+			if err != nil {
+				return err
+			}
+		}
+
+		for _, rule := range hostVethConf.Rules {
+			_, err = utils.EnsureIPRule(ctx, rule)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

daemon/rule_linux_test.go

@@ -0,0 +1,61 @@
+//go:build linux
+
+package daemon
+
+import (
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vishvananda/netlink"
+
+	"github.com/AliyunContainerService/terway/plugin/datapath"
+	"github.com/AliyunContainerService/terway/plugin/driver/types"
+	terwayTypes "github.com/AliyunContainerService/terway/types"
+)
+
+func TestGenerateConfig(t *testing.T) {
+	setUp := &types.SetupConfig{
+		ContainerIPNet: &terwayTypes.IPNetSet{
+			IPv4: &net.IPNet{
+				IP:   net.ParseIP("10.0.0.2"),
+				Mask: net.CIDRMask(32, 32),
+			},
+			IPv6: &net.IPNet{
+				IP:   net.ParseIP("fd0::2"),
+				Mask: net.CIDRMask(128, 128),
+			},
+		},
+		GatewayIP: &terwayTypes.IPSet{
+			IPv4: net.ParseIP("169.254.169.254"),
+			IPv6: net.ParseIP("fd0::1"),
+		},
+		ENIIndex: 1,
+	}
+	eni := &netlink.GenericLink{
+		LinkAttrs: netlink.LinkAttrs{
+			Index: 1,
+			Name:  "eth1",
+		},
+	}
+	eniConf := datapath.GenerateENICfgForPolicy(setUp, eni, 1)
+
+	assert.Equal(t, []*netlink.Route{
+		{
+			Dst: &net.IPNet{
+				IP:   net.ParseIP("10.0.0.2"),
+				Mask: net.CIDRMask(32, 32),
+			},
+			Gw:    net.ParseIP("169.254.169.254"),
+			Table: 1,
+		},
+		{
+			Dst: &net.IPNet{
+				IP:   net.ParseIP("fd0::2"),
+				Mask: net.CIDRMask(128, 128),
+			},
+			Gw:    net.ParseIP("fd0::1"),
+			Table: 1,
+		},
+	}, eniConf.Routes)
+}

daemon/rule_unsupported.go

@@ -0,0 +1,11 @@
+//go:build !linux
+
+package daemon
+
+import (
+	"github.com/AliyunContainerService/terway/types/daemon"
+)
+
+func ruleSync(ctx context.Context, res daemon.PodResources) error {
+	return nil
+}

plugin/datapath/policy_router_linux.go

@@ -172,7 +172,7 @@ func generateContCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, mac net
 	return contCfg
 }
 
-func generateHostPeerCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf {
+func GenerateHostPeerCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf {
 	var addrs []*netlink.Addr
 	var routes []*netlink.Route
 	var rules []*netlink.Rule
@@ -247,7 +247,7 @@ func generateHostPeerCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, tab
 	}
 }
 
-func generateENICfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf {
+func GenerateENICfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf {
 	var routes []*netlink.Route
 	var rules []*netlink.Rule
 	var neighs []*netlink.Neigh
@@ -377,7 +377,7 @@ func (d *PolicyRoute) Setup(ctx context.Context, cfg *types.SetupConfig, netNS n
 
 	table := utils.GetRouteTableID(eni.Attrs().Index)
 
-	eniCfg := generateENICfgForPolicy(cfg, eni, table)
+	eniCfg := GenerateENICfgForPolicy(cfg, eni, table)
 	err = nic.Setup(ctx, eni, eniCfg)
 	if err != nil {
 		return fmt.Errorf("setup eni config, %w", err)
@@ -390,7 +390,7 @@ func (d *PolicyRoute) Setup(ctx context.Context, cfg *types.SetupConfig, netNS n
 		}
 	}
 
-	hostVETHCfg := generateHostPeerCfgForPolicy(cfg, hostVETH, table)
+	hostVETHCfg := GenerateHostPeerCfgForPolicy(cfg, hostVETH, table)
 	err = nic.Setup(ctx, hostVETH, hostVETHCfg)
 	if err != nil {
 		return fmt.Errorf("setup host veth config, %w", err)

plugin/driver/utils/utils_linux.go

@@ -174,6 +174,9 @@ func NewIPNet1(ipNet *terwayTypes.IPNetSet) []*netlink.Addr {
 }
 
 func NewIPNetToMaxMask(ipNet *terwayTypes.IPNetSet) []*netlink.Addr {
+	if ipNet == nil {
+		return nil
+	}
 	var addrs []*netlink.Addr
 	if ipNet.IPv4 != nil {
 		addrs = append(addrs, &netlink.Addr{IPNet: NewIPNetWithMaxMask(ipNet.IPv4)})