diff --git a/cmd/terway/main.go b/cmd/terway/main.go index 806b2e2c..72f1d4f4 100644 --- a/cmd/terway/main.go +++ b/cmd/terway/main.go @@ -57,6 +57,7 @@ func main() { log.Info(version.Version) ctx := ctrl.SetupSignalHandler() + ctx = ctrl.LoggerInto(ctx, ctrl.Log) err = daemon.Run(ctx, utils.NormalizePath(defaultSocketPath), readonlyListen, utils.NormalizePath(configFilePath), daemonMode) if err != nil { diff --git a/daemon/daemon.go b/daemon/daemon.go index 0d21c579..964497bf 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -563,6 +563,10 @@ func (n *networkService) gcPods(ctx context.Context) error { podID := utils.PodInfoKey(podRes.PodInfo.Namespace, podRes.PodInfo.Name) if _, ok := exist[podID]; ok { + err = ruleSync(ctx, podRes) + if err != nil { + serviceLog.Error(err, "error sync pod rule") + } continue } diff --git a/daemon/rule_linux.go b/daemon/rule_linux.go new file mode 100644 index 00000000..0748efa3 --- /dev/null +++ b/daemon/rule_linux.go @@ -0,0 +1,120 @@ +package daemon + +import ( + "context" + "encoding/json" + + "github.com/samber/lo" + "github.com/vishvananda/netlink" + + "github.com/AliyunContainerService/terway/pkg/link" + "github.com/AliyunContainerService/terway/pkg/utils/nodecap" + "github.com/AliyunContainerService/terway/plugin/datapath" + "github.com/AliyunContainerService/terway/plugin/driver/types" + "github.com/AliyunContainerService/terway/plugin/driver/utils" + "github.com/AliyunContainerService/terway/rpc" + terwayTypes "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/daemon" +) + +func ruleSync(ctx context.Context, res daemon.PodResources) error { + if res.PodInfo == nil { + return nil + } + + if res.PodInfo.PodNetworkType != daemon.PodNetworkTypeENIMultiIP { + return nil + } + + switch nodecap.GetNodeCapabilities(nodecap.NodeCapabilityDataPath) { + case "datapathv2", "veth", "": + default: + return nil + } + + netConf := make([]*rpc.NetConf, 0) + err := json.Unmarshal([]byte(res.NetConf), &netConf) + if err != nil { + return nil + } + + links, err := netlink.LinkList() + if err != nil { + return err + } + + for _, conf := range netConf { + if conf.BasicInfo == nil || conf.ENIInfo == nil || + conf.BasicInfo.PodIP == nil { + continue + } + ifName := "eth0" + if conf.IfName != "" { + ifName = conf.IfName + } + + hostVethName, _ := link.VethNameForPod(res.PodInfo.Name, res.PodInfo.Namespace, ifName, "cali") + + // check host veth ,make sure pod is present + hostVeth, ok := lo.Find(links, func(item netlink.Link) bool { + return hostVethName == item.Attrs().Name + }) + if !ok { + continue + } + + eni, ok := lo.Find(links, func(item netlink.Link) bool { + if _, ok := item.(*netlink.Device); !ok { + return false + } + return item.Attrs().HardwareAddr.String() == conf.ENIInfo.MAC + }) + if !ok { + continue + } + + setUp := &types.SetupConfig{ + ContainerIPNet: &terwayTypes.IPNetSet{}, + GatewayIP: &terwayTypes.IPSet{}, + ENIIndex: eni.Attrs().Index, + } + if conf.BasicInfo.PodIP.IPv4 != "" { + setUp.ContainerIPNet.SetIPNet(conf.BasicInfo.PodIP.IPv4 + "/32") + } + if conf.BasicInfo.PodIP.IPv6 != "" { + setUp.ContainerIPNet.SetIPNet(conf.BasicInfo.PodIP.IPv6 + "/128") + } + setUp.GatewayIP.SetIP(conf.BasicInfo.GatewayIP.IPv4) + setUp.GatewayIP.SetIP(conf.BasicInfo.GatewayIP.IPv6) + + // 1. route point to hostVeth + table := utils.GetRouteTableID(eni.Attrs().Index) + + eniConf := datapath.GenerateENICfgForPolicy(setUp, eni, table) + hostVethConf := datapath.GenerateHostPeerCfgForPolicy(setUp, hostVeth, table) + + // default via 10.xx.xx.253 dev eth1 onlink table 1003 + for _, route := range eniConf.Routes { + _, err = utils.EnsureRoute(ctx, route) + if err != nil { + return err + } + } + // 10.xx.xx.xx dev calixx scope link + for _, route := range hostVethConf.Routes { + _, err = utils.EnsureRoute(ctx, route) + if err != nil { + return err + } + } + + for _, rule := range hostVethConf.Rules { + _, err = utils.EnsureIPRule(ctx, rule) + if err != nil { + return err + } + } + } + + return nil +} diff --git a/daemon/rule_linux_test.go b/daemon/rule_linux_test.go new file mode 100644 index 00000000..e351e96b --- /dev/null +++ b/daemon/rule_linux_test.go @@ -0,0 +1,94 @@ +//go:build linux + +package daemon + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vishvananda/netlink" + + "github.com/AliyunContainerService/terway/plugin/datapath" + "github.com/AliyunContainerService/terway/plugin/driver/types" + terwayTypes "github.com/AliyunContainerService/terway/types" +) + +func TestGenerateConfig(t *testing.T) { + setUp := &types.SetupConfig{ + ContainerIPNet: &terwayTypes.IPNetSet{ + IPv4: &net.IPNet{ + IP: net.ParseIP("10.0.0.2"), + Mask: net.CIDRMask(32, 32), + }, + IPv6: &net.IPNet{ + IP: net.ParseIP("fd0::2"), + Mask: net.CIDRMask(128, 128), + }, + }, + GatewayIP: &terwayTypes.IPSet{ + IPv4: net.ParseIP("169.254.169.254"), + IPv6: net.ParseIP("fd0::1"), + }, + ENIIndex: 1, + } + eni := &netlink.GenericLink{ + LinkAttrs: netlink.LinkAttrs{ + Index: 1, + Name: "eth1", + }, + } + eniConf := datapath.GenerateENICfgForPolicy(setUp, eni, 1) + + assert.True(t, eniConf.Routes[0].Equal(netlink.Route{ + Dst: &net.IPNet{ + IP: net.ParseIP("0.0.0.0"), + Mask: net.CIDRMask(0, 0), + }, + Gw: net.ParseIP("169.254.169.254"), + Table: 1, + LinkIndex: 1, + Scope: netlink.SCOPE_UNIVERSE, + Flags: int(netlink.FLAG_ONLINK), + })) + assert.True(t, eniConf.Routes[2].Equal(netlink.Route{ + Dst: &net.IPNet{ + IP: net.ParseIP("::"), + Mask: net.CIDRMask(0, 0), + }, + Gw: net.ParseIP("fd0::1"), + Table: 1, + LinkIndex: 1, + Scope: netlink.SCOPE_UNIVERSE, + Flags: int(netlink.FLAG_ONLINK), + })) +} + +func TestGenerateHostPeerCfgForPolicy(t *testing.T) { + setUp := &types.SetupConfig{ + ContainerIPNet: &terwayTypes.IPNetSet{ + IPv4: &net.IPNet{ + IP: net.ParseIP("10.0.0.2"), + Mask: net.CIDRMask(32, 32), + }, + IPv6: &net.IPNet{ + IP: net.ParseIP("fd0::2"), + Mask: net.CIDRMask(128, 128), + }, + }, + GatewayIP: &terwayTypes.IPSet{ + IPv4: net.ParseIP("169.254.169.254"), + IPv6: net.ParseIP("fd0::1"), + }, + ENIIndex: 1, + } + veth := &netlink.GenericLink{ + LinkAttrs: netlink.LinkAttrs{ + Index: 1, + Name: "calixxx", + }, + } + vethConf := datapath.GenerateHostPeerCfgForPolicy(setUp, veth, 1) + assert.Equal(t, 2, len(vethConf.Routes)) + assert.Equal(t, 4, len(vethConf.Rules)) +} diff --git a/daemon/rule_unsupported.go b/daemon/rule_unsupported.go new file mode 100644 index 00000000..84aab1df --- /dev/null +++ b/daemon/rule_unsupported.go @@ -0,0 +1,13 @@ +//go:build !linux + +package daemon + +import ( + "context" + + "github.com/AliyunContainerService/terway/types/daemon" +) + +func ruleSync(ctx context.Context, res daemon.PodResources) error { + return nil +} diff --git a/plugin/datapath/policy_router_linux.go b/plugin/datapath/policy_router_linux.go index ff959ccc..d073ea01 100644 --- a/plugin/datapath/policy_router_linux.go +++ b/plugin/datapath/policy_router_linux.go @@ -172,7 +172,7 @@ func generateContCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, mac net return contCfg } -func generateHostPeerCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf { +func GenerateHostPeerCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf { var addrs []*netlink.Addr var routes []*netlink.Route var rules []*netlink.Rule @@ -247,7 +247,7 @@ func generateHostPeerCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, tab } } -func generateENICfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf { +func GenerateENICfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf { var routes []*netlink.Route var rules []*netlink.Rule var neighs []*netlink.Neigh @@ -377,7 +377,7 @@ func (d *PolicyRoute) Setup(ctx context.Context, cfg *types.SetupConfig, netNS n table := utils.GetRouteTableID(eni.Attrs().Index) - eniCfg := generateENICfgForPolicy(cfg, eni, table) + eniCfg := GenerateENICfgForPolicy(cfg, eni, table) err = nic.Setup(ctx, eni, eniCfg) if err != nil { return fmt.Errorf("setup eni config, %w", err) @@ -390,7 +390,7 @@ func (d *PolicyRoute) Setup(ctx context.Context, cfg *types.SetupConfig, netNS n } } - hostVETHCfg := generateHostPeerCfgForPolicy(cfg, hostVETH, table) + hostVETHCfg := GenerateHostPeerCfgForPolicy(cfg, hostVETH, table) err = nic.Setup(ctx, hostVETH, hostVETHCfg) if err != nil { return fmt.Errorf("setup host veth config, %w", err) diff --git a/plugin/driver/utils/utils_linux.go b/plugin/driver/utils/utils_linux.go index 5f0ac1da..88243d5e 100644 --- a/plugin/driver/utils/utils_linux.go +++ b/plugin/driver/utils/utils_linux.go @@ -174,6 +174,9 @@ func NewIPNet1(ipNet *terwayTypes.IPNetSet) []*netlink.Addr { } func NewIPNetToMaxMask(ipNet *terwayTypes.IPNetSet) []*netlink.Addr { + if ipNet == nil { + return nil + } var addrs []*netlink.Addr if ipNet.IPv4 != nil { addrs = append(addrs, &netlink.Addr{IPNet: NewIPNetWithMaxMask(ipNet.IPv4)})