From efda5443bb4196a0a89ceb38f1f2779c3754d362 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Fri, 8 Mar 2024 16:26:52 +0800 Subject: [PATCH] daemon: simplfy the initialization Signed-off-by: l1b0k --- daemon/daemon.go | 274 +++++++++-------- daemon/daemon_test.go | 258 ++++++++++++++++ go.mod | 3 +- go.sum | 5 +- pkg/factory/fake/fake_factory.go | 154 ---------- pkg/factory/mocks/Factory.go | 261 +++++++++++++++++ pkg/factory/types.go | 2 + pkg/k8s/k8s.go | 2 + pkg/k8s/mocks/Kubernetes.go | 389 +++++++++++++++++++++++++ pkg/utils/nodecap/node_capabilities.go | 5 + 10 files changed, 1070 insertions(+), 283 deletions(-) create mode 100644 daemon/daemon_test.go delete mode 100644 pkg/factory/fake/fake_factory.go create mode 100644 pkg/factory/mocks/Factory.go create mode 100644 pkg/k8s/mocks/Kubernetes.go diff --git a/daemon/daemon.go b/daemon/daemon.go index 47d2528b..58c8eb06 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -73,8 +73,6 @@ type networkService struct { pendingPods sync.Map sync.RWMutex - enableTrunk bool - enableIPv4, enableIPv6 bool ipamType types.IPAMType @@ -807,21 +805,10 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( meta := instance.GetInstanceMeta() var ( - enableIPv4, enableIPv6 bool - trunkENIID = "" - nodeAnnotations = map[string]string{} + trunkENIID = "" + nodeAnnotations = map[string]string{} ) - switch config.IPStack { - case "ipv4": - enableIPv4 = true - case "dual": - enableIPv4 = true - enableIPv6 = true - case "ipv6": - enableIPv6 = true - } - var providers []credential.Interface if string(config.AccessID) != "" && string(config.AccessSecret) != "" { providers = append(providers, credential.NewAKPairProvider(string(config.AccessID), string(config.AccessSecret))) @@ -846,31 +833,7 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( return nil, fmt.Errorf("upable get instance limit, %w", err) } - if enableIPv6 { - if !limit.SupportIPv6() { - enableIPv6 = false - serviceLog.Info("instance is not support ipv6", "instanceType", meta.InstanceType) - } else if daemonMode == daemon.ModeENIMultiIP && !limit.SupportMultiIPIPv6() { - enableIPv6 = false - serviceLog.Info("instance is not support multi ipv6", "instanceType", meta.InstanceType) - } - } - if limit.TrunkPod() <= 0 { - config.EnableENITrunking = false - } - - if config.EnableERDMA { - if limit.ERDMARes() <= 0 { - serviceLog.Info("instance is not support erdma", "instanceType", meta.InstanceType) - config.EnableERDMA = false - } else { - ok := nodecap.GetNodeCapabilities(nodecap.NodeCapabilityERDMA) - if ok == "" { - config.EnableERDMA = false - serviceLog.Info("os is not support erdma") - } - } - } + enableIPv4, enableIPv6 := checkInstance(limit, daemonMode, config) netSrv.enableIPv4 = enableIPv4 netSrv.enableIPv6 = enableIPv6 @@ -914,71 +877,15 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( factory = aliyun.NewAliyun(ctx, aliyunClient, eni2.NewENIMetadata(enableIPv4, enableIPv6), vswPool, eniConfig) } - // init trunk if config.EnableENITrunking { - preferTrunkID := netSrv.k8s.GetTrunkID() - if preferTrunkID == "" && config.WaitTrunkENI { - preferTrunkID, err = netSrv.k8s.WaitTrunkReady() - if err != nil { - return nil, fmt.Errorf("error wait trunk ready, %w", err) - } + trunkENIID, err = initTrunk(config, poolConfig, netSrv.k8s, factory) + if err != nil { + return nil, err } - - if !config.WaitTrunkENI { - enis, err := factory.GetAttachedNetworkInterface(preferTrunkID) - if err != nil { - return nil, fmt.Errorf("error get attached eni, %w", err) - } - found := false - for _, eni := range enis { - if eni.Trunk && eni.ID == preferTrunkID { - if eni.ERdma { - serviceLog.Info("erdma eni on trunk mode, disable erdma") - config.EnableERDMA = false - } - found = true - - trunkENIID = preferTrunkID - netSrv.enableTrunk = true - - nodeAnnotations[types.TrunkOn] = preferTrunkID - nodeAnnotations[string(types.MemberENIIPTypeIPs)] = strconv.Itoa(poolConfig.MaxMemberENI) - break - } - } - if !found { - if poolConfig.MaxENI > len(enis) { - v6 := 0 - if enableIPv6 { - v6 = 1 - } - eni, _, _, err := factory.CreateNetworkInterface(1, v6, "trunk") - if err != nil { - if eni != nil { - _ = factory.DeleteNetworkInterface(eni.ID) - } - - return nil, fmt.Errorf("error create trunk eni, %w", err) - } - - trunkENIID = eni.ID - netSrv.enableTrunk = true - - nodeAnnotations[types.TrunkOn] = eni.ID - nodeAnnotations[string(types.MemberENIIPTypeIPs)] = strconv.Itoa(poolConfig.MaxMemberENI) - } else { - serviceLog.Info("no trunk eni found, fallback to non-trunk mode") - - config.EnableENITrunking = false - config.DisableDevicePlugin = true - } - } + if trunkENIID == "" { + serviceLog.Info("no trunk eni found, fallback to non-trunk mode") } else { - // WaitTrunkENI enabled, we believe what we got. - trunkENIID = preferTrunkID - netSrv.enableTrunk = true - - nodeAnnotations[types.TrunkOn] = preferTrunkID + nodeAnnotations[types.TrunkOn] = trunkENIID nodeAnnotations[string(types.MemberENIIPTypeIPs)] = strconv.Itoa(poolConfig.MaxMemberENI) } } @@ -1014,33 +921,9 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( nodeAnnotations[string(types.ERDMAIPTypeIPs)] = strconv.Itoa(realRdmaCount) poolConfig.ERdmaCapacity = realRdmaCount } - - } - - if !(daemonMode == daemon.ModeENIMultiIP && !config.EnableENITrunking) { - if !config.DisableDevicePlugin { - res := deviceplugin.ENITypeENI - capacity := poolConfig.MaxENI - if config.EnableENITrunking { - res = deviceplugin.ENITypeMember - capacity = poolConfig.MaxMemberENI - } - - dp := deviceplugin.NewENIDevicePlugin(capacity, res) - go dp.Serve() - } } - if config.EnableERDMA { - if !config.DisableDevicePlugin { - res := deviceplugin.ENITypeERDMA - capacity := poolConfig.ERdmaCapacity - if capacity > 0 { - dp := deviceplugin.NewENIDevicePlugin(capacity, res) - go dp.Serve() - } - } - } + runDevicePlugin(daemonMode, config, poolConfig) // ensure node annotations err = netSrv.k8s.PatchNodeAnnotations(nodeAnnotations) @@ -1179,6 +1062,143 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( return netSrv, nil } +func checkInstance(limit *client.Limits, daemonMode string, config *daemon.Config) (bool, bool) { + var enableIPv4, enableIPv6 bool + switch config.IPStack { + case "ipv4": + enableIPv4 = true + case "dual": + enableIPv4 = true + enableIPv6 = true + case "ipv6": + enableIPv6 = true + } + + if enableIPv6 { + if !limit.SupportIPv6() { + enableIPv6 = false + serviceLog.Info("instance is not support ipv6") + } else if daemonMode == daemon.ModeENIMultiIP && !limit.SupportMultiIPIPv6() { + enableIPv6 = false + serviceLog.Info("instance is not support multi ipv6") + } + } + + if config.EnableENITrunking && limit.TrunkPod() <= 0 { + config.EnableENITrunking = false + serviceLog.Info("instance is not support trunk") + } + + if config.EnableERDMA { + if limit.ERDMARes() <= 0 { + config.EnableERDMA = false + serviceLog.Info("instance is not support erdma") + } else { + ok := nodecap.GetNodeCapabilities(nodecap.NodeCapabilityERDMA) + if ok == "" { + config.EnableERDMA = false + serviceLog.Info("os is not support erdma") + } + } + } + return enableIPv4, enableIPv6 +} + +// initTrunk to ensure trunk eni is present. Return eni id if found. +func initTrunk(config *daemon.Config, poolConfig *types.PoolConfig, k8sClient k8s.Kubernetes, f factory.Factory) (string, error) { + var err error + + // get eni id form node annotation + preferTrunkID := k8sClient.GetTrunkID() + + if config.WaitTrunkENI { + // at this mode , we retreat id ONLY by node annotation + if preferTrunkID == "" { + preferTrunkID, err = k8sClient.WaitTrunkReady() + if err != nil { + return "", fmt.Errorf("error wait trunk ready, %w", err) + } + } + return preferTrunkID, nil + } + + // already exclude the primary eni + enis, err := f.GetAttachedNetworkInterface(preferTrunkID) + if err != nil { + return "", fmt.Errorf("error get attached eni, %w", err) + } + + // get attached trunk eni + preferred := lo.Filter(enis, func(ni *daemon.ENI, idx int) bool { return ni.Trunk && ni.ID == preferTrunkID }) + if len(preferred) > 0 { + // found the eni + trunk := preferred[0] + if trunk.ERdma { + serviceLog.Info("erdma eni on trunk mode, disable erdma") + config.EnableERDMA = false + } + + return trunk.ID, nil + } + + // choose one + attachedTrunk := lo.Filter(enis, func(ni *daemon.ENI, idx int) bool { return ni.Trunk }) + if len(attachedTrunk) > 0 { + trunk := attachedTrunk[0] + if trunk.ERdma { + serviceLog.Info("erdma eni on trunk mode, disable erdma") + config.EnableERDMA = false + } + + return trunk.ID, nil + } + + // we have to create one if possible + if poolConfig.MaxENI <= len(enis) { + config.EnableENITrunking = false + return "", nil + } + + v6 := 0 + if poolConfig.EnableIPv6 { + v6 = 1 + } + trunk, _, _, err := f.CreateNetworkInterface(1, v6, "trunk") + if err != nil { + if trunk != nil { + _ = f.DeleteNetworkInterface(trunk.ID) + } + + return "", fmt.Errorf("error create trunk eni, %w", err) + } + + return trunk.ID, nil +} + +func runDevicePlugin(daemonMode string, config *daemon.Config, poolConfig *types.PoolConfig) { + switch daemonMode { + case daemon.ModeVPC, daemon.ModeENIOnly: + dp := deviceplugin.NewENIDevicePlugin(poolConfig.MaxENI, deviceplugin.ENITypeENI) + go dp.Serve() + case daemon.ModeENIMultiIP: + if config.EnableENITrunking { + dp := deviceplugin.NewENIDevicePlugin(poolConfig.MaxMemberENI, deviceplugin.ENITypeMember) + go dp.Serve() + } + } + + if config.EnableERDMA { + if !config.DisableDevicePlugin { + res := deviceplugin.ENITypeERDMA + capacity := poolConfig.ERdmaCapacity + if capacity > 0 { + dp := deviceplugin.NewENIDevicePlugin(capacity, res) + go dp.Serve() + } + } + } +} + func getPodResources(list []interface{}) []daemon.PodResources { var res []daemon.PodResources for _, resObj := range list { diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go new file mode 100644 index 00000000..5331e342 --- /dev/null +++ b/daemon/daemon_test.go @@ -0,0 +1,258 @@ +package daemon + +import ( + "net/netip" + "testing" + + "github.com/AliyunContainerService/terway/pkg/aliyun/client" + factorymocks "github.com/AliyunContainerService/terway/pkg/factory/mocks" + k8smocks "github.com/AliyunContainerService/terway/pkg/k8s/mocks" + "github.com/AliyunContainerService/terway/pkg/utils/nodecap" + "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/daemon" + + "github.com/stretchr/testify/assert" +) + +func Test_checkInstance1(t *testing.T) { + nodecap.SetNodeCapabilities("erdma", "true") + + type args struct { + limit *client.Limits + daemonMode string + config *daemon.Config + } + tests := []struct { + name string + args args + v4 bool + v6 bool + trunking bool + erdma bool + }{ + { + name: "unsupported instance", + args: args{ + limit: &client.Limits{}, + daemonMode: "ENIMultiIP", + config: &daemon.Config{ + IPStack: "dual", + EnableENITrunking: true, + EnableERDMA: true, + }, + }, + v4: true, + v6: false, + trunking: false, + erdma: false, + }, + { + name: "supported instance", + args: args{ + limit: &client.Limits{ + Adapters: 10, + TotalAdapters: 15, + IPv4PerAdapter: 10, + IPv6PerAdapter: 10, + MemberAdapterLimit: 10, + MaxMemberAdapterLimit: 10, + ERdmaAdapters: 2, + }, + daemonMode: "ENIMultiIP", + config: &daemon.Config{ + IPStack: "dual", + EnableENITrunking: true, + EnableERDMA: true, + }, + }, + v4: true, + v6: true, + trunking: true, + erdma: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := checkInstance(tt.args.limit, tt.args.daemonMode, tt.args.config) + assert.Equalf(t, tt.v4, got, "v4(%v, %v, %v)", tt.args.limit, tt.args.daemonMode, tt.args.config) + assert.Equalf(t, tt.v6, got1, "v6(%v, %v, %v)", tt.args.limit, tt.args.daemonMode, tt.args.config) + assert.Equalf(t, tt.trunking, tt.args.config.EnableENITrunking, "trunking(%v, %v, %v)", tt.args.limit, tt.args.daemonMode, tt.args.config) + assert.Equalf(t, tt.erdma, tt.args.config.EnableERDMA, "erdma(%v, %v, %v)", tt.args.limit, tt.args.daemonMode, tt.args.config) + }) + } +} + +func Test_initTrunk(t *testing.T) { + type args struct { + config *daemon.Config + poolConfig *types.PoolConfig + k8sClient *k8smocks.Kubernetes + f *factorymocks.Factory + } + tests := []struct { + name string + args args + preStart func(args) + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "empty trunk id should create new trunk", + args: args{ + config: &daemon.Config{ + IPStack: "dual", + EnableENITrunking: true, + EnableERDMA: true, + }, + poolConfig: &types.PoolConfig{ + MaxENI: 2, + }, + k8sClient: k8smocks.NewKubernetes(t), + f: factorymocks.NewFactory(t), + }, + preStart: func(args args) { + args.k8sClient.On("GetTrunkID").Return("") + args.f.On("CreateNetworkInterface", 1, 0, "trunk").Return(&daemon.ENI{ + ID: "eni-1", + MAC: "", + SecurityGroupIDs: nil, + Trunk: true, + ERdma: false, + PrimaryIP: types.IPSet{}, + GatewayIP: types.IPSet{}, + VSwitchCIDR: types.IPNetSet{}, + VSwitchID: "", + }, []netip.Addr{}, []netip.Addr{}, nil) + args.f.On("GetAttachedNetworkInterface", "").Return([]*daemon.ENI{ + { + ID: "eni-1", + MAC: "", + SecurityGroupIDs: nil, + Trunk: false, + ERdma: false, + PrimaryIP: types.IPSet{}, + GatewayIP: types.IPSet{}, + VSwitchCIDR: types.IPNetSet{}, + VSwitchID: "", + }, + }, nil) + }, + want: "eni-1", + wantErr: assert.NoError, + }, { + name: "reuse exist trunk eni", + args: args{ + config: &daemon.Config{ + IPStack: "dual", + EnableENITrunking: true, + EnableERDMA: true, + }, + poolConfig: &types.PoolConfig{ + MaxENI: 2, + }, + k8sClient: k8smocks.NewKubernetes(t), + f: factorymocks.NewFactory(t), + }, + preStart: func(args args) { + args.k8sClient.On("GetTrunkID").Return("") + args.f.On("GetAttachedNetworkInterface", "").Return([]*daemon.ENI{ + { + ID: "eni-1", + MAC: "", + SecurityGroupIDs: nil, + Trunk: true, + ERdma: false, + PrimaryIP: types.IPSet{}, + GatewayIP: types.IPSet{}, + VSwitchCIDR: types.IPNetSet{}, + VSwitchID: "", + }, + }, nil) + }, + want: "eni-1", + wantErr: assert.NoError, + }, { + name: "disable trunk if can not create more", + args: args{ + config: &daemon.Config{ + IPStack: "dual", + EnableENITrunking: true, + EnableERDMA: true, + }, + poolConfig: &types.PoolConfig{ + MaxENI: 2, + }, + k8sClient: k8smocks.NewKubernetes(t), + f: factorymocks.NewFactory(t), + }, + preStart: func(args args) { + args.k8sClient.On("GetTrunkID").Return("") + args.f.On("GetAttachedNetworkInterface", "").Return([]*daemon.ENI{ + { + ID: "eni-1", + }, + { + ID: "eni-2", + }, + }, nil) + }, + want: "", + wantErr: assert.NoError, + }, { + name: "enable wait trunk ready", + args: args{ + config: &daemon.Config{ + IPStack: "dual", + EnableENITrunking: true, + EnableERDMA: true, + WaitTrunkENI: true, + }, + poolConfig: &types.PoolConfig{ + MaxENI: 2, + }, + k8sClient: k8smocks.NewKubernetes(t), + f: factorymocks.NewFactory(t), + }, + preStart: func(args args) { + args.k8sClient.On("GetTrunkID").Return("eni-1") + }, + want: "eni-1", + wantErr: assert.NoError, + }, { + name: "enable wait trunk ready, get from remote", + args: args{ + config: &daemon.Config{ + IPStack: "dual", + EnableENITrunking: true, + EnableERDMA: true, + WaitTrunkENI: true, + }, + poolConfig: &types.PoolConfig{ + MaxENI: 2, + }, + k8sClient: k8smocks.NewKubernetes(t), + f: factorymocks.NewFactory(t), + }, + preStart: func(args args) { + args.k8sClient.On("GetTrunkID").Return("") + args.k8sClient.On("WaitTrunkReady").Return("eni-1", nil) + }, + want: "eni-1", + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.preStart(tt.args) + + got, err := initTrunk(tt.args.config, tt.args.poolConfig, tt.args.k8sClient, tt.args.f) + if !tt.wantErr(t, err) { + return + } + assert.Equal(t, tt.want, got) + if got == "" { + assert.False(t, tt.args.config.EnableENITrunking) + } + }) + } +} diff --git a/go.mod b/go.mod index 0b390fcd..85d977a6 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/samber/lo v1.39.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.6.1 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.9.0 github.com/vishvananda/netlink v1.2.1-beta.2 go.uber.org/atomic v1.9.0 golang.org/x/net v0.19.0 @@ -106,6 +106,7 @@ require ( github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734 // indirect github.com/segmentio/go-snakecase v1.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/vladimirvivien/gexe v0.1.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/go.sum b/go.sum index d160378b..2168bda9 100644 --- a/go.sum +++ b/go.sum @@ -783,6 +783,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -792,8 +794,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= diff --git a/pkg/factory/fake/fake_factory.go b/pkg/factory/fake/fake_factory.go deleted file mode 100644 index 205f5944..00000000 --- a/pkg/factory/fake/fake_factory.go +++ /dev/null @@ -1,154 +0,0 @@ -package fake - -import ( - "fmt" - "net/netip" - "sync" - - "github.com/google/uuid" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog/v2" - - "github.com/AliyunContainerService/terway/pkg/factory" - "github.com/AliyunContainerService/terway/types/daemon" -) - -type ENI struct { - ID string - - IPv4 sets.Set[netip.Addr] - IPv6 sets.Set[netip.Addr] -} - -var _ factory.Factory = &FakeFactory{} - -type FakeFactory struct { - sync.Mutex - ENIS map[string]*ENI - - ipv4Next netip.Addr - ipv6Next netip.Addr -} - -func NewFakeFactory() *FakeFactory { - return &FakeFactory{ - ENIS: map[string]*ENI{}, - ipv4Next: netip.MustParseAddr("10.0.0.0"), - ipv6Next: netip.MustParseAddr("fd00::0"), - } -} - -func (f *FakeFactory) CreateNetworkInterface(ipv4, ipv6 int, eniType string) (*daemon.ENI, []netip.Addr, []netip.Addr, error) { - if ipv4 <= 0 { - return nil, nil, nil, fmt.Errorf("ipv4 must be greater than 0") - } - - f.Lock() - defer f.Unlock() - - n := &ENI{ - ID: uuid.NewString(), - IPv4: make(sets.Set[netip.Addr]), - IPv6: make(sets.Set[netip.Addr]), - } - - f.ENIS[n.ID] = n - - var v4, v6 []netip.Addr - for i := 0; i < ipv4; i++ { - ip := f.ipv4Next.Next() - n.IPv4.Insert(ip) - f.ipv4Next = ip - v4 = append(v4, ip) - klog.Infof("fake factory create eni %s ipv4 %s", n.ID, ip.String()) - } - - for i := 0; i < ipv6; i++ { - ip := f.ipv6Next.Next() - n.IPv6.Insert(ip) - f.ipv6Next = ip - v6 = append(v6, ip) - klog.Infof("fake factory create eni %s ipv6 %s", n.ID, ip.String()) - } - - return &daemon.ENI{ID: n.ID}, v4, v6, nil -} - -func (f *FakeFactory) AssignNIPv4(eniID string, count int, mac string) ([]netip.Addr, error) { - f.Lock() - defer f.Unlock() - - n, ok := f.ENIS[eniID] - if !ok { - return nil, fmt.Errorf("eni %s not found", eniID) - } - - var v4 []netip.Addr - for i := 0; i < count; i++ { - ip := f.ipv4Next.Next() - n.IPv4.Insert(ip) - f.ipv4Next = ip - v4 = append(v4, ip) - klog.Infof("fake factory assign eni %s ipv4 %s", n.ID, ip.String()) - } - return v4, nil -} - -func (f *FakeFactory) AssignNIPv6(eniID string, count int, mac string) ([]netip.Addr, error) { - f.Lock() - defer f.Unlock() - - n, ok := f.ENIS[eniID] - if !ok { - return nil, fmt.Errorf("eni %s not found", eniID) - } - - var v6 []netip.Addr - for i := 0; i < count; i++ { - ip := f.ipv6Next.Next() - n.IPv6.Insert(ip) - f.ipv6Next = ip - v6 = append(v6, ip) - klog.Infof("fake factory assign eni %s ipv6 %s", n.ID, ip.String()) - } - return v6, nil -} - -func (f *FakeFactory) DeleteNetworkInterface(eniID string) error { - f.Lock() - defer f.Unlock() - delete(f.ENIS, eniID) - return nil -} - -func (f *FakeFactory) UnAssignNIPv4(eniID string, ips []netip.Addr, mac string) error { - f.Lock() - defer f.Unlock() - - e, ok := f.ENIS[eniID] - if !ok { - return fmt.Errorf("eni not found") - } - e.IPv4.Delete(ips...) - return nil -} - -func (f *FakeFactory) UnAssignNIPv6(eniID string, ips []netip.Addr, mac string) error { - f.Lock() - defer f.Unlock() - - e, ok := f.ENIS[eniID] - if !ok { - return fmt.Errorf("eni not found") - } - e.IPv6.Delete(ips...) - return nil -} - -func (f *FakeFactory) LoadNetworkInterface(mac string) ([]netip.Addr, []netip.Addr, error) { - return nil, nil, nil -} - -func (f *FakeFactory) GetAttachedNetworkInterface(preferTrunkID string) ([]*daemon.ENI, error) { - return nil, nil -} diff --git a/pkg/factory/mocks/Factory.go b/pkg/factory/mocks/Factory.go new file mode 100644 index 00000000..14e6978d --- /dev/null +++ b/pkg/factory/mocks/Factory.go @@ -0,0 +1,261 @@ +// Code generated by mockery v2.42.0. DO NOT EDIT. + +package mocks + +import ( + daemon "github.com/AliyunContainerService/terway/types/daemon" + + mock "github.com/stretchr/testify/mock" + + netip "net/netip" +) + +// Factory is an autogenerated mock type for the Factory type +type Factory struct { + mock.Mock +} + +// AssignNIPv4 provides a mock function with given fields: eniID, count, mac +func (_m *Factory) AssignNIPv4(eniID string, count int, mac string) ([]netip.Addr, error) { + ret := _m.Called(eniID, count, mac) + + if len(ret) == 0 { + panic("no return value specified for AssignNIPv4") + } + + var r0 []netip.Addr + var r1 error + if rf, ok := ret.Get(0).(func(string, int, string) ([]netip.Addr, error)); ok { + return rf(eniID, count, mac) + } + if rf, ok := ret.Get(0).(func(string, int, string) []netip.Addr); ok { + r0 = rf(eniID, count, mac) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]netip.Addr) + } + } + + if rf, ok := ret.Get(1).(func(string, int, string) error); ok { + r1 = rf(eniID, count, mac) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AssignNIPv6 provides a mock function with given fields: eniID, count, mac +func (_m *Factory) AssignNIPv6(eniID string, count int, mac string) ([]netip.Addr, error) { + ret := _m.Called(eniID, count, mac) + + if len(ret) == 0 { + panic("no return value specified for AssignNIPv6") + } + + var r0 []netip.Addr + var r1 error + if rf, ok := ret.Get(0).(func(string, int, string) ([]netip.Addr, error)); ok { + return rf(eniID, count, mac) + } + if rf, ok := ret.Get(0).(func(string, int, string) []netip.Addr); ok { + r0 = rf(eniID, count, mac) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]netip.Addr) + } + } + + if rf, ok := ret.Get(1).(func(string, int, string) error); ok { + r1 = rf(eniID, count, mac) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CreateNetworkInterface provides a mock function with given fields: ipv4, ipv6, eniType +func (_m *Factory) CreateNetworkInterface(ipv4 int, ipv6 int, eniType string) (*daemon.ENI, []netip.Addr, []netip.Addr, error) { + ret := _m.Called(ipv4, ipv6, eniType) + + if len(ret) == 0 { + panic("no return value specified for CreateNetworkInterface") + } + + var r0 *daemon.ENI + var r1 []netip.Addr + var r2 []netip.Addr + var r3 error + if rf, ok := ret.Get(0).(func(int, int, string) (*daemon.ENI, []netip.Addr, []netip.Addr, error)); ok { + return rf(ipv4, ipv6, eniType) + } + if rf, ok := ret.Get(0).(func(int, int, string) *daemon.ENI); ok { + r0 = rf(ipv4, ipv6, eniType) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*daemon.ENI) + } + } + + if rf, ok := ret.Get(1).(func(int, int, string) []netip.Addr); ok { + r1 = rf(ipv4, ipv6, eniType) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).([]netip.Addr) + } + } + + if rf, ok := ret.Get(2).(func(int, int, string) []netip.Addr); ok { + r2 = rf(ipv4, ipv6, eniType) + } else { + if ret.Get(2) != nil { + r2 = ret.Get(2).([]netip.Addr) + } + } + + if rf, ok := ret.Get(3).(func(int, int, string) error); ok { + r3 = rf(ipv4, ipv6, eniType) + } else { + r3 = ret.Error(3) + } + + return r0, r1, r2, r3 +} + +// DeleteNetworkInterface provides a mock function with given fields: eniID +func (_m *Factory) DeleteNetworkInterface(eniID string) error { + ret := _m.Called(eniID) + + if len(ret) == 0 { + panic("no return value specified for DeleteNetworkInterface") + } + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(eniID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GetAttachedNetworkInterface provides a mock function with given fields: preferTrunkID +func (_m *Factory) GetAttachedNetworkInterface(preferTrunkID string) ([]*daemon.ENI, error) { + ret := _m.Called(preferTrunkID) + + if len(ret) == 0 { + panic("no return value specified for GetAttachedNetworkInterface") + } + + var r0 []*daemon.ENI + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*daemon.ENI, error)); ok { + return rf(preferTrunkID) + } + if rf, ok := ret.Get(0).(func(string) []*daemon.ENI); ok { + r0 = rf(preferTrunkID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*daemon.ENI) + } + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(preferTrunkID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// LoadNetworkInterface provides a mock function with given fields: mac +func (_m *Factory) LoadNetworkInterface(mac string) ([]netip.Addr, []netip.Addr, error) { + ret := _m.Called(mac) + + if len(ret) == 0 { + panic("no return value specified for LoadNetworkInterface") + } + + var r0 []netip.Addr + var r1 []netip.Addr + var r2 error + if rf, ok := ret.Get(0).(func(string) ([]netip.Addr, []netip.Addr, error)); ok { + return rf(mac) + } + if rf, ok := ret.Get(0).(func(string) []netip.Addr); ok { + r0 = rf(mac) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]netip.Addr) + } + } + + if rf, ok := ret.Get(1).(func(string) []netip.Addr); ok { + r1 = rf(mac) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).([]netip.Addr) + } + } + + if rf, ok := ret.Get(2).(func(string) error); ok { + r2 = rf(mac) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// UnAssignNIPv4 provides a mock function with given fields: eniID, ips, mac +func (_m *Factory) UnAssignNIPv4(eniID string, ips []netip.Addr, mac string) error { + ret := _m.Called(eniID, ips, mac) + + if len(ret) == 0 { + panic("no return value specified for UnAssignNIPv4") + } + + var r0 error + if rf, ok := ret.Get(0).(func(string, []netip.Addr, string) error); ok { + r0 = rf(eniID, ips, mac) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UnAssignNIPv6 provides a mock function with given fields: eniID, ips, mac +func (_m *Factory) UnAssignNIPv6(eniID string, ips []netip.Addr, mac string) error { + ret := _m.Called(eniID, ips, mac) + + if len(ret) == 0 { + panic("no return value specified for UnAssignNIPv6") + } + + var r0 error + if rf, ok := ret.Get(0).(func(string, []netip.Addr, string) error); ok { + r0 = rf(eniID, ips, mac) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewFactory creates a new instance of Factory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewFactory(t interface { + mock.TestingT + Cleanup(func()) +}) *Factory { + mock := &Factory{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/factory/types.go b/pkg/factory/types.go index 0c384a9b..a23f310a 100644 --- a/pkg/factory/types.go +++ b/pkg/factory/types.go @@ -1,3 +1,5 @@ +//go:generate mockery --name Factory + package factory import ( diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go index 2a3ad31d..278d667c 100644 --- a/pkg/k8s/k8s.go +++ b/pkg/k8s/k8s.go @@ -1,3 +1,5 @@ +//go:generate mockery --name Kubernetes + package k8s import ( diff --git a/pkg/k8s/mocks/Kubernetes.go b/pkg/k8s/mocks/Kubernetes.go new file mode 100644 index 00000000..e7d8f129 --- /dev/null +++ b/pkg/k8s/mocks/Kubernetes.go @@ -0,0 +1,389 @@ +// Code generated by mockery v2.42.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + client "sigs.k8s.io/controller-runtime/pkg/client" + + daemon "github.com/AliyunContainerService/terway/types/daemon" + + mock "github.com/stretchr/testify/mock" + + types "github.com/AliyunContainerService/terway/types" + + v1 "k8s.io/api/core/v1" +) + +// Kubernetes is an autogenerated mock type for the Kubernetes type +type Kubernetes struct { + mock.Mock +} + +// GetClient provides a mock function with given fields: +func (_m *Kubernetes) GetClient() client.Client { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetClient") + } + + var r0 client.Client + if rf, ok := ret.Get(0).(func() client.Client); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(client.Client) + } + } + + return r0 +} + +// GetDynamicConfigWithName provides a mock function with given fields: ctx, name +func (_m *Kubernetes) GetDynamicConfigWithName(ctx context.Context, name string) (string, error) { + ret := _m.Called(ctx, name) + + if len(ret) == 0 { + panic("no return value specified for GetDynamicConfigWithName") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (string, error)); ok { + return rf(ctx, name) + } + if rf, ok := ret.Get(0).(func(context.Context, string) string); ok { + r0 = rf(ctx, name) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetLocalPods provides a mock function with given fields: +func (_m *Kubernetes) GetLocalPods() ([]*daemon.PodInfo, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetLocalPods") + } + + var r0 []*daemon.PodInfo + var r1 error + if rf, ok := ret.Get(0).(func() ([]*daemon.PodInfo, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() []*daemon.PodInfo); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*daemon.PodInfo) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetNodeCidr provides a mock function with given fields: +func (_m *Kubernetes) GetNodeCidr() *types.IPNetSet { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetNodeCidr") + } + + var r0 *types.IPNetSet + if rf, ok := ret.Get(0).(func() *types.IPNetSet); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.IPNetSet) + } + } + + return r0 +} + +// GetNodeDynamicConfigLabel provides a mock function with given fields: +func (_m *Kubernetes) GetNodeDynamicConfigLabel() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetNodeDynamicConfigLabel") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// GetPod provides a mock function with given fields: ctx, namespace, name, cache +func (_m *Kubernetes) GetPod(ctx context.Context, namespace string, name string, cache bool) (*daemon.PodInfo, error) { + ret := _m.Called(ctx, namespace, name, cache) + + if len(ret) == 0 { + panic("no return value specified for GetPod") + } + + var r0 *daemon.PodInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, bool) (*daemon.PodInfo, error)); ok { + return rf(ctx, namespace, name, cache) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, bool) *daemon.PodInfo); ok { + r0 = rf(ctx, namespace, name, cache) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*daemon.PodInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, bool) error); ok { + r1 = rf(ctx, namespace, name, cache) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetServiceCIDR provides a mock function with given fields: +func (_m *Kubernetes) GetServiceCIDR() *types.IPNetSet { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetServiceCIDR") + } + + var r0 *types.IPNetSet + if rf, ok := ret.Get(0).(func() *types.IPNetSet); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.IPNetSet) + } + } + + return r0 +} + +// GetTrunkID provides a mock function with given fields: +func (_m *Kubernetes) GetTrunkID() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetTrunkID") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// PatchNodeAnnotations provides a mock function with given fields: anno +func (_m *Kubernetes) PatchNodeAnnotations(anno map[string]string) error { + ret := _m.Called(anno) + + if len(ret) == 0 { + panic("no return value specified for PatchNodeAnnotations") + } + + var r0 error + if rf, ok := ret.Get(0).(func(map[string]string) error); ok { + r0 = rf(anno) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// PatchNodeIPResCondition provides a mock function with given fields: status, reason, message +func (_m *Kubernetes) PatchNodeIPResCondition(status v1.ConditionStatus, reason string, message string) error { + ret := _m.Called(status, reason, message) + + if len(ret) == 0 { + panic("no return value specified for PatchNodeIPResCondition") + } + + var r0 error + if rf, ok := ret.Get(0).(func(v1.ConditionStatus, string, string) error); ok { + r0 = rf(status, reason, message) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// PatchPodIPInfo provides a mock function with given fields: info, ips +func (_m *Kubernetes) PatchPodIPInfo(info *daemon.PodInfo, ips string) error { + ret := _m.Called(info, ips) + + if len(ret) == 0 { + panic("no return value specified for PatchPodIPInfo") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*daemon.PodInfo, string) error); ok { + r0 = rf(info, ips) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// PodExist provides a mock function with given fields: namespace, name +func (_m *Kubernetes) PodExist(namespace string, name string) (bool, error) { + ret := _m.Called(namespace, name) + + if len(ret) == 0 { + panic("no return value specified for PodExist") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(string, string) (bool, error)); ok { + return rf(namespace, name) + } + if rf, ok := ret.Get(0).(func(string, string) bool); ok { + r0 = rf(namespace, name) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(namespace, name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// RecordNodeEvent provides a mock function with given fields: eventType, reason, message +func (_m *Kubernetes) RecordNodeEvent(eventType string, reason string, message string) { + _m.Called(eventType, reason, message) +} + +// RecordPodEvent provides a mock function with given fields: podName, podNamespace, eventType, reason, message +func (_m *Kubernetes) RecordPodEvent(podName string, podNamespace string, eventType string, reason string, message string) error { + ret := _m.Called(podName, podNamespace, eventType, reason, message) + + if len(ret) == 0 { + panic("no return value specified for RecordPodEvent") + } + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, string, string, string) error); ok { + r0 = rf(podName, podNamespace, eventType, reason, message) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SetCustomStatefulWorkloadKinds provides a mock function with given fields: kinds +func (_m *Kubernetes) SetCustomStatefulWorkloadKinds(kinds []string) error { + ret := _m.Called(kinds) + + if len(ret) == 0 { + panic("no return value specified for SetCustomStatefulWorkloadKinds") + } + + var r0 error + if rf, ok := ret.Get(0).(func([]string) error); ok { + r0 = rf(kinds) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SetNodeAllocatablePod provides a mock function with given fields: count +func (_m *Kubernetes) SetNodeAllocatablePod(count int) error { + ret := _m.Called(count) + + if len(ret) == 0 { + panic("no return value specified for SetNodeAllocatablePod") + } + + var r0 error + if rf, ok := ret.Get(0).(func(int) error); ok { + r0 = rf(count) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// WaitTrunkReady provides a mock function with given fields: +func (_m *Kubernetes) WaitTrunkReady() (string, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for WaitTrunkReady") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func() (string, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewKubernetes creates a new instance of Kubernetes. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewKubernetes(t interface { + mock.TestingT + Cleanup(func()) +}) *Kubernetes { + mock := &Kubernetes{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/utils/nodecap/node_capabilities.go b/pkg/utils/nodecap/node_capabilities.go index f31e0e72..3d9915e4 100644 --- a/pkg/utils/nodecap/node_capabilities.go +++ b/pkg/utils/nodecap/node_capabilities.go @@ -27,6 +27,11 @@ func init() { } } +// SetNodeCapabilities set node capability , test purpose +func SetNodeCapabilities(capName, val string) { + cachedNodeCapabilities[capName] = val +} + func GetNodeCapabilities(capName string) string { return cachedNodeCapabilities[capName] }