From 4860e6df621c937ea0e6dd47828d269d4219eba8 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Tue, 2 Jan 2024 13:29:27 +0800 Subject: [PATCH] refactoring the pool - Allow pod level config to select eni - EIP feature is removed --- .gitignore | 1 + cmd/terway-cli/cmd.go | 88 +- cmd/terway-cli/main.go | 5 +- cmd/terway/main.go | 9 +- daemon/config.go | 22 +- daemon/config_test.go | 10 +- daemon/context.go | 23 - daemon/daemon.go | 1602 ++++++++---------------- daemon/daemon_test.go | 91 -- daemon/daemon_unwindows.go | 7 +- daemon/daemon_windows.go | 8 +- daemon/eip.go | 141 --- daemon/eni-multi-ip.go | 1165 ----------------- daemon/eni-multi-ip_unwindows.go | 14 - daemon/eni-multi-ip_windows.go | 80 -- daemon/eni.go | 401 ------ daemon/resource_manager.go | 18 +- daemon/server.go | 62 +- daemon/veth.go | 46 - docs/trouble-shotting.md | 0 examples/maxpods/maxpods.go | 23 +- go.mod | 50 +- go.sum | 126 +- pkg/aliyun/aliyun.go | 562 --------- pkg/aliyun/client/ecs.go | 209 ++-- pkg/aliyun/client/errors/errors.go | 11 + pkg/aliyun/client/fake/fake_client.go | 54 +- pkg/aliyun/client/interface.go | 8 - pkg/aliyun/client/interface_default.go | 10 +- pkg/aliyun/client/types.go | 4 +- pkg/aliyun/client/vpc.go | 184 --- pkg/aliyun/client/vsw_default.go | 15 +- pkg/aliyun/credential/types.go | 3 +- pkg/aliyun/eip.go | 315 ----- pkg/aliyun/{ => eni}/eni.go | 46 +- pkg/aliyun/{ => instance}/instance.go | 11 +- pkg/aliyun/metadata/metadata.go | 93 +- pkg/backoff/backoff.go | 7 + pkg/controller/delegate/deleg.go | 19 +- pkg/controller/node/node_controller.go | 11 +- pkg/controller/pool/eni_mgr.go | 10 +- pkg/controller/vswitch/vswitch.go | 5 +- pkg/eni/conditiontype_string.go | 26 + pkg/eni/ipstatus_string.go | 26 + pkg/eni/local.go | 847 +++++++++++++ pkg/eni/local_test.go | 85 ++ pkg/eni/manager.go | 352 ++++++ pkg/eni/manager_test.go | 192 +++ pkg/eni/remote.go | 233 ++++ pkg/eni/remote_test.go | 70 ++ pkg/eni/status_string.go | 26 + pkg/eni/trunk.go | 71 ++ pkg/eni/types.go | 259 ++++ pkg/eni/types_test.go | 97 ++ pkg/eni/veth.go | 96 ++ pkg/factory/aliyun/aliyun.go | 466 +++++++ pkg/factory/fake/fake_factory.go | 154 +++ pkg/factory/types.go | 23 + pkg/ip/ip.go | 38 +- pkg/ip/ip_cilium.go | 32 - pkg/ipam/ipam.go | 33 - {daemon => pkg/k8s}/k8s.go | 833 ++++++------ pkg/link/interface_linux.go | 6 +- pkg/logger/log.go | 10 - pkg/pool/pool.go | 736 ----------- pkg/pool/pool_test.go | 335 ----- pkg/pool/queue.go | 115 -- pkg/pool/queue_test.go | 90 -- pkg/tracing/rpc.go | 25 +- pkg/tracing/tracing.go | 30 +- pkg/utils/config.go | 15 - pkg/utils/k8s.go | 5 + plugin/terway/cni_linux.go | 8 + rpc/rpc.pb.go | 195 ++- rpc/rpc.proto | 2 - rpc/rpc_grpc.pb.go | 27 +- rpc/tracing.pb.go | 304 ++--- rpc/tracing.proto | 24 +- rpc/tracing_grpc.pb.go | 45 +- types/config.go | 6 + types/daemon/cni.go | 9 + types/daemon/config.go | 10 +- types/{ => daemon}/res.go | 13 +- types/daemon/types.go | 190 +++ types/errors.go | 35 + types/types.go | 271 +--- types/types_test.go | 88 +- 87 files changed, 5055 insertions(+), 7067 deletions(-) delete mode 100644 daemon/context.go delete mode 100644 daemon/daemon_test.go delete mode 100644 daemon/eip.go delete mode 100644 daemon/eni-multi-ip.go delete mode 100644 daemon/eni-multi-ip_unwindows.go delete mode 100644 daemon/eni-multi-ip_windows.go delete mode 100644 daemon/eni.go delete mode 100644 daemon/veth.go create mode 100644 docs/trouble-shotting.md delete mode 100644 pkg/aliyun/aliyun.go delete mode 100644 pkg/aliyun/client/vpc.go delete mode 100644 pkg/aliyun/eip.go rename pkg/aliyun/{ => eni}/eni.go (68%) rename pkg/aliyun/{ => instance}/instance.go (94%) create mode 100644 pkg/eni/conditiontype_string.go create mode 100644 pkg/eni/ipstatus_string.go create mode 100644 pkg/eni/local.go create mode 100644 pkg/eni/local_test.go create mode 100644 pkg/eni/manager.go create mode 100644 pkg/eni/manager_test.go create mode 100644 pkg/eni/remote.go create mode 100644 pkg/eni/remote_test.go create mode 100644 pkg/eni/status_string.go create mode 100644 pkg/eni/trunk.go create mode 100644 pkg/eni/types.go create mode 100644 pkg/eni/types_test.go create mode 100644 pkg/eni/veth.go create mode 100644 pkg/factory/aliyun/aliyun.go create mode 100644 pkg/factory/fake/fake_factory.go create mode 100644 pkg/factory/types.go delete mode 100644 pkg/ipam/ipam.go rename {daemon => pkg/k8s}/k8s.go (67%) delete mode 100644 pkg/pool/pool.go delete mode 100644 pkg/pool/pool_test.go delete mode 100644 pkg/pool/queue.go delete mode 100644 pkg/pool/queue_test.go delete mode 100644 pkg/utils/config.go create mode 100644 types/daemon/cni.go rename types/{ => daemon}/res.go (88%) create mode 100644 types/daemon/types.go create mode 100644 types/errors.go diff --git a/.gitignore b/.gitignore index b3c9ef8e..83d7c115 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea vendor **/*.exe +.run diff --git a/cmd/terway-cli/cmd.go b/cmd/terway-cli/cmd.go index 210d6bfc..922f80b6 100644 --- a/cmd/terway-cli/cmd.go +++ b/cmd/terway-cli/cmd.go @@ -99,23 +99,6 @@ func runShow(cmd *cobra.Command, args []string) error { return err } -const ( - mappingTableHeaderStatus = "Status" - mappingTableHeaderPodName = "Pod Name" - mappingTableHeaderResourceID = "Res ID" - mappingTableHeaderFactoryResourceID = "Factory Res ID" - - mappingStringErrorExists = "error exists in mapping" -) - -var ( - mappingStatus = map[rpc.ResourceMappingType]string{ - rpc.ResourceMappingType_MappingTypeNormal: "Normal", - rpc.ResourceMappingType_MappingTypeIdle: "Idle", - rpc.ResourceMappingType_MappingTypeError: "ERROR", - } -) - func runMapping(cmd *cobra.Command, args []string) error { placeholder := &rpc.Placeholder{} result, err := client.GetResourceMapping(ctx, placeholder) @@ -123,40 +106,51 @@ func runMapping(cmd *cobra.Command, args []string) error { return err } - tableData := pterm.TableData{ - { - mappingTableHeaderStatus, - mappingTableHeaderPodName, - mappingTableHeaderResourceID, - mappingTableHeaderFactoryResourceID, - }, - } - - for _, v := range result.Info { - clr := pterm.FgDefault - switch v.Type { - case rpc.ResourceMappingType_MappingTypeNormal: - // Idle - if v.PodName == "" { - v.Type = rpc.ResourceMappingType_MappingTypeIdle - clr = pterm.FgLightCyan - } - case rpc.ResourceMappingType_MappingTypeError: - clr = pterm.FgLightRed - err = fmt.Errorf(mappingStringErrorExists) + // + for i, r := range result.Info { + items := []pterm.BulletListItem{ + { + Level: 0, + Text: fmt.Sprintf("slot %d", i), + BulletStyle: pterm.NewStyle(pterm.FgRed), + }, + { + Level: 1, + Text: r.NetworkInterfaceID, + Bullet: "-", + }, + { + Level: 1, + Text: r.MAC, + Bullet: "-", + }, + { + Level: 1, + Text: r.Status, + Bullet: "-", + }, + { + Level: 1, + Text: fmt.Sprintf("Type %s", r.Type), + Bullet: "-", + }, + { + Level: 1, + Text: fmt.Sprintf("InhibitExpireAt %s", r.AllocInhibitExpireAt), + Bullet: "-", + }, } - row := []string{ - clr.Sprint(mappingStatus[v.Type]), - clr.Sprint(v.PodName), - clr.Sprint(v.ResourceName), - clr.Sprint(v.FactoryResourceName), + for _, v := range r.Info { + items = append(items, pterm.BulletListItem{ + Level: 2, + Text: v, + }) } - tableData = append(tableData, row) - } - if err := pterm.DefaultTable.WithHasHeader().WithData(tableData).Render(); err != nil { - return err + if err := pterm.DefaultBulletList.WithItems(items).Render(); err != nil { + return err + } } return err diff --git a/cmd/terway-cli/main.go b/cmd/terway-cli/main.go index 714ec7ad..2cef75e1 100644 --- a/cmd/terway-cli/main.go +++ b/cmd/terway-cli/main.go @@ -3,8 +3,8 @@ package main import ( "context" "fmt" - "log" "net" + "os" "time" "github.com/spf13/cobra" @@ -99,6 +99,7 @@ func init() { func main() { if err := rootCmd.Execute(); err != nil { - log.Fatalf("terway-cli error: %s", err) + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) } } diff --git a/cmd/terway/main.go b/cmd/terway/main.go index 167c4cf7..b8aea814 100644 --- a/cmd/terway/main.go +++ b/cmd/terway/main.go @@ -4,6 +4,7 @@ import ( "flag" "math/rand" "os" + "strings" "time" "github.com/AliyunContainerService/terway/daemon" @@ -42,13 +43,19 @@ func main() { fs.StringVar(&daemonMode, "daemon-mode", "VPC", "terway network mode") fs.StringVar(&logLevel, "log-level", "info", "terway log level") fs.StringVar(&readonlyListen, "readonly-listen", utils.NormalizePath(debugSocketPath), "terway readonly listen") + ctrl.RegisterFlags(fs) + klog.InitFlags(fs) + err := fs.Parse(os.Args[1:]) if err != nil { panic(err) } + if strings.ToLower(logLevel) == "debug" { + _ = fs.Set("v", "5") + } ctx := ctrl.SetupSignalHandler() - err = daemon.Run(ctx, utils.NormalizePath(defaultSocketPath), readonlyListen, utils.NormalizePath(defaultConfigPath), daemonMode, logLevel) + err = daemon.Run(ctx, utils.NormalizePath(defaultSocketPath), readonlyListen, utils.NormalizePath(defaultConfigPath), daemonMode) if err != nil { klog.Fatal(err) diff --git a/daemon/config.go b/daemon/config.go index 75762111..d1095659 100644 --- a/daemon/config.go +++ b/daemon/config.go @@ -1,7 +1,10 @@ package daemon import ( - "github.com/AliyunContainerService/terway/pkg/aliyun" + "context" + + "github.com/AliyunContainerService/terway/pkg/aliyun/instance" + "github.com/AliyunContainerService/terway/pkg/k8s" "github.com/AliyunContainerService/terway/pkg/utils" "github.com/AliyunContainerService/terway/types" "github.com/AliyunContainerService/terway/types/daemon" @@ -9,24 +12,27 @@ import ( // getDynamicConfig returns (config, label, error) specified in node // ("", "", nil) for no dynamic config for this node -func getDynamicConfig(k8s Kubernetes) (string, string, error) { +func getDynamicConfig(ctx context.Context, k8s k8s.Kubernetes) (string, string, error) { label := k8s.GetNodeDynamicConfigLabel() if label == "" { return "", "", nil } - cfg, err := k8s.GetDynamicConfigWithName(label) + cfg, err := k8s.GetDynamicConfigWithName(ctx, label) return cfg, label, err } // the actual size for pool is minIdle and maxIdle -func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *aliyun.Limits) (*types.PoolConfig, error) { +func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *instance.Limits) (*types.PoolConfig, error) { + poolConfig := &types.PoolConfig{ SecurityGroupIDs: cfg.GetSecurityGroups(), VSwitchSelectionPolicy: cfg.VSwitchSelectionPolicy, DisableSecurityGroupCheck: cfg.DisableSecurityGroupCheck, + BatchSize: 10, } + if cfg.ENITags == nil { cfg.ENITags = make(map[string]string) } @@ -39,7 +45,7 @@ func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *aliyun.Limits) maxMemberENI := 0 switch daemonMode { - case daemonModeVPC, daemonModeENIOnly: + case daemon.ModeVPC, daemon.ModeENIOnly: maxENI = limit.Adapters maxENI = int(float64(maxENI)*cfg.EniCapRatio) + cfg.EniCapShift - 1 @@ -67,12 +73,12 @@ func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *aliyun.Limits) } maxMemberENI = limit.MemberAdapterLimit - if cfg.ENICapPolicy == types.ENICapPolicyPreferTrunk { + if cfg.ENICapPolicy == daemon.ENICapPolicyPreferTrunk { maxMemberENI = limit.MaxMemberAdapterLimit } poolConfig.MaxIPPerENI = 1 - case daemonModeENIMultiIP: + case daemon.ModeENIMultiIP: maxENI = limit.Adapters maxENI = int(float64(maxENI)*cfg.EniCapRatio) + cfg.EniCapShift - 1 @@ -121,7 +127,7 @@ func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *aliyun.Limits) } if requireMeta { - ins := aliyun.GetInstanceMeta() + ins := instance.GetInstanceMeta() zone := ins.ZoneID if cfg.VSwitches != nil { zoneVswitchs, ok := cfg.VSwitches[zone] diff --git a/daemon/config_test.go b/daemon/config_test.go index 125ac7a9..85ff32fe 100644 --- a/daemon/config_test.go +++ b/daemon/config_test.go @@ -5,12 +5,12 @@ import ( "github.com/stretchr/testify/assert" - "github.com/AliyunContainerService/terway/pkg/aliyun" + "github.com/AliyunContainerService/terway/pkg/aliyun/instance" "github.com/AliyunContainerService/terway/types/daemon" ) func init() { - aliyun.Test = true + instance.Test = true } func TestGetPoolConfigWithVPCMode(t *testing.T) { @@ -20,7 +20,7 @@ func TestGetPoolConfigWithVPCMode(t *testing.T) { EniCapRatio: 1, RegionID: "foo", } - limit := &aliyun.Limits{ + limit := &instance.Limits{ Adapters: 10, MemberAdapterLimit: 5, } @@ -37,7 +37,7 @@ func TestGetPoolConfigWithENIOnlyMode(t *testing.T) { EniCapRatio: 1, RegionID: "foo", } - limit := &aliyun.Limits{ + limit := &instance.Limits{ Adapters: 10, MemberAdapterLimit: 5, } @@ -54,7 +54,7 @@ func TestGetPoolConfigWithENIMultiIPMode(t *testing.T) { EniCapRatio: 1, RegionID: "foo", } - limit := &aliyun.Limits{ + limit := &instance.Limits{ Adapters: 10, IPv4PerAdapter: 5, MemberAdapterLimit: 5, diff --git a/daemon/context.go b/daemon/context.go deleted file mode 100644 index b3cecc97..00000000 --- a/daemon/context.go +++ /dev/null @@ -1,23 +0,0 @@ -package daemon - -import ( - "github.com/AliyunContainerService/terway/pkg/logger" - "github.com/AliyunContainerService/terway/types" - - "github.com/sirupsen/logrus" - "golang.org/x/net/context" -) - -type networkContext struct { - context.Context - resources []types.ResourceItem - pod *types.PodInfo - k8sService Kubernetes -} - -func (networkContext *networkContext) Log() *logrus.Entry { - return logger.DefaultLogger. - WithField("podName", networkContext.pod.Name). - WithField("podNs", networkContext.pod.Namespace). - WithField("resources", networkContext.resources) -} diff --git a/daemon/daemon.go b/daemon/daemon.go index 334a7ff2..30ad6ba9 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -3,32 +3,27 @@ package daemon import ( "context" "encoding/json" + "errors" "fmt" - "net" - "os" - "sort" + "net/netip" "strconv" "strings" "sync" "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/util/retry" - "github.com/AliyunContainerService/terway/deviceplugin" - "github.com/AliyunContainerService/terway/pkg/aliyun" "github.com/AliyunContainerService/terway/pkg/aliyun/client" "github.com/AliyunContainerService/terway/pkg/aliyun/credential" + eni2 "github.com/AliyunContainerService/terway/pkg/aliyun/eni" + "github.com/AliyunContainerService/terway/pkg/aliyun/instance" "github.com/AliyunContainerService/terway/pkg/apis/alibabacloud.com/v1beta1" "github.com/AliyunContainerService/terway/pkg/apis/crds" - podENITypes "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" "github.com/AliyunContainerService/terway/pkg/backoff" vswpool "github.com/AliyunContainerService/terway/pkg/controller/vswitch" - terwayIP "github.com/AliyunContainerService/terway/pkg/ip" - "github.com/AliyunContainerService/terway/pkg/link" - "github.com/AliyunContainerService/terway/pkg/logger" + "github.com/AliyunContainerService/terway/pkg/eni" + "github.com/AliyunContainerService/terway/pkg/factory/aliyun" + "github.com/AliyunContainerService/terway/pkg/k8s" "github.com/AliyunContainerService/terway/pkg/metric" - "github.com/AliyunContainerService/terway/pkg/pool" "github.com/AliyunContainerService/terway/pkg/storage" "github.com/AliyunContainerService/terway/pkg/tracing" "github.com/AliyunContainerService/terway/pkg/utils" @@ -36,24 +31,18 @@ import ( "github.com/AliyunContainerService/terway/types" "github.com/AliyunContainerService/terway/types/daemon" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" k8sErr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/flowcontrol" + "k8s.io/client-go/util/retry" + logf "sigs.k8s.io/controller-runtime/pkg/log" ) const ( - daemonModeVPC = "VPC" - daemonModeENIMultiIP = "ENIMultiIP" - daemonModeENIOnly = "ENIOnly" - - gcPeriod = 5 * time.Minute - poolCheckPeriod = 10 * time.Minute - - conditionFalse = "false" - conditionTrue = "true" + gcPeriod = 5 * time.Minute networkServiceName = "default" tracingKeyName = "name" @@ -63,6 +52,7 @@ const ( tracingKeyPendingPodsCount = "pending_pods_count" commandMapping = "mapping" + commandResDB = "resdb" IfEth0 = "eth0" ) @@ -71,146 +61,60 @@ type networkService struct { daemonMode string configFilePath string - k8s Kubernetes - resourceDB storage.Storage - vethResMgr ResourceManager - eniResMgr ResourceManager - eniIPResMgr ResourceManager - eipResMgr ResourceManager - //networkResourceMgr ResourceManager - mgrForResource map[string]ResourceManager - pendingPods sync.Map + k8s k8s.Kubernetes + resourceDB storage.Storage + + eniMgr *eni.Manager + pendingPods sync.Map sync.RWMutex enableTrunk bool - ipFamily *types.IPFamily - ipamType types.IPAMType - eniCapPolicy types.ENICapPolicy + enableIPv4, enableIPv6 bool + + ipamType types.IPAMType + + wg sync.WaitGroup rpc.UnimplementedTerwayBackendServer } -var serviceLog = logger.DefaultLogger.WithField("subSys", "network-service") +var serviceLog = logf.Log.WithName("server") var _ rpc.TerwayBackendServer = (*networkService)(nil) -func (n *networkService) getResourceManagerForRes(resType string) ResourceManager { - return n.mgrForResource[resType] -} - // return resource relation in db, or return nil. -func (n *networkService) getPodResource(info *types.PodInfo) (types.PodResources, error) { - obj, err := n.resourceDB.Get(podInfoKey(info.Namespace, info.Name)) +func (n *networkService) getPodResource(info *daemon.PodInfo) (daemon.PodResources, error) { + obj, err := n.resourceDB.Get(utils.PodInfoKey(info.Namespace, info.Name)) if err == nil { - return obj.(types.PodResources), nil + return obj.(daemon.PodResources), nil } - if err == storage.ErrNotFound { - return types.PodResources{}, nil + if errors.Is(err, storage.ErrNotFound) { + return daemon.PodResources{}, nil } - return types.PodResources{}, err + return daemon.PodResources{}, err } -func (n *networkService) deletePodResource(info *types.PodInfo) error { - key := podInfoKey(info.Namespace, info.Name) +func (n *networkService) deletePodResource(info *daemon.PodInfo) error { + key := utils.PodInfoKey(info.Namespace, info.Name) return n.resourceDB.Delete(key) } -func (n *networkService) allocateVeth(ctx *networkContext, old *types.PodResources) (*types.Veth, error) { - oldVethRes := old.GetResourceItemByType(types.ResourceTypeVeth) - oldVethID := "" - if old.PodInfo != nil { - if len(oldVethRes) == 0 { - ctx.Log().Debugf("veth for pod %s is zero", podInfoKey(old.PodInfo.Namespace, old.PodInfo.Name)) - } else if len(oldVethRes) > 1 { - ctx.Log().Warnf("veth for pod %s is more than one", podInfoKey(old.PodInfo.Namespace, old.PodInfo.Name)) - } else { - oldVethID = oldVethRes[0].ID - } - } - - res, err := n.vethResMgr.Allocate(ctx, oldVethID) - if err != nil { - return nil, err - } - return res.(*types.Veth), nil -} - -func (n *networkService) allocateENI(ctx *networkContext, old *types.PodResources) (*types.ENI, error) { - oldENIRes := old.GetResourceItemByType(types.ResourceTypeENI) - oldENIID := "" - if old.PodInfo != nil { - if len(oldENIRes) == 0 { - ctx.Log().Debugf("eniip for pod %s is zero", podInfoKey(old.PodInfo.Namespace, old.PodInfo.Name)) - } else if len(oldENIRes) > 1 { - ctx.Log().Warnf("eniip for pod %s is more than one", podInfoKey(old.PodInfo.Namespace, old.PodInfo.Name)) - } else { - oldENIID = oldENIRes[0].ID - } - } - - res, err := n.eniResMgr.Allocate(ctx, oldENIID) - if err != nil { - return nil, err - } - return res.(*types.ENI), nil -} - -func (n *networkService) allocateENIMultiIP(ctx *networkContext, old *types.PodResources) (*types.ENIIP, error) { - oldENIIPRes := old.GetResourceItemByType(types.ResourceTypeENIIP) - oldENIIPID := "" - if old.PodInfo != nil { - if len(oldENIIPRes) == 0 { - ctx.Log().Debugf("eniip for pod %s is zero", podInfoKey(old.PodInfo.Namespace, old.PodInfo.Name)) - } else if len(oldENIIPRes) > 1 { - ctx.Log().Warnf("eniip for pod %s is more than one", podInfoKey(old.PodInfo.Namespace, old.PodInfo.Name)) - } else { - oldENIIPID = oldENIIPRes[0].ID - } - } - - res, err := n.eniIPResMgr.Allocate(ctx, oldENIIPID) - if err != nil { - return nil, err - } - return res.(*types.ENIIP), nil -} - -func (n *networkService) allocateEIP(ctx *networkContext, old *types.PodResources) (*types.EIP, error) { - oldEIPRes := old.GetResourceItemByType(types.ResourceTypeEIP) - oldEIPID := "" - if old.PodInfo != nil { - if len(oldEIPRes) == 0 { - ctx.Log().Debugf("eip for pod %s is zero", podInfoKey(old.PodInfo.Namespace, old.PodInfo.Name)) - } else if len(oldEIPRes) > 1 { - ctx.Log().Warnf("eip for pod %s is more than one", podInfoKey(old.PodInfo.Namespace, old.PodInfo.Name)) - } else { - oldEIPID = oldEIPRes[0].ID - } - } - - res, err := n.eipResMgr.Allocate(ctx, oldEIPID) - if err != nil { - return nil, err - } - return res.(*types.EIP), nil -} - func (n *networkService) AllocIP(ctx context.Context, r *rpc.AllocIPRequest) (*rpc.AllocIPReply, error) { - serviceLog.WithFields(map[string]interface{}{ - "pod": podInfoKey(r.K8SPodNamespace, r.K8SPodName), - "containerID": r.K8SPodInfraContainerId, - "netNS": r.Netns, - "ifName": r.IfName, - }).Info("alloc ip req") - - _, exist := n.pendingPods.LoadOrStore(podInfoKey(r.K8SPodNamespace, r.K8SPodName), struct{}{}) + podID := utils.PodInfoKey(r.K8SPodNamespace, r.K8SPodName) + l := logf.FromContext(ctx) + l.Info("alloc ip req") + + _, exist := n.pendingPods.LoadOrStore(podID, struct{}{}) if exist { - return nil, fmt.Errorf("pod %s resource processing", podInfoKey(r.K8SPodNamespace, r.K8SPodName)) + return nil, &types.Error{ + Code: types.PodIsProcessing, + Msg: fmt.Sprintf("Pod %s request is processing", podID), + } } defer func() { - n.pendingPods.Delete(podInfoKey(r.K8SPodNamespace, r.K8SPodName)) + n.pendingPods.Delete(podID) }() n.RLock() @@ -219,284 +123,181 @@ func (n *networkService) AllocIP(ctx context.Context, r *rpc.AllocIPRequest) (*r start = time.Now() err error ) + + reply := &rpc.AllocIPReply{ + Success: true, + IPv4: n.enableIPv4, + IPv6: n.enableIPv6, + } + defer func() { metric.RPCLatency.WithLabelValues("AllocIP", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) + + l.Info("alloc ip", "reply", fmt.Sprintf("%v", reply), "err", fmt.Sprintf("%v", err)) }() // 0. Get pod Info - podinfo, err := n.k8s.GetPod(r.K8SPodNamespace, r.K8SPodName, true) + pod, err := n.k8s.GetPod(ctx, r.K8SPodNamespace, r.K8SPodName, true) if err != nil { - return nil, errors.Wrapf(err, "error get pod info for: %+v", r) + return nil, &types.Error{ + Code: types.InvalidArgsErrCode, + Msg: err.Error(), + R: err, + } } // 1. Init Context - networkContext := &networkContext{ - Context: ctx, - resources: []types.ResourceItem{}, - pod: podinfo, - k8sService: n.k8s, - } - allocIPReply := &rpc.AllocIPReply{IPv4: n.ipFamily.IPv4, IPv6: n.ipFamily.IPv6} - - defer func() { - // roll back allocated resource when error - if err != nil { - networkContext.Log().Errorf("alloc result with error, %+v", err) - for _, res := range networkContext.resources { - err = n.deletePodResource(podinfo) - networkContext.Log().Errorf("rollback res[%v] with error, %+v", res, err) - mgr := n.getResourceManagerForRes(res.Type) - if mgr == nil { - networkContext.Log().Warnf("error cleanup allocated network resource %s, %s: %v", res.ID, res.Type, err) - continue - } - err = mgr.Release(networkContext, res) - if err != nil { - networkContext.Log().Infof("rollback res error: %+v", err) - } - } - } else { - networkContext.Log().Infof("alloc result: %+v", allocIPReply) - for _, netConfig := range allocIPReply.NetConfs { - if netConfig.IfName != IfEth0 && netConfig.IfName != "" { - continue - } - if netConfig.BasicInfo == nil || netConfig.BasicInfo.PodIP == nil { - continue - } - var ips []string - if netConfig.BasicInfo.PodIP.IPv4 != "" { - ips = append(ips, netConfig.BasicInfo.PodIP.IPv4) - } - if netConfig.BasicInfo.PodIP.IPv6 != "" { - ips = append(ips, netConfig.BasicInfo.PodIP.IPv6) - } - _ = n.k8s.PatchPodIPInfo(podinfo, strings.Join(ips, ",")) - } - } - }() + cni := &daemon.CNI{ + PodName: r.K8SPodName, + PodNamespace: r.K8SPodNamespace, + PodID: podID, + PodUID: pod.PodUID, + NetNSPath: r.Netns, + } // 2. Find old resource info - oldRes, err := n.getPodResource(podinfo) + oldRes, err := n.getPodResource(pod) if err != nil { - return nil, errors.Wrapf(err, "error get pod resources from db for pod %+v", podinfo) + return nil, &types.Error{ + Code: types.InternalError, + Msg: err.Error(), + R: err, + } } - if !n.verifyPodNetworkType(podinfo.PodNetworkType) { - return nil, fmt.Errorf("unexpect pod network type allocate, maybe daemon mode changed: %+v", podinfo.PodNetworkType) + if !n.verifyPodNetworkType(pod.PodNetworkType) { + return nil, &types.Error{ + Code: types.InvalidArgsErrCode, + Msg: fmt.Sprintf("Unexpected network type, maybe daemon mode changed"), + } } + + var resourceRequests []eni.ResourceRequest + var netConf []*rpc.NetConf // 3. Allocate network resource for pod - switch podinfo.PodNetworkType { - case podNetworkTypeENIMultiIP: - allocIPReply.IPType = rpc.IPType_TypeENIMultiIP - var netConfs []*rpc.NetConf - netConfs, err = n.multiIPFromCRD(podinfo, true) - if err != nil { - return nil, err - } - netConf = append(netConf, netConfs...) + switch pod.PodNetworkType { + case daemon.PodNetworkTypeENIMultiIP: + reply.IPType = rpc.IPType_TypeENIMultiIP - defaultIfSet := false - for _, cfg := range netConf { - if defaultIf(cfg.IfName) { - defaultIfSet = true - } - } - if !defaultIfSet { - // alloc eniip - var eniIP *types.ENIIP - eniIP, err = n.allocateENIMultiIP(networkContext, &oldRes) - if err != nil { - return nil, fmt.Errorf("error get allocated eniip ip for: %+v, result: %+v", podinfo, err) - } - newRes := types.PodResources{ - PodInfo: podinfo, - Resources: eniIP.ToResItems(), - NetNs: func(s string) *string { - return &s - }(r.Netns), - ContainerID: func(s string) *string { - return &s - }(r.K8SPodInfraContainerId), - } - networkContext.resources = append(networkContext.resources, newRes.Resources...) - if n.eipResMgr != nil && podinfo.EipInfo.PodEip { - podinfo.PodIPs = eniIP.IPSet - var eipRes *types.EIP - eipRes, err = n.allocateEIP(networkContext, &oldRes) - if err != nil { - return nil, fmt.Errorf("error get allocated eip for: %+v, result: %+v", podinfo, err) - } - eipResItem := eipRes.ToResItems() - newRes.Resources = append(newRes.Resources, eipResItem...) - networkContext.resources = append(networkContext.resources, eipResItem...) - } - err = n.resourceDB.Put(podInfoKey(podinfo.Namespace, podinfo.Name), newRes) - if err != nil { - return nil, errors.Wrapf(err, "error put resource into store") + if pod.PodENI { + resourceRequests = append(resourceRequests, &eni.RemoteIPRequest{}) + } else { + req := &eni.LocalIPRequest{} + + if len(oldRes.GetResourceItemByType(daemon.ResourceTypeENIIP)) == 1 { + old := oldRes.GetResourceItemByType(daemon.ResourceTypeENIIP)[0] + + setRequest(req, old) } - netConf = append(netConf, &rpc.NetConf{ - BasicInfo: &rpc.BasicInfo{ - PodIP: eniIP.IPSet.ToRPC(), - PodCIDR: eniIP.ENI.VSwitchCIDR.ToRPC(), - GatewayIP: eniIP.ENI.GatewayIP.ToRPC(), - ServiceCIDR: n.k8s.GetServiceCIDR().ToRPC(), - }, - ENIInfo: &rpc.ENIInfo{ - MAC: eniIP.ENI.MAC, - Trunk: false, - }, - Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, - NetworkPriority: podinfo.NetworkPriority, - }, - IfName: "", - ExtraRoutes: nil, - DefaultRoute: true, - }) + resourceRequests = append(resourceRequests, req) } + case daemon.PodNetworkTypeVPCENI: + reply.IPType = rpc.IPType_TypeVPCENI - err = defaultForNetConf(netConf) - if err != nil { - return nil, err - } - allocIPReply.Success = true - case podNetworkTypeVPCENI: - allocIPReply.IPType = rpc.IPType_TypeVPCENI - if n.ipamType == types.IPAMTypeCRD { - var netConfs []*rpc.NetConf - netConfs, err = n.exclusiveENIFromCRD(podinfo, true) - if err != nil { - return nil, err - } - netConf = append(netConf, netConfs...) + if pod.PodENI || n.ipamType == types.IPAMTypeCRD { + resourceRequests = append(resourceRequests, &eni.RemoteIPRequest{}) } else { - var eni *types.ENI - eni, err = n.allocateENI(networkContext, &oldRes) - if err != nil { - return nil, fmt.Errorf("error get allocated vpc ENI ip for: %+v, result: %+v", podinfo, err) - } - newRes := types.PodResources{ - PodInfo: podinfo, - Resources: eni.ToResItems(), - NetNs: func(s string) *string { - return &s - }(r.Netns), - ContainerID: func(s string) *string { - return &s - }(r.K8SPodInfraContainerId), - } - networkContext.resources = append(networkContext.resources, newRes.Resources...) - if n.eipResMgr != nil && podinfo.EipInfo.PodEip { - podinfo.PodIPs = eni.PrimaryIP - var eipRes *types.EIP - eipRes, err = n.allocateEIP(networkContext, &oldRes) - if err != nil { - return nil, fmt.Errorf("error get allocated eip for: %+v, result: %+v", podinfo, err) - } - eipResItem := eipRes.ToResItems() - newRes.Resources = append(newRes.Resources, eipResItem...) - networkContext.resources = append(networkContext.resources, eipResItem...) - } - err = n.resourceDB.Put(podInfoKey(podinfo.Namespace, podinfo.Name), newRes) - if err != nil { - return nil, errors.Wrapf(err, "error put resource into store") + req := &eni.LocalIPRequest{} + + if len(oldRes.GetResourceItemByType(daemon.ResourceTypeENI)) == 1 { + old := oldRes.GetResourceItemByType(daemon.ResourceTypeENI)[0] + + setRequest(req, old) } - netConf = append(netConf, &rpc.NetConf{ - BasicInfo: &rpc.BasicInfo{ - PodIP: eni.PrimaryIP.ToRPC(), - PodCIDR: eni.VSwitchCIDR.ToRPC(), - GatewayIP: eni.GatewayIP.ToRPC(), - ServiceCIDR: n.k8s.GetServiceCIDR().ToRPC(), - }, - ENIInfo: &rpc.ENIInfo{ - MAC: eni.MAC, - Trunk: false, - }, - Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, - NetworkPriority: podinfo.NetworkPriority, - }, - IfName: "", - ExtraRoutes: nil, - DefaultRoute: true, - }) + resourceRequests = append(resourceRequests, req) } - allocIPReply.Success = true - case podNetworkTypeVPCIP: - allocIPReply.IPType = rpc.IPType_TypeVPCIP - var vpcVeth *types.Veth - vpcVeth, err = n.allocateVeth(networkContext, &oldRes) - if err != nil { - return nil, fmt.Errorf("error get allocated vpc ip for: %+v, result: %+v", podinfo, err) + case daemon.PodNetworkTypeVPCIP: + reply.IPType = rpc.IPType_TypeVPCIP + resourceRequests = append(resourceRequests, &eni.VethRequest{}) + default: + return nil, &types.Error{ + Code: types.InternalError, + Msg: "Unknown pod network type", } - newRes := types.PodResources{ - PodInfo: podinfo, - Resources: vpcVeth.ToResItems(), - NetNs: func(s string) *string { - return &s - }(r.Netns), - ContainerID: func(s string) *string { - return &s - }(r.K8SPodInfraContainerId), + } + + var networkResource []daemon.ResourceItem + + resp, err := n.eniMgr.Allocate(ctx, cni, &eni.AllocRequest{ + ResourceRequests: resourceRequests, + }) + if err != nil { + _ = n.eniMgr.Release(ctx, cni, &eni.ReleaseRequest{ + NetworkResources: resp, + }) + return nil, err + } + + for _, res := range resp { + netConf = append(netConf, res.ToRPC()...) + networkResource = append(networkResource, res.ToStore()...) + } + + for _, c := range netConf { + if c.BasicInfo == nil { + c.BasicInfo = &rpc.BasicInfo{} } - networkContext.resources = append(networkContext.resources, newRes.Resources...) - err = n.resourceDB.Put(podInfoKey(podinfo.Namespace, podinfo.Name), newRes) - if err != nil { - return nil, errors.Wrapf(err, "error put resource into store") + c.BasicInfo.ServiceCIDR = n.k8s.GetServiceCIDR().ToRPC() + if pod.PodNetworkType == daemon.PodNetworkTypeVPCIP { + c.BasicInfo.PodCIDR = n.k8s.GetNodeCidr().ToRPC() + } + c.Pod = &rpc.Pod{ + Ingress: pod.TcIngress, + Egress: pod.TcEgress, + NetworkPriority: pod.NetworkPriority, + } + } + + err = defaultForNetConf(netConf) + if err != nil { + return nil, err + } + + out, err := json.Marshal(netConf) + if err != nil { + return nil, &types.Error{ + Code: types.InternalError, + R: err, } - netConf = append(netConf, &rpc.NetConf{ - BasicInfo: &rpc.BasicInfo{ - PodIP: nil, - PodCIDR: n.k8s.GetNodeCidr().ToRPC(), - GatewayIP: nil, - ServiceCIDR: n.k8s.GetServiceCIDR().ToRPC(), - }, - ENIInfo: nil, - Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, - NetworkPriority: podinfo.NetworkPriority, - }, - IfName: "", - ExtraRoutes: nil, - DefaultRoute: true, - }) - allocIPReply.Success = true - default: - return nil, fmt.Errorf("not support pod network type") } - // 4. grpc connection - if ctx.Err() != nil { - err = ctx.Err() - return nil, fmt.Errorf("error on grpc connection, %w", err) + // 4. Record resource info + newRes := daemon.PodResources{ + PodInfo: pod, + Resources: networkResource, + NetNs: &r.Netns, + ContainerID: &r.K8SPodInfraContainerId, + NetConf: string(out), } - allocIPReply.NetConfs = netConf - allocIPReply.EnableTrunking = n.enableTrunk + err = n.resourceDB.Put(podID, newRes) + if err != nil { + return nil, err + } - // 5. return allocate result - return allocIPReply, err + reply.NetConfs = netConf + reply.Success = true + + return reply, nil } func (n *networkService) ReleaseIP(ctx context.Context, r *rpc.ReleaseIPRequest) (*rpc.ReleaseIPReply, error) { - serviceLog.WithFields(map[string]interface{}{ - "pod": podInfoKey(r.K8SPodNamespace, r.K8SPodName), - "containerID": r.K8SPodInfraContainerId, - }).Info("release ip req") + podID := utils.PodInfoKey(r.K8SPodNamespace, r.K8SPodName) + l := logf.FromContext(ctx) + l.Info("release ip req") - _, exist := n.pendingPods.LoadOrStore(podInfoKey(r.K8SPodNamespace, r.K8SPodName), struct{}{}) + _, exist := n.pendingPods.LoadOrStore(podID, struct{}{}) if exist { - return nil, fmt.Errorf("pod %s resource processing", podInfoKey(r.K8SPodNamespace, r.K8SPodName)) + return nil, &types.Error{ + Code: types.PodIsProcessing, + Msg: fmt.Sprintf("Pod %s request is processing", podID), + } } defer func() { - n.pendingPods.Delete(podInfoKey(r.K8SPodNamespace, r.K8SPodName)) + n.pendingPods.Delete(podID) }() n.RLock() @@ -505,250 +306,147 @@ func (n *networkService) ReleaseIP(ctx context.Context, r *rpc.ReleaseIPRequest) start = time.Now() err error ) + + reply := &rpc.ReleaseIPReply{ + Success: true, + IPv4: n.enableIPv4, + IPv6: n.enableIPv6, + } + defer func() { metric.RPCLatency.WithLabelValues("ReleaseIP", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) + l.Info("release ip", "reply", fmt.Sprintf("%v", reply), "err", fmt.Sprintf("%v", err)) }() // 0. Get pod Info - podinfo, err := n.k8s.GetPod(r.K8SPodNamespace, r.K8SPodName, true) + pod, err := n.k8s.GetPod(ctx, r.K8SPodNamespace, r.K8SPodName, true) if err != nil { - return nil, errors.Wrapf(err, "error get pod info for: %+v", r) + if k8sErr.IsNotFound(err) { + return reply, nil + } + return nil, err } - // 1. Init Context - netCtx := &networkContext{ - Context: ctx, - resources: []types.ResourceItem{}, - pod: podinfo, - k8sService: n.k8s, - } - releaseReply := &rpc.ReleaseIPReply{ - Success: true, - IPv4: n.ipFamily.IPv4, - IPv6: n.ipFamily.IPv6, + cni := &daemon.CNI{ + PodName: r.K8SPodName, + PodNamespace: r.K8SPodNamespace, + PodID: podID, + PodUID: pod.PodUID, } - defer func() { - if err != nil { - netCtx.Log().Errorf("release result with error, %+v", err) - } else { - netCtx.Log().Infof("release result: %+v", releaseReply) - } - }() + // 1. Init Context - oldRes, err := n.getPodResource(podinfo) + oldRes, err := n.getPodResource(pod) if err != nil { return nil, err } - if !n.verifyPodNetworkType(podinfo.PodNetworkType) { - netCtx.Log().Warnf("unexpect pod network type release, maybe daemon mode changed: %+v", podinfo.PodNetworkType) - return releaseReply, nil + if !n.verifyPodNetworkType(pod.PodNetworkType) { + return nil, fmt.Errorf("unexpect pod network type allocate, maybe daemon mode changed: %+v", pod.PodNetworkType) } + if oldRes.ContainerID != nil { if r.K8SPodInfraContainerId != *oldRes.ContainerID { - netCtx.Log().Warnf("cni request not macth stored resource, expect %s, got %s, ignored", *oldRes.ContainerID, r.K8SPodInfraContainerId) - return releaseReply, nil + l.Info("cni request not match stored resource, ignored", "old", *oldRes.ContainerID) + return reply, nil } } - for _, res := range oldRes.Resources { - //record old resource for pod - netCtx.resources = append(netCtx.resources, res) - mgr := n.getResourceManagerForRes(res.Type) - if mgr == nil { - netCtx.Log().Warnf("error cleanup allocated network resource %s, %s: %v", res.ID, res.Type, err) - continue - } - if podinfo.IPStickTime == 0 { - if err = mgr.Release(netCtx, res); err != nil && err != pool.ErrInvalidState { - return nil, errors.Wrapf(err, "error release request network resource for: %+v", r) + if pod.IPStickTime == 0 { + for _, resource := range oldRes.Resources { + res := parseNetworkResource(resource) + if res == nil { + continue } - if err = n.deletePodResource(podinfo); err != nil { - return nil, errors.Wrapf(err, "error delete resource from db: %+v", r) + + err = n.eniMgr.Release(ctx, cni, &eni.ReleaseRequest{ + NetworkResources: []eni.NetworkResource{res}, + }) + if err != nil { + return nil, err } } + err = n.deletePodResource(pod) + if err != nil { + return nil, fmt.Errorf("error delete pod resource: %w", err) + } } - if netCtx.Err() != nil { - err = ctx.Err() - return nil, fmt.Errorf("error on grpc connection, %w", err) - } - - return releaseReply, nil + return reply, nil } func (n *networkService) GetIPInfo(ctx context.Context, r *rpc.GetInfoRequest) (*rpc.GetInfoReply, error) { - serviceLog.Debugf("GetIPInfo request: %+v", r) - // 0. Get pod Info - podinfo, err := n.k8s.GetPod(r.K8SPodNamespace, r.K8SPodName, true) - if err != nil { - return nil, errors.Wrapf(err, "error get pod info for: %+v", r) - } + podID := utils.PodInfoKey(r.K8SPodNamespace, r.K8SPodName) + log := logf.FromContext(ctx) + log.Info("get ip req") - if !n.verifyPodNetworkType(podinfo.PodNetworkType) { - return nil, fmt.Errorf("unexpect pod network type get info, maybe daemon mode changed: %+v", podinfo.PodNetworkType) - } - - // 1. Init Context - networkContext := &networkContext{ - Context: ctx, - resources: []types.ResourceItem{}, - pod: podinfo, - k8sService: n.k8s, + _, exist := n.pendingPods.LoadOrStore(podID, struct{}{}) + if exist { + return nil, &types.Error{ + Code: types.PodIsProcessing, + Msg: fmt.Sprintf("Pod %s request is processing", podID), + } } - - getIPInfoResult := &rpc.GetInfoReply{IPv4: n.ipFamily.IPv4, IPv6: n.ipFamily.IPv6} - defer func() { - networkContext.Log().Debugf("getIpInfo result: %+v", getIPInfoResult) + n.pendingPods.Delete(podID) }() n.RLock() - podRes, err := n.getPodResource(podinfo) - n.RUnlock() + defer n.RUnlock() + + var err error + + // 0. Get pod Info + pod, err := n.k8s.GetPod(ctx, r.K8SPodNamespace, r.K8SPodName, true) if err != nil { - networkContext.Log().Errorf("failed to get pod info : %+v", err) - return getIPInfoResult, err + return nil, &types.Error{ + Code: types.InvalidArgsErrCode, + Msg: err.Error(), + R: err, + } } - if podRes.ContainerID != nil { - if r.K8SPodInfraContainerId != *podRes.ContainerID { - networkContext.Log().Warnf("cni request not macth stored resource, expect %s, got %s, ignored", *podRes.ContainerID, r.K8SPodInfraContainerId) - return getIPInfoResult, nil - } + // 1. Init Context + reply := &rpc.GetInfoReply{ + Success: true, + IPv4: n.enableIPv4, + IPv6: n.enableIPv6, } - var netConf []*rpc.NetConf - // 2. return network info for pod - switch podinfo.PodNetworkType { - case podNetworkTypeENIMultiIP: - getIPInfoResult.IPType = rpc.IPType_TypeENIMultiIP - netConfs, err2 := n.multiIPFromCRD(podinfo, false) - if err != nil { - if k8sErr.IsNotFound(err2) { - getIPInfoResult.Error = rpc.Error_ErrCRDNotFound - } - return getIPInfoResult, nil + // 2. Find old resource info + oldRes, err := n.getPodResource(pod) + if err != nil { + return nil, &types.Error{ + Code: types.InternalError, + Msg: err.Error(), + R: err, } - netConf = append(netConf, netConfs...) + } - defaultIfSet := false - for _, cfg := range netConf { - if defaultIf(cfg.IfName) { - defaultIfSet = true - } + if !n.verifyPodNetworkType(pod.PodNetworkType) { + return nil, &types.Error{ + Code: types.InvalidArgsErrCode, + Msg: fmt.Sprintf("Unexpected network type, maybe daemon mode changed"), } - if !defaultIfSet { - resItems := podRes.GetResourceItemByType(types.ResourceTypeENIIP) - if len(resItems) > 0 { - // only have one - res, err := n.eniIPResMgr.Stat(networkContext, resItems[0].ID) - if err == nil { - eniIP := res.(*types.ENIIP) - - netConf = append(netConf, &rpc.NetConf{ - BasicInfo: &rpc.BasicInfo{ - PodIP: eniIP.IPSet.ToRPC(), - PodCIDR: eniIP.ENI.VSwitchCIDR.ToRPC(), - GatewayIP: eniIP.ENI.GatewayIP.ToRPC(), - ServiceCIDR: n.k8s.GetServiceCIDR().ToRPC(), - }, - ENIInfo: &rpc.ENIInfo{ - MAC: eniIP.ENI.MAC, - Trunk: false, - }, - Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, - NetworkPriority: podinfo.NetworkPriority, - }, - IfName: "", - ExtraRoutes: nil, - }) + } - } else { - serviceLog.Debugf("failed to get res stat %s", resItems[0].ID) - } - } - } - err = defaultForNetConf(netConf) - if err != nil { - return getIPInfoResult, err - } - case podNetworkTypeVPCIP: - getIPInfoResult.IPType = rpc.IPType_TypeVPCIP - netConf = append(netConf, &rpc.NetConf{ - BasicInfo: &rpc.BasicInfo{ - PodIP: nil, - PodCIDR: n.k8s.GetNodeCidr().ToRPC(), - GatewayIP: nil, - ServiceCIDR: n.k8s.GetServiceCIDR().ToRPC(), - }, - Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, - NetworkPriority: podinfo.NetworkPriority, - }, - DefaultRoute: true, - }) - case podNetworkTypeVPCENI: - getIPInfoResult.IPType = rpc.IPType_TypeVPCENI - if n.ipamType == types.IPAMTypeCRD { - netConfs, err2 := n.exclusiveENIFromCRD(podinfo, false) - if err2 != nil { - if k8sErr.IsNotFound(err2) { - getIPInfoResult.Error = rpc.Error_ErrCRDNotFound - } - return getIPInfoResult, nil - } - netConf = append(netConf, netConfs...) - } else { - resItems := podRes.GetResourceItemByType(types.ResourceTypeENI) - if len(resItems) > 0 { - // only have one - res, err := n.eniResMgr.Stat(networkContext, resItems[0].ID) - if err == nil { - eni := res.(*types.ENI) - - netConf = append(netConf, &rpc.NetConf{ - BasicInfo: &rpc.BasicInfo{ - PodIP: eni.PrimaryIP.ToRPC(), - PodCIDR: eni.VSwitchCIDR.ToRPC(), - GatewayIP: eni.GatewayIP.ToRPC(), - ServiceCIDR: n.k8s.GetServiceCIDR().ToRPC(), - }, - ENIInfo: &rpc.ENIInfo{ - MAC: eni.MAC, - Trunk: podinfo.PodENI && n.enableTrunk && eni.Trunk, - }, - Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, - NetworkPriority: podinfo.NetworkPriority, - }, - IfName: "", - ExtraRoutes: nil, - DefaultRoute: true, - }) - } else { - serviceLog.Debugf("failed to get res stat %s", resItems[0].ID) - } - } - } - default: - return getIPInfoResult, errors.Errorf("unknown or unsupport network type for: %v", r) + netConf := make([]*rpc.NetConf, 0, 0) + + err = json.Unmarshal([]byte(oldRes.NetConf), &netConf) + if err != nil { + // ignore for not found + } else { + reply.NetConfs = netConf } - getIPInfoResult.NetConfs = netConf - getIPInfoResult.EnableTrunking = n.enableTrunk + reply.Success = true - return getIPInfoResult, nil + return reply, nil } func (n *networkService) RecordEvent(_ context.Context, r *rpc.EventRequest) (*rpc.EventReply, error) { - eventType := eventTypeNormal + eventType := corev1.EventTypeNormal if r.EventType == rpc.EventType_EventTypeWarning { - eventType = eventTypeWarning + eventType = corev1.EventTypeWarning } reply := &rpc.EventReply{ @@ -774,349 +472,94 @@ func (n *networkService) RecordEvent(_ context.Context, r *rpc.EventRequest) (*r } func (n *networkService) verifyPodNetworkType(podNetworkMode string) bool { - return (n.daemonMode == daemonModeVPC && //vpc - (podNetworkMode == podNetworkTypeVPCENI || podNetworkMode == podNetworkTypeVPCIP)) || + return (n.daemonMode == daemon.ModeVPC && //vpc + (podNetworkMode == daemon.PodNetworkTypeVPCENI || podNetworkMode == daemon.PodNetworkTypeVPCIP)) || // eni-multi-ip - (n.daemonMode == daemonModeENIMultiIP && podNetworkMode == podNetworkTypeENIMultiIP) || + (n.daemonMode == daemon.ModeENIMultiIP && podNetworkMode == daemon.PodNetworkTypeENIMultiIP) || // eni-only - (n.daemonMode == daemonModeENIOnly && podNetworkMode == podNetworkTypeVPCENI) -} - -func (n *networkService) startGarbageCollectionLoop() { - // period do network resource gc - gcTicker := time.NewTicker(gcPeriod) - go func() { - for range gcTicker.C { - serviceLog.Debugf("do resource gc on node") - n.Lock() - pods, err := n.k8s.GetLocalPods() - if err != nil { - serviceLog.Warnf("error get local pods for gc") - n.Unlock() - continue - } - podKeyMap := make(map[string]bool) - - for _, pod := range pods { - if !pod.SandboxExited { - podKeyMap[podInfoKey(pod.Namespace, pod.Name)] = true - } - } - - var ( - inUseSet = make(map[string]map[string]types.ResourceItem) - expireSet = make(map[string]map[string]types.ResourceItem) - relateExpireList = make([]string, 0) - ) - - resRelateList, err := n.resourceDB.List() - if err != nil { - serviceLog.Warnf("error list resource db for gc") - n.Unlock() - continue - } - - for _, resRelateObj := range resRelateList { - resRelate := resRelateObj.(types.PodResources) - _, podExist := podKeyMap[podInfoKey(resRelate.PodInfo.Namespace, resRelate.PodInfo.Name)] - if !podExist { - // check kbe-api again - _, err := n.k8s.GetPod(resRelate.PodInfo.Namespace, resRelate.PodInfo.Name, false) - if !k8sErr.IsNotFound(err) { - continue - } - serviceLog.Infof("found pod %s not exist, will cleanup related resource", podInfoKey(resRelate.PodInfo.Namespace, resRelate.PodInfo.Name)) - - if resRelate.PodInfo.IPStickTime != 0 { - // delay resource garbage collection for sticky ip - resRelate.PodInfo.IPStickTime = 0 - if err = n.resourceDB.Put(podInfoKey(resRelate.PodInfo.Namespace, resRelate.PodInfo.Name), - resRelate); err != nil { - serviceLog.Warnf("error store pod info to resource db") - } - podExist = true - } else { - relateExpireList = append(relateExpireList, podInfoKey(resRelate.PodInfo.Namespace, resRelate.PodInfo.Name)) - } - } - for _, res := range resRelate.Resources { - if _, ok := inUseSet[res.Type]; !ok { - inUseSet[res.Type] = make(map[string]types.ResourceItem) - expireSet[res.Type] = make(map[string]types.ResourceItem) - } - // already in use by others - if _, ok := inUseSet[res.Type][res.ID]; ok { - continue - } - if podExist { - // remove resource from expirelist - delete(expireSet[res.Type], res.ID) - inUseSet[res.Type][res.ID] = res - } else { - if _, ok := inUseSet[res.Type][res.ID]; !ok { - expireSet[res.Type][res.ID] = res - } - } - } - } - gcDone := true - for mgrType := range inUseSet { - mgr, ok := n.mgrForResource[mgrType] - if ok { - serviceLog.Debugf("start garbage collection for %v, list: %+v， %+v", mgrType, inUseSet[mgrType], expireSet[mgrType]) - err = mgr.GarbageCollection(inUseSet[mgrType], expireSet[mgrType]) - if err != nil { - serviceLog.Warnf("error do garbage collection for %+v, inuse: %v, expire: %v, err: %v", mgrType, inUseSet[mgrType], expireSet[mgrType], err) - gcDone = false - } - } - } - if gcDone { - func() { - resMap, ok := expireSet[types.ResourceTypeENIIP] - if !ok { - return - } - for resID := range resMap { - // try clean ip rules - list := strings.SplitAfterN(resID, ".", 2) - if len(list) <= 1 { - serviceLog.Debugf("skip gc res id %s", resID) - continue - } - serviceLog.Debugf("checking ip %s", list[1]) - _, addr, err := net.ParseCIDR(fmt.Sprintf("%s/32", list[1])) - if err != nil { - serviceLog.Errorf("failed parse ip %s", list[1]) - return - } - // try clean all - err = link.DeleteIPRulesByIP(addr) - if err != nil { - serviceLog.Errorf("failed release ip rules %v", err) - } - err = link.DeleteRouteByIP(addr) - if err != nil { - serviceLog.Errorf("failed delete route %v", err) - } - } - }() - - for _, relate := range relateExpireList { - err = n.resourceDB.Delete(relate) - if err != nil { - serviceLog.Warnf("error delete resource db relation: %v", err) - } - } - } - n.Unlock() - } - }() + (n.daemonMode == daemon.ModeENIOnly && podNetworkMode == daemon.PodNetworkTypeVPCENI) } -func (n *networkService) startPeriodCheck() { - // check pool - func() { - serviceLog.Debugf("compare poll with metadata") - podMapping, err := n.GetResourceMapping() +func (n *networkService) startGarbageCollectionLoop(ctx context.Context) { + _ = wait.PollUntilContextCancel(ctx, gcPeriod, true, func(ctx context.Context) (done bool, err error) { + err = n.gcPods(ctx) if err != nil { - serviceLog.Error(err) - return - } - for _, res := range podMapping { - if res.Valid { - continue - } - if res.Name == "" || res.Namespace == "" { - // just log - serviceLog.Warnf("found resource invalid %s %s", res.LocalResID, res.RemoteResID) - } else { - _ = tracing.RecordPodEvent(res.Name, res.Namespace, corev1.EventTypeWarning, "ResourceInvalid", fmt.Sprintf("resource %s", res.LocalResID)) - } - } - }() -} - -// requestCRD get crd from api -// note: need tolerate crd is not exist, so contained can del pod normally -func (n *networkService) requestCRD(podInfo *types.PodInfo, waitReady bool) (*podENITypes.PodENI, error) { - if n.ipamType == types.IPAMTypeCRD || podInfo.PodENI && n.enableTrunk { - var podENI *podENITypes.PodENI - var err error - if waitReady { - podENI, err = n.k8s.WaitPodENIInfo(podInfo) - } else { - podENI, err = n.k8s.GetPodENIInfo(podInfo) - } - if err != nil { - return nil, err - } - if len(podENI.Spec.Allocations) <= 0 { - return nil, fmt.Errorf("podENI has no allocation info") + serviceLog.Error(err, "error garbage collection") } - - return podENI, nil - } - return nil, nil + return false, nil + }) } -func (n *networkService) multiIPFromCRD(podInfo *types.PodInfo, waitReady bool) ([]*rpc.NetConf, error) { - var netConf []*rpc.NetConf +func (n *networkService) gcPods(ctx context.Context) error { + n.Lock() + defer n.Unlock() - var nodeTrunkENI *types.ENI - podEni, err := n.requestCRD(podInfo, waitReady) + pods, err := n.k8s.GetLocalPods() if err != nil { - return nil, fmt.Errorf("error wait pod eni info, %w", err) + return err } - if podEni == nil { - return nil, nil - } - nodeTrunkENI = n.eniIPResMgr.(*eniIPResourceManager).trunkENI - if nodeTrunkENI == nil || nodeTrunkENI.ID != podEni.Status.TrunkENIID { - return nil, fmt.Errorf("pod status eni parent not match instance trunk eni") - } - // for now only ipvlan is supported - - // call api to get eni info - for _, alloc := range podEni.Spec.Allocations { - podIP := &rpc.IPSet{} - cidr := &rpc.IPSet{} - gw := &rpc.IPSet{} - if alloc.IPv4 != "" { - podIP.IPv4 = alloc.IPv4 - cidr.IPv4 = alloc.IPv4CIDR - gw.IPv4 = terwayIP.DeriveGatewayIP(alloc.IPv4CIDR) + exist := make(map[string]bool) - if cidr.IPv4 == "" || gw.IPv4 == "" { - return nil, fmt.Errorf("empty cidr or gateway") - } - } - if alloc.IPv6 != "" { - podIP.IPv6 = alloc.IPv6 - cidr.IPv6 = alloc.IPv6CIDR - gw.IPv6 = terwayIP.DeriveGatewayIP(alloc.IPv6CIDR) - - if cidr.IPv6 == "" || gw.IPv6 == "" { - return nil, fmt.Errorf("empty cidr or gateway") - } - } - eniInfo := &rpc.ENIInfo{ - MAC: nodeTrunkENI.MAC, // set trunk eni mac - Trunk: true, - GatewayIP: nodeTrunkENI.GatewayIP.ToRPC(), - } - info, ok := podEni.Status.ENIInfos[alloc.ENI.ID] - if !ok { - return nil, fmt.Errorf("error get podENI status") + for _, pod := range pods { + if !pod.SandboxExited { + exist[utils.PodInfoKey(pod.Namespace, pod.Name)] = true } - vid := uint32(info.Vid) - eniInfo.Vid = vid - - netConf = append(netConf, &rpc.NetConf{ - BasicInfo: &rpc.BasicInfo{ - PodIP: podIP, - PodCIDR: cidr, - GatewayIP: gw, - ServiceCIDR: n.k8s.GetServiceCIDR().ToRPC(), - }, - ENIInfo: eniInfo, - Pod: &rpc.Pod{ - Ingress: podInfo.TcIngress, - Egress: podInfo.TcEgress, - NetworkPriority: podInfo.NetworkPriority, - }, - IfName: alloc.Interface, - ExtraRoutes: parseExtraRoute(alloc.ExtraRoutes), - DefaultRoute: alloc.DefaultRoute, - }) } - return netConf, nil -} - -func (n *networkService) exclusiveENIFromCRD(podInfo *types.PodInfo, waitReady bool) ([]*rpc.NetConf, error) { - var netConf []*rpc.NetConf - - var nodeTrunkENI *types.ENI - podEni, err := n.requestCRD(podInfo, waitReady) + objList, err := n.resourceDB.List() if err != nil { - return nil, fmt.Errorf("error wait pod eni info, %w", err) + return err } + podResources := getPodResources(objList) - if n.enableTrunk { - nodeTrunkENI = n.eniResMgr.(*eniResourceManager).trunkENI - if nodeTrunkENI == nil || nodeTrunkENI.ID != podEni.Status.TrunkENIID { - return nil, fmt.Errorf("pod status eni parent not match instance trunk eni") + for _, podRes := range podResources { + podID := utils.PodInfoKey(podRes.PodInfo.Namespace, podRes.PodInfo.Name) + if _, ok := exist[podID]; ok { + continue } - } - - // call api to get eni info - for _, alloc := range podEni.Spec.Allocations { - podIP := &rpc.IPSet{} - cidr := &rpc.IPSet{} - gw := &rpc.IPSet{} - if alloc.IPv4 != "" { - podIP.IPv4 = alloc.IPv4 - cidr.IPv4 = alloc.IPv4CIDR - gw.IPv4 = terwayIP.DeriveGatewayIP(alloc.IPv4CIDR) + // that is old logic ... keep it + if podRes.PodInfo.IPStickTime != 0 { + podRes.PodInfo.IPStickTime = 0 - if cidr.IPv4 == "" || gw.IPv4 == "" { - return nil, fmt.Errorf("empty cidr or gateway") + err = n.resourceDB.Put(podID, podRes) + if err != nil { + return err } + continue } - if alloc.IPv6 != "" { - podIP.IPv6 = alloc.IPv6 - cidr.IPv6 = alloc.IPv6CIDR - gw.IPv6 = terwayIP.DeriveGatewayIP(alloc.IPv6CIDR) - if cidr.IPv6 == "" || gw.IPv6 == "" { - return nil, fmt.Errorf("empty cidr or gateway") + for _, resource := range podRes.Resources { + res := parseNetworkResource(resource) + if res == nil { + continue } - } - eniInfo := &rpc.ENIInfo{ - MAC: alloc.ENI.MAC, - Trunk: false, - } - if n.enableTrunk { - eniInfo.MAC = nodeTrunkENI.MAC // set trunk eni mac - eniInfo.Trunk = true - info, ok := podEni.Status.ENIInfos[alloc.ENI.ID] - if !ok { - return nil, fmt.Errorf("error get podENI status") + err = n.eniMgr.Release(ctx, &daemon.CNI{ + PodName: podRes.PodInfo.Name, + PodNamespace: podRes.PodInfo.Namespace, + PodID: podID, + PodUID: podRes.PodInfo.PodUID, + }, &eni.ReleaseRequest{ + NetworkResources: []eni.NetworkResource{res}, + }) + if err != nil { + return err } - eniInfo.Vid = uint32(info.Vid) - eniInfo.GatewayIP = nodeTrunkENI.GatewayIP.ToRPC() } - netConf = append(netConf, &rpc.NetConf{ - BasicInfo: &rpc.BasicInfo{ - PodIP: podIP, - PodCIDR: cidr, - GatewayIP: gw, - ServiceCIDR: n.k8s.GetServiceCIDR().ToRPC(), - }, - ENIInfo: eniInfo, - Pod: &rpc.Pod{ - Ingress: podInfo.TcIngress, - Egress: podInfo.TcEgress, - NetworkPriority: podInfo.NetworkPriority, - }, - IfName: alloc.Interface, - ExtraRoutes: parseExtraRoute(alloc.ExtraRoutes), - DefaultRoute: alloc.DefaultRoute, - }) - } - err = defaultForNetConf(netConf) - if err != nil { - return nil, err + + err = n.deletePodResource(podRes.PodInfo) + if err != nil { + return err + } + serviceLog.Info("removed pod", podID) } - return netConf, nil + return nil } func (n *networkService) migrateEIP(ctx context.Context, objs []interface{}) error { once := sync.Once{} for _, resObj := range objs { - podRes, ok := resObj.(types.PodResources) + podRes, ok := resObj.(daemon.PodResources) if !ok { continue } @@ -1124,7 +567,7 @@ func (n *networkService) migrateEIP(ctx context.Context, objs []interface{}) err continue } for _, eipRes := range podRes.Resources { - if eipRes.Type != types.ResourceTypeEIP { + if eipRes.Type != daemon.ResourceTypeEIP { continue } allocType := v1beta1.IPAllocTypeAuto @@ -1149,14 +592,14 @@ func (n *networkService) migrateEIP(ctx context.Context, objs []interface{}) err ctx, cancel := context.WithTimeout(ctx, 60*time.Second) - l := serviceLog.WithField("name", fmt.Sprintf("%s/%s", podRes.PodInfo.Namespace, podRes.PodInfo.Name)) + //l := serviceLog.WithField("name", fmt.Sprintf("%s/%s", podRes.PodInfo.Namespace, podRes.PodInfo.Name)) c := n.k8s.GetClient() podEIP := &v1beta1.PodEIP{} err = c.Get(ctx, k8stypes.NamespacedName{Namespace: podRes.PodInfo.Namespace, Name: podRes.PodInfo.Name}, podEIP) if err == nil { cancel() - l.Info("skip create podEIP, already exist") + //l.Info("skip create podEIP, already exist") continue } if !k8sErr.IsNotFound(err) { @@ -1196,7 +639,7 @@ func (n *networkService) migrateEIP(ctx context.Context, objs []interface{}) err }, } - l.Infof("create podEIP for %v", podRes) + //l.Infof("create podEIP for %v", podRes) err := c.Create(ctx, podEIP) if k8sErr.IsAlreadyExists(err) { @@ -1243,7 +686,7 @@ func (n *networkService) Trace() []tracing.MapKeyValueEntry { } for _, v := range resList { - res := v.(types.PodResources) + res := v.(daemon.PodResources) var resources []string for _, v := range res.Resources { @@ -1263,6 +706,17 @@ func (n *networkService) Execute(cmd string, _ []string, message chan<- string) case commandMapping: mapping, err := n.GetResourceMapping() message <- fmt.Sprintf("mapping: %v, err: %s\n", mapping, err) + case commandResDB: + n.RLock() + defer n.RUnlock() + objList, err := n.resourceDB.List() + if err != nil { + message <- fmt.Sprintf("%s\n", err) + } else { + out, _ := json.Marshal(objList) + message <- string(out) + } + default: message <- "can't recognize command\n" } @@ -1270,113 +724,22 @@ func (n *networkService) Execute(cmd string, _ []string, message chan<- string) close(message) } -func (n *networkService) GetResourceMapping() ([]*tracing.PodMapping, error) { - var poolStats tracing.ResourcePoolStats - var err error - - n.RLock() - // get []ResourceMapping - switch n.daemonMode { - case daemonModeENIMultiIP: - poolStats, err = n.eniIPResMgr.GetResourceMapping() - case daemonModeVPC: - n.RUnlock() - return nil, nil - case daemonModeENIOnly: - poolStats, err = n.eniResMgr.GetResourceMapping() +func (n *networkService) GetResourceMapping() ([]*rpc.ResourceMapping, error) { + var mapping []*rpc.ResourceMapping + for _, status := range n.eniMgr.Status() { + mapping = append(mapping, toRPCMapping(status)) } - if err != nil { - n.RUnlock() - return nil, err - } - // pod related res - pods, err := n.resourceDB.List() - n.RUnlock() - if err != nil { - return nil, err - } - - return toResMapping(poolStats, pods) -} - -// toResMapping toResMapping -func toResMapping(poolStats tracing.ResourcePoolStats, pods []interface{}) ([]*tracing.PodMapping, error) { - // three way compare, use resource id as key - - all := map[string]*tracing.PodMapping{} - - for _, res := range poolStats.GetLocal() { - old, ok := all[res.GetID()] - if !ok { - all[res.GetID()] = &tracing.PodMapping{ - LocalResID: res.GetID(), - } - continue - } - old.LocalResID = res.GetID() - } - - for _, res := range poolStats.GetRemote() { - old, ok := all[res.GetID()] - if !ok { - all[res.GetID()] = &tracing.PodMapping{ - RemoteResID: res.GetID(), - } - continue - } - old.RemoteResID = res.GetID() - } - - for _, pod := range pods { - p := pod.(types.PodResources) - for _, res := range p.Resources { - if res.Type == types.ResourceTypeEIP { - continue - } - old, ok := all[res.ID] - if !ok { - all[res.ID] = &tracing.PodMapping{ - Name: p.PodInfo.Name, - Namespace: p.PodInfo.Namespace, - PodBindResID: res.ID, - } - continue - } - old.Name = p.PodInfo.Name - old.Namespace = p.PodInfo.Namespace - old.PodBindResID = res.ID - if old.PodBindResID == old.LocalResID && old.LocalResID == old.RemoteResID { - old.Valid = true - } - } - } - - mapping := make([]*tracing.PodMapping, 0, len(all)) - for _, res := range all { - // idle - if res.Name == "" && res.LocalResID == res.RemoteResID { - res.Valid = true - } - mapping = append(mapping, res) - } - - sort.Slice(mapping, func(i, j int) bool { - if mapping[i].Name != mapping[j].Name { - return mapping[i].Name > mapping[j].Name - } - return mapping[i].RemoteResID < mapping[j].RemoteResID - }) return mapping, nil } -func newNetworkService(ctx context.Context, configFilePath, daemonMode string) (rpc.TerwayBackendServer, error) { - serviceLog.Debugf("start network service with: %s, %s", configFilePath, daemonMode) +func newNetworkService(ctx context.Context, configFilePath, daemonMode string) (*networkService, error) { + serviceLog.Info("start network service", "config", configFilePath, "daemonMode", daemonMode) netSrv := &networkService{ configFilePath: configFilePath, pendingPods: sync.Map{}, } - if daemonMode == daemonModeENIMultiIP || daemonMode == daemonModeVPC || daemonMode == daemonModeENIOnly { + if daemonMode == daemon.ModeENIMultiIP || daemonMode == daemon.ModeVPC || daemonMode == daemon.ModeENIOnly { netSrv.daemonMode = daemonMode } else { return nil, fmt.Errorf("unsupport daemon mode") @@ -1389,15 +752,15 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( return nil, err } - netSrv.k8s, err = newK8S(daemonMode, globalConfig) + netSrv.k8s, err = k8s.NewK8S(daemonMode, globalConfig) if err != nil { return nil, fmt.Errorf("error init k8s: %w", err) } // load dynamic config - dynamicCfg, nodeLabel, err := getDynamicConfig(netSrv.k8s) + dynamicCfg, _, err := getDynamicConfig(ctx, netSrv.k8s) if err != nil { - serviceLog.Warnf("get dynamic config error: %s. fallback to default config", err.Error()) + //serviceLog.Warnf("get dynamic config error: %s. fallback to default config", err.Error()) dynamicCfg = "" } @@ -1407,9 +770,9 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( } if len(dynamicCfg) == 0 { - serviceLog.Infof("got config: %+v from: %+v", config, configFilePath) + serviceLog.Info("got config", "config", fmt.Sprintf("%+v", config)) } else { - serviceLog.Infof("got config: %+v from %+v, with dynamic config %+v", config, configFilePath, nodeLabel) + serviceLog.Info("got config", "config", fmt.Sprintf("%+v", config), "dynamicConfig", fmt.Sprintf("%+v", dynamicCfg)) } config.Populate() @@ -1421,10 +784,17 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( backoff.OverrideBackoff(config.BackoffOverride) _ = netSrv.k8s.SetCustomStatefulWorkloadKinds(config.CustomStatefulWorkloadKinds) netSrv.ipamType = config.IPAMType - netSrv.eniCapPolicy = config.ENICapPolicy - ipFamily := types.NewIPFamilyFromIPStack(types.IPStack(config.IPStack)) - netSrv.ipFamily = ipFamily + var enableIPv4, enableIPv6 bool + switch config.IPStack { + case "ipv4": + enableIPv4 = true + case "dual": + enableIPv4 = true + enableIPv6 = true + case "ipv6": + enableIPv6 = true + } var providers []credential.Interface if string(config.AccessID) != "" && string(config.AccessSecret) != "" { @@ -1442,20 +812,20 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( flowcontrol.NewTokenBucketRateLimiter(8, 10), flowcontrol.NewTokenBucketRateLimiter(4, 5)) if err != nil { - return nil, errors.Wrapf(err, "error create aliyun client") + return nil, err } - limit, err := aliyun.GetLimit(aliyunClient, config.InstanceType) + limit, err := instance.GetLimit(aliyunClient, config.InstanceType) if err != nil { return nil, fmt.Errorf("upable get instance limit, %w", err) } - if ipFamily.IPv6 { + if enableIPv6 { if !limit.SupportIPv6() { - ipFamily.IPv6 = false - serviceLog.Warnf("instance %s is not support ipv6", config.InstanceType) - } else if daemonMode == daemonModeENIMultiIP && !limit.SupportMultiIPIPv6() { - ipFamily.IPv6 = false - serviceLog.Warnf("instance %s is not support ipv6", config.InstanceType) + enableIPv6 = false + serviceLog.Info("instance is not support ipv6", "instanceType", config.InstanceType) + } else if daemonMode == daemon.ModeENIMultiIP && !limit.SupportMultiIPIPv6() { + enableIPv6 = false + serviceLog.Info("instance is not support multi ipv6", "instanceType", config.InstanceType) } } @@ -1463,19 +833,17 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( config.EnableENITrunking = false } - ecs := aliyun.NewAliyunImpl(aliyunClient, config.EnableENITrunking && !config.WaitTrunkENI, ipFamily, config.ENITagFilter) - netSrv.resourceDB, err = storage.NewDiskStorage( resDBName, utils.NormalizePath(resDBPath), json.Marshal, func(bytes []byte) (interface{}, error) { - resourceRel := &types.PodResources{} + resourceRel := &daemon.PodResources{} err = json.Unmarshal(bytes, resourceRel) if err != nil { - return nil, errors.Wrapf(err, "error unmarshal pod relate resource") + return nil, err } return *resourceRel, nil }) if err != nil { - return nil, errors.Wrapf(err, "error init resource manager storage") + return nil, err } nodeAnnotations := map[string]string{} @@ -1483,15 +851,35 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( // get pool config poolConfig, err := getPoolConfig(config, daemonMode, limit) if err != nil { - return nil, errors.Wrapf(err, "error get pool config") + return nil, err + } + poolConfig.EnableIPv4 = enableIPv4 + poolConfig.EnableIPv6 = enableIPv6 + + netSrv.enableIPv4 = enableIPv4 + netSrv.enableIPv6 = enableIPv6 + + // fall back to use primary eni's sg + if len(poolConfig.SecurityGroupIDs) == 0 { + enis, err := aliyunClient.DescribeNetworkInterface(ctx, "", nil, poolConfig.InstanceID, "Primary", "", nil) + if err != nil { + return nil, err + } + if len(enis) == 0 { + return nil, fmt.Errorf("no primary eni found") + } + poolConfig.SecurityGroupIDs = enis[0].SecurityGroupIDs } - serviceLog.Infof("init pool config: %+v", poolConfig) + + serviceLog.Info("pool config", "pool", fmt.Sprintf("%+v", poolConfig)) vswPool, err := vswpool.NewSwitchPool(100, "10m") if err != nil { return nil, fmt.Errorf("error init vsw pool, %w", err) } + factory := aliyun.NewAliyun(ctx, aliyunClient, eni2.NewENIMetadata(poolConfig.EnableIPv4, poolConfig.EnableIPv6), vswPool, poolConfig) + // init trunk if config.EnableENITrunking { preferTrunkID := netSrv.k8s.GetTrunkID() @@ -1503,7 +891,7 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( } if !config.WaitTrunkENI { - enis, err := ecs.GetAttachedENIs(ctx, false, preferTrunkID) + enis, err := factory.GetAttachedNetworkInterface(preferTrunkID) if err != nil { return nil, fmt.Errorf("error get attached eni, %w", err) } @@ -1522,16 +910,17 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( } if !found { if poolConfig.MaxENI > len(enis) { - vsw, err := vswPool.GetOne(ctx, ecs, poolConfig.ZoneID, poolConfig.VSwitchOptions, &vswpool.SelectOptions{ - VSwitchSelectPolicy: vswpool.VSwitchSelectionPolicyMost, - }) - if err != nil { - return nil, fmt.Errorf("error get vsw, %w", err) + v6 := 0 + if enableIPv6 { + v6 = 1 } - - eni, err := ecs.AllocateENI(ctx, vsw.ID, poolConfig.SecurityGroupIDs, poolConfig.InstanceID, true, 1, poolConfig.ENITags) + eni, _, _, err := factory.CreateNetworkInterface(1, v6, "trunk") if err != nil { - return nil, fmt.Errorf("error allocate eni, %w", err) + if eni != nil { + _ = factory.DeleteNetworkInterface(eni.ID) + } + + return nil, fmt.Errorf("error create trunk eni, %w", err) } poolConfig.TrunkENIID = eni.ID @@ -1540,7 +929,7 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( nodeAnnotations[types.TrunkOn] = eni.ID nodeAnnotations[string(types.MemberENIIPTypeIPs)] = strconv.Itoa(poolConfig.MaxMemberENI) } else { - serviceLog.Warnf("no trunk eni found, fallback to non-trunk mode") + serviceLog.Info("no trunk eni found, fallback to non-trunk mode") config.EnableENITrunking = false config.DisableDevicePlugin = true @@ -1556,11 +945,11 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( } } - if daemonMode != daemonModeVPC { + if daemonMode != daemon.ModeVPC { nodeAnnotations[string(types.NormalIPTypeIPs)] = strconv.Itoa(poolConfig.Capacity) } - if !(daemonMode == daemonModeENIMultiIP && !config.EnableENITrunking) { + if !(daemonMode == daemon.ModeENIMultiIP && !config.EnableENITrunking) { if !config.DisableDevicePlugin { res := deviceplugin.ENITypeENI capacity := poolConfig.MaxENI @@ -1581,12 +970,12 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( } localResource := make(map[string]map[string]resourceManagerInitItem) - resObjList, err := netSrv.resourceDB.List() + objList, err := netSrv.resourceDB.List() if err != nil { - return nil, errors.Wrapf(err, "error list resource relation db") + return nil, err } - for _, resObj := range resObjList { - podRes := resObj.(types.PodResources) + for _, resObj := range objList { + podRes := resObj.(daemon.PodResources) for _, res := range podRes.Resources { if localResource[res.Type] == nil { localResource[res.Type] = make(map[string]resourceManagerInitItem) @@ -1595,85 +984,86 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( } } + podResources := getPodResources(objList) + if config.EnableEIPMigrate { - err = netSrv.migrateEIP(ctx, resObjList) + err = netSrv.migrateEIP(ctx, objList) if err != nil { return nil, err } - serviceLog.Infof("eip migrate finished") + serviceLog.Info("eip migrate finished") } resStr, err := json.Marshal(localResource) if err != nil { return nil, err } - serviceLog.Debugf("local resources to restore: %s", resStr) + serviceLog.Info("local resources to restore", "resources", string(resStr)) err = preStartResourceManager(daemonMode, netSrv.k8s) if err != nil { return nil, err } - switch daemonMode { - case daemonModeVPC: - //init ENI - netSrv.eniResMgr, err = newENIResourceManager(poolConfig, ecs, localResource[types.ResourceTypeENI], ipFamily, netSrv.k8s, netSrv.ipamType) - if err != nil { - return nil, errors.Wrapf(err, "error init ENI resource manager") - } + var eniList []eni.NetworkInterface - netSrv.vethResMgr, err = newVPCResourceManager() - if err != nil { - return nil, errors.Wrapf(err, "error init vpc resource manager") - } + attached, err := factory.GetAttachedNetworkInterface(poolConfig.TrunkENIID) + if err != nil { + return nil, err + } - netSrv.mgrForResource = map[string]ResourceManager{ - types.ResourceTypeENI: netSrv.eniResMgr, - types.ResourceTypeVeth: netSrv.vethResMgr, - } + if daemonMode == daemon.ModeVPC { + eniList = append(eniList, &eni.Veth{}) + } - case daemonModeENIMultiIP: - //init ENI multi ip - netSrv.eniIPResMgr, err = newENIIPResourceManager(poolConfig, ecs, netSrv.k8s, localResource[types.ResourceTypeENIIP], ipFamily) - if err != nil { - return nil, errors.Wrapf(err, "error init ENI ip resource manager") - } - netSrv.mgrForResource = map[string]ResourceManager{ - types.ResourceTypeENIIP: netSrv.eniIPResMgr, - } - if config.EnableEIPPool == conditionTrue && !config.EnableEIPMigrate { - netSrv.eipResMgr = newEipResourceManager(ecs, netSrv.k8s, config.AllowEIPRob == conditionTrue) - netSrv.mgrForResource[types.ResourceTypeEIP] = netSrv.eipResMgr - } + if daemonMode == daemon.ModeENIOnly { + if config.IPAMType == types.IPAMTypeCRD { + if !config.EnableENITrunking { + eniList = append(eniList, eni.NewRemote(netSrv.k8s.GetClient(), nil)) + } else { + for _, ni := range attached { + if !ni.Trunk { + continue + } + lo := eni.NewLocal(ni, "trunk", factory, poolConfig) + eniList = append(eniList, eni.NewTrunk(netSrv.k8s.GetClient(), lo)) + } + } + } else { + // the legacy mode + for _, ni := range attached { + eniList = append(eniList, eni.NewLocal(ni, "secondary", factory, poolConfig)) + } - case daemonModeENIOnly: - //init eni - netSrv.eniResMgr, err = newENIResourceManager(poolConfig, ecs, localResource[types.ResourceTypeENI], ipFamily, netSrv.k8s, netSrv.ipamType) - if err != nil { - return nil, errors.Wrapf(err, "error init eni resource manager") + for i := 0; i < (poolConfig.MaxENI - len(attached)); i++ { + eniList = append(eniList, eni.NewLocal(nil, "secondary", factory, poolConfig)) + } } - netSrv.mgrForResource = map[string]ResourceManager{ - types.ResourceTypeENIIP: netSrv.eniIPResMgr, + } else { + for _, ni := range attached { + if config.EnableENITrunking && ni.Trunk && poolConfig.TrunkENIID == ni.ID { + lo := eni.NewLocal(ni, "trunk", factory, poolConfig) + eniList = append(eniList, eni.NewTrunk(netSrv.k8s.GetClient(), lo)) + } else { + eniList = append(eniList, eni.NewLocal(ni, "secondary", factory, poolConfig)) + } } - if config.EnableEIPPool == conditionTrue && !config.EnableENITrunking && !config.EnableEIPMigrate { - netSrv.eipResMgr = newEipResourceManager(ecs, netSrv.k8s, config.AllowEIPRob == conditionTrue) - netSrv.mgrForResource[types.ResourceTypeEIP] = netSrv.eipResMgr + + for i := 0; i < (poolConfig.MaxENI - len(attached)); i++ { + eniList = append(eniList, eni.NewLocal(nil, "secondary", factory, poolConfig)) } - default: - panic("unsupported daemon mode" + daemonMode) + } + + eniManager := eni.NewManager(poolConfig.MinPoolSize, poolConfig.MaxPoolSize, poolConfig.Capacity, 30*time.Second, eniList, netSrv.k8s) + netSrv.eniMgr = eniManager + err = eniManager.Run(ctx, &netSrv.wg, podResources) + if err != nil { + return nil, err } if config.IPAMType != types.IPAMTypeCRD { //start gc loop - netSrv.startGarbageCollectionLoop() - period := poolCheckPeriod - periodCfg := os.Getenv("POOL_CHECK_PERIOD_SECONDS") - periodSeconds, err := strconv.Atoi(periodCfg) - if err == nil { - period = time.Duration(periodSeconds) * time.Second - } - - go wait.JitterUntil(netSrv.startPeriodCheck, period, 1, true, wait.NeverStop) + go netSrv.startGarbageCollectionLoop(ctx) } // register for tracing @@ -1684,19 +1074,69 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) ( return netSrv, nil } -func parseExtraRoute(routes []podENITypes.Route) []*rpc.Route { - if routes == nil { - return nil - } - var res []*rpc.Route - for _, r := range routes { - res = append(res, &rpc.Route{ - Dst: r.Dst, - }) +func getPodResources(list []interface{}) []daemon.PodResources { + var res []daemon.PodResources + for _, resObj := range list { + res = append(res, resObj.(daemon.PodResources)) } return res } +func parseNetworkResource(item daemon.ResourceItem) eni.NetworkResource { + switch item.Type { + case daemon.ResourceTypeENIIP, daemon.ResourceTypeENI: + var v4, v6 netip.Addr + if item.IPv4 != "" { + v4, _ = netip.ParseAddr(item.IPv4) + } + if item.IPv6 != "" { + v6, _ = netip.ParseAddr(item.IPv6) + } + + return &eni.LocalIPResource{ + ENI: daemon.ENI{ + ID: item.ENIID, + MAC: item.ENIMAC, + }, + IP: types.IPSet2{ + IPv4: v4, + IPv6: v6, + }, + } + } + return nil +} + +func extractIPs(old daemon.ResourceItem) (ipv4, ipv6 netip.Addr, eniID string) { + ipv4, _ = netip.ParseAddr(old.IPv4) + ipv6, _ = netip.ParseAddr(old.IPv6) + eniID = old.ENIID + return ipv4, ipv6, eniID +} + +func setRequest(req *eni.LocalIPRequest, old daemon.ResourceItem) { + ipv4, ipv6, eniID := extractIPs(old) + req.IPv4 = ipv4 + req.IPv6 = ipv6 + req.NetworkInterfaceID = eniID +} + +func toRPCMapping(res eni.Status) *rpc.ResourceMapping { + rMapping := rpc.ResourceMapping{ + NetworkInterfaceID: res.NetworkInterfaceID, + MAC: res.MAC, + Type: res.Type, + AllocInhibitExpireAt: res.AllocInhibitExpireAt, + Status: res.Status, + } + + for _, v := range res.Usage { + rMapping.Info = append(rMapping.Info, strings.Join(v, " ")) + } + + return &rMapping +} + // set default val for netConf func defaultForNetConf(netConf []*rpc.NetConf) error { // ignore netConf check diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go deleted file mode 100644 index ded5c7b9..00000000 --- a/daemon/daemon_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package daemon - -import ( - "testing" - - "github.com/AliyunContainerService/terway/pkg/tracing" - "github.com/AliyunContainerService/terway/types" - "github.com/stretchr/testify/assert" -) - -func Test_toResMapping(t *testing.T) { - pool := tracing.FakeResourcePoolStats{ - Local: map[string]types.Res{ - "idle": &types.FakeRes{ - ID: "idle", - Type: "", - Status: types.ResStatusIdle, - }, "inuse": &types.FakeRes{ - ID: "inuse", - Type: "", - Status: types.ResStatusInUse, - }, "invalid-remote": &types.FakeRes{ - ID: "invalid-remote", - Type: "", - Status: types.ResStatusInvalid, - }, - }, - Remote: map[string]types.Res{ - "idle": &types.FakeRes{ - ID: "idle", - Type: "", - Status: types.ResStatusIdle, - }, - "inuse": &types.FakeRes{ - ID: "inuse", - Type: "", - Status: types.ResStatusInUse, - }, "invalid-lo": &types.FakeRes{ - ID: "invalid-lo", - Type: "", - Status: types.ResStatusInvalid, - }, - }, - } - pods := []types.PodResources{ - { - PodInfo: &types.PodInfo{ - Name: "inuse", - Namespace: "inuse", - }, - Resources: []types.ResourceItem{{ - Type: "", - ID: "inuse", - }}, - }, - { - PodInfo: &types.PodInfo{ - Name: "invalid-remote", - Namespace: "invalid-remote", - }, - Resources: []types.ResourceItem{{ - Type: "", - ID: "invalid-remote", - }}, - }, - { - PodInfo: &types.PodInfo{ - Name: "invalid-lo", - Namespace: "invalid-lo", - }, - Resources: []types.ResourceItem{{ - Type: "", - ID: "invalid-lo", - }}, - }, - } - list := []interface{}{} - for _, p := range pods { - list = append(list, p) - } - mapping, err := toResMapping(&pool, list) - assert.NoError(t, err) - for _, m := range mapping { - switch m.Name { - case "inuse", "idle": - assert.True(t, m.Valid) - case "invalid-remote", "invalid-lo": - assert.False(t, m.Valid) - } - } -} diff --git a/daemon/daemon_unwindows.go b/daemon/daemon_unwindows.go index 3d382383..e1c3dd64 100644 --- a/daemon/daemon_unwindows.go +++ b/daemon/daemon_unwindows.go @@ -1,8 +1,11 @@ //go:build !windows -// +build !windows package daemon -func preStartResourceManager(daemonMode string, k8s Kubernetes) error { +import ( + "github.com/AliyunContainerService/terway/pkg/k8s" +) + +func preStartResourceManager(daemonMode string, k8s k8s.Kubernetes) error { return nil } diff --git a/daemon/daemon_windows.go b/daemon/daemon_windows.go index 404305a9..1f3d36c3 100644 --- a/daemon/daemon_windows.go +++ b/daemon/daemon_windows.go @@ -7,12 +7,14 @@ import ( "github.com/pkg/errors" "github.com/AliyunContainerService/terway/pkg/aliyun/metadata" + "github.com/AliyunContainerService/terway/pkg/k8s" "github.com/AliyunContainerService/terway/pkg/windows/apis" "github.com/AliyunContainerService/terway/pkg/windows/iface" "github.com/AliyunContainerService/terway/pkg/windows/ip" + "github.com/AliyunContainerService/terway/types/daemon" ) -func preStartResourceManager(daemonMode string, k8s Kubernetes) error { +func preStartResourceManager(daemonMode string, k8s k8s.Kubernetes) error { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() @@ -22,7 +24,7 @@ func preStartResourceManager(daemonMode string, k8s Kubernetes) error { } switch daemonMode { - case daemonModeVPC: + case daemon.ModeVPC: var nwIface, err = iface.GetInterfaceByMAC(primaryMac, true) if err != nil { return errors.Wrap(err, "error getting interface") @@ -39,7 +41,7 @@ func preStartResourceManager(daemonMode string, k8s Kubernetes) error { if err != nil { return errors.Wrapf(err, "error adding network: %s", nw.Format(apis.HNS)) } - case daemonModeENIOnly, daemonModeENIMultiIP: + case daemon.ModeENIOnly, daemon.ModeENIMultiIP: // NB(thxCode): create a fake network to allow service connection var assistantIface, err = iface.GetInterfaceByMAC(primaryMac, true) if err != nil { diff --git a/daemon/eip.go b/daemon/eip.go deleted file mode 100644 index 9b6319be..00000000 --- a/daemon/eip.go +++ /dev/null @@ -1,141 +0,0 @@ -package daemon - -import ( - "context" - "fmt" - "net" - - "github.com/AliyunContainerService/terway/pkg/aliyun" - "github.com/AliyunContainerService/terway/pkg/ipam" - "github.com/AliyunContainerService/terway/pkg/logger" - "github.com/AliyunContainerService/terway/pkg/tracing" - - "github.com/AliyunContainerService/terway/types" -) - -var eipLog = logger.DefaultLogger - -// eip resource manager for pod public ip address -type eipResourceManager struct { - ecs ipam.API - k8s Kubernetes - allowEipRob bool -} - -func newEipResourceManager(e ipam.API, k Kubernetes, allowEipRob bool) ResourceManager { - return &eipResourceManager{ - ecs: e, - k8s: k, - allowEipRob: allowEipRob, - } -} - -func (e *eipResourceManager) Allocate(context *networkContext, prefer string) (types.NetworkResource, error) { - eipLog.Infof("Allocate EIP: %v, %v, %s", context.pod, context.resources, prefer) - if context.pod == nil { - return nil, fmt.Errorf("invalid pod info: %v", context.pod) - } - var ( - eniID string - eniIP net.IP - err error - ) - ctx := context.Context - podIP := context.pod.PodIPs.IPv4 - if podIP == nil { - return nil, fmt.Errorf("pod ipv4 addr is empty") - } - for _, res := range context.resources { - switch res.Type { - case types.ResourceTypeENI, types.ResourceTypeENIIP: - eniID, err = e.ecs.QueryEniIDByIP(ctx, aliyun.GetInstanceMeta().VPCID, podIP) - if err != nil { - return nil, fmt.Errorf("error Query ENI by pod IP, %v, %w", context.pod, err) - } - eniIP = podIP - } - } - if eniID == "" && eniIP == nil { - return nil, fmt.Errorf("pod network mode not support EIP associate") - } - eipID := context.pod.EipInfo.PodEipID - if eipID == "" { - eipLog.Infof("eip id empty pod") - eipID = prefer - } - eipInfo, err := e.ecs.AllocateEipAddress(ctx, context.pod.EipInfo.PodEipBandWidth, context.pod.EipInfo.PodEipChargeType, - eipID, eniID, eniIP, e.allowEipRob, context.pod.EipInfo.PodEipISP, context.pod.EipInfo.PodEipBandwidthPackageID, context.pod.EipInfo.PodEipPoolID) - if err != nil { - return nil, fmt.Errorf("error allocate eip info: %w", err) - } - - // set eip to delete if pod not specific eip id - if context.pod.EipInfo.PodEipID == "" { - eipInfo.Delete = true - } - context.pod.EipInfo.PodEipIP = eipInfo.Address.String() - err = e.k8s.PatchEipInfo(context.pod) - if err != nil { - var err1 error - if eipInfo.Delete { - err1 = e.ecs.ReleaseEipAddress(ctx, eipInfo.ID, eniID, eniIP) - } else { - err1 = e.ecs.UnassociateEipAddress(ctx, eipInfo.ID, eniID, eniIP.String()) - } - if err1 != nil { - eipLog.Errorf("error rollback eip: %v", err1) - } - return nil, fmt.Errorf("error patch pod info: %w", err) - } - return eipInfo, nil -} - -func (e *eipResourceManager) Release(context *networkContext, resItem types.ResourceItem) error { - if resItem.ExtraEipInfo == nil { - return nil - } - eipLog.Infof("release eip: %v, %v", resItem.ID, resItem.ExtraEipInfo) - ctx := context.Context - - if resItem.ExtraEipInfo.Delete { - err := e.ecs.ReleaseEipAddress(ctx, resItem.ID, resItem.ExtraEipInfo.AssociateENI, resItem.ExtraEipInfo.AssociateENIIP) - if err != nil { - return err - } - } else { - err := e.ecs.UnassociateEipAddress(ctx, resItem.ID, resItem.ExtraEipInfo.AssociateENI, resItem.ExtraEipInfo.AssociateENIIP.String()) - if err != nil { - return err - } - } - return nil -} - -func (e *eipResourceManager) GarbageCollection(inUseResSet map[string]types.ResourceItem, expireResSet map[string]types.ResourceItem) error { - for expireRes, expireItem := range expireResSet { - if expireItem.ExtraEipInfo == nil { - continue - } - eipLog.Infof("release eip: %v, %v", expireRes, expireItem) - if expireItem.ExtraEipInfo.Delete { - err := e.ecs.ReleaseEipAddress(context.Background(), expireRes, expireItem.ExtraEipInfo.AssociateENI, expireItem.ExtraEipInfo.AssociateENIIP) - if err != nil { - return err - } - } else { - err := e.ecs.UnassociateEipAddress(context.Background(), expireRes, expireItem.ExtraEipInfo.AssociateENI, expireItem.ExtraEipInfo.AssociateENIIP.String()) - if err != nil { - return err - } - } - } - return nil -} - -func (e *eipResourceManager) Stat(context *networkContext, resID string) (types.NetworkResource, error) { - return nil, nil -} - -func (e *eipResourceManager) GetResourceMapping() (tracing.ResourcePoolStats, error) { - return nil, fmt.Errorf("eip resource manager store network resource") -} diff --git a/daemon/eni-multi-ip.go b/daemon/eni-multi-ip.go deleted file mode 100644 index 23f71b94..00000000 --- a/daemon/eni-multi-ip.go +++ /dev/null @@ -1,1165 +0,0 @@ -package daemon - -import ( - "context" - "fmt" - "net" - "sort" - "strings" - "sync" - "time" - - apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" - terwayIP "github.com/AliyunContainerService/terway/pkg/ip" - "github.com/AliyunContainerService/terway/pkg/ipam" - "github.com/AliyunContainerService/terway/pkg/logger" - "github.com/AliyunContainerService/terway/pkg/metric" - "github.com/AliyunContainerService/terway/pkg/pool" - "github.com/AliyunContainerService/terway/pkg/tracing" - "github.com/AliyunContainerService/terway/pkg/utils" - "github.com/AliyunContainerService/terway/types" - - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - corev1 "k8s.io/api/core/v1" -) - -type AllocCtx struct { - Trace []Trace -} - -func (a *AllocCtx) String() string { - var s []string - for _, t := range a.Trace { - s = append(s, t.Msg) - } - return strings.Join(s, "\n") -} - -type Trace struct { - Msg string -} - -var eniIPLog = logger.DefaultLogger - -const ( - maxEniOperating = 3 - maxIPBacklog = 10 -) - -const ( - eniIPAllocInhibitTimeout = 10 * time.Minute - - typeNameENIIP = "eniip" - poolNameENIIP = "eniip" - factoryNameENIIP = "eniip" - - tracingKeyENIMaxIP = "eni_max_ip" - tracingKeyENICount = "eni_count" - tracingKeySecondaryIPCount = "secondary_ip_count" - - commandAudit = "audit" -) - -const timeFormat = "2006-01-02 15:04:05" - -type eniIPFactory struct { - name string - enableTrunk bool - trunkOnEni string - eniFactory *eniFactory - enis []*ENI - maxENI chan struct{} - eniMaxIP int - nodeMaxENI int - eniOperChan chan struct{} - ipResultChan chan *ENIIP - sync.RWMutex - // metrics - metricENICount prometheus.Gauge - - ipFamily *types.IPFamily -} - -// ENIIP the secondary ip of eni -type ENIIP struct { - *types.ENIIP - err error -} - -// ENI to hold ENI's secondary config -type ENI struct { - lock sync.Mutex - *types.ENI - ips []*ENIIP - pending int - ipBacklog chan struct{} - ecs ipam.API - done chan struct{} - // Unix timestamp to mark when this ENI can allocate Pod IP. - ipAllocInhibitExpireAt time.Time -} - -func (e *ENI) getIPCountLocked() int { - return e.pending + len(e.ips) -} - -// eni ip allocator -func (e *ENI) allocateWorker(resultChan chan<- *ENIIP) { - for { - toAllocate := 0 - select { - case <-e.done: - return - case <-e.ipBacklog: - toAllocate = 1 - } - // wait 300ms for aggregation the cni request - time.Sleep(300 * time.Millisecond) - popAll: - for { - select { - case <-e.ipBacklog: - toAllocate++ - default: - break popAll - } - if toAllocate >= maxIPBacklog { - break - } - } - eniIPLog.Debugf("allocate %v ips for eni", toAllocate) - v4, v6, err := e.ecs.AssignNIPsForENI(context.Background(), e.ENI.ID, e.ENI.MAC, toAllocate) - eniIPLog.Debugf("allocated ips for eni: eni = %+v, v4 = %+v,v6 = %+v, err = %v", e.ENI, v4, v6, err) - if err != nil { - eniIPLog.Errorf("error allocate ips for eni: %v", err) - metric.ENIIPFactoryIPAllocCount.WithLabelValues(e.MAC, metric.ENIIPAllocActionFail).Add(float64(toAllocate)) - for i := 0; i < toAllocate; i++ { - resultChan <- &ENIIP{ - ENIIP: &types.ENIIP{ - ENI: e.ENI, - }, - err: errors.Errorf("error assign ip for ENI: %v", err), - } - } - } else { - metric.ENIIPFactoryIPAllocCount.WithLabelValues(e.MAC, metric.ENIIPAllocActionSucceed).Add(float64(toAllocate)) - for _, ip := range types.MergeIPs(v4, v6) { - resultChan <- &ENIIP{ - ENIIP: &types.ENIIP{ - ENI: e.ENI, - IPSet: ip, - }, - err: nil, - } - } - } - } -} - -func (f *eniIPFactory) getEnis(ctx *AllocCtx) ([]*ENI, error) { - var ( - enis []*ENI - pendingEnis []*ENI - ) - enisLen := len(f.enis) - // If there is only one switch, then no need for ordering. - if enisLen <= 1 { - return f.enis, nil - } - // pre sort eni by ip count to balance ip allocation on enis - sort.Slice(f.enis, func(i, j int) bool { - return f.enis[i].getIPCountLocked() < f.enis[j].getIPCountLocked() - }) - - // If VSwitchSelectionPolicy is ordered, then call f.eniFactory.GetVSwitches() API to get a switch slice - // in descending order per each switch's available IP count. - vSwitches, err := f.eniFactory.GetVSwitches() - eniIPLog.Infof("adjusted vswitch slice: %+v, original eni slice: %s", vSwitches, func(enis []*ENI) string { - var vsw []string - for i := 0; i < len(enis); i++ { - if enis[i] == nil || enis[i].ENI == nil { - continue - } - vsw = append(vsw, enis[i].VSwitchID) - } - return strings.Join(vsw, ",") - }(f.enis)) - if err != nil { - eniIPLog.Errorf("error to get vswitch slice: %v, instead use original eni slice in eniIPFactory: %v", err, f.enis) - return f.enis, err - } - for _, vsw := range vSwitches { - for _, eni := range f.enis { - if eni.ENI == nil { - pendingEnis = append(pendingEnis, eni) - continue - } - if vsw.id == eni.VSwitchID { - enis = append(enis, eni) - } - } - } - enis = append(enis, pendingEnis...) - if len(enis) == 0 && len(f.enis) != 0 { - ctx.Trace = append(ctx.Trace, Trace{ - Msg: "current eni have vSwitch not as expected, please check eni-config", - }) - } - - return enis, nil -} - -func (f *eniIPFactory) submit(ctx *AllocCtx) error { - f.Lock() - defer f.Unlock() - var enis []*ENI - enis, _ = f.getEnis(ctx) - for _, eni := range enis { - eniIPLog.Infof("check existing eni: %+v", eni) - eni.lock.Lock() - now := time.Now() - if eni.ENI != nil { - eniIPLog.Infof("check if the current eni is in the time window for IP allocation inhibition: "+ - "eni = %+v, vsw= %s, now = %s, expireAt = %s", eni, eni.VSwitchID, now.Format(timeFormat), eni.ipAllocInhibitExpireAt.Format(timeFormat)) - } - // if the current eni has been inhibited for Pod IP allocation, then skip current eni. - if now.Before(eni.ipAllocInhibitExpireAt) && eni.ENI != nil { - eni.lock.Unlock() - - // vsw have insufficient ip - ctx.Trace = append(ctx.Trace, Trace{Msg: fmt.Sprintf("eni %s vSwitch %s have insufficient IP, next check %s", eni.ID, eni.VSwitchID, eni.ipAllocInhibitExpireAt)}) - - eniIPLog.Debugf("skip IP allocation: eni = %+v, vsw = %s", eni, eni.VSwitchID) - continue - } - - eniIPLog.Debugf("check if the current eni will reach eni IP quota with new pending IP added: "+ - "eni = %+v, eni.pending = %d, len(eni.ips) = %d, eni.MaxIPs = %d", eni, eni.pending, len(eni.ips), f.eniMaxIP) - if eni.getIPCountLocked() < f.eniMaxIP { - select { - case eni.ipBacklog <- struct{}{}: - default: - eni.lock.Unlock() - continue - } - eniIPLog.Debugf("submit eniip to the eni backlog: %+v", eni) - eni.pending++ - eni.lock.Unlock() - return nil - } - eni.lock.Unlock() - } - return errors.Errorf("trigger ENIIP throttle, max operating concurrent: %v", maxIPBacklog) -} - -func (f *eniIPFactory) ipExhaustErrorLocked(apiErr error) *types.IPInsufficientError { - // apiErr already be Insufficient error, should be return by eniFactory - if err, ok := apiErr.(*types.IPInsufficientError); ok { - return err - } - // ordered policy has vswitches ip count - if f.eniFactory.vswitchSelectionPolicy == types.VSwitchSelectionPolicyOrdered { - vswitches, err := f.eniFactory.GetVSwitches() - if err != nil { - eniIPLog.Warnf("cannot get vswitches %v on judging error type", err) - return nil - } - // node already has free eni slot, should retry in eniFactory - if len(f.enis) < f.nodeMaxENI { - return nil - } - var allEniIDs []string - var allEniVswitches = map[string]int{} - for _, eni := range f.enis { - if eni.ENI == nil { - // node still has pending eni maybe can allocate eniip - return nil - } - for _, vsw := range vswitches { - if eni.VSwitchID == vsw.id { - // the node still has some eni can allocate eniip - if eni.getIPCountLocked() < f.eniMaxIP && vsw.ipCount >= 0 && !strings.Contains(apiErr.Error(), vsw.id) { - return nil - } - allEniIDs = append(allEniIDs, eni.ID) - allEniVswitches[vsw.id] = 0 - } - } - } - return &types.IPInsufficientError{ - Err: apiErr, - Reason: fmt.Sprintf("ENI: %v on vswitches: %v \n all has no available ips to allocate", strings.Join(allEniIDs, ","), allEniVswitches), - } - } - return nil -} - -func (f *eniIPFactory) popResult() (ip *types.ENIIP, err error) { - result := <-f.ipResultChan - eniIPLog.Debugf("pop result from resultChan: %+v", result) - if result.ENIIP == nil || result.err != nil { - // There are two error cases: - // Error Case 1. The ENI-associated VSwitch has no available IP for Pod IP allocation. - // Error Case 2. The IP number allocated has reached ENI quota. - f.Lock() - defer f.Unlock() - if result.ENIIP != nil && result.ENI != nil { - for _, eni := range f.enis { - // eni is not ready - if eni == nil { - continue - } - if eni.MAC == result.ENI.MAC { - eni.pending-- - // if an error message with InvalidVSwitchIDIPNotEnough returned, then mark the ENI as IP allocation inhibited. - if strings.Contains(result.err.Error(), apiErr.InvalidVSwitchIDIPNotEnough) { - eni.ipAllocInhibitExpireAt = time.Now().Add(eniIPAllocInhibitTimeout) - eniIPLog.Infof("eni's associated vswitch %s has no available IP, set eni ipAllocInhibitExpireAt = %s", - eni.VSwitchID, eni.ipAllocInhibitExpireAt.Format(timeFormat)) - ipExhaustErr := f.ipExhaustErrorLocked(result.err) - if ipExhaustErr != nil { - result.err = ipExhaustErr - } - } - } - } - } - if strings.Contains(result.err.Error(), apiErr.ErrSecurityGroupInstanceLimitExceed) { - return nil, &types.IPInsufficientError{ - Err: err, - Reason: "security group of eni exceeded max ip limit", - } - } - if _, ok := result.err.(*types.IPInsufficientError); ok { - return nil, result.err - } - return nil, errors.Errorf("error allocate ip from eni: %v", result.err) - } - f.Lock() - defer f.Unlock() - for _, eni := range f.enis { - if eni.ENI != nil && eni.MAC == result.ENI.MAC { - eni.pending-- - eni.lock.Lock() - eni.ips = append(eni.ips, result) - eni.lock.Unlock() - metric.ENIIPFactoryIPCount.WithLabelValues(f.name, eni.MAC, fmt.Sprint(f.eniMaxIP)).Inc() - return result.ENIIP, nil - } - } - return nil, errors.Errorf("unexpected eni ip allocated: %v", result) -} - -func (f *eniIPFactory) Create(count int) ([]types.NetworkResource, error) { - ctx := &AllocCtx{} - var ( - ipResult []types.NetworkResource - err error - waiting int - ) - defer func() { - if len(ipResult) == 0 { - eniIPLog.Debugf("create result: %v, error: %v", ipResult, err) - } else { - for _, ip := range ipResult { - eniIPLog.Debugf("create result nil: %+v, error: %v", ip, err) - } - } - }() - - // find for available ENIs and submit for ip allocation - for ; waiting < count; waiting++ { - err = f.submit(ctx) - if err != nil { - break - } - } - // there are (count - waiting) ips still wait for allocation - // create a new ENI with initial ip count (count - waiting) as initENIIPCount - // initENIIPCount can't be greater than eniMaxIP and maxIPBacklog - initENIIPCount := count - waiting - if initENIIPCount > f.eniMaxIP { - initENIIPCount = f.eniMaxIP - } - if initENIIPCount > maxIPBacklog { - initENIIPCount = maxIPBacklog - } - if initENIIPCount > 0 { - eniIPLog.Debugf("create eni async, ip count: %+v", initENIIPCount) - _, err = f.createENIAsync(initENIIPCount) - if err == nil { - waiting += initENIIPCount - } else { - eniIPLog.Errorf("error create eni async: %+v", err) - } - } - - // no ip has been created - if waiting == 0 { - return ipResult, errors.Errorf("error submit ip create request: %v,%s", err, ctx.String()) - } - - var ip *types.ENIIP - for ; waiting > 0; waiting-- { // receive allocate result - ip, err = f.popResult() - if err != nil { - eniIPLog.Errorf("error allocate ip address: %+v", err) - } else { - ipResult = append(ipResult, ip) - } - } - if len(ipResult) == 0 { - if _, ok := err.(*types.IPInsufficientError); ok { - return nil, err - } - return ipResult, errors.Errorf("error allocate ip address: %v", err) - } - - return ipResult, nil -} - -func (f *eniIPFactory) Dispose(res types.NetworkResource) (err error) { - defer func() { - eniIPLog.Debugf("dispose result: %v, error: %v", res.GetResourceID(), err != nil) - }() - ip := res.(*types.ENIIP) - if ip.ENI == nil { - return nil - } - var ( - eni *ENI - eniip *ENIIP - ) - f.RLock() - for _, e := range f.enis { - if ip.ENI.ID == e.ID { - eni = e - e.lock.Lock() - for _, eip := range e.ips { - if eip.IPSet.String() == ip.IPSet.String() { - eniip = eip - } - } - e.lock.Unlock() - } - } - f.RUnlock() - if eni == nil || eniip == nil { - return fmt.Errorf("invalid resource to dispose") - } - - eni.lock.Lock() - if len(eni.ips) == 1 { - if f.enableTrunk && eni.Trunk { - eni.lock.Unlock() - return fmt.Errorf("trunk ENI %+v will not dispose", eni.ID) - } - - if eni.pending > 0 { - eni.lock.Unlock() - return fmt.Errorf("ENI have pending ips to be allocate") - } - // block ip allocate - eni.pending = f.eniMaxIP - eni.lock.Unlock() - - f.Lock() - for i, e := range f.enis { - if ip.ENI.ID == e.ID { - close(eni.done) - f.enis[len(f.enis)-1], f.enis[i] = f.enis[i], f.enis[len(f.enis)-1] - f.enis = f.enis[:len(f.enis)-1] - break - } - } - f.metricENICount.Dec() - f.Unlock() - - err = f.destroyENICompartment(ip.ENI) - if err != nil { - return fmt.Errorf("error destroy eni compartment before disposing: %v", err) - } - - f.eniOperChan <- struct{}{} - // only remain ENI main ip address, release the ENI interface - err = f.eniFactory.Dispose(ip.ENI) - <-f.eniOperChan - if err != nil { - return fmt.Errorf("error dispose ENI for eniip, %v", err) - } - <-f.maxENI - return nil - } - eni.lock.Unlock() - - // main ip of ENI, raise put_it_back error - if ip.ENI.PrimaryIP.IPv4.Equal(ip.IPSet.IPv4) { - // if in dual-stack and have no ipv6 address may need add one ip for it - return fmt.Errorf("ip to be release is primary ip of ENI") - } - - var v4, v6 []net.IP - if ip.IPSet.IPv4 != nil { - v4 = append(v4, ip.IPSet.IPv4) - } - if ip.IPSet.IPv6 != nil { - v6 = append(v6, ip.IPSet.IPv6) - } - err = f.eniFactory.ecs.UnAssignIPsForENI(context.Background(), ip.ENI.ID, ip.ENI.MAC, v4, v6) - if err != nil { - return fmt.Errorf("error unassign eniip, %v", err) - } - eni.lock.Lock() - for i, e := range eni.ips { - if e.IPSet.IPv4.Equal(eniip.IPSet.IPv4) { - eni.ips[len(eni.ips)-1], eni.ips[i] = eni.ips[i], eni.ips[len(eni.ips)-1] - eni.ips = eni.ips[:len(eni.ips)-1] - break - } - } - eni.lock.Unlock() - metric.ENIIPFactoryIPCount.WithLabelValues(f.name, eni.MAC, fmt.Sprint(f.eniMaxIP)).Dec() - return nil -} - -// Check resource in remote -func (f *eniIPFactory) Check(res types.NetworkResource) error { - eniIP, ok := res.(*types.ENIIP) - if !ok { - return fmt.Errorf("unsupported type %T", res) - } - - ipv4, ipv6, err := f.eniFactory.ecs.GetENIIPs(context.Background(), eniIP.ENI.MAC) - if err != nil { - return err - } - if utils.IsWindowsOS() { - // NB(thxCode): don't assign the primary IP of the assistant eni. - ipv4, ipv6 = dropPrimaryIP(eniIP.ENI, ipv4, ipv6) - } - - if eniIP.IPSet.IPv4 != nil { - if !terwayIP.IPsIntersect([]net.IP{eniIP.IPSet.IPv4}, ipv4) { - return apiErr.ErrNotFound - } - } - - if eniIP.IPSet.IPv6 != nil { - if !terwayIP.IPsIntersect([]net.IP{eniIP.IPSet.IPv6}, ipv6) { - return apiErr.ErrNotFound - } - } - - return nil -} - -// ListResource load all eni info from metadata -func (f *eniIPFactory) ListResource() (map[string]types.NetworkResource, error) { - f.RLock() - defer f.RUnlock() - - // list all resource status in our pool - mapping := make(map[string]types.NetworkResource) - var inUseENIIPs []*types.ENIIP - for _, e := range f.enis { - e.lock.Lock() - for _, eniIP := range e.ips { - inUseENIIPs = append(inUseENIIPs, eniIP.ENIIP) - } - - e.lock.Unlock() - } - ctx := context.Background() - macs, err := f.eniFactory.ecs.GetSecondaryENIMACs(ctx) - if err != nil { - return nil, err - } - - for _, mac := range macs { - // get secondary ips from one mac - ipv4s, ipv6s, err := f.eniFactory.ecs.GetENIIPs(ctx, mac) - if err != nil { - if errors.Is(err, apiErr.ErrNotFound) { - continue - } - return nil, err - } - ipv4Set := terwayIP.ToIPMap(ipv4s) - ipv6Set := terwayIP.ToIPMap(ipv6s) - - for _, eniIP := range inUseENIIPs { - if eniIP.ENI.MAC != mac { - continue - } - - var v4, v6 net.IP - if eniIP.IPSet.IPv4 != nil { - _, ok := ipv4Set[eniIP.IPSet.IPv4.String()] - if ok { - v4 = eniIP.IPSet.IPv4 - } - } - if eniIP.IPSet.IPv6 != nil { - _, ok := ipv6Set[eniIP.IPSet.IPv6.String()] - if ok { - v6 = eniIP.IPSet.IPv6 - } - } - - tmp := types.ENIIP{ - ENI: &types.ENI{ - MAC: mac, - }, - IPSet: types.IPSet{ - IPv4: v4, - IPv6: v6, - }, - } - - mapping[tmp.GetResourceID()] = &tmp - } - } - - return mapping, nil -} - -func (f *eniIPFactory) Reconcile() { - // check security group - err := f.eniFactory.ecs.CheckEniSecurityGroup(context.Background(), f.eniFactory.securityGroupIDs) - if err != nil { - _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, "ResourceInvalid", fmt.Sprintf("eni has misconfiged security group. %s", err.Error())) - } -} - -func (f *eniIPFactory) initialENI(eni *ENI, ipCount int) { - if utils.IsWindowsOS() { - // NB(thxCode): create eni with one more IP in windows at initialization. - ipCount++ - } - rawEni, err := f.eniFactory.CreateWithIPCount(ipCount, false) - var ipv4s []net.IP - var ipv6s []net.IP - // eni operate finished - <-f.eniOperChan - if err != nil || len(rawEni) != 1 { - // create eni failed, put quota back - <-f.maxENI - } else { - var ok bool - eni.ENI, ok = rawEni[0].(*types.ENI) - if !ok { - err = fmt.Errorf("error get type ENI from factory, got: %+v, rollback it", rawEni) - eniIPLog.Error(err) - errDispose := f.eniFactory.Dispose(rawEni[0]) - if errDispose != nil { - eniIPLog.Errorf("rollback %+v failed", rawEni) - } - <-f.maxENI - } else { - ipv4s, ipv6s, err = f.eniFactory.ecs.GetENIIPs(context.Background(), eni.MAC) - if err != nil { - eniIPLog.Errorf("error get eni secondary address: %+v, rollback it", err) - errDispose := f.eniFactory.Dispose(rawEni[0]) - if errDispose != nil { - eniIPLog.Errorf("rollback %+v failed", rawEni) - } - <-f.maxENI - } - if f.ipFamily.IPv4 && f.ipFamily.IPv6 { - if len(ipv4s) != len(ipv6s) { - eniIPLog.Errorf("error get eni secondary address: ipv4 ipv6 length is not equal, rollback it") - errDispose := f.eniFactory.Dispose(rawEni[0]) - if errDispose != nil { - eniIPLog.Errorf("rollback %+v failed", rawEni) - } - <-f.maxENI - } - } - err = f.setupENICompartment(eni.ENI) - if err != nil { - eniIPLog.Errorf("error setup eni compartment after initialization: %v", err) - errDispose := f.eniFactory.Dispose(rawEni[0]) - if errDispose != nil { - eniIPLog.Errorf("rollback %+v failed", rawEni) - } - <-f.maxENI - } - } - } - - eniIPLog.Debugf("eni initial finished: %+v, err: %+v", eni, err) - - if err != nil { - eni.lock.Lock() - //failed all pending on this initial eni - for i := 0; i < eni.pending; i++ { - eniip := &ENIIP{ - ENIIP: &types.ENIIP{ - ENI: nil, - }, - err: fmt.Errorf("error initial ENI: %w", err), - } - if _, ok := err.(*types.IPInsufficientError); ok { - eniip.err = err - } - f.ipResultChan <- eniip - } - // disable eni for submit - eni.pending = f.eniMaxIP - eni.lock.Unlock() - - // remove from eni list - f.Lock() - for i, e := range f.enis { - if e == eni { - f.enis[len(f.enis)-1], f.enis[i] = f.enis[i], f.enis[len(f.enis)-1] - f.enis = f.enis[:len(f.enis)-1] - break - } - } - f.metricENICount.Dec() - f.Unlock() - - return - } - - eni.lock.Lock() - if utils.IsWindowsOS() { - // NB(thxCode): don't assign the primary IP of the assistant eni. - ipv4s, ipv6s = dropPrimaryIP(eni.ENI, ipv4s, ipv6s) - } - eniIPLog.Infof("allocate status on async eni: %+v, pending: %v, ips: %v, backlog: %v", - eni, eni.pending, ipv4s, len(eni.ipBacklog)) - - for _, ipSet := range types.MergeIPs(ipv4s, ipv6s) { - eniIP := &types.ENIIP{ - ENI: eni.ENI, - IPSet: ipSet, - } - - f.ipResultChan <- &ENIIP{ - ENIIP: eniIP, - err: nil, - } - } - - eni.lock.Unlock() - go eni.allocateWorker(f.ipResultChan) -} - -func (f *eniIPFactory) createENIAsync(initIPs int) (*ENI, error) { - eni := &ENI{ - ENI: nil, - ips: make([]*ENIIP, 0), - pending: initIPs, - ipBacklog: make(chan struct{}, maxIPBacklog), - ecs: f.eniFactory.ecs, - done: make(chan struct{}, 1), - } - select { - case f.maxENI <- struct{}{}: - select { - case f.eniOperChan <- struct{}{}: - default: - <-f.maxENI - return nil, fmt.Errorf("trigger ENI throttle, max operating concurrent: %v", maxEniOperating) - } - go f.initialENI(eni, eni.pending) - default: - return nil, &types.IPInsufficientError{ - Err: fmt.Errorf("max ENI exceeded"), - Reason: "all ENIs bind on host can not allocate ip address, and node has no slot to allocate ENI", - } - } - f.Lock() - f.enis = append(f.enis, eni) - f.Unlock() - // metric - f.metricENICount.Inc() - return eni, nil -} - -func (f *eniIPFactory) Config() []tracing.MapKeyValueEntry { - config := []tracing.MapKeyValueEntry{ - {Key: tracingKeyName, Value: f.name}, - {Key: tracingKeyENIMaxIP, Value: fmt.Sprint(f.eniMaxIP)}, - } - - return config -} - -func (f *eniIPFactory) Trace() []tracing.MapKeyValueEntry { - var trace []tracing.MapKeyValueEntry - - secIPCount := 0 - - trace = append(trace, tracing.MapKeyValueEntry{Key: tracingKeyENICount, Value: fmt.Sprint(len(f.enis))}) - trace = append(trace, tracing.MapKeyValueEntry{Key: tracingKeySecondaryIPCount, Value: ""}) // placeholder - - for _, v := range f.enis { - secIPCount += len(v.ips) - trace = append(trace, tracing.MapKeyValueEntry{ - Key: fmt.Sprintf("eni/%s/pending", v.MAC), - Value: fmt.Sprint(v.pending), - }) - - var secIPs []string - for _, v := range v.ips { - secIPs = append(secIPs, v.IPSet.String()) - } - - trace = append(trace, tracing.MapKeyValueEntry{ - Key: fmt.Sprintf("eni/%s/secondary_ips", v.MAC), - Value: strings.Join(secIPs, " "), - }) - - trace = append(trace, tracing.MapKeyValueEntry{ - Key: fmt.Sprintf("eni/%s/ip_alloc_inhibit_expire_at", v.MAC), - Value: v.ipAllocInhibitExpireAt.Format(timeFormat), - }) - } - - trace[1].Value = fmt.Sprint(secIPCount) - - return trace -} - -func (f *eniIPFactory) Execute(cmd string, _ []string, message chan<- string) { - switch cmd { - case commandAudit: // check account - f.checkAccount(message) - case commandMapping: - mapping, err := f.ListResource() - message <- fmt.Sprintf("mapping: %v, err: %s\n", mapping, err) - default: - message <- "can't recognize command\n" - } - - close(message) -} - -func (f *eniIPFactory) checkAccount(message chan<- string) { - // get ENIs via Aliyun API - message <- "fetching attached ENIs from aliyun\n" - ctx := context.Background() - enis, err := f.eniFactory.ecs.GetAttachedENIs(ctx, false, f.trunkOnEni) - if err != nil { - message <- fmt.Sprintf("error while fetching from remote: %s\n", err.Error()) - return - } - - message <- fmt.Sprintf("%d enis fetched\n", len(enis)) - - f.Lock() - // check local enis count - if len(f.enis) != len(enis) { - message <- fmt.Sprintf("remote eni count not equal to local: remote %d, local: %d\n", len(enis), len(f.enis)) - } - - // diff - diffMap := make(map[string]*ENI) - - for _, v := range f.enis { - diffMap[v.ID] = v - } - f.Unlock() - - // range for remote enis - for _, v := range enis { - message <- fmt.Sprintf("checking eni %s(%s)\n", v.ID, v.MAC) - - eni, ok := diffMap[v.ID] - if !ok { - message <- fmt.Sprintf("remote eni %s(%s) not found on local machine\n", v.ID, v.MAC) - continue - } - - // diff secondary ips - remoteIPs, _, err := f.eniFactory.ecs.GetENIIPs(ctx, v.MAC) - if err != nil { - message <- fmt.Sprintf("error while fetching ips: %s", err.Error()) - return - } - if utils.IsWindowsOS() { - // NB(thxCode): don't assign the primary IP of the assistant eni. - remoteIPs, _ = dropPrimaryIP(v, remoteIPs, nil) - } - - diffIPMap := make(map[string]struct{}) - - for _, ip := range eni.ips { - diffIPMap[ip.IPSet.IPv4.String()] = struct{}{} - } - - // range for remote ips - for _, ip := range remoteIPs { - _, ok := diffIPMap[ip.String()] - if !ok { - message <- fmt.Sprintf("ip %s attached to eni %s(%s) not found on local machine\n", ip.String(), v.ID, v.MAC) - } - } - } - - message <- "done.\n" -} - -type eniIPResourceManager struct { - trunkENI *types.ENI - pool pool.ObjectPool -} - -func newENIIPResourceManager(poolConfig *types.PoolConfig, ecs ipam.API, k8s Kubernetes, allocatedResources map[string]resourceManagerInitItem, ipFamily *types.IPFamily) (ResourceManager, error) { - eniFactory, err := newENIFactory(poolConfig, ecs) - if err != nil { - return nil, fmt.Errorf("error get ENI factory for eniip factory, %w", err) - } - - factory := &eniIPFactory{ - name: factoryNameENIIP, - eniFactory: eniFactory, - trunkOnEni: poolConfig.TrunkENIID, - enableTrunk: poolConfig.TrunkENIID != "", - enis: []*ENI{}, - eniOperChan: make(chan struct{}, maxEniOperating), - ipResultChan: make(chan *ENIIP, maxIPBacklog), - ipFamily: ipFamily, - maxENI: make(chan struct{}, poolConfig.MaxENI), - eniMaxIP: poolConfig.MaxIPPerENI, - nodeMaxENI: poolConfig.MaxENI, - } - - // eniip factory metrics - factory.metricENICount = metric.ENIIPFactoryENICount.WithLabelValues(factory.name, fmt.Sprint(poolConfig.MaxENI)) - var trunkENI *types.ENI - poolCfg := pool.Config{ - Name: poolNameENIIP, - Type: typeNameENIIP, - MaxIdle: poolConfig.MaxPoolSize, - MinIdle: poolConfig.MinPoolSize, - Factory: factory, - Capacity: poolConfig.Capacity, - IPConditionHandler: k8s.PatchNodeIPResCondition, - Initializer: func(holder pool.ResourceHolder) error { - ctx := context.Background() - // not use main ENI for ENI multiple ip allocate - enis, err := ecs.GetAttachedENIs(ctx, false, factory.trunkOnEni) - if err != nil { - return fmt.Errorf("error get attach ENI on pool init, %w", err) - } - - if factory.trunkOnEni != "" { - found := false - for _, eni := range enis { - if eni.Trunk && factory.trunkOnEni == eni.ID { - found = true - trunkENI = eni - break - } - } - if !found { - return fmt.Errorf("trunk eni %s not found", factory.trunkOnEni) - } - } - - for _, eni := range enis { - ipv4s, ipv6s, err := ecs.GetENIIPs(ctx, eni.MAC) - if err != nil { - return fmt.Errorf("error get ENI's ip on pool init, %w", err) - } - err = factory.setupENICompartment(eni) - if err != nil { - // NB(thxCode): an unbinding eni stuck and then block starting, - // we just ignore this kind of error. - if strings.Contains(err.Error(), "no interface with given MAC") { - continue - } - return errors.Wrap(err, "error setup eni compartment") - } - if utils.IsWindowsOS() { - // NB(thxCode): don't assign the primary IP of one assistant eni. - ipv4s, ipv6s = dropPrimaryIP(eni, ipv4s, ipv6s) - } - poolENI := &ENI{ - ENI: eni, - ips: []*ENIIP{}, - ecs: ecs, - ipBacklog: make(chan struct{}, maxIPBacklog), - done: make(chan struct{}, 1), - } - factory.enis = append(factory.enis, poolENI) - factory.metricENICount.Inc() - if ipFamily.IPv4 && !ipFamily.IPv6 { - for _, ip := range ipv4s { - eniIP := &types.ENIIP{ - ENI: eni, - IPSet: types.IPSet{IPv4: ip}, - } - res, ok := allocatedResources[eniIP.GetResourceID()] - - poolENI.ips = append(poolENI.ips, &ENIIP{ - ENIIP: eniIP, - }) - metric.ENIIPFactoryIPCount.WithLabelValues(factory.name, poolENI.MAC, fmt.Sprint(poolConfig.MaxENI)).Inc() - - if !ok { - holder.AddIdle(eniIP) - } else { - holder.AddInuse(eniIP, podInfoKey(res.podInfo.Namespace, res.podInfo.Name)) - } - } - } else { - v4Map := terwayIP.ToIPMap(ipv4s) - v6Map := terwayIP.ToIPMap(ipv6s) - - // put all local res in - for id, res := range allocatedResources { - if res.item.ENIMAC != eni.MAC { - continue - } - ipSet := types.IPSet{} - eniIP := &types.ENIIP{ - ENI: eni, - IPSet: *ipSet.SetIP(res.item.IPv4).SetIP(res.item.IPv6), - } - - poolENI.ips = append(poolENI.ips, &ENIIP{ - ENIIP: eniIP, - }) - metric.ENIIPFactoryIPCount.WithLabelValues(factory.name, poolENI.MAC, fmt.Sprint(poolConfig.MaxENI)).Inc() - - holder.AddInuse(eniIP, podInfoKey(res.podInfo.Namespace, res.podInfo.Name)) - - if ipSet.IPv4 != nil { - delete(v4Map, ipSet.IPv4.String()) - } - if ipSet.IPv6 != nil { - delete(v6Map, ipSet.IPv6.String()) - } - delete(allocatedResources, id) - } - var v4List, v6List []net.IP - for _, v4 := range v4Map { - v4List = append(v4List, v4) - } - for _, v6 := range v6Map { - v6List = append(v6List, v6) - } - for _, unUsed := range types.MergeIPs(v4List, v6List) { - eniIP := &types.ENIIP{ - ENI: eni, - IPSet: unUsed, - } - poolENI.ips = append(poolENI.ips, &ENIIP{ - ENIIP: eniIP, - }) - metric.ENIIPFactoryIPCount.WithLabelValues(factory.name, poolENI.MAC, fmt.Sprint(poolConfig.MaxENI)).Inc() - - if ipFamily.IPv4 && ipFamily.IPv6 && (unUsed.IPv6 == nil || unUsed.IPv4 == nil) { - holder.AddInvalid(eniIP) - } else { - holder.AddIdle(eniIP) - } - } - } - - eniIPLog.Debugf("init factory's exist ENI: %+v", poolENI) - select { - case factory.maxENI <- struct{}{}: - default: - eniIPLog.Warnf("exist enis already over eni limits, maxENI config will not be available") - } - go poolENI.allocateWorker(factory.ipResultChan) - } - return nil - }, - } - p, err := pool.NewSimpleObjectPool(poolCfg) - if err != nil { - return nil, err - } - mgr := &eniIPResourceManager{ - trunkENI: trunkENI, - pool: p, - } - - _ = tracing.Register(tracing.ResourceTypeFactory, factory.name, factory) - return mgr, nil -} - -func (m *eniIPResourceManager) Allocate(ctx *networkContext, prefer string) (types.NetworkResource, error) { - return m.pool.Acquire(ctx, prefer, podInfoKey(ctx.pod.Namespace, ctx.pod.Name)) -} - -func (m *eniIPResourceManager) Release(context *networkContext, resItem types.ResourceItem) error { - if context != nil && context.pod != nil { - return m.pool.ReleaseWithReservation(resItem.ID, context.pod.IPStickTime) - } - return m.pool.Release(resItem.ID) -} - -func (m *eniIPResourceManager) GarbageCollection(inUseResSet map[string]types.ResourceItem, expireResSet map[string]types.ResourceItem) error { - for expireRes, expireItem := range expireResSet { - if _, err := m.pool.Stat(expireRes); err == nil { - err = m.Release(nil, expireItem) - if err != nil { - return err - } - } - } - return nil -} - -func (m *eniIPResourceManager) Stat(context *networkContext, resID string) (types.NetworkResource, error) { - return m.pool.Stat(resID) -} - -func (m *eniIPResourceManager) GetResourceMapping() (tracing.ResourcePoolStats, error) { - return m.pool.GetResourceMapping() -} - -func dropPrimaryIP(eni *types.ENI, ipv4s, ipv6s []net.IP) ([]net.IP, []net.IP) { - if eni == nil { - return ipv4s, ipv6s - } - - var ipv4sLen = len(ipv4s) - if ipv4Primary := eni.PrimaryIP.IPv4; len(ipv4Primary) != 0 && ipv4sLen != 0 { - var ipv4sNew []net.IP - for idx, ipv4 := range ipv4s { - if ipv4.Equal(ipv4Primary) { - switch idx { - case 0: - ipv4sNew = ipv4s[1:] - case ipv4sLen - 1: - ipv4sNew = ipv4s[:ipv4sLen-1] - default: - ipv4sNew = append(ipv4s[0:idx], ipv4s[idx+1:]...) - } - break - } - } - ipv4s = ipv4sNew - } - - var ipv6sLen = len(ipv6s) - if ipv6Primary := eni.PrimaryIP.IPv6; len(ipv6Primary) != 0 && ipv6sLen != 0 { - var ipv6sNew []net.IP - for idx, ipv6 := range ipv6s { - if ipv6.Equal(ipv6Primary) { - switch idx { - case 0: - ipv6sNew = ipv6s[1:] - case ipv6sLen - 1: - ipv6sNew = ipv6s[:ipv6sLen-1] - default: - ipv6sNew = append(ipv6s[0:idx], ipv6s[idx+1:]...) - } - break - } - } - ipv6s = ipv6sNew - } - - return ipv4s, ipv6s -} diff --git a/daemon/eni-multi-ip_unwindows.go b/daemon/eni-multi-ip_unwindows.go deleted file mode 100644 index bde9a233..00000000 --- a/daemon/eni-multi-ip_unwindows.go +++ /dev/null @@ -1,14 +0,0 @@ -//go:build !windows -// +build !windows - -package daemon - -import "github.com/AliyunContainerService/terway/types" - -func (f *eniIPFactory) setupENICompartment(eni *types.ENI) error { - return nil -} - -func (f *eniIPFactory) destroyENICompartment(eni *types.ENI) error { - return nil -} diff --git a/daemon/eni-multi-ip_windows.go b/daemon/eni-multi-ip_windows.go deleted file mode 100644 index 9b3902aa..00000000 --- a/daemon/eni-multi-ip_windows.go +++ /dev/null @@ -1,80 +0,0 @@ -package daemon - -import ( - "context" - "strings" - "time" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/wait" - - "github.com/AliyunContainerService/terway/pkg/windows/apis" - "github.com/AliyunContainerService/terway/pkg/windows/iface" - "github.com/AliyunContainerService/terway/types" -) - -func (f *eniIPFactory) setupENICompartment(eni *types.ENI) (err error) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - var nwIface *iface.Interface - _ = wait.PollUntil(2*time.Second, func() (bool, error) { - nwIface, err = iface.GetInterfaceByMAC(eni.MAC, true) - if err != nil { - return false, nil - } - return true, nil - }, ctx.Done()) - if err != nil { - return errors.Wrapf(err, "error getting interface by mac %s", eni.MAC) - } - - dnsConfig, _ := iface.GetDNSConfigurationByMAC(eni.MAC, iface.IPv4) - var nwSubnet = eni.VSwitchCIDR.IPv4 - var nw = &apis.Network{ - Name: nwIface.Name, - AdapterName: nwIface.Alias, - AdapterMAC: nwIface.MacAddress, - Subnet: *nwSubnet, - Gateway: eni.GatewayIP.IPv4, - DNS: dnsConfig, - } - err = apis.AddTunnelHNSNetwork(ctx, nw) - if err != nil { - return errors.Wrapf(err, "error adding network: %s", nw.Format(apis.HNS)) - } - - return -} - -func (f *eniIPFactory) destroyENICompartment(eni *types.ENI) (err error) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - var nwIface *iface.Interface - _ = wait.PollImmediateUntil(2*time.Second, func() (bool, error) { - nwIface, err = iface.GetInterfaceByMAC(eni.MAC, true) - if err != nil && !strings.Contains(err.Error(), "no interface with given MAC") { - err = nil - return false, nil - } - return true, nil - }, ctx.Done()) - if err != nil { - return errors.Wrapf(err, "error getting interface by mac %s", eni.MAC) - } - // go away if not found - if nwIface == nil { - return - } - - var nw = &apis.Network{ - Name: nwIface.Name, - } - err = apis.DeleteHNSNetwork(ctx, nw) - if err != nil { - return errors.Wrap(err, "error deleting network") - } - - return -} diff --git a/daemon/eni.go b/daemon/eni.go deleted file mode 100644 index f816aa10..00000000 --- a/daemon/eni.go +++ /dev/null @@ -1,401 +0,0 @@ -package daemon - -import ( - "context" - "fmt" - "math/rand" - "sort" - "strings" - "sync" - "time" - - "github.com/samber/lo" - - apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" - - "github.com/AliyunContainerService/terway/pkg/ipam" - "github.com/AliyunContainerService/terway/pkg/logger" - "github.com/AliyunContainerService/terway/pkg/pool" - "github.com/AliyunContainerService/terway/pkg/tracing" - "github.com/AliyunContainerService/terway/types" - - "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" -) - -var eniLog = logger.DefaultLogger - -const ( - // vSwitchIPCntTimeout is the duration for the vswitchCnt content's effectiveness - vSwitchIPCntTimeout = 10 * time.Minute - - typeNameENI = "eni" - poolNameENI = "eni" - factoryNameENI = "eni" - - tracingKeyVSwitches = "vswitches" - tracingKeyVSwitchSelectionPolicy = "vswitch_selection_policy" - tracingKeyCacheExpireAt = "cache_expire_at" -) - -type eniResourceManager struct { - pool pool.ObjectPool - ecs ipam.API - trunkENI *types.ENI -} - -func newENIResourceManager(poolConfig *types.PoolConfig, ecs ipam.API, allocatedResources map[string]resourceManagerInitItem, ipFamily *types.IPFamily, k8s Kubernetes, ipamType types.IPAMType) (ResourceManager, error) { - eniLog.Debugf("new ENI Resource Manager, pool config: %+v, allocated resources: %+v", poolConfig, allocatedResources) - factory, err := newENIFactory(poolConfig, ecs) - if err != nil { - return nil, errors.Wrapf(err, "error create ENI factory") - } - - _ = tracing.Register(tracing.ResourceTypeFactory, factoryNameENI, factory) - - var trunkENI *types.ENI - - poolCfg := pool.Config{ - Name: poolNameENI, - Type: typeNameENI, - MaxIdle: poolConfig.MaxPoolSize, - MinIdle: poolConfig.MinPoolSize, - Capacity: poolConfig.Capacity, - Factory: factory, - IPConditionHandler: k8s.PatchNodeIPResCondition, - Initializer: func(holder pool.ResourceHolder) error { - if ipamType == types.IPAMTypeCRD { - return nil - } - ctx := context.Background() - enis, err := ecs.GetAttachedENIs(ctx, false, factory.trunkOnEni) - if err != nil { - return fmt.Errorf("error get attach ENI on pool init, %w", err) - } - - if factory.trunkOnEni != "" { - found := false - for _, eni := range enis { - if eni.Trunk && factory.trunkOnEni == eni.ID { - found = true - trunkENI = eni - break - } - } - if !found { - return fmt.Errorf("trunk eni %s not found", factory.trunkOnEni) - } - } - - for _, e := range enis { - if ipFamily.IPv6 { - _, ipv6, err := ecs.GetENIIPs(ctx, e.MAC) - if err != nil || len(ipv6) == 0 { - return errors.Wrapf(err, "error get eni ip") - } - e.PrimaryIP.IPv6 = ipv6[0] - } - if item, ok := allocatedResources[e.GetResourceID()]; ok { - holder.AddInuse(e, podInfoKey(item.podInfo.Namespace, item.podInfo.Name)) - } else { - holder.AddIdle(e) - } - } - return nil - }, - } - - p, err := pool.NewSimpleObjectPool(poolCfg) - if err != nil { - return nil, err - } - mgr := &eniResourceManager{ - pool: p, - ecs: ecs, - trunkENI: trunkENI, - } - - return mgr, nil -} - -func (m *eniResourceManager) Allocate(ctx *networkContext, prefer string) (types.NetworkResource, error) { - return m.pool.Acquire(ctx, prefer, podInfoKey(ctx.pod.Namespace, ctx.pod.Name)) -} - -func (m *eniResourceManager) Release(context *networkContext, resItem types.ResourceItem) error { - if context != nil && context.pod != nil { - return m.pool.ReleaseWithReservation(resItem.ID, context.pod.IPStickTime) - } - return m.pool.Release(resItem.ID) -} - -func (m *eniResourceManager) GarbageCollection(inUseResSet map[string]types.ResourceItem, expireResSet map[string]types.ResourceItem) error { - for expireRes, expireItem := range expireResSet { - if _, err := m.pool.Stat(expireRes); err == nil { - err = m.Release(nil, expireItem) - if err != nil { - return err - } - } - } - return nil -} - -func (m *eniResourceManager) Stat(context *networkContext, resID string) (types.NetworkResource, error) { - return m.pool.Stat(resID) -} - -func (m *eniResourceManager) GetResourceMapping() (tracing.ResourcePoolStats, error) { - return m.pool.GetResourceMapping() -} - -type vswitch struct { - id string - ipCount int -} - -func (v *vswitch) String() string { - return fmt.Sprintf("%s(%d)", v.id, v.ipCount) -} - -type eniFactory struct { - name string - enableTrunk bool - trunkOnEni string - vSwitchOptions []string - eniTags map[string]string - securityGroupIDs []string - instanceID string - ecs ipam.API - vswitchCnt []vswitch - tsExpireAt time.Time - vswitchSelectionPolicy string - disableSecurityGroupCheck bool - sync.RWMutex -} - -func newENIFactory(poolConfig *types.PoolConfig, ecs ipam.API) (*eniFactory, error) { - if len(poolConfig.SecurityGroupIDs) == 0 { - securityGroups, err := ecs.GetAttachedSecurityGroups(context.Background(), poolConfig.InstanceID) - if err != nil { - return nil, errors.Wrapf(err, "error get security group on factory init") - } - poolConfig.SecurityGroupIDs = securityGroups - } - return &eniFactory{ - name: factoryNameENI, - vSwitchOptions: poolConfig.VSwitchOptions, - eniTags: poolConfig.ENITags, - securityGroupIDs: poolConfig.SecurityGroupIDs, - trunkOnEni: poolConfig.TrunkENIID, - enableTrunk: poolConfig.TrunkENIID != "", - instanceID: poolConfig.InstanceID, - ecs: ecs, - vswitchCnt: make([]vswitch, 0), - vswitchSelectionPolicy: poolConfig.VSwitchSelectionPolicy, - disableSecurityGroupCheck: poolConfig.DisableSecurityGroupCheck, - }, nil -} - -func (f *eniFactory) GetVSwitches() ([]vswitch, error) { - - var vSwitches []vswitch - - vswCnt := len(f.vSwitchOptions) - // If there is ONLY ONE vswitch, then there is no need for ordering per switches' available IP counts, - // return the slice with only this vswitch. - if vswCnt == 1 { - return []vswitch{{ - id: f.vSwitchOptions[0], - ipCount: 0, - }}, nil - } - - if f.vswitchSelectionPolicy == types.VSwitchSelectionPolicyRandom { - vSwitches = lo.Map(f.vSwitchOptions, func(item string, index int) vswitch { - return vswitch{ - id: item, - ipCount: 0, - } - }) - rand.Seed(time.Now().UnixNano()) - rand.Shuffle(vswCnt, func(i, j int) { vSwitches[i], vSwitches[j] = vSwitches[j], vSwitches[i] }) - return vSwitches, nil - } - - if f.vswitchSelectionPolicy == types.VSwitchSelectionPolicyOrdered { - // If VSwitchSelectionPolicy is ordered, then call f.ecs.DescribeVSwitch API to get the switch's available IP count - // PS: this is only feasible for systems with RAM policy for VPC API permission. - // Use f.vswitchCnt to track IP count + vswitch ID - var ( - start = time.Now() - err error - ) - // If f.vswitchCnt is empty, then fill in the map with switch + switch's available IP count. - f.Lock() - if (len(f.vswitchCnt) == 0 && f.tsExpireAt.IsZero()) || start.After(f.tsExpireAt) { - f.vswitchCnt = make([]vswitch, 0) - // Loop vsw slice to get each vsw's available IP count. - for _, vswID := range f.vSwitchOptions { - var vsw *vpc.VSwitch - vsw, err = f.ecs.DescribeVSwitchByID(context.Background(), vswID) - if err != nil { - f.vswitchCnt = append(f.vswitchCnt, vswitch{ - id: vswID, - ipCount: 0, - }) - } else { - f.vswitchCnt = append(f.vswitchCnt, vswitch{ - id: vswID, - ipCount: int(vsw.AvailableIpAddressCount), - }) - } - } - if err == nil { - // don't cache result when error - f.tsExpireAt = time.Now().Add(vSwitchIPCntTimeout) - } - } - - if len(f.vswitchCnt) > 0 { - sort.Slice(f.vswitchCnt, func(i, j int) bool { - return f.vswitchCnt[i].ipCount > f.vswitchCnt[j].ipCount - }) - vSwitches = f.vswitchCnt - } else { - vSwitches = lo.Map(f.vSwitchOptions, func(item string, index int) vswitch { - return vswitch{ - id: item, - ipCount: 0, - } - }) - } - f.Unlock() - } - - return vSwitches, nil -} - -func (f *eniFactory) Create(int) ([]types.NetworkResource, error) { - return f.CreateWithIPCount(1, false) -} - -func (f *eniFactory) CreateWithIPCount(count int, trunk bool) ([]types.NetworkResource, error) { - vSwitches, _ := f.GetVSwitches() - eniLog.Infof("adjusted vswitch slice: %+v", vSwitches) - - tags := map[string]string{ - types.NetworkInterfaceTagCreatorKey: types.NetworkInterfaceTagCreatorValue, - } - for k, v := range f.eniTags { - tags[k] = v - } - eni, err := f.ecs.AllocateENI(context.Background(), vSwitches[0].id, f.securityGroupIDs, f.instanceID, trunk, count, tags) - if err != nil { - if strings.Contains(err.Error(), apiErr.InvalidVSwitchIDIPNotEnough) { - reportIPExhaustive := false - if len(vSwitches) == 1 { - reportIPExhaustive = true - } - if f.vswitchSelectionPolicy == types.VSwitchSelectionPolicyOrdered { - reportIPExhaustive = true - } - if reportIPExhaustive { - return nil, &types.IPInsufficientError{ - Err: err, - Reason: fmt.Sprintf("all configure vswitches: %v has no available ip address", vSwitches)} - } - } else if strings.Contains(err.Error(), apiErr.ErrEniPerInstanceLimitExceeded) { - return nil, &types.IPInsufficientError{ - Err: err, - Reason: fmt.Sprintf("instance %v exceeded max eni limit", f.instanceID)} - } else if strings.Contains(err.Error(), apiErr.ErrSecurityGroupInstanceLimitExceed) { - return nil, &types.IPInsufficientError{ - Err: err, - Reason: fmt.Sprintf("security group %v exceeded max ip limit", f.securityGroupIDs)} - } - return nil, err - } - return []types.NetworkResource{eni}, nil -} - -func (f *eniFactory) Dispose(resource types.NetworkResource) error { - eni := resource.(*types.ENI) - if f.enableTrunk && eni.Trunk { - return fmt.Errorf("trunk ENI %+v will not dispose", eni.ID) - } - return f.ecs.FreeENI(context.Background(), eni.ID, f.instanceID) -} - -func (f *eniFactory) Config() []tracing.MapKeyValueEntry { - config := []tracing.MapKeyValueEntry{ - {Key: tracingKeyName, Value: f.name}, - {Key: tracingKeyVSwitches, Value: strings.Join(f.vSwitchOptions, " ")}, - {Key: tracingKeyVSwitchSelectionPolicy, Value: f.vswitchSelectionPolicy}, - } - - return config -} - -func (f *eniFactory) Trace() []tracing.MapKeyValueEntry { - trace := []tracing.MapKeyValueEntry{ - {Key: tracingKeyCacheExpireAt, Value: fmt.Sprint(f.tsExpireAt)}, - } - - for _, cnt := range f.vswitchCnt { - key := fmt.Sprintf("vswitch/%s/ip_count", cnt.id) - trace = append(trace, tracing.MapKeyValueEntry{ - Key: key, - Value: fmt.Sprint(cnt), - }) - } - - return trace -} - -func (f *eniFactory) Execute(cmd string, _ []string, message chan<- string) { - switch cmd { - case commandMapping: - mapping, err := f.ListResource() - message <- fmt.Sprintf("mapping: %v, err: %s\n", mapping, err) - default: - message <- "can't recognize command\n" - } - - close(message) -} - -func (f *eniFactory) Check(res types.NetworkResource) error { - eni, ok := res.(*types.ENI) - if !ok { - return fmt.Errorf("unsupported type %T", res) - } - _, err := f.ecs.GetENIByMac(context.Background(), eni.MAC) - return err -} - -func (f *eniFactory) ListResource() (map[string]types.NetworkResource, error) { - enis, err := f.ecs.GetAttachedENIs(context.Background(), false, f.trunkOnEni) - if err != nil { - return nil, err - } - - mapping := make(map[string]types.NetworkResource, len(enis)) - for i := 0; i < len(enis); i++ { - mapping[enis[i].GetResourceID()] = enis[i] - } - - return mapping, nil -} - -func (f *eniFactory) Reconcile() { - // check security group - if f.disableSecurityGroupCheck { - return - } - err := f.ecs.CheckEniSecurityGroup(context.Background(), f.securityGroupIDs) - if err != nil { - _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, "ResourceInvalid", fmt.Sprintf("eni has misconfiged security group. %s", err.Error())) - } -} diff --git a/daemon/resource_manager.go b/daemon/resource_manager.go index dc4340eb..d58810b0 100644 --- a/daemon/resource_manager.go +++ b/daemon/resource_manager.go @@ -1,9 +1,7 @@ package daemon import ( - "github.com/AliyunContainerService/terway/pkg/tracing" - - "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/daemon" ) const ( @@ -12,16 +10,6 @@ const ( ) type resourceManagerInitItem struct { - item types.ResourceItem - podInfo *types.PodInfo -} - -// ResourceManager Allocate/Release/Pool/Stick/GC pod resource -// managed pod and resource relationship -type ResourceManager interface { - Allocate(context *networkContext, prefer string) (types.NetworkResource, error) - Release(context *networkContext, resItem types.ResourceItem) error - GarbageCollection(inUseResSet map[string]types.ResourceItem, expireResSet map[string]types.ResourceItem) error - Stat(context *networkContext, resID string) (types.NetworkResource, error) - tracing.ResourceMappingHandler + item daemon.ResourceItem + podInfo *daemon.PodInfo } diff --git a/daemon/server.go b/daemon/server.go index cc7c0010..bd8f3020 100644 --- a/daemon/server.go +++ b/daemon/server.go @@ -12,6 +12,10 @@ import ( "runtime" "strings" "syscall" + "time" + + "github.com/go-logr/logr" + logf "sigs.k8s.io/controller-runtime/pkg/log" "github.com/AliyunContainerService/terway/pkg/logger" "github.com/AliyunContainerService/terway/pkg/metric" @@ -21,10 +25,11 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - log "github.com/sirupsen/logrus" "google.golang.org/grpc" ) +const daemonRPCTimeout = 118 * time.Second + // stackTriger print golang stack trace to log func stackTriger() { sigchain := make(chan os.Signal, 1) @@ -42,7 +47,7 @@ func stackTriger() { bufferLen *= 2 } buf = buf[:stackSize] - log.Printf("dump stacks: %s\n", string(buf)) + logger.DefaultLogger.Printf("dump stacks: %s\n", string(buf)) } }(sigchain) @@ -50,18 +55,8 @@ func stackTriger() { } // Run terway daemon -func Run(ctx context.Context, socketFilePath, debugSocketListen, configFilePath, daemonMode, logLevel string) error { - level, err := log.ParseLevel(logLevel) - if err != nil { - return fmt.Errorf("error set log level: %s, %w", logLevel, err) - } - logger.DefaultLogger.SetLevel(level) - if !utils.IsWindowsOS() { - // NB(thxCode): hcsshim lib introduces much noise. - log.SetLevel(level) - } - - err = os.MkdirAll(filepath.Dir(socketFilePath), 0700) +func Run(ctx context.Context, socketFilePath, debugSocketListen, configFilePath, daemonMode string) error { + err := os.MkdirAll(filepath.Dir(socketFilePath), 0700) if err != nil { return fmt.Errorf("error create socket dir: %s, %w", filepath.Dir(socketFilePath), err) } @@ -77,13 +72,15 @@ func Run(ctx context.Context, socketFilePath, debugSocketListen, configFilePath, return fmt.Errorf("error listen at %s: %v", socketFilePath, err) } - networkService, err := newNetworkService(ctx, configFilePath, daemonMode) + svc, err := newNetworkService(ctx, configFilePath, daemonMode) if err != nil { return err } - grpcServer := grpc.NewServer() - rpc.RegisterTerwayBackendServer(grpcServer, networkService) + grpcServer := grpc.NewServer(grpc.ChainUnaryInterceptor( + cniInterceptor, + )) + rpc.RegisterTerwayBackendServer(grpcServer, svc) rpc.RegisterTerwayTracingServer(grpcServer, tracing.DefaultRPCServer()) stop := make(chan struct{}) @@ -95,10 +92,10 @@ func Run(ctx context.Context, socketFilePath, debugSocketListen, configFilePath, } go func() { - serviceLog.Infof("start serving on %s", socketFilePath) + serviceLog.Info("start serving", "path", socketFilePath) err = grpcServer.Serve(l) if err != nil { - log.Errorf("error start grpc server: %v", err) + logger.DefaultLogger.Errorf("error start grpc server: %v", err) close(stop) } }() @@ -108,6 +105,9 @@ func Run(ctx context.Context, socketFilePath, debugSocketListen, configFilePath, case <-stop: } grpcServer.Stop() + + svc.wg.Wait() + return nil } @@ -143,7 +143,7 @@ func runDebugServer(debugSocketListen string) error { go func() { err := http.Serve(l, http.DefaultServeMux) if err != nil { - log.Errorf("error start debug server: %v", err) + logger.DefaultLogger.Errorf("error start debug server: %v", err) } }() @@ -164,3 +164,25 @@ func registerPrometheus() { prometheus.MustRegister(metric.ENIIPFactoryENICount) prometheus.MustRegister(metric.ENIIPFactoryIPAllocCount) } + +func cniInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + ctx, cancel := context.WithTimeout(ctx, daemonRPCTimeout) + defer cancel() + + switch req.(type) { + case *rpc.AllocIPRequest: + r := req.(*rpc.AllocIPRequest) + l := logf.FromContext(ctx, "pod", utils.PodInfoKey(r.K8SPodNamespace, r.K8SPodName), "containerID", r.K8SPodInfraContainerId) + ctx = logr.NewContext(ctx, l) + case *rpc.ReleaseIPRequest: + r := req.(*rpc.ReleaseIPRequest) + l := logf.FromContext(ctx, "pod", utils.PodInfoKey(r.K8SPodNamespace, r.K8SPodName), "containerID", r.K8SPodInfraContainerId) + ctx = logr.NewContext(ctx, l) + case *rpc.GetInfoRequest: + r := req.(*rpc.GetInfoRequest) + l := logf.FromContext(ctx, "pod", utils.PodInfoKey(r.K8SPodNamespace, r.K8SPodName), "containerID", r.K8SPodInfraContainerId) + ctx = logr.NewContext(ctx, l) + default: + } + return handler(ctx, req) +} diff --git a/daemon/veth.go b/daemon/veth.go deleted file mode 100644 index ccb010c0..00000000 --- a/daemon/veth.go +++ /dev/null @@ -1,46 +0,0 @@ -package daemon - -import ( - "github.com/AliyunContainerService/terway/pkg/tracing" - - "github.com/AliyunContainerService/terway/pkg/link" - "github.com/AliyunContainerService/terway/types" -) - -const ( - defaultPrefix = "cali" -) - -type vethResourceManager struct { -} - -func (*vethResourceManager) Allocate(context *networkContext, prefer string) (types.NetworkResource, error) { - vethName, _ := link.VethNameForPod(context.pod.Name, context.pod.Namespace, "", defaultPrefix) - return &types.Veth{ - HostVeth: vethName, - }, nil -} - -func (*vethResourceManager) Release(context *networkContext, resItem types.ResourceItem) error { - return nil -} - -func (f *vethResourceManager) GarbageCollection(inUseResSet map[string]types.ResourceItem, expireResSet map[string]types.ResourceItem) error { - return nil -} - -func (f *vethResourceManager) GetResourceMapping() (tracing.ResourcePoolStats, error) { - return nil, nil -} - -func (f *vethResourceManager) Stat(context *networkContext, resID string) (types.NetworkResource, error) { - vethName, _ := link.VethNameForPod(context.pod.Name, context.pod.Namespace, "", defaultPrefix) - return &types.Veth{ - HostVeth: vethName, - }, nil -} - -func newVPCResourceManager() (ResourceManager, error) { - mgr := &vethResourceManager{} - return mgr, nil -} diff --git a/docs/trouble-shotting.md b/docs/trouble-shotting.md new file mode 100644 index 00000000..e69de29b diff --git a/examples/maxpods/maxpods.go b/examples/maxpods/maxpods.go index b2b3b925..786218ec 100644 --- a/examples/maxpods/maxpods.go +++ b/examples/maxpods/maxpods.go @@ -6,13 +6,12 @@ import ( "io" "log" - "github.com/AliyunContainerService/terway/pkg/aliyun" - "github.com/AliyunContainerService/terway/pkg/aliyun/client" - "github.com/AliyunContainerService/terway/pkg/aliyun/credential" - "github.com/AliyunContainerService/terway/types" - "github.com/sirupsen/logrus" "k8s.io/client-go/util/flowcontrol" + + "github.com/AliyunContainerService/terway/pkg/aliyun/client" + "github.com/AliyunContainerService/terway/pkg/aliyun/credential" + "github.com/AliyunContainerService/terway/pkg/aliyun/instance" ) var ( @@ -37,7 +36,7 @@ func main() { flag.Parse() log.SetOutput(io.Discard) logrus.SetOutput(io.Discard) - ins := aliyun.GetInstanceMeta() + ins := instance.GetInstanceMeta() providers := []credential.Interface{ credential.NewAKPairProvider(accessKeyID, accessKeySecret), @@ -57,22 +56,14 @@ func main() { panic(err) } - f := &types.IPFamily{} - if ipStack == "ipv4" || ipStack == "dual" { - f.IPv4 = true - } - if ipStack == "dual" || ipStack == "ipv6" { - f.IPv6 = true - } - if mode == "terway-eniip" { - limit, err := aliyun.GetLimit(api, aliyun.GetInstanceMeta().InstanceType) + limit, err := instance.GetLimit(api, instance.GetInstanceMeta().InstanceType) if err != nil { panic(err) } fmt.Println(limit.IPv4PerAdapter * (limit.Adapters - 1)) } else if mode == "terway-eni" { - limit, err := aliyun.GetLimit(api, aliyun.GetInstanceMeta().InstanceType) + limit, err := instance.GetLimit(api, instance.GetInstanceMeta().InstanceType) if err != nil { panic(err) } diff --git a/go.mod b/go.mod index f66a0aa3..76b143af 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/AliyunContainerService/terway -go 1.18 +go 1.21 require ( github.com/Jeffail/gabs/v2 v2.7.0 @@ -14,6 +14,7 @@ require ( github.com/denverdino/aliyungo v0.0.0-20201215054313-f635de23c5e0 github.com/docker/docker v20.10.20+incompatible github.com/evanphx/json-patch v5.6.0+incompatible + github.com/go-logr/logr v1.3.0 github.com/go-playground/mold/v4 v4.2.0 github.com/go-playground/validator/v10 v10.11.1 github.com/google/uuid v1.3.0 @@ -22,28 +23,28 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.1 github.com/pterm/pterm v0.12.62 - github.com/samber/lo v1.38.1 - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.1 github.com/vishvananda/netlink v1.2.1-beta.2 go.uber.org/atomic v1.9.0 - golang.org/x/net v0.17.0 - golang.org/x/sync v0.2.0 - golang.org/x/sys v0.13.0 + golang.org/x/net v0.19.0 + golang.org/x/sync v0.5.0 + golang.org/x/sys v0.15.0 + golang.org/x/time v0.3.0 gomodules.xyz/jsonpatch/v2 v2.3.0 - google.golang.org/grpc v1.51.0 - google.golang.org/protobuf v1.30.0 + google.golang.org/grpc v1.56.3 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.27.2 - k8s.io/apiextensions-apiserver v0.27.2 - k8s.io/apimachinery v0.27.2 - k8s.io/client-go v0.27.2 - k8s.io/code-generator v0.27.2 + k8s.io/api v0.27.9 + k8s.io/apiextensions-apiserver v0.27.9 + k8s.io/apimachinery v0.27.9 + k8s.io/client-go v0.27.9 + k8s.io/code-generator v0.27.9 k8s.io/klog/v2 v2.90.1 - k8s.io/kubelet v0.21.4 + k8s.io/kubelet v0.27.9 k8s.io/utils v0.0.0-20230209194617-a36077c30491 - sigs.k8s.io/controller-runtime v0.15.0 + sigs.k8s.io/controller-runtime v0.15.3 sigs.k8s.io/e2e-framework v0.0.7 sigs.k8s.io/yaml v1.3.0 ) @@ -61,7 +62,6 @@ require ( github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect @@ -106,21 +106,19 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 // indirect - golang.org/x/mod v0.10.0 // indirect - golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.9.1 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.12.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/component-base v0.27.2 // indirect + k8s.io/component-base v0.27.9 // indirect k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index 8548eb5b..e0db3967 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,5 @@ atomicgo.dev/assert v0.0.2 h1:FiKeMiZSgRrZsPo9qn/7vmr7mCsh5SZyXY4YGYiYwrg= +atomicgo.dev/assert v0.0.2/go.mod h1:ut4NcI3QDdJtlmAxQULOmA13Gz6e2DWbSAS8RUOmNYQ= atomicgo.dev/cursor v0.1.1 h1:0t9sxQomCTRh5ug+hAMCs59x/UmC9QL6Ci5uosINKD4= atomicgo.dev/cursor v0.1.1/go.mod h1:Lr4ZJB3U7DfPPOkbH7/6TOtJ4vFGHlgj1nc+n900IpU= atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8= @@ -35,7 +36,6 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= @@ -57,6 +57,7 @@ github.com/MarvinJWendt/testza v0.2.12/go.mod h1:JOIegYyV7rX+7VZ9r77L/eH6CfJHHzX github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/2oUqKc6bF2c= github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE= github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4= +github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= @@ -98,6 +99,7 @@ github.com/aliyun/alibaba-cloud-sdk-go v1.62.215/go.mod h1:Api2AkmMgGaSUAhmk76oa github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -140,11 +142,7 @@ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= @@ -260,7 +258,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= @@ -305,7 +302,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= @@ -333,10 +329,10 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -369,6 +365,7 @@ github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= @@ -448,6 +445,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= +github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -528,6 +526,7 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= +github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -537,6 +536,7 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -580,7 +580,6 @@ github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2J github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -616,6 +615,7 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -718,14 +718,13 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0= github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs= -github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= -github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= @@ -744,8 +743,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -843,6 +842,7 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -861,12 +861,11 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -902,8 +901,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -941,7 +940,6 @@ golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= @@ -949,15 +947,15 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -971,8 +969,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1022,7 +1020,6 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1034,7 +1031,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1053,36 +1049,33 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1133,8 +1126,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1188,8 +1181,8 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1208,9 +1201,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1224,9 +1216,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1280,35 +1271,31 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.21.4/go.mod h1:fTVGP+M4D8+00FN2cMnJqk/eb/GH53bvmNs2SVTmpFk= -k8s.io/api v0.27.2 h1:+H17AJpUMvl+clT+BPnKf0E3ksMAzoBBg7CntpSuADo= -k8s.io/api v0.27.2/go.mod h1:ENmbocXfBT2ADujUXcBhHV55RIT31IIEvkntP6vZKS4= -k8s.io/apiextensions-apiserver v0.27.2 h1:iwhyoeS4xj9Y7v8YExhUwbVuBhMr3Q4bd/laClBV6Bo= -k8s.io/apiextensions-apiserver v0.27.2/go.mod h1:Oz9UdvGguL3ULgRdY9QMUzL2RZImotgxvGjdWRq6ZXQ= +k8s.io/api v0.27.9 h1:zelL2mPSOAgcItlCwIzy75/wl4Rt9kSKLMQYhAE2tA4= +k8s.io/api v0.27.9/go.mod h1:H6ZBAkYZ3B/BbyS6Us85MJas3OJp4xteQF4+kZHaw6g= +k8s.io/apiextensions-apiserver v0.27.9 h1:z7f95DVIrNkscuV2f/JdZNpjOxpUYeeRYcmrn0Wixas= +k8s.io/apiextensions-apiserver v0.27.9/go.mod h1:0/FD0h+vIInWSBUFv3sFgAkU5Zfd74Mwz0dOHWxr7/k= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.21.4/go.mod h1:H/IM+5vH9kZRNJ4l3x/fXP/5bOPJaVP/guptnZPeCFI= -k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg= -k8s.io/apimachinery v0.27.2/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= +k8s.io/apimachinery v0.27.9 h1:IOGjTX8IiPVpl7qOVOvXiiTQ+0G4k68/t5OMTWZkdas= +k8s.io/apimachinery v0.27.9/go.mod h1:EIXLxLt/b1muPITiF5zlrcm7I+YnXsIgM+0GdnPTQvA= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.21.4/go.mod h1:t0/eMKyUAq/DoQ7vW8NVVA00/nomlwC+eInsS8PxSew= -k8s.io/client-go v0.27.2 h1:vDLSeuYvCHKeoQRhCXjxXO45nHVv2Ip4Fe0MfioMrhE= -k8s.io/client-go v0.27.2/go.mod h1:tY0gVmUsHrAmjzHX9zs7eCjxcBsf8IiNe7KQ52biTcQ= +k8s.io/client-go v0.27.9 h1:eSGXZbMVyrFXQ7VZBtyJ1bPtnsApO/U/oJq0arkZ27c= +k8s.io/client-go v0.27.9/go.mod h1:I3POCgjLe2aLPpBSME+WQq64IGjOZ3CQUxp0MQdpVE8= k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= -k8s.io/code-generator v0.27.2 h1:RmK0CnU5qRaK6WRtSyWNODmfTZNoJbrizpVcsgbtrvI= -k8s.io/code-generator v0.27.2/go.mod h1:DPung1sI5vBgn4AGKtlPRQAyagj/ir/4jI55ipZHVww= +k8s.io/code-generator v0.27.9 h1:lm1+gQLzA4ElK8FtItUbFG1PpZ+Us1fSfn9RlRsy1ko= +k8s.io/code-generator v0.27.9/go.mod h1:NEx95JBRos8MSki+CuSoiEyKk6yv1rC4z/eY8DCZ/Rw= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.21.4/go.mod h1:ZKG0eHVX+tUDcaoIGpU3Vtk4TIjMddN9uhEWDmW6Nyg= -k8s.io/component-base v0.27.2 h1:neju+7s/r5O4x4/txeUONNTS9r1HsPbyoPBAtHsDCpo= -k8s.io/component-base v0.27.2/go.mod h1:5UPk7EjfgrfgRIuDBFtsEFAe4DAvP3U+M8RTzoSJkpo= +k8s.io/component-base v0.27.9 h1:2WGfRevgs/06+lVnXZCcLEf1XE7Cdts96zRdvkQjl8k= +k8s.io/component-base v0.27.9/go.mod h1:XGpxAlwW8CdfTCogVqmHPu6ZSGlGEMsvpOcWSveWRzY= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= @@ -1321,16 +1308,14 @@ k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= -k8s.io/kubelet v0.21.4 h1:LuGtuAeghCMf+IDoh7bpvFQNge1+GTujpR/qfQe/xfk= -k8s.io/kubelet v0.21.4/go.mod h1:kgXUz8upYNIngMSEZP1rpg2kp4gfUrsB7ir5u9Cm4HE= +k8s.io/kubelet v0.27.9 h1:KT26xwkGOTaq8rLNpdKK7G4cdSw/4EfHyxGSPhX2T3o= +k8s.io/kubelet v0.27.9/go.mod h1:dZ01D6udy2TGcZHd7uAv3iQRCO62wR9Q9sm0BhI+xIs= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= @@ -1340,8 +1325,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.15.0 h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0UPojU= -sigs.k8s.io/controller-runtime v0.15.0/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= +sigs.k8s.io/controller-runtime v0.15.3 h1:L+t5heIaI3zeejoIyyvLQs5vTVu/67IU2FfisVzFlBc= +sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= sigs.k8s.io/e2e-framework v0.0.7 h1:nMv2oSPBLWARse2aBoqX5Wq3ox67w8jrhTGWGpccWDQ= sigs.k8s.io/e2e-framework v0.0.7/go.mod h1:hdwYGVQg4bvDAah5eidNf2/qkG35qHjzuyMVr2A3oiY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= @@ -1349,7 +1334,6 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h6 sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/pkg/aliyun/aliyun.go b/pkg/aliyun/aliyun.go deleted file mode 100644 index ab775360..00000000 --- a/pkg/aliyun/aliyun.go +++ /dev/null @@ -1,562 +0,0 @@ -package aliyun - -import ( - "context" - "fmt" - "net" - "strings" - "sync" - "time" - - "k8s.io/apimachinery/pkg/util/uuid" - - "github.com/AliyunContainerService/terway/pkg/aliyun/client" - apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" - "github.com/AliyunContainerService/terway/pkg/backoff" - "github.com/AliyunContainerService/terway/pkg/ip" - "github.com/AliyunContainerService/terway/pkg/ipam" - "github.com/AliyunContainerService/terway/pkg/logger" - "github.com/AliyunContainerService/terway/pkg/metric" - "github.com/AliyunContainerService/terway/pkg/tracing" - "github.com/AliyunContainerService/terway/types" - - "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" - "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" - k8sErr "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" -) - -var log = logger.DefaultLogger.WithField("subSys", "openAPI") - -var _ ipam.API = &Impl{} - -type Impl struct { - privateIPMutex sync.RWMutex - metadata ENIInfoGetter - - // fixme remove when metadata support eni type field - eniTypeAttr bool - - ipFamily *types.IPFamily - - tagFilter map[string]string // eg. "TagKey": "creator", "TagValue": "terway" - - *client.OpenAPI -} - -// NewAliyunImpl return new API implement object -func NewAliyunImpl(openAPI *client.OpenAPI, needENITypeAttr bool, ipFamily *types.IPFamily, tagFilter map[string]string) ipam.API { - return &Impl{ - metadata: NewENIMetadata(ipFamily), - ipFamily: ipFamily, - eniTypeAttr: needENITypeAttr, - OpenAPI: openAPI, - tagFilter: tagFilter, - } -} - -// AllocateENI for instance -func (e *Impl) AllocateENI(ctx context.Context, vSwitch string, securityGroups []string, instanceID string, trunk bool, ipCount int, eniTags map[string]string) (*types.ENI, error) { - if vSwitch == "" || len(securityGroups) == 0 || instanceID == "" { - return nil, fmt.Errorf("invalid eni args for allocate") - } - - ipv4Count, ipv6Count := 0, 0 - if e.ipFamily.IPv4 { - ipv4Count = ipCount - } - if e.ipFamily.IPv6 { - ipv6Count = ipCount - } - - resp, err := e.CreateNetworkInterface(ctx, trunk, vSwitch, securityGroups, "", ipv4Count, ipv6Count, eniTags) - if err != nil { - return nil, err - } - defer func() { - if err != nil { - rollBackCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - eniDestroy := &types.ENI{ - ID: resp.NetworkInterfaceID, - } - if err = e.destroyInterface(rollBackCtx, eniDestroy.ID, instanceID, ""); err != nil { - fmtErr := fmt.Sprintf("error rollback interface, may cause eni leak: %+v", err) - _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, - tracing.AllocResourceFailed, fmtErr) - logrus.Error(fmtErr) - } - } - }() - - var innerErr error - err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.WaitENIStatus), func(ctx context.Context) (bool, error) { - innerErr = e.AttachNetworkInterface(ctx, resp.NetworkInterfaceID, instanceID, "") - if innerErr != nil { - return false, nil - } - return true, nil - }) - if err != nil { - fmtErr := fmt.Sprintf("error attach ENI, %v", innerErr) - _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, - tracing.AllocResourceFailed, fmtErr) - return nil, fmt.Errorf("%s, %w", fmtErr, err) - } - - logrus.Debugf("wait network interface attach: %v, %v", resp.NetworkInterfaceID, instanceID) - - start := time.Now() - // bind status is async api, sleep for first bind status inspect - time.Sleep(backoff.Backoff(backoff.WaitENIStatus).Duration) - eniStatus, err := e.WaitForNetworkInterface(ctx, resp.NetworkInterfaceID, client.ENIStatusInUse, backoff.Backoff(backoff.WaitENIStatus), false) - metric.OpenAPILatency.WithLabelValues("WaitForNetworkInterfaceBind/"+string(client.ENIStatusInUse), fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - - if err != nil { - return nil, err - } - - var eni *types.ENI - // backoff get eni config - err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.WaitENIStatus), - func(ctx context.Context) (done bool, err error) { - eni, innerErr = e.metadata.GetENIByMac(eniStatus.MacAddress) - if innerErr != nil || eni.ID != resp.NetworkInterfaceID { - logrus.Warnf("error get eni config by mac: %v, retrying...", innerErr) - return false, nil - } - eni.Trunk = eniStatus.Type == client.ENITypeTrunk - return true, nil - }, - ) - if err != nil { - return nil, fmt.Errorf("error get eni config, %v, %w", innerErr, err) - } - - return eni, nil -} - -func (e *Impl) FreeENI(ctx context.Context, eniID, instanceID string) error { - return e.destroyInterface(ctx, eniID, instanceID, "") -} - -func (e *Impl) destroyInterface(ctx context.Context, eniID, instanceID, trunkENIID string) error { - var innerErr error - err := wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIRelease), - func(ctx context.Context) (done bool, err error) { - innerErr = e.DetachNetworkInterface(ctx, eniID, instanceID, trunkENIID) - if innerErr != nil { - return false, nil - } - return true, nil - }, - ) - if err != nil { - fmtErr := fmt.Sprintf("cannot detach eni, %+v", innerErr) - _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, - tracing.DisposeResourceFailed, fmtErr) - } - - time.Sleep(backoff.Backoff(backoff.WaitENIStatus).Duration) - - // backoff delete network interface - err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIOps), - func(ctx context.Context) (done bool, err error) { - innerErr = e.DeleteNetworkInterface(context.Background(), eniID) - if innerErr != nil { - return false, nil - } - return true, nil - }, - ) - if err != nil { - fmtErr := fmt.Sprintf("cannot delete eni: %v %v", err, innerErr) - _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, - tracing.DisposeResourceFailed, fmtErr) - return fmt.Errorf("%s, %w", fmtErr, err) - } - return nil -} - -// GetAttachedENIs of instanceId -// containsMainENI is contains the main interface(eth0) of instance -func (e *Impl) GetAttachedENIs(ctx context.Context, containsMainENI bool, trunkENIID string) ([]*types.ENI, error) { - enis, err := e.metadata.GetENIs(containsMainENI) - if err != nil { - return nil, fmt.Errorf("error get eni config by mac, %w", err) - } - - var eniIDs []string - enisMap := map[string]*types.ENI{} - for _, eni := range enis { - eniIDs = append(eniIDs, eni.ID) - enisMap[eni.ID] = eni - - if trunkENIID == eni.ID { - eni.Trunk = true - } - } - - var result []*types.ENI - if (e.eniTypeAttr || len(e.tagFilter) > 0) && len(eniIDs) > 0 { - if trunkENIID == "" || len(e.tagFilter) > 0 { - eniSet, err := e.DescribeNetworkInterface(ctx, "", eniIDs, "", "", "", e.tagFilter) - if err != nil { - return nil, err - } - for _, eni := range eniSet { - e, ok := enisMap[eni.NetworkInterfaceID] - if !ok { - continue - } - e.Trunk = eni.Type == client.ENITypeTrunk - - // take to intersect - result = append(result, e) - } - } else { - result = enis - } - } else { - result = enis - } - return result, nil -} - -func (e *Impl) GetSecondaryENIMACs(ctx context.Context) ([]string, error) { - return e.metadata.GetSecondaryENIMACs() -} - -func (e *Impl) GetENIIPs(ctx context.Context, mac string) ([]net.IP, []net.IP, error) { - e.privateIPMutex.RLock() - defer e.privateIPMutex.RUnlock() - - var ipv4, ipv6 []net.IP - var err error - if e.ipFamily.IPv4 { - ipv4, err = e.metadata.GetENIPrivateAddressesByMAC(mac) - if err != nil { - return nil, nil, err - } - } - if e.ipFamily.IPv6 { - ipv6, err = e.metadata.GetENIPrivateIPv6AddressesByMAC(mac) - if err != nil { - return nil, nil, err - } - } - return ipv4, ipv6, nil -} - -func (e *Impl) AssignNIPsForENI(ctx context.Context, eniID, mac string, count int) ([]net.IP, []net.IP, error) { - if eniID == "" || mac == "" || count <= 0 { - return nil, nil, fmt.Errorf("args error") - } - e.privateIPMutex.Lock() - defer e.privateIPMutex.Unlock() - - var wg sync.WaitGroup - var ipv4s, ipv6s []net.IP - var err, v4Err, v6Err error - - wrap := func(e error) error { - err = e - return err - } - - defer func() { - if err == nil { - return - } - fmtErr := fmt.Errorf("error assign %d address for eniID: %v, %w", count, eniID, err) - _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, - tracing.AllocResourceFailed, fmtErr.Error()) - - // rollback ips - rollBackCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - roleBackErr := e.unAssignIPsForENIUnSafe(rollBackCtx, eniID, mac, ipv4s, ipv6s) - if roleBackErr != nil { - fmtErr = fmt.Errorf("roll back failed %s, %w", fmtErr, roleBackErr) - log.Error(fmtErr.Error()) - _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, - tracing.AllocResourceFailed, fmtErr.Error()) - } - }() - - if e.ipFamily.IPv4 { - var innerErr error - idempotentKey := string(uuid.NewUUID()) - err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIOps), func(ctx context.Context) (bool, error) { - ipv4s, innerErr = e.AssignPrivateIPAddress(ctx, eniID, count, idempotentKey) - if innerErr != nil { - if apiErr.ErrAssert(apiErr.InvalidVSwitchIDIPNotEnough, innerErr) { - return false, innerErr - } - return false, nil - } - return true, nil - }) - if err != nil { - return nil, nil, wrap(fmt.Errorf("%w, innerErr %v", err, innerErr)) - } - if len(ipv4s) != count { - return nil, nil, wrap(fmt.Errorf("openAPI return IP error.Want %d got %d", count, len(ipv4s))) - } - wg.Add(1) - go func() { - defer wg.Done() - v4Err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.MetaAssignPrivateIP), - func(ctx context.Context) (bool, error) { - var remoteIPs []net.IP - remoteIPs, innerErr = e.metadata.GetENIPrivateAddressesByMAC(mac) - if innerErr != nil { - return false, nil - } - if !ip.IPsIntersect(remoteIPs, ipv4s) { - innerErr = fmt.Errorf("ip is not present in metadataAPI,expect %s got %s", ipv4s, remoteIPs) - return false, nil - } - return true, nil - }, - ) - if v4Err != nil { - v4Err = fmt.Errorf("%w, metadataAPI %v", v4Err, innerErr) - } - }() - } - - if e.ipFamily.IPv6 { - var innerErr error - idempotentKey := string(uuid.NewUUID()) - err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIOps), func(ctx context.Context) (bool, error) { - ipv6s, innerErr = e.AssignIpv6Addresses(ctx, eniID, count, idempotentKey) - if innerErr != nil { - if apiErr.ErrAssert(apiErr.InvalidVSwitchIDIPNotEnough, innerErr) { - return false, innerErr - } - return false, nil - } - return true, nil - }) - if err != nil { - return ipv4s, ipv6s, wrap(fmt.Errorf("%w, innerErr %v", err, innerErr)) - } - if len(ipv6s) != count { - return ipv4s, ipv6s, wrap(fmt.Errorf("openAPI return IP error.Want %d got %d", count, len(ipv6s))) - } - wg.Add(1) - go func() { - defer wg.Done() - v6Err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.MetaAssignPrivateIP), - func(ctx context.Context) (bool, error) { - var remoteIPs []net.IP - remoteIPs, innerErr = e.metadata.GetENIPrivateIPv6AddressesByMAC(mac) - if innerErr != nil { - return false, nil - } - if !ip.IPsIntersect(remoteIPs, ipv6s) { - innerErr = fmt.Errorf("ip is not present in metadataAPI,expect %s got %s", ipv6s, remoteIPs) - return false, nil - } - return true, nil - }, - ) - if v6Err != nil { - v6Err = fmt.Errorf("%w, metadataAPI %v", v6Err, innerErr) - } - }() - } - wg.Wait() - - err = k8sErr.NewAggregate([]error{v4Err, v6Err}) - - return ipv4s, ipv6s, err -} - -func (e *Impl) UnAssignIPsForENI(ctx context.Context, eniID, mac string, ipv4s []net.IP, ipv6s []net.IP) error { - e.privateIPMutex.Lock() - defer e.privateIPMutex.Unlock() - - return e.unAssignIPsForENIUnSafe(ctx, eniID, mac, ipv4s, ipv6s) -} - -func (e *Impl) unAssignIPsForENIUnSafe(ctx context.Context, eniID, mac string, ipv4s []net.IP, ipv6s []net.IP) error { - if eniID == "" || mac == "" { - return fmt.Errorf("args error") - } - - var errs []error - - if len(ipv4s) > 0 { - var innerErr error - - err := wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIOps), func(ctx context.Context) (bool, error) { - innerErr = e.UnAssignPrivateIPAddresses(ctx, eniID, ipv4s) - if innerErr != nil { - return false, nil - } - return true, nil - }) - if err != nil { - errs = append(errs, err, innerErr) - } - } - - if len(ipv6s) > 0 { - var innerErr error - - err := wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.ENIOps), func(ctx context.Context) (bool, error) { - innerErr = e.UnAssignIpv6Addresses(ctx, eniID, ipv6s) - if innerErr != nil { - return false, nil - } - return true, nil - }) - if err != nil { - errs = append(errs, err, innerErr) - } - } - - if len(errs) > 0 { - fmtErr := fmt.Sprintf("error unassign address for eniID: %v, %v", eniID, k8sErr.NewAggregate(errs)) - _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, - tracing.DisposeResourceFailed, fmtErr) - return k8sErr.NewAggregate(errs) - } - - start := time.Now() - - // unassignPrivateIpAddresses is async api, sleep for first ip addr inspect - time.Sleep(backoff.Backoff(backoff.MetaUnAssignPrivateIP).Duration) - // backoff get interface addresses - var innerErr error - - err := wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.MetaUnAssignPrivateIP), - func(ctx context.Context) (bool, error) { - if len(ipv4s) > 0 { - var remoteIPs []net.IP - remoteIPs, innerErr = e.metadata.GetENIPrivateAddressesByMAC(mac) - if innerErr != nil { - return false, nil - } - if ip.IPsIntersect(remoteIPs, ipv4s) { - innerErr = fmt.Errorf("ip is present in metadataAPI,expect %s be removed, got %s", ipv4s, remoteIPs) - return false, nil - } - } - if len(ipv6s) > 0 { - var remoteIPs []net.IP - remoteIPs, innerErr = e.metadata.GetENIPrivateIPv6AddressesByMAC(mac) - if innerErr != nil { - return false, nil - } - if ip.IPsIntersect(remoteIPs, ipv6s) { - innerErr = fmt.Errorf("ip is present in metadataAPI,expect %s be removed, got %s", ipv6s, remoteIPs) - return false, nil - } - } - return true, nil - }, - ) - metric.OpenAPILatency.WithLabelValues("UnassignPrivateIpAddressesAsync", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - fmtErr := fmt.Sprintf("error unassign eni private address for %s, %v", eniID, innerErr) - _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, - tracing.DisposeResourceFailed, fmtErr) - return fmt.Errorf("%s, %w", fmtErr, err) - } - return nil -} - -func (e *Impl) GetENIByMac(ctx context.Context, mac string) (*types.ENI, error) { - eni, err := e.metadata.GetENIByMac(mac) - if err != nil { - return nil, fmt.Errorf("error get eni config by mac, %w", err) - } - return eni, nil -} - -func (e *Impl) GetAttachedSecurityGroups(ctx context.Context, instanceID string) ([]string, error) { - var ids []string - insType, err := e.GetInstanceAttributesType(ctx, instanceID) - if err != nil { - // fallback to deprecated DescribeInstanceAttribute - start := time.Now() - req := ecs.CreateDescribeInstanceAttributeRequest() - req.InstanceId = instanceID - resp, err := e.ClientSet.ECS().DescribeInstanceAttribute(req) - metric.OpenAPILatency.WithLabelValues("DescribeInstanceAttribute", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - return nil, fmt.Errorf("error describe instance attribute for security group: %s,%w", instanceID, err) - } - ids = resp.SecurityGroupIds.SecurityGroupId - } else { - ids = insType.SecurityGroupIds.SecurityGroupId - } - if len(ids) > 0 { - return ids, nil - } - return nil, fmt.Errorf("error get instance security groups: %s", instanceID) -} - -func (e *Impl) GetInstanceAttributesType(ctx context.Context, instanceID string) (*ecs.Instance, error) { - req := ecs.CreateDescribeInstancesRequest() - req.InstanceIds = fmt.Sprintf("[%q]", instanceID) - - start := time.Now() - resp, err := e.ClientSet.ECS().DescribeInstances(req) - metric.OpenAPILatency.WithLabelValues("DescribeInstances", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - return nil, err - } - if len(resp.Instances.Instance) != 1 { - return nil, fmt.Errorf("error get instanceAttributesType with instanceID %s: expected 1 but got %d", instanceID, len(resp.Instances.Instance)) - } - return &resp.Instances.Instance[0], nil -} - -func (e *Impl) QueryEniIDByIP(ctx context.Context, vpcID string, address net.IP) (string, error) { - req := ecs.CreateDescribeNetworkInterfacesRequest() - req.VpcId = vpcID - req.PrivateIpAddress = &[]string{address.String()} - - resp, err := e.ClientSet.ECS().DescribeNetworkInterfaces(req) - if err != nil || len(resp.NetworkInterfaceSets.NetworkInterfaceSet) != 1 { - return "", fmt.Errorf("error describe network interfaces from ip: %v, %v, %v", address, err, resp) - } - return resp.NetworkInterfaceSets.NetworkInterfaceSet[0].NetworkInterfaceId, nil -} - -// CheckEniSecurityGroup will sync eni's security with ecs's security group -func (e *Impl) CheckEniSecurityGroup(ctx context.Context, sg []string) error { - instanceID := GetInstanceMeta().InstanceID - - // get all attached eni - eniList, err := e.DescribeNetworkInterface(ctx, "", nil, instanceID, "", "", nil) - if err != nil { - logrus.WithField(client.LogFieldInstanceID, instanceID).Warn(err) - return nil - } - sgSet := sets.NewString(sg...) - - var errs []error - for _, eni := range eniList { - if eni.Type == string(client.ENITypePrimary) { - continue - } - eniSgSet := sets.NewString(eni.SecurityGroupIDs...) - if sgSet.Intersection(eniSgSet).Len() > 0 { - continue - } - err := fmt.Errorf("found eni %s security group [%s] mismatch witch ecs security group [%s]."+ - "If you can confirm config is correct, you can ignore this", eni.NetworkInterfaceID, - strings.Join(eni.SecurityGroupIDs, ","), strings.Join(sg, ",")) - logrus.WithField("instance", instanceID).Warn(err) - - errs = append(errs, err) - } - return k8sErr.NewAggregate(errs) -} diff --git a/pkg/aliyun/client/ecs.go b/pkg/aliyun/client/ecs.go index 6fed2827..485ad27d 100644 --- a/pkg/aliyun/client/ecs.go +++ b/pkg/aliyun/client/ecs.go @@ -3,23 +3,22 @@ package client import ( "context" "fmt" - "net" + "net/netip" + "strings" "time" - "github.com/AliyunContainerService/terway/pkg/backoff" - "github.com/AliyunContainerService/terway/pkg/tracing" - corev1 "k8s.io/api/core/v1" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/flowcontrol" + logf "sigs.k8s.io/controller-runtime/pkg/log" apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" "github.com/AliyunContainerService/terway/pkg/aliyun/credential" + "github.com/AliyunContainerService/terway/pkg/backoff" "github.com/AliyunContainerService/terway/pkg/ip" "github.com/AliyunContainerService/terway/pkg/metric" - - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" - "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/flowcontrol" ) var _ VSwitch = &OpenAPI{} @@ -42,6 +41,12 @@ func New(c credential.Client, readOnly, mutating flowcontrol.RateLimiter) (*Open // CreateNetworkInterface instanceType Secondary Trunk func (a *OpenAPI) CreateNetworkInterface(ctx context.Context, trunk bool, vSwitch string, securityGroups []string, resourceGroupID string, ipCount, ipv6Count int, eniTags map[string]string) (*NetworkInterface, error) { + l := logf.FromContext(ctx).WithValues( + LogFieldAPI, "CreateNetworkInterface", + LogFieldVSwitchID, vSwitch, + LogFieldSgID, securityGroups, + LogFieldResourceGroupID, resourceGroupID, + ) req := ecs.CreateCreateNetworkInterfaceRequest() req.ClientToken = string(uuid.NewUUID()) req.VSwitchId = vSwitch @@ -69,12 +74,6 @@ func (a *OpenAPI) CreateNetworkInterface(ctx context.Context, trunk bool, vSwitc } req.Tag = &tags - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "CreateNetworkInterface", - LogFieldVSwitchID: vSwitch, - LogFieldSgID: securityGroups, - LogFieldResourceGroupID: resourceGroupID, - }) var ( innerErr error resp *ecs.CreateNetworkInterfaceResponse @@ -85,25 +84,26 @@ func (a *OpenAPI) CreateNetworkInterface(ctx context.Context, trunk bool, vSwitc resp, innerErr = a.ClientSet.ECS().CreateNetworkInterface(req) metric.OpenAPILatency.WithLabelValues("CreateNetworkInterface", fmt.Sprint(innerErr != nil)).Observe(metric.MsSince(start)) if innerErr != nil { - if apiErr.ErrAssert(apiErr.InvalidVSwitchIDIPNotEnough, innerErr) { + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(innerErr)).Error(innerErr, "failed") + + if apiErr.ErrAssert(apiErr.ErrForbidden, innerErr) || + apiErr.ErrAssert(apiErr.ErrSecurityGroupInstanceLimitExceed, innerErr) || + apiErr.ErrAssert(apiErr.InvalidVSwitchIDIPNotEnough, innerErr) { return false, innerErr } - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(innerErr)).Errorf("error create ENI, %s", innerErr.Error()) return false, nil } return true, nil }) if err != nil { - fmtErr := fmt.Sprintf("error create ENI, %v", innerErr) - _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, - tracing.AllocResourceFailed, fmtErr) + fmtErr := fmt.Sprintf("error create eni, %v", innerErr) return nil, fmt.Errorf("%s, %w", fmtErr, err) } - l.WithFields(map[string]interface{}{ - LogFieldENIID: resp.NetworkInterfaceId, - LogFieldRequestID: resp.RequestId, - }).Info("create ENI") + l.WithValues( + LogFieldENIID, resp.NetworkInterfaceId, + LogFieldRequestID, resp.RequestId, + ).Info("create ENI") return FromCreateResp(resp), err } @@ -134,17 +134,17 @@ func (a *OpenAPI) DescribeNetworkInterface(ctx context.Context, vpcID string, en req.MaxResults = requests.NewInteger(maxSinglePageSize) - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "DescribeNetworkInterfaces", - LogFieldENIID: eniID, - LogFieldInstanceID: instanceID, - }) + l := logf.FromContext(ctx).WithValues( + LogFieldAPI, "DescribeNetworkInterfaces", + LogFieldENIID, eniID, + LogFieldInstanceID, instanceID) + a.ReadOnlyRateLimiter.Accept() start := time.Now() resp, err := a.ClientSet.ECS().DescribeNetworkInterfaces(req) metric.OpenAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) if err != nil { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Warn(err) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "error describe eni") return nil, err } for _, r := range resp.NetworkInterfaceSets.NetworkInterfaceSet { @@ -166,20 +166,19 @@ func (a *OpenAPI) AttachNetworkInterface(ctx context.Context, eniID, instanceID, req.InstanceId = instanceID req.TrunkNetworkInstanceId = trunkENIID - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "AttachNetworkInterface", - LogFieldENIID: eniID, - LogFieldInstanceID: instanceID, - }) + l := logf.FromContext(ctx).WithValues(LogFieldAPI, "AttachNetworkInterface", + LogFieldENIID, eniID, + LogFieldInstanceID, instanceID) + a.MutatingRateLimiter.Accept() start := time.Now() resp, err := a.ClientSet.ECS().AttachNetworkInterface(req) metric.OpenAPILatency.WithLabelValues("AttachNetworkInterface", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) if err != nil { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Warnf("attach ENI failed, %s", err.Error()) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "attach ENI failed") return err } - l.WithField(LogFieldRequestID, resp.RequestId).Infof("attach eni") + l.WithValues(LogFieldRequestID, resp.RequestId).Info("attach eni") return nil } @@ -190,11 +189,11 @@ func (a *OpenAPI) DetachNetworkInterface(ctx context.Context, eniID, instanceID, req.InstanceId = instanceID req.TrunkNetworkInstanceId = trunkENIID - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "DetachNetworkInterface", - LogFieldENIID: eniID, - LogFieldInstanceID: instanceID, - }) + l := logf.FromContext(ctx).WithValues( + LogFieldAPI, "DetachNetworkInterface", + LogFieldENIID, eniID, + LogFieldInstanceID, instanceID, + ) a.MutatingRateLimiter.Accept() start := time.Now() resp, err := a.ClientSet.ECS().DetachNetworkInterface(req) @@ -203,10 +202,10 @@ func (a *OpenAPI) DetachNetworkInterface(ctx context.Context, eniID, instanceID, if apiErr.ErrAssert(apiErr.ErrInvalidENINotFound, err) { return nil } - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Errorf("detach eni failed, %v", err) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "detach ENI failed") return err } - l.WithField(LogFieldRequestID, resp.RequestId).Infof("detach eni") + l.WithValues(LogFieldRequestID, resp.RequestId).Info("detach eni") return nil } @@ -215,19 +214,19 @@ func (a *OpenAPI) DeleteNetworkInterface(ctx context.Context, eniID string) erro req := ecs.CreateDeleteNetworkInterfaceRequest() req.NetworkInterfaceId = eniID - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "DeleteNetworkInterface", - LogFieldENIID: eniID, - }) + l := logf.FromContext(ctx).WithValues( + LogFieldAPI, "DeleteNetworkInterface", + LogFieldENIID, eniID, + ) a.MutatingRateLimiter.Accept() start := time.Now() resp, err := a.ClientSet.ECS().DeleteNetworkInterface(req) metric.OpenAPILatency.WithLabelValues("DeleteNetworkInterface", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) if err != nil { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Errorf("delete eni failed, %v", err) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "delete eni failed") return err } - l.WithField(LogFieldRequestID, resp.RequestId).Infof("delete eni") + l.WithValues(LogFieldRequestID, resp.RequestId).Info("delete eni") return nil } @@ -264,30 +263,30 @@ func (a *OpenAPI) WaitForNetworkInterface(ctx context.Context, eniID string, sta } // AssignPrivateIPAddress assign secondary ip -func (a *OpenAPI) AssignPrivateIPAddress(ctx context.Context, eniID string, count int, idempotentKey string) ([]net.IP, error) { +func (a *OpenAPI) AssignPrivateIPAddress(ctx context.Context, eniID string, count int, idempotentKey string) ([]netip.Addr, error) { req := ecs.CreateAssignPrivateIpAddressesRequest() req.NetworkInterfaceId = eniID req.SecondaryPrivateIpAddressCount = requests.NewInteger(count) req.ClientToken = idempotentKey - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "AssignPrivateIpAddresses", - LogFieldENIID: eniID, - LogFieldSecondaryIPCount: count, - }) + l := logf.FromContext(ctx).WithValues( + LogFieldAPI, "AssignPrivateIpAddresses", + LogFieldENIID, eniID, + LogFieldSecondaryIPCount, count, + ) start := time.Now() resp, err := a.ClientSet.ECS().AssignPrivateIpAddresses(req) metric.OpenAPILatency.WithLabelValues("AssignPrivateIpAddresses", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) if err != nil { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Warnf("assign private ip failed, %s", err.Error()) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "failed") return nil, err } - ips, err := ip.ToIPs(resp.AssignedPrivateIpAddressesSet.PrivateIpSet.PrivateIpAddress) + ips, err := ip.ToIPAddrs(resp.AssignedPrivateIpAddressesSet.PrivateIpSet.PrivateIpAddress) if err != nil { - l.WithField(LogFieldRequestID, resp.RequestId).Errorf("assign private ip, %v", resp.AssignedPrivateIpAddressesSet.PrivateIpSet.PrivateIpAddress) + l.WithValues(LogFieldRequestID, resp.RequestId).Error(err, "failed") return nil, err } - l.WithField(LogFieldRequestID, resp.RequestId).Infof("assign private ip, %v", resp.AssignedPrivateIpAddressesSet.PrivateIpSet.PrivateIpAddress) + l.WithValues(LogFieldRequestID, resp.RequestId).Info("assign private ip", "ips", strings.Join(resp.AssignedPrivateIpAddressesSet.PrivateIpSet.PrivateIpAddress, ",")) return ips, nil } @@ -295,100 +294,96 @@ func (a *OpenAPI) AssignPrivateIPAddress(ctx context.Context, eniID string, coun // UnAssignPrivateIPAddresses remove ip from eni // return ok if 1. eni is released 2. ip is already released 3. release success // for primaryIP err is InvalidIp.IpUnassigned -func (a *OpenAPI) UnAssignPrivateIPAddresses(ctx context.Context, eniID string, ips []net.IP) error { +func (a *OpenAPI) UnAssignPrivateIPAddresses(ctx context.Context, eniID string, ips []netip.Addr) error { if len(ips) == 0 { return nil } req := ecs.CreateUnassignPrivateIpAddressesRequest() req.NetworkInterfaceId = eniID - str := ip.IPs2str(ips) + str := ip.IPAddrs2str(ips) req.PrivateIpAddress = &str - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "UnassignPrivateIpAddresses", - LogFieldENIID: eniID, - }) + l := logf.FromContext(ctx).WithValues( + LogFieldAPI, "UnassignPrivateIpAddresses", + LogFieldENIID, eniID, + LogFieldIPs, strings.Join(str, ","), + ) start := time.Now() resp, err := a.ClientSet.ECS().UnassignPrivateIpAddresses(req) metric.OpenAPILatency.WithLabelValues("UnassignPrivateIpAddresses", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) if err != nil { - if apiErr.ErrAssert(apiErr.ErrInvalidIPIPUnassigned, err) { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Infof("unassign private ip ,%s", str) + if apiErr.ErrAssert(apiErr.ErrInvalidIPIPUnassigned, err) || apiErr.ErrAssert(apiErr.ErrInvalidENINotFound, err) { + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Info("success") return nil } - if apiErr.ErrAssert(apiErr.ErrInvalidENINotFound, err) { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Infof("unassign private ip ,%s", str) - return nil - } - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Warnf("unassign private ip failed,%s %s", str, err.Error()) + + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "unassign private ip failed") return err } - l.WithField(LogFieldRequestID, resp.RequestId).Infof("unassign private ip ,%s", str) + l.WithValues(LogFieldRequestID, resp.RequestId).Info("success") return nil } // AssignIpv6Addresses assign ipv6 address -func (a *OpenAPI) AssignIpv6Addresses(ctx context.Context, eniID string, count int, idempotentKey string) ([]net.IP, error) { +func (a *OpenAPI) AssignIpv6Addresses(ctx context.Context, eniID string, count int, idempotentKey string) ([]netip.Addr, error) { req := ecs.CreateAssignIpv6AddressesRequest() req.NetworkInterfaceId = eniID req.Ipv6AddressCount = requests.NewInteger(count) req.ClientToken = idempotentKey - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "AssignIpv6Addresses", - LogFieldENIID: eniID, - LogFieldSecondaryIPCount: count, - }) + l := logf.FromContext(ctx).WithValues( + LogFieldAPI, "AssignIpv6Addresses", + LogFieldENIID, eniID, + LogFieldSecondaryIPCount, count, + ) start := time.Now() resp, err := a.ClientSet.ECS().AssignIpv6Addresses(req) metric.OpenAPILatency.WithLabelValues("AssignIpv6Addresses", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) if err != nil { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Warnf("assign private ip failed, %s", err.Error()) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "failed") return nil, err } - ips, err := ip.ToIPs(resp.Ipv6Sets.Ipv6Address) + ips, err := ip.ToIPAddrs(resp.Ipv6Sets.Ipv6Address) if err != nil { - l.WithField(LogFieldRequestID, resp.RequestId).Errorf("assign private ip, %v", resp.Ipv6Sets.Ipv6Address) + l.WithValues(LogFieldRequestID, resp.RequestId).Error(err, "failed") return nil, err } - l.WithField(LogFieldRequestID, resp.RequestId).Infof("assign ipv6 ip, %v", resp.Ipv6Sets.Ipv6Address) + l.WithValues(LogFieldRequestID, resp.RequestId).Info("assign ipv6 ip", "ips", strings.Join(resp.Ipv6Sets.Ipv6Address, ",")) return ips, nil } // UnAssignIpv6Addresses remove ip from eni // return ok if 1. eni is released 2. ip is already released 3. release success -func (a *OpenAPI) UnAssignIpv6Addresses(ctx context.Context, eniID string, ips []net.IP) error { +func (a *OpenAPI) UnAssignIpv6Addresses(ctx context.Context, eniID string, ips []netip.Addr) error { if len(ips) == 0 { return nil } req := ecs.CreateUnassignIpv6AddressesRequest() req.NetworkInterfaceId = eniID - str := ip.IPs2str(ips) + str := ip.IPAddrs2str(ips) req.Ipv6Address = &str - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "UnassignIpv6Addresses", - LogFieldENIID: eniID, - }) + l := logf.FromContext(ctx).WithValues( + LogFieldAPI, "UnassignIpv6Addresses", + LogFieldENIID, eniID, + LogFieldIPs, strings.Join(str, ","), + ) start := time.Now() resp, err := a.ClientSet.ECS().UnassignIpv6Addresses(req) metric.OpenAPILatency.WithLabelValues("UnassignIpv6Addresses", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) if err != nil { - if apiErr.ErrAssert(apiErr.ErrInvalidIPIPUnassigned, err) { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Infof("unassign private ip ,%s", str) - return nil - } - if apiErr.ErrAssert(apiErr.ErrInvalidENINotFound, err) { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Infof("unassign private ip ,%s", str) + if apiErr.ErrAssert(apiErr.ErrInvalidIPIPUnassigned, err) || apiErr.ErrAssert(apiErr.ErrInvalidENINotFound, err) { + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Info("success") return nil } - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Warnf("unassign private ipv6 failed,%s %s", str, err.Error()) + + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "unassign ipv6 ip failed") return err } - l.WithField(LogFieldRequestID, resp.RequestId).Infof("unassign ipv6 ip ,%s", str) + l.WithValues(LogFieldRequestID, resp.RequestId).Info("success") return nil } @@ -408,11 +403,11 @@ func (a *OpenAPI) DescribeInstanceTypes(ctx context.Context, types []string) ([] resp, err := a.ClientSet.ECS().DescribeInstanceTypes(req) metric.OpenAPILatency.WithLabelValues("DescribeInstanceTypes", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "DescribeInstanceTypes", - }) + l := logf.FromContext(ctx).WithValues( + LogFieldAPI, "DescribeInstanceTypes", + ) if err != nil { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "describe instance types failed") return nil, err } @@ -435,13 +430,13 @@ func (a *OpenAPI) ModifyNetworkInterfaceAttribute(ctx context.Context, eniID str resp, err := a.ClientSet.ECS().ModifyNetworkInterfaceAttribute(req) metric.OpenAPILatency.WithLabelValues("ModifyNetworkInterfaceAttribute", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "ModifyNetworkInterfaceAttribute", - }) + l := logf.FromContext(ctx).WithValues( + LogFieldAPI, "ModifyNetworkInterfaceAttribute", + ) if err != nil { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "modify securityGroup failed") return err } - l.WithField(LogFieldRequestID, resp.RequestId).Infof("modify securityGroup %s", securityGroupIDs) + l.WithValues(LogFieldRequestID, resp.RequestId).Info("modify securityGroup", "ids", strings.Join(securityGroupIDs, ",")) return nil } diff --git a/pkg/aliyun/client/errors/errors.go b/pkg/aliyun/client/errors/errors.go index c8e56328..3460d978 100644 --- a/pkg/aliyun/client/errors/errors.go +++ b/pkg/aliyun/client/errors/errors.go @@ -2,11 +2,14 @@ package errors import ( "errors" + "fmt" apiErr "github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors" ) const ( + ErrForbidden = "Forbidden.RAM" + // InvalidVSwitchIDIPNotEnough AssignPrivateIpAddresses const error message // Reference: https://help.aliyun.com/document_detail/85917.html InvalidVSwitchIDIPNotEnough = "InvalidVSwitchId.IpNotEnough" @@ -83,3 +86,11 @@ func ErrRequestID(err error) string { } return "" } + +func ErrFormat(err error) string { + respErr, ok := err.(apiErr.Error) + if !ok { + return err.Error() + } + return fmt.Sprintf("errCode: %s, msg: %s", respErr.ErrorCode(), respErr.Message()) +} diff --git a/pkg/aliyun/client/fake/fake_client.go b/pkg/aliyun/client/fake/fake_client.go index 9bd599d9..8f49254b 100644 --- a/pkg/aliyun/client/fake/fake_client.go +++ b/pkg/aliyun/client/fake/fake_client.go @@ -4,17 +4,16 @@ import ( "context" "errors" "fmt" - "net" + "net/netip" "sync" - "github.com/AliyunContainerService/terway/pkg/aliyun/client" - apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" - terwayIP "github.com/AliyunContainerService/terway/pkg/ip" - "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" + + "github.com/AliyunContainerService/terway/pkg/aliyun/client" + apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" ) var _ client.VSwitch = &OpenAPI{} @@ -26,8 +25,8 @@ type OpenAPI struct { VSwitches map[string]vpc.VSwitch ENIs map[string]*client.NetworkInterface - IPAM map[string]net.IP // index by vSwitch id - IPAMV6 map[string]net.IP // index by vSwitch id + IPAM map[string]netip.Addr // index by vSwitch id + IPAMV6 map[string]netip.Addr // index by vSwitch id } func New() *OpenAPI { @@ -35,8 +34,8 @@ func New() *OpenAPI { Mutex: sync.Mutex{}, VSwitches: map[string]vpc.VSwitch{}, ENIs: map[string]*client.NetworkInterface{}, - IPAM: map[string]net.IP{}, - IPAMV6: map[string]net.IP{}, + IPAM: map[string]netip.Addr{}, + IPAMV6: map[string]netip.Addr{}, } } @@ -189,7 +188,7 @@ func (o *OpenAPI) WaitForNetworkInterface(ctx context.Context, eniID string, sta return nil, apiErr.ErrNotFound } -func (o *OpenAPI) AssignPrivateIPAddress(ctx context.Context, eniID string, count int, idempotentKey string) ([]net.IP, error) { +func (o *OpenAPI) AssignPrivateIPAddress(ctx context.Context, eniID string, count int, idempotent string) ([]netip.Addr, error) { o.Lock() defer o.Unlock() @@ -197,7 +196,7 @@ func (o *OpenAPI) AssignPrivateIPAddress(ctx context.Context, eniID string, coun if !ok { return nil, apiErr.ErrNotFound } - var ips []net.IP + var ips []netip.Addr for i := 0; i < count; i++ { ip := o.nextIP(eni.VSwitchID) ips = append(ips, ip) @@ -210,16 +209,16 @@ func (o *OpenAPI) AssignPrivateIPAddress(ctx context.Context, eniID string, coun return ips, nil } -func (o *OpenAPI) UnAssignPrivateIPAddresses(ctx context.Context, eniID string, ips []net.IP) error { +func (o *OpenAPI) UnAssignPrivateIPAddresses(ctx context.Context, eniID string, ips []netip.Addr) error { return nil } -func (o *OpenAPI) AssignIpv6Addresses(ctx context.Context, eniID string, count int, idempotentKey string) ([]net.IP, error) { +func (o *OpenAPI) AssignIpv6Addresses(ctx context.Context, eniID string, count int, idempotentKey string) ([]netip.Addr, error) { eni, ok := o.ENIs[eniID] if !ok { return nil, apiErr.ErrNotFound } - var ips []net.IP + var ips []netip.Addr for i := 0; i < count; i++ { ip := o.nextIPV6(eni.VSwitchID) ips = append(ips, ip) @@ -232,7 +231,7 @@ func (o *OpenAPI) AssignIpv6Addresses(ctx context.Context, eniID string, count i return ips, nil } -func (o *OpenAPI) UnAssignIpv6Addresses(ctx context.Context, eniID string, ips []net.IP) error { +func (o *OpenAPI) UnAssignIpv6Addresses(ctx context.Context, eniID string, ips []netip.Addr) error { return nil } @@ -258,38 +257,39 @@ func (o *OpenAPI) ModifyNetworkInterfaceAttribute(ctx context.Context, eniID str return nil } -func (o *OpenAPI) nextIP(vSwitchID string) net.IP { +func (o *OpenAPI) nextIP(vSwitchID string) netip.Addr { pre, ok := o.IPAM[vSwitchID] if ok { - o.IPAM[vSwitchID] = terwayIP.GetNextIP(pre) + o.IPAM[vSwitchID] = pre.Next() return o.IPAM[vSwitchID] } vsw, ok := o.VSwitches[vSwitchID] if !ok { - return nil + return netip.Addr{} } - ip, _, err := net.ParseCIDR(vsw.CidrBlock) + prefix, err := netip.ParsePrefix(vsw.CidrBlock) if err != nil { - return nil + return netip.Addr{} } - o.IPAM[vSwitchID] = terwayIP.GetNextIP(ip) + o.IPAM[vSwitchID] = prefix.Addr().Next() return o.IPAM[vSwitchID] } -func (o *OpenAPI) nextIPV6(vSwitchID string) net.IP { +func (o *OpenAPI) nextIPV6(vSwitchID string) netip.Addr { pre, ok := o.IPAMV6[vSwitchID] if ok { - o.IPAMV6[vSwitchID] = terwayIP.GetNextIP(pre) + o.IPAMV6[vSwitchID] = pre.Next() return o.IPAMV6[vSwitchID] } vsw, ok := o.VSwitches[vSwitchID] if !ok { - return nil + return netip.Addr{} } - ip, _, err := net.ParseCIDR(vsw.Ipv6CidrBlock) + + prefix, err := netip.ParsePrefix(vsw.Ipv6CidrBlock) if err != nil { - return nil + return netip.Addr{} } - o.IPAMV6[vSwitchID] = terwayIP.GetNextIP(ip) + o.IPAMV6[vSwitchID] = prefix.Addr().Next() return o.IPAMV6[vSwitchID] } diff --git a/pkg/aliyun/client/interface.go b/pkg/aliyun/client/interface.go index 89b8441c..61fd51b2 100644 --- a/pkg/aliyun/client/interface.go +++ b/pkg/aliyun/client/interface.go @@ -9,11 +9,3 @@ import ( type VSwitch interface { DescribeVSwitchByID(ctx context.Context, vSwitchID string) (*vpc.VSwitch, error) } - -type EIP interface { - AllocateEIPAddress(bandwidth, chargeType, isp string) (*vpc.AllocateEipAddressResponse, error) - AssociateEIPAddress(eipID, eniID, privateIP string) error - UnAssociateEIPAddress(eipID, eniID, eniIP string) error - ReleaseEIPAddress(eipID string) error - AddCommonBandwidthPackageIP(eipID, packageID string) error -} diff --git a/pkg/aliyun/client/interface_default.go b/pkg/aliyun/client/interface_default.go index 027eee23..5497c8f8 100644 --- a/pkg/aliyun/client/interface_default.go +++ b/pkg/aliyun/client/interface_default.go @@ -4,7 +4,7 @@ package client import ( "context" - "net" + "net/netip" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "k8s.io/apimachinery/pkg/util/wait" @@ -17,10 +17,10 @@ type ENI interface { DetachNetworkInterface(ctx context.Context, eniID, instanceID, trunkENIID string) error DeleteNetworkInterface(ctx context.Context, eniID string) error WaitForNetworkInterface(ctx context.Context, eniID string, status string, backoff wait.Backoff, ignoreNotExist bool) (*NetworkInterface, error) - AssignPrivateIPAddress(ctx context.Context, eniID string, count int, idempotent string) ([]net.IP, error) - UnAssignPrivateIPAddresses(ctx context.Context, eniID string, ips []net.IP) error - AssignIpv6Addresses(ctx context.Context, eniID string, count int, idempotentKey string) ([]net.IP, error) - UnAssignIpv6Addresses(ctx context.Context, eniID string, ips []net.IP) error + AssignPrivateIPAddress(ctx context.Context, eniID string, count int, idempotent string) ([]netip.Addr, error) + UnAssignPrivateIPAddresses(ctx context.Context, eniID string, ips []netip.Addr) error + AssignIpv6Addresses(ctx context.Context, eniID string, count int, idempotentKey string) ([]netip.Addr, error) + UnAssignIpv6Addresses(ctx context.Context, eniID string, ips []netip.Addr) error ModifyNetworkInterfaceAttribute(ctx context.Context, eniID string, securityGroupIDs []string) error } diff --git a/pkg/aliyun/client/types.go b/pkg/aliyun/client/types.go index 52f66105..5cd6920b 100644 --- a/pkg/aliyun/client/types.go +++ b/pkg/aliyun/client/types.go @@ -5,12 +5,9 @@ import ( "math/rand" "time" - "github.com/AliyunContainerService/terway/pkg/logger" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" ) -var log = logger.DefaultLogger.WithField("subSys", "openAPI") - // log fields const ( LogFieldAPI = "api" @@ -18,6 +15,7 @@ const ( LogFieldInstanceID = "instanceID" LogFieldSecondaryIPCount = "secondaryIPCount" LogFieldENIID = "eni" + LogFieldIPs = "ips" LogFieldEIPID = "eip" LogFieldPrivateIP = "privateIP" LogFieldVSwitchID = "vSwitchID" diff --git a/pkg/aliyun/client/vpc.go b/pkg/aliyun/client/vpc.go deleted file mode 100644 index 5dc4bb43..00000000 --- a/pkg/aliyun/client/vpc.go +++ /dev/null @@ -1,184 +0,0 @@ -package client - -import ( - "fmt" - "time" - - apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" - "github.com/AliyunContainerService/terway/pkg/backoff" - "github.com/AliyunContainerService/terway/pkg/metric" - "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" - "k8s.io/client-go/util/retry" -) - -// AllocateEIPAddress create EIP -func (a *OpenAPI) AllocateEIPAddress(bandwidth, chargeType, isp string) (*vpc.AllocateEipAddressResponse, error) { - req := vpc.CreateAllocateEipAddressRequest() - req.Bandwidth = bandwidth - req.InternetChargeType = chargeType - req.ISP = isp - - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "AllocateEipAddress", - }) - start := time.Now() - resp, err := a.ClientSet.VPC().AllocateEipAddress(req) - metric.OpenAPILatency.WithLabelValues("AllocateEipAddress", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - l.WithFields(map[string]interface{}{ - LogFieldRequestID: apiErr.ErrRequestID(err)}).Errorf("alloc EIP faild, %s", err.Error()) - return nil, fmt.Errorf("error create EIP, bandwidth %s, %w", bandwidth, err) - } - l.WithFields(map[string]interface{}{ - LogFieldEIPID: resp.AllocationId, - LogFieldRequestID: resp.RequestId}).Infof("alloc EIP %s", resp.EipAddress) - return resp, nil -} - -// AssociateEIPAddress bind eip to ip -func (a *OpenAPI) AssociateEIPAddress(eipID, eniID, privateIP string) error { - req := vpc.CreateAssociateEipAddressRequest() - req.AllocationId = eipID - req.InstanceId = eniID - req.PrivateIpAddress = privateIP - req.InstanceType = EIPInstanceTypeNetworkInterface - - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "AssociateEipAddress", - LogFieldEIPID: eipID, - LogFieldENIID: eniID, - }) - start := time.Now() - - return retry.OnError(backoff.Backoff(backoff.DefaultKey), func(err error) bool { - return apiErr.ErrAssert(apiErr.ErrTaskConflict, err) - }, func() error { - resp, err := a.ClientSet.VPC().AssociateEipAddress(req) - metric.OpenAPILatency.WithLabelValues("AssociateEipAddress", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - l.WithFields(map[string]interface{}{ - LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("associate EIP to %s failed, %s", privateIP, err.Error()) - - return err - } - l.WithFields(map[string]interface{}{ - LogFieldRequestID: resp.RequestId}).Infof("associate EIP to %s", privateIP) - return nil - }) -} - -// UnAssociateEIPAddress un-bind eip -func (a *OpenAPI) UnAssociateEIPAddress(eipID, eniID, eniIP string) error { - req := vpc.CreateUnassociateEipAddressRequest() - req.AllocationId = eipID - req.InstanceId = eniID - req.PrivateIpAddress = eniIP - req.InstanceType = EIPInstanceTypeNetworkInterface - - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "UnassociateEipAddress", - LogFieldEIPID: eipID, - }) - - return retry.OnError(backoff.Backoff(backoff.DefaultKey), func(err error) bool { - return apiErr.ErrAssert(apiErr.ErrTaskConflict, err) - }, func() error { - start := time.Now() - resp, err := a.ClientSet.VPC().UnassociateEipAddress(req) - metric.OpenAPILatency.WithLabelValues("UnassociateEipAddress", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - l.WithFields(map[string]interface{}{ - LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("unassociate EIP failed, %s", err.Error()) - if apiErr.ErrAssert(apiErr.ErrInvalidAllocationIDNotFound, err) || - apiErr.ErrAssert(apiErr.ErrIncorrectEIPStatus, err) { - return nil - } - return fmt.Errorf("error unassociate EIP %s, %w", eipID, err) - } - l.WithFields(map[string]interface{}{ - LogFieldRequestID: resp.RequestId}).Info("unassociate EIP") - return nil - }) -} - -// ReleaseEIPAddress delete EIP -func (a *OpenAPI) ReleaseEIPAddress(eipID string) error { - req := vpc.CreateReleaseEipAddressRequest() - req.AllocationId = eipID - - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "ReleaseEipAddress", - LogFieldEIPID: eipID, - }) - - return retry.OnError(backoff.Backoff(backoff.DefaultKey), func(err error) bool { - return apiErr.ErrAssert(apiErr.ErrTaskConflict, err) - }, func() error { - start := time.Now() - resp, err := a.ClientSet.VPC().ReleaseEipAddress(req) - metric.OpenAPILatency.WithLabelValues("ReleaseEipAddress", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - l.WithFields(map[string]interface{}{ - LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("release EIP failed, %s", err.Error()) - return err - } - l.WithFields(map[string]interface{}{ - LogFieldRequestID: resp.RequestId}).Info("release EIP") - return nil - }) -} - -// AddCommonBandwidthPackageIP add EIP to bandwidth package -func (a *OpenAPI) AddCommonBandwidthPackageIP(eipID, packageID string) error { - req := vpc.CreateAddCommonBandwidthPackageIpRequest() - req.BandwidthPackageId = packageID - req.IpInstanceId = eipID - - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "AddCommonBandwidthPackageIp", - LogFieldEIPID: eipID, - }) - return retry.OnError(backoff.Backoff(backoff.DefaultKey), func(err error) bool { - return apiErr.ErrAssert(apiErr.ErrTaskConflict, err) - }, func() error { - start := time.Now() - resp, err := a.ClientSet.VPC().AddCommonBandwidthPackageIp(req) - metric.OpenAPILatency.WithLabelValues("AddCommonBandwidthPackageIp", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - l.WithFields(map[string]interface{}{ - LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("add eip failed, %s", err.Error()) - return err - } - l.WithFields(map[string]interface{}{ - LogFieldRequestID: resp.RequestId}).Info("add eip success") - return nil - }) -} - -// RemoveCommonBandwidthPackageIP remove EIP from bandwidth package -func (a *OpenAPI) RemoveCommonBandwidthPackageIP(eipID, packageID string) error { - req := vpc.CreateRemoveCommonBandwidthPackageIpRequest() - req.BandwidthPackageId = packageID - req.IpInstanceId = eipID - - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "RemoveCommonBandwidthPackageIp", - LogFieldEIPID: eipID, - }) - - return retry.OnError(backoff.Backoff(backoff.DefaultKey), func(err error) bool { - return apiErr.ErrAssert(apiErr.ErrTaskConflict, err) - }, func() error { - start := time.Now() - resp, err := a.ClientSet.VPC().RemoveCommonBandwidthPackageIp(req) - metric.OpenAPILatency.WithLabelValues("RemoveCommonBandwidthPackageIp", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - l.WithFields(map[string]interface{}{ - LogFieldRequestID: apiErr.ErrRequestID(err)}).Warnf("remove eip failed, %s", err.Error()) - return err - } - l.WithFields(map[string]interface{}{ - LogFieldRequestID: resp.RequestId}).Info("remove eip success") - return nil - }) -} diff --git a/pkg/aliyun/client/vsw_default.go b/pkg/aliyun/client/vsw_default.go index 52ed69a9..ac67ef2e 100644 --- a/pkg/aliyun/client/vsw_default.go +++ b/pkg/aliyun/client/vsw_default.go @@ -7,9 +7,11 @@ import ( "fmt" "time" + "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" + logf "sigs.k8s.io/controller-runtime/pkg/log" + apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" "github.com/AliyunContainerService/terway/pkg/metric" - "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" ) // DescribeVSwitchByID get vsw by id @@ -17,19 +19,18 @@ func (a *OpenAPI) DescribeVSwitchByID(ctx context.Context, vSwitchID string) (*v req := vpc.CreateDescribeVSwitchesRequest() req.VSwitchId = vSwitchID - l := log.WithFields(map[string]interface{}{ - LogFieldAPI: "DescribeVSwitches", - LogFieldVSwitchID: vSwitchID, - }) + l := logf.FromContext(ctx).WithValues( + LogFieldAPI, "DescribeVSwitches", + LogFieldVSwitchID, vSwitchID, + ) start := time.Now() resp, err := a.ClientSet.VPC().DescribeVSwitches(req) metric.OpenAPILatency.WithLabelValues("DescribeVSwitches", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) if err != nil { - l.WithField(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err) + l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "DescribeVSwitches failed") return nil, err } - l.WithField(LogFieldRequestID, resp.RequestId).Debugf("DescribeVSwitches: vsw slice = %+v, err = %v", resp.VSwitches.VSwitch, err) if len(resp.VSwitches.VSwitch) == 0 { return nil, apiErr.ErrNotFound } diff --git a/pkg/aliyun/credential/types.go b/pkg/aliyun/credential/types.go index e3a651dd..be8208c3 100644 --- a/pkg/aliyun/credential/types.go +++ b/pkg/aliyun/credential/types.go @@ -3,8 +3,9 @@ package credential import ( "time" - "github.com/AliyunContainerService/terway/pkg/logger" "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth" + + "github.com/AliyunContainerService/terway/pkg/logger" ) var log = logger.DefaultLogger.WithField("subSys", "credential") diff --git a/pkg/aliyun/eip.go b/pkg/aliyun/eip.go deleted file mode 100644 index 9aee7359..00000000 --- a/pkg/aliyun/eip.go +++ /dev/null @@ -1,315 +0,0 @@ -package aliyun - -import ( - "context" - "fmt" - "net" - "strconv" - "time" - - "github.com/AliyunContainerService/terway/pkg/aliyun/client" - apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" - "github.com/AliyunContainerService/terway/pkg/backoff" - "github.com/AliyunContainerService/terway/pkg/ip" - "github.com/AliyunContainerService/terway/pkg/metric" - "github.com/AliyunContainerService/terway/types" - - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" - "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/util/wait" -) - -func (e *Impl) AllocateEipAddress(ctx context.Context, bandwidth int, chargeType types.InternetChargeType, eipID, eniID string, eniIP net.IP, allowRob bool, isp, bandwidthPackageID, eipPoolID string) (*types.EIP, error) { - var ( - eipInfo *types.EIP - err error - ) - var eni *client.NetworkInterface - eni, err = e.WaitForNetworkInterface(ctx, eniID, "", backoff.Backoff(backoff.ENIOps), false) - if err != nil { - return nil, err - } - eipBindedPrivateIP := eniIP.String() - // get privateIP form eip , if privateIP is eni's primary ip , it's empty - if eni.PrivateIPAddress == eniIP.String() { - eipBindedPrivateIP = "" - } - - if eipID == "" { - var eips []vpc.EipAddress - eips, err = e.describeEipAddresses("", eniID) - if err != nil { - return nil, err - } - - // 1. check if eni have bind eip already,if true, use it directly - if len(eips) > 0 { - for _, eip := range eips { - if eip.PrivateIpAddress == eipBindedPrivateIP { - publicIP, err := ip.ToIP(eip.IpAddress) - if err != nil { - return nil, fmt.Errorf("eip %s, %w", eip.AllocationId, err) - } - log.WithFields(map[string]interface{}{ - client.LogFieldEIPID: eip.AllocationId, - client.LogFieldENIID: eniID, - client.LogFieldPrivateIP: eniIP, - }).Infof("resuse binded eip") - return &types.EIP{ - ID: eip.AllocationId, - Address: publicIP, - AssociateENI: eniID, - AssociateENIIP: eniIP, - }, nil - } - } - } - // 2. create eip and bind to eni - var resp *vpc.AllocateEipAddressResponse - resp, err = e.AllocateEIPAddress(strconv.Itoa(bandwidth), string(chargeType), isp) - if err != nil { - return nil, err - } - eipAddress := net.ParseIP(resp.EipAddress) - if eipAddress == nil { - return nil, fmt.Errorf("invalid eip address %s, %s", resp.AllocationId, resp.EipAddress) - } - eipInfo = &types.EIP{ - ID: resp.AllocationId, - Address: eipAddress, - Delete: true, - AssociateENI: eniID, - AssociateENIIP: eniIP, - } - - defer func() { - if err != nil { - err = e.ReleaseEipAddress(ctx, eipInfo.ID, eniID, eniIP) - if err != nil { - log.Errorf("error rollback eip: %+v, %+v, may cause eip leak...", resp.AllocationId, resp.EipAddress) - } - } - }() - - // add eip to bandwidth package - if bandwidthPackageID != "" { - err = e.AddCommonBandwidthPackageIP(eipInfo.ID, bandwidthPackageID) - if err != nil { - return nil, err - } - } - - } else { - var eips []vpc.EipAddress - eips, err = e.describeEipAddresses(eipID, "") - if err != nil { - return nil, err - } - if len(eips) == 0 { - return nil, fmt.Errorf("can not found eip %s", eipID) - } - // 1. check this eip is bind as expected - eip := eips[0] - var eipAddress net.IP - eipAddress, err = ip.ToIP(eip.IpAddress) - if err != nil { - return nil, fmt.Errorf("eip %s, %w", eip.AllocationId, err) - } - - eipInfo = &types.EIP{ - ID: eipID, - Address: eipAddress, - AssociateENI: eniID, - AssociateENIIP: eniIP, - } - - // check eni is already bind the same eip - if eip.InstanceId == eniID && - eip.PrivateIpAddress == eipBindedPrivateIP { - return eipInfo, nil - } - - if allowRob && eip.Status == eipStatusInUse { - err = e.UnAssociateEIPAddress(eipID, "", "") - if err != nil { - return nil, fmt.Errorf("error unassocicate previous eip address, %v", err) - } - time.Sleep(3 * time.Second) - - start := time.Now() - _, err = e.WaitForEIP(eipID, eipStatusAvailable, backoff.Backoff(backoff.WaitENIStatus)) - metric.OpenAPILatency.WithLabelValues("UnassociateEipAddress/Async", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - return nil, fmt.Errorf("error wait for eip to status Available: %v", err) - } - } else if eip.Status != eipStatusAvailable { - err = fmt.Errorf("eip id: %v status is not Available", eipID) - return nil, err - } - } - logrus.Debugf("get eip info: %+v", eipInfo) - - // bind eip to eni/secondary address - err = e.AssociateEIPAddress(eipInfo.ID, eniID, eniIP.String()) - if err != nil { - err = fmt.Errorf("error associate eip:%v to eni:%v.%v, err: %v", eipInfo, eniID, eniIP, err) - return nil, err - } - time.Sleep(3 * time.Second) - - start := time.Now() - _, err = e.WaitForEIP(eipInfo.ID, "InUse", backoff.Backoff(backoff.WaitENIStatus)) - metric.OpenAPILatency.WithLabelValues("AssociateEipAddress/Async", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - return nil, fmt.Errorf("wait for eip error: %v", err) - } - return eipInfo, nil -} - -// UnassociateEipAddress un associate eip -// 1. if eni is deleted eip auto unassociated -// 2. if eip is deleted , return code is InvalidAllocationId.NotFound -// 3. if eip is not bind ,return code is IncorrectEipStatus -func (e *Impl) UnassociateEipAddress(ctx context.Context, eipID, eniID, eniIP string) error { - var innerErr error - err := wait.ExponentialBackoff(backoff.Backoff(backoff.ENIOps), - func() (done bool, err error) { - // we check eip binding is not changed - var eips []vpc.EipAddress - eips, innerErr = e.describeEipAddresses(eipID, "") - if innerErr != nil { - return false, nil - } - if len(eips) == 0 { - return true, nil - } - eip := eips[0] - // eip is bind to other ins - if eip.InstanceId != eniID { - return true, nil - } - // eip bind to eni primary address, the eip.PrivateIPAddress is empty - if eip.PrivateIpAddress != "" { - if eniIP != eip.PrivateIpAddress { - return true, nil - } - } - - innerErr = e.UnAssociateEIPAddress(eipID, eniID, eniIP) - if innerErr != nil { - return false, nil - } - return true, nil - }, - ) - if err != nil { - return fmt.Errorf("error unassociate eip address: %v, %v, %v,%v,%w", eipID, eniID, eniIP, innerErr, err) - } - return nil -} - -func (e *Impl) ReleaseEipAddress(ctx context.Context, eipID, eniID string, eniIP net.IP) error { - eip, err := e.WaitForEIP(eipID, "", backoff.Backoff(backoff.WaitENIStatus)) - if err != nil { - return fmt.Errorf("error release eip: %w", err) - } - // eip already released - if eip == nil { - return nil - } - logrus.Infof("got eip info to release: %+v", eip) - if eip.Status != eipStatusAvailable { - // detach eip from specify eni - err = e.UnassociateEipAddress(ctx, eipID, eniID, eniIP.String()) - if err == nil { - time.Sleep(3 * time.Second) - start := time.Now() - eip, err = e.WaitForEIP(eipID, eipStatusAvailable, backoff.Backoff(backoff.WaitENIStatus)) - metric.OpenAPILatency.WithLabelValues("UnassociateEipAddress/Async", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - logrus.Errorf("wait timeout UnassociateEipAddress for eni: %v, %v, %v", eniID, eniIP, err) - } - } else { - logrus.Errorf("error UnassociateEipAddress for eni: %v, %v, %v", eniID, eniIP, err) - } - } - var innerErr error - if eip.Status == eipStatusAvailable { - if eip.BandwidthPackageId != "" { - err = e.RemoveCommonBandwidthPackageIP(eip.AllocationId, eip.BandwidthPackageId) - if err != nil { - if !apiErr.ErrAssert(apiErr.ErrIPNotInCbwp, err) { - return err - } - } - } - err = wait.ExponentialBackoff(backoff.Backoff(backoff.ENIRelease), func() (done bool, err error) { - innerErr = e.ReleaseEIPAddress(eip.AllocationId) - if innerErr != nil { - return false, nil - } - return true, nil - }) - if err != nil { - return fmt.Errorf("%w, %s", err, innerErr) - } - } - return err -} - -// WaitForEIP wait status of eni, ignore status if is empty -func (e *Impl) WaitForEIP(eipID string, status string, backoff wait.Backoff) (*vpc.EipAddress, error) { - var eip *vpc.EipAddress - var innerErr error - err := wait.ExponentialBackoff(backoff, - func() (done bool, err error) { - var eips []vpc.EipAddress - eips, innerErr = e.describeEipAddresses(eipID, "") - if innerErr != nil { - return false, nil - } - if len(eips) == 0 { - return false, apiErr.ErrNotFound - } - eip = &eips[0] - if status != "" { - if eip.Status != status { - return false, nil - } - } - return true, nil - }, - ) - if err != nil { - err = fmt.Errorf("failed to wait eip %s status %s error: %v %w", eipID, status, innerErr, err) - } - - return eip, err -} - -func (e *Impl) describeEipAddresses(eipID, eniID string) ([]vpc.EipAddress, error) { - req := vpc.CreateDescribeEipAddressesRequest() - req.AllocationId = eipID - if eniID != "" { - req.AssociatedInstanceType = client.EIPInstanceTypeNetworkInterface - req.AssociatedInstanceId = eniID - } - req.PageSize = requests.NewInteger(100) - - l := log.WithFields(map[string]interface{}{client.LogFieldAPI: "DescribeEipAddresses", client.LogFieldEIPID: eipID, client.LogFieldENIID: eniID}) - start := time.Now() - resp, err := e.ClientSet.VPC().DescribeEipAddresses(req) - metric.OpenAPILatency.WithLabelValues("DescribeEipAddresses", fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) - if err != nil { - l.WithFields(map[string]interface{}{client.LogFieldRequestID: apiErr.ErrRequestID(err)}).Warn(err) - return nil, err - } - l.WithFields(map[string]interface{}{client.LogFieldRequestID: resp.RequestId}).Debugf("get eip len %d", len(resp.EipAddresses.EipAddress)) - return resp.EipAddresses.EipAddress, nil -} - -const ( - eipStatusInUse = "InUse" - eipStatusAvailable = "Available" -) diff --git a/pkg/aliyun/eni.go b/pkg/aliyun/eni/eni.go similarity index 68% rename from pkg/aliyun/eni.go rename to pkg/aliyun/eni/eni.go index 7015f41b..33443b9e 100644 --- a/pkg/aliyun/eni.go +++ b/pkg/aliyun/eni/eni.go @@ -1,34 +1,40 @@ -package aliyun +package eni import ( "fmt" "net" + "net/netip" + "github.com/AliyunContainerService/terway/pkg/aliyun/instance" "github.com/AliyunContainerService/terway/pkg/aliyun/metadata" "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/daemon" ) // ENIInfoGetter interface to get eni information type ENIInfoGetter interface { - GetENIByMac(mac string) (*types.ENI, error) - GetENIPrivateAddressesByMAC(mac string) ([]net.IP, error) - GetENIPrivateIPv6AddressesByMAC(mac string) ([]net.IP, error) - GetENIs(containsMainENI bool) ([]*types.ENI, error) + GetENIByMac(mac string) (*daemon.ENI, error) + + GetENIPrivateAddressesByMACv2(mac string) ([]netip.Addr, error) + GetENIPrivateIPv6AddressesByMACv2(mac string) ([]netip.Addr, error) + + GetENIs(containsMainENI bool) ([]*daemon.ENI, error) GetSecondaryENIMACs() ([]string, error) } type ENIMetadata struct { - ipFamily *types.IPFamily + ipv4, ipv6 bool } -func NewENIMetadata(ipFamily *types.IPFamily) *ENIMetadata { +func NewENIMetadata(ipv4, ipv6 bool) *ENIMetadata { return &ENIMetadata{ - ipFamily: ipFamily, + ipv4: ipv4, + ipv6: ipv6, } } -func (e *ENIMetadata) GetENIByMac(mac string) (*types.ENI, error) { - eni := types.ENI{ +func (e *ENIMetadata) GetENIByMac(mac string) (*daemon.ENI, error) { + eni := daemon.ENI{ MAC: mac, } var err error @@ -58,7 +64,7 @@ func (e *ENIMetadata) GetENIByMac(mac string) (*types.ENI, error) { var v6gw net.IP var vSwitchIPv6CIDR *net.IPNet - if e.ipFamily.IPv6 { + if e.ipv6 { v6gw, err = metadata.GetENIV6Gateway(mac) if err != nil { return nil, fmt.Errorf("error get eni ipv6 gateway from metaserver, mac: %s, %w", mac, err) @@ -89,18 +95,18 @@ func (e *ENIMetadata) GetENIByMac(mac string) (*types.ENI, error) { return &eni, nil } -func (e *ENIMetadata) GetENIPrivateAddressesByMAC(mac string) ([]net.IP, error) { - return metadata.GetENIPrivateIPs(mac) +func (e *ENIMetadata) GetENIPrivateAddressesByMACv2(mac string) ([]netip.Addr, error) { + return metadata.GetIPv4ByMac(mac) } -func (e *ENIMetadata) GetENIPrivateIPv6AddressesByMAC(mac string) ([]net.IP, error) { - return metadata.GetENIPrivateIPv6IPs(mac) +func (e *ENIMetadata) GetENIPrivateIPv6AddressesByMACv2(mac string) ([]netip.Addr, error) { + return metadata.GetIPv6ByMac(mac) } -func (e *ENIMetadata) GetENIs(containsMainENI bool) ([]*types.ENI, error) { - var enis []*types.ENI +func (e *ENIMetadata) GetENIs(containsMainENI bool) ([]*daemon.ENI, error) { + var enis []*daemon.ENI - mainENIMac := GetInstanceMeta().PrimaryMAC + mainENIMac := instance.GetInstanceMeta().PrimaryMAC macs, err := metadata.GetENIsMAC() if err != nil { @@ -114,7 +120,7 @@ func (e *ENIMetadata) GetENIs(containsMainENI bool) ([]*types.ENI, error) { } eni, err := e.GetENIByMac(mac) if err != nil { - return nil, fmt.Errorf("error get eni info by mac: %s from metadata, %w", mac, err) + return nil, err } enis = append(enis, eni) } @@ -125,7 +131,7 @@ func (e *ENIMetadata) GetENIs(containsMainENI bool) ([]*types.ENI, error) { func (e *ENIMetadata) GetSecondaryENIMACs() ([]string, error) { var result []string - mainENIMac := GetInstanceMeta().PrimaryMAC + mainENIMac := instance.GetInstanceMeta().PrimaryMAC macs, err := metadata.GetENIsMAC() if err != nil { diff --git a/pkg/aliyun/instance.go b/pkg/aliyun/instance/instance.go similarity index 94% rename from pkg/aliyun/instance.go rename to pkg/aliyun/instance/instance.go index 122c5c3d..a7e2ca2d 100644 --- a/pkg/aliyun/instance.go +++ b/pkg/aliyun/instance/instance.go @@ -1,4 +1,4 @@ -package aliyun +package instance import ( "context" @@ -8,7 +8,6 @@ import ( "github.com/AliyunContainerService/terway/pkg/aliyun/client" "github.com/AliyunContainerService/terway/pkg/aliyun/metadata" "github.com/AliyunContainerService/terway/pkg/logger" - "github.com/AliyunContainerService/terway/pkg/utils" ) var defaultIns *Instance @@ -168,10 +167,10 @@ func GetLimit(client client.ECS, instanceType string) (*Limits, error) { limits.Store(instanceType, &Limits{ Adapters: adapterLimit, TotalAdapters: instanceTypeInfo.EniTotalQuantity, - IPv4PerAdapter: utils.Minimal(ipv4PerAdapter), - IPv6PerAdapter: utils.Minimal(ipv6PerAdapter), - MemberAdapterLimit: utils.Minimal(memberAdapterLimit), - MaxMemberAdapterLimit: utils.Minimal(maxMemberAdapterLimit), + IPv4PerAdapter: max(ipv4PerAdapter, 0), + IPv6PerAdapter: max(ipv6PerAdapter, 0), + MemberAdapterLimit: max(memberAdapterLimit, 0), + MaxMemberAdapterLimit: max(maxMemberAdapterLimit, 0), InstanceBandwidthRx: instanceTypeInfo.InstanceBandwidthRx, InstanceBandwidthTx: instanceTypeInfo.InstanceBandwidthTx, }) diff --git a/pkg/aliyun/metadata/metadata.go b/pkg/aliyun/metadata/metadata.go index 5c5f4cfe..9df89ae3 100644 --- a/pkg/aliyun/metadata/metadata.go +++ b/pkg/aliyun/metadata/metadata.go @@ -7,6 +7,7 @@ import ( "io" "net" "net/http" + "net/netip" "strings" "time" @@ -51,12 +52,12 @@ func getValue(url string) (string, error) { }() resp, err := http.DefaultClient.Get(url) if err != nil { - return "", err + return "", fmt.Errorf("error get url: %s from metaserver. %w", url, err) } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { - return "", apiErr.ErrNotFound + return "", fmt.Errorf("error get url: %s from metaserver, code: %v, %w", url, resp.StatusCode, apiErr.ErrNotFound) } if resp.StatusCode >= http.StatusBadRequest { return "", fmt.Errorf("error get url: %s from metaserver, code: %v", url, resp.StatusCode) @@ -85,12 +86,12 @@ func getArray(url string) ([]string, error) { resp, err := http.DefaultClient.Get(url) if err != nil { - return []string{}, err + return []string{}, fmt.Errorf("error get url: %s from metaserver. %w", url, err) } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { - return nil, apiErr.ErrNotFound + return []string{}, fmt.Errorf("error get url: %s from metaserver, code: %v, %w", url, resp.StatusCode, apiErr.ErrNotFound) } if resp.StatusCode >= http.StatusBadRequest { return []string{}, fmt.Errorf("error get url: %s from metaserver, code: %v", url, resp.StatusCode) @@ -157,6 +158,15 @@ func GetENIPrimaryIP(mac string) (net.IP, error) { return ip.ToIP(addr) } +// GetENIPrimaryAddr by mac +func GetENIPrimaryAddr(mac string) (netip.Addr, error) { + addr, err := getValue(fmt.Sprintf(metadataBase+eniAddrPath, mac)) + if err != nil { + return netip.Addr{}, err + } + return netip.ParseAddr(addr) +} + // GetENIPrivateIPs by mac func GetENIPrivateIPs(mac string) ([]net.IP, error) { addressStrList := &[]string{} @@ -179,6 +189,20 @@ func GetENIPrivateIPs(mac string) ([]net.IP, error) { return ips, nil } +func GetIPv4ByMac(mac string) ([]netip.Addr, error) { + addressStrList := &[]string{} + ipsStr, err := getValue(fmt.Sprintf(metadataBase+eniPrivateIPs, mac)) + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(ipsStr), addressStrList) + if err != nil { + return nil, fmt.Errorf("failed to parse ip, %s, %w", ipsStr, err) + } + + return ip.ToIPAddrs(*addressStrList) +} + // GetENIPrivateIPv6IPs by mac return [2408::28eb] func GetENIPrivateIPv6IPs(mac string) ([]net.IP, error) { ipsStr, err := getValue(fmt.Sprintf(metadataBase+eniPrivateV6IPs, mac)) @@ -204,6 +228,31 @@ func GetENIPrivateIPv6IPs(mac string) ([]net.IP, error) { return ips, nil } +// GetIPv6ByMac by mac return [2408::28eb] +func GetIPv6ByMac(mac string) ([]netip.Addr, error) { + ipsStr, err := getValue(fmt.Sprintf(metadataBase+eniPrivateV6IPs, mac)) + if err != nil { + // metadata return 404 when no ipv6 is allocated + if errors.Is(err, apiErr.ErrNotFound) { + return nil, nil + } + return nil, err + } + ipsStr = strings.ReplaceAll(ipsStr, "[", "") + ipsStr = strings.ReplaceAll(ipsStr, "]", "") + + var result []netip.Addr + for _, addr := range strings.Split(ipsStr, ",") { + i, err := netip.ParseAddr(strings.TrimSpace(addr)) + if err != nil { + return nil, err + } + result = append(result, i) + } + + return result, nil +} + // GetENIGateway return gateway ip by mac func GetENIGateway(mac string) (net.IP, error) { addr, err := getValue(fmt.Sprintf(metadataBase+eniGatewayPath, mac)) @@ -213,6 +262,15 @@ func GetENIGateway(mac string) (net.IP, error) { return ip.ToIP(addr) } +// GetENIGatewayAddr return gateway ip by mac +func GetENIGatewayAddr(mac string) (netip.Addr, error) { + addr, err := getValue(fmt.Sprintf(metadataBase+eniGatewayPath, mac)) + if err != nil { + return netip.Addr{}, err + } + return netip.ParseAddr(addr) +} + // GetVSwitchCIDR return vSwitch cidr by mac func GetVSwitchCIDR(mac string) (*net.IPNet, error) { addr, err := getValue(fmt.Sprintf(metadataBase+eniVSwitchCIDRPath, mac)) @@ -226,6 +284,15 @@ func GetVSwitchCIDR(mac string) (*net.IPNet, error) { return ipNet, nil } +// GetVSwitchPrefix return vSwitch cidr by mac +func GetVSwitchPrefix(mac string) (netip.Prefix, error) { + addr, err := getValue(fmt.Sprintf(metadataBase+eniVSwitchCIDRPath, mac)) + if err != nil { + return netip.Prefix{}, err + } + return netip.ParsePrefix(addr) +} + // GetVSwitchIPv6CIDR return vSwitch cidr by mac func GetVSwitchIPv6CIDR(mac string) (*net.IPNet, error) { addr, err := getValue(fmt.Sprintf(metadataBase+eniVSwitchIPv6CIDRPath, mac)) @@ -239,6 +306,15 @@ func GetVSwitchIPv6CIDR(mac string) (*net.IPNet, error) { return ipNet, nil } +// GetVSwitchIPv6Prefix return vSwitch cidr by mac +func GetVSwitchIPv6Prefix(mac string) (netip.Prefix, error) { + addr, err := getValue(fmt.Sprintf(metadataBase+eniVSwitchIPv6CIDRPath, mac)) + if err != nil { + return netip.Prefix{}, err + } + return netip.ParsePrefix(addr) +} + // GetENIV6Gateway return gateway ip by mac func GetENIV6Gateway(mac string) (net.IP, error) { addr, err := getValue(fmt.Sprintf(metadataBase+eniV6GatewayPath, mac)) @@ -248,6 +324,15 @@ func GetENIV6Gateway(mac string) (net.IP, error) { return ip.ToIP(addr) } +// GetENIV6GatewayAddr return gateway ip by mac +func GetENIV6GatewayAddr(mac string) (netip.Addr, error) { + addr, err := getValue(fmt.Sprintf(metadataBase+eniV6GatewayPath, mac)) + if err != nil { + return netip.Addr{}, err + } + return netip.ParseAddr(addr) +} + // GetENIVSwitchID by mac func GetENIVSwitchID(mac string) (string, error) { return getValue(fmt.Sprintf(metadataBase+eniVSwitchPath, mac)) diff --git a/pkg/backoff/backoff.go b/pkg/backoff/backoff.go index 62be637c..b07bf6a4 100644 --- a/pkg/backoff/backoff.go +++ b/pkg/backoff/backoff.go @@ -12,6 +12,7 @@ const ( ENICreate = "eni_create" ENIOps = "eni_ops" ENIRelease = "eni_release" + ENIIPOps = "eni_ip_ops" WaitENIStatus = "wait_eni_status" WaitPodENIStatus = "wait_podeni_status" MetaAssignPrivateIP = "meta_assign_private_ip" @@ -44,6 +45,12 @@ var backoffMap = map[string]wait.Backoff{ Jitter: 0.5, Steps: 8, }, + ENIIPOps: { + Duration: time.Second * 5, + Factor: 1, + Jitter: 1.3, + Steps: 6, + }, WaitENIStatus: { Duration: time.Second * 5, Factor: 1.5, diff --git a/pkg/controller/delegate/deleg.go b/pkg/controller/delegate/deleg.go index 6a4a3609..d8c51b77 100644 --- a/pkg/controller/delegate/deleg.go +++ b/pkg/controller/delegate/deleg.go @@ -2,17 +2,18 @@ package delegate import ( "context" - "net" + "net/netip" - aliyunClient "github.com/AliyunContainerService/terway/pkg/aliyun/client" - register "github.com/AliyunContainerService/terway/pkg/controller" - "github.com/AliyunContainerService/terway/pkg/controller/common" - "github.com/AliyunContainerService/terway/pkg/controller/node" corev1 "k8s.io/api/core/v1" k8sErr "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + aliyunClient "github.com/AliyunContainerService/terway/pkg/aliyun/client" + register "github.com/AliyunContainerService/terway/pkg/controller" + "github.com/AliyunContainerService/terway/pkg/controller/common" + "github.com/AliyunContainerService/terway/pkg/controller/node" + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" "k8s.io/apimachinery/pkg/util/wait" @@ -135,22 +136,22 @@ func (d *Delegate) DescribeInstanceTypes(ctx context.Context, types []string) ([ return d.defaultClient.DescribeInstanceTypes(ctx, types) } -func (d *Delegate) AssignPrivateIPAddress(ctx context.Context, eniID string, count int, idempotentKey string) ([]net.IP, error) { +func (d *Delegate) AssignPrivateIPAddress(ctx context.Context, eniID string, count int, idempotent string) ([]netip.Addr, error) { //TODO implement me panic("implement me") } -func (d *Delegate) UnAssignPrivateIPAddresses(ctx context.Context, eniID string, ips []net.IP) error { +func (d *Delegate) UnAssignPrivateIPAddresses(ctx context.Context, eniID string, ips []netip.Addr) error { //TODO implement me panic("implement me") } -func (d *Delegate) AssignIpv6Addresses(ctx context.Context, eniID string, count int, idempotentKey string) ([]net.IP, error) { +func (d *Delegate) AssignIpv6Addresses(ctx context.Context, eniID string, count int, idempotentKey string) ([]netip.Addr, error) { //TODO implement me panic("implement me") } -func (d *Delegate) UnAssignIpv6Addresses(ctx context.Context, eniID string, ips []net.IP) error { +func (d *Delegate) UnAssignIpv6Addresses(ctx context.Context, eniID string, ips []netip.Addr) error { //TODO implement me panic("implement me") } diff --git a/pkg/controller/node/node_controller.go b/pkg/controller/node/node_controller.go index 4403fc16..8e403b76 100644 --- a/pkg/controller/node/node_controller.go +++ b/pkg/controller/node/node_controller.go @@ -6,8 +6,10 @@ import ( "reflect" "time" - "github.com/AliyunContainerService/terway/pkg/aliyun" + "k8s.io/client-go/util/retry" + aliyunClient "github.com/AliyunContainerService/terway/pkg/aliyun/client" + "github.com/AliyunContainerService/terway/pkg/aliyun/instance" "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" register "github.com/AliyunContainerService/terway/pkg/controller" "github.com/AliyunContainerService/terway/pkg/controller/common" @@ -16,7 +18,6 @@ import ( "github.com/AliyunContainerService/terway/types" "github.com/AliyunContainerService/terway/types/controlplane" "github.com/AliyunContainerService/terway/types/daemon" - "k8s.io/client-go/util/retry" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" corev1 "k8s.io/api/core/v1" @@ -37,7 +38,7 @@ const controllerName = "node" func init() { register.Add(controllerName, func(mgr manager.Manager, ctrlCtx *register.ControllerCtx) error { - _, err := aliyun.GetLimit(ctrlCtx.AliyunClient, "") + _, err := instance.GetLimit(ctrlCtx.AliyunClient, "") if err != nil { return err } @@ -248,7 +249,7 @@ func (m *ReconcileNode) ensureResourceLimit(ctx context.Context, node *corev1.No if instanceType == "" { return fmt.Errorf("get label %s is empty", corev1.LabelInstanceTypeStable) } - l, err := aliyun.GetLimit(m.aliyun, instanceType) + l, err := instance.GetLimit(m.aliyun, instanceType) if err != nil { return err } @@ -278,7 +279,7 @@ func (m *ReconcileNode) initENIManagerForNode(ctx context.Context, node *corev1. if instanceType == "" { return fmt.Errorf("get label %s is empty", corev1.LabelInstanceTypeStable) } - limit, err := aliyun.GetLimit(m.aliyun, instanceType) + limit, err := instance.GetLimit(m.aliyun, instanceType) if err != nil { return err } diff --git a/pkg/controller/pool/eni_mgr.go b/pkg/controller/pool/eni_mgr.go index 361e8cd3..3ab96921 100644 --- a/pkg/controller/pool/eni_mgr.go +++ b/pkg/controller/pool/eni_mgr.go @@ -20,7 +20,7 @@ import ( "context" "errors" "fmt" - "net" + "net/netip" "strings" "time" @@ -263,19 +263,19 @@ func (m *Manager) DescribeInstanceTypes(ctx context.Context, types []string) ([] panic("implement me") } -func (m *Manager) AssignPrivateIPAddress(ctx context.Context, eniID string, count int, idempotentKey string) ([]net.IP, error) { +func (m *Manager) AssignPrivateIPAddress(ctx context.Context, eniID string, count int, idempotent string) ([]netip.Addr, error) { panic("implement me") } -func (m *Manager) UnAssignPrivateIPAddresses(ctx context.Context, eniID string, ips []net.IP) error { +func (m *Manager) UnAssignPrivateIPAddresses(ctx context.Context, eniID string, ips []netip.Addr) error { panic("implement me") } -func (m *Manager) AssignIpv6Addresses(ctx context.Context, eniID string, count int, idempotentKey string) ([]net.IP, error) { +func (m *Manager) AssignIpv6Addresses(ctx context.Context, eniID string, count int, idempotentKey string) ([]netip.Addr, error) { panic("implement me") } -func (m *Manager) UnAssignIpv6Addresses(ctx context.Context, eniID string, ips []net.IP) error { +func (m *Manager) UnAssignIpv6Addresses(ctx context.Context, eniID string, ips []netip.Addr) error { //TODO implement me panic("implement me") } diff --git a/pkg/controller/vswitch/vswitch.go b/pkg/controller/vswitch/vswitch.go index 6d60e09a..068f358b 100644 --- a/pkg/controller/vswitch/vswitch.go +++ b/pkg/controller/vswitch/vswitch.go @@ -18,6 +18,7 @@ package vswitch import ( "context" + "errors" "fmt" "math/rand" "sort" @@ -29,6 +30,8 @@ import ( "github.com/AliyunContainerService/terway/pkg/aliyun/client" ) +var ErrNoAvailableVSwitch = errors.New("no available vSwitch") + // Switch hole all switch info from both terway config and podNetworking type Switch struct { ID string @@ -128,7 +131,7 @@ func (s *SwitchPool) GetOne(ctx context.Context, client client.VSwitch, zone str return vsw, nil } - return nil, fmt.Errorf("no available vSwitch for zone %s, vswList %v", zone, ids) + return nil, fmt.Errorf("no available vSwitch for zone %s, vswList %v, %w", zone, ids, ErrNoAvailableVSwitch) } // GetByID will get vSwitch info from local store or openAPI diff --git a/pkg/eni/conditiontype_string.go b/pkg/eni/conditiontype_string.go new file mode 100644 index 00000000..22cf18dd --- /dev/null +++ b/pkg/eni/conditiontype_string.go @@ -0,0 +1,26 @@ +// Code generated by "stringer -type=ConditionType -trimprefix=Condition"; DO NOT EDIT. + +package eni + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Full-0] + _ = x[ResourceTypeMismatch-1] + _ = x[NetworkInterfaceMismatch-2] + _ = x[InsufficientVSwitchIP-3] +} + +const _ConditionType_name = "FullResourceTypeMismatchNetworkInterfaceMismatchInsufficientVSwitchIP" + +var _ConditionType_index = [...]uint8{0, 4, 24, 48, 69} + +func (i ConditionType) String() string { + if i < 0 || i >= ConditionType(len(_ConditionType_index)-1) { + return "ConditionType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _ConditionType_name[_ConditionType_index[i]:_ConditionType_index[i+1]] +} diff --git a/pkg/eni/ipstatus_string.go b/pkg/eni/ipstatus_string.go new file mode 100644 index 00000000..8b3f81bd --- /dev/null +++ b/pkg/eni/ipstatus_string.go @@ -0,0 +1,26 @@ +// Code generated by "stringer -type=ipStatus -trimprefix=ipStatus"; DO NOT EDIT. + +package eni + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[ipStatusInit-0] + _ = x[ipStatusValid-1] + _ = x[ipStatusInvalid-2] + _ = x[ipStatusDeleting-3] +} + +const _ipStatus_name = "InitValidInvalidDeleting" + +var _ipStatus_index = [...]uint8{0, 4, 9, 16, 24} + +func (i ipStatus) String() string { + if i < 0 || i >= ipStatus(len(_ipStatus_index)-1) { + return "ipStatus(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _ipStatus_name[_ipStatus_index[i]:_ipStatus_index[i+1]] +} diff --git a/pkg/eni/local.go b/pkg/eni/local.go new file mode 100644 index 00000000..2ba10006 --- /dev/null +++ b/pkg/eni/local.go @@ -0,0 +1,847 @@ +package eni + +//go:generate stringer -type=status -trimprefix=status + +import ( + "context" + "fmt" + "net/netip" + "sort" + "sync" + "time" + + "golang.org/x/time/rate" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" + "github.com/AliyunContainerService/terway/pkg/factory" + "github.com/AliyunContainerService/terway/pkg/tracing" + "github.com/AliyunContainerService/terway/rpc" + "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/daemon" +) + +var _ NetworkInterface = &Local{} +var _ Usage = &Local{} +var _ ReportStatus = &Trunk{} + +type eniStatus int + +const ( + statusInit eniStatus = iota + statusCreating + statusInUse + statusDeleting +) + +var rateLimit = rate.Every(1 * time.Minute / 10) + +var _ ResourceRequest = &LocalIPRequest{} + +type LocalIPRequest struct { + NetworkInterfaceID string + IPv4 netip.Addr + IPv6 netip.Addr +} + +func (l *LocalIPRequest) ResourceType() ResourceType { + return ResourceTypeLocalIP +} + +var _ NetworkResource = &LocalIPResource{} + +type LocalIPResource struct { + PodID string + + ENI daemon.ENI + + IP types.IPSet2 +} + +func (l *LocalIPResource) ResourceType() ResourceType { + return ResourceTypeLocalIP +} + +func (l *LocalIPResource) ToStore() []daemon.ResourceItem { + if l == nil { + return nil + } + r := daemon.ResourceItem{ + Type: daemon.ResourceTypeENIIP, + ID: fmt.Sprintf("%s.%s", l.ENI.MAC, l.IP.String()), + ENIID: l.ENI.ID, + ENIMAC: l.ENI.MAC, + IPv4: l.IP.GetIPv4(), + IPv6: l.IP.GetIPv6(), + } + + return []daemon.ResourceItem{r} +} + +func (l *LocalIPResource) ToRPC() []*rpc.NetConf { + cfg := &rpc.NetConf{ + BasicInfo: &rpc.BasicInfo{ + PodIP: l.IP.ToRPC(), + PodCIDR: l.ENI.VSwitchCIDR.ToRPC(), + GatewayIP: l.ENI.GatewayIP.ToRPC(), + ServiceCIDR: nil, + }, + ENIInfo: &rpc.ENIInfo{ + MAC: l.ENI.MAC, + Trunk: false, + Vid: 0, + GatewayIP: l.ENI.GatewayIP.ToRPC(), + }, + Pod: nil, + IfName: "", + ExtraRoutes: nil, + DefaultRoute: true, + } + + return []*rpc.NetConf{cfg} +} + +type Local struct { + batchSize int + + cap int + allocatingV4, allocatingV6 int + + eni *daemon.ENI + ipAllocInhibitExpireAt time.Time + + eniType string + + enableIPv4, enableIPv6 bool + ipv4, ipv6 Set + rateLimitEni, rateLimitv4, rateLimitv6 *rate.Limiter + + cond *sync.Cond + + status eniStatus + + factory factory.Factory +} + +func NewLocal(eni *daemon.ENI, eniType string, factory factory.Factory, poolConfig *types.PoolConfig) *Local { + l := &Local{ + eni: eni, + batchSize: poolConfig.BatchSize, + cap: poolConfig.MaxIPPerENI, + cond: sync.NewCond(&sync.Mutex{}), + ipv4: make(Set), + ipv6: make(Set), + eniType: eniType, + enableIPv4: poolConfig.EnableIPv4, + enableIPv6: poolConfig.EnableIPv6, + factory: factory, + + rateLimitEni: rate.NewLimiter(rateLimit, 2), + rateLimitv4: rate.NewLimiter(rateLimit, 2), + rateLimitv6: rate.NewLimiter(rateLimit, 2), + } + + return l +} + +// Run initialize the local eni +func (l *Local) Run(ctx context.Context, podResources []daemon.PodResources, wg *sync.WaitGroup) error { + err := l.load(podResources) + if err != nil { + return err + } + + wg.Add(2) + + go func() { + defer wg.Done() + l.factoryAllocWorker(ctx) + }() + go func() { + defer wg.Done() + l.factoryDisposeWorker(ctx) + }() + + go l.notify(ctx) + + go wait.JitterUntil(l.sync, 1*time.Minute, 1.0, true, ctx.Done()) + + return nil +} + +func (l *Local) notify(ctx context.Context) { + select { + case <-ctx.Done(): + l.cond.Broadcast() + } +} + +func (l *Local) load(podResources []daemon.PodResources) error { + if l.eni == nil { + return nil + } + + // sync ips + ipv4, ipv6, err := l.factory.LoadNetworkInterface(l.eni.MAC) + if err != nil { + return err + } + + logf.Log.Info("load eni", "eni", l.eni.ID, "mac", l.eni.MAC, "ipv4", ipv4, "ipv6", ipv6) + + primary, err := netip.ParseAddr(l.eni.PrimaryIP.IPv4.String()) + if err != nil { + return err + } + + for _, v := range ipv4 { + l.ipv4.Add(NewValidIP(v, netip.MustParseAddr(v.String()) == primary)) + } + l.ipv6.PutValid(ipv6...) + l.status = statusInUse + + // allocate to previous pods + for _, podResource := range podResources { + podID := podResource.PodInfo.Namespace + "/" + podResource.PodInfo.Name + + for _, res := range podResource.Resources { + switch res.Type { + case daemon.ResourceTypeENIIP, daemon.ResourceTypeENI: + default: + continue + } + + if res.ENIID != l.eni.ID { + continue + } + + logf.Log.Info("exist pod", "pod", podID, "ipv4", res.IPv4, "ipv6", res.IPv6, "eni", l.eni.ID) + + // belong to this eni + if res.IPv4 != "" { + ip, err := netip.ParseAddr(res.IPv4) + if err != nil { + return &types.Error{ + Code: types.InvalidDataType, + Msg: err.Error(), + R: err, + } + } + v, ok := l.ipv4[ip] + if !ok { + continue + } + v.Allocate(podID) + } + if res.IPv6 != "" { + ip, err := netip.ParseAddr(res.IPv6) + if err != nil { + return &types.Error{ + Code: types.InvalidDataType, + Msg: err.Error(), + R: err, + } + } + v, ok := l.ipv6[ip] + if !ok { + continue + } + v.Allocate(podID) + } + } + } + + return nil +} + +// sync periodically sync the ip from remote +func (l *Local) sync() { + l.cond.L.Lock() + defer l.cond.L.Unlock() + + if l.eni == nil || l.status != statusInUse { + return + } + + ipv4, ipv6, err := l.factory.LoadNetworkInterface(l.eni.MAC) + if err != nil { + return + } + + syncIPLocked(l.ipv4, ipv4) + syncIPLocked(l.ipv6, ipv6) + + l.cond.Broadcast() +} + +func (l *Local) Allocate(ctx context.Context, cni *daemon.CNI, request ResourceRequest) (chan *AllocResp, []Trace) { + if request.ResourceType() != ResourceTypeLocalIP { + return nil, []Trace{{Condition: ResourceTypeMismatch}} + } + + l.cond.L.Lock() + defer l.cond.L.Unlock() + + switch l.status { + case statusDeleting: + return nil, nil + } + + lo, ok := request.(*LocalIPRequest) + if !ok { + return nil, []Trace{{Condition: ResourceTypeMismatch}} + } + + if lo.NetworkInterfaceID != "" && l.eni.ID != lo.NetworkInterfaceID { + return nil, []Trace{{Condition: NetworkInterfaceMismatch}} + } + + log := logf.FromContext(ctx) + + expectV4 := 0 + expectV6 := 0 + + if l.enableIPv4 { + ipv4 := l.ipv4.PeekAvailable(cni.PodID, lo.IPv4) + if ipv4 == nil && len(l.ipv4)+l.allocatingV4 >= l.cap { + return nil, []Trace{{Condition: Full}} + } else if ipv4 == nil { + expectV4 = 1 + } + } + + if l.enableIPv6 { + ipv6 := l.ipv6.PeekAvailable(cni.PodID, lo.IPv6) + if ipv6 == nil && len(l.ipv6)+l.allocatingV6 >= l.cap { + return nil, []Trace{{Condition: Full}} + } else if ipv6 == nil { + expectV6 = 1 + } + } + + if expectV4 > 0 && expectV6 > 0 && l.ipAllocInhibitExpireAt.After(time.Now()) { + log.Info("eni alloc inhibit", "expire", l.ipAllocInhibitExpireAt.String()) + return nil, []Trace{{Condition: InsufficientVSwitchIP, Reason: fmt.Sprintf("alloc inhibit, expire at %s", l.ipAllocInhibitExpireAt.String())}} + } + + l.allocatingV4 += expectV4 + l.allocatingV6 += expectV6 + + l.cond.Broadcast() + + respCh := make(chan *AllocResp) + + go l.allocWorker(ctx, cni, lo, respCh, func() { + // current roll back ip at same time + l.allocatingV4 -= expectV4 + l.allocatingV4 = max(l.allocatingV4, 0) + l.allocatingV6 -= expectV6 + l.allocatingV6 = max(l.allocatingV6, 0) + log.Info("rollback ipv4", "ipv4", expectV4) + }) + + return respCh, nil +} + +// Release take the cni Del request and release resource to pool +func (l *Local) Release(ctx context.Context, cni *daemon.CNI, request ResourceRequest) bool { + if request.ResourceType() != ResourceTypeLocalIP { + return false + } + + l.cond.L.Lock() + defer l.cond.L.Unlock() + + res := request.(*LocalIPResource) + + if l.eni == nil || l.eni.ID != res.ENI.ID { + return false + } + + log := logf.FromContext(ctx) + + if res.IP.IPv4.IsValid() { + l.ipv4.Release(cni.PodID, res.IP.IPv4) + + log.Info("release ipv4", "ipv4", res.IP.IPv4) + } + if res.IP.IPv6.IsValid() { + l.ipv4.Release(cni.PodID, res.IP.IPv6) + + log.Info("release ipv6", "ipv6", res.IP.IPv6) + } + + return true +} + +// Priority for local resource only +func (l *Local) Priority() int { + l.cond.L.Lock() + defer l.cond.L.Unlock() + + // unInitiated eni has the lower priority + prio := 0 + switch l.status { + case statusDeleting: + return -100 + case statusInUse: + prio = 50 + case statusCreating: + prio = 10 + case statusInit: + prio = 0 + } + + if l.enableIPv4 { + prio += len(l.ipv4) + } + if l.enableIPv6 { + prio += len(l.ipv6) + } + return prio +} + +// allocWorker started with each Allocate call +func (l *Local) allocWorker(ctx context.Context, cni *daemon.CNI, request *LocalIPRequest, respCh chan *AllocResp, onErrLocked func()) { + done := make(chan struct{}) + defer close(done) + + l.cond.L.Lock() + defer l.cond.L.Unlock() + + go func() { + select { + case <-ctx.Done(): + l.cond.L.Lock() + l.cond.Broadcast() + l.cond.L.Unlock() + case <-done: + } + }() + + log := logf.FromContext(ctx) + for { + resp := &AllocResp{} + + var ip types.IPSet2 + var ipv4, ipv6 *IP + if l.enableIPv4 { + ipv4 = l.ipv4.PeekAvailable(cni.PodID, request.IPv4) + if ipv4 == nil { + l.cond.Wait() + continue + } + ip.IPv4 = ipv4.ip + } + if l.enableIPv6 { + ipv6 = l.ipv6.PeekAvailable(cni.PodID, request.IPv6) + if ipv6 == nil { + l.cond.Wait() + continue + } + ip.IPv6 = ipv6.ip + } + + resp.NetworkConfigs = append(resp.NetworkConfigs, &LocalIPResource{ + ENI: *l.eni, + IP: ip, + }) + + log.Info("allocWorker got ip", "eni", l.eni.ID, "ipv4", ip.IPv4.String(), "ipv6", ip.IPv6.String()) + + select { + case <-ctx.Done(): + // parent cancel the context, so no need to send the chain + onErrLocked() + + case respCh <- resp: + // mark the ip as allocated + if ipv4 != nil { + ipv4.Allocate(cni.PodID) + } + if ipv6 != nil { + ipv6.Allocate(cni.PodID) + } + } + + return + } +} + +func (l *Local) factoryAllocWorker(ctx context.Context) { + l.cond.L.Lock() + + log := logf.FromContext(ctx) + for { + if log.V(4).Enabled() { + log.V(4).Info("call allocWorker") + } + + select { + case <-ctx.Done(): + l.cond.L.Unlock() + return + default: + } + + if l.allocatingV4 <= 0 && l.allocatingV6 <= 0 { + l.cond.Wait() + continue + } + + switch l.status { + case statusCreating, statusDeleting: + l.cond.Wait() + continue + case statusInit, statusInUse: + } + + // wait a small period + l.cond.L.Unlock() + time.Sleep(300 * time.Millisecond) + l.cond.L.Lock() + + if l.eni == nil { + // create eni + v4Count := min(l.batchSize, max(l.allocatingV4, 1)) + v6Count := min(l.batchSize, l.allocatingV6) + + l.status = statusCreating + l.cond.L.Unlock() + + err := l.rateLimitEni.Wait(ctx) + if err != nil { + log.Error(err, "wait for rate limit failed") + l.cond.L.Lock() + continue + } + eni, ipv4Set, ipv6Set, err := l.factory.CreateNetworkInterface(v4Count, v6Count, l.eniType) + if err != nil { + log.Error(err, "create eni failed") + + l.cond.L.Lock() + l.errorHandleLocked(err) + + l.eni = eni + + // if create failed, mark eni as deleting + if eni != nil { + l.status = statusDeleting + l.cond.Broadcast() + } else { + l.status = statusInit + } + continue + } + + l.cond.L.Lock() + + l.eni = eni + + l.allocatingV4 -= v4Count + l.allocatingV6 -= v6Count + + l.allocatingV4 = max(l.allocatingV4, 0) + l.allocatingV6 = max(l.allocatingV6, 0) + + primary, err := netip.ParseAddr(eni.PrimaryIP.IPv4.String()) + if err == nil { + for _, v := range ipv4Set { + l.ipv4.Add(NewValidIP(v, netip.MustParseAddr(v.String()) == primary)) + } + } + + l.ipv6.PutValid(ipv6Set...) + + l.status = statusInUse + } else { + eniID := l.eni.ID + v4Count := min(l.batchSize, l.allocatingV4) + v6Count := min(l.batchSize, l.allocatingV6) + + if v4Count > 0 { + l.cond.L.Unlock() + + err := l.rateLimitv4.Wait(ctx) + if err != nil { + log.Error(err, "wait for rate limit failed") + l.cond.L.Lock() + continue + } + ipv4Set, err := l.factory.AssignNIPv4(eniID, v4Count, l.eni.MAC) + + l.cond.L.Lock() + + if err != nil { + log.Error(err, "assign ipv4 failed", "eni", eniID) + l.ipv4.PutDeleting(ipv4Set...) + + l.errorHandleLocked(err) + + continue + } + + l.allocatingV4 -= v4Count + l.allocatingV4 = max(l.allocatingV4, 0) + + l.ipv4.PutValid(ipv4Set...) + } + + if v6Count > 0 { + l.cond.L.Unlock() + + err := l.rateLimitv6.Wait(ctx) + if err != nil { + log.Error(err, "wait for rate limit failed") + l.cond.L.Lock() + continue + } + ipv6Set, err := l.factory.AssignNIPv6(eniID, v6Count, l.eni.MAC) + + l.cond.L.Lock() + + if err != nil { + log.Error(err, "assign ipv6 failed", "eni", eniID) + + l.ipv6.PutDeleting(ipv6Set...) + + l.errorHandleLocked(err) + + continue + } + + l.allocatingV6 -= v6Count + l.allocatingV6 = max(l.allocatingV6, 0) + + l.ipv6.PutValid(ipv6Set...) + } + } + + l.cond.Broadcast() + } +} + +func (l *Local) Dispose(n int) int { + l.cond.L.Lock() + defer l.cond.L.Unlock() + + if l.eni == nil || l.status != statusInUse { + return 0 + } + + log := logf.Log.WithValues("eni", l.eni.ID, "mac", l.eni.MAC) + + // 1. check if can dispose the eni + if n >= max(len(l.ipv4), len(l.ipv6)) { + if l.eni.Type != "trunk" && len(l.ipv4.InUse()) == 0 && len(l.ipv6.InUse()) == 0 { + log.Info("dispose eni") + l.status = statusDeleting + l.cond.Broadcast() + return max(len(l.ipv4), len(l.ipv6)) + } + } + + // 2. dispose invalid first + // it is not take into account + for _, v := range l.ipv4 { + if v.InUse() || v.Valid() { + continue + } + log.Info("dispose invalid ipv4", "ip", v.ip.String()) + v.Dispose() + } + for _, v := range l.ipv6 { + if v.InUse() || v.Valid() { + continue + } + log.Info("dispose invalid ipv6", "ip", v.ip.String()) + v.Dispose() + } + + // 3. dispose idle + left := min(len(l.ipv4.Idles()), n) + + for i := 0; i < left; i++ { + for _, v := range l.ipv4 { + if v.InUse() || v.Deleting() { + continue + } + v.Dispose() // small problem for primary ip + break + } + } + + left6 := min(len(l.ipv6.Idles()), n) + + for i := 0; i < left6; i++ { + for _, v := range l.ipv6 { + if v.InUse() || v.Deleting() { + continue + } + v.Dispose() + break + } + } + + return max(left, left6) +} + +func (l *Local) factoryDisposeWorker(ctx context.Context) { + l.cond.L.Lock() + + log := logf.FromContext(ctx) + for { + select { + case <-ctx.Done(): + l.cond.L.Unlock() + return + default: + } + + if l.eni == nil { + l.cond.Wait() + continue + } + + if l.status == statusDeleting { + // remove the eni + + l.cond.L.Unlock() + + err := l.rateLimitEni.Wait(ctx) + if err != nil { + log.Error(err, "wait for rate limit failed") + l.cond.L.Lock() + continue + } + err = l.factory.DeleteNetworkInterface(l.eni.ID) + l.cond.L.Lock() + + if err != nil { + continue + } + + l.eni = nil + l.ipv4 = make(Set) + l.ipv6 = make(Set) + l.status = statusInit + l.ipAllocInhibitExpireAt = time.Time{} + + l.cond.Broadcast() + continue + } + + toDelete4 := l.ipv4.Deleting() + toDelete6 := l.ipv6.Deleting() + + if toDelete4 == nil && toDelete6 == nil { + l.cond.Wait() + continue + } + + if len(toDelete4) > l.batchSize { + toDelete4 = toDelete4[:l.batchSize] + } + if len(toDelete6) > l.batchSize { + toDelete6 = toDelete6[:l.batchSize] + } + + if len(toDelete4) > 0 { + l.cond.L.Unlock() + err := l.factory.UnAssignNIPv4(l.eni.ID, toDelete4, l.eni.MAC) + l.cond.L.Lock() + + if err == nil { + l.ipv4.Delete(toDelete4...) + } + } + + if len(toDelete6) > 0 { + l.cond.L.Unlock() + err := l.factory.UnAssignNIPv6(l.eni.ID, toDelete6, l.eni.MAC) + l.cond.L.Lock() + + if err == nil { + l.ipv6.Delete(toDelete6...) + } + } + } +} + +func (l *Local) errorHandleLocked(err error) { + if err == nil { + return + } + + if apiErr.ErrAssert(apiErr.InvalidVSwitchIDIPNotEnough, err) { + l.ipAllocInhibitExpireAt = time.Now().Add(10 * time.Minute) + } + + return +} + +func (l *Local) Usage() (int, int, error) { + // return idle and inUse resource + l.cond.L.Lock() + defer l.cond.L.Unlock() + + if l.eni == nil || l.status != statusInUse { + return 0, 0, nil + } + + var idles, inUse int + if l.enableIPv4 { + idles = len(l.ipv4.Idles()) + inUse = len(l.ipv4.InUse()) + } else if l.enableIPv6 { + idles = len(l.ipv6.Idles()) + inUse = len(l.ipv6.InUse()) + } + return idles, inUse, nil +} + +func (l *Local) Status() Status { + l.cond.L.Lock() + defer l.cond.L.Unlock() + + s := Status{ + Status: l.status.String(), + Type: l.eniType, + AllocInhibitExpireAt: l.ipAllocInhibitExpireAt.String(), + } + if l.eni == nil { + return s + } + + s.MAC = l.eni.MAC + s.NetworkInterfaceID = l.eni.ID + + usage := make([][]string, 0, len(l.ipv4)+len(l.ipv6)) + for _, v := range l.ipv4 { + usage = append(usage, []string{v.ip.String(), v.podID, v.status.String()}) + } + for _, v := range l.ipv6 { + usage = append(usage, []string{v.ip.String(), v.podID, v.status.String()}) + } + + sort.Slice(usage, func(i, j int) bool { + return usage[i][0] > usage[j][0] + }) + s.Usage = usage + return s +} + +// syncIPLocked will mark ip as invalid , if not found in remote +func syncIPLocked(lo Set, remote []netip.Addr) { + s := sets.New[netip.Addr](remote...) + for _, v := range lo { + if !s.Has(v.ip) { + logf.Log.Info("remote ip gone, mark as invalid", "ip", v.ip.String()) + _ = tracing.RecordNodeEvent(corev1.EventTypeWarning, string(types.NotFoundInMetadata), fmt.Sprintf("Mark as invalid, ip: %s", v.ip.String())) + v.SetInvalid() + } + } +} diff --git a/pkg/eni/local_test.go b/pkg/eni/local_test.go new file mode 100644 index 00000000..a2fe5a6d --- /dev/null +++ b/pkg/eni/local_test.go @@ -0,0 +1,85 @@ +package eni + +import ( + "context" + "net/netip" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/time/rate" + + "github.com/AliyunContainerService/terway/pkg/factory" + "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/daemon" +) + +func NewLocalTest(eni *daemon.ENI, factory factory.Factory, poolConfig *types.PoolConfig) *Local { + l := &Local{ + eni: eni, + batchSize: poolConfig.BatchSize, + cap: poolConfig.MaxIPPerENI, + cond: sync.NewCond(&sync.Mutex{}), + ipv4: make(Set), + ipv6: make(Set), + enableIPv4: poolConfig.EnableIPv4, + enableIPv6: poolConfig.EnableIPv6, + factory: factory, + + rateLimitEni: rate.NewLimiter(100, 100), + rateLimitv4: rate.NewLimiter(100, 100), + rateLimitv6: rate.NewLimiter(100, 100), + } + + return l +} + +func TestLocal_Release_ValidIPv4(t *testing.T) { + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{}) + request := &LocalIPResource{ + ENI: daemon.ENI{ID: "eni-1"}, + IP: types.IPSet2{IPv4: netip.MustParseAddr("192.0.2.1")}, + } + cni := &daemon.CNI{PodID: "pod-1"} + + local.ipv4.Add(NewValidIP(netip.MustParseAddr("192.0.2.1"), false)) + local.ipv4[netip.MustParseAddr("192.0.2.1")].Allocate("pod-1") + + assert.True(t, local.Release(context.Background(), cni, request)) +} + +func TestLocal_Release_ValidIPv6(t *testing.T) { + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{}) + request := &LocalIPResource{ + ENI: daemon.ENI{ID: "eni-1"}, + IP: types.IPSet2{IPv6: netip.MustParseAddr("fd00:46dd:e::1")}, + } + cni := &daemon.CNI{PodID: "pod-1"} + + local.ipv6.Add(NewValidIP(netip.MustParseAddr("fd00:46dd:e::1"), false)) + local.ipv6[netip.MustParseAddr("fd00:46dd:e::1")].Allocate("pod-1") + + assert.True(t, local.Release(context.Background(), cni, request)) +} + +func TestLocal_Release_NilENI(t *testing.T) { + local := NewLocalTest(nil, nil, &types.PoolConfig{}) + request := &LocalIPResource{ + ENI: daemon.ENI{ID: "eni-1"}, + IP: types.IPSet2{IPv4: netip.MustParseAddr("192.0.2.1")}, + } + cni := &daemon.CNI{PodID: "pod-1"} + + assert.False(t, local.Release(context.Background(), cni, request)) +} + +func TestLocal_Release_DifferentENIID(t *testing.T) { + local := NewLocalTest(&daemon.ENI{ID: "eni-1"}, nil, &types.PoolConfig{}) + request := &LocalIPResource{ + ENI: daemon.ENI{ID: "eni-2"}, + IP: types.IPSet2{IPv4: netip.MustParseAddr("192.0.2.1")}, + } + cni := &daemon.CNI{PodID: "pod-1"} + + assert.False(t, local.Release(context.Background(), cni, request)) +} diff --git a/pkg/eni/manager.go b/pkg/eni/manager.go new file mode 100644 index 00000000..5b6cc289 --- /dev/null +++ b/pkg/eni/manager.go @@ -0,0 +1,352 @@ +package eni + +import ( + "context" + "fmt" + "sort" + "sync" + "time" + + "go.uber.org/atomic" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/AliyunContainerService/terway/pkg/k8s" + "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/daemon" +) + +var mgrLog = logf.Log.WithName("eni-manager") + +var ipExhaustiveConditionPeriod = 10 * time.Minute + +type NodeConditionHandler func(status corev1.ConditionStatus, reason, message string) error + +type NodeCondition struct { + factoryIPExhaustive *atomic.Bool + factoryIPExhaustiveTimer *time.Timer + + handler NodeConditionHandler +} + +func (n *NodeCondition) Run() { + for range n.factoryIPExhaustiveTimer.C { + if n.factoryIPExhaustive.Load() { + if n.handler != nil { + if err := n.handler(corev1.ConditionTrue, types.IPResSufficientReason, + fmt.Sprintf("node has sufficient IP or pass the exhaustive period: %v", ipExhaustiveConditionPeriod)); err != nil { + mgrLog.Error(err, "set IPExhaustive condition failed") + } + } + n.factoryIPExhaustive.Store(false) + } + } +} + +func (n *NodeCondition) SetIPExhaustive() { + if n.handler == nil { + return + } + + if !n.factoryIPExhaustive.Load() { + n.factoryIPExhaustive.Store(true) + if err := n.handler(corev1.ConditionFalse, types.IPResInsufficientReason, + fmt.Sprintf("node has insufficient IP")); err != nil { + mgrLog.Error(err, "set IPExhaustive condition failed") + } + n.factoryIPExhaustiveTimer.Reset(ipExhaustiveConditionPeriod) + } +} + +func (n *NodeCondition) UnsetIPExhaustive() { + if n.factoryIPExhaustive.Load() { + n.factoryIPExhaustiveTimer.Reset(0) + } +} + +type Usage interface { + Usage() (int, int, error) +} + +type ReportStatus interface { + Status() Status +} +type NetworkInterface interface { + Allocate(ctx context.Context, cni *daemon.CNI, request ResourceRequest) (chan *AllocResp, []Trace) + Release(ctx context.Context, cni *daemon.CNI, request ResourceRequest) bool + Priority() int + Dispose(n int) int + Run(ctx context.Context, podResources []daemon.PodResources, wg *sync.WaitGroup) error +} + +type ByPriority []NetworkInterface + +func (n ByPriority) Len() int { + return len(n) +} + +func (n ByPriority) Less(i, j int) bool { + return n[i].Priority() > n[j].Priority() +} + +func (n ByPriority) Swap(i, j int) { n[i], n[j] = n[j], n[i] } + +type Manager struct { + sync.RWMutex + networkInterfaces []NetworkInterface + + minIdles int + maxIdles int + total int + + syncPeriod time.Duration + + k8s k8s.Kubernetes + + node *NodeCondition +} + +func (m *Manager) Run(ctx context.Context, wg *sync.WaitGroup, podResources []daemon.PodResources) error { + // 1. load all eni + for _, ni := range m.networkInterfaces { + err := ni.Run(ctx, podResources, wg) + if err != nil { + return err + } + } + + go m.node.Run() + + // 2. start a goroutine to sync pool + if m.syncPeriod > 0 { + go wait.JitterUntil(func() { + m.syncPool(ctx) + }, m.syncPeriod, 1.0, true, ctx.Done()) + } + return nil +} + +// Allocate find the resource manager and send the request to it. +// Caller should roll back the allocated resource if any error happen. +func (m *Manager) Allocate(ctx context.Context, cni *daemon.CNI, req *AllocRequest) (NetworkResources, error) { + result := make([]NetworkResource, 0, len(req.ResourceRequests)) + + resultCh := make(chan NetworkResources) + done := make(chan struct{}) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + // start a goroutine to collect the result + for { + select { + case <-ctx.Done(): + close(done) + return + case resp, ok := <-resultCh: + if !ok { + close(done) + return + } + result = append(result, resp...) + } + } + }() + + wg := sync.WaitGroup{} + + var traces []Trace + + m.Lock() + sort.Sort(ByPriority(m.networkInterfaces)) + + var err error + for _, request := range req.ResourceRequests { + + var ch chan *AllocResp + for _, ni := range m.networkInterfaces { + var tr []Trace + ch, tr = ni.Allocate(ctx, cni, request) + if ch != nil { + break + } + traces = append(traces, tr...) + } + + if ch == nil { + m.Unlock() + // no eni can handle the allocation + for _, t := range traces { + if t.Condition == InsufficientVSwitchIP { + m.node.SetIPExhaustive() + break + } + } + return nil, fmt.Errorf("no eni can handle the allocation") + } + + wg.Add(1) + + go func() { + defer wg.Done() + + select { + case <-ctx.Done(): + break + case resp := <-ch: + // if any err happen, cancel the context + if resp.Err != nil { + err = resp.Err + + cancel() + break + } + resultCh <- resp.NetworkConfigs + } + }() + } + m.Unlock() + + wg.Wait() + + // already send , close it + close(resultCh) + <-done + + if err == nil && ctx.Err() != nil { + err = ctx.Err() + } + + return result, err +} + +// Release find the resource manager and send the request to it. +func (m *Manager) Release(ctx context.Context, cni *daemon.CNI, req *ReleaseRequest) error { + m.RLock() + defer m.RUnlock() + + for _, networkResource := range req.NetworkResources { + for _, ni := range m.networkInterfaces { + ok := ni.Release(ctx, cni, networkResource) + if ok { + if networkResource.ResourceType() == ResourceTypeLocalIP { + m.node.UnsetIPExhaustive() + } + break + } + } + } + + // assume resource is released, as no backend can handle the resource. + return nil +} + +func (m *Manager) Status() []Status { + m.RLock() + defer m.RUnlock() + + var result []Status + + for _, v := range m.networkInterfaces { + s, ok := v.(ReportStatus) + if !ok { + continue + } + result = append(result, s.Status()) + } + + return result +} + +func (m *Manager) syncPool(ctx context.Context) { + m.Lock() + sort.Sort(sort.Reverse(ByPriority(m.networkInterfaces))) + + var idles, inuses int + for _, ni := range m.networkInterfaces { + usage, ok := ni.(Usage) + if !ok { + continue + } + idle, inuse, err := usage.Usage() + if err != nil { + mgrLog.Error(err, "sync pool error") + continue + } + idles += idle + inuses += inuse + } + + toDel := idles - m.maxIdles + if toDel > 0 { + mgrLog.Info("sync pool", "toDel", toDel) + for _, ni := range m.networkInterfaces { + if toDel <= 0 { + break + } + toDel -= ni.Dispose(toDel) + } + } + + m.Unlock() + + if idles+inuses >= m.total { + return + } + + toAdd := m.minIdles - idles + + if toAdd <= 0 { + return + } + + wg := sync.WaitGroup{} + + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + mgrLog.Info("sync pool", "toAdd", toAdd) + + for i := 0; i < toAdd; i++ { + wg.Add(1) + + go func() { + defer wg.Done() + + _, err := m.Allocate(ctx, &daemon.CNI{}, &AllocRequest{ + ResourceRequests: []ResourceRequest{ + &LocalIPRequest{}, + }, + }) + if err != nil { + mgrLog.Error(err, "sync pool error") + } + }() + } + + wg.Wait() +} + +func NewManager(minIdles, maxIdles, total int, syncPeriod time.Duration, networkInterfaces []NetworkInterface, k8s k8s.Kubernetes) *Manager { + if syncPeriod < 2*time.Minute && syncPeriod > 0 { + syncPeriod = 2 * time.Minute + } + + return &Manager{ + networkInterfaces: networkInterfaces, + minIdles: minIdles, + maxIdles: maxIdles, + total: total, + syncPeriod: syncPeriod, + k8s: k8s, + + node: &NodeCondition{ + factoryIPExhaustiveTimer: time.NewTimer(0), + factoryIPExhaustive: atomic.NewBool(true), + handler: k8s.PatchNodeIPResCondition, + }, + } +} diff --git a/pkg/eni/manager_test.go b/pkg/eni/manager_test.go new file mode 100644 index 00000000..e2e058e2 --- /dev/null +++ b/pkg/eni/manager_test.go @@ -0,0 +1,192 @@ +package eni + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/daemon" +) + +var _ NetworkInterface = &timeOut{} + +type timeOut struct { +} + +func (o *timeOut) Allocate(ctx context.Context, cni *daemon.CNI, request ResourceRequest) (chan *AllocResp, []Trace) { + return make(chan *AllocResp), nil +} + +func (o *timeOut) Release(ctx context.Context, cni *daemon.CNI, request ResourceRequest) bool { + return true +} + +func (o *timeOut) Priority() int { + return 0 +} + +func (o *timeOut) Dispose(n int) int { + return 0 +} + +func (o *timeOut) Run(ctx context.Context, podResources []daemon.PodResources, wg *sync.WaitGroup) error { + return nil +} + +var _ NetworkInterface = &success{} + +type success struct{} + +func (s *success) Allocate(ctx context.Context, cni *daemon.CNI, request ResourceRequest) (chan *AllocResp, []Trace) { + ch := make(chan *AllocResp) + + go func() { + ch <- &AllocResp{ + NetworkConfigs: []NetworkResource{ + &LocalIPResource{ + PodID: "", + ENI: daemon.ENI{}, + IP: types.IPSet2{}, + }}, + } + }() + return ch, nil +} + +func (s *success) Release(ctx context.Context, cni *daemon.CNI, request ResourceRequest) bool { + return true +} + +func (s *success) Priority() int { + return 0 +} + +func (s *success) Dispose(n int) int { + return 0 +} + +func (s *success) Run(ctx context.Context, podResources []daemon.PodResources, wg *sync.WaitGroup) error { + return nil +} + +func TestManagerAllocateReturnsResourcesWhenSuccessful(t *testing.T) { + mockNI := &success{} + manager := NewManager(0, 0, 0, 0, []NetworkInterface{mockNI}, &FakeK8s{}) + + resources, err := manager.Allocate(context.Background(), &daemon.CNI{}, &AllocRequest{ + ResourceRequests: []ResourceRequest{&LocalIPRequest{}}, + }) + + assert.Nil(t, err) + assert.NotNil(t, resources) +} + +func TestManagerAllocateReturnsErrorWhenNoBackendCanHandleAllocation(t *testing.T) { + manager := NewManager(0, 0, 0, 0, []NetworkInterface{}, &FakeK8s{}) + + _, err := manager.Allocate(context.Background(), &daemon.CNI{}, &AllocRequest{ + ResourceRequests: []ResourceRequest{&LocalIPRequest{}}, + }) + + assert.NotNil(t, err) +} + +func TestManagerAllocateWithTimeoutWhenAllocationFails(t *testing.T) { + mockNI := &timeOut{} + manager := NewManager(0, 0, 0, 0, []NetworkInterface{mockNI}, &FakeK8s{}) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, err := manager.Allocate(ctx, &daemon.CNI{}, &AllocRequest{ + ResourceRequests: []ResourceRequest{&LocalIPRequest{}}, + }) + assert.NotNil(t, err) +} + +type FakeK8s struct{} + +func (f *FakeK8s) GetLocalPods() ([]*daemon.PodInfo, error) { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) GetPod(ctx context.Context, namespace, name string, cache bool) (*daemon.PodInfo, error) { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) GetServiceCIDR() *types.IPNetSet { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) GetNodeCidr() *types.IPNetSet { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) SetNodeAllocatablePod(count int) error { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) PatchNodeAnnotations(anno map[string]string) error { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) PatchPodIPInfo(info *daemon.PodInfo, ips string) error { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) PatchNodeIPResCondition(status corev1.ConditionStatus, reason, message string) error { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) RecordNodeEvent(eventType, reason, message string) { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) RecordPodEvent(podName, podNamespace, eventType, reason, message string) error { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) GetNodeDynamicConfigLabel() string { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) GetDynamicConfigWithName(ctx context.Context, name string) (string, error) { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) SetCustomStatefulWorkloadKinds(kinds []string) error { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) WaitTrunkReady() (string, error) { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) GetTrunkID() string { + //TODO implement me + panic("implement me") +} + +func (f *FakeK8s) GetClient() client.Client { + //TODO implement me + panic("implement me") +} diff --git a/pkg/eni/remote.go b/pkg/eni/remote.go new file mode 100644 index 00000000..fb61f32d --- /dev/null +++ b/pkg/eni/remote.go @@ -0,0 +1,233 @@ +package eni + +import ( + "context" + "fmt" + "sync" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + podENITypes "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" + "github.com/AliyunContainerService/terway/pkg/backoff" + terwayIP "github.com/AliyunContainerService/terway/pkg/ip" + "github.com/AliyunContainerService/terway/rpc" + "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/daemon" +) + +var _ ResourceRequest = &RemoteIPRequest{} + +type RemoteIPRequest struct{} + +func (l *RemoteIPRequest) ResourceType() ResourceType { + return ResourceTypeRemoteIP +} + +var _ NetworkResource = &RemoteIPResource{} + +type RemoteIPResource struct { + trunkENI daemon.ENI + podENI podENITypes.PodENI +} + +func (l *RemoteIPResource) ToStore() []daemon.ResourceItem { + return nil +} + +func (l *RemoteIPResource) ToRPC() []*rpc.NetConf { + var netConf []*rpc.NetConf + for _, alloc := range l.podENI.Spec.Allocations { + podIP := &rpc.IPSet{} + cidr := &rpc.IPSet{} + gw := &rpc.IPSet{} + if alloc.IPv4 != "" { + podIP.IPv4 = alloc.IPv4 + cidr.IPv4 = alloc.IPv4CIDR + gw.IPv4 = terwayIP.DeriveGatewayIP(alloc.IPv4CIDR) + + if cidr.IPv4 == "" || gw.IPv4 == "" { + return nil + } + } + if alloc.IPv6 != "" { + podIP.IPv6 = alloc.IPv6 + cidr.IPv6 = alloc.IPv6CIDR + gw.IPv6 = terwayIP.DeriveGatewayIP(alloc.IPv6CIDR) + + if cidr.IPv6 == "" || gw.IPv6 == "" { + return nil + } + } + eniInfo := &rpc.ENIInfo{ + MAC: alloc.ENI.MAC, + } + if l.trunkENI.ID != "" { + eniInfo.Trunk = true + eniInfo.MAC = l.trunkENI.MAC + eniInfo.GatewayIP = l.trunkENI.GatewayIP.ToRPC() + + info, ok := l.podENI.Status.ENIInfos[alloc.ENI.ID] + if !ok { + return nil + } + vid := uint32(info.Vid) + eniInfo.Vid = vid + } + + netConf = append(netConf, &rpc.NetConf{ + BasicInfo: &rpc.BasicInfo{ + PodIP: podIP, + PodCIDR: cidr, + GatewayIP: gw, + }, + ENIInfo: eniInfo, + IfName: alloc.Interface, + ExtraRoutes: parseExtraRoute(alloc.ExtraRoutes), + DefaultRoute: alloc.DefaultRoute, + }) + } + return netConf +} + +func (l *RemoteIPResource) ResourceType() ResourceType { + return ResourceTypeRemoteIP +} + +var _ NetworkInterface = &Remote{} + +type Remote struct { + trunkENI *daemon.ENI // for nil , this is not a trunk + client client.Client +} + +func NewRemote(client client.Client, trunkENI *daemon.ENI) *Remote { + return &Remote{ + trunkENI: trunkENI, + client: client, + } +} + +func (r *Remote) Run(ctx context.Context, podResources []daemon.PodResources, wg *sync.WaitGroup) error { + return nil +} + +func (r *Remote) Priority() int { + return 100 +} + +func (r *Remote) Allocate(ctx context.Context, cni *daemon.CNI, request ResourceRequest) (chan *AllocResp, []Trace) { + if request.ResourceType() != ResourceTypeRemoteIP { + return nil, []Trace{{Condition: ResourceTypeMismatch}} + } + + resp := make(chan *AllocResp) + + go func() { + l := logf.FromContext(ctx) + + var podENI *podENITypes.PodENI + var err, innerErr error + err = wait.ExponentialBackoffWithContext(ctx, backoff.Backoff(backoff.WaitPodENIStatus), func(ctx context.Context) (bool, error) { + podENI, innerErr = getPodENI(ctx, r.client, cni.PodNamespace, cni.PodName) + if innerErr != nil { + innerErr = &types.Error{ + Code: types.PodENINotReady, + Msg: "Get PodENI error", + R: innerErr, + } + return false, nil + } + if !podENI.DeletionTimestamp.IsZero() { + innerErr = &types.Error{ + Code: types.PodENINotReady, + Msg: "DeletionTimestamp is not zero", + } + return false, nil + } + if podENI.Status.Phase != podENITypes.ENIPhaseBind { + innerErr = &types.Error{ + Code: types.PodENINotReady, + Msg: fmt.Sprintf("PodENI Phase is %s", podENI.Status.Phase), + } + return false, nil + } + if cni.PodUID != "" { + if podENI.Annotations[types.PodUID] != cni.PodUID { + return false, nil + } + } + if podENI.Status.TrunkENIID != r.trunkENI.ID { + innerErr = &types.Error{ + Code: types.PodENINotReady, + Msg: fmt.Sprintf("PodENI used a different trunk %s", podENI.Status.TrunkENIID), + } + return false, innerErr + } + + if len(podENI.Spec.Allocations) == 0 { + innerErr = &types.Error{ + Code: types.PodENINotReady, + Msg: fmt.Sprintf("PodENI has empty allocations"), + } + return false, innerErr + } + return true, nil + }) + + var networkResources NetworkResources + if podENI != nil { + remote := &RemoteIPResource{ + podENI: *podENI, + } + if r.trunkENI != nil { + remote.trunkENI = *r.trunkENI + } + l.Info("get pod eni success", "uid", podENI.UID) + + networkResources = append(networkResources, remote) + } + + select { + case <-ctx.Done(): + case resp <- &AllocResp{ + Err: err, + NetworkConfigs: networkResources, + }: + } + }() + + return resp, nil +} + +func (r *Remote) Release(ctx context.Context, cni *daemon.CNI, request ResourceRequest) bool { + return false +} + +func (r *Remote) Dispose(n int) int { + return 0 +} + +func getPodENI(ctx context.Context, c client.Client, namespace, name string) (*podENITypes.PodENI, error) { + obj := &podENITypes.PodENI{} + err := c.Get(ctx, k8stypes.NamespacedName{Namespace: namespace, Name: name}, obj, &client.GetOptions{Raw: &metav1.GetOptions{ + ResourceVersion: "0", + }}) + return obj, err +} + +func parseExtraRoute(routes []podENITypes.Route) []*rpc.Route { + if routes == nil { + return nil + } + var res []*rpc.Route + for _, r := range routes { + res = append(res, &rpc.Route{ + Dst: r.Dst, + }) + } + return res +} diff --git a/pkg/eni/remote_test.go b/pkg/eni/remote_test.go new file mode 100644 index 00000000..934b796f --- /dev/null +++ b/pkg/eni/remote_test.go @@ -0,0 +1,70 @@ +package eni + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/daemon" + + podENITypes "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" +) + +func TestToRPC(t *testing.T) { + t.Run("test with valid IPv4 and IPv6 allocations", func(t *testing.T) { + l := &RemoteIPResource{ + podENI: podENITypes.PodENI{ + Spec: podENITypes.PodENISpec{ + Allocations: []podENITypes.Allocation{ + { + IPv4: "192.168.1.1", + IPv4CIDR: "192.168.1.0/24", + IPv6: "fd00:db8::1", + IPv6CIDR: "fd00:db8::/64", + ENI: podENITypes.ENI{ + ID: "eni-11", + MAC: "00:00:00:00:00:00", + }, + Interface: "eth0", + ExtraRoutes: []podENITypes.Route{}, + DefaultRoute: true, + }, + }, + }, + Status: podENITypes.PodENIStatus{ + Phase: "", + InstanceID: "i-123456", + TrunkENIID: "eni-12345678", + Msg: "", + PodLastSeen: metav1.Time{}, + ENIInfos: map[string]podENITypes.ENIInfo{ + "eni-11": {}, + }, + }, + }, + trunkENI: daemon.ENI{ + ID: "eni-12345678", + MAC: "00:00:00:00:00:00", + GatewayIP: types.IPSet{ + IPv4: net.ParseIP("192.168.1.253"), + IPv6: net.ParseIP("fd00:db8::fffe"), + }, + }, + } + + // Call ToRPC and check the result + result := l.ToRPC() + assert.NotNil(t, result) + assert.Equal(t, 1, len(result)) + assert.Equal(t, "192.168.1.1", result[0].BasicInfo.PodIP.IPv4) + assert.Equal(t, "fd00:db8::1", result[0].BasicInfo.PodIP.IPv6) + assert.Equal(t, "192.168.1.0/24", result[0].BasicInfo.PodCIDR.IPv4) + assert.Equal(t, "fd00:db8::/64", result[0].BasicInfo.PodCIDR.IPv6) + assert.Equal(t, "00:00:00:00:00:00", result[0].ENIInfo.MAC) + assert.Equal(t, "eth0", result[0].IfName) + assert.Equal(t, true, result[0].DefaultRoute) + }) +} diff --git a/pkg/eni/status_string.go b/pkg/eni/status_string.go new file mode 100644 index 00000000..e8d24344 --- /dev/null +++ b/pkg/eni/status_string.go @@ -0,0 +1,26 @@ +// Code generated by "stringer -type=eniStatus -trimprefix=eniStatus"; DO NOT EDIT. + +package eni + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[statusInit-0] + _ = x[statusCreating-1] + _ = x[statusInUse-2] + _ = x[statusDeleting-3] +} + +const _status_name = "InitCreatingInUseDeleting" + +var _status_index = [...]uint8{0, 4, 12, 17, 25} + +func (i eniStatus) String() string { + if i < 0 || i >= eniStatus(len(_status_index)-1) { + return "eniStatus(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _status_name[_status_index[i]:_status_index[i+1]] +} diff --git a/pkg/eni/trunk.go b/pkg/eni/trunk.go new file mode 100644 index 00000000..8c5718ca --- /dev/null +++ b/pkg/eni/trunk.go @@ -0,0 +1,71 @@ +package eni + +import ( + "context" + "sync" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/AliyunContainerService/terway/types/daemon" +) + +var _ NetworkInterface = &Trunk{} +var _ Usage = &Trunk{} +var _ ReportStatus = &Trunk{} + +type Trunk struct { + trunkENI *daemon.ENI + + remote *Remote + local *Local +} + +func NewTrunk(client client.Client, local *Local) *Trunk { + return &Trunk{ + trunkENI: local.eni, + local: local, + remote: NewRemote(client, local.eni), + } +} + +func (r *Trunk) Run(ctx context.Context, podResources []daemon.PodResources, wg *sync.WaitGroup) error { + return r.local.Run(ctx, podResources, wg) +} + +func (r *Trunk) Priority() int { + return 100 +} + +func (r *Trunk) Allocate(ctx context.Context, cni *daemon.CNI, request ResourceRequest) (chan *AllocResp, []Trace) { + switch request.ResourceType() { + case ResourceTypeLocalIP: + return r.local.Allocate(ctx, cni, request) + case ResourceTypeRemoteIP: + return r.remote.Allocate(ctx, cni, request) + default: + return nil, []Trace{{Condition: ResourceTypeMismatch}} + } +} + +func (r *Trunk) Release(ctx context.Context, cni *daemon.CNI, request ResourceRequest) bool { + switch request.ResourceType() { + case ResourceTypeLocalIP: + return r.local.Release(ctx, cni, request) + case ResourceTypeRemoteIP: + return r.remote.Release(ctx, cni, request) + default: + return false + } +} + +func (r *Trunk) Dispose(n int) int { + return r.local.Dispose(n) +} + +func (r *Trunk) Status() Status { + return r.local.Status() +} + +func (r *Trunk) Usage() (int, int, error) { + return r.local.Usage() +} diff --git a/pkg/eni/types.go b/pkg/eni/types.go new file mode 100644 index 00000000..e448ff99 --- /dev/null +++ b/pkg/eni/types.go @@ -0,0 +1,259 @@ +package eni + +import ( + "net/netip" + "time" + + "github.com/AliyunContainerService/terway/rpc" + "github.com/AliyunContainerService/terway/types/daemon" +) + +//go:generate stringer -type=ipStatus -trimprefix=ipStatus +//go:generate stringer -type=ConditionType -trimprefix=Condition + +type ipStatus int + +const ( + ipStatusInit ipStatus = iota + ipStatusValid + ipStatusInvalid + ipStatusDeleting +) + +type IP struct { + ip netip.Addr + primary bool + + podID string + + status ipStatus +} + +func (ip *IP) String() string { + if ip == nil { + return "" + } + return ip.ip.String() +} + +func NewIP(ip netip.Addr, primary bool) *IP { + return &IP{ + ip: ip, + primary: primary, + status: ipStatusInit, + } +} + +func NewValidIP(ip netip.Addr, primary bool) *IP { + return &IP{ + ip: ip, + primary: primary, + status: ipStatusValid, + } +} + +func (ip *IP) Primary() bool { + return ip.primary +} + +func (ip *IP) Valid() bool { + return ip.status == ipStatusValid +} + +func (ip *IP) Deleting() bool { + return ip.status == ipStatusDeleting +} + +func (ip *IP) InUse() bool { + return ip.podID != "" +} + +func (ip *IP) Allocate(podID string) { + ip.podID = podID +} + +func (ip *IP) Release(podID string) { + if ip.podID != podID { + return + } + ip.podID = "" +} + +func (ip *IP) Dispose() { + if ip.primary { + return + } + ip.status = ipStatusDeleting +} + +func (ip *IP) SetInvalid() { + ip.status = ipStatusInvalid +} + +func (ip *IP) Allocatable() bool { + return ip.Valid() && !ip.InUse() +} + +type Set map[any]*IP + +func (s Set) Idles() []*IP { + var result []*IP + for _, v := range s { + if !v.InUse() { + result = append(result, v) + } + } + return result +} + +func (s Set) InUse() []*IP { + var result []*IP + for _, v := range s { + if v.InUse() { + result = append(result, v) + } + } + return result +} + +func (s Set) Allocatable() []*IP { + var result []*IP + for _, v := range s { + if v.Allocatable() { + result = append(result, v) + } + } + return result +} + +func (s Set) PeekAvailable(podID string, prefer netip.Addr) *IP { + if podID != "" && prefer.IsValid() { + if v, ok := s[prefer]; ok { + if v.podID == podID { + return v + } + } + } + for _, v := range s { + if v.Allocatable() { + return v + } + } + return nil +} + +func (s Set) Add(ip *IP) { + s[ip.ip] = ip +} + +func (s Set) PutValid(ip ...netip.Addr) { + for _, v := range ip { + s[v] = &IP{ip: v, status: ipStatusValid} + } +} + +func (s Set) PutDeleting(ip ...netip.Addr) { + for _, v := range ip { + s[v] = &IP{ip: v, status: ipStatusDeleting} + } +} + +func (s Set) Delete(ip ...netip.Addr) { + for _, v := range ip { + delete(s, v) + } +} + +func (s Set) Release(podID string, ip netip.Addr) { + i, ok := s[ip] + if ok { + i.Release(podID) + } +} + +func (s Set) Deleting() []netip.Addr { + var result []netip.Addr + for _, v := range s { + if v.Deleting() { + result = append(result, v.ip) + } + } + return result +} + +func (s Set) ByPodID(podID string) *IP { + for _, v := range s { + if v.podID == podID { + return v + } + } + return nil +} + +type ResourceType int + +const ( + ResourceTypeLocalIP = 1 << iota + ResourceTypeVeth + ResourceTypeRemoteIP + ResourceTypeRDMA +) + +type ResourceRequest interface { + ResourceType() ResourceType +} + +// AllocRequest represent a bunch of resource must be met. +type AllocRequest struct { + ResourceRequests []ResourceRequest +} + +type AllocResp struct { + Err error + + NetworkConfigs NetworkResources +} + +type ReleaseRequest struct { + NetworkResources []NetworkResource +} + +type NetworkResources []NetworkResource + +type NetworkResource interface { + ResourceType() ResourceType + + ToRPC() []*rpc.NetConf + + ToStore() []daemon.ResourceItem +} + +type Status struct { + NetworkInterfaceID string + MAC string + Type string + AllocInhibitExpireAt string + + Usage [][]string + Status string +} + +type Trace struct { + Condition ConditionType + Reason string +} + +type Condition struct { + ConditionType ConditionType + Reason string + Last time.Time +} + +type ConditionType int + +const ( + Full ConditionType = iota + ResourceTypeMismatch + NetworkInterfaceMismatch + InsufficientVSwitchIP +) diff --git a/pkg/eni/types_test.go b/pkg/eni/types_test.go new file mode 100644 index 00000000..ec28afc6 --- /dev/null +++ b/pkg/eni/types_test.go @@ -0,0 +1,97 @@ +package eni + +import ( + "net/netip" + "reflect" + "testing" +) + +func Test_syncIPLocked(t *testing.T) { + type args struct { + lo Set + remote []netip.Addr + } + tests := []struct { + name string + args args + expect Set + }{ + { + name: "test invalid ip", + args: args{ + lo: Set{ + netip.MustParseAddr("127.0.0.1"): &IP{ + ip: netip.MustParseAddr("127.0.0.1"), + primary: true, + }, + }, + remote: nil, + }, + expect: Set{ + netip.MustParseAddr("127.0.0.1"): &IP{ + ip: netip.MustParseAddr("127.0.0.1"), + primary: true, + status: ipStatusInvalid, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + syncIPLocked(tt.args.lo, tt.args.remote) + if !reflect.DeepEqual(tt.args.lo, tt.expect) { + t.Errorf("syncIPLocked() = %v, want %v", tt.args.lo, tt.expect) + } + }) + } +} + +func TestSet_Allocatable(t *testing.T) { + tests := []struct { + name string + set Set + expected []*IP + }{ + { + name: "Allocatable IPs", + set: Set{ + netip.MustParseAddr("192.0.2.1"): NewValidIP(netip.MustParseAddr("192.0.2.1"), false), + netip.MustParseAddr("192.0.2.2"): NewValidIP(netip.MustParseAddr("192.0.2.2"), false), + }, + expected: []*IP{ + NewValidIP(netip.MustParseAddr("192.0.2.1"), false), + NewValidIP(netip.MustParseAddr("192.0.2.2"), false), + }, + }, + { + name: "No Allocatable IPs", + set: Set{ + netip.MustParseAddr("192.0.2.1"): &IP{ + ip: netip.MustParseAddr("192.0.2.1"), + primary: false, + status: ipStatusDeleting, + }, + netip.MustParseAddr("192.0.2.2"): &IP{ + ip: netip.MustParseAddr("192.0.2.2"), + primary: false, + status: ipStatusInvalid, + }, + }, + expected: nil, + }, + { + name: "Empty Set", + set: Set{}, + expected: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.set.Allocatable() + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("Allocatable() = %v, want %v", result, tt.expected) + } + }) + } +} diff --git a/pkg/eni/veth.go b/pkg/eni/veth.go new file mode 100644 index 00000000..afa3b91f --- /dev/null +++ b/pkg/eni/veth.go @@ -0,0 +1,96 @@ +package eni + +import ( + "context" + "sync" + + "github.com/AliyunContainerService/terway/pkg/link" + "github.com/AliyunContainerService/terway/rpc" + "github.com/AliyunContainerService/terway/types/daemon" +) + +var _ ResourceRequest = &VethRequest{} + +type VethRequest struct{} + +func (l *VethRequest) ResourceType() ResourceType { + return ResourceTypeVeth +} + +type VethResource struct { + Name string +} + +func (v VethResource) ResourceType() ResourceType { + return ResourceTypeVeth +} + +func (v VethResource) ToRPC() []*rpc.NetConf { + return []*rpc.NetConf{{ + BasicInfo: nil, + ENIInfo: nil, + Pod: nil, + IfName: "", + ExtraRoutes: nil, + DefaultRoute: true, + }} +} + +func (v VethResource) ToStore() []daemon.ResourceItem { + r := daemon.ResourceItem{ + Type: daemon.ResourceTypeVeth, + ID: v.Name, + } + + return []daemon.ResourceItem{r} +} + +var _ NetworkResource = &VethResource{} + +var _ NetworkInterface = &Veth{} + +const ( + defaultPrefix = "cali" +) + +type Veth struct{} + +func (r *Veth) Allocate(ctx context.Context, cni *daemon.CNI, request ResourceRequest) (chan *AllocResp, []Trace) { + if request.ResourceType() != ResourceTypeVeth { + return nil, nil + } + ch := make(chan *AllocResp) + + go func() { + name, _ := link.VethNameForPod(cni.PodName, cni.PodNamespace, "", defaultPrefix) + var nfs []NetworkResource + nfs = append(nfs, &VethResource{Name: name}) + + select { + case <-ctx.Done(): + + case ch <- &AllocResp{ + NetworkConfigs: nfs, + }: + + } + }() + + return ch, nil +} + +func (r *Veth) Release(ctx context.Context, cni *daemon.CNI, request ResourceRequest) bool { + return request.ResourceType() == ResourceTypeVeth +} + +func (r *Veth) Priority() int { + return 0 +} + +func (r *Veth) Dispose(n int) int { + return 0 +} + +func (r *Veth) Run(ctx context.Context, podResources []daemon.PodResources, wg *sync.WaitGroup) error { + return nil +} diff --git a/pkg/factory/aliyun/aliyun.go b/pkg/factory/aliyun/aliyun.go new file mode 100644 index 00000000..120c1f26 --- /dev/null +++ b/pkg/factory/aliyun/aliyun.go @@ -0,0 +1,466 @@ +package aliyun + +import ( + "context" + "net/netip" + "time" + + "github.com/google/uuid" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" + + "github.com/AliyunContainerService/terway/pkg/aliyun/client" + apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" + "github.com/AliyunContainerService/terway/pkg/aliyun/eni" + "github.com/AliyunContainerService/terway/pkg/aliyun/metadata" + "github.com/AliyunContainerService/terway/pkg/backoff" + vswpool "github.com/AliyunContainerService/terway/pkg/controller/vswitch" + "github.com/AliyunContainerService/terway/pkg/factory" + "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/types/daemon" +) + +const ( + metadataPollInterval = time.Second * 1 + metadataWaitTimeout = time.Second * 10 +) + +var _ factory.Factory = &Aliyun{} + +// Aliyun the local eni factory impl for aliyun. +type Aliyun struct { + ctx context.Context + + enableIPv4, enableIPv6 bool + + instanceID string + zoneID string + + openAPI interface { + client.ENI + client.VSwitch + } + getter eni.ENIInfoGetter + + vsw *vswpool.SwitchPool + + vSwitchOptions []string + securityGroupIDs []string + resourceGroupID string + + eniTags map[string]string + + eniTypeAttr bool + eniTagFilter map[string]string +} + +func NewAliyun(ctx context.Context, openAPI *client.OpenAPI, getter eni.ENIInfoGetter, vsw *vswpool.SwitchPool, cfg *types.PoolConfig) *Aliyun { + + return &Aliyun{ + ctx: ctx, + openAPI: openAPI, + getter: getter, + vsw: vsw, + enableIPv4: cfg.EnableIPv4, + enableIPv6: cfg.EnableIPv6, + instanceID: cfg.InstanceID, + zoneID: cfg.ZoneID, + vSwitchOptions: cfg.VSwitchOptions, + securityGroupIDs: cfg.SecurityGroupIDs, + resourceGroupID: cfg.ResourceGroupID, + eniTags: cfg.ENITags, + } +} + +func (a *Aliyun) CreateNetworkInterface(ipv4, ipv6 int, eniType string) (*daemon.ENI, []netip.Addr, []netip.Addr, error) { + ctx, cancel := context.WithTimeout(a.ctx, time.Second*60) + defer cancel() + + // 1. create eni + var eni *client.NetworkInterface + var vswID string + + err := wait.ExponentialBackoffWithContext(a.ctx, backoff.Backoff(backoff.ENICreate), func(ctx context.Context) (bool, error) { + vsw, innerErr := a.vsw.GetOne(ctx, a.openAPI, a.zoneID, a.vSwitchOptions) + if innerErr != nil { + return false, innerErr + } + vswID = vsw.ID + + trunk := false + if eniType == "trunk" { + trunk = true + } + eni, innerErr = a.openAPI.CreateNetworkInterface(ctx, trunk, vswID, a.securityGroupIDs, a.resourceGroupID, ipv4, ipv6, a.eniTags) + if innerErr != nil { + if apiErr.ErrAssert(apiErr.ErrForbidden, innerErr) { + return true, innerErr + } + if apiErr.ErrAssert(apiErr.ErrSecurityGroupInstanceLimitExceed, innerErr) { + return true, innerErr + } + if apiErr.ErrAssert(apiErr.InvalidVSwitchIDIPNotEnough, innerErr) { + a.vsw.Block(vswID) + return false, nil + } + return false, nil + } + + return true, nil + }) + + if err != nil { + return nil, nil, nil, err + } + + r := &daemon.ENI{ + ID: eni.NetworkInterfaceID, + MAC: eni.MacAddress, + VSwitchID: eni.VSwitchID, + Type: eni.Type, + } + + r.PrimaryIP.SetIP(eni.PrivateIPAddress) + + v4Set, err := func() ([]netip.Addr, error) { + var ips []netip.Addr + for _, v := range eni.PrivateIPSets { + addr, err := netip.ParseAddr(v.PrivateIpAddress) + if err != nil { + return nil, err + } + ips = append(ips, addr) + } + return ips, nil + }() + if err != nil { + return r, nil, nil, err + } + + v6Set, err := func() ([]netip.Addr, error) { + var ips []netip.Addr + for _, v := range eni.IPv6Set { + addr, err := netip.ParseAddr(v.Ipv6Address) + if err != nil { + return nil, err + } + ips = append(ips, addr) + } + return ips, nil + }() + if err != nil { + return r, nil, nil, err + } + + // 2. attach eni + err = a.openAPI.AttachNetworkInterface(ctx, eni.NetworkInterfaceID, a.instanceID, "") + if err != nil { + return r, nil, nil, err + } + + // 3. wait metadata ready & update cidr + err = validateIPInMetadata(ctx, v4Set, func() []netip.Addr { + exists, err := metadata.GetIPv4ByMac(r.MAC) + if err != nil { + klog.Errorf("metadata: error get eni private ip: %v", err) + } + return exists + }) + if err != nil { + return r, nil, nil, err + } + + prefix, err := metadata.GetVSwitchCIDR(eni.MacAddress) + if err != nil { + return nil, nil, nil, err + } + r.VSwitchCIDR.SetIPNet(prefix.String()) + + gw, err := metadata.GetENIGatewayAddr(eni.MacAddress) + if err != nil { + return nil, nil, nil, err + } + r.GatewayIP.SetIP(gw.String()) + + if ipv6 > 0 { + prefix, err = metadata.GetVSwitchIPv6CIDR(eni.MacAddress) + if err != nil { + return nil, nil, nil, err + } + r.VSwitchCIDR.SetIPNet(prefix.String()) + + gw, err = metadata.GetENIV6GatewayAddr(eni.MacAddress) + if err != nil { + return nil, nil, nil, err + } + r.GatewayIP.SetIP(gw.String()) + + err = validateIPInMetadata(ctx, v6Set, func() []netip.Addr { + exists, err := metadata.GetIPv4ByMac(r.MAC) + if err != nil { + klog.Errorf("metadata: error get eni private ip: %v", err) + } + return exists + }) + if err != nil { + return r, nil, nil, err + } + } + + return r, v4Set, v6Set, nil +} + +func (a *Aliyun) AssignNIPv4(eniID string, count int, mac string) ([]netip.Addr, error) { + // 1. assign ip + var ips []netip.Addr + var err, innerErr error + idempotentKey := uuid.NewString() + + err = wait.ExponentialBackoffWithContext(a.ctx, backoff.Backoff(backoff.ENIIPOps), func(ctx context.Context) (bool, error) { + ips, innerErr = a.openAPI.AssignPrivateIPAddress(ctx, eniID, count, idempotentKey) + if innerErr != nil { + if apiErr.ErrAssert(apiErr.ErrForbidden, innerErr) { + return true, innerErr + } + if apiErr.ErrAssert(apiErr.InvalidVSwitchIDIPNotEnough, innerErr) { + return true, innerErr + } + return false, nil + } + return true, nil + }) + + if err != nil { + if innerErr != nil { + return ips, innerErr + } + return nil, err + } + + // 2. wait ip ready in metadata + // TODO: support rollback single ip + err = validateIPInMetadata(a.ctx, ips, func() []netip.Addr { + exists, err := metadata.GetIPv4ByMac(mac) + if err != nil { + klog.Errorf("metadata: error get eni private ip: %v", err) + } + return exists + }) + + return ips, err +} + +func (a *Aliyun) AssignNIPv6(eniID string, count int, mac string) ([]netip.Addr, error) { + // 1. assign ip + var ips []netip.Addr + var err, innerErr error + idempotentKey := uuid.NewString() + + err = wait.ExponentialBackoffWithContext(a.ctx, backoff.Backoff(backoff.ENIIPOps), func(ctx context.Context) (bool, error) { + ips, innerErr = a.openAPI.AssignIpv6Addresses(ctx, eniID, count, idempotentKey) + if innerErr != nil { + if apiErr.ErrAssert(apiErr.ErrForbidden, innerErr) { + return true, innerErr + } + if apiErr.ErrAssert(apiErr.InvalidVSwitchIDIPNotEnough, innerErr) { + return true, innerErr + } + return false, nil + } + return true, nil + }) + + if err != nil { + if innerErr != nil { + return ips, innerErr + } + return nil, err + } + + // 2. wait ip ready in metadata + // TODO: support rollback single ip + err = validateIPInMetadata(a.ctx, ips, func() []netip.Addr { + exists, err := metadata.GetIPv6ByMac(mac) + if err != nil { + klog.Errorf("metadata: error get eni private ip: %v", err) + } + return exists + }) + + return ips, err +} + +func (a *Aliyun) UnAssignNIPv4(eniID string, ips []netip.Addr, mac string) error { + var err, innerErr error + + err = wait.ExponentialBackoffWithContext(a.ctx, backoff.Backoff(backoff.ENIIPOps), func(ctx context.Context) (bool, error) { + innerErr = a.openAPI.UnAssignPrivateIPAddresses(ctx, eniID, ips) + if innerErr != nil { + if apiErr.ErrAssert(apiErr.ErrForbidden, innerErr) { + return true, innerErr + } + return false, nil + } + return true, nil + }) + + if err != nil { + if innerErr != nil { + return innerErr + } + return err + } + + err = validateIPNotInMetadata(a.ctx, ips, func() []netip.Addr { + var exists []netip.Addr + exists, innerErr = metadata.GetIPv4ByMac(mac) + if innerErr != nil { + klog.Errorf("metadata: error get eni private ip: %v", innerErr) + return ips + } + return exists + }) + + return err +} + +func (a *Aliyun) UnAssignNIPv6(eniID string, ips []netip.Addr, mac string) error { + var err, innerErr error + + err = wait.ExponentialBackoffWithContext(a.ctx, backoff.Backoff(backoff.ENIIPOps), func(ctx context.Context) (bool, error) { + innerErr = a.openAPI.UnAssignIpv6Addresses(ctx, eniID, ips) + if innerErr != nil { + if apiErr.ErrAssert(apiErr.ErrForbidden, innerErr) { + return true, innerErr + } + return false, nil + } + return true, nil + }) + + if err != nil { + if innerErr != nil { + return innerErr + } + return err + } + + err = validateIPNotInMetadata(a.ctx, ips, func() []netip.Addr { + var exists []netip.Addr + exists, innerErr = metadata.GetIPv6ByMac(mac) + if innerErr != nil { + klog.Errorf("metadata: error get eni private ip: %v", innerErr) + return ips + } + return exists + }) + + return err +} + +func (a *Aliyun) DeleteNetworkInterface(eniID string) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + err := a.openAPI.DetachNetworkInterface(ctx, eniID, a.instanceID, "") + if err != nil { + return err + } + time.Sleep(time.Second * 5) + err = a.openAPI.DeleteNetworkInterface(ctx, eniID) + return err +} + +func (a *Aliyun) LoadNetworkInterface(mac string) (ipv4Set []netip.Addr, ipv6Set []netip.Addr, err error) { + + if a.enableIPv4 { + ipv4Set, err = a.getter.GetENIPrivateAddressesByMACv2(mac) + if err != nil { + return + } + } + + if a.enableIPv6 { + ipv6Set, err = a.getter.GetENIPrivateIPv6AddressesByMACv2(mac) + } + + return +} + +func (a *Aliyun) GetAttachedNetworkInterface(trunkENIID string) ([]*daemon.ENI, error) { + enis, err := a.getter.GetENIs(false) + if err != nil { + return nil, err + } + + if len(enis) == 0 { + return nil, err + } + + var eniIDs []string + var trunkFound bool + idMap := map[string]*daemon.ENI{} + for _, eni := range enis { + eniIDs = append(eniIDs, eni.ID) + idMap[eni.ID] = eni + + if trunkENIID == eni.ID { + eni.Trunk = true + + trunkFound = true + } + } + + var result []*daemon.ENI + + if a.eniTypeAttr || len(a.eniTagFilter) > 0 { + if !trunkFound || len(a.eniTagFilter) > 0 { + var innerErr error + var eniSet []*client.NetworkInterface + err = wait.ExponentialBackoffWithContext(a.ctx, backoff.Backoff(backoff.ENIIPOps), func(ctx context.Context) (bool, error) { + eniSet, innerErr = a.openAPI.DescribeNetworkInterface(ctx, "", eniIDs, "", "", "", a.eniTagFilter) + if apiErr.ErrAssert(apiErr.ErrForbidden, innerErr) { + return true, innerErr + } + return false, nil + }) + if err != nil { + if innerErr != nil { + return nil, innerErr + } + return nil, err + } + for _, eni := range eniSet { + e, ok := idMap[eni.NetworkInterfaceID] + if !ok { + continue + } + e.Trunk = eni.Type == client.ENITypeTrunk + + // take to intersect + result = append(result, e) + } + } else { + result = enis + } + } else { + result = enis + } + return result, nil +} + +func validateIPInMetadata(ctx context.Context, expect []netip.Addr, getExist func() []netip.Addr) error { + return wait.PollUntilContextTimeout(ctx, metadataPollInterval, metadataWaitTimeout, false, func(ctx context.Context) (bool, error) { + exists := getExist() + return sets.New[netip.Addr](exists...).HasAll(expect...), nil + }) +} + +func validateIPNotInMetadata(ctx context.Context, gone []netip.Addr, getExist func() []netip.Addr) error { + return wait.PollUntilContextTimeout(ctx, metadataPollInterval, metadataWaitTimeout, false, func(ctx context.Context) (bool, error) { + exists := getExist() + + return !sets.New[netip.Addr](exists...).HasAny(gone...), nil + }) +} diff --git a/pkg/factory/fake/fake_factory.go b/pkg/factory/fake/fake_factory.go new file mode 100644 index 00000000..205f5944 --- /dev/null +++ b/pkg/factory/fake/fake_factory.go @@ -0,0 +1,154 @@ +package fake + +import ( + "fmt" + "net/netip" + "sync" + + "github.com/google/uuid" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + + "github.com/AliyunContainerService/terway/pkg/factory" + "github.com/AliyunContainerService/terway/types/daemon" +) + +type ENI struct { + ID string + + IPv4 sets.Set[netip.Addr] + IPv6 sets.Set[netip.Addr] +} + +var _ factory.Factory = &FakeFactory{} + +type FakeFactory struct { + sync.Mutex + ENIS map[string]*ENI + + ipv4Next netip.Addr + ipv6Next netip.Addr +} + +func NewFakeFactory() *FakeFactory { + return &FakeFactory{ + ENIS: map[string]*ENI{}, + ipv4Next: netip.MustParseAddr("10.0.0.0"), + ipv6Next: netip.MustParseAddr("fd00::0"), + } +} + +func (f *FakeFactory) CreateNetworkInterface(ipv4, ipv6 int, eniType string) (*daemon.ENI, []netip.Addr, []netip.Addr, error) { + if ipv4 <= 0 { + return nil, nil, nil, fmt.Errorf("ipv4 must be greater than 0") + } + + f.Lock() + defer f.Unlock() + + n := &ENI{ + ID: uuid.NewString(), + IPv4: make(sets.Set[netip.Addr]), + IPv6: make(sets.Set[netip.Addr]), + } + + f.ENIS[n.ID] = n + + var v4, v6 []netip.Addr + for i := 0; i < ipv4; i++ { + ip := f.ipv4Next.Next() + n.IPv4.Insert(ip) + f.ipv4Next = ip + v4 = append(v4, ip) + klog.Infof("fake factory create eni %s ipv4 %s", n.ID, ip.String()) + } + + for i := 0; i < ipv6; i++ { + ip := f.ipv6Next.Next() + n.IPv6.Insert(ip) + f.ipv6Next = ip + v6 = append(v6, ip) + klog.Infof("fake factory create eni %s ipv6 %s", n.ID, ip.String()) + } + + return &daemon.ENI{ID: n.ID}, v4, v6, nil +} + +func (f *FakeFactory) AssignNIPv4(eniID string, count int, mac string) ([]netip.Addr, error) { + f.Lock() + defer f.Unlock() + + n, ok := f.ENIS[eniID] + if !ok { + return nil, fmt.Errorf("eni %s not found", eniID) + } + + var v4 []netip.Addr + for i := 0; i < count; i++ { + ip := f.ipv4Next.Next() + n.IPv4.Insert(ip) + f.ipv4Next = ip + v4 = append(v4, ip) + klog.Infof("fake factory assign eni %s ipv4 %s", n.ID, ip.String()) + } + return v4, nil +} + +func (f *FakeFactory) AssignNIPv6(eniID string, count int, mac string) ([]netip.Addr, error) { + f.Lock() + defer f.Unlock() + + n, ok := f.ENIS[eniID] + if !ok { + return nil, fmt.Errorf("eni %s not found", eniID) + } + + var v6 []netip.Addr + for i := 0; i < count; i++ { + ip := f.ipv6Next.Next() + n.IPv6.Insert(ip) + f.ipv6Next = ip + v6 = append(v6, ip) + klog.Infof("fake factory assign eni %s ipv6 %s", n.ID, ip.String()) + } + return v6, nil +} + +func (f *FakeFactory) DeleteNetworkInterface(eniID string) error { + f.Lock() + defer f.Unlock() + delete(f.ENIS, eniID) + return nil +} + +func (f *FakeFactory) UnAssignNIPv4(eniID string, ips []netip.Addr, mac string) error { + f.Lock() + defer f.Unlock() + + e, ok := f.ENIS[eniID] + if !ok { + return fmt.Errorf("eni not found") + } + e.IPv4.Delete(ips...) + return nil +} + +func (f *FakeFactory) UnAssignNIPv6(eniID string, ips []netip.Addr, mac string) error { + f.Lock() + defer f.Unlock() + + e, ok := f.ENIS[eniID] + if !ok { + return fmt.Errorf("eni not found") + } + e.IPv6.Delete(ips...) + return nil +} + +func (f *FakeFactory) LoadNetworkInterface(mac string) ([]netip.Addr, []netip.Addr, error) { + return nil, nil, nil +} + +func (f *FakeFactory) GetAttachedNetworkInterface(preferTrunkID string) ([]*daemon.ENI, error) { + return nil, nil +} diff --git a/pkg/factory/types.go b/pkg/factory/types.go new file mode 100644 index 00000000..0c384a9b --- /dev/null +++ b/pkg/factory/types.go @@ -0,0 +1,23 @@ +package factory + +import ( + "net/netip" + + "github.com/AliyunContainerService/terway/types/daemon" +) + +type Factory interface { + CreateNetworkInterface(ipv4, ipv6 int, eniType string) (*daemon.ENI, []netip.Addr, []netip.Addr, error) + AssignNIPv4(eniID string, count int, mac string) ([]netip.Addr, error) + AssignNIPv6(eniID string, count int, mac string) ([]netip.Addr, error) + + // UnAssignNIPv4 unassign ip from eni, the primary ip is not allowed to unassign + UnAssignNIPv4(eniID string, ips []netip.Addr, mac string) error + UnAssignNIPv6(eniID string, ips []netip.Addr, mac string) error + + DeleteNetworkInterface(eniID string) error + + LoadNetworkInterface(mac string) ([]netip.Addr, []netip.Addr, error) + + GetAttachedNetworkInterface(preferTrunkID string) ([]*daemon.ENI, error) +} diff --git a/pkg/ip/ip.go b/pkg/ip/ip.go index 89bd5eab..66d8bb86 100644 --- a/pkg/ip/ip.go +++ b/pkg/ip/ip.go @@ -3,6 +3,7 @@ package ip import ( "fmt" "net" + "net/netip" "k8s.io/apimachinery/pkg/util/sets" ) @@ -16,10 +17,10 @@ func ToIP(addr string) (net.IP, error) { return ip, nil } -func ToIPs(addrs []string) ([]net.IP, error) { - var result []net.IP +func ToIPAddrs(addrs []string) ([]netip.Addr, error) { + var result []netip.Addr for _, addr := range addrs { - i, err := ToIP(addr) + i, err := netip.ParseAddr(addr) if err != nil { return nil, err } @@ -28,33 +29,19 @@ func ToIPs(addrs []string) ([]net.IP, error) { return result, nil } -func ToIPMap(addrs []net.IP) map[string]net.IP { - result := make(map[string]net.IP) - - for _, addr := range addrs { - result[addr.String()] = addr - } - - return result -} - func IPv6(ip net.IP) bool { return ip.To4() == nil } -// NetEqual returns true if both IPNet are equal -func NetEqual(ipn1 *net.IPNet, ipn2 *net.IPNet) bool { - if ipn1 == ipn2 { - return true - } - if ipn1 == nil || ipn2 == nil { - return false +func IPs2str(ips []net.IP) []string { + var result []string + for _, ip := range ips { + result = append(result, ip.String()) } - - return ipn1.String() == ipn2.String() + return result } -func IPs2str(ips []net.IP) []string { +func IPAddrs2str(ips []netip.Addr) []string { var result []string for _, ip := range ips { result = append(result, ip.String()) @@ -67,11 +54,6 @@ func IPsIntersect(a []net.IP, b []net.IP) bool { return sets.NewString(IPs2str(a)...).HasAny(IPs2str(b)...) } -// IPsHasAll return true if all b is in a -func IPsHasAll(a []net.IP, b []net.IP) bool { - return sets.NewString(IPs2str(a)...).HasAll(IPs2str(b)...) -} - // DeriveGatewayIP gateway ip from cidr func DeriveGatewayIP(cidr string) string { if cidr == "" { diff --git a/pkg/ip/ip_cilium.go b/pkg/ip/ip_cilium.go index 4d0ba61e..1a168c35 100644 --- a/pkg/ip/ip_cilium.go +++ b/pkg/ip/ip_cilium.go @@ -15,7 +15,6 @@ package ip import ( - "encoding/binary" "math/big" "net" ) @@ -75,37 +74,6 @@ func GetIPAtIndex(ipNet net.IPNet, index int64) net.IP { return nil } -// GetNextIP returns the next IP from the given IP address. If the given IP is -// the last IP of a v4 or v6 range, the same IP is returned. -func GetNextIP(ip net.IP) net.IP { - if ip.Equal(upperIPv4) || ip.Equal(upperIPv6) { - return ip - } - - nextIP := make(net.IP, len(ip)) - switch len(ip) { - case net.IPv4len: - ipU32 := binary.BigEndian.Uint32(ip) - ipU32++ - binary.BigEndian.PutUint32(nextIP, ipU32) - return nextIP - case net.IPv6len: - ipU64 := binary.BigEndian.Uint64(ip[net.IPv6len/2:]) - ipU64++ - binary.BigEndian.PutUint64(nextIP[net.IPv6len/2:], ipU64) - if ipU64 == 0 { - ipU64 = binary.BigEndian.Uint64(ip[:net.IPv6len/2]) - ipU64++ - binary.BigEndian.PutUint64(nextIP[:net.IPv6len/2], ipU64) - } else { - copy(nextIP[:net.IPv6len/2], ip[:net.IPv6len/2]) - } - return nextIP - default: - return ip - } -} - type netWithRange struct { First *net.IP Last *net.IP diff --git a/pkg/ipam/ipam.go b/pkg/ipam/ipam.go deleted file mode 100644 index a89d3f32..00000000 --- a/pkg/ipam/ipam.go +++ /dev/null @@ -1,33 +0,0 @@ -package ipam - -import ( - "context" - "net" - - "github.com/AliyunContainerService/terway/types" - "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" - "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" -) - -// API the interface of ecs operation set -type API interface { - AllocateENI(ctx context.Context, vSwitch string, securityGroup []string, instanceID string, trunk bool, ipCount int, eniTags map[string]string) (*types.ENI, error) - GetAttachedENIs(ctx context.Context, containsMainENI bool, trunkENIID string) ([]*types.ENI, error) - GetSecondaryENIMACs(ctx context.Context) ([]string, error) - GetENIByMac(ctx context.Context, mac string) (*types.ENI, error) - FreeENI(ctx context.Context, eniID string, instanceID string) error - GetENIIPs(ctx context.Context, mac string) ([]net.IP, []net.IP, error) - AssignNIPsForENI(ctx context.Context, eniID, mac string, count int) ([]net.IP, []net.IP, error) - UnAssignIPsForENI(ctx context.Context, eniID, mac string, ipv4s []net.IP, ipv6s []net.IP) error - GetAttachedSecurityGroups(ctx context.Context, instanceID string) ([]string, error) - CheckEniSecurityGroup(ctx context.Context, sgIDs []string) error - DescribeInstanceTypes(ctx context.Context, types []string) ([]ecs.InstanceType, error) - - // FIXME remove vendor for vpc - DescribeVSwitchByID(ctx context.Context, vSwitch string) (*vpc.VSwitch, error) - // EIP - AllocateEipAddress(ctx context.Context, bandwidth int, chargeType types.InternetChargeType, eipID, eniID string, eniIP net.IP, allowRob bool, isp, bandwidthPackageID, poolID string) (*types.EIP, error) - UnassociateEipAddress(ctx context.Context, eipID, eniID, eniIP string) error - ReleaseEipAddress(ctx context.Context, eipID, eniID string, eniIP net.IP) error - QueryEniIDByIP(ctx context.Context, vpcID string, address net.IP) (string, error) -} diff --git a/daemon/k8s.go b/pkg/k8s/k8s.go similarity index 67% rename from daemon/k8s.go rename to pkg/k8s/k8s.go index 5e869e43..af9ff90d 100644 --- a/daemon/k8s.go +++ b/pkg/k8s/k8s.go @@ -1,8 +1,9 @@ -package daemon +package k8s import ( "context" "encoding/json" + "errors" "fmt" "net" "os" @@ -12,8 +13,6 @@ import ( "time" "unicode" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -26,11 +25,11 @@ import ( "k8s.io/client-go/kubernetes/scheme" typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/AliyunContainerService/terway/deviceplugin" - podENITypes "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" "github.com/AliyunContainerService/terway/pkg/backoff" "github.com/AliyunContainerService/terway/pkg/storage" "github.com/AliyunContainerService/terway/pkg/tracing" @@ -41,36 +40,58 @@ import ( ) const ( - podNetworkTypeVPCIP = "VPCIP" - podNetworkTypeVPCENI = "VPCENI" - podNetworkTypeENIMultiIP = "ENIMultiIP" - dbPath = "/var/lib/cni/terway/pod.db" - dbName = "pods" + k8sSystemNamespace = "kube-system" + k8sKubeadmConfigmap = "kubeadm-config" + k8sKubeadmConfigmapNetworking = "MasterConfiguration" + k8sKubeadmConfigmapClusterconfiguration = "ClusterConfiguration" + + podNeedEni = "k8s.aliyun.com/ENI" + podIngressBandwidth = "k8s.aliyun.com/ingress-bandwidth" //deprecated + podEgressBandwidth = "k8s.aliyun.com/egress-bandwidth" //deprecated + + defaultStickTimeForSts = 5 * time.Minute + + dbPath = "/var/lib/cni/terway/pod.db" + dbName = "pods" eventTypeNormal = corev1.EventTypeNormal eventTypeWarning = corev1.EventTypeWarning labelDynamicConfig = "terway-config" + + ConditionFalse = "false" + conditionTrue = "true" +) + +// bandwidth limit unit +const ( + BYTE = 1 << (10 * iota) + KILOBYTE + MEGABYTE + GIGABYTE + TERABYTE +) + +var ( + storageCleanTimeout = 1 * time.Hour + storageCleanPeriod = 5 * time.Minute ) // Kubernetes operation set type Kubernetes interface { - GetLocalPods() ([]*types.PodInfo, error) - GetPod(namespace, name string, cache bool) (*types.PodInfo, error) + GetLocalPods() ([]*daemon.PodInfo, error) + GetPod(ctx context.Context, namespace, name string, cache bool) (*daemon.PodInfo, error) GetServiceCIDR() *types.IPNetSet GetNodeCidr() *types.IPNetSet SetNodeAllocatablePod(count int) error - PatchEipInfo(info *types.PodInfo) error PatchNodeAnnotations(anno map[string]string) error - PatchPodIPInfo(info *types.PodInfo, ips string) error + PatchPodIPInfo(info *daemon.PodInfo, ips string) error PatchNodeIPResCondition(status corev1.ConditionStatus, reason, message string) error - WaitPodENIInfo(info *types.PodInfo) (podEni *podENITypes.PodENI, err error) - GetPodENIInfo(info *types.PodInfo) (podEni *podENITypes.PodENI, err error) RecordNodeEvent(eventType, reason, message string) RecordPodEvent(podName, podNamespace, eventType, reason, message string) error GetNodeDynamicConfigLabel() string - GetDynamicConfigWithName(name string) (string, error) + GetDynamicConfigWithName(ctx context.Context, name string) (string, error) SetCustomStatefulWorkloadKinds(kinds []string) error WaitTrunkReady() (string, error) @@ -79,6 +100,97 @@ type Kubernetes interface { GetClient() client.Client } +// NewK8S return Kubernetes service by pod spec and daemon mode +func NewK8S(daemonMode string, globalConfig *daemon.Config) (Kubernetes, error) { + restConfig := ctrl.GetConfigOrDie() + restConfig.QPS = globalConfig.KubeClientQPS + restConfig.Burst = globalConfig.KubeClientBurst + restConfig.UserAgent = version.UA + + c, err := client.New(restConfig, client.Options{ + Scheme: types.Scheme, + Mapper: types.NewRESTMapper(), + }) + if err != nil { + return nil, err + } + utils.RegisterClients(restConfig) + + nodeName := os.Getenv("NODE_NAME") + if nodeName == "" { + return nil, fmt.Errorf("failed to get NODE_NAME") + } + + daemonNamespace := os.Getenv("POD_NAMESPACE") + if len(daemonNamespace) == 0 { + daemonNamespace = "kube-system" + klog.Info("POD_NAMESPACE is not set in environment variables, use kube-system as default namespace") + } + + node, err := getNode(context.Background(), c, nodeName) + if err != nil { + return nil, fmt.Errorf("error retrieving node spec for '%s': %w", nodeName, err) + } + + var nodeCidr *types.IPNetSet + if daemonMode == daemon.ModeVPC { + // vpc mode not support ipv6 + nodeCidr, err = nodeCidrFromAPIServer(utils.K8sClient, nodeName) + if err != nil { + return nil, fmt.Errorf("error retrieving node cidr for '%s': %w", nodeName, err) + } + } + + storage, err := storage.NewDiskStorage(dbName, utils.NormalizePath(dbPath), serialize, deserialize) + if err != nil { + return nil, fmt.Errorf("error creating storage: %w", err) + } + + broadcaster := record.NewBroadcaster() + source := corev1.EventSource{Component: "terway-daemon"} + recorder := broadcaster.NewRecorder(scheme.Scheme, source) + + sink := &typedv1.EventSinkImpl{ + Interface: typedv1.New(utils.K8sClient.CoreV1().RESTClient()).Events(""), + } + broadcaster.StartRecordingToSink(sink) + + k8sObj := &k8s{ + client: c, + mode: daemonMode, + node: node, + nodeName: nodeName, + nodeCIDR: nodeCidr, + daemonNamespace: daemonNamespace, + storage: storage, + broadcaster: broadcaster, + recorder: recorder, + Locker: &sync.RWMutex{}, + } + + svcCIDR := &types.IPNetSet{} + cidrs := strings.Split(globalConfig.ServiceCIDR, ",") + + for _, cidr := range cidrs { + svcCIDR.SetIPNet(cidr) + } + err = k8sObj.setSvcCIDR(svcCIDR) + if err != nil { + return nil, err + } + + go func() { + for range time.Tick(storageCleanPeriod) { + err := k8sObj.clean() + if err != nil { + klog.Errorf("error cleanup k8s pod local storage, %v", err) + } + } + }() + + return k8sObj, err +} + type k8s struct { client client.Client @@ -88,10 +200,9 @@ type k8s struct { mode string nodeName string daemonNamespace string - nodeCidr *types.IPNetSet + nodeCIDR *types.IPNetSet node *corev1.Node - svcCidr *types.IPNetSet - apiConnTime time.Time + svcCIDR *types.IPNetSet statefulWorkloadKindSet sets.Set[string] sync.Locker @@ -196,30 +307,15 @@ func (k *k8s) setSvcCIDR(svcCidr *types.IPNetSet) error { if svcCidr.IPv4 == nil { svcCidr.IPv4, err = serviceCidrFromAPIServer(k.client) if err != nil { - return errors.Wrap(err, "failed getting service cidr") + return fmt.Errorf("error retrieving service cidr: %w", err) } } - k.svcCidr = svcCidr + k.svcCIDR = svcCidr return nil } -func (k *k8s) PatchEipInfo(info *types.PodInfo) error { - pod, err := getPod(context.Background(), k.client, info.Namespace, info.Name, true) - if err != nil || pod == nil { - return err - } - - if pod.Annotations == nil { - pod.Annotations = map[string]string{} - } - pod.Annotations[podEipAddress] = info.EipInfo.PodEipIP - - annotationPatchStr := fmt.Sprintf(`{"metadata":{"annotations":{"%v":"%v"}}}`, podEipAddress, info.EipInfo.PodEipIP) - return k.client.Patch(context.Background(), pod, client.RawPatch(k8stypes.MergePatchType, []byte(annotationPatchStr))) -} - -func (k *k8s) PatchPodIPInfo(info *types.PodInfo, ips string) error { +func (k *k8s) PatchPodIPInfo(info *daemon.PodInfo, ips string) error { pod, err := getPod(context.Background(), k.client, info.Namespace, info.Name, true) if err != nil || pod == nil { return err @@ -232,48 +328,6 @@ func (k *k8s) PatchPodIPInfo(info *types.PodInfo, ips string) error { return k.client.Patch(context.Background(), pod, client.RawPatch(k8stypes.MergePatchType, []byte(annotationPatchStr))) } -func (k *k8s) WaitPodENIInfo(info *types.PodInfo) (podEni *podENITypes.PodENI, err error) { - err = wait.ExponentialBackoff(backoff.Backoff(backoff.WaitPodENIStatus), func() (bool, error) { - podEni, err = getPodENI(context.Background(), k.client, info.Namespace, info.Name) - if err != nil { - if apierrors.IsNotFound(err) { - // wait pod eni exist - return false, nil - } - return false, errors.Wrapf(err, "error get pod eni info") - } - if podEni.Status.Phase != podENITypes.ENIPhaseBind { - // wait pod eni bind - return false, nil - } - if info.PodUID != "" { - if podEni.Annotations[types.PodUID] != info.PodUID { - return false, nil - } - } - - if !podEni.DeletionTimestamp.IsZero() { - return false, nil - } - return true, nil - }) - return podEni, err -} - -func (k *k8s) GetPodENIInfo(info *types.PodInfo) (podEni *podENITypes.PodENI, err error) { - err = wait.ExponentialBackoff(backoff.Backoff(backoff.WaitPodENIStatus), func() (bool, error) { - podEni, err = getPodENI(context.Background(), k.client, info.Namespace, info.Name) - if err != nil { - if apierrors.IsNotFound(err) { - return true, err - } - return false, fmt.Errorf("error get pod eni info, %w", err) - } - return true, nil - }) - return podEni, err -} - func (k *k8s) WaitTrunkReady() (string, error) { id := "" err := wait.ExponentialBackoff(backoff.Backoff(backoff.DefaultKey), func() (bool, error) { @@ -298,189 +352,185 @@ func (k *k8s) GetClient() client.Client { return k.client } -// newK8S return Kubernetes service by pod spec and daemon mode -func newK8S(daemonMode string, globalConfig *daemon.Config) (Kubernetes, error) { - restConfig := ctrl.GetConfigOrDie() - restConfig.QPS = globalConfig.KubeClientQPS - restConfig.Burst = globalConfig.KubeClientBurst - restConfig.UserAgent = version.UA +func (k *k8s) GetPod(ctx context.Context, namespace, name string, cache bool) (*daemon.PodInfo, error) { + pod, err := getPod(ctx, k.client, namespace, name, cache) + key := utils.PodInfoKey(namespace, name) + if err != nil { + if apierrors.IsNotFound(err) { + obj, innerErr := k.storage.Get(key) + if innerErr == nil { + item := obj.(*storageItem) + return item.Pod, nil + } - c, err := client.New(restConfig, client.Options{ - Scheme: types.Scheme, - Mapper: types.NewRESTMapper(), - }) + if !errors.Is(innerErr, storage.ErrNotFound) { + return nil, innerErr + } + } + return nil, err + } + podInfo := convertPod(k.mode, k.statefulWorkloadKindSet, pod) + item := &storageItem{ + Pod: podInfo, + } + err = k.storage.Put(key, item) if err != nil { return nil, err } - utils.RegisterClients(restConfig) + return podInfo, nil +} - nodeName := os.Getenv("NODE_NAME") - if nodeName == "" { - return nil, fmt.Errorf("failed to get NODE_NAME") - } +func (k *k8s) GetNodeCidr() *types.IPNetSet { + return k.nodeCIDR +} - daemonNamespace := os.Getenv("POD_NAMESPACE") - if len(daemonNamespace) == 0 { - daemonNamespace = "kube-system" - log.Warnf("POD_NAMESPACE is not set in environment variables, use kube-system as default namespace") +func (k *k8s) GetLocalPods() ([]*daemon.PodInfo, error) { + options := metav1.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.nodeName", k.nodeName).String(), + ResourceVersion: "0", } + podList := &corev1.PodList{} + err := k.client.List(context.Background(), podList, &client.ListOptions{Raw: &options}) - node, err := getNode(context.Background(), c, nodeName) if err != nil { - return nil, errors.Wrap(err, "failed getting node") + return nil, fmt.Errorf("error retrieving pod list for '%s': %w", k.nodeName, err) } - - var nodeCidr *types.IPNetSet - if daemonMode == daemonModeVPC { - // vpc mode not support ipv6 - nodeCidr, err = nodeCidrFromAPIServer(utils.K8sClient, nodeName) - if err != nil { - return nil, errors.Wrap(err, "failed getting node cidr") + var ret []*daemon.PodInfo + for _, pod := range podList.Items { + if types.IgnoredByTerway(pod.Labels) { + continue } - } - storage, err := storage.NewDiskStorage(dbName, utils.NormalizePath(dbPath), serialize, deserialize) - if err != nil { - return nil, errors.Wrapf(err, "failed init db storage with path %s and bucket %s", dbPath, dbName) + podInfo := convertPod(k.mode, k.statefulWorkloadKindSet, &pod) + ret = append(ret, podInfo) } - broadcaster := record.NewBroadcaster() - source := corev1.EventSource{Component: "terway-daemon"} - recorder := broadcaster.NewRecorder(scheme.Scheme, source) + return ret, nil +} - sink := &typedv1.EventSinkImpl{ - Interface: typedv1.New(utils.K8sClient.CoreV1().RESTClient()).Events(""), - } - broadcaster.StartRecordingToSink(sink) +func (k *k8s) GetServiceCIDR() *types.IPNetSet { + return k.svcCIDR +} - k8sObj := &k8s{ - client: c, - mode: daemonMode, - node: node, - nodeName: nodeName, - nodeCidr: nodeCidr, - daemonNamespace: daemonNamespace, - storage: storage, - broadcaster: broadcaster, - recorder: recorder, - apiConnTime: time.Now(), - Locker: &sync.RWMutex{}, - } +func (k *k8s) SetNodeAllocatablePod(count int) error { + return nil +} - svcCIDR := &types.IPNetSet{} - cidrs := strings.Split(globalConfig.ServiceCIDR, ",") +// clean up storage +// tag the object as deletion when found pod not exist +// the tagged object will be deleted on secondary scan +func (k *k8s) clean() error { - for _, cidr := range cidrs { - svcCIDR.SetIPNet(cidr) + list, err := k.storage.List() + if err != nil { + return err } - err = k8sObj.setSvcCIDR(svcCIDR) + + localPods, err := k.GetLocalPods() if err != nil { - return nil, err + return fmt.Errorf("error get local pods: %w", err) } + podsMap := make(map[string]*daemon.PodInfo) - go func() { - for range time.Tick(storageCleanPeriod) { - err := k8sObj.clean() - if err != nil { - log.Errorf("error cleanup k8s pod local storage, %v", err) + for _, pod := range localPods { + key := utils.PodInfoKey(pod.Namespace, pod.Name) + podsMap[key] = pod + } + + for _, obj := range list { + item := obj.(*storageItem) + key := utils.PodInfoKey(item.Pod.Namespace, item.Pod.Name) + + if _, exists := podsMap[key]; exists { + if item.deletionTime != nil { + item.deletionTime = nil + if err := k.storage.Put(key, item); err != nil { + return fmt.Errorf("error save storage: %w", err) + } } + continue } - }() - return k8sObj, err -} + if item.deletionTime == nil { + now := time.Now() + item.deletionTime = &now + if err := k.storage.Put(key, item); err != nil { + return fmt.Errorf("error save storage: %w", err) + } + continue + } -func nodeCidrFromAPIServer(client kubernetes.Interface, nodeName string) (*types.IPNetSet, error) { - node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("error retrieving node spec for '%s': %v", nodeName, err) + if time.Since(*item.deletionTime) > storageCleanTimeout { + if err := k.storage.Delete(key); err != nil { + return fmt.Errorf("error delete storage: %w", err) + } + } } - if node.Spec.PodCIDR == "" { - return nil, fmt.Errorf("node %q pod cidr not assigned", nodeName) - } - podCIDR := &types.IPNetSet{} - for _, cidr := range node.Spec.PodCIDRs { - podCIDR.SetIPNet(cidr) - } - podCIDR.SetIPNet(node.Spec.PodCIDR) - - return podCIDR, nil + return nil } -func parseCidr(cidrString string) (*net.IPNet, error) { - _, cidr, err := net.ParseCIDR(cidrString) - return cidr, err +func (k *k8s) RecordNodeEvent(eventType, reason, message string) { + ref := &corev1.ObjectReference{ + Kind: "Node", + Name: k.node.Name, + UID: k.node.UID, + Namespace: "", + } + + k.recorder.Event(ref, eventType, reason, message) } -func serviceCidrFromAPIServer(c client.Client) (*net.IPNet, error) { - kubeadmConfigMap, err := getCM(context.Background(), c, k8sSystemNamespace, k8sKubeadmConfigmap) +func (k *k8s) RecordPodEvent(podName, podNamespace, eventType, reason, message string) error { + pod, err := getPod(context.Background(), k.client, podNamespace, podName, true) if err != nil { - return nil, err + return err } - kubeNetworkingConfig, ok := kubeadmConfigMap.Data[k8sKubeadmConfigmapNetworking] - if !ok { - kubeNetworkingConfig, ok = kubeadmConfigMap.Data[k8sKubeadmConfigmapClusterconfiguration] - if !ok { - return nil, fmt.Errorf("cannot found kubeproxy config for svc cidr") - } + ref := &corev1.ObjectReference{ + Kind: "Pod", + Name: pod.Name, + UID: pod.UID, + Namespace: pod.Namespace, } - configMap := make(map[interface{}]interface{}) + k.recorder.Event(ref, eventType, reason, message) + return nil +} - err = yaml.Unmarshal([]byte(kubeNetworkingConfig), &configMap) - if err != nil { - return nil, errors.Wrapf(err, "error get networking config from configmap") +// GetNodeDynamicConfigLabel returns value with label config +func (k *k8s) GetNodeDynamicConfigLabel() string { + // use node cached in newK8s() + cfgName, ok := k.node.Labels[labelDynamicConfig] + if !ok { + return "" } - if networkingObj, ok := configMap["networking"]; ok { - if networkingMap, ok := networkingObj.(map[interface{}]interface{}); ok { - if svcObj, ok := networkingMap["serviceSubnet"]; ok { - if svcStr, ok := svcObj.(string); ok { - return parseCidr(svcStr) - } - } - } - } - return nil, fmt.Errorf("cannot found kubeproxy config for svc cidr") + return cfgName } -const k8sSystemNamespace = "kube-system" -const k8sKubeadmConfigmap = "kubeadm-config" -const k8sKubeadmConfigmapNetworking = "MasterConfiguration" -const k8sKubeadmConfigmapClusterconfiguration = "ClusterConfiguration" - -const podNeedEni = "k8s.aliyun.com/ENI" -const podIngressBandwidth = "k8s.aliyun.com/ingress-bandwidth" //deprecated -const podEgressBandwidth = "k8s.aliyun.com/egress-bandwidth" //deprecated - -const podWithEip = "k8s.aliyun.com/pod-with-eip" -const eciWithEip = "k8s.aliyun.com/eci-with-eip" // to adopt ask annotation -const podEipBandwidth = "k8s.aliyun.com/eip-bandwidth" -const podEipChargeType = "k8s.aliyun.com/eip-charge-type" -const podEipInternetChargeType = "k8s.aliyun.com/eip-internet-charge-type" // to adopt ask annotation -const podEciEipInstanceID = "k8s.aliyun.com/eci-eip-instanceid" // to adopt ask annotation -const podPodEipInstanceID = "k8s.aliyun.com/pod-eip-instanceid" -const podEipAddress = "k8s.aliyun.com/allocated-eipAddress" -const eipBandwidthPackageID = "k8s.aliyun.com/eip-common-bandwidth-package-id" -const eipISP = "k8s.aliyun.com/eip-isp" -const eipPublicIPAddressPoolID = "k8s.aliyun.com/eip-public-ip-address-pool-id" +// GetDynamicConfigWithName gets the Dynamic Config's content with its ConfigMap name +func (k *k8s) GetDynamicConfigWithName(ctx context.Context, name string) (string, error) { + cfgMap, err := getCM(ctx, k.client, k.daemonNamespace, name) + if err != nil { + return "", err + } -const defaultStickTimeForSts = 5 * time.Minute + content, ok := cfgMap.Data["eni_conf"] + if ok { + return content, nil + } -var ( - storageCleanTimeout = 1 * time.Hour - storageCleanPeriod = 5 * time.Minute -) + return "", errors.New("configmap not included eni_conf") +} func podNetworkType(daemonMode string, pod *corev1.Pod) string { switch daemonMode { - case daemonModeENIMultiIP: - return podNetworkTypeENIMultiIP - case daemonModeVPC: + case daemon.ModeENIMultiIP: + return daemon.PodNetworkTypeENIMultiIP + case daemon.ModeVPC: podAnnotation := pod.GetAnnotations() useENI := false - if needEni, ok := podAnnotation[podNeedEni]; ok && (needEni != "" && needEni != conditionFalse && needEni != "0") { + if needEni, ok := podAnnotation[podNeedEni]; ok && (needEni != "" && needEni != ConditionFalse && needEni != "0") { useENI = true } @@ -492,18 +542,18 @@ func podNetworkType(daemonMode string, pod *corev1.Pod) string { } if useENI { - return podNetworkTypeVPCENI + return daemon.PodNetworkTypeVPCENI } - return podNetworkTypeVPCIP - case daemonModeENIOnly: - return podNetworkTypeVPCENI + return daemon.PodNetworkTypeVPCIP + case daemon.ModeENIOnly: + return daemon.PodNetworkTypeVPCENI } panic(fmt.Errorf("unknown daemon mode %s", daemonMode)) } -func convertPod(daemonMode string, statefulWorkloadKindSet sets.Set[string], pod *corev1.Pod) *types.PodInfo { - pi := &types.PodInfo{ +func convertPod(daemonMode string, statefulWorkloadKindSet sets.Set[string], pod *corev1.Pod) *daemon.PodInfo { + pi := &daemon.PodInfo{ Name: pod.Name, Namespace: pod.Namespace, PodIPs: types.IPSet{}, @@ -535,54 +585,6 @@ func convertPod(daemonMode string, statefulWorkloadKindSet sets.Set[string], pod } } - if eipAnnotation, ok := podAnnotation[podWithEip]; ok && eipAnnotation == conditionTrue { - pi.EipInfo.PodEip = true - pi.EipInfo.PodEipBandWidth = 5 - pi.EipInfo.PodEipChargeType = types.PayByTraffic - } - if eipAnnotation, ok := podAnnotation[eciWithEip]; ok && eipAnnotation == conditionTrue { - pi.EipInfo.PodEip = true - pi.EipInfo.PodEipBandWidth = 5 - pi.EipInfo.PodEipChargeType = types.PayByTraffic - } - - if eipAnnotation, ok := podAnnotation[podEipBandwidth]; ok { - eipBandwidth, err := strconv.Atoi(eipAnnotation) - if err != nil { - log.Errorf("error convert eip bandwidth: %v", eipBandwidth) - } else { - pi.EipInfo.PodEipBandWidth = eipBandwidth - } - } - - if eipAnnotation, ok := podAnnotation[podEipChargeType]; ok { - pi.EipInfo.PodEipChargeType = types.InternetChargeType(eipAnnotation) - } - if eipAnnotation, ok := podAnnotation[podEipInternetChargeType]; ok { - pi.EipInfo.PodEipChargeType = types.InternetChargeType(eipAnnotation) - } - - if eipAnnotation, ok := podAnnotation[podEciEipInstanceID]; ok && eipAnnotation != "" { - pi.EipInfo.PodEip = true - pi.EipInfo.PodEipID = eipAnnotation - } - - if eipAnnotation, ok := podAnnotation[podPodEipInstanceID]; ok && eipAnnotation != "" { - pi.EipInfo.PodEip = true - pi.EipInfo.PodEipID = eipAnnotation - } - - if eipAnnotation, ok := podAnnotation[eipISP]; ok && eipAnnotation != "" { - pi.EipInfo.PodEipISP = eipAnnotation - } - - if eipAnnotation, ok := podAnnotation[eipBandwidthPackageID]; ok && eipAnnotation != "" { - pi.EipInfo.PodEipBandwidthPackageID = eipAnnotation - } - if eipAnnotation, ok := podAnnotation[eipPublicIPAddressPoolID]; ok && eipAnnotation != "" { - pi.EipInfo.PodEipPoolID = eipAnnotation - } - pi.SandboxExited = pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodSucceeded if podENI, ok := podAnnotation[types.PodENI]; ok { @@ -627,55 +629,30 @@ func parseBool(s string) bool { return b } -// bandwidth limit unit -const ( - BYTE = 1 << (10 * iota) - KILOBYTE - MEGABYTE - GIGABYTE - TERABYTE -) - -func parseBandwidth(s string) (uint64, error) { - // when bandwidth is "", return - if len(s) == 0 { - return 0, fmt.Errorf("invalid bandwidth %s", s) - } - - s = strings.TrimSpace(s) - s = strings.ToUpper(s) - - i := strings.IndexFunc(s, unicode.IsLetter) - - bytesString, multiple := s[:i], s[i:] - bytes, err := strconv.ParseFloat(bytesString, 64) - if err != nil || bytes <= 0 { - return 0, fmt.Errorf("invalid bandwidth %s", s) - } - - switch multiple { - case "T", "TB", "TIB": - return uint64(bytes * TERABYTE), nil - case "G", "GB", "GIB": - return uint64(bytes * GIGABYTE), nil - case "M", "MB", "MIB": - return uint64(bytes * MEGABYTE), nil - case "K", "KB", "KIB": - return uint64(bytes * KILOBYTE), nil - case "B", "": - return uint64(bytes), nil - default: - return 0, fmt.Errorf("invalid bandwidth %s", s) - } +func getNode(ctx context.Context, c client.Client, nodeName string) (*corev1.Node, error) { + obj := &corev1.Node{} + err := c.Get(ctx, k8stypes.NamespacedName{Name: nodeName}, obj, &client.GetOptions{Raw: &metav1.GetOptions{ + ResourceVersion: "0", + }}) + return obj, err } -type storageItem struct { - Pod *types.PodInfo - deletionTime *time.Time +func getPod(ctx context.Context, c client.Client, namespace, name string, cache bool) (*corev1.Pod, error) { + opt := &metav1.GetOptions{} + if cache { + opt.ResourceVersion = "0" + } + obj := &corev1.Pod{} + err := c.Get(ctx, k8stypes.NamespacedName{Namespace: namespace, Name: name}, obj, &client.GetOptions{Raw: opt}) + return obj, err } -func podInfoKey(namespace, name string) string { - return fmt.Sprintf("%s/%s", namespace, name) +func getCM(ctx context.Context, c client.Client, namespace, name string) (*corev1.ConfigMap, error) { + obj := &corev1.ConfigMap{} + err := c.Get(ctx, k8stypes.NamespacedName{Namespace: namespace, Name: name}, obj, &client.GetOptions{Raw: &metav1.GetOptions{ + ResourceVersion: "0", + }}) + return obj, err } func serialize(item interface{}) ([]byte, error) { @@ -690,207 +667,95 @@ func deserialize(data []byte) (interface{}, error) { return item, nil } -func (k *k8s) GetPod(namespace, name string, cache bool) (*types.PodInfo, error) { - pod, err := getPod(context.Background(), k.client, namespace, name, cache) +func nodeCidrFromAPIServer(client kubernetes.Interface, nodeName string) (*types.IPNetSet, error) { + node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil { - if apierrors.IsNotFound(err) { - key := podInfoKey(namespace, name) - obj, err := k.storage.Get(key) - if err == nil { - item := obj.(*storageItem) - return item.Pod, nil - } - - if err != storage.ErrNotFound { - return nil, err - } - } - return nil, err + return nil, fmt.Errorf("error retrieving node spec for '%s': %v", nodeName, err) } - podInfo := convertPod(k.mode, k.statefulWorkloadKindSet, pod) - item := &storageItem{ - Pod: podInfo, + if node.Spec.PodCIDR == "" { + return nil, fmt.Errorf("node %q pod cidr not assigned", nodeName) } - - if err := k.storage.Put(podInfoKey(podInfo.Namespace, podInfo.Name), item); err != nil { - return nil, err + podCIDR := &types.IPNetSet{} + for _, cidr := range node.Spec.PodCIDRs { + podCIDR.SetIPNet(cidr) } - return podInfo, nil -} + podCIDR.SetIPNet(node.Spec.PodCIDR) -func (k *k8s) GetNodeCidr() *types.IPNetSet { - return k.nodeCidr + return podCIDR, nil } -func (k *k8s) GetLocalPods() ([]*types.PodInfo, error) { - options := metav1.ListOptions{ - FieldSelector: fields.OneTermEqualSelector("spec.nodeName", k.nodeName).String(), - ResourceVersion: "0", - } - podList := &corev1.PodList{} - err := k.client.List(context.Background(), podList, &client.ListOptions{Raw: &options}) +func parseCidr(cidrString string) (*net.IPNet, error) { + _, cidr, err := net.ParseCIDR(cidrString) + return cidr, err +} +func serviceCidrFromAPIServer(c client.Client) (*net.IPNet, error) { + kubeadmConfigMap, err := getCM(context.Background(), c, k8sSystemNamespace, k8sKubeadmConfigmap) if err != nil { - return nil, errors.Wrapf(err, "failed listting pods on %s from apiserver", k.nodeName) + return nil, err } - var ret []*types.PodInfo - for _, pod := range podList.Items { - if types.IgnoredByTerway(pod.Labels) { - continue - } - podInfo := convertPod(k.mode, k.statefulWorkloadKindSet, &pod) - ret = append(ret, podInfo) + kubeNetworkingConfig, ok := kubeadmConfigMap.Data[k8sKubeadmConfigmapNetworking] + if !ok { + kubeNetworkingConfig, ok = kubeadmConfigMap.Data[k8sKubeadmConfigmapClusterconfiguration] + if !ok { + return nil, fmt.Errorf("cannot found kubeproxy config for svc cidr") + } } - return ret, nil -} - -func (k *k8s) GetServiceCIDR() *types.IPNetSet { - return k.svcCidr -} - -func (k *k8s) SetNodeAllocatablePod(count int) error { - return nil -} - -// clean up storage -// tag the object as deletion when found pod not exist -// the tagged object will be deleted on secondary scan -func (k *k8s) clean() error { - - list, err := k.storage.List() - if err != nil { - return err - } + configMap := make(map[interface{}]interface{}) - localPods, err := k.GetLocalPods() + err = yaml.Unmarshal([]byte(kubeNetworkingConfig), &configMap) if err != nil { - return errors.Wrap(err, "error get local pods") - } - podsMap := make(map[string]*types.PodInfo) - - for _, pod := range localPods { - key := podInfoKey(pod.Namespace, pod.Name) - podsMap[key] = pod + return nil, fmt.Errorf("configmap unmarshal failed, %w", err) } - for _, obj := range list { - item := obj.(*storageItem) - key := podInfoKey(item.Pod.Namespace, item.Pod.Name) - - if _, exists := podsMap[key]; exists { - if item.deletionTime != nil { - item.deletionTime = nil - if err := k.storage.Put(key, item); err != nil { - return errors.Wrapf(err, "error save storage") + if networkingObj, ok := configMap["networking"]; ok { + if networkingMap, ok := networkingObj.(map[interface{}]interface{}); ok { + if svcObj, ok := networkingMap["serviceSubnet"]; ok { + if svcStr, ok := svcObj.(string); ok { + return parseCidr(svcStr) } } - continue - } - - if item.deletionTime == nil { - now := time.Now() - item.deletionTime = &now - if err := k.storage.Put(key, item); err != nil { - return errors.Wrap(err, "error save storage") - } - continue - } - - if time.Since(*item.deletionTime) > storageCleanTimeout { - if err := k.storage.Delete(key); err != nil { - return errors.Wrap(err, "error delete storage") - } } } - return nil -} - -func (k *k8s) RecordNodeEvent(eventType, reason, message string) { - ref := &corev1.ObjectReference{ - Kind: "Node", - Name: k.node.Name, - UID: k.node.UID, - Namespace: "", - } - - k.recorder.Event(ref, eventType, reason, message) -} - -func (k *k8s) RecordPodEvent(podName, podNamespace, eventType, reason, message string) error { - pod, err := getPod(context.Background(), k.client, podNamespace, podName, true) - if err != nil { - return err - } - - ref := &corev1.ObjectReference{ - Kind: "Pod", - Name: pod.Name, - UID: pod.UID, - Namespace: pod.Namespace, - } - - k.recorder.Event(ref, eventType, reason, message) - return nil + return nil, fmt.Errorf("cannot found kubeproxy config for svc cidr") } -// GetNodeDynamicConfigLabel returns value with label config -func (k *k8s) GetNodeDynamicConfigLabel() string { - // use node cached in newK8s() - cfgName, ok := k.node.Labels[labelDynamicConfig] - if !ok { - return "" +func parseBandwidth(s string) (uint64, error) { + // when bandwidth is "", return + if len(s) == 0 { + return 0, fmt.Errorf("invalid bandwidth %s", s) } - return cfgName -} + s = strings.TrimSpace(s) + s = strings.ToUpper(s) -// GetDynamicConfigWithName gets the Dynamic Config's content with its ConfigMap name -func (k *k8s) GetDynamicConfigWithName(name string) (string, error) { - cfgMap, err := getCM(context.Background(), k.client, k.daemonNamespace, name) - if err != nil { - return "", err - } + i := strings.IndexFunc(s, unicode.IsLetter) - content, ok := cfgMap.Data["eni_conf"] - if ok { - return content, nil + bytesString, multiple := s[:i], s[i:] + bytes, err := strconv.ParseFloat(bytesString, 64) + if err != nil || bytes <= 0 { + return 0, fmt.Errorf("invalid bandwidth %s", s) } - return "", errors.New("configmap not included eni_conf") -} - -func getNode(ctx context.Context, c client.Client, nodeName string) (*corev1.Node, error) { - obj := &corev1.Node{} - err := c.Get(ctx, k8stypes.NamespacedName{Name: nodeName}, obj, &client.GetOptions{Raw: &metav1.GetOptions{ - ResourceVersion: "0", - }}) - return obj, err -} - -func getPod(ctx context.Context, c client.Client, namespace, name string, cache bool) (*corev1.Pod, error) { - opt := &metav1.GetOptions{} - if cache { - opt.ResourceVersion = "0" + switch multiple { + case "T", "TB", "TIB": + return uint64(bytes * TERABYTE), nil + case "G", "GB", "GIB": + return uint64(bytes * GIGABYTE), nil + case "M", "MB", "MIB": + return uint64(bytes * MEGABYTE), nil + case "K", "KB", "KIB": + return uint64(bytes * KILOBYTE), nil + case "B", "": + return uint64(bytes), nil + default: + return 0, fmt.Errorf("invalid bandwidth %s", s) } - obj := &corev1.Pod{} - err := c.Get(ctx, k8stypes.NamespacedName{Namespace: namespace, Name: name}, obj, &client.GetOptions{Raw: opt}) - return obj, err -} - -func getCM(ctx context.Context, c client.Client, namespace, name string) (*corev1.ConfigMap, error) { - obj := &corev1.ConfigMap{} - err := c.Get(ctx, k8stypes.NamespacedName{Namespace: namespace, Name: name}, obj, &client.GetOptions{Raw: &metav1.GetOptions{ - ResourceVersion: "0", - }}) - return obj, err } -func getPodENI(ctx context.Context, c client.Client, namespace, name string) (*podENITypes.PodENI, error) { - obj := &podENITypes.PodENI{} - err := c.Get(ctx, k8stypes.NamespacedName{Namespace: namespace, Name: name}, obj, &client.GetOptions{Raw: &metav1.GetOptions{ - ResourceVersion: "0", - }}) - return obj, err +type storageItem struct { + Pod *daemon.PodInfo + deletionTime *time.Time } diff --git a/pkg/link/interface_linux.go b/pkg/link/interface_linux.go index a98787bf..6817ca36 100644 --- a/pkg/link/interface_linux.go +++ b/pkg/link/interface_linux.go @@ -8,8 +8,8 @@ import ( "os" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" + "k8s.io/klog/v2" ) // GetDeviceNumber get interface device number by mac address @@ -43,7 +43,7 @@ func DeleteIPRulesByIP(addr *net.IPNet) error { } for _, r := range rules { if ipNetEqual(addr, r.Src) || ipNetEqual(addr, r.Dst) { - log.Infof("del ip rule %s", r.String()) + klog.Infof("del ip rule %s", r.String()) err := netlink.RuleDel(&r) if err == nil { continue @@ -71,7 +71,7 @@ func DeleteRouteByIP(addr *net.IPNet) error { } for _, r := range routes { if r.Dst != nil && r.Dst.IP.Equal(addr.IP) { - log.Infof("del route %s", r.String()) + klog.Infof("del route %s", r.String()) err := netlink.RouteDel(&r) if err != nil { return err diff --git a/pkg/logger/log.go b/pkg/logger/log.go index 3a961913..45273224 100644 --- a/pkg/logger/log.go +++ b/pkg/logger/log.go @@ -62,13 +62,3 @@ func NewDefaultLogger() *logrus.Logger { l.SetFormatter(&Format{}) return l } - -func SetLevel(level string) error { - l, err := logrus.ParseLevel(level) - if err != nil { - return err - } - DefaultLogger.Infof("set log level %s", level) - DefaultLogger.SetLevel(l) - return nil -} diff --git a/pkg/pool/pool.go b/pkg/pool/pool.go deleted file mode 100644 index d8f7f45a..00000000 --- a/pkg/pool/pool.go +++ /dev/null @@ -1,736 +0,0 @@ -package pool - -import ( - "context" - "errors" - "fmt" - "strings" - "sync" - "time" - - "go.uber.org/atomic" - corev1 "k8s.io/api/core/v1" - - apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" - "github.com/AliyunContainerService/terway/pkg/logger" - "github.com/AliyunContainerService/terway/pkg/metric" - "github.com/AliyunContainerService/terway/pkg/tracing" - "github.com/AliyunContainerService/terway/types" - - "github.com/prometheus/client_golang/prometheus" - "k8s.io/apimachinery/pkg/util/wait" -) - -var log = logger.DefaultLogger.WithField("subSys", "pool") - -// Errors of pool -var ( - ErrNoAvailableResource = errors.New("no available resource") - ErrInvalidState = errors.New("invalid state") - ErrNotFound = errors.New("not found") - ErrContextDone = errors.New("context done") - ErrInvalidArguments = errors.New("invalid arguments") -) - -const ( - // CheckIdleInterval the interval of check and process idle eni - CheckIdleInterval = 2 * time.Minute - defaultPoolBackoff = 1 * time.Minute - - tracingKeyName = "name" - tracingKeyMaxIdle = "max_idle" - tracingKeyMinIdle = "min_idle" - tracingKeyCapacity = "capacity" - tracingKeyIdle = "idle" - tracingKeyInuse = "inuse" - - commandMapping = "mapping" - - ipExhaustiveConditionPeriod = 10 * time.Minute -) - -// ResUsage ResUsage -type ResUsage struct { - ID string - Type string - Status types.ResStatus -} - -func (r *ResUsage) GetID() string { - return r.ID -} - -func (r *ResUsage) GetType() string { - return r.Type -} - -func (r *ResUsage) GetStatus() types.ResStatus { - return r.Status -} - -// Usage store res booth local and remote -type Usage struct { - Local map[string]types.Res - Remote map[string]types.Res -} - -func (u *Usage) GetLocal() map[string]types.Res { - return u.Local -} - -func (u *Usage) GetRemote() map[string]types.Res { - return u.Remote -} - -// ObjectPool object pool interface -type ObjectPool interface { - Acquire(ctx context.Context, resID, idempotentKey string) (types.NetworkResource, error) - ReleaseWithReservation(resID string, reservation time.Duration) error - Release(resID string) error - AcquireAny(ctx context.Context, idempotentKey string) (types.NetworkResource, error) - Stat(resID string) (types.NetworkResource, error) - GetName() string - tracing.ResourceMappingHandler -} - -// ResourceHolder interface to initialize pool -type ResourceHolder interface { - AddIdle(resource types.NetworkResource) - AddInvalid(resource types.NetworkResource) - AddInuse(resource types.NetworkResource, idempotentKey string) -} - -// ObjectFactory interface of network resource object factory -type ObjectFactory interface { - // Create res with count - Create(count int) ([]types.NetworkResource, error) - Dispose(types.NetworkResource) error - ListResource() (map[string]types.NetworkResource, error) - Check(types.NetworkResource) error - // Reconcile run periodicity - Reconcile() -} - -type simpleObjectPool struct { - name string - inuse map[string]poolItem - idle *priorityQueue - invalid map[string]poolItem // hole invalid also idle resource - lock sync.Mutex - factory ObjectFactory - maxIdle int - minIdle int - capacity int - notifyCh chan interface{} - // concurrency to create resource. tokenCh = capacity - (idle + inuse + dispose) - tokenCh chan struct{} - backoffTime time.Duration - // metrics - metricIdle prometheus.Gauge - metricTotal prometheus.Gauge - metricDisposed prometheus.Counter - // node conditions - factoryIPExhaustive *atomic.Bool - factoryIPExhaustiveTimer *time.Timer - IPConditionHandler NodeConditionHandler -} - -// Config configuration of pool -type Config struct { - Name string - Type string - Factory ObjectFactory - Initializer Initializer - MinIdle int - MaxIdle int - Capacity int - IPConditionHandler NodeConditionHandler -} - -type poolItem struct { - res types.NetworkResource - reservation time.Time - idempotentKey string -} - -func (i *poolItem) lessThan(other *poolItem) bool { - return i.reservation.Before(other.reservation) -} - -// Initializer of pool -type Initializer func(holder ResourceHolder) error - -type NodeConditionHandler func(status corev1.ConditionStatus, reason, message string) error - -// NewSimpleObjectPool return an object pool implement -func NewSimpleObjectPool(cfg Config) (ObjectPool, error) { - if cfg.MinIdle > cfg.MaxIdle { - return nil, ErrInvalidArguments - } - - if cfg.MaxIdle > cfg.Capacity { - return nil, ErrInvalidArguments - } - - pool := &simpleObjectPool{ - name: cfg.Name, - factory: cfg.Factory, - inuse: make(map[string]poolItem), - idle: newPriorityQueue(), - invalid: make(map[string]poolItem), - maxIdle: cfg.MaxIdle, - minIdle: cfg.MinIdle, - capacity: cfg.Capacity, - notifyCh: make(chan interface{}, 1), - tokenCh: make(chan struct{}, cfg.Capacity), - backoffTime: defaultPoolBackoff, - factoryIPExhaustiveTimer: time.NewTimer(0), - factoryIPExhaustive: atomic.NewBool(true), - IPConditionHandler: cfg.IPConditionHandler, - // create metrics with labels in the pool struct - // and it will show in metrics even if it has not been triggered yet - metricIdle: metric.ResourcePoolIdle.WithLabelValues(cfg.Name, cfg.Type, fmt.Sprint(cfg.Capacity), - fmt.Sprint(cfg.MaxIdle), fmt.Sprint(cfg.MinIdle)), - metricTotal: metric.ResourcePoolTotal.WithLabelValues(cfg.Name, cfg.Type, fmt.Sprint(cfg.Capacity), - fmt.Sprint(cfg.MaxIdle), fmt.Sprint(cfg.MinIdle)), - metricDisposed: metric.ResourcePoolDisposed.WithLabelValues(cfg.Name, cfg.Type, fmt.Sprint(cfg.Capacity), - fmt.Sprint(cfg.MaxIdle), fmt.Sprint(cfg.MinIdle)), - } - - if cfg.Initializer != nil { - if err := cfg.Initializer(pool); err != nil { - return nil, err - } - } - - if err := pool.preload(); err != nil { - return nil, err - } - - log.WithFields(map[string]interface{}{ - "capacity": pool.capacity, - "maxIdle": pool.maxIdle, - "minIdle": pool.minIdle, - "idle": pool.idle.Size(), - "inUse": len(pool.inuse), - "invalid": len(pool.invalid), - }).Infof("pool initial state ,idle: %s, inuse: %s, invalid: %s", queueKeys(pool.idle), mapKeys(pool.inuse), mapKeys(pool.invalid)) - - go pool.startCheckIdleTicker() - go pool.checkIPExhaustive() - - _ = tracing.Register(tracing.ResourceTypeResourcePool, pool.name, pool) - return pool, nil -} - -func (p *simpleObjectPool) startCheckIdleTicker() { - tick := make(chan struct{}) - go wait.JitterUntil(func() { - tick <- struct{}{} - }, CheckIdleInterval, 0.2, true, wait.NeverStop) - reconcileTick := make(chan struct{}) - go wait.JitterUntil(func() { - reconcileTick <- struct{}{} - }, time.Hour, 0.2, true, wait.NeverStop) - for { - select { - case <-tick: - p.checkResSync() // make sure pool is synced - p.checkIdle() - p.checkInsufficient() - case <-p.notifyCh: - p.checkIdle() - p.checkInsufficient() - case <-reconcileTick: - p.factory.Reconcile() - } - } -} - -func (p *simpleObjectPool) setIPExhaustive(err *types.IPInsufficientError) { - if p.IPConditionHandler == nil { - return - } - if err := p.IPConditionHandler(corev1.ConditionFalse, types.IPResInsufficientReason, - fmt.Sprintf("node has insufficient IP, error: %v", err.Reason)); err != nil { - log.Errorf("set IPExhaustive condition failed: %v", err) - } - if !p.factoryIPExhaustive.Load() { - p.factoryIPExhaustive.Store(true) - } - p.factoryIPExhaustiveTimer.Reset(ipExhaustiveConditionPeriod) -} - -func (p *simpleObjectPool) unsetIPExhaustive() { - if p.factoryIPExhaustive.Load() { - p.factoryIPExhaustiveTimer.Reset(0) - } -} - -func (p *simpleObjectPool) checkIPExhaustive() { - for range p.factoryIPExhaustiveTimer.C { - if p.factoryIPExhaustive.Load() { - if p.IPConditionHandler != nil { - if err := p.IPConditionHandler(corev1.ConditionTrue, types.IPResSufficientReason, - fmt.Sprintf("node has sufficient IP or pass the exhaustive period: %v", ipExhaustiveConditionPeriod)); err != nil { - log.Errorf("set IPExhaustive condition failed: %v", err) - } - } - p.factoryIPExhaustive.Store(false) - } - } -} - -func mapKeys(m map[string]poolItem) string { - var keys []string - for k := range m { - keys = append(keys, k) - } - return strings.Join(keys, ", ") -} - -func queueKeys(q *priorityQueue) string { - var keys []string - for i := 0; i < q.size; i++ { - keys = append(keys, q.slots[i].res.GetResourceID()) - } - return strings.Join(keys, ", ") -} - -func (p *simpleObjectPool) tooManyIdleLocked() bool { - return p.idle.Size() > p.maxIdle || (p.idle.Size() > 0 && p.sizeLocked() > p.capacity) -} - -func (p *simpleObjectPool) needAddition() int { - p.lock.Lock() - defer p.lock.Unlock() - addition := p.minIdle - p.idle.Size() - if addition > (p.capacity - p.sizeLocked()) { - return p.capacity - p.sizeLocked() - } - return addition -} - -func (p *simpleObjectPool) peekOverfullIdle() *poolItem { - p.lock.Lock() - defer p.lock.Unlock() - - if !p.tooManyIdleLocked() { - return nil - } - - item := p.idle.Peek() - if item == nil { - return nil - } - - if item.reservation.After(time.Now()) { - return nil - } - return p.idle.Pop() -} - -// found resources that can be disposed, put them into dispose channel -func (p *simpleObjectPool) checkIdle() { - for { - item := p.peekOverfullIdle() - if item == nil { - break - } - - p.metricIdle.Dec() - p.metricTotal.Dec() - - res := item.res - log.Infof("try dispose res %+v", res) - err := p.factory.Dispose(res) - if err == nil { - p.tokenCh <- struct{}{} - p.backoffTime = defaultPoolBackoff - // one item popped from idle and total - p.metricDisposed.Inc() - } else { - log.Warnf("error dispose res: %+v", err) - p.backoffTime = p.backoffTime * 2 - p.AddIdle(res) - time.Sleep(p.backoffTime) - } - } -} - -func (p *simpleObjectPool) checkInsufficient() { - addition := p.needAddition() - if addition <= 0 { - return - } - var tokenAcquired int - for i := 0; i < addition; i++ { - // pending resources - select { - case <-p.tokenCh: - tokenAcquired++ - default: - continue - } - } - log.Debugf("token acquired count: %v", tokenAcquired) - if tokenAcquired <= 0 { - return - } - resList, err := p.factory.Create(tokenAcquired) - if err != nil { - inSufErr, ok := err.(*types.IPInsufficientError) - if ok { - p.setIPExhaustive(inSufErr) - } - log.Errorf("error add idle network resources: %v", err) - } - if tokenAcquired == len(resList) { - p.backoffTime = defaultPoolBackoff - } - for _, res := range resList { - log.Infof("add resource %s to pool idle", res.GetResourceID()) - p.AddIdle(res) - tokenAcquired-- - } - for i := 0; i < tokenAcquired; i++ { - // release token - p.tokenCh <- struct{}{} - } - if tokenAcquired != 0 { - log.Debugf("token acquired left: %d, err: %v", tokenAcquired, err) - p.notify() - } - - if err != nil { - p.backoffTime = p.backoffTime * 2 - time.Sleep(p.backoffTime) - } -} - -func (p *simpleObjectPool) preload() error { - p.lock.Lock() - defer p.lock.Unlock() - - tokenCount := p.capacity - p.sizeLocked() - for i := 0; i < tokenCount; i++ { - p.tokenCh <- struct{}{} - } - - return nil -} - -func (p *simpleObjectPool) sizeLocked() int { - return p.idle.Size() + len(p.inuse) + len(p.invalid) -} - -func (p *simpleObjectPool) getOneLocked(resID string) *poolItem { - if len(resID) > 0 { - item := p.idle.Rob(resID) - if item != nil { - return item - } - } - return p.idle.Pop() -} - -func (p *simpleObjectPool) Acquire(ctx context.Context, resID, idempotentKey string) (types.NetworkResource, error) { - if p.factoryIPExhaustive.Load() { - // slowdown ip allocation on ip exhaustive - time.Sleep(10 * time.Second) - } - p.lock.Lock() - if resItem, ok := p.inuse[resID]; ok && resItem.idempotentKey == idempotentKey { - p.lock.Unlock() - return resItem.res, nil - } - - if p.idle.Size() > 0 { - res := p.getOneLocked(resID).res - p.inuse[res.GetResourceID()] = poolItem{res: res, idempotentKey: idempotentKey} - p.lock.Unlock() - log.Infof("acquire (expect %s): return idle %s", resID, res.GetResourceID()) - p.metricIdle.Dec() - p.notify() - return res, nil - } - size := p.sizeLocked() - if size >= p.capacity { - p.lock.Unlock() - p.setIPExhaustive(&types.IPInsufficientError{ - Err: ErrNoAvailableResource, - Reason: fmt.Sprintf("exceed node ip resource capacity: %v", p.capacity), - }) - log.Infof("acquire (expect %s), size %d, capacity %d: return err %v", resID, size, p.capacity, ErrNoAvailableResource) - return nil, ErrNoAvailableResource - } - - p.lock.Unlock() - - select { - case <-p.tokenCh: - // should we pass ctx into factory.Create? - res, err := p.factory.Create(1) - if err != nil || len(res) == 0 { - p.tokenCh <- struct{}{} - inSufErr, ok := err.(*types.IPInsufficientError) - if ok { - p.setIPExhaustive(inSufErr) - } - return nil, fmt.Errorf("error create from factory: %v", err) - } - log.Infof("acquire (expect %s): return newly %s", resID, res[0].GetResourceID()) - p.AddInuse(res[0], idempotentKey) - return res[0], nil - case <-ctx.Done(): - log.Infof("acquire (expect %s): return err %v", resID, ErrContextDone) - return nil, ErrContextDone - } -} - -func (p *simpleObjectPool) AcquireAny(ctx context.Context, idempotentKey string) (types.NetworkResource, error) { - return p.Acquire(ctx, "", idempotentKey) -} - -func (p *simpleObjectPool) Stat(resID string) (types.NetworkResource, error) { - p.lock.Lock() - defer p.lock.Unlock() - v, ok := p.inuse[resID] - if ok { - return v.res, nil - } - vv := p.idle.Find(resID) - if vv != nil { - return vv.res, nil - } - - return nil, ErrNotFound -} - -func (p *simpleObjectPool) GetName() string { - return p.name -} - -func (p *simpleObjectPool) Config() []tracing.MapKeyValueEntry { - config := []tracing.MapKeyValueEntry{ - {Key: tracingKeyName, Value: p.name}, - {Key: tracingKeyMaxIdle, Value: fmt.Sprint(p.maxIdle)}, - {Key: tracingKeyMinIdle, Value: fmt.Sprint(p.minIdle)}, - {Key: tracingKeyCapacity, Value: fmt.Sprint(p.capacity)}, - } - - return config -} - -func (p *simpleObjectPool) Trace() []tracing.MapKeyValueEntry { - trace := []tracing.MapKeyValueEntry{ - {Key: tracingKeyIdle, Value: queueKeys(p.idle)}, - {Key: tracingKeyInuse, Value: mapKeys(p.inuse)}, - } - - return trace -} - -func (p *simpleObjectPool) Execute(cmd string, _ []string, message chan<- string) { - switch cmd { - case commandMapping: - mapping, err := p.GetResourceMapping() - message <- fmt.Sprintf("mapping: %v, err: %s\n", mapping, err) - default: - message <- "can't recognize command\n" - } - - close(message) -} - -func (p *simpleObjectPool) notify() { - select { - case p.notifyCh <- true: - default: - } -} - -func (p *simpleObjectPool) ReleaseWithReservation(resID string, reservation time.Duration) error { - p.lock.Lock() - defer p.lock.Unlock() - res, ok := p.inuse[resID] - if !ok { - log.Infof("release %s: return err %v", resID, ErrInvalidState) - return ErrInvalidState - } - log.Infof("release %s, reservation %v: return success", resID, reservation) - delete(p.inuse, resID) - - // check metadata - err := p.factory.Check(res.res) - if errors.Is(err, apiErr.ErrNotFound) { - log.Warnf("release %s, resource not exist in metadata, ignored", resID) - if err = p.factory.Dispose(res.res); err == nil { - p.tokenCh <- struct{}{} - p.metricTotal.Dec() - p.metricDisposed.Inc() - return nil - } - log.Warnf("release %s, err %v", resID, err) - - // put resource to invalid - p.invalid[resID] = res - return nil - } - - reserveTo := time.Now() - if reservation > 0 { - reserveTo = reserveTo.Add(reservation) - } - p.idle.Push(&poolItem{res: res.res, reservation: reserveTo}) - p.unsetIPExhaustive() - p.metricIdle.Inc() - p.notify() - return nil -} - -func (p *simpleObjectPool) Release(resID string) error { - return p.ReleaseWithReservation(resID, time.Duration(0)) -} - -func (p *simpleObjectPool) AddIdle(resource types.NetworkResource) { - p.lock.Lock() - defer p.lock.Unlock() - p.idle.Push(&poolItem{res: resource, reservation: time.Now()}) - // assume AddIdle() adds a resource that not exists in the pool before - // both add total and idle gauge - p.metricTotal.Inc() - p.metricIdle.Inc() -} - -func (p *simpleObjectPool) AddInvalid(resource types.NetworkResource) { - p.lock.Lock() - defer p.lock.Unlock() - - p.invalid[resource.GetResourceID()] = poolItem{ - res: resource, - } - p.metricTotal.Inc() - p.metricIdle.Inc() // use idle metric here -} - -func (p *simpleObjectPool) AddInuse(res types.NetworkResource, idempotentKey string) { - p.lock.Lock() - defer p.lock.Unlock() - p.inuse[res.GetResourceID()] = poolItem{ - res: res, - idempotentKey: idempotentKey, - } - // assume AddInuse() adds a resource that not exists in the pool before - p.metricTotal.Inc() -} - -func (p *simpleObjectPool) GetResourceMapping() (tracing.ResourcePoolStats, error) { - p.lock.Lock() - defer p.lock.Unlock() - - usage, err := p.getResUsage() - if err != nil { - return nil, err - } - - return usage, nil -} - -// checkResSync will check pool res witch metadata. make sure pool is synced -func (p *simpleObjectPool) checkResSync() { - p.lock.Lock() - defer p.lock.Unlock() - - for key, invalid := range p.invalid { - l := log.WithFields(map[string]interface{}{ - "id": invalid.res.GetResourceID(), - "reason": "invalid", - }) - err := p.factory.Dispose(invalid.res) - if err != nil { - l.Warnf("dispose failed %s", err.Error()) - continue - } - l.Infof("dispose succeed") - delete(p.invalid, key) - - p.tokenCh <- struct{}{} - p.metricTotal.Dec() - p.metricDisposed.Inc() - } - - usage, err := p.getResUsage() - if err != nil { - log.Error(err) - return - } - for _, r := range usage.Local { - _, ok := usage.Remote[r.GetID()] - if ok { - continue - } - // res store in pool but can not find in remote(metadata) - if r.GetStatus() == types.ResStatusIdle { - log.Warnf("res %s, type %s is removed from remote,mark as invalid", r.GetID(), r.GetType()) - invalid := p.idle.Rob(r.GetID()) - p.invalid[r.GetID()] = *invalid - } - log.Errorf("res %s, type %s is removed from remote,but is in use", r.GetID(), r.GetType()) - } -} - -// getResUsage get current usage -func (p *simpleObjectPool) getResUsage() (*Usage, error) { - localRes := make(map[string]types.Res) - // idle - for i := 0; i < p.idle.size; i++ { - item := p.idle.slots[i] - localRes[item.res.GetResourceID()] = &ResUsage{ - ID: item.res.GetResourceID(), - Type: item.res.GetType(), - Status: types.ResStatusIdle, - } - } - // inuse - for _, v := range p.inuse { - localRes[v.res.GetResourceID()] = &ResUsage{ - ID: v.res.GetResourceID(), - Type: v.res.GetType(), - Status: types.ResStatusInUse, - } - } - // invalid - for _, v := range p.invalid { - localRes[v.res.GetResourceID()] = &ResUsage{ - ID: v.res.GetResourceID(), - Type: v.res.GetType(), - Status: types.ResStatusInvalid, - } - } - - factoryRes, err := p.factory.ListResource() - if err != nil { - return nil, err - } - remoteRes := make(map[string]types.Res) - - // map to factory - for _, v := range factoryRes { - status := types.ResStatusInvalid - lo, ok := localRes[v.GetResourceID()] - if ok { - status = lo.GetStatus() - } - - remoteRes[v.GetResourceID()] = &ResUsage{ - ID: v.GetResourceID(), - Status: status, - } - } - - return &Usage{ - Local: localRes, - Remote: remoteRes, - }, nil -} diff --git a/pkg/pool/pool_test.go b/pkg/pool/pool_test.go deleted file mode 100644 index a9afce86..00000000 --- a/pkg/pool/pool_test.go +++ /dev/null @@ -1,335 +0,0 @@ -package pool - -import ( - "context" - "fmt" - "os" - "sync" - "testing" - "time" - - "github.com/sirupsen/logrus" - - "github.com/AliyunContainerService/terway/types" - "github.com/stretchr/testify/assert" -) - -type mockNetworkResource struct { - ID string -} - -func (n *mockNetworkResource) ToResItems() []types.ResourceItem { - return []types.ResourceItem{ - { - Type: n.GetType(), - ID: n.GetResourceID(), - }, - } -} - -func (n mockNetworkResource) GetResourceID() string { - return n.ID -} - -func (n mockNetworkResource) GetType() string { - return "mock" -} - -type mockObjectFactory struct { - createDelay time.Duration - disposeDeplay time.Duration - err error - totalCreated int - totalDisposed int - idBegin int - - Res map[string]string - - lock sync.Mutex -} - -func newMockObjectFactory(id int) *mockObjectFactory { - return &mockObjectFactory{ - idBegin: id, - Res: map[string]string{}, - } -} - -func (f *mockObjectFactory) ListResource() (map[string]types.NetworkResource, error) { - f.lock.Lock() - defer f.lock.Unlock() - - mapping := make(map[string]types.NetworkResource) - for _, id := range f.Res { - mapping[id] = &mockNetworkResource{ - ID: id, - } - } - return mapping, nil -} - -func (f *mockObjectFactory) Create(count int) ([]types.NetworkResource, error) { - time.Sleep(f.createDelay) - if f.err != nil { - return nil, f.err - } - f.lock.Lock() - defer f.lock.Unlock() - - var result []types.NetworkResource - for i := 0; i < count; i++ { - f.totalCreated++ - f.idBegin++ - res := &mockNetworkResource{ID: fmt.Sprintf("%d", f.idBegin)} - f.Res[res.GetResourceID()] = res.GetResourceID() - result = append(result, res) - } - - return result, nil -} - -func (f *mockObjectFactory) Put(count int) ([]types.NetworkResource, error) { - f.lock.Lock() - defer f.lock.Unlock() - - var result []types.NetworkResource - for i := 0; i < count; i++ { - f.idBegin++ - res := &mockNetworkResource{ID: fmt.Sprintf("%d", f.idBegin)} - f.Res[res.GetResourceID()] = res.GetResourceID() - result = append(result, res) - } - - return result, nil -} - -func (f *mockObjectFactory) Dispose(in types.NetworkResource) error { - time.Sleep(f.disposeDeplay) - f.lock.Lock() - defer f.lock.Unlock() - f.totalDisposed++ - - res, ok := in.(*mockNetworkResource) - if !ok { - return fmt.Errorf("err type") - } - delete(f.Res, res.ID) - return f.err -} - -func (f *mockObjectFactory) Check(in types.NetworkResource) error { - return nil -} - -func (f *mockObjectFactory) Reconcile() {} - -func (f *mockObjectFactory) getTotalDisposed() int { - f.lock.Lock() - defer f.lock.Unlock() - return f.totalDisposed -} - -func (f *mockObjectFactory) getTotalCreated() int { - f.lock.Lock() - defer f.lock.Unlock() - return f.totalCreated -} - -func TestInitializerWithoutAutoCreate(t *testing.T) { - factory := newMockObjectFactory(1000) - createPool(factory, 3, 5, 3, 0) - time.Sleep(time.Second) - assert.Equal(t, 0, factory.getTotalCreated()) - assert.Equal(t, 0, factory.getTotalDisposed()) -} - -func TestInitializerWithAutoCreate(t *testing.T) { - factory := newMockObjectFactory(1000) - createPool(factory, 3, 5, 0, 0) - time.Sleep(time.Second) - assert.Equal(t, 3, factory.getTotalCreated()) - assert.Equal(t, 0, factory.getTotalDisposed()) -} - -func createPool(factory *mockObjectFactory, minIdle, maxIdle, initIdle, initInuse int) ObjectPool { - cfg := Config{ - Factory: factory, - Initializer: func(holder ResourceHolder) error { - idleRes, err := factory.Put(initIdle) - if err != nil { - panic(err) - } - for _, res := range idleRes { - holder.AddIdle(res) - } - - inuseRes, err := factory.Put(initInuse) - if err != nil { - panic(err) - } - for _, res := range inuseRes { - holder.AddInuse(res, "") - } - - return nil - }, - MinIdle: minIdle, - MaxIdle: maxIdle, - Capacity: 10, - } - pool, err := NewSimpleObjectPool(cfg) - if err != nil { - panic(err) - } - return pool -} - -func TestMain(m *testing.M) { - logrus.SetLevel(logrus.DebugLevel) - os.Exit(m.Run()) -} - -func TestInitializerExceedMaxIdle(t *testing.T) { - factory := newMockObjectFactory(1000) - createPool(factory, 3, 5, 6, 0) - time.Sleep(1 * time.Second) - assert.Equal(t, 0, factory.getTotalCreated()) - assert.Equal(t, 1, factory.getTotalDisposed()) -} - -func TestInitializerExceedCapacity(t *testing.T) { - factory := newMockObjectFactory(1000) - createPool(factory, 3, 5, 1, 10) - time.Sleep(time.Second) - assert.Equal(t, 0, factory.getTotalCreated()) - assert.Equal(t, 1, factory.getTotalDisposed()) -} - -func TestAcquireIdle(t *testing.T) { - factory := newMockObjectFactory(1000) - pool := createPool(factory, 0, 5, 3, 0) - _, err := pool.Acquire(context.Background(), "", "") - assert.Nil(t, err) - assert.Equal(t, 0, factory.getTotalCreated()) -} - -func TestAutoAddition(t *testing.T) { - factory := newMockObjectFactory(1000) - pool := createPool(factory, 3, 5, 0, 0) - time.Sleep(1 * time.Second) - _, err := pool.Acquire(context.Background(), "", "") - assert.Nil(t, err) - _, err = pool.Acquire(context.Background(), "", "") - assert.Nil(t, err) - _, err = pool.Acquire(context.Background(), "", "") - assert.Nil(t, err) - time.Sleep(1 * time.Second) - assert.Equal(t, 6, factory.getTotalCreated()) -} - -func TestAcquireNonExists(t *testing.T) { - factory := newMockObjectFactory(1000) - pool := createPool(factory, 0, 5, 3, 0) - _, err := pool.Acquire(context.Background(), "1000", "") - assert.Nil(t, err) - assert.Equal(t, 0, factory.getTotalCreated()) -} - -func TestAcquireExists(t *testing.T) { - factory := newMockObjectFactory(0) - pool := createPool(factory, 0, 5, 3, 0) - res, err := pool.Acquire(context.Background(), "2", "") - assert.Nil(t, err) - assert.Equal(t, 0, factory.getTotalCreated()) - assert.Equal(t, "2", res.GetResourceID()) -} - -func TestConcurrencyAcquireNoMoreThanCapacity(t *testing.T) { - factory := newMockObjectFactory(0) - - pool := createPool(factory, 0, 5, 1, 0) - wg := sync.WaitGroup{} - for i := 0; i < 10; i++ { - wg.Add(1) - ctx, cancel := context.WithTimeout(context.Background(), 11*time.Second) - go func() { - _, err := pool.Acquire(ctx, "", "") - cancel() - assert.Nil(t, err) - wg.Done() - }() - } - wg.Wait() -} - -func TestConcurrencyAcquireMoreThanCapacity(t *testing.T) { - factory := newMockObjectFactory(1000) - - pool := createPool(factory, 3, 5, 3, 0) - wg := sync.WaitGroup{} - for i := 0; i < 20; i++ { - wg.Add(1) - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - go func() { - res, _ := pool.Acquire(ctx, "", "") - t.Logf("concurrency acquire resource: %+v", res) - cancel() - wg.Done() - }() - } - wg.Wait() - assert.Equal(t, 7, factory.getTotalCreated()) -} - -func TestRelease(t *testing.T) { - factory := newMockObjectFactory(1000) - - pool := createPool(factory, 0, 5, 3, 0) - n1, _ := pool.Acquire(context.Background(), "", "") - n2, _ := pool.Acquire(context.Background(), "", "") - n3, _ := pool.Acquire(context.Background(), "", "") - n4, _ := pool.Acquire(context.Background(), "", "") - n5, _ := pool.Acquire(context.Background(), "", "") - n6, _ := pool.Acquire(context.Background(), "", "") - assert.Equal(t, 3, factory.getTotalCreated()) - err := pool.Release(n1.GetResourceID()) - assert.Equal(t, err, nil) - err = pool.Release(n2.GetResourceID()) - assert.Equal(t, err, nil) - err = pool.Release(n3.GetResourceID()) - assert.Equal(t, err, nil) - time.Sleep(1 * time.Second) - assert.Equal(t, 0, factory.getTotalDisposed()) - err = pool.Release(n4.GetResourceID()) - assert.Equal(t, err, nil) - err = pool.Release(n5.GetResourceID()) - assert.Equal(t, err, nil) - time.Sleep(1 * time.Second) - assert.Equal(t, 0, factory.getTotalDisposed()) - err = pool.Release(n6.GetResourceID()) - assert.Equal(t, err, nil) - time.Sleep(1 * time.Second) - assert.Equal(t, 1, factory.getTotalDisposed()) -} - -func TestReleaseInvalid(t *testing.T) { - factory := newMockObjectFactory(1000) - pool := createPool(factory, 3, 5, 3, 0) - err := pool.Release("not-exists") - assert.Equal(t, err, ErrInvalidState) -} - -func TestGetResourceMapping(t *testing.T) { - factory := newMockObjectFactory(1000) - pool := createPool(factory, 3, 5, 3, 2) - - for i := 0; i < 5; i++ { - _, err := pool.Acquire(context.Background(), "", "") - assert.Equal(t, nil, err) - } - - mapping, err := pool.GetResourceMapping() - assert.Equal(t, nil, err) - assert.NotNil(t, mapping.GetLocal()) - assert.NotNil(t, mapping.GetRemote()) -} diff --git a/pkg/pool/queue.go b/pkg/pool/queue.go deleted file mode 100644 index aaec6ec8..00000000 --- a/pkg/pool/queue.go +++ /dev/null @@ -1,115 +0,0 @@ -package pool - -type priorityQueue struct { - slots []*poolItem - size int - capacity int -} - -func newPriorityQueue() *priorityQueue { - return &priorityQueue{ - capacity: 10, - size: 0, - slots: make([]*poolItem, 10), - } -} - -func (q *priorityQueue) Pop() *poolItem { - if q.size == 0 { - return nil - } - ret := q.slots[0] - q.slots[0] = q.slots[q.size-1] - q.size-- - q.bubbleDown(0) - return ret -} - -func (q *priorityQueue) bubbleUp(index int) { - for index > 0 { - parent := (index - 1) / 2 - if !q.slots[index].lessThan(q.slots[parent]) { - break - } - q.swap(index, parent) - index = parent - } -} - -func (q *priorityQueue) swap(x, y int) { - q.slots[x], q.slots[y] = q.slots[y], q.slots[x] -} - -func (q *priorityQueue) bubbleDown(index int) { - for index < q.size { - left := index*2 + 1 - right := index*2 + 2 - var minChild int - if left < q.size && right < q.size { - if q.slots[left].lessThan(q.slots[right]) { - minChild = left - } else { - minChild = right - } - } else if left < q.size { - minChild = left - } else if right < q.size { - minChild = right - } else { - break - } - if q.slots[minChild].lessThan(q.slots[index]) { - //fmt.Printf("min: %d\n", min) - q.swap(index, minChild) - index = minChild - } else { - break - } - } -} - -func (q *priorityQueue) Peek() *poolItem { - if q.size == 0 { - return nil - } - return q.slots[0] -} - -func (q *priorityQueue) Rob(id string) *poolItem { - for i := 0; i < q.size; i++ { - item := q.slots[i] - if item.res.GetResourceID() == id { - q.slots[i] = q.slots[q.size-1] - q.size-- - q.bubbleDown(i) - return item - } - } - - return nil -} - -func (q *priorityQueue) Find(id string) *poolItem { - for i := 0; i < q.size; i++ { - if q.slots[i].res.GetResourceID() == id { - return q.slots[i] - } - } - return nil -} - -func (q *priorityQueue) Push(item *poolItem) { - q.slots[q.size] = item - q.size++ - q.bubbleUp(q.size - 1) - if q.size >= q.capacity { - q.capacity *= 2 - newSlots := make([]*poolItem, q.capacity) - copy(newSlots, q.slots) - q.slots = newSlots - } -} - -func (q *priorityQueue) Size() int { - return q.size -} diff --git a/pkg/pool/queue_test.go b/pkg/pool/queue_test.go deleted file mode 100644 index 5df56d0b..00000000 --- a/pkg/pool/queue_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package pool - -import ( - "fmt" - "testing" - "time" - - "github.com/AliyunContainerService/terway/types" - "github.com/stretchr/testify/assert" -) - -func TestInit(t *testing.T) { - queue := newPriorityQueue() - assert.Zero(t, queue.Size()) -} - -func createNetworkResource(id string) types.NetworkResource { - return &mockNetworkResource{ID: id} -} - -func createPoolItem(id int) *poolItem { - return &poolItem{res: createNetworkResource(fmt.Sprintf("%d", id)), reservation: time.Now().Add(time.Hour * time.Duration(id))} -} - -func TestPop(t *testing.T) { - queue := newPriorityQueue() - for i := 0; i < 100; i++ { - item := createPoolItem(i) - queue.Push(item) - } - i := 0 - for { - item := queue.Pop() - if item == nil { - break - } - assert.Equal(t, fmt.Sprintf("%d", i), item.res.GetResourceID()) - i++ - } - assert.Equal(t, 100, i) -} - -func TestPush(t *testing.T) { - queue := newPriorityQueue() - for i := 0; i < 10; i += 2 { - item := createPoolItem(i) - queue.Push(item) - } - for i := 1; i < 10; i += 2 { - item := createPoolItem(i) - queue.Push(item) - } - - i := 0 - for { - item := queue.Pop() - if item == nil { - break - } - assert.Equal(t, fmt.Sprintf("%d", i), item.res.GetResourceID()) - i++ - } - assert.Equal(t, 10, i) -} - -func TestRob(t *testing.T) { - queue := newPriorityQueue() - for i := 0; i < 100; i += 2 { - item := createPoolItem(i) - queue.Push(item) - } - assert.Nil(t, queue.Rob("5")) - item := queue.Rob("6") - - assert.Equal(t, "6", item.res.GetResourceID()) - assert.Equal(t, 49, queue.Size()) -} - -func TestFind(t *testing.T) { - queue := newPriorityQueue() - for i := 0; i < 100; i += 2 { - item := createPoolItem(i) - queue.Push(item) - } - assert.Nil(t, queue.Find("5")) - item := queue.Find("6") - - assert.Equal(t, "6", item.res.GetResourceID()) - assert.Equal(t, 50, queue.Size()) -} diff --git a/pkg/tracing/rpc.go b/pkg/tracing/rpc.go index 44def1bb..5a736e68 100644 --- a/pkg/tracing/rpc.go +++ b/pkg/tracing/rpc.go @@ -83,36 +83,17 @@ func (t *tracingRPC) ResourceExecute(request *rpc.ResourceExecuteRequest, server return nil } -func (t *tracingRPC) GetResourceMapping(_ context.Context, _ *rpc.Placeholder) (*rpc.PodResourceMappingReply, error) { +func (t *tracingRPC) GetResourceMapping(_ context.Context, _ *rpc.Placeholder) (*rpc.ResourceMappingReply, error) { mapping, err := t.tracer.GetResourceMapping() if err != nil { return nil, err } - var info []*rpc.PodResourceMapping - for _, m := range mapping { - info = append(info, toRPCMapping(*m)) - } - - return &rpc.PodResourceMappingReply{ - Info: info, + return &rpc.ResourceMappingReply{ + Info: mapping, }, nil } -func toRPCMapping(res PodMapping) *rpc.PodResourceMapping { - rMapping := rpc.PodResourceMapping{ - Type: rpc.ResourceMappingType_MappingTypeNormal, - PodName: res.Name, - ResourceName: res.LocalResID, - FactoryResourceName: res.RemoteResID, - } - if !res.Valid { - rMapping.Type = rpc.ResourceMappingType_MappingTypeError - } - - return &rMapping -} - func toRPCEntry(entry MapKeyValueEntry) *rpc.MapKeyValueEntry { return &rpc.MapKeyValueEntry{ Key: entry.Key, diff --git a/pkg/tracing/tracing.go b/pkg/tracing/tracing.go index a40681e6..1932c3e0 100644 --- a/pkg/tracing/tracing.go +++ b/pkg/tracing/tracing.go @@ -5,7 +5,8 @@ import ( "fmt" "sync" - "github.com/AliyunContainerService/terway/types" + "github.com/AliyunContainerService/terway/rpc" + "github.com/AliyunContainerService/terway/types/daemon" ) const ( @@ -22,17 +23,6 @@ const ( AllocResourceFailed = "AllocResourceFailed" ) -// PodMapping PodMapping -type PodMapping struct { - Name string - Namespace string - Valid bool - - PodBindResID string - LocalResID string - RemoteResID string -} - var ( defaultTracer Tracer ) @@ -56,23 +46,23 @@ type TraceHandler interface { // ResourcePoolStats define two pool lo and remote type ResourcePoolStats interface { - GetLocal() map[string]types.Res - GetRemote() map[string]types.Res + GetLocal() map[string]daemon.Res + GetRemote() map[string]daemon.Res } // FakeResourcePoolStats for test type FakeResourcePoolStats struct { - Local map[string]types.Res - Remote map[string]types.Res + Local map[string]daemon.Res + Remote map[string]daemon.Res } // GetLocal GetLocal -func (f *FakeResourcePoolStats) GetLocal() map[string]types.Res { +func (f *FakeResourcePoolStats) GetLocal() map[string]daemon.Res { return f.Local } // GetRemote GetRemote -func (f *FakeResourcePoolStats) GetRemote() map[string]types.Res { +func (f *FakeResourcePoolStats) GetRemote() map[string]daemon.Res { return f.Remote } @@ -83,7 +73,7 @@ type ResourceMappingHandler interface { // ResMapping ResMapping type ResMapping interface { - GetResourceMapping() ([]*PodMapping, error) + GetResourceMapping() ([]*rpc.ResourceMapping, error) } // PodEventRecorder records event on pod @@ -256,7 +246,7 @@ func (t *Tracer) RecordNodeEvent(eventType, reason, message string) error { // GetResourceMapping gives the resource mapping from the handler // if the handler has not been registered, there will be error -func (t *Tracer) GetResourceMapping() ([]*PodMapping, error) { +func (t *Tracer) GetResourceMapping() ([]*rpc.ResourceMapping, error) { if t.resourceMapping == nil { return nil, errors.New("no resource mapping handler registered") } diff --git a/pkg/utils/config.go b/pkg/utils/config.go deleted file mode 100644 index cea1ab88..00000000 --- a/pkg/utils/config.go +++ /dev/null @@ -1,15 +0,0 @@ -package utils - -func Minimal(a int) int { - if a < 0 { - return 0 - } - return a -} - -func Max(a, b int) int { - if a > b { - return a - } - return b -} diff --git a/pkg/utils/k8s.go b/pkg/utils/k8s.go index 0474c6d1..d04d2c26 100644 --- a/pkg/utils/k8s.go +++ b/pkg/utils/k8s.go @@ -1,6 +1,7 @@ package utils import ( + "fmt" "time" corev1 "k8s.io/api/core/v1" @@ -75,6 +76,10 @@ func PodSandboxExited(p *corev1.Pod) bool { } } +func PodInfoKey(namespace, name string) string { + return fmt.Sprintf("%s/%s", namespace, name) +} + var ( // DefaultPatchBackoff for patch status field DefaultPatchBackoff = wait.Backoff{ diff --git a/plugin/terway/cni_linux.go b/plugin/terway/cni_linux.go index d67cb30f..e7d75c98 100644 --- a/plugin/terway/cni_linux.go +++ b/plugin/terway/cni_linux.go @@ -158,6 +158,11 @@ func doCmdAdd(ctx context.Context, logger *logrus.Entry, client rpc.TerwayBacken ipv4, ipv6 := allocResult.IPv4, allocResult.IPv6 + if !ipv4 && !ipv6 { + err = fmt.Errorf("cmdAdd: alloc ip no valid ip type") + return + } + hostIPSet, err := utils.GetHostIP(ipv4, ipv6) if err != nil { return @@ -421,6 +426,9 @@ func doCmdCheck(ctx context.Context, logger *logrus.Entry, client rpc.TerwayBack } ipv4, ipv6 := getResult.IPv4, getResult.IPv6 + if !ipv4 && !ipv6 { + return fmt.Errorf("cmdCheck: no valid ip type") + } hostIPSet, err := utils.GetHostIP(ipv4, ipv6) if err != nil { diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go index f64e2592..975a8513 100644 --- a/rpc/rpc.pb.go +++ b/rpc/rpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.0 -// protoc v3.19.4 +// protoc-gen-go v1.31.0 +// protoc v4.25.1 // source: rpc.proto package rpc @@ -434,12 +434,11 @@ type AllocIPReply struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` - IPType IPType `protobuf:"varint,2,opt,name=IPType,proto3,enum=rpc.IPType" json:"IPType,omitempty"` - IPv4 bool `protobuf:"varint,3,opt,name=IPv4,proto3" json:"IPv4,omitempty"` - IPv6 bool `protobuf:"varint,4,opt,name=IPv6,proto3" json:"IPv6,omitempty"` - NetConfs []*NetConf `protobuf:"bytes,5,rep,name=NetConfs,proto3" json:"NetConfs,omitempty"` - EnableTrunking bool `protobuf:"varint,6,opt,name=EnableTrunking,proto3" json:"EnableTrunking,omitempty"` + Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` + IPType IPType `protobuf:"varint,2,opt,name=IPType,proto3,enum=rpc.IPType" json:"IPType,omitempty"` + IPv4 bool `protobuf:"varint,3,opt,name=IPv4,proto3" json:"IPv4,omitempty"` + IPv6 bool `protobuf:"varint,4,opt,name=IPv6,proto3" json:"IPv6,omitempty"` + NetConfs []*NetConf `protobuf:"bytes,5,rep,name=NetConfs,proto3" json:"NetConfs,omitempty"` } func (x *AllocIPReply) Reset() { @@ -509,13 +508,6 @@ func (x *AllocIPReply) GetNetConfs() []*NetConf { return nil } -func (x *AllocIPReply) GetEnableTrunking() bool { - if x != nil { - return x.EnableTrunking - } - return false -} - type BasicInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1011,13 +1003,12 @@ type GetInfoReply struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - IPType IPType `protobuf:"varint,1,opt,name=IPType,proto3,enum=rpc.IPType" json:"IPType,omitempty"` - Success bool `protobuf:"varint,2,opt,name=Success,proto3" json:"Success,omitempty"` - IPv4 bool `protobuf:"varint,3,opt,name=IPv4,proto3" json:"IPv4,omitempty"` - IPv6 bool `protobuf:"varint,4,opt,name=IPv6,proto3" json:"IPv6,omitempty"` - NetConfs []*NetConf `protobuf:"bytes,5,rep,name=NetConfs,proto3" json:"NetConfs,omitempty"` - EnableTrunking bool `protobuf:"varint,6,opt,name=EnableTrunking,proto3" json:"EnableTrunking,omitempty"` - Error Error `protobuf:"varint,7,opt,name=Error,proto3,enum=rpc.Error" json:"Error,omitempty"` + IPType IPType `protobuf:"varint,1,opt,name=IPType,proto3,enum=rpc.IPType" json:"IPType,omitempty"` + Success bool `protobuf:"varint,2,opt,name=Success,proto3" json:"Success,omitempty"` + IPv4 bool `protobuf:"varint,3,opt,name=IPv4,proto3" json:"IPv4,omitempty"` + IPv6 bool `protobuf:"varint,4,opt,name=IPv6,proto3" json:"IPv6,omitempty"` + NetConfs []*NetConf `protobuf:"bytes,5,rep,name=NetConfs,proto3" json:"NetConfs,omitempty"` + Error Error `protobuf:"varint,7,opt,name=Error,proto3,enum=rpc.Error" json:"Error,omitempty"` } func (x *GetInfoReply) Reset() { @@ -1087,13 +1078,6 @@ func (x *GetInfoReply) GetNetConfs() []*NetConf { return nil } -func (x *GetInfoReply) GetEnableTrunking() bool { - if x != nil { - return x.EnableTrunking - } - return false -} - func (x *GetInfoReply) GetError() Error { if x != nil { return x.Error @@ -1276,7 +1260,7 @@ var file_rpc_proto_rawDesc = []byte{ 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x0b, 0x45, 0x78, 0x74, 0x72, 0x61, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, - 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x22, 0xc7, 0x01, 0x0a, + 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x22, 0x9f, 0x01, 0x0a, 0x0c, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x49, 0x50, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x23, 0x0a, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, @@ -1286,85 +1270,80 @@ var file_rpc_proto_rawDesc = []byte{ 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x36, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, 0x36, 0x12, 0x28, 0x0a, 0x08, 0x4e, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x65, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x08, 0x4e, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x73, 0x12, 0x26, - 0x0a, 0x0e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x72, 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x72, - 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x22, 0xab, 0x01, 0x0a, 0x09, 0x42, 0x61, 0x73, 0x69, 0x63, - 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x20, 0x0a, 0x05, 0x50, 0x6f, 0x64, 0x49, 0x50, 0x18, 0x01, 0x20, + 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x08, 0x4e, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x73, 0x22, 0xab, + 0x01, 0x0a, 0x09, 0x42, 0x61, 0x73, 0x69, 0x63, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x20, 0x0a, 0x05, + 0x50, 0x6f, 0x64, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, 0x70, + 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, 0x52, 0x05, 0x50, 0x6f, 0x64, 0x49, 0x50, 0x12, 0x24, + 0x0a, 0x07, 0x50, 0x6f, 0x64, 0x43, 0x49, 0x44, 0x52, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, 0x52, 0x07, 0x50, 0x6f, 0x64, + 0x43, 0x49, 0x44, 0x52, 0x12, 0x28, 0x0a, 0x09, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x49, + 0x50, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, + 0x53, 0x65, 0x74, 0x52, 0x09, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x49, 0x50, 0x12, 0x2c, + 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x49, 0x44, 0x52, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, 0x52, - 0x05, 0x50, 0x6f, 0x64, 0x49, 0x50, 0x12, 0x24, 0x0a, 0x07, 0x50, 0x6f, 0x64, 0x43, 0x49, 0x44, - 0x52, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, - 0x53, 0x65, 0x74, 0x52, 0x07, 0x50, 0x6f, 0x64, 0x43, 0x49, 0x44, 0x52, 0x12, 0x28, 0x0a, 0x09, - 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x49, 0x50, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, 0x52, 0x09, 0x47, 0x61, 0x74, - 0x65, 0x77, 0x61, 0x79, 0x49, 0x50, 0x12, 0x2c, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x43, 0x49, 0x44, 0x52, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, 0x70, - 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, 0x52, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x43, 0x49, 0x44, 0x52, 0x22, 0x6d, 0x0a, 0x07, 0x45, 0x4e, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x10, 0x0a, 0x03, 0x4d, 0x41, 0x43, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x4d, 0x41, - 0x43, 0x12, 0x14, 0x0a, 0x05, 0x54, 0x72, 0x75, 0x6e, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x05, 0x54, 0x72, 0x75, 0x6e, 0x6b, 0x12, 0x10, 0x0a, 0x03, 0x56, 0x69, 0x64, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x56, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x09, 0x47, 0x61, 0x74, - 0x65, 0x77, 0x61, 0x79, 0x49, 0x50, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, - 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, 0x52, 0x09, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x49, 0x50, 0x22, 0x19, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, - 0x44, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x44, 0x73, 0x74, 0x22, 0x61, - 0x0a, 0x03, 0x50, 0x6f, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, - 0x16, 0x0a, 0x06, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x06, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x4e, 0x65, 0x74, 0x77, 0x6f, - 0x72, 0x6b, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0f, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, - 0x79, 0x22, 0x93, 0x02, 0x0a, 0x10, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x50, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, - 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x73, 0x50, - 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, - 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x12, 0x36, 0x0a, 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x43, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x06, 0x49, 0x50, 0x54, 0x79, - 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, - 0x50, 0x54, 0x79, 0x70, 0x65, 0x52, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x12, 0x26, 0x0a, - 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, 0x52, 0x08, 0x49, 0x50, 0x76, - 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, 0x12, - 0x16, 0x0a, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x9e, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x6c, 0x65, - 0x61, 0x73, 0x65, 0x49, 0x50, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, - 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x12, 0x26, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, - 0x65, 0x74, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x22, 0x0a, 0x0c, - 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, - 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x34, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, - 0x49, 0x50, 0x76, 0x34, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x36, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, 0x36, 0x22, 0x92, 0x01, 0x0a, 0x0e, 0x47, 0x65, 0x74, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x4b, - 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x4b, - 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, - 0x6e, 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, - 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0xe9, 0x01, - 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x23, - 0x0a, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, - 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x52, 0x06, 0x49, 0x50, 0x54, - 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, - 0x04, 0x49, 0x50, 0x76, 0x34, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, - 0x34, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x36, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x04, 0x49, 0x50, 0x76, 0x36, 0x12, 0x28, 0x0a, 0x08, 0x4e, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, - 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x65, - 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x08, 0x4e, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x73, 0x12, - 0x26, 0x0a, 0x0e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x72, 0x75, 0x6e, 0x6b, 0x69, 0x6e, - 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x54, - 0x72, 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x20, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x49, 0x44, 0x52, 0x22, 0x6d, 0x0a, 0x07, + 0x45, 0x4e, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x10, 0x0a, 0x03, 0x4d, 0x41, 0x43, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x4d, 0x41, 0x43, 0x12, 0x14, 0x0a, 0x05, 0x54, 0x72, 0x75, + 0x6e, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x54, 0x72, 0x75, 0x6e, 0x6b, 0x12, + 0x10, 0x0a, 0x03, 0x56, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x56, 0x69, + 0x64, 0x12, 0x28, 0x0a, 0x09, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x49, 0x50, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, + 0x52, 0x09, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x49, 0x50, 0x22, 0x19, 0x0a, 0x05, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x44, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x44, 0x73, 0x74, 0x22, 0x61, 0x0a, 0x03, 0x50, 0x6f, 0x64, 0x12, 0x18, 0x0a, + 0x07, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, + 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x45, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, + 0x28, 0x0a, 0x0f, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, + 0x74, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, + 0x6b, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x22, 0x93, 0x02, 0x0a, 0x10, 0x52, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x50, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, + 0x0a, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, + 0x0a, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x4b, 0x38, 0x73, 0x50, + 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, + 0x49, 0x6e, 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, + 0x12, 0x23, 0x0a, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x0b, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x52, 0x06, 0x49, + 0x50, 0x54, 0x79, 0x70, 0x65, 0x12, 0x26, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, + 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, + 0x53, 0x65, 0x74, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x18, 0x0a, + 0x07, 0x4d, 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x4d, 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, + 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, + 0x9e, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x50, 0x52, 0x65, 0x70, + 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x26, 0x0a, 0x08, + 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, + 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, + 0x41, 0x64, 0x64, 0x72, 0x12, 0x22, 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, + 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, + 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x34, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, 0x34, 0x12, 0x12, 0x0a, 0x04, + 0x49, 0x50, 0x76, 0x36, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, 0x36, + 0x22, 0x92, 0x01, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, + 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, + 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x36, 0x0a, + 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x4b, + 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0xc1, 0x01, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x23, 0x0a, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x53, + 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x34, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, 0x34, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, + 0x36, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, 0x36, 0x12, 0x28, 0x0a, + 0x08, 0x4e, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x0c, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x08, 0x4e, + 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x73, 0x12, 0x20, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xec, 0x01, 0x0a, 0x0c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x32, 0x0a, 0x0b, 0x45, 0x76, diff --git a/rpc/rpc.proto b/rpc/rpc.proto index dd765968..de387798 100644 --- a/rpc/rpc.proto +++ b/rpc/rpc.proto @@ -42,7 +42,6 @@ message AllocIPReply { bool IPv4 = 3; bool IPv6 = 4; repeated NetConf NetConfs = 5; - bool EnableTrunking = 6; } message BasicInfo { @@ -106,7 +105,6 @@ message GetInfoReply { bool IPv4 = 3; bool IPv6 = 4; repeated NetConf NetConfs = 5; - bool EnableTrunking = 6; Error Error = 7; } diff --git a/rpc/rpc_grpc.pb.go b/rpc/rpc_grpc.pb.go index f9da6ab3..3358131d 100644 --- a/rpc/rpc_grpc.pb.go +++ b/rpc/rpc_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.19.4 +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.25.1 // source: rpc.proto package rpc @@ -18,6 +18,13 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + TerwayBackend_AllocIP_FullMethodName = "/rpc.TerwayBackend/AllocIP" + TerwayBackend_ReleaseIP_FullMethodName = "/rpc.TerwayBackend/ReleaseIP" + TerwayBackend_GetIPInfo_FullMethodName = "/rpc.TerwayBackend/GetIPInfo" + TerwayBackend_RecordEvent_FullMethodName = "/rpc.TerwayBackend/RecordEvent" +) + // TerwayBackendClient is the client API for TerwayBackend service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -38,7 +45,7 @@ func NewTerwayBackendClient(cc grpc.ClientConnInterface) TerwayBackendClient { func (c *terwayBackendClient) AllocIP(ctx context.Context, in *AllocIPRequest, opts ...grpc.CallOption) (*AllocIPReply, error) { out := new(AllocIPReply) - err := c.cc.Invoke(ctx, "/rpc.TerwayBackend/AllocIP", in, out, opts...) + err := c.cc.Invoke(ctx, TerwayBackend_AllocIP_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -47,7 +54,7 @@ func (c *terwayBackendClient) AllocIP(ctx context.Context, in *AllocIPRequest, o func (c *terwayBackendClient) ReleaseIP(ctx context.Context, in *ReleaseIPRequest, opts ...grpc.CallOption) (*ReleaseIPReply, error) { out := new(ReleaseIPReply) - err := c.cc.Invoke(ctx, "/rpc.TerwayBackend/ReleaseIP", in, out, opts...) + err := c.cc.Invoke(ctx, TerwayBackend_ReleaseIP_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -56,7 +63,7 @@ func (c *terwayBackendClient) ReleaseIP(ctx context.Context, in *ReleaseIPReques func (c *terwayBackendClient) GetIPInfo(ctx context.Context, in *GetInfoRequest, opts ...grpc.CallOption) (*GetInfoReply, error) { out := new(GetInfoReply) - err := c.cc.Invoke(ctx, "/rpc.TerwayBackend/GetIPInfo", in, out, opts...) + err := c.cc.Invoke(ctx, TerwayBackend_GetIPInfo_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -65,7 +72,7 @@ func (c *terwayBackendClient) GetIPInfo(ctx context.Context, in *GetInfoRequest, func (c *terwayBackendClient) RecordEvent(ctx context.Context, in *EventRequest, opts ...grpc.CallOption) (*EventReply, error) { out := new(EventReply) - err := c.cc.Invoke(ctx, "/rpc.TerwayBackend/RecordEvent", in, out, opts...) + err := c.cc.Invoke(ctx, TerwayBackend_RecordEvent_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -122,7 +129,7 @@ func _TerwayBackend_AllocIP_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/rpc.TerwayBackend/AllocIP", + FullMethod: TerwayBackend_AllocIP_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TerwayBackendServer).AllocIP(ctx, req.(*AllocIPRequest)) @@ -140,7 +147,7 @@ func _TerwayBackend_ReleaseIP_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/rpc.TerwayBackend/ReleaseIP", + FullMethod: TerwayBackend_ReleaseIP_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TerwayBackendServer).ReleaseIP(ctx, req.(*ReleaseIPRequest)) @@ -158,7 +165,7 @@ func _TerwayBackend_GetIPInfo_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/rpc.TerwayBackend/GetIPInfo", + FullMethod: TerwayBackend_GetIPInfo_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TerwayBackendServer).GetIPInfo(ctx, req.(*GetInfoRequest)) @@ -176,7 +183,7 @@ func _TerwayBackend_RecordEvent_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/rpc.TerwayBackend/RecordEvent", + FullMethod: TerwayBackend_RecordEvent_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TerwayBackendServer).RecordEvent(ctx, req.(*EventRequest)) diff --git a/rpc/tracing.pb.go b/rpc/tracing.pb.go index 49a53f04..403e15b7 100644 --- a/rpc/tracing.pb.go +++ b/rpc/tracing.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.0 -// protoc v3.19.4 +// protoc-gen-go v1.31.0 +// protoc v4.25.1 // source: tracing.proto package rpc @@ -20,55 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type ResourceMappingType int32 - -const ( - ResourceMappingType_MappingTypeNormal ResourceMappingType = 0 // Pod -> Resource -> Factory - ResourceMappingType_MappingTypeIdle ResourceMappingType = 1 // x -> Resource -> Factory - ResourceMappingType_MappingTypeError ResourceMappingType = 2 // x -> x -> Factory -) - -// Enum value maps for ResourceMappingType. -var ( - ResourceMappingType_name = map[int32]string{ - 0: "MappingTypeNormal", - 1: "MappingTypeIdle", - 2: "MappingTypeError", - } - ResourceMappingType_value = map[string]int32{ - "MappingTypeNormal": 0, - "MappingTypeIdle": 1, - "MappingTypeError": 2, - } -) - -func (x ResourceMappingType) Enum() *ResourceMappingType { - p := new(ResourceMappingType) - *p = x - return p -} - -func (x ResourceMappingType) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (ResourceMappingType) Descriptor() protoreflect.EnumDescriptor { - return file_tracing_proto_enumTypes[0].Descriptor() -} - -func (ResourceMappingType) Type() protoreflect.EnumType { - return &file_tracing_proto_enumTypes[0] -} - -func (x ResourceMappingType) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use ResourceMappingType.Descriptor instead. -func (ResourceMappingType) EnumDescriptor() ([]byte, []int) { - return file_tracing_proto_rawDescGZIP(), []int{0} -} - type Placeholder struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -570,19 +521,21 @@ func (x *ResourceTraceReply) GetTrace() []*MapKeyValueEntry { return nil } -type PodResourceMapping struct { +type ResourceMapping struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Type ResourceMappingType `protobuf:"varint,1,opt,name=type,proto3,enum=rpc.ResourceMappingType" json:"type,omitempty"` - PodName string `protobuf:"bytes,2,opt,name=PodName,proto3" json:"PodName,omitempty"` - ResourceName string `protobuf:"bytes,3,opt,name=ResourceName,proto3" json:"ResourceName,omitempty"` - FactoryResourceName string `protobuf:"bytes,4,opt,name=FactoryResourceName,proto3" json:"FactoryResourceName,omitempty"` + NetworkInterfaceID string `protobuf:"bytes,1,opt,name=NetworkInterfaceID,proto3" json:"NetworkInterfaceID,omitempty"` + MAC string `protobuf:"bytes,2,opt,name=MAC,proto3" json:"MAC,omitempty"` + Type string `protobuf:"bytes,3,opt,name=Type,proto3" json:"Type,omitempty"` + AllocInhibitExpireAt string `protobuf:"bytes,4,opt,name=AllocInhibitExpireAt,proto3" json:"AllocInhibitExpireAt,omitempty"` + Status string `protobuf:"bytes,5,opt,name=Status,proto3" json:"Status,omitempty"` + Info []string `protobuf:"bytes,6,rep,name=Info,proto3" json:"Info,omitempty"` } -func (x *PodResourceMapping) Reset() { - *x = PodResourceMapping{} +func (x *ResourceMapping) Reset() { + *x = ResourceMapping{} if protoimpl.UnsafeEnabled { mi := &file_tracing_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -590,13 +543,13 @@ func (x *PodResourceMapping) Reset() { } } -func (x *PodResourceMapping) String() string { +func (x *ResourceMapping) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PodResourceMapping) ProtoMessage() {} +func (*ResourceMapping) ProtoMessage() {} -func (x *PodResourceMapping) ProtoReflect() protoreflect.Message { +func (x *ResourceMapping) ProtoReflect() protoreflect.Message { mi := &file_tracing_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -608,49 +561,63 @@ func (x *PodResourceMapping) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PodResourceMapping.ProtoReflect.Descriptor instead. -func (*PodResourceMapping) Descriptor() ([]byte, []int) { +// Deprecated: Use ResourceMapping.ProtoReflect.Descriptor instead. +func (*ResourceMapping) Descriptor() ([]byte, []int) { return file_tracing_proto_rawDescGZIP(), []int{10} } -func (x *PodResourceMapping) GetType() ResourceMappingType { +func (x *ResourceMapping) GetNetworkInterfaceID() string { if x != nil { - return x.Type + return x.NetworkInterfaceID + } + return "" +} + +func (x *ResourceMapping) GetMAC() string { + if x != nil { + return x.MAC } - return ResourceMappingType_MappingTypeNormal + return "" } -func (x *PodResourceMapping) GetPodName() string { +func (x *ResourceMapping) GetType() string { if x != nil { - return x.PodName + return x.Type } return "" } -func (x *PodResourceMapping) GetResourceName() string { +func (x *ResourceMapping) GetAllocInhibitExpireAt() string { if x != nil { - return x.ResourceName + return x.AllocInhibitExpireAt } return "" } -func (x *PodResourceMapping) GetFactoryResourceName() string { +func (x *ResourceMapping) GetStatus() string { if x != nil { - return x.FactoryResourceName + return x.Status } return "" } -type PodResourceMappingReply struct { +func (x *ResourceMapping) GetInfo() []string { + if x != nil { + return x.Info + } + return nil +} + +type ResourceMappingReply struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Info []*PodResourceMapping `protobuf:"bytes,1,rep,name=info,proto3" json:"info,omitempty"` + Info []*ResourceMapping `protobuf:"bytes,1,rep,name=info,proto3" json:"info,omitempty"` } -func (x *PodResourceMappingReply) Reset() { - *x = PodResourceMappingReply{} +func (x *ResourceMappingReply) Reset() { + *x = ResourceMappingReply{} if protoimpl.UnsafeEnabled { mi := &file_tracing_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -658,13 +625,13 @@ func (x *PodResourceMappingReply) Reset() { } } -func (x *PodResourceMappingReply) String() string { +func (x *ResourceMappingReply) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PodResourceMappingReply) ProtoMessage() {} +func (*ResourceMappingReply) ProtoMessage() {} -func (x *PodResourceMappingReply) ProtoReflect() protoreflect.Message { +func (x *ResourceMappingReply) ProtoReflect() protoreflect.Message { mi := &file_tracing_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -676,12 +643,12 @@ func (x *PodResourceMappingReply) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PodResourceMappingReply.ProtoReflect.Descriptor instead. -func (*PodResourceMappingReply) Descriptor() ([]byte, []int) { +// Deprecated: Use ResourceMappingReply.ProtoReflect.Descriptor instead. +func (*ResourceMappingReply) Descriptor() ([]byte, []int) { return file_tracing_proto_rawDescGZIP(), []int{11} } -func (x *PodResourceMappingReply) GetInfo() []*PodResourceMapping { +func (x *ResourceMappingReply) GetInfo() []*ResourceMapping { if x != nil { return x.Info } @@ -729,58 +696,53 @@ var file_tracing_proto_rawDesc = []byte{ 0x72, 0x63, 0x65, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x2b, 0x0a, 0x05, 0x54, 0x72, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x61, 0x70, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x05, 0x54, 0x72, 0x61, 0x63, 0x65, 0x22, 0xb2, 0x01, 0x0a, 0x12, 0x50, - 0x6f, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x12, 0x2c, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x61, - 0x70, 0x70, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, - 0x18, 0x0a, 0x07, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x30, 0x0a, - 0x13, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x46, 0x61, 0x63, 0x74, - 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, - 0x46, 0x0a, 0x17, 0x50, 0x6f, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x61, - 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x2b, 0x0a, 0x04, 0x69, 0x6e, - 0x66, 0x6f, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x50, - 0x6f, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x2a, 0x57, 0x0a, 0x13, 0x52, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, 0x15, - 0x0a, 0x11, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x4e, 0x6f, 0x72, - 0x6d, 0x61, 0x6c, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, - 0x54, 0x79, 0x70, 0x65, 0x49, 0x64, 0x6c, 0x65, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x4d, 0x61, - 0x70, 0x70, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x02, - 0x32, 0xbe, 0x03, 0x0a, 0x0d, 0x54, 0x65, 0x72, 0x77, 0x61, 0x79, 0x54, 0x72, 0x61, 0x63, 0x69, - 0x6e, 0x67, 0x12, 0x3e, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x10, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x6c, 0x61, - 0x63, 0x65, 0x68, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x1a, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x70, - 0x6c, 0x79, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x12, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, - 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x4e, 0x61, 0x6d, 0x65, - 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x4b, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1c, 0x2e, 0x72, 0x70, - 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x4e, 0x61, - 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, - 0x70, 0x6c, 0x79, 0x12, 0x49, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x54, 0x72, 0x61, 0x63, 0x65, 0x12, 0x1c, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x4b, - 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x65, 0x12, 0x1b, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, - 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x45, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x30, 0x01, 0x12, 0x44, 0x0a, 0x12, 0x47, - 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x12, 0x10, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x6c, 0x61, 0x63, 0x65, 0x68, 0x6f, 0x6c, - 0x64, 0x65, 0x72, 0x1a, 0x1c, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x6f, 0x64, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x70, 0x6c, - 0x79, 0x42, 0x07, 0x5a, 0x05, 0x2e, 0x3b, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x74, 0x72, 0x79, 0x52, 0x05, 0x54, 0x72, 0x61, 0x63, 0x65, 0x22, 0xc7, 0x01, 0x0a, 0x0f, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x2e, + 0x0a, 0x12, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, + 0x63, 0x65, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x4e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x49, 0x44, 0x12, 0x10, + 0x0a, 0x03, 0x4d, 0x41, 0x43, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x4d, 0x41, 0x43, + 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x32, 0x0a, 0x14, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x49, 0x6e, 0x68, + 0x69, 0x62, 0x69, 0x74, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x14, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x49, 0x6e, 0x68, 0x69, 0x62, 0x69, 0x74, + 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x12, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, + 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x40, 0x0a, 0x14, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x28, 0x0a, 0x04, + 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, + 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x32, 0xbb, 0x03, 0x0a, 0x0d, 0x54, 0x65, 0x72, 0x77, 0x61, + 0x79, 0x54, 0x72, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x12, 0x3e, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x10, 0x2e, 0x72, + 0x70, 0x63, 0x2e, 0x50, 0x6c, 0x61, 0x63, 0x65, 0x68, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x1a, 0x18, + 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x54, 0x79, + 0x70, 0x65, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x4b, 0x0a, 0x11, + 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x1c, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x49, 0x0a, 0x10, 0x47, 0x65, 0x74, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x61, 0x63, 0x65, 0x12, 0x1c, 0x2e, + 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, + 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x72, 0x70, + 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x12, 0x4b, 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x1b, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x30, + 0x01, 0x12, 0x41, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x10, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x6c, + 0x61, 0x63, 0x65, 0x68, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x1a, 0x19, 0x2e, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x42, 0x07, 0x5a, 0x05, 0x2e, 0x3b, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -795,45 +757,42 @@ func file_tracing_proto_rawDescGZIP() []byte { return file_tracing_proto_rawDescData } -var file_tracing_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_tracing_proto_msgTypes = make([]protoimpl.MessageInfo, 12) var file_tracing_proto_goTypes = []interface{}{ - (ResourceMappingType)(0), // 0: rpc.ResourceMappingType - (*Placeholder)(nil), // 1: rpc.Placeholder - (*ResourcesTypesReply)(nil), // 2: rpc.ResourcesTypesReply - (*ResourcesNamesReply)(nil), // 3: rpc.ResourcesNamesReply - (*ResourceTypeRequest)(nil), // 4: rpc.ResourceTypeRequest - (*ResourceTypeNameRequest)(nil), // 5: rpc.ResourceTypeNameRequest - (*ResourceExecuteRequest)(nil), // 6: rpc.ResourceExecuteRequest - (*ResourceExecuteReply)(nil), // 7: rpc.ResourceExecuteReply - (*MapKeyValueEntry)(nil), // 8: rpc.MapKeyValueEntry - (*ResourceConfigReply)(nil), // 9: rpc.ResourceConfigReply - (*ResourceTraceReply)(nil), // 10: rpc.ResourceTraceReply - (*PodResourceMapping)(nil), // 11: rpc.PodResourceMapping - (*PodResourceMappingReply)(nil), // 12: rpc.PodResourceMappingReply + (*Placeholder)(nil), // 0: rpc.Placeholder + (*ResourcesTypesReply)(nil), // 1: rpc.ResourcesTypesReply + (*ResourcesNamesReply)(nil), // 2: rpc.ResourcesNamesReply + (*ResourceTypeRequest)(nil), // 3: rpc.ResourceTypeRequest + (*ResourceTypeNameRequest)(nil), // 4: rpc.ResourceTypeNameRequest + (*ResourceExecuteRequest)(nil), // 5: rpc.ResourceExecuteRequest + (*ResourceExecuteReply)(nil), // 6: rpc.ResourceExecuteReply + (*MapKeyValueEntry)(nil), // 7: rpc.MapKeyValueEntry + (*ResourceConfigReply)(nil), // 8: rpc.ResourceConfigReply + (*ResourceTraceReply)(nil), // 9: rpc.ResourceTraceReply + (*ResourceMapping)(nil), // 10: rpc.ResourceMapping + (*ResourceMappingReply)(nil), // 11: rpc.ResourceMappingReply } var file_tracing_proto_depIdxs = []int32{ - 8, // 0: rpc.ResourceConfigReply.Config:type_name -> rpc.MapKeyValueEntry - 8, // 1: rpc.ResourceTraceReply.Trace:type_name -> rpc.MapKeyValueEntry - 0, // 2: rpc.PodResourceMapping.type:type_name -> rpc.ResourceMappingType - 11, // 3: rpc.PodResourceMappingReply.info:type_name -> rpc.PodResourceMapping - 1, // 4: rpc.TerwayTracing.GetResourceTypes:input_type -> rpc.Placeholder - 4, // 5: rpc.TerwayTracing.GetResources:input_type -> rpc.ResourceTypeRequest - 5, // 6: rpc.TerwayTracing.GetResourceConfig:input_type -> rpc.ResourceTypeNameRequest - 5, // 7: rpc.TerwayTracing.GetResourceTrace:input_type -> rpc.ResourceTypeNameRequest - 6, // 8: rpc.TerwayTracing.ResourceExecute:input_type -> rpc.ResourceExecuteRequest - 1, // 9: rpc.TerwayTracing.GetResourceMapping:input_type -> rpc.Placeholder - 2, // 10: rpc.TerwayTracing.GetResourceTypes:output_type -> rpc.ResourcesTypesReply - 3, // 11: rpc.TerwayTracing.GetResources:output_type -> rpc.ResourcesNamesReply - 9, // 12: rpc.TerwayTracing.GetResourceConfig:output_type -> rpc.ResourceConfigReply - 10, // 13: rpc.TerwayTracing.GetResourceTrace:output_type -> rpc.ResourceTraceReply - 7, // 14: rpc.TerwayTracing.ResourceExecute:output_type -> rpc.ResourceExecuteReply - 12, // 15: rpc.TerwayTracing.GetResourceMapping:output_type -> rpc.PodResourceMappingReply - 10, // [10:16] is the sub-list for method output_type - 4, // [4:10] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 7, // 0: rpc.ResourceConfigReply.Config:type_name -> rpc.MapKeyValueEntry + 7, // 1: rpc.ResourceTraceReply.Trace:type_name -> rpc.MapKeyValueEntry + 10, // 2: rpc.ResourceMappingReply.info:type_name -> rpc.ResourceMapping + 0, // 3: rpc.TerwayTracing.GetResourceTypes:input_type -> rpc.Placeholder + 3, // 4: rpc.TerwayTracing.GetResources:input_type -> rpc.ResourceTypeRequest + 4, // 5: rpc.TerwayTracing.GetResourceConfig:input_type -> rpc.ResourceTypeNameRequest + 4, // 6: rpc.TerwayTracing.GetResourceTrace:input_type -> rpc.ResourceTypeNameRequest + 5, // 7: rpc.TerwayTracing.ResourceExecute:input_type -> rpc.ResourceExecuteRequest + 0, // 8: rpc.TerwayTracing.GetResourceMapping:input_type -> rpc.Placeholder + 1, // 9: rpc.TerwayTracing.GetResourceTypes:output_type -> rpc.ResourcesTypesReply + 2, // 10: rpc.TerwayTracing.GetResources:output_type -> rpc.ResourcesNamesReply + 8, // 11: rpc.TerwayTracing.GetResourceConfig:output_type -> rpc.ResourceConfigReply + 9, // 12: rpc.TerwayTracing.GetResourceTrace:output_type -> rpc.ResourceTraceReply + 6, // 13: rpc.TerwayTracing.ResourceExecute:output_type -> rpc.ResourceExecuteReply + 11, // 14: rpc.TerwayTracing.GetResourceMapping:output_type -> rpc.ResourceMappingReply + 9, // [9:15] is the sub-list for method output_type + 3, // [3:9] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_tracing_proto_init() } @@ -963,7 +922,7 @@ func file_tracing_proto_init() { } } file_tracing_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PodResourceMapping); i { + switch v := v.(*ResourceMapping); i { case 0: return &v.state case 1: @@ -975,7 +934,7 @@ func file_tracing_proto_init() { } } file_tracing_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PodResourceMappingReply); i { + switch v := v.(*ResourceMappingReply); i { case 0: return &v.state case 1: @@ -992,14 +951,13 @@ func file_tracing_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_tracing_proto_rawDesc, - NumEnums: 1, + NumEnums: 0, NumMessages: 12, NumExtensions: 0, NumServices: 1, }, GoTypes: file_tracing_proto_goTypes, DependencyIndexes: file_tracing_proto_depIdxs, - EnumInfos: file_tracing_proto_enumTypes, MessageInfos: file_tracing_proto_msgTypes, }.Build() File_tracing_proto = out.File diff --git a/rpc/tracing.proto b/rpc/tracing.proto index 10acd543..1dc5cfbd 100644 --- a/rpc/tracing.proto +++ b/rpc/tracing.proto @@ -8,7 +8,7 @@ service TerwayTracing { rpc GetResourceConfig(ResourceTypeNameRequest) returns (ResourceConfigReply); rpc GetResourceTrace(ResourceTypeNameRequest) returns (ResourceTraceReply); rpc ResourceExecute(ResourceExecuteRequest) returns (stream ResourceExecuteReply); - rpc GetResourceMapping(Placeholder) returns (PodResourceMappingReply); + rpc GetResourceMapping(Placeholder) returns (ResourceMappingReply); } message Placeholder {} @@ -53,19 +53,15 @@ message ResourceTraceReply { repeated MapKeyValueEntry Trace = 1; } -enum ResourceMappingType { - MappingTypeNormal = 0; // Pod -> Resource -> Factory - MappingTypeIdle = 1; // x -> Resource -> Factory - MappingTypeError = 2; // x -> x -> Factory +message ResourceMapping { + string NetworkInterfaceID = 1; + string MAC = 2; + string Type = 3; + string AllocInhibitExpireAt = 4 ; + string Status = 5; + repeated string Info = 6; } -message PodResourceMapping { - ResourceMappingType type = 1; - string PodName = 2; - string ResourceName = 3; - string FactoryResourceName = 4; +message ResourceMappingReply { + repeated ResourceMapping info = 1; } - -message PodResourceMappingReply { - repeated PodResourceMapping info = 1; -} \ No newline at end of file diff --git a/rpc/tracing_grpc.pb.go b/rpc/tracing_grpc.pb.go index 0d0f7488..95f9cf37 100644 --- a/rpc/tracing_grpc.pb.go +++ b/rpc/tracing_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.19.4 +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.25.1 // source: tracing.proto package rpc @@ -18,6 +18,15 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + TerwayTracing_GetResourceTypes_FullMethodName = "/rpc.TerwayTracing/GetResourceTypes" + TerwayTracing_GetResources_FullMethodName = "/rpc.TerwayTracing/GetResources" + TerwayTracing_GetResourceConfig_FullMethodName = "/rpc.TerwayTracing/GetResourceConfig" + TerwayTracing_GetResourceTrace_FullMethodName = "/rpc.TerwayTracing/GetResourceTrace" + TerwayTracing_ResourceExecute_FullMethodName = "/rpc.TerwayTracing/ResourceExecute" + TerwayTracing_GetResourceMapping_FullMethodName = "/rpc.TerwayTracing/GetResourceMapping" +) + // TerwayTracingClient is the client API for TerwayTracing service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -27,7 +36,7 @@ type TerwayTracingClient interface { GetResourceConfig(ctx context.Context, in *ResourceTypeNameRequest, opts ...grpc.CallOption) (*ResourceConfigReply, error) GetResourceTrace(ctx context.Context, in *ResourceTypeNameRequest, opts ...grpc.CallOption) (*ResourceTraceReply, error) ResourceExecute(ctx context.Context, in *ResourceExecuteRequest, opts ...grpc.CallOption) (TerwayTracing_ResourceExecuteClient, error) - GetResourceMapping(ctx context.Context, in *Placeholder, opts ...grpc.CallOption) (*PodResourceMappingReply, error) + GetResourceMapping(ctx context.Context, in *Placeholder, opts ...grpc.CallOption) (*ResourceMappingReply, error) } type terwayTracingClient struct { @@ -40,7 +49,7 @@ func NewTerwayTracingClient(cc grpc.ClientConnInterface) TerwayTracingClient { func (c *terwayTracingClient) GetResourceTypes(ctx context.Context, in *Placeholder, opts ...grpc.CallOption) (*ResourcesTypesReply, error) { out := new(ResourcesTypesReply) - err := c.cc.Invoke(ctx, "/rpc.TerwayTracing/GetResourceTypes", in, out, opts...) + err := c.cc.Invoke(ctx, TerwayTracing_GetResourceTypes_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -49,7 +58,7 @@ func (c *terwayTracingClient) GetResourceTypes(ctx context.Context, in *Placehol func (c *terwayTracingClient) GetResources(ctx context.Context, in *ResourceTypeRequest, opts ...grpc.CallOption) (*ResourcesNamesReply, error) { out := new(ResourcesNamesReply) - err := c.cc.Invoke(ctx, "/rpc.TerwayTracing/GetResources", in, out, opts...) + err := c.cc.Invoke(ctx, TerwayTracing_GetResources_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -58,7 +67,7 @@ func (c *terwayTracingClient) GetResources(ctx context.Context, in *ResourceType func (c *terwayTracingClient) GetResourceConfig(ctx context.Context, in *ResourceTypeNameRequest, opts ...grpc.CallOption) (*ResourceConfigReply, error) { out := new(ResourceConfigReply) - err := c.cc.Invoke(ctx, "/rpc.TerwayTracing/GetResourceConfig", in, out, opts...) + err := c.cc.Invoke(ctx, TerwayTracing_GetResourceConfig_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -67,7 +76,7 @@ func (c *terwayTracingClient) GetResourceConfig(ctx context.Context, in *Resourc func (c *terwayTracingClient) GetResourceTrace(ctx context.Context, in *ResourceTypeNameRequest, opts ...grpc.CallOption) (*ResourceTraceReply, error) { out := new(ResourceTraceReply) - err := c.cc.Invoke(ctx, "/rpc.TerwayTracing/GetResourceTrace", in, out, opts...) + err := c.cc.Invoke(ctx, TerwayTracing_GetResourceTrace_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -75,7 +84,7 @@ func (c *terwayTracingClient) GetResourceTrace(ctx context.Context, in *Resource } func (c *terwayTracingClient) ResourceExecute(ctx context.Context, in *ResourceExecuteRequest, opts ...grpc.CallOption) (TerwayTracing_ResourceExecuteClient, error) { - stream, err := c.cc.NewStream(ctx, &TerwayTracing_ServiceDesc.Streams[0], "/rpc.TerwayTracing/ResourceExecute", opts...) + stream, err := c.cc.NewStream(ctx, &TerwayTracing_ServiceDesc.Streams[0], TerwayTracing_ResourceExecute_FullMethodName, opts...) if err != nil { return nil, err } @@ -106,9 +115,9 @@ func (x *terwayTracingResourceExecuteClient) Recv() (*ResourceExecuteReply, erro return m, nil } -func (c *terwayTracingClient) GetResourceMapping(ctx context.Context, in *Placeholder, opts ...grpc.CallOption) (*PodResourceMappingReply, error) { - out := new(PodResourceMappingReply) - err := c.cc.Invoke(ctx, "/rpc.TerwayTracing/GetResourceMapping", in, out, opts...) +func (c *terwayTracingClient) GetResourceMapping(ctx context.Context, in *Placeholder, opts ...grpc.CallOption) (*ResourceMappingReply, error) { + out := new(ResourceMappingReply) + err := c.cc.Invoke(ctx, TerwayTracing_GetResourceMapping_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -124,7 +133,7 @@ type TerwayTracingServer interface { GetResourceConfig(context.Context, *ResourceTypeNameRequest) (*ResourceConfigReply, error) GetResourceTrace(context.Context, *ResourceTypeNameRequest) (*ResourceTraceReply, error) ResourceExecute(*ResourceExecuteRequest, TerwayTracing_ResourceExecuteServer) error - GetResourceMapping(context.Context, *Placeholder) (*PodResourceMappingReply, error) + GetResourceMapping(context.Context, *Placeholder) (*ResourceMappingReply, error) mustEmbedUnimplementedTerwayTracingServer() } @@ -147,7 +156,7 @@ func (UnimplementedTerwayTracingServer) GetResourceTrace(context.Context, *Resou func (UnimplementedTerwayTracingServer) ResourceExecute(*ResourceExecuteRequest, TerwayTracing_ResourceExecuteServer) error { return status.Errorf(codes.Unimplemented, "method ResourceExecute not implemented") } -func (UnimplementedTerwayTracingServer) GetResourceMapping(context.Context, *Placeholder) (*PodResourceMappingReply, error) { +func (UnimplementedTerwayTracingServer) GetResourceMapping(context.Context, *Placeholder) (*ResourceMappingReply, error) { return nil, status.Errorf(codes.Unimplemented, "method GetResourceMapping not implemented") } func (UnimplementedTerwayTracingServer) mustEmbedUnimplementedTerwayTracingServer() {} @@ -173,7 +182,7 @@ func _TerwayTracing_GetResourceTypes_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/rpc.TerwayTracing/GetResourceTypes", + FullMethod: TerwayTracing_GetResourceTypes_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TerwayTracingServer).GetResourceTypes(ctx, req.(*Placeholder)) @@ -191,7 +200,7 @@ func _TerwayTracing_GetResources_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/rpc.TerwayTracing/GetResources", + FullMethod: TerwayTracing_GetResources_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TerwayTracingServer).GetResources(ctx, req.(*ResourceTypeRequest)) @@ -209,7 +218,7 @@ func _TerwayTracing_GetResourceConfig_Handler(srv interface{}, ctx context.Conte } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/rpc.TerwayTracing/GetResourceConfig", + FullMethod: TerwayTracing_GetResourceConfig_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TerwayTracingServer).GetResourceConfig(ctx, req.(*ResourceTypeNameRequest)) @@ -227,7 +236,7 @@ func _TerwayTracing_GetResourceTrace_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/rpc.TerwayTracing/GetResourceTrace", + FullMethod: TerwayTracing_GetResourceTrace_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TerwayTracingServer).GetResourceTrace(ctx, req.(*ResourceTypeNameRequest)) @@ -266,7 +275,7 @@ func _TerwayTracing_GetResourceMapping_Handler(srv interface{}, ctx context.Cont } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/rpc.TerwayTracing/GetResourceMapping", + FullMethod: TerwayTracing_GetResourceMapping_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TerwayTracingServer).GetResourceMapping(ctx, req.(*Placeholder)) diff --git a/types/config.go b/types/config.go index 02d500bc..4545afe5 100644 --- a/types/config.go +++ b/types/config.go @@ -2,10 +2,14 @@ package types // PoolConfig configuration of pool and resource factory type PoolConfig struct { + EnableIPv4 bool + EnableIPv6 bool + Capacity int // the max res can hold in the pool MaxENI int // the max eni terway can be created (already exclude main eni) MaxMemberENI int // the max member eni can be created MaxIPPerENI int + BatchSize int MaxPoolSize int MinPoolSize int @@ -21,4 +25,6 @@ type PoolConfig struct { DisableSecurityGroupCheck bool TrunkENIID string + + ResourceGroupID string } diff --git a/types/daemon/cni.go b/types/daemon/cni.go new file mode 100644 index 00000000..4875ab1c --- /dev/null +++ b/types/daemon/cni.go @@ -0,0 +1,9 @@ +package daemon + +type CNI struct { + PodName string + PodNamespace string + PodID string + PodUID string + NetNSPath string +} diff --git a/types/daemon/config.go b/types/daemon/config.go index e5c26ac7..34b5d91b 100644 --- a/types/daemon/config.go +++ b/types/daemon/config.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/AliyunContainerService/terway/pkg/aliyun/instance" "github.com/AliyunContainerService/terway/types/secret" jsonpatch "github.com/evanphx/json-patch" @@ -11,7 +12,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - "github.com/AliyunContainerService/terway/pkg/aliyun" "github.com/AliyunContainerService/terway/types" "github.com/AliyunContainerService/terway/types/route" ) @@ -45,7 +45,7 @@ type Config struct { EnableENITrunking bool `yaml:"enable_eni_trunking" json:"enable_eni_trunking"` CustomStatefulWorkloadKinds []string `yaml:"custom_stateful_workload_kinds" json:"custom_stateful_workload_kinds"` IPAMType types.IPAMType `yaml:"ipam_type" json:"ipam_type"` // crd or default - ENICapPolicy types.ENICapPolicy `yaml:"eni_cap_policy" json:"eni_cap_policy"` // prefer trunk or secondary + ENICapPolicy ENICapPolicy `yaml:"eni_cap_policy" json:"eni_cap_policy"` // prefer trunk or secondary BackoffOverride map[string]wait.Backoff `json:"backoff_override,omitempty"` ExtraRoutes []route.Route `json:"extra_routes,omitempty"` DisableDevicePlugin bool `json:"disable_device_plugin"` @@ -83,12 +83,12 @@ func (c *Config) GetExtraRoutes() []string { func (c *Config) Populate() { if c.RegionID == "" { - c.RegionID = aliyun.GetInstanceMeta().RegionID + c.RegionID = instance.GetInstanceMeta().RegionID } if c.InstanceType == "" { // TODO support retreat from node label - c.InstanceType = aliyun.GetInstanceMeta().InstanceType + c.InstanceType = instance.GetInstanceMeta().InstanceType } if c.EniCapRatio == 0 { @@ -96,7 +96,7 @@ func (c *Config) Populate() { } if c.VSwitchSelectionPolicy == "" { - c.VSwitchSelectionPolicy = types.VSwitchSelectionPolicyRandom + c.VSwitchSelectionPolicy = VSwitchSelectionPolicyRandom } if c.IPStack == "" { diff --git a/types/res.go b/types/daemon/res.go similarity index 88% rename from types/res.go rename to types/daemon/res.go index 5f50a6f5..d5010086 100644 --- a/types/res.go +++ b/types/daemon/res.go @@ -1,12 +1,15 @@ -package types +package daemon import ( "net" "time" + + "github.com/AliyunContainerService/terway/types" ) // PodEipInfo store pod eip info // NOTE: this is the type store in db +// DEPRECATED type PodEipInfo struct { PodEip bool PodEipID string @@ -27,10 +30,10 @@ type PodInfo struct { TcIngress uint64 TcEgress uint64 PodNetworkType string - PodIP string // used for eip and mip - PodIPs IPSet // used for eip and mip + PodIP string // used for eip and mip + PodIPs types.IPSet // used for eip and mip SandboxExited bool - EipInfo PodEipInfo + EipInfo PodEipInfo // deprecated IPStickTime time.Duration PodENI bool PodUID string @@ -63,6 +66,8 @@ type PodResources struct { PodInfo *PodInfo NetNs *string ContainerID *string + + NetConf string } // GetResourceItemByType get pod resource by resource type diff --git a/types/daemon/types.go b/types/daemon/types.go new file mode 100644 index 00000000..e76615f4 --- /dev/null +++ b/types/daemon/types.go @@ -0,0 +1,190 @@ +package daemon + +import ( + "fmt" + + "github.com/AliyunContainerService/terway/types" +) + +const ( + PodNetworkTypeVPCIP = "VPCIP" + PodNetworkTypeVPCENI = "VPCENI" + PodNetworkTypeENIMultiIP = "ENIMultiIP" +) + +// DEPRECATED +type InternetChargeType string + +// EIP pay type +const ( + PayByBandwidth = InternetChargeType("PayByBandwidth") + PayByTraffic = InternetChargeType("PayByTraffic") +) + +// network resource type +const ( + ResourceTypeVeth = "veth" + ResourceTypeENI = "eni" + ResourceTypeENIIP = "eniIp" + ResourceTypeEIP = "eip" +) +const ( + ModeVPC = "VPC" + ModeENIMultiIP = "ENIMultiIP" + ModeENIOnly = "ENIOnly" +) + +// Vswitch Selection Policy +const ( + VSwitchSelectionPolicyRandom = "random" + VSwitchSelectionPolicyOrdered = "ordered" +) + +// ENICapPolicy how eni cap is calculated +type ENICapPolicy string + +// how eni cap is calculated +const ( + ENICapPolicyPreferTrunk = "preferTrunk" + ENICapPolicyDefault = "" +) + +// ENI aliyun ENI resource +type ENI struct { + ID string + MAC string + SecurityGroupIDs []string + + Trunk bool + + PrimaryIP types.IPSet + GatewayIP types.IPSet + + VSwitchCIDR types.IPNetSet + + VSwitchID string + + Type string +} + +// GetResourceID return mac address of eni +func (e *ENI) GetResourceID() string { + return e.MAC +} + +// GetType return type name +func (e *ENI) GetType() string { + return ResourceTypeENI +} + +func (e *ENI) ToResItems() []ResourceItem { + return []ResourceItem{ + { + Type: e.GetType(), + ID: e.GetResourceID(), + ENIID: e.ID, + ENIMAC: e.MAC, + IPv4: e.PrimaryIP.GetIPv4(), + IPv6: e.PrimaryIP.GetIPv6(), + }, + } +} + +// ENIIP aliyun secondary IP resource +type ENIIP struct { + ENI *ENI + IPSet types.IPSet +} + +// GetResourceID return mac address of eni and secondary ip address +func (e *ENIIP) GetResourceID() string { + return fmt.Sprintf("%s.%s", e.ENI.GetResourceID(), e.IPSet.String()) +} + +// GetType return type name +func (e *ENIIP) GetType() string { + return ResourceTypeENIIP +} + +func (e *ENIIP) ToResItems() []ResourceItem { + return []ResourceItem{ + { + Type: e.GetType(), + ID: e.GetResourceID(), + ENIID: e.ENI.ID, + ENIMAC: e.ENI.MAC, + IPv4: e.IPSet.GetIPv4(), + IPv6: e.IPSet.GetIPv6(), + }, + } +} + +// Veth veth pair resource on system +type Veth struct { + HostVeth string +} + +// GetResourceID return host veth name of veth resource +func (e *Veth) GetResourceID() string { + return e.HostVeth +} + +// GetType return type name +func (e *Veth) GetType() string { + return ResourceTypeVeth +} + +func (e *Veth) ToResItems() []ResourceItem { + return []ResourceItem{ + { + Type: e.GetType(), + ID: e.GetResourceID(), + }, + } +} + +// NetworkResource interface of network resources +type NetworkResource interface { + GetResourceID() string + GetType() string + ToResItems() []ResourceItem +} + +// Res is the func for res +type Res interface { + GetID() string + GetType() string + GetStatus() ResStatus +} + +// ResStatus ResStatus +type ResStatus int + +// ResStatus +const ( + ResStatusInvalid ResStatus = iota + ResStatusIdle + ResStatusInUse +) + +// FakeRes for test +type FakeRes struct { + ID string + Type string + Status ResStatus +} + +// GetID GetID +func (r *FakeRes) GetID() string { + return r.ID +} + +// GetType GetType +func (r *FakeRes) GetType() string { + return r.Type +} + +// GetStatus GetStatus +func (r *FakeRes) GetStatus() ResStatus { + return r.Status +} diff --git a/types/errors.go b/types/errors.go new file mode 100644 index 00000000..1c482d3b --- /dev/null +++ b/types/errors.go @@ -0,0 +1,35 @@ +package types + +import ( + "fmt" +) + +const ( + InternalError ErrCode = "InternalError" + InvalidArgsErrCode ErrCode = "InvalidArgs" + InvalidDataType ErrCode = "InvalidDataType" + + PodIsProcessing ErrCode = "PodIsProcessing" + + NotFoundInMetadata ErrCode = "NotFoundInMetadata" + + OpenAPIErrCode ErrCode = "OpenAPIErr" + PodENINotReady ErrCode = "PodENINotReady" +) + +type ErrCode string + +type Error struct { + Code ErrCode + Msg string + + R error +} + +func (e *Error) Error() string { + return fmt.Sprintf("code: %s, msg: %s", e.Code, e.Msg) +} + +func (e *Error) Unwrap() error { + return e.R +} diff --git a/types/types.go b/types/types.go index 3c24618e..dc180a56 100644 --- a/types/types.go +++ b/types/types.go @@ -17,37 +17,14 @@ limitations under the License. package types import ( - "fmt" "net" + "net/netip" "strings" terwayIP "github.com/AliyunContainerService/terway/pkg/ip" - "github.com/AliyunContainerService/terway/pkg/utils" "github.com/AliyunContainerService/terway/rpc" ) -type InternetChargeType string - -// EIP pay type -const ( - PayByBandwidth = InternetChargeType("PayByBandwidth") - PayByTraffic = InternetChargeType("PayByTraffic") -) - -// network resource type -const ( - ResourceTypeVeth = "veth" - ResourceTypeENI = "eni" - ResourceTypeENIIP = "eniIp" - ResourceTypeEIP = "eip" -) - -// Vswitch Selection Policy -const ( - VSwitchSelectionPolicyRandom = "random" - VSwitchSelectionPolicyOrdered = "ordered" -) - // IPStack is the ip family type type IPStack string @@ -58,11 +35,6 @@ const ( IPStackIPv6 IPStack = "ipv6" ) -type IPFamily struct { - IPv4 bool - IPv6 bool -} - // IPAMType how terway deal with ip resource type IPAMType string @@ -73,28 +45,48 @@ const ( IPAMTypeDefault = "" ) -// ENICapPolicy how eni cap is calculated -type ENICapPolicy string +type IPSet2 struct { + IPv4 netip.Addr + IPv6 netip.Addr +} -// how eni cap is calculated -const ( - ENICapPolicyPreferTrunk = "preferTrunk" - ENICapPolicyDefault = "" -) +func (i *IPSet2) String() string { + var result []string + if i.IPv4.IsValid() { + result = append(result, i.IPv4.String()) + } + if i.IPv6.IsValid() { + result = append(result, i.IPv6.String()) + } + return strings.Join(result, "-") +} -// NewIPFamilyFromIPStack parse IPStack to IPFamily -func NewIPFamilyFromIPStack(ipStack IPStack) *IPFamily { - f := &IPFamily{} - switch ipStack { - case IPStackIPv4: - f.IPv4 = true - case IPStackDual: - f.IPv4 = true - f.IPv6 = true - case IPStackIPv6: - f.IPv6 = true +func (i *IPSet2) ToRPC() *rpc.IPSet { + var ipv4, ipv6 string + if i.IPv4.IsValid() { + ipv4 = i.IPv4.String() + } + if i.IPv6.IsValid() { + ipv6 = i.IPv6.String() + } + return &rpc.IPSet{ + IPv4: ipv4, + IPv6: ipv6, } - return f +} + +func (i *IPSet2) GetIPv4() string { + if !i.IPv4.IsValid() { + return "" + } + return i.IPv4.String() +} + +func (i *IPSet2) GetIPv6() string { + if !i.IPv6.IsValid() { + return "" + } + return i.IPv6.String() } // IPSet is the type hole both ipv4 and ipv6 net.IP @@ -155,17 +147,6 @@ func (i *IPSet) GetIPv6() string { return i.IPv6.String() } -func MergeIPs(a, b []net.IP) []IPSet { - result := make([]IPSet, utils.Max(len(a), len(b))) - for i, ip := range a { - result[i].IPv4 = ip - } - for i, ip := range b { - result[i].IPv6 = ip - } - return result -} - type IPNetSet struct { IPv4 *net.IPNet IPv6 *net.IPNet @@ -184,6 +165,7 @@ func (i *IPNetSet) ToRPC() *rpc.IPSet { IPv6: ipv6, } } + func (i *IPNetSet) String() string { if i == nil { return "" @@ -213,174 +195,3 @@ func (i *IPNetSet) SetIPNet(str string) *IPNetSet { i.IPv4 = ipNet return i } - -// ENI aliyun ENI resource -type ENI struct { - ID string - MAC string - SecurityGroupIDs []string - - Trunk bool - - PrimaryIP IPSet - GatewayIP IPSet - - VSwitchCIDR IPNetSet - - VSwitchID string -} - -// GetResourceID return mac address of eni -func (e *ENI) GetResourceID() string { - return e.MAC -} - -// GetType return type name -func (e *ENI) GetType() string { - return ResourceTypeENI -} - -func (e *ENI) ToResItems() []ResourceItem { - return []ResourceItem{ - { - Type: e.GetType(), - ID: e.GetResourceID(), - ENIID: e.ID, - ENIMAC: e.MAC, - IPv4: e.PrimaryIP.GetIPv4(), - IPv6: e.PrimaryIP.GetIPv6(), - }, - } -} - -// ENIIP aliyun secondary IP resource -type ENIIP struct { - ENI *ENI - IPSet IPSet -} - -// GetResourceID return mac address of eni and secondary ip address -func (e *ENIIP) GetResourceID() string { - return fmt.Sprintf("%s.%s", e.ENI.GetResourceID(), e.IPSet.String()) -} - -// GetType return type name -func (e *ENIIP) GetType() string { - return ResourceTypeENIIP -} - -func (e *ENIIP) ToResItems() []ResourceItem { - return []ResourceItem{ - { - Type: e.GetType(), - ID: e.GetResourceID(), - ENIID: e.ENI.ID, - ENIMAC: e.ENI.MAC, - IPv4: e.IPSet.GetIPv4(), - IPv6: e.IPSet.GetIPv6(), - }, - } -} - -// Veth veth pair resource on system -type Veth struct { - HostVeth string -} - -// GetResourceID return host veth name of veth resource -func (e *Veth) GetResourceID() string { - return e.HostVeth -} - -// GetType return type name -func (e *Veth) GetType() string { - return ResourceTypeVeth -} - -func (e *Veth) ToResItems() []ResourceItem { - return []ResourceItem{ - { - Type: e.GetType(), - ID: e.GetResourceID(), - }, - } -} - -// EIP Aliyun public ip -type EIP struct { - ID string - Address net.IP - Delete bool // delete related eip on pod deletion - AssociateENI string - AssociateENIIP net.IP -} - -// GetResourceID return eip id -func (e *EIP) GetResourceID() string { - return e.ID -} - -// GetType return type name -func (e *EIP) GetType() string { - return ResourceTypeEIP -} - -func (e *EIP) ToResItems() []ResourceItem { - return []ResourceItem{ - { - Type: e.GetType(), - ID: e.GetResourceID(), - ExtraEipInfo: &ExtraEipInfo{ - Delete: e.Delete, - AssociateENI: e.AssociateENI, - AssociateENIIP: e.AssociateENIIP, - }, - }, - } -} - -// NetworkResource interface of network resources -type NetworkResource interface { - GetResourceID() string - GetType() string - ToResItems() []ResourceItem -} - -// Res is the func for res -type Res interface { - GetID() string - GetType() string - GetStatus() ResStatus -} - -// ResStatus ResStatus -type ResStatus int - -// ResStatus -const ( - ResStatusInvalid ResStatus = iota - ResStatusIdle - ResStatusInUse -) - -// FakeRes for test -type FakeRes struct { - ID string - Type string - Status ResStatus -} - -// GetID GetID -func (r *FakeRes) GetID() string { - return r.ID -} - -// GetType GetType -func (r *FakeRes) GetType() string { - return r.Type -} - -// GetStatus GetStatus -func (r *FakeRes) GetStatus() ResStatus { - return r.Status -} diff --git a/types/types_test.go b/types/types_test.go index 1947c2f2..a6e932eb 100644 --- a/types/types_test.go +++ b/types/types_test.go @@ -1,15 +1,16 @@ -package types +package types_test import ( - "net" - "reflect" + "fmt" "testing" "github.com/stretchr/testify/assert" + + "github.com/AliyunContainerService/terway/types" ) func TestIPSet_SetIP(t *testing.T) { - ipSet := &IPSet{} + ipSet := &types.IPSet{} assert.Equal(t, "127.0.0.1", ipSet.SetIP("127.0.0.1").IPv4.String()) assert.NotNil(t, ipSet.IPv4) assert.Equal(t, "127.0.0.1", ipSet.SetIP("127.0.0.x").IPv4.String()) @@ -18,56 +19,8 @@ func TestIPSet_SetIP(t *testing.T) { assert.NotNil(t, ipSet.IPv6) } -func TestMergeIPs(t *testing.T) { - type args struct { - a []net.IP - b []net.IP - } - tests := []struct { - name string - args args - want []IPSet - }{ - { - name: "ipv4", - args: args{ - a: []net.IP{net.ParseIP("127.0.0.1")}, - b: nil, - }, - want: []IPSet{ - { - IPv4: net.ParseIP("127.0.0.1"), - }, - }, - }, - { - name: "ipv6", - args: args{ - a: nil, - b: []net.IP{net.ParseIP("::1")}, - }, - want: []IPSet{ - { - IPv6: net.ParseIP("::1"), - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := MergeIPs(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.want) { - t.Errorf("MergeIPs() = %v, want %v", got, tt.want) - } - }) - } - - result := MergeIPs([]net.IP{net.ParseIP("127.0.0.1")}, - []net.IP{net.ParseIP("::1"), net.ParseIP("fd::1")}) - assert.Equal(t, 2, len(result)) -} - func TestIPNetSet_SetIPNet(t *testing.T) { - ipNetSet := &IPNetSet{} + ipNetSet := &types.IPNetSet{} assert.Equal(t, "127.0.0.1/32", ipNetSet.SetIPNet("127.0.0.1/32").IPv4.String()) assert.NotNil(t, ipNetSet.IPv4) assert.Equal(t, "127.0.0.0/24", ipNetSet.SetIPNet("127.0.0.1/24").IPv4.String()) @@ -77,3 +30,32 @@ func TestIPNetSet_SetIPNet(t *testing.T) { assert.Equal(t, "fd00::/120", ipNetSet.SetIPNet("fd00::/120").IPv6.String()) assert.NotNil(t, ipNetSet.IPv6) } + +func TestErrorReturnsCorrectErrorMessage(t *testing.T) { + err := &types.Error{ + Code: types.InternalError, + Msg: "An internal error occurred", + } + + assert.Equal(t, "code: InternalError, msg: An internal error occurred", err.Error()) +} + +func TestErrorUnwrapReturnsUnderlyingError(t *testing.T) { + underlyingError := fmt.Errorf("underlying error") + err := &types.Error{ + Code: types.InternalError, + Msg: "An internal error occurred", + R: underlyingError, + } + + assert.Equal(t, underlyingError, err.Unwrap()) +} + +func TestErrorUnwrapReturnsNilWhenNoUnderlyingError(t *testing.T) { + err := &types.Error{ + Code: types.InternalError, + Msg: "An internal error occurred", + } + + assert.Nil(t, err.Unwrap()) +}