diff --git a/cmd/aws-k8s-agent/main.go b/cmd/aws-k8s-agent/main.go index 2880209c4f..a21f8fa057 100644 --- a/cmd/aws-k8s-agent/main.go +++ b/cmd/aws-k8s-agent/main.go @@ -164,7 +164,7 @@ func _main() int { } // Pool manager - go ipamContext.StartNodeIPPoolManager() + go ipamContext.StartNodeIPPoolManager(context.Background()) if !utils.GetBoolAsStringEnvVar(envDisableMetrics, false) { // Prometheus metrics diff --git a/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go b/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go index 81da89c7ac..0bb263e2f3 100644 --- a/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go +++ b/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go @@ -1,8 +1,22 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. +// + // Code generated by MockGen. DO NOT EDIT. -// Source: driver.go +// Source: github.com/aws/amazon-vpc-cni-k8s/cmd/routed-eni-cni-plugin/driver (interfaces: NetworkAPIs) -// Package mocks is a generated GoMock package. -package mocks +// Package mock_driver is a generated GoMock package. +package mock_driver import ( reflect "reflect" @@ -37,57 +51,57 @@ func (m *MockNetworkAPIs) EXPECT() *MockNetworkAPIsMockRecorder { } // SetupBranchENIPodNetwork mocks base method. -func (m *MockNetworkAPIs) SetupBranchENIPodNetwork(vethMetadata driver.VirtualInterfaceMetadata, netnsPath string, vlanID int, eniMAC, subnetGW string, parentIfIndex, mtu int, podSGEnforcingMode sgpp.EnforcingMode, log logger.Logger) error { +func (m *MockNetworkAPIs) SetupBranchENIPodNetwork(arg0 driver.VirtualInterfaceMetadata, arg1 string, arg2 int, arg3, arg4 string, arg5, arg6 int, arg7 sgpp.EnforcingMode, arg8 logger.Logger) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetupBranchENIPodNetwork", vethMetadata, netnsPath, vlanID, eniMAC, subnetGW, parentIfIndex, mtu, podSGEnforcingMode, log) + ret := m.ctrl.Call(m, "SetupBranchENIPodNetwork", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) ret0, _ := ret[0].(error) return ret0 } // SetupBranchENIPodNetwork indicates an expected call of SetupBranchENIPodNetwork. -func (mr *MockNetworkAPIsMockRecorder) SetupBranchENIPodNetwork(vethMetadata, netnsPath, vlanID, eniMAC, subnetGW, parentIfIndex, mtu, podSGEnforcingMode, log interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) SetupBranchENIPodNetwork(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupBranchENIPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupBranchENIPodNetwork), vethMetadata, netnsPath, vlanID, eniMAC, subnetGW, parentIfIndex, mtu, podSGEnforcingMode, log) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupBranchENIPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupBranchENIPodNetwork), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) } // SetupPodNetwork mocks base method. -func (m *MockNetworkAPIs) SetupPodNetwork(vethMetadata []driver.VirtualInterfaceMetadata, netnsPath string, mtu int, log logger.Logger) error { +func (m *MockNetworkAPIs) SetupPodNetwork(arg0 []driver.VirtualInterfaceMetadata, arg1 string, arg2 int, arg3 logger.Logger) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetupPodNetwork", vethMetadata, netnsPath, mtu, log) + ret := m.ctrl.Call(m, "SetupPodNetwork", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) return ret0 } // SetupPodNetwork indicates an expected call of SetupPodNetwork. -func (mr *MockNetworkAPIsMockRecorder) SetupPodNetwork(vethMetadata, netnsPath, mtu, log interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) SetupPodNetwork(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupPodNetwork), vethMetadata, netnsPath, mtu, log) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupPodNetwork), arg0, arg1, arg2, arg3) } // TeardownBranchENIPodNetwork mocks base method. -func (m *MockNetworkAPIs) TeardownBranchENIPodNetwork(vethMetadata driver.VirtualInterfaceMetadata, vlanID int, podSGEnforcingMode sgpp.EnforcingMode, log logger.Logger) error { +func (m *MockNetworkAPIs) TeardownBranchENIPodNetwork(arg0 driver.VirtualInterfaceMetadata, arg1 int, arg2 sgpp.EnforcingMode, arg3 logger.Logger) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TeardownBranchENIPodNetwork", vethMetadata, vlanID, podSGEnforcingMode, log) + ret := m.ctrl.Call(m, "TeardownBranchENIPodNetwork", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) return ret0 } // TeardownBranchENIPodNetwork indicates an expected call of TeardownBranchENIPodNetwork. -func (mr *MockNetworkAPIsMockRecorder) TeardownBranchENIPodNetwork(vethMetadata, vlanID, podSGEnforcingMode, log interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) TeardownBranchENIPodNetwork(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TeardownBranchENIPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).TeardownBranchENIPodNetwork), vethMetadata, vlanID, podSGEnforcingMode, log) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TeardownBranchENIPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).TeardownBranchENIPodNetwork), arg0, arg1, arg2, arg3) } // TeardownPodNetwork mocks base method. -func (m *MockNetworkAPIs) TeardownPodNetwork(vethMetadata []driver.VirtualInterfaceMetadata, log logger.Logger) error { +func (m *MockNetworkAPIs) TeardownPodNetwork(arg0 []driver.VirtualInterfaceMetadata, arg1 logger.Logger) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TeardownPodNetwork", vethMetadata, log) + ret := m.ctrl.Call(m, "TeardownPodNetwork", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // TeardownPodNetwork indicates an expected call of TeardownPodNetwork. -func (mr *MockNetworkAPIsMockRecorder) TeardownPodNetwork(vethMetadata, log interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) TeardownPodNetwork(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TeardownPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).TeardownPodNetwork), vethMetadata, log) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TeardownPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).TeardownPodNetwork), arg0, arg1) } diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index aeca531168..d6bb59877f 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -61,16 +61,22 @@ const ( // AllocENI need to choose a first free device number between 0 and maxENI // 100 is a hard limit because we use vlanID + 100 for pod networking table names - maxENIs = 100 - clusterNameEnvVar = "CLUSTER_NAME" - eniNodeTagKey = "node.k8s.amazonaws.com/instance_id" - eniCreatedAtTagKey = "node.k8s.amazonaws.com/createdAt" - eniClusterTagKey = "cluster.k8s.amazonaws.com/name" - eniOwnerTagKey = "eks:eni:owner" - eniOwnerTagValue = "amazon-vpc-cni" - additionalEniTagsEnvVar = "ADDITIONAL_ENI_TAGS" - reservedTagKeyPrefix = "k8s.amazonaws.com" - subnetDiscoveryTagKey = "kubernetes.io/role/cni" + maxENIs = 100 + clusterNameEnvVar = "CLUSTER_NAME" + + // clusterTagKeyPrefix is the prefix for the cluster-specific subnet tags + clusterTagKeyPrefix = "kubernetes.io/cluster/" + + eniNodeTagKey = "node.k8s.amazonaws.com/instance_id" + eniCreatedAtTagKey = "node.k8s.amazonaws.com/createdAt" + eniClusterTagKey = "cluster.k8s.amazonaws.com/name" + eniOwnerTagKey = "eks:eni:owner" + eniOwnerTagValue = "amazon-vpc-cni" + additionalEniTagsEnvVar = "ADDITIONAL_ENI_TAGS" + reservedTagKeyPrefix = "k8s.amazonaws.com" + subnetDiscoveryTagKey = "kubernetes.io/role/cni" + subnetDiscoveryTagValueIncluded = "1" + subnetDiscoveryTagValueExcluded = "0" // UnknownInstanceType indicates that the instance type is not yet supported UnknownInstanceType = "vpc ip resource(eni ip limit): unknown instance type" @@ -105,43 +111,43 @@ var log = logger.Get() // APIs defines interfaces calls for adding/getting/deleting ENIs/secondary IPs. The APIs are not thread-safe. type APIs interface { // AllocENI creates an ENI and attaches it to the instance - AllocENI(sg []*string, eniCfgSubnet string, numIPs int, networkCard int) (eni string, err error) + AllocENI(ctx context.Context, sg []*string, eniCfgSubnet string, numIPs int, networkCard int) (eni string, err error) // FreeENI detaches ENI interface and deletes it - FreeENI(eniName string) error + FreeENI(ctx context.Context, eniName string) error // TagENI Tags ENI with current tags to contain expected tags. - TagENI(eniID string, currentTags map[string]string) error + TagENI(ctx context.Context, eniID string, currentTags map[string]string) error // GetAttachedENIs retrieves eni information from instance metadata service GetAttachedENIs() (eniList []ENIMetadata, err error) // GetIPv4sFromEC2 returns the IPv4 addresses for a given ENI - GetIPv4sFromEC2(eniID string) (addrList []ec2types.NetworkInterfacePrivateIpAddress, err error) + GetIPv4sFromEC2(ctx context.Context, eniID string) (addrList []ec2types.NetworkInterfacePrivateIpAddress, err error) // GetIPv4PrefixesFromEC2 returns the IPv4 prefixes for a given ENI - GetIPv4PrefixesFromEC2(eniID string) (addrList []ec2types.Ipv4PrefixSpecification, err error) + GetIPv4PrefixesFromEC2(ctx context.Context, eniID string) (addrList []ec2types.Ipv4PrefixSpecification, err error) // GetIPv6PrefixesFromEC2 returns the IPv6 prefixes for a given ENI - GetIPv6PrefixesFromEC2(eniID string) (addrList []ec2types.Ipv6PrefixSpecification, err error) + GetIPv6PrefixesFromEC2(ctx context.Context, eniID string) (addrList []ec2types.Ipv6PrefixSpecification, err error) // DescribeAllENIs calls EC2 and returns a fully populated DescribeAllENIsResult struct and an error - DescribeAllENIs() (DescribeAllENIsResult, error) + DescribeAllENIs(ctx context.Context) (DescribeAllENIsResult, error) // AllocIPAddress allocates an IP address for an ENI - AllocIPAddress(eniID string) error + AllocIPAddress(ctx context.Context, eniID string) error // AllocIPAddresses allocates numIPs IP addresses on a ENI - AllocIPAddresses(eniID string, numIPs int) (*ec2.AssignPrivateIpAddressesOutput, error) + AllocIPAddresses(ctx context.Context, eniID string, numIPs int) (*ec2.AssignPrivateIpAddressesOutput, error) // DeallocIPAddresses deallocates the list of IP addresses from a ENI - DeallocIPAddresses(eniID string, ips []string) error + DeallocIPAddresses(ctx context.Context, eniID string, ips []string) error // DeallocPrefixAddresses deallocates the list of IP addresses from a ENI - DeallocPrefixAddresses(eniID string, ips []string) error + DeallocPrefixAddresses(ctx context.Context, eniID string, ips []string) error - //AllocIPv6Prefixes allocates IPv6 prefixes to the ENI passed in - AllocIPv6Prefixes(eniID string) ([]*string, error) + // AllocIPv6Prefixes allocates IPv6 prefixes to the ENI passed in + AllocIPv6Prefixes(ctx context.Context, eniID string) ([]*string, error) // GetVPCIPv4CIDRs returns VPC's IPv4 CIDRs from instance metadata GetVPCIPv4CIDRs() ([]string, error) @@ -191,34 +197,47 @@ type APIs interface { // WaitForENIAndIPsAttached waits until the ENI has been attached and the secondary IPs have been added WaitForENIAndIPsAttached(eni string, wantedSecondaryIPs int) (ENIMetadata, error) - //IsPrimaryENI + // IsPrimaryENI IsPrimaryENI(eniID string) bool - //RefreshSGIDs - RefreshSGIDs(mac string, ds *datastore.DataStoreAccess) error + // RefreshSGIDs + RefreshSGIDs(ctx context.Context, mac string, ds *datastore.DataStoreAccess) error - //GetInstanceHypervisorFamily returns the hypervisor family for the instance + // RefreshCustomSGIDs discovers and refreshes security groups tagged with kubernetes.io/role/cni=1 + RefreshCustomSGIDs(ctx context.Context, dsAccess *datastore.DataStoreAccess) error + + // GetInstanceHypervisorFamily returns the hypervisor family for the instance GetInstanceHypervisorFamily() string - //GetInstanceType returns the EC2 instance type + // GetInstanceType returns the EC2 instance type GetInstanceType() string - //Update cached prefix delegation flag + // Update cached prefix delegation flag InitCachedPrefixDelegation(bool) // GetInstanceID returns the instance ID GetInstanceID() string // FetchInstanceTypeLimits Verify if the InstanceNetworkingLimits has the ENI limits else make EC2 call to fill cache. - FetchInstanceTypeLimits() error + FetchInstanceTypeLimits(ctx context.Context) error IsPrefixDelegationSupported() bool + + // GetENISubnetID gets the subnet ID for an ENI from AWS + GetENISubnetID(ctx context.Context, eniID string) (string, error) + + // GetVpcSubnets returns all subnets in the VPC + GetVpcSubnets(ctx context.Context) ([]ec2types.Subnet, error) + + // IsSubnetExcluded returns if a subnet is excluded for pod IPs based on its tags + IsSubnetExcluded(ctx context.Context, subnetID string) (bool, error) } // EC2InstanceMetadataCache caches instance metadata type EC2InstanceMetadataCache struct { // metadata info securityGroups StringSet + customSecurityGroups StringSet subnetID string localIPv4 net.IP v4Enabled bool @@ -340,7 +359,7 @@ func (ss *StringSet) Difference(other *StringSet) *StringSet { other.RLock() defer ss.RUnlock() defer other.RUnlock() - //example: s1 = {a1, a2, a3} s2 = {a1, a2, a4, a5} s1.Difference(s2) = {a3} s2.Difference(s1) = {a4, a5} + // example: s1 = {a1, a2, a3} s2 = {a1, a2, a4, a5} s1.Difference(s2) = {a3} s2.Difference(s1) = {a4, a5} return &StringSet{data: ss.data.Difference(other.data)} } @@ -427,7 +446,7 @@ func New(useSubnetDiscovery, useCustomNetworking, disableLeakedENICleanup, v4Ena // Clean up leaked ENIs in the background if !disableLeakedENICleanup { - go wait.Forever(cache.cleanUpLeakedENIs, time.Hour) + go wait.Forever(func() { cache.cleanUpLeakedENIs(context.Background()) }, time.Hour) } return cache, nil } @@ -513,75 +532,246 @@ func (cache *EC2InstanceMetadataCache) initWithEC2Metadata(ctx context.Context) return nil } -// RefreshSGIDs retrieves security groups -func (cache *EC2InstanceMetadataCache) RefreshSGIDs(mac string, dsAccess *datastore.DataStoreAccess) error { - ctx := context.TODO() +// discoverCustomSecurityGroups discovers security groups with the cni-role tag +func (cache *EC2InstanceMetadataCache) discoverCustomSecurityGroups(ctx context.Context) ([]string, error) { + describeSGInput := &ec2.DescribeSecurityGroupsInput{ + Filters: []ec2types.Filter{ + { + Name: aws.String("vpc-id"), + Values: []string{cache.vpcID}, + }, + { + Name: aws.String("tag:" + subnetDiscoveryTagKey), + Values: []string{subnetDiscoveryTagValueIncluded}, + }, + }, + } - sgIDs, err := cache.imds.GetSecurityGroupIDs(ctx, mac) + result, err := cache.ec2SVC.DescribeSecurityGroups(ctx, describeSGInput) if err != nil { - awsAPIErrInc("GetSecurityGroupIDs", err) - return err + return nil, fmt.Errorf("discoverCustomSecurityGroups: unable to describe security groups: %v", err) + } + + var sgIDs []string + for _, sg := range result.SecurityGroups { + sgIDs = append(sgIDs, *sg.GroupId) + } + + return sgIDs, nil +} + +// GetENISubnetID gets the subnet ID for an ENI from AWS +func (cache *EC2InstanceMetadataCache) GetENISubnetID(ctx context.Context, eniID string) (string, error) { + describeInput := &ec2.DescribeNetworkInterfacesInput{ + NetworkInterfaceIds: []string{eniID}, + } + + result, err := cache.ec2SVC.DescribeNetworkInterfaces(ctx, describeInput) + if err != nil { + return "", fmt.Errorf("getENISubnetID: unable to describe network interface: %v", err) + } + + if len(result.NetworkInterfaces) == 0 { + return "", fmt.Errorf("getENISubnetID: no interfaces found") } - newSGs := StringSet{} - newSGs.Set(sgIDs) - addedSGs := newSGs.Difference(&cache.securityGroups) - addedSGsCount := 0 - deletedSGs := cache.securityGroups.Difference(&newSGs) - deletedSGsCount := 0 + return *result.NetworkInterfaces[0].SubnetId, nil +} + +// Helper function to check if an ENI is in a secondary subnet +func (cache *EC2InstanceMetadataCache) isENIInSecondarySubnet(ctx context.Context, eniID string) bool { + eniSubnetID, err := cache.GetENISubnetID(ctx, eniID) + return err == nil && eniSubnetID != cache.subnetID +} + +// Helper function to get ENIs that match specific criteria +func (cache *EC2InstanceMetadataCache) getFilteredENIs(ctx context.Context, store *datastore.DataStore, onlySecondarySubnets bool) []string { + eniInfos := store.GetENIInfos() + var eniIDs []string + + for eniID := range eniInfos.ENIs { + if eniInfo, ok := eniInfos.ENIs[eniID]; ok { + // Skip primary ENI for secondary subnet operations + if onlySecondarySubnets && eniInfo.IsPrimary { + continue + } + + isSecondarySubnet := cache.isENIInSecondarySubnet(ctx, eniID) + + // Filter based on subnet type + if onlySecondarySubnets && !isSecondarySubnet { + continue + } else if !onlySecondarySubnets && isSecondarySubnet { + // When we want primary subnet ENIs, skip secondary subnet ENIs + continue + } + + eniIDs = append(eniIDs, eniID) + } + } + // Apply standard filters (unmanaged and multi-card ENIs) + newENIs := StringSet{} + newENIs.Set(eniIDs) + filteredENIs := newENIs.Difference(&cache.unmanagedENIs) + + return filteredENIs.SortedList() +} + +// Helper function to apply security groups to a list of ENIs +func (cache *EC2InstanceMetadataCache) applySecurityGroupsToENIs(ctx context.Context, eniIDs []string, sgIDs []string, logPrefix string) { + for _, eniID := range eniIDs { + log.Debugf("%s ENI %s with security groups %v", logPrefix, eniID, sgIDs) + + attributeInput := &ec2.ModifyNetworkInterfaceAttributeInput{ + Groups: sgIDs, + NetworkInterfaceId: aws.String(eniID), + } + start := time.Now() + _, err := cache.ec2SVC.ModifyNetworkInterfaceAttribute(ctx, attributeInput) + prometheusmetrics.Ec2ApiReq.WithLabelValues("ModifyNetworkInterfaceAttribute").Inc() + prometheusmetrics.AwsAPILatency.WithLabelValues("ModifyNetworkInterfaceAttribute", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) + + if err != nil { + if errors.As(err, &awsAPIError) { + if awsAPIError.ErrorCode() == "InvalidNetworkInterfaceID.NotFound" { + awsAPIErrInc("IMDSMetaDataOutOfSync", err) + } + } + checkAPIErrorAndBroadcastEvent(err, "ec2:ModifyNetworkInterfaceAttribute") + awsAPIErrInc("ModifyNetworkInterfaceAttribute", err) + prometheusmetrics.Ec2ApiErr.WithLabelValues("ModifyNetworkInterfaceAttribute").Inc() + log.Warnf("%s: unable to update ENI %s security groups: %v", logPrefix, eniID, err) + } + } +} + +// Helper function to detect and log security group changes +func (cache *EC2InstanceMetadataCache) detectSecurityGroupChanges(newSGs []string, currentSGs *StringSet, sgType string) (int, int) { + newSGSet := StringSet{} + newSGSet.Set(newSGs) + addedSGs := newSGSet.Difference(currentSGs) + deletedSGs := currentSGs.Difference(&newSGSet) + + addedCount := 0 for _, sg := range addedSGs.SortedList() { - log.Infof("Found %s, added to ipamd cache", sg) - addedSGsCount++ + log.Infof("Found %s SG %s, added to ipamd cache", sgType, sg) + addedCount++ } + + deletedCount := 0 for _, sg := range deletedSGs.SortedList() { - log.Infof("Removed %s from ipamd cache", sg) - deletedSGsCount++ + log.Infof("Removed %s SG %s from ipamd cache", sgType, sg) + deletedCount++ } - cache.securityGroups.Set(sgIDs) - if !cache.useCustomNetworking && (addedSGsCount != 0 || deletedSGsCount != 0) { - var eniIDs []string + return addedCount, deletedCount +} + +// RefreshCustomSGIDs discovers and refreshes security groups tagged for use with the CNI +func (cache *EC2InstanceMetadataCache) RefreshCustomSGIDs(ctx context.Context, dsAccess *datastore.DataStoreAccess) error { + sgIDs, err := cache.discoverCustomSecurityGroups(ctx) + if err != nil { + awsAPIErrInc("DiscoverCustomSecurityGroups", err) + log.Warnf("Failed to discover custom security groups: %v", err) + log.Info("Falling back to using primary security groups for ENIs in secondary subnets") + // Clear custom security groups cache to trigger fallback behavior + cache.customSecurityGroups.Set([]string{}) + + // Apply primary security groups to ENIs in secondary subnets as fallback + cache.applyFallbackSecurityGroupsForAllDatastores(ctx, dsAccess) + return err + } + + // Check if no custom security groups were found (empty list) + if len(sgIDs) == 0 { + log.Info("No custom security groups found, using primary security groups for ENIs in secondary subnets") + + // Clear custom security groups cache + cache.customSecurityGroups.Set([]string{}) + + // Apply primary security groups to ENIs in secondary subnets as fallback + cache.applyFallbackSecurityGroupsForAllDatastores(ctx, dsAccess) + + return nil + } + + addedCount, deletedCount := cache.detectSecurityGroupChanges(sgIDs, &cache.customSecurityGroups, "custom") + cache.customSecurityGroups.Set(sgIDs) + + // If there are changes, update ENIs in secondary subnets + if addedCount != 0 || deletedCount != 0 { + var eniIDs []string for _, ds := range dsAccess.DataStores { - eniInfos := ds.GetENIInfos() - for eniID := range eniInfos.ENIs { - eniIDs = append(eniIDs, eniID) - } + eniIDs = append(eniIDs, cache.getFilteredENIs(ctx, ds, true)...) // only secondary subnet ENIs } + cache.applySecurityGroupsToENIs(ctx, eniIDs, sgIDs, "Update") + } - newENIs := StringSet{} - newENIs.Set(eniIDs) - filteredENIs := newENIs.Difference(&cache.unmanagedENIs) + return nil +} - // This will update SG for managed ENIs created by EKS. - for _, eniID := range filteredENIs.SortedList() { - log.Debugf("Update ENI %s", eniID) +// applyFallbackSecurityGroupsForAllDatastores applies primary security groups to ENIs in secondary subnets across all datastores +func (cache *EC2InstanceMetadataCache) applyFallbackSecurityGroupsForAllDatastores(ctx context.Context, dsAccess *datastore.DataStoreAccess) { + log.Info("Applying primary security groups as fallback for ENIs in secondary subnets across all datastores") - attributeInput := &ec2.ModifyNetworkInterfaceAttributeInput{ - Groups: sgIDs, - NetworkInterfaceId: aws.String(eniID), - } - start := time.Now() - _, err = cache.ec2SVC.ModifyNetworkInterfaceAttribute(context.Background(), attributeInput) - prometheusmetrics.Ec2ApiReq.WithLabelValues("ModifyNetworkInterfaceAttribute").Inc() - prometheusmetrics.AwsAPILatency.WithLabelValues("ModifyNetworkInterfaceAttribute", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) + primarySGs := cache.securityGroups.SortedList() + if len(primarySGs) == 0 { + log.Warn("No primary security groups available for fallback") + } - if err != nil { - if errors.As(err, &awsAPIError) { - if awsAPIError.ErrorCode() == "InvalidNetworkInterfaceID.NotFound" { - awsAPIErrInc("IMDSMetaDataOutOfSync", err) + var eniIDs []string + for _, ds := range dsAccess.DataStores { + eniIDs = append(eniIDs, cache.getFilteredENIs(ctx, ds, true)...) // only secondary subnet ENIs + } + + cache.applySecurityGroupsToENIs(ctx, eniIDs, primarySGs, "Applying primary security groups to") +} + +// RefreshSGIDs retrieves security groups +func (cache *EC2InstanceMetadataCache) RefreshSGIDs(ctx context.Context, mac string, dsAccess *datastore.DataStoreAccess) error { + sgIDs, err := cache.imds.GetSecurityGroupIDs(ctx, mac) + if err != nil { + awsAPIErrInc("GetSecurityGroupIDs", err) + return err + } + + addedCount, deletedCount := cache.detectSecurityGroupChanges(sgIDs, &cache.securityGroups, "primary") + cache.securityGroups.Set(sgIDs) + + if !cache.useCustomNetworking && (addedCount != 0 || deletedCount != 0) { + var eniIDs []string + + // When subnet discovery is enabled, only apply primary SGs to primary subnet ENIs + if cache.useSubnetDiscovery { + for _, ds := range dsAccess.DataStores { + // Get only primary subnet ENIs (onlySecondarySubnets=false) + primarySubnetENIs := cache.getFilteredENIs(ctx, ds, false) + for _, eniID := range primarySubnetENIs { + // Filter out unmanaged ENIs + if !cache.unmanagedENIs.Has(eniID) { + eniIDs = append(eniIDs, eniID) } } - checkAPIErrorAndBroadcastEvent(err, "ec2:ModifyNetworkInterfaceAttribute") - awsAPIErrInc("ModifyNetworkInterfaceAttribute", err) - prometheusmetrics.Ec2ApiErr.WithLabelValues("ModifyNetworkInterfaceAttribute").Inc() - // No need to return error here since retry will happen in 30 seconds and also - // If update failed due to stale ENI then returning error will prevent updating SG - // for following ENIs since the list is sorted - log.Debugf("refreshSGIDs: unable to update the ENI %s SG - %v", eniID, err) } + } else { + // Original behavior: apply to all managed ENIs when subnet discovery is disabled + for _, ds := range dsAccess.DataStores { + eniInfos := ds.GetENIInfos() + for eniID := range eniInfos.ENIs { + eniIDs = append(eniIDs, eniID) + } + } + + newENIs := StringSet{} + newENIs.Set(eniIDs) + filteredENIs := newENIs.Difference(&cache.unmanagedENIs) + eniIDs = filteredENIs.SortedList() } + + // Apply security groups to the filtered ENIs + cache.applySecurityGroupsToENIs(ctx, eniIDs, sgIDs, "Update") } return nil } @@ -796,13 +986,13 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat } // awsGetFreeDeviceNumber calls EC2 API DescribeInstances to get the next free device index -func (cache *EC2InstanceMetadataCache) awsGetFreeDeviceNumber(networkCard int) (int, error) { +func (cache *EC2InstanceMetadataCache) awsGetFreeDeviceNumber(ctx context.Context, networkCard int) (int, error) { input := &ec2.DescribeInstancesInput{ InstanceIds: []string{cache.instanceID}, } start := time.Now() - result, err := cache.ec2SVC.DescribeInstances(context.Background(), input) + result, err := cache.ec2SVC.DescribeInstances(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("DescribeInstances").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("DescribeInstances", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -844,16 +1034,15 @@ func (cache *EC2InstanceMetadataCache) awsGetFreeDeviceNumber(networkCard int) ( // AllocENI creates an ENI and attaches it to the instance // returns: newly created ENI ID -func (cache *EC2InstanceMetadataCache) AllocENI(sg []*string, eniCfgSubnet string, numIPs int, networkCard int) (string, error) { - - eniID, err := cache.createENI(sg, eniCfgSubnet, numIPs) +func (cache *EC2InstanceMetadataCache) AllocENI(ctx context.Context, sg []*string, eniCfgSubnet string, numIPs int, networkCard int) (string, error) { + eniID, err := cache.createENI(ctx, sg, eniCfgSubnet, numIPs) if err != nil { return "", errors.Wrap(err, "AllocENI: failed to create ENI") } - attachmentID, err := cache.attachENI(eniID, networkCard) + attachmentID, err := cache.attachENI(ctx, eniID, networkCard) if err != nil { - derr := cache.deleteENI(eniID, maxENIBackoffDelay) + derr := cache.deleteENI(ctx, eniID, maxENIBackoffDelay) if derr != nil { awsUtilsErrInc("AllocENIDeleteErr", err) log.Errorf("Failed to delete newly created untagged ENI! %v", err) @@ -871,14 +1060,14 @@ func (cache *EC2InstanceMetadataCache) AllocENI(sg []*string, eniCfgSubnet strin } start := time.Now() - _, err = cache.ec2SVC.ModifyNetworkInterfaceAttribute(context.Background(), attributeInput) + _, err = cache.ec2SVC.ModifyNetworkInterfaceAttribute(ctx, attributeInput) prometheusmetrics.Ec2ApiReq.WithLabelValues("ModifyNetworkInterfaceAttribute").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("ModifyNetworkInterfaceAttribute", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { checkAPIErrorAndBroadcastEvent(err, "ec2:ModifyNetworkInterfaceAttribute") awsAPIErrInc("ModifyNetworkInterfaceAttribute", err) prometheusmetrics.Ec2ApiErr.WithLabelValues("ModifyNetworkInterfaceAttribute").Inc() - err := cache.FreeENI(eniID) + err := cache.FreeENI(ctx, eniID) if err != nil { awsUtilsErrInc("ENICleanupUponModifyNetworkErr", err) } @@ -890,9 +1079,9 @@ func (cache *EC2InstanceMetadataCache) AllocENI(sg []*string, eniCfgSubnet strin } // attachENI calls EC2 API to attach the ENI and returns the attachment id -func (cache *EC2InstanceMetadataCache) attachENI(eniID string, networkCard int) (string, error) { +func (cache *EC2InstanceMetadataCache) attachENI(ctx context.Context, eniID string, networkCard int) (string, error) { // attach to instance - freeDevice, err := cache.awsGetFreeDeviceNumber(networkCard) + freeDevice, err := cache.awsGetFreeDeviceNumber(ctx, networkCard) if err != nil { return "", errors.Wrap(err, "attachENI: failed to get a free device number") } @@ -904,7 +1093,7 @@ func (cache *EC2InstanceMetadataCache) attachENI(eniID string, networkCard int) NetworkCardIndex: aws.Int32(int32(networkCard)), } start := time.Now() - attachOutput, err := cache.ec2SVC.AttachNetworkInterface(context.Background(), attachInput) + attachOutput, err := cache.ec2SVC.AttachNetworkInterface(ctx, attachInput) prometheusmetrics.Ec2ApiReq.WithLabelValues("AttachNetworkInterface").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("AttachNetworkInterface", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -922,7 +1111,6 @@ func (cache *EC2InstanceMetadataCache) attachENI(eniID string, networkCard int) // Returns: // - []ec2types.TagSpecification: Returns the tags by converting it into AWS SDK class func (cache *EC2InstanceMetadataCache) createENITags() []ec2types.TagSpecification { - tags := map[string]string{ eniCreatedAtTagKey: time.Now().Format(time.RFC3339), } @@ -962,11 +1150,11 @@ func (cache *EC2InstanceMetadataCache) createENIInput(eniDescription string, tag } // return ENI id, error -func (cache *EC2InstanceMetadataCache) createENI(sg []*string, eniCfgSubnet string, numIPs int) (string, error) { +func (cache *EC2InstanceMetadataCache) createENI(ctx context.Context, sg []*string, eniCfgSubnet string, numIPs int) (string, error) { eniDescription := eniDescriptionPrefix + cache.instanceID tags := cache.createENITags() - var needIPs = numIPs + needIPs := numIPs if !cache.v6Enabled { ipLimit := cache.GetENIIPv4Limit() @@ -987,39 +1175,79 @@ func (cache *EC2InstanceMetadataCache) createENI(sg []*string, eniCfgSubnet stri input = createENIUsingCustomCfg(sg, eniCfgSubnet, input) log.Infof("Creating ENI with security groups: %v in subnet: %s", input.Groups, aws.ToString(input.SubnetId)) - networkInterfaceID, err = cache.tryCreateNetworkInterface(input) + networkInterfaceID, err = cache.tryCreateNetworkInterface(ctx, input) if err == nil { return networkInterfaceID, nil } } else { - if cache.useSubnetDiscovery && !cache.v6Enabled { - subnetResult, vpcErr := cache.getVpcSubnets() + if cache.useSubnetDiscovery { + subnetResult, vpcErr := cache.GetVpcSubnets(ctx) if vpcErr != nil { log.Warnf("Failed to call ec2:DescribeSubnets: %v", vpcErr) log.Info("Defaulting to same subnet as the primary interface for the new ENI") - networkInterfaceID, err = cache.tryCreateNetworkInterface(input) + + // Even in fallback, check if primary subnet is excluded + excluded, checkErr := cache.IsSubnetExcluded(ctx, cache.subnetID) + if checkErr != nil { + log.Warnf("Failed to check if primary subnet is excluded: %v. Proceeding with ENI creation attempt.", checkErr) + } else if excluded { + // Primary subnet is explicitly excluded + return "", fmt.Errorf("primary subnet is tagged with kubernetes.io/role/cni=0 - no valid subnets available for ENI creation") + } + + networkInterfaceID, err = cache.tryCreateNetworkInterface(ctx, input) if err == nil { return networkInterfaceID, nil } } else { + validSubnetsFound := false for _, subnet := range subnetResult { - if *subnet.SubnetId != cache.subnetID { - if !validTag(subnet) { - continue + // Check tag for all subnets including primary + isPrimarySubnet := *subnet.SubnetId == cache.subnetID + if !validTag(subnet, isPrimarySubnet) { + // Log when primary subnet is excluded + if isPrimarySubnet { + log.Infof("Primary subnet %s is excluded from ENI creation", cache.subnetID) } + continue } - log.Infof("Creating ENI with security groups: %v in subnet: %s", input.Groups, aws.ToString(input.SubnetId)) + validSubnetsFound = true + // If this is a secondary subnet and we have custom security groups, use those instead + // We already determined isPrimarySubnet above, just reuse the variable + if !isPrimarySubnet && len(cache.customSecurityGroups.SortedList()) > 0 { + log.Infof("Using custom security groups for ENI in secondary subnet %s", *subnet.SubnetId) + input.Groups = cache.customSecurityGroups.SortedList() + } else if !isPrimarySubnet { + // Secondary subnet but no custom security groups available - use primary SGs as fallback + log.Infof("No custom security groups available, using primary security groups for ENI in secondary subnet %s", *subnet.SubnetId) + } + log.Infof("Creating ENI with security groups: %v in subnet: %s", input.Groups, aws.ToString(subnet.SubnetId)) input.SubnetId = subnet.SubnetId - networkInterfaceID, err = cache.tryCreateNetworkInterface(input) + networkInterfaceID, err = cache.tryCreateNetworkInterface(ctx, input) if err == nil { return networkInterfaceID, nil } } + + // If no valid subnets found, return appropriate error + if !validSubnetsFound { + return "", fmt.Errorf("no valid subnets available for ENI creation - all subnets are either not tagged or tagged with kubernetes.io/role/cni=0") + } } } else { log.Info("Using same security group config as the primary interface for the new ENI") - networkInterfaceID, err = cache.tryCreateNetworkInterface(input) + // When subnet discovery is disabled, check if primary subnet is excluded + excluded, checkErr := cache.IsSubnetExcluded(ctx, cache.subnetID) + if checkErr != nil { + // If we can't determine exclusion status, log warning and proceed + log.Warnf("Failed to check if primary subnet is excluded: %v. Proceeding with ENI creation attempt.", checkErr) + } else if excluded { + // Primary subnet is explicitly excluded + return "", fmt.Errorf("primary subnet is tagged with kubernetes.io/role/cni=0 and subnet discovery is disabled - no valid subnets available for ENI creation") + } + + networkInterfaceID, err = cache.tryCreateNetworkInterface(ctx, input) if err == nil { return networkInterfaceID, nil } @@ -1028,7 +1256,7 @@ func (cache *EC2InstanceMetadataCache) createENI(sg []*string, eniCfgSubnet stri return "", errors.Wrap(err, "failed to create network interface") } -func (cache *EC2InstanceMetadataCache) getVpcSubnets() ([]ec2types.Subnet, error) { +func (cache *EC2InstanceMetadataCache) GetVpcSubnets(ctx context.Context) ([]ec2types.Subnet, error) { describeSubnetInput := &ec2.DescribeSubnetsInput{ Filters: []ec2types.Filter{ { @@ -1043,7 +1271,7 @@ func (cache *EC2InstanceMetadataCache) getVpcSubnets() ([]ec2types.Subnet, error } start := time.Now() - subnetResult, err := cache.ec2SVC.DescribeSubnets(context.Background(), describeSubnetInput) + subnetResult, err := cache.ec2SVC.DescribeSubnets(ctx, describeSubnetInput) prometheusmetrics.Ec2ApiReq.WithLabelValues("DescribeSubnets").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("DescribeSubnets", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -1061,15 +1289,71 @@ func (cache *EC2InstanceMetadataCache) getVpcSubnets() ([]ec2types.Subnet, error return subnetResult.Subnets, nil } -func validTag(subnet ec2types.Subnet) bool { - for _, tag := range subnet.Tags { - if *tag.Key == subnetDiscoveryTagKey { - return true +// validTag checks if subnet should be used for ENI/IP allocation +// For primary subnet: include by default (no tag), exclude only if tag value is "0" +// For secondary subnets: exclude by default (no tag), include only if tag exists with non-"0" value +// If the subnet has cluster-specific tags, it will only be used by the matching cluster +func validTag(subnet ec2types.Subnet, isPrimarySubnet bool) bool { + // Get cluster name for cluster-specific tag checks + localClusterName := os.Getenv(clusterNameEnvVar) + localClusterTagKey := clusterTagKeyPrefix + localClusterName + + // Parse subnet tags + cniTagValue := getTagValue(subnet.Tags, subnetDiscoveryTagKey) + hasClusterTags, belongsToThisCluster := checkClusterTags(subnet.Tags, localClusterTagKey) + + // Rule 1: CNI tag with value "0" always excludes the subnet + if cniTagValue == subnetDiscoveryTagValueExcluded { + log.Debugf("Subnet %s has %s=0 tag, excluding it from ENI creation", *subnet.SubnetId, subnetDiscoveryTagKey) + return false + } + + // Rule 2: Check CNI tag requirements based on subnet type + hasCniTag := cniTagValue != "" + if !hasCniTag { + if isPrimarySubnet { + // Primary subnets are included by default (backwards compatibility) + log.Debugf("Primary subnet %s has no %s tag, including it for ENI creation (backwards compatibility)", *subnet.SubnetId, subnetDiscoveryTagKey) + } else { + // Secondary subnets require explicit opt-in via CNI tag + log.Debugf("Subnet %s has no %s tag, excluding it from ENI creation", *subnet.SubnetId, subnetDiscoveryTagKey) + return false } } + + // Rule 3: Check cluster-specific tags + if !hasClusterTags || belongsToThisCluster { + return true + } + + // Subnet has cluster tags but not for this cluster + log.Debugf("Subnet %s does not belong to this cluster, excluding it from ENI creation", *subnet.SubnetId) return false } +// getTagValue returns the value of a specific tag key, or empty string if not found +func getTagValue(tags []ec2types.Tag, key string) string { + for _, tag := range tags { + if tag.Key != nil && *tag.Key == key && tag.Value != nil { + return *tag.Value + } + } + return "" +} + +// checkClusterTags checks if subnet has cluster-specific tags and if it belongs to the current cluster +func checkClusterTags(tags []ec2types.Tag, localClusterTagKey string) (hasClusterTags bool, belongsToThisCluster bool) { + for _, tag := range tags { + if tag.Key != nil && strings.HasPrefix(*tag.Key, clusterTagKeyPrefix) { + hasClusterTags = true + if *tag.Key == localClusterTagKey && tag.Value != nil && *tag.Value == "shared" { + belongsToThisCluster = true + } + } + } + return +} + func createENIUsingCustomCfg(sg []*string, eniCfgSubnet string, input *ec2.CreateNetworkInterfaceInput) *ec2.CreateNetworkInterfaceInput { log.Info("Using a custom network config for the new ENI") @@ -1083,9 +1367,9 @@ func createENIUsingCustomCfg(sg []*string, eniCfgSubnet string, input *ec2.Creat return input } -func (cache *EC2InstanceMetadataCache) tryCreateNetworkInterface(input *ec2.CreateNetworkInterfaceInput) (string, error) { +func (cache *EC2InstanceMetadataCache) tryCreateNetworkInterface(ctx context.Context, input *ec2.CreateNetworkInterfaceInput) (string, error) { start := time.Now() - result, err := cache.ec2SVC.CreateNetworkInterface(context.Background(), input) + result, err := cache.ec2SVC.CreateNetworkInterface(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("CreateNetworkInterface").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("CreateNetworkInterface", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err == nil { @@ -1117,7 +1401,7 @@ func (cache *EC2InstanceMetadataCache) buildENITags() map[string]string { return tags } -func (cache *EC2InstanceMetadataCache) TagENI(eniID string, currentTags map[string]string) error { +func (cache *EC2InstanceMetadataCache) TagENI(ctx context.Context, eniID string, currentTags map[string]string) error { tagChanges := make(map[string]string) for tagKey, tagValue := range cache.buildENITags() { if currentTagValue, ok := currentTags[tagKey]; !ok || currentTagValue != tagValue { @@ -1136,7 +1420,7 @@ func (cache *EC2InstanceMetadataCache) TagENI(eniID string, currentTags map[stri log.Debugf("Tagging ENI %s with missing tags: %v", eniID, tagChanges) return retry.NWithBackoff(retry.NewSimpleBackoff(500*time.Millisecond, maxENIBackoffDelay, 0.3, 2), 5, func() error { start := time.Now() - _, err := cache.ec2SVC.CreateTags(context.Background(), input) + _, err := cache.ec2SVC.CreateTags(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("CreateTags").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -1164,15 +1448,15 @@ func awsUtilsErrInc(fn string, err error) { } // FreeENI detaches and deletes the ENI interface -func (cache *EC2InstanceMetadataCache) FreeENI(eniName string) error { - return cache.freeENI(eniName, 2*time.Second, maxENIBackoffDelay) +func (cache *EC2InstanceMetadataCache) FreeENI(ctx context.Context, eniName string) error { + return cache.freeENI(ctx, eniName, 2*time.Second, maxENIBackoffDelay) } -func (cache *EC2InstanceMetadataCache) freeENI(eniName string, sleepDelayAfterDetach time.Duration, maxBackoffDelay time.Duration) error { +func (cache *EC2InstanceMetadataCache) freeENI(ctx context.Context, eniName string, sleepDelayAfterDetach time.Duration, maxBackoffDelay time.Duration) error { log.Infof("Trying to free ENI: %s", eniName) // Find out attachment - attachID, err := cache.getENIAttachmentID(eniName) + attachID, err := cache.getENIAttachmentID(ctx, eniName) if err != nil { if err == ErrENINotFound { log.Infof("ENI %s not found. It seems to be already freed", eniName) @@ -1191,7 +1475,7 @@ func (cache *EC2InstanceMetadataCache) freeENI(eniName string, sleepDelayAfterDe // Retry detaching the ENI from the instance err = retry.NWithBackoff(retry.NewSimpleBackoff(time.Millisecond*200, maxBackoffDelay, 0.15, 2.0), maxENIEC2APIRetries, func() error { start := time.Now() - _, ec2Err := cache.ec2SVC.DetachNetworkInterface(context.Background(), detachInput) + _, ec2Err := cache.ec2SVC.DetachNetworkInterface(ctx, detachInput) prometheusmetrics.Ec2ApiReq.WithLabelValues("DetachNetworkInterface").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("DetachNetworkInterface", fmt.Sprint(ec2Err != nil), awsReqStatus(ec2Err)).Observe(msSince(start)) if ec2Err != nil { @@ -1204,7 +1488,6 @@ func (cache *EC2InstanceMetadataCache) freeENI(eniName string, sleepDelayAfterDe log.Infof("Successfully detached ENI: %s", eniName) return nil }) - if err != nil { log.Errorf("Failed to detach ENI %s %v", eniName, err) return ErrUnableToDetachENI @@ -1212,7 +1495,7 @@ func (cache *EC2InstanceMetadataCache) freeENI(eniName string, sleepDelayAfterDe // It does take awhile for EC2 to detach ENI from instance, so we wait 2s before trying the delete. time.Sleep(sleepDelayAfterDetach) - err = cache.deleteENI(eniName, maxBackoffDelay) + err = cache.deleteENI(ctx, eniName, maxBackoffDelay) if err != nil { awsUtilsErrInc("FreeENIDeleteErr", err) return errors.Wrapf(err, "FreeENI: failed to free ENI: %s", eniName) @@ -1223,13 +1506,13 @@ func (cache *EC2InstanceMetadataCache) freeENI(eniName string, sleepDelayAfterDe } // getENIAttachmentID calls EC2 to fetch the attachmentID of a given ENI -func (cache *EC2InstanceMetadataCache) getENIAttachmentID(eniID string) (*string, error) { +func (cache *EC2InstanceMetadataCache) getENIAttachmentID(ctx context.Context, eniID string) (*string, error) { eniIds := make([]*string, 0) eniIds = append(eniIds, aws.String(eniID)) input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: aws.ToStringSlice(eniIds)} start := time.Now() - result, err := cache.ec2SVC.DescribeNetworkInterfaces(context.Background(), input) + result, err := cache.ec2SVC.DescribeNetworkInterfaces(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("DescribeNetworkInterfaces").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -1260,14 +1543,14 @@ func (cache *EC2InstanceMetadataCache) getENIAttachmentID(eniID string) (*string return attachID, nil } -func (cache *EC2InstanceMetadataCache) deleteENI(eniName string, maxBackoffDelay time.Duration) error { +func (cache *EC2InstanceMetadataCache) deleteENI(ctx context.Context, eniName string, maxBackoffDelay time.Duration) error { log.Debugf("Trying to delete ENI: %s", eniName) deleteInput := &ec2.DeleteNetworkInterfaceInput{ NetworkInterfaceId: aws.String(eniName), } err := retry.NWithBackoff(retry.NewSimpleBackoff(time.Millisecond*500, maxBackoffDelay, 0.15, 2.0), maxENIEC2APIRetries, func() error { start := time.Now() - _, ec2Err := cache.ec2SVC.DeleteNetworkInterface(context.Background(), deleteInput) + _, ec2Err := cache.ec2SVC.DeleteNetworkInterface(ctx, deleteInput) prometheusmetrics.Ec2ApiReq.WithLabelValues("DeleteNetworkInterface").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("DeleteNetworkInterface", fmt.Sprint(ec2Err != nil), awsReqStatus(ec2Err)).Observe(msSince(start)) if ec2Err != nil { @@ -1291,13 +1574,13 @@ func (cache *EC2InstanceMetadataCache) deleteENI(eniName string, maxBackoffDelay } // GetIPv4sFromEC2 calls EC2 and returns a list of all addresses on the ENI -func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(eniID string) (addrList []ec2types.NetworkInterfacePrivateIpAddress, err error) { +func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(ctx context.Context, eniID string) (addrList []ec2types.NetworkInterfacePrivateIpAddress, err error) { eniIds := make([]*string, 0) eniIds = append(eniIds, aws.String(eniID)) input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: aws.ToStringSlice(eniIds)} start := time.Now() - result, err := cache.ec2SVC.DescribeNetworkInterfaces(context.Background(), input) + result, err := cache.ec2SVC.DescribeNetworkInterfaces(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("DescribeNetworkInterfaces").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -1323,12 +1606,12 @@ func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(eniID string) (addrList [ } // GetIPv4PrefixesFromEC2 calls EC2 and returns a list of all addresses on the ENI -func (cache *EC2InstanceMetadataCache) GetIPv4PrefixesFromEC2(eniID string) (addrList []ec2types.Ipv4PrefixSpecification, err error) { +func (cache *EC2InstanceMetadataCache) GetIPv4PrefixesFromEC2(ctx context.Context, eniID string) (addrList []ec2types.Ipv4PrefixSpecification, err error) { eniIds := []*string{aws.String(eniID)} input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: aws.ToStringSlice(eniIds)} start := time.Now() - result, err := cache.ec2SVC.DescribeNetworkInterfaces(context.Background(), input) + result, err := cache.ec2SVC.DescribeNetworkInterfaces(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("DescribeNetworkInterfaces").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -1336,7 +1619,6 @@ func (cache *EC2InstanceMetadataCache) GetIPv4PrefixesFromEC2(eniID string) (add if awsAPIError.ErrorCode() == "InvalidNetworkInterfaceID.NotFound" { return nil, ErrENINotFound } - } checkAPIErrorAndBroadcastEvent(err, "ec2:DescribeNetworkInterfaces") awsAPIErrInc("DescribeNetworkInterfaces", err) @@ -1355,12 +1637,12 @@ func (cache *EC2InstanceMetadataCache) GetIPv4PrefixesFromEC2(eniID string) (add } // GetIPv6PrefixesFromEC2 calls EC2 and returns a list of all addresses on the ENI -func (cache *EC2InstanceMetadataCache) GetIPv6PrefixesFromEC2(eniID string) (addrList []ec2types.Ipv6PrefixSpecification, err error) { +func (cache *EC2InstanceMetadataCache) GetIPv6PrefixesFromEC2(ctx context.Context, eniID string) (addrList []ec2types.Ipv6PrefixSpecification, err error) { eniIds := []*string{aws.String(eniID)} input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: aws.ToStringSlice(eniIds)} start := time.Now() - result, err := cache.ec2SVC.DescribeNetworkInterfaces(context.Background(), input) + result, err := cache.ec2SVC.DescribeNetworkInterfaces(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("DescribeNetworkInterfaces").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -1368,7 +1650,6 @@ func (cache *EC2InstanceMetadataCache) GetIPv6PrefixesFromEC2(eniID string) (add if awsAPIError.ErrorCode() == "InvalidNetworkInterfaceID.NotFound" { return nil, ErrENINotFound } - } checkAPIErrorAndBroadcastEvent(err, "ec2:DescribeNetworkInterfaces") awsAPIErrInc("DescribeNetworkInterfaces", err) @@ -1386,7 +1667,7 @@ func (cache *EC2InstanceMetadataCache) GetIPv6PrefixesFromEC2(eniID string) (add } // DescribeAllENIs calls EC2 to refresh the ENIMetadata and tags for all attached ENIs -func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult, error) { +func (cache *EC2InstanceMetadataCache) DescribeAllENIs(ctx context.Context) (DescribeAllENIsResult, error) { // Fetch all local ENI info from metadata allENIs, err := cache.GetAttachedENIs() @@ -1428,7 +1709,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult, for retryCount := 0; retryCount < maxENIEC2APIRetries && len(eniIDs) > 0; retryCount++ { input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: eniIDs} start := time.Now() - ec2Response, err = cache.ec2SVC.DescribeNetworkInterfaces(context.Background(), input) + ec2Response, err = cache.ec2SVC.DescribeNetworkInterfaces(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("DescribeNetworkInterfaces").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err == nil { @@ -1634,7 +1915,7 @@ func logOutOfSyncState(eniID string, imdsIPv4s, ec2IPv4s []ec2types.NetworkInter } // AllocIPAddress allocates an IP address for an ENI -func (cache *EC2InstanceMetadataCache) AllocIPAddress(eniID string) error { +func (cache *EC2InstanceMetadataCache) AllocIPAddress(ctx context.Context, eniID string) error { log.Infof("Trying to allocate an IP address on ENI: %s", eniID) input := &ec2.AssignPrivateIpAddressesInput{ @@ -1643,7 +1924,7 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddress(eniID string) error { } start := time.Now() - output, err := cache.ec2SVC.AssignPrivateIpAddresses(context.Background(), input) + output, err := cache.ec2SVC.AssignPrivateIpAddresses(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("AssignPrivateIpAddresses").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("AssignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -1658,7 +1939,7 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddress(eniID string) error { return nil } -func (cache *EC2InstanceMetadataCache) FetchInstanceTypeLimits() error { +func (cache *EC2InstanceMetadataCache) FetchInstanceTypeLimits(ctx context.Context) error { _, ok := vpc.GetInstance(cache.instanceType) if ok { return nil @@ -1666,7 +1947,7 @@ func (cache *EC2InstanceMetadataCache) FetchInstanceTypeLimits() error { log.Debugf("Instance type limits are missing from vpc_ip_limits.go hence making an EC2 call to fetch the limits") describeInstanceTypesInput := &ec2.DescribeInstanceTypesInput{InstanceTypes: []ec2types.InstanceType{ec2types.InstanceType(cache.instanceType)}} - output, err := cache.ec2SVC.DescribeInstanceTypes(context.Background(), describeInstanceTypesInput) + output, err := cache.ec2SVC.DescribeInstanceTypes(ctx, describeInstanceTypesInput) prometheusmetrics.Ec2ApiReq.WithLabelValues("DescribeInstanceTypes").Inc() if err != nil || len(output.InstanceTypes) != 1 { prometheusmetrics.Ec2ApiErr.WithLabelValues("DescribeInstanceTypes").Inc() @@ -1691,7 +1972,7 @@ func (cache *EC2InstanceMetadataCache) FetchInstanceTypeLimits() error { NetworkCardIndex: int64(*info.NetworkInfo.NetworkCards[idx].NetworkCardIndex), } } - //Not checking for empty hypervisorType since have seen certain instances not getting this filled. + // Not checking for empty hypervisorType since have seen certain instances not getting this filled. if instanceType != "" && eniLimit > 0 && ipv4Limit > 0 { vpc.SetInstance(instanceType, eniLimit, ipv4Limit, defaultNetworkCardIndex, networkCards, hypervisorType, isBareMetalInstance) } else { @@ -1766,8 +2047,8 @@ func (cache *EC2InstanceMetadataCache) IsPrefixDelegationSupported() bool { } // AllocIPAddresses allocates numIPs of IP address on an ENI -func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int) (*ec2.AssignPrivateIpAddressesOutput, error) { - var needIPs = numIPs +func (cache *EC2InstanceMetadataCache) AllocIPAddresses(ctx context.Context, eniID string, numIPs int) (*ec2.AssignPrivateIpAddressesOutput, error) { + needIPs := numIPs ipLimit := cache.GetENIIPv4Limit() @@ -1799,7 +2080,7 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int } start := time.Now() - output, err := cache.ec2SVC.AssignPrivateIpAddresses(context.Background(), input) + output, err := cache.ec2SVC.AssignPrivateIpAddresses(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("AssignPrivateIpAddresses").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("AssignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -1819,14 +2100,14 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int return output, nil } -func (cache *EC2InstanceMetadataCache) AllocIPv6Prefixes(eniID string) ([]*string, error) { - //We only need to allocate one IPv6 prefix per ENI. +func (cache *EC2InstanceMetadataCache) AllocIPv6Prefixes(ctx context.Context, eniID string) ([]*string, error) { + // We only need to allocate one IPv6 prefix per ENI. input := &ec2.AssignIpv6AddressesInput{ NetworkInterfaceId: aws.String(eniID), Ipv6PrefixCount: aws.Int32(1), } start := time.Now() - output, err := cache.ec2SVC.AssignIpv6Addresses(context.Background(), input) + output, err := cache.ec2SVC.AssignIpv6Addresses(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("AssignIpv6Addresses").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("AssignIpv6AddressesWithContext", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -1872,8 +2153,8 @@ func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, want eniIPCount = len(returnedENI.IPv6Addresses) } } else { - //Ignore primary IP of the ENI - //wantedCidrs will be at most 1 less then the IP limit for the ENI because of the primary IP in secondary pod + // Ignore primary IP of the ENI + // wantedCidrs will be at most 1 less then the IP limit for the ENI because of the primary IP in secondary pod eniIPCount = len(returnedENI.IPv4Addresses) - 1 } @@ -1916,7 +2197,7 @@ func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, want } // DeallocIPAddresses frees IP address on an ENI -func (cache *EC2InstanceMetadataCache) DeallocIPAddresses(eniID string, ips []string) error { +func (cache *EC2InstanceMetadataCache) DeallocIPAddresses(ctx context.Context, eniID string, ips []string) error { if len(ips) == 0 { return nil } @@ -1928,7 +2209,7 @@ func (cache *EC2InstanceMetadataCache) DeallocIPAddresses(eniID string, ips []st } start := time.Now() - _, err := cache.ec2SVC.UnassignPrivateIpAddresses(context.Background(), input) + _, err := cache.ec2SVC.UnassignPrivateIpAddresses(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("UnassignPrivateIpAddresses").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("UnassignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -1942,51 +2223,98 @@ func (cache *EC2InstanceMetadataCache) DeallocIPAddresses(eniID string, ips []st return nil } -// DeallocPrefixAddresses frees Prefixes on an ENI -func (cache *EC2InstanceMetadataCache) DeallocPrefixAddresses(eniID string, prefixes []string) error { +// DeallocPrefixAddresses frees Prefixes on an ENI (supports both IPv4 and IPv6) +func (cache *EC2InstanceMetadataCache) DeallocPrefixAddresses(ctx context.Context, eniID string, prefixes []string) error { if len(prefixes) == 0 { return nil } log.Infof("Trying to unassign the following Prefixes %v from ENI %s", prefixes, eniID) - input := &ec2.UnassignPrivateIpAddressesInput{ - NetworkInterfaceId: aws.String(eniID), - Ipv4Prefixes: prefixes, + // Separate IPv4 and IPv6 prefixes + var ipv4Prefixes []string + var ipv6Prefixes []string + + for _, prefix := range prefixes { + // Parse the CIDR to determine if it's IPv4 or IPv6 + _, cidr, err := net.ParseCIDR(prefix) + if err != nil { + log.Warnf("Failed to parse CIDR %s: %v", prefix, err) + continue + } + + if cidr.IP.To4() != nil { + ipv4Prefixes = append(ipv4Prefixes, prefix) + } else { + ipv6Prefixes = append(ipv6Prefixes, prefix) + } } - start := time.Now() - _, err := cache.ec2SVC.UnassignPrivateIpAddresses(context.Background(), input) - prometheusmetrics.Ec2ApiReq.WithLabelValues("UnassignPrivateIpAddresses").Inc() - prometheusmetrics.AwsAPILatency.WithLabelValues("UnassignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) - if err != nil { - checkAPIErrorAndBroadcastEvent(err, "ec2:UnassignPrivateIpAddresses") - awsAPIErrInc("UnassignPrivateIpAddresses", err) - prometheusmetrics.Ec2ApiErr.WithLabelValues("UnassignPrivateIpAddresses").Inc() - log.Errorf("Failed to deallocate a Prefixes address %v", err) - return errors.Wrap(err, fmt.Sprintf("deallocate prefix: failed to deallocate Prefix addresses: %v", prefixes)) + // Handle IPv4 prefixes using UnassignPrivateIpAddresses API + if len(ipv4Prefixes) > 0 { + log.Debugf("Deallocating IPv4 prefixes: %v", ipv4Prefixes) + input := &ec2.UnassignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv4Prefixes: ipv4Prefixes, + } + + start := time.Now() + _, err := cache.ec2SVC.UnassignPrivateIpAddresses(ctx, input) + prometheusmetrics.Ec2ApiReq.WithLabelValues("UnassignPrivateIpAddresses").Inc() + prometheusmetrics.AwsAPILatency.WithLabelValues("UnassignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) + if err != nil { + checkAPIErrorAndBroadcastEvent(err, "ec2:UnassignPrivateIpAddresses") + awsAPIErrInc("UnassignPrivateIpAddresses", err) + prometheusmetrics.Ec2ApiErr.WithLabelValues("UnassignPrivateIpAddresses").Inc() + log.Errorf("Failed to deallocate IPv4 Prefixes %v: %v", ipv4Prefixes, err) + return errors.Wrap(err, fmt.Sprintf("deallocate IPv4 prefix: failed to deallocate IPv4 Prefix addresses: %v", ipv4Prefixes)) + } + log.Debugf("Successfully freed IPv4 Prefixes %v from ENI %s", ipv4Prefixes, eniID) + } + + // Handle IPv6 prefixes using UnassignIpv6Addresses API + if len(ipv6Prefixes) > 0 { + log.Debugf("Deallocating IPv6 prefixes: %v", ipv6Prefixes) + input := &ec2.UnassignIpv6AddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv6Prefixes: ipv6Prefixes, + } + + start := time.Now() + _, err := cache.ec2SVC.UnassignIpv6Addresses(ctx, input) + prometheusmetrics.Ec2ApiReq.WithLabelValues("UnassignIpv6Addresses").Inc() + prometheusmetrics.AwsAPILatency.WithLabelValues("UnassignIpv6Addresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) + if err != nil { + checkAPIErrorAndBroadcastEvent(err, "ec2:UnassignIpv6Addresses") + awsAPIErrInc("UnassignIpv6Addresses", err) + prometheusmetrics.Ec2ApiErr.WithLabelValues("UnassignIpv6Addresses").Inc() + log.Errorf("Failed to deallocate IPv6 Prefixes %v: %v", ipv6Prefixes, err) + return errors.Wrap(err, fmt.Sprintf("deallocate IPv6 prefix: failed to deallocate IPv6 Prefix addresses: %v", ipv6Prefixes)) + } + log.Debugf("Successfully freed IPv6 Prefixes %v from ENI %s", ipv6Prefixes, eniID) } - log.Debugf("Successfully freed Prefixes %v from ENI %s", prefixes, eniID) + + log.Debugf("Successfully freed all Prefixes %v from ENI %s", prefixes, eniID) return nil } -func (cache *EC2InstanceMetadataCache) cleanUpLeakedENIs() { - cache.cleanUpLeakedENIsInternal(time.Duration(rand.Intn(eniCleanupStartupDelayMax)) * time.Second) +func (cache *EC2InstanceMetadataCache) cleanUpLeakedENIs(ctx context.Context) { + cache.cleanUpLeakedENIsInternal(ctx, time.Duration(rand.Intn(eniCleanupStartupDelayMax))*time.Second) } -func (cache *EC2InstanceMetadataCache) cleanUpLeakedENIsInternal(startupDelay time.Duration) { +func (cache *EC2InstanceMetadataCache) cleanUpLeakedENIsInternal(ctx context.Context, startupDelay time.Duration) { rand.Seed(time.Now().UnixNano()) log.Infof("Will attempt to clean up AWS CNI leaked ENIs after waiting %s.", startupDelay) time.Sleep(startupDelay) log.Debug("Checking for leaked AWS CNI ENIs.") - networkInterfaces, err := cache.getLeakedENIs() + networkInterfaces, err := cache.getLeakedENIs(ctx) if err != nil { log.Warnf("Unable to get leaked ENIs: %v", err) } else { // Clean up all the leaked ones we found for _, networkInterface := range networkInterfaces { eniID := aws.ToString(networkInterface.NetworkInterfaceId) - err = cache.deleteENI(eniID, maxENIBackoffDelay) + err = cache.deleteENI(ctx, eniID, maxENIBackoffDelay) if err != nil { awsUtilsErrInc("cleanUpLeakedENIDeleteErr", err) log.Warnf("Failed to clean up leaked ENI %s: %v", eniID, err) @@ -1997,7 +2325,7 @@ func (cache *EC2InstanceMetadataCache) cleanUpLeakedENIsInternal(startupDelay ti } } -func (cache *EC2InstanceMetadataCache) tagENIcreateTS(eniID string, maxBackoffDelay time.Duration) { +func (cache *EC2InstanceMetadataCache) tagENIcreateTS(ctx context.Context, eniID string, maxBackoffDelay time.Duration) { // Tag the ENI with "node.k8s.amazonaws.com/createdAt=currentTime" tags := []ec2types.Tag{ { @@ -2017,7 +2345,7 @@ func (cache *EC2InstanceMetadataCache) tagENIcreateTS(eniID string, maxBackoffDe _ = retry.NWithBackoff(retry.NewSimpleBackoff(500*time.Millisecond, maxBackoffDelay, 0.3, 2), 5, func() error { start := time.Now() - _, err := cache.ec2SVC.CreateTags(context.Background(), input) + _, err := cache.ec2SVC.CreateTags(ctx, input) prometheusmetrics.Ec2ApiReq.WithLabelValues("CreateTags").Inc() prometheusmetrics.AwsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { @@ -2034,7 +2362,7 @@ func (cache *EC2InstanceMetadataCache) tagENIcreateTS(eniID string, maxBackoffDe // getLeakedENIs calls DescribeNetworkInterfaces to get all available ENIs that were allocated by // the AWS CNI plugin, but were not deleted. -func (cache *EC2InstanceMetadataCache) getLeakedENIs() ([]ec2types.NetworkInterface, error) { +func (cache *EC2InstanceMetadataCache) getLeakedENIs(ctx context.Context) ([]ec2types.NetworkInterface, error) { leakedENIFilters := []ec2types.Filter{ { Name: aws.String("tag-key"), @@ -2080,7 +2408,7 @@ func (cache *EC2InstanceMetadataCache) getLeakedENIs() ([]ec2types.NetworkInterf parsedTime, err := time.Parse(time.RFC3339, value) if err != nil { log.Warnf("ParsedTime format %s is wrong so retagging with current TS", parsedTime) - cache.tagENIcreateTS(aws.ToString(networkInterface.NetworkInterfaceId), maxENIBackoffDelay) + cache.tagENIcreateTS(ctx, aws.ToString(networkInterface.NetworkInterfaceId), maxENIBackoffDelay) } if time.Since(parsedTime) < eniDeleteCooldownTime { log.Infof("Found an ENI created less than 5 minutes ago, so not cleaning it up") @@ -2091,7 +2419,7 @@ func (cache *EC2InstanceMetadataCache) getLeakedENIs() ([]ec2types.NetworkInterf /* Set a time if we didn't find one. This is to prevent accidentally deleting ENIs that are in the * process of being attached by CNI versions v1.5.x or earlier. */ - cache.tagENIcreateTS(aws.ToString(networkInterface.NetworkInterfaceId), maxENIBackoffDelay) + cache.tagENIcreateTS(ctx, aws.ToString(networkInterface.NetworkInterfaceId), maxENIBackoffDelay) return nil } networkInterfaces = append(networkInterfaces, networkInterface) @@ -2253,3 +2581,35 @@ func checkAPIErrorAndBroadcastEvent(err error, api string) { } } } + +// IsSubnetExcluded checks if a subnet is excluded by examining its kubernetes.io/role/cni tag +func (cache *EC2InstanceMetadataCache) IsSubnetExcluded(ctx context.Context, subnetID string) (bool, error) { + // Get all VPC subnets with their tags + subnets, err := cache.GetVpcSubnets(ctx) + if err != nil { + return false, fmt.Errorf("failed to get VPC subnets: %v", err) + } + + // Find the specific subnet and check its tags + for _, subnet := range subnets { + if *subnet.SubnetId == subnetID { + // Check if the subnet has the exclusion tag kubernetes.io/role/cni=0 + for _, tag := range subnet.Tags { + if *tag.Key == "kubernetes.io/role/cni" { + tagValue := *tag.Value + excluded := tagValue == "0" + log.Debugf("IsSubnetExcluded: subnet %s has tag kubernetes.io/role/cni=%s, excluded=%t", subnetID, tagValue, excluded) + return excluded, nil + } + } + + // If no kubernetes.io/role/cni tag found, subnet is not explicitly excluded + log.Debugf("IsSubnetExcluded: subnet %s has no kubernetes.io/role/cni tag, not excluded", subnetID) + return false, nil + } + } + + // Subnet not found in VPC + log.Warnf("IsSubnetExcluded: subnet %s not found in VPC", subnetID) + return false, fmt.Errorf("subnet %s not found in VPC", subnetID) +} diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go index 84e840271e..935fe8f1b8 100644 --- a/pkg/awsutils/awsutils_test.go +++ b/pkg/awsutils/awsutils_test.go @@ -36,6 +36,7 @@ import ( "github.com/stretchr/testify/assert" mock_ec2wrapper "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2wrapper/mocks" + "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/eventrecorder" "github.com/aws/amazon-vpc-cni-k8s/pkg/vpc" @@ -353,7 +354,7 @@ func TestAWSGetFreeDeviceNumberOnErr(t *testing.T) { mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any()).Return(nil, errors.New("error on DescribeInstances")) cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2} - _, err := cache.awsGetFreeDeviceNumber(0) + _, err := cache.awsGetFreeDeviceNumber(context.Background(), 0) assert.Error(t, err) } @@ -373,12 +374,13 @@ func TestAWSGetFreeDeviceNumberNoDevice(t *testing.T) { result := &ec2.DescribeInstancesOutput{Reservations: []ec2types.Reservation{{ Instances: []ec2types.Instance{{ NetworkInterfaces: ec2ENIs, - }}}}} + }}, + }}} mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2} - _, err := cache.awsGetFreeDeviceNumber(0) + _, err := cache.awsGetFreeDeviceNumber(context.Background(), 0) assert.Error(t, err) } @@ -445,7 +447,7 @@ func TestGetENIAttachmentID(t *testing.T) { mockEC2.EXPECT().DescribeNetworkInterfaces(gomock.Any(), gomock.Any(), gomock.Any()).Return(tc.output, tc.err) cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2} - id, err := cache.getENIAttachmentID("test-eni") + id, err := cache.getENIAttachmentID(context.Background(), "test-eni") assert.Equal(t, tc.expErr, err) assert.Equal(t, tc.expID, id) } @@ -506,7 +508,7 @@ func TestDescribeAllENIs(t *testing.T) { _ = os.Setenv(utils.EnvEnableImdsOnlyMode, "true") } vpc.SetInstance("test", 4, 10, 0, []vpc.NetworkCard{{MaximumNetworkInterfaces: 4, NetworkCardIndex: 0}}, "nitro", false) - metaData, err := cache.DescribeAllENIs() + metaData, err := cache.DescribeAllENIs(context.Background()) if !tc.expEC2call { _ = os.Unsetenv(utils.EnvEnableImdsOnlyMode) } @@ -551,12 +553,14 @@ func TestAllocENI(t *testing.T) { ec2ENIs = append(ec2ENIs, ec2ENI) result := &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}} + Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}, + } mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) attachmentID := "eni-attach-58ddda9d" attachResult := &ec2.AttachNetworkInterfaceOutput{ - AttachmentId: &attachmentID} + AttachmentId: &attachmentID, + } mockEC2.EXPECT().AttachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(attachResult, nil) mockEC2.EXPECT().ModifyNetworkInterfaceAttribute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) @@ -567,7 +571,7 @@ func TestAllocENI(t *testing.T) { useSubnetDiscovery: true, } - _, err := cache.AllocENI(nil, "", 5, 0) + _, err := cache.AllocENI(context.Background(), nil, "", 5, 0) assert.NoError(t, err) } @@ -605,7 +609,8 @@ func TestAllocENINoFreeDevice(t *testing.T) { ec2ENIs = append(ec2ENIs, ec2ENI) } result := &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}} + Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}, + } mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) mockEC2.EXPECT().DeleteNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) @@ -617,7 +622,7 @@ func TestAllocENINoFreeDevice(t *testing.T) { useSubnetDiscovery: true, } - _, err := cache.AllocENI(nil, "", 5, 0) + _, err := cache.AllocENI(context.Background(), nil, "", 5, 0) assert.Error(t, err) } @@ -657,7 +662,8 @@ func TestAllocENIMaxReached(t *testing.T) { ec2ENIs = append(ec2ENIs, ec2ENI) result := &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}} + Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}, + } mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) mockEC2.EXPECT().AttachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("AttachmentLimitExceeded")) @@ -670,7 +676,7 @@ func TestAllocENIMaxReached(t *testing.T) { useSubnetDiscovery: true, } - _, err := cache.AllocENI(nil, "", 5, 0) + _, err := cache.AllocENI(context.Background(), nil, "", 5, 0) assert.Error(t, err) } @@ -708,16 +714,18 @@ func TestAllocENIWithIPAddresses(t *testing.T) { ec2ENIs = append(ec2ENIs, ec2ENI) result := &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}} + Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}, + } mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) attachmentID := "eni-attach-58ddda9d" attachResult := &ec2.AttachNetworkInterfaceOutput{ - AttachmentId: &attachmentID} + AttachmentId: &attachmentID, + } mockEC2.EXPECT().AttachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(attachResult, nil) mockEC2.EXPECT().ModifyNetworkInterfaceAttribute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", useSubnetDiscovery: true} - _, err := cache.AllocENI(nil, subnetID, 5, 0) + _, err := cache.AllocENI(context.Background(), nil, subnetID, 5, 0) assert.NoError(t, err) // when required IP numbers(50) is higher than ENI's limit(49) @@ -727,7 +735,7 @@ func TestAllocENIWithIPAddresses(t *testing.T) { mockEC2.EXPECT().AttachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(attachResult, nil) mockEC2.EXPECT().ModifyNetworkInterfaceAttribute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) cache = &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", useSubnetDiscovery: true} - _, err = cache.AllocENI(nil, subnetID, 49, 0) + _, err = cache.AllocENI(context.Background(), nil, subnetID, 49, 0) assert.NoError(t, err) } @@ -761,7 +769,7 @@ func TestAllocENIWithIPAddressesAlreadyFull(t *testing.T) { instanceType: "t3.xlarge", useSubnetDiscovery: true, } - _, err := cache.AllocENI(nil, "", 14, 0) + _, err := cache.AllocENI(context.Background(), nil, "", 14, 0) assert.Error(t, err) } @@ -800,11 +808,13 @@ func TestAllocENIWithPrefixAddresses(t *testing.T) { ec2ENIs = append(ec2ENIs, ec2ENI) result := &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}} + Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}, + } mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) attachmentID := "eni-attach-58ddda9d" attachResult := &ec2.AttachNetworkInterfaceOutput{ - AttachmentId: &attachmentID} + AttachmentId: &attachmentID, + } mockEC2.EXPECT().AttachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(attachResult, nil) mockEC2.EXPECT().ModifyNetworkInterfaceAttribute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) @@ -815,7 +825,7 @@ func TestAllocENIWithPrefixAddresses(t *testing.T) { enablePrefixDelegation: true, useSubnetDiscovery: true, } - _, err := cache.AllocENI(nil, subnetID, 1, 0) + _, err := cache.AllocENI(context.Background(), nil, subnetID, 1, 0) assert.NoError(t, err) } @@ -850,7 +860,7 @@ func TestAllocENIWithPrefixesAlreadyFull(t *testing.T) { enablePrefixDelegation: true, useSubnetDiscovery: true, } - _, err := cache.AllocENI(nil, "", 1, 0) + _, err := cache.AllocENI(context.Background(), nil, "", 1, 0) assert.Error(t, err) } @@ -861,7 +871,8 @@ func TestFreeENI(t *testing.T) { attachmentID := eniAttachID attachment := &ec2types.NetworkInterfaceAttachment{AttachmentId: &attachmentID} result := &ec2.DescribeNetworkInterfacesOutput{ - NetworkInterfaces: []ec2types.NetworkInterface{{Attachment: attachment}}} + NetworkInterfaces: []ec2types.NetworkInterface{{Attachment: attachment}}, + } mockEC2.EXPECT().DescribeNetworkInterfaces(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) mockEC2.EXPECT().DetachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) mockEC2.EXPECT().DeleteNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) @@ -870,7 +881,7 @@ func TestFreeENI(t *testing.T) { ec2SVC: mockEC2, } - err := cache.freeENI("test-eni", time.Millisecond, time.Millisecond) + err := cache.freeENI(context.Background(), "test-eni", time.Millisecond, time.Millisecond) assert.NoError(t, err) } @@ -881,7 +892,8 @@ func TestFreeENIRetry(t *testing.T) { attachmentID := eniAttachID attachment := &ec2types.NetworkInterfaceAttachment{AttachmentId: &attachmentID} result := &ec2.DescribeNetworkInterfacesOutput{ - NetworkInterfaces: []ec2types.NetworkInterface{{Attachment: attachment}}} + NetworkInterfaces: []ec2types.NetworkInterface{{Attachment: attachment}}, + } mockEC2.EXPECT().DescribeNetworkInterfaces(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) // retry 2 times @@ -893,7 +905,7 @@ func TestFreeENIRetry(t *testing.T) { ec2SVC: mockEC2, } - err := cache.freeENI("test-eni", time.Millisecond, time.Millisecond) + err := cache.freeENI(context.Background(), "test-eni", time.Millisecond, time.Millisecond) assert.NoError(t, err) } @@ -935,7 +947,8 @@ func TestFreeENIRetryMax(t *testing.T) { attachmentID := eniAttachID attachment := &ec2types.NetworkInterfaceAttachment{AttachmentId: &attachmentID} result := &ec2.DescribeNetworkInterfacesOutput{ - NetworkInterfaces: []ec2types.NetworkInterface{{Attachment: attachment}}} + NetworkInterfaces: []ec2types.NetworkInterface{{Attachment: attachment}}, + } mockEC2.EXPECT().DescribeNetworkInterfaces(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) mockEC2.EXPECT().DetachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) @@ -947,7 +960,7 @@ func TestFreeENIRetryMax(t *testing.T) { ec2SVC: mockEC2, } - err := cache.freeENI("test-eni", time.Millisecond, time.Millisecond) + err := cache.freeENI(context.Background(), "test-eni", time.Millisecond, time.Millisecond) assert.Error(t, err) } @@ -961,7 +974,7 @@ func TestFreeENIDescribeErr(t *testing.T) { ec2SVC: mockEC2, } - err := cache.FreeENI("test-eni") + err := cache.FreeENI(context.Background(), "test-eni") assert.Error(t, err) } @@ -970,9 +983,11 @@ func TestDescribeInstanceTypes(t *testing.T) { defer ctrl.Finish() mockEC2.EXPECT().DescribeInstanceTypes(gomock.Any(), gomock.Any(), gomock.Any()).Return(&ec2.DescribeInstanceTypesOutput{ InstanceTypes: []ec2types.InstanceTypeInfo{ - {InstanceType: "not-there", NetworkInfo: &ec2types.NetworkInfo{ - MaximumNetworkInterfaces: aws.Int32(9), - Ipv4AddressesPerInterface: aws.Int32(99)}, + { + InstanceType: "not-there", NetworkInfo: &ec2types.NetworkInfo{ + MaximumNetworkInterfaces: aws.Int32(9), + Ipv4AddressesPerInterface: aws.Int32(99), + }, }, }, NextToken: nil, @@ -980,7 +995,7 @@ func TestDescribeInstanceTypes(t *testing.T) { cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2} cache.instanceType = "not-there" - err := cache.FetchInstanceTypeLimits() + err := cache.FetchInstanceTypeLimits(context.Background()) assert.NoError(t, err) value := cache.GetENILimit() assert.Equal(t, 9, value) @@ -995,7 +1010,7 @@ func TestAllocIPAddress(t *testing.T) { mockEC2.EXPECT().AssignPrivateIpAddresses(gomock.Any(), gomock.Any(), gomock.Any()).Return(&ec2.AssignPrivateIpAddressesOutput{}, nil) cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2} - err := cache.AllocIPAddress("eni-id") + err := cache.AllocIPAddress(context.Background(), "eni-id") assert.NoError(t, err) } @@ -1006,7 +1021,7 @@ func TestAllocIPAddressOnErr(t *testing.T) { mockEC2.EXPECT().AssignPrivateIpAddresses(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("Error on AssignPrivateIpAddressesWithContext")) cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2} - err := cache.AllocIPAddress("eni-id") + err := cache.AllocIPAddress(context.Background(), "eni-id") assert.Error(t, err) } @@ -1022,7 +1037,7 @@ func TestAllocIPAddresses(t *testing.T) { mockEC2.EXPECT().AssignPrivateIpAddresses(gomock.Any(), input, gomock.Any()).Return(nil, nil) cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge"} - _, err := cache.AllocIPAddresses(eniID, 5) + _, err := cache.AllocIPAddresses(context.Background(), eniID, 5) assert.NoError(t, err) // when required IP numbers(50) is higher than ENI's limit(49) @@ -1038,11 +1053,11 @@ func TestAllocIPAddresses(t *testing.T) { mockEC2.EXPECT().AssignPrivateIpAddresses(gomock.Any(), input, gomock.Any()).Return(&output, nil) cache = &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge"} - _, err = cache.AllocIPAddresses(eniID, 50) + _, err = cache.AllocIPAddresses(context.Background(), eniID, 50) assert.NoError(t, err) // Adding 0 should do nothing - _, err = cache.AllocIPAddresses(eniID, 0) + _, err = cache.AllocIPAddresses(context.Background(), eniID, 0) assert.NoError(t, err) } @@ -1059,7 +1074,7 @@ func TestAllocIPAddressesAlreadyFull(t *testing.T) { retErr := &smithy.GenericAPIError{Code: "PrivateIpAddressLimitExceeded", Message: "Too many IPs already allocated"} mockEC2.EXPECT().AssignPrivateIpAddresses(gomock.Any(), input, gomock.Any()).Return(nil, retErr) // If EC2 says that all IPs are already attached, then DS is out of sync so alloc will fail - _, err := cache.AllocIPAddresses(eniID, 14) + _, err := cache.AllocIPAddresses(context.Background(), eniID, 14) assert.Error(t, err) } @@ -1067,7 +1082,7 @@ func TestAllocPrefixAddresses(t *testing.T) { ctrl, mockEC2 := setup(t) defer ctrl.Finish() - //Allocate 1 prefix for the ENI + // Allocate 1 prefix for the ENI input := &ec2.AssignPrivateIpAddressesInput{ NetworkInterfaceId: aws.String(eniID), Ipv4PrefixCount: aws.Int32(1), @@ -1075,11 +1090,11 @@ func TestAllocPrefixAddresses(t *testing.T) { mockEC2.EXPECT().AssignPrivateIpAddresses(gomock.Any(), input, gomock.Any()).Return(nil, nil) cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", enablePrefixDelegation: true} - _, err := cache.AllocIPAddresses(eniID, 1) + _, err := cache.AllocIPAddresses(context.Background(), eniID, 1) assert.NoError(t, err) // Adding 0 should do nothing - _, err = cache.AllocIPAddresses(eniID, 0) + _, err = cache.AllocIPAddresses(context.Background(), eniID, 0) assert.NoError(t, err) } @@ -1096,7 +1111,7 @@ func TestAllocPrefixesAlreadyFull(t *testing.T) { retErr := &smithy.GenericAPIError{Code: "PrivateIpAddressLimitExceeded", Message: "Too many IPs already allocated"} mockEC2.EXPECT().AssignPrivateIpAddresses(gomock.Any(), input, gomock.Any()).Return(nil, retErr) // If EC2 says that all IPs are already attached, then DS is out of sync so alloc will fail - _, err := cache.AllocIPAddresses(eniID, 1) + _, err := cache.AllocIPAddresses(context.Background(), eniID, 1) assert.Error(t, err) } @@ -1309,8 +1324,10 @@ func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) { metadataMACPath + eni2MAC + metaDataPrefixPath: eniPrefixes, }) } - cache := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2, - enablePrefixDelegation: true, v4Enabled: tt.args.v4Enabled, v6Enabled: tt.args.v6Enabled} + cache := &EC2InstanceMetadataCache{ + imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2, + enablePrefixDelegation: true, v4Enabled: tt.args.v4Enabled, v6Enabled: tt.args.v6Enabled, + } gotEniMetadata, err := cache.waitForENIAndIPsAttached(tt.args.eni, tt.args.wantedSecondaryIPs, tt.args.maxBackoffDelay) if (err != nil) != tt.wantErr { t.Errorf("waitForENIAndIPsAttached() error = %+v, wantErr %+v", err, tt.wantErr) @@ -1352,11 +1369,12 @@ func TestEC2InstanceMetadataCache_cleanUpLeakedENIsInternal(t *testing.T) { cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2} // Test checks that both mocks gets called. - cache.cleanUpLeakedENIsInternal(time.Millisecond) + cache.cleanUpLeakedENIsInternal(context.Background(), time.Millisecond) } func setupDescribeNetworkInterfacesPagesWithContextMock( - t *testing.T, mockEC2 *mock_ec2wrapper.MockEC2, interfaces []ec2types.NetworkInterface, err error, times int) { + t *testing.T, mockEC2 *mock_ec2wrapper.MockEC2, interfaces []ec2types.NetworkInterface, err error, times int, +) { mockEC2.EXPECT(). DescribeNetworkInterfaces(gomock.Any(), gomock.Any(), gomock.Any()). Times(times). @@ -1889,7 +1907,7 @@ func TestEC2InstanceMetadataCache_getLeakedENIs(t *testing.T) { }) } cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, clusterName: tt.fields.clusterName, vpcID: vpcID} - got, err := cache.getLeakedENIs() + got, err := cache.getLeakedENIs(context.Background()) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { @@ -2054,7 +2072,7 @@ func TestEC2InstanceMetadataCache_TagENI(t *testing.T) { clusterName: tt.fields.clusterName, additionalENITags: tt.fields.additionalENITags, } - err := cache.TagENI(tt.args.eniID, tt.args.currentTags) + err := cache.TagENI(context.Background(), tt.args.eniID, tt.args.currentTags) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { @@ -2236,3 +2254,1367 @@ func Test_loadAdditionalENITags(t *testing.T) { }) } } + +func TestValidTagWithClusterSpecificTags(t *testing.T) { + // Save original environment and restore it after the test + originalClusterName := os.Getenv(clusterNameEnvVar) + defer os.Setenv(clusterNameEnvVar, originalClusterName) + + // Set a test cluster name + const testClusterName = "my-example-cluster" + os.Setenv(clusterNameEnvVar, testClusterName) + + tests := []struct { + name string + subnet ec2types.Subnet + isPrimarySubnet bool + want bool + description string + }{ + { + name: "subnet-123: subnet with CNI tag but no cluster tag", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-123"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + isPrimarySubnet: false, + want: true, + description: "Should be available to all clusters when no cluster tags present", + }, + { + name: "subnet-456: subnet with CNI tag and different cluster's tag", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-456"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + { + Key: aws.String("kubernetes.io/cluster/some-other-cluster"), + Value: aws.String("shared"), + }, + }, + }, + isPrimarySubnet: false, + want: false, + description: "Should not be available to our cluster when tagged for a different cluster", + }, + { + name: "subnet-789: subnet with CNI tag and multiple cluster tags", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-789"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + { + Key: aws.String("kubernetes.io/cluster/" + testClusterName), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/cluster/some-other-cluster"), + Value: aws.String("shared"), + }, + }, + }, + isPrimarySubnet: false, + want: true, + description: "Should be available to our cluster when tagged for multiple clusters including ours", + }, + { + name: "subnet-abc: subnet with cluster tag but no CNI tag", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-abc"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/cluster/" + testClusterName), + Value: aws.String("shared"), + }, + }, + }, + isPrimarySubnet: false, + want: false, + description: "Should NOT be available to any cluster when CNI tag is missing", + }, + { + name: "primary subnet with wrong cluster value", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-def"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + { + Key: aws.String("kubernetes.io/cluster/" + testClusterName), + Value: aws.String("not-shared"), // Wrong value + }, + }, + }, + isPrimarySubnet: true, + want: false, + description: "Should NOT be available when cluster tag has wrong value", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := validTag(tt.subnet, tt.isPrimarySubnet) + assert.Equal(t, tt.want, got, tt.description) + }) + } +} + +func TestValidTag(t *testing.T) { + tests := []struct { + name string + subnet ec2types.Subnet + isPrimarySubnet bool + want bool + }{ + { + name: "primary subnet with tag value 0 - should exclude", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-123"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + }, + isPrimarySubnet: true, + want: false, + }, + { + name: "primary subnet with tag value 1 - should include", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-123"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + isPrimarySubnet: true, + want: true, + }, + { + name: "primary subnet with empty tag value - should include", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-123"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String(""), + }, + }, + }, + isPrimarySubnet: true, + want: true, + }, + { + name: "primary subnet with nil tag value - should include", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-123"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: nil, + }, + }, + }, + isPrimarySubnet: true, + want: true, + }, + { + name: "primary subnet without the tag - should include (backwards compatible)", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-123"), + Tags: []ec2types.Tag{ + { + Key: aws.String("some-other-tag"), + Value: aws.String("value"), + }, + }, + }, + isPrimarySubnet: true, + want: true, + }, + { + name: "secondary subnet without the tag - should exclude", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-123"), + Tags: []ec2types.Tag{ + { + Key: aws.String("some-other-tag"), + Value: aws.String("value"), + }, + }, + }, + isPrimarySubnet: false, + want: false, + }, + { + name: "secondary subnet with tag value 0 - should exclude", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-123"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + }, + isPrimarySubnet: false, + want: false, + }, + { + name: "secondary subnet with tag value 1 - should include", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-123"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + isPrimarySubnet: false, + want: true, + }, + { + name: "secondary subnet with tag value non-zero - should include", + subnet: ec2types.Subnet{ + SubnetId: aws.String("subnet-123"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("true"), + }, + }, + }, + isPrimarySubnet: false, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := validTag(tt.subnet, tt.isPrimarySubnet) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestIsSubnetExcluded(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + tests := []struct { + name string + subnetTags []ec2types.Tag + describeError error + want bool + wantErr bool + }{ + { + name: "subnet with tag value 0 - excluded", + subnetTags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + want: true, + wantErr: false, + }, + { + name: "subnet with tag value 1 - not excluded", + subnetTags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + want: false, + wantErr: false, + }, + { + name: "subnet without tag - not excluded", + subnetTags: []ec2types.Tag{ + { + Key: aws.String("other-tag"), + Value: aws.String("value"), + }, + }, + want: false, + wantErr: false, + }, + { + name: "GetVpcSubnets API error", + describeError: errors.New("API error"), + want: false, + wantErr: true, + }, + { + name: "no subnets returned", + subnetTags: []ec2types.Tag{}, + want: false, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + vpcID: "vpc-12345", + availabilityZone: "us-west-2a", + } + + if tt.describeError != nil { + // Mock DescribeSubnets to return error + mockEC2.EXPECT().DescribeSubnets(gomock.Any(), gomock.Any()).Return(nil, tt.describeError) + } else { + // Mock DescribeSubnets for GetVpcSubnets + subnetResult := &ec2.DescribeSubnetsOutput{ + Subnets: []ec2types.Subnet{ + { + SubnetId: aws.String(subnetID), + Tags: tt.subnetTags, + VpcId: aws.String("vpc-12345"), + }, + }, + } + mockEC2.EXPECT().DescribeSubnets(gomock.Any(), gomock.Any()).Return(subnetResult, nil) + } + + got, err := cache.IsSubnetExcluded(context.Background(), subnetID) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} + +func TestAllocENIWithSubnetExclusion(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + mockMetadata := testMetadata(nil) + + tests := []struct { + name string + subnets []ec2types.Subnet + useSubnetDiscovery bool + expectError bool + errorContains string + }{ + { + name: "multiple subnets with primary excluded", + subnets: []ec2types.Subnet{ + { + SubnetId: aws.String(subnetID), + AvailableIpAddressCount: aws.Int32(100), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + }, + { + SubnetId: aws.String("subnet-secondary"), + AvailableIpAddressCount: aws.Int32(100), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + }, + useSubnetDiscovery: true, + expectError: false, + }, + { + name: "all subnets excluded", + subnets: []ec2types.Subnet{ + { + SubnetId: aws.String(subnetID), + AvailableIpAddressCount: aws.Int32(100), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + }, + { + SubnetId: aws.String("subnet-secondary"), + AvailableIpAddressCount: aws.Int32(100), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + }, + }, + useSubnetDiscovery: true, + expectError: true, + errorContains: "no valid subnets available for ENI creation", + }, + { + name: "subnet discovery disabled with primary excluded", + subnets: []ec2types.Subnet{ + { + SubnetId: aws.String(subnetID), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + }, + }, + useSubnetDiscovery: false, + expectError: true, + errorContains: "primary subnet is tagged with kubernetes.io/role/cni=0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.useSubnetDiscovery { + subnetResult := &ec2.DescribeSubnetsOutput{Subnets: tt.subnets} + mockEC2.EXPECT().DescribeSubnets(gomock.Any(), gomock.Any(), gomock.Any()).Return(subnetResult, nil) + } else { + // When subnet discovery is disabled, it will check if primary subnet is excluded + primarySubnetResult := &ec2.DescribeSubnetsOutput{Subnets: tt.subnets} + mockEC2.EXPECT().DescribeSubnets(gomock.Any(), gomock.Any()).Return(primarySubnetResult, nil) + } + + if !tt.expectError { + // Setup successful ENI creation + cureniID := eniID + eni := ec2.CreateNetworkInterfaceOutput{NetworkInterface: &ec2types.NetworkInterface{NetworkInterfaceId: &cureniID}} + mockEC2.EXPECT().CreateNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(&eni, nil) + + // Mock DescribeInstances for free device number + ec2ENIs := make([]ec2types.InstanceNetworkInterface, 0) + deviceNum1 := int32(0) + ec2ENI := ec2types.InstanceNetworkInterface{Attachment: &ec2types.InstanceNetworkInterfaceAttachment{DeviceIndex: &deviceNum1}} + ec2ENIs = append(ec2ENIs, ec2ENI) + result := &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}, + } + mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) + + // Mock AttachNetworkInterface + attachmentID := "eni-attach-123" + attachResult := &ec2.AttachNetworkInterfaceOutput{AttachmentId: &attachmentID} + mockEC2.EXPECT().AttachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(attachResult, nil) + mockEC2.EXPECT().ModifyNetworkInterfaceAttribute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + } + + cache := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + imds: TypedIMDS{mockMetadata}, + instanceType: "c5n.18xlarge", + useSubnetDiscovery: tt.useSubnetDiscovery, + subnetID: subnetID, + } + + _, err := cache.AllocENI(context.Background(), nil, "", 5, 0) + if tt.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errorContains) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestAllocENIWithSubnetDiscoveryIPv6(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + mockMetadata := testMetadata(nil) + + // Define test variables + primaryENI := "eni-primary" + clusterName := "test-cluster" + + tests := []struct { + name string + subnets []ec2types.Subnet + useSubnetDiscovery bool + v6Enabled bool + expectError bool + errorContains string + }{ + { + name: "IPv6 enabled with subnet discovery and primary excluded", + subnets: []ec2types.Subnet{ + { + SubnetId: aws.String(subnetID), + AvailableIpAddressCount: aws.Int32(100), + Ipv6CidrBlockAssociationSet: []ec2types.SubnetIpv6CidrBlockAssociation{ + { + Ipv6CidrBlock: aws.String("2001:db8::/64"), + }, + }, + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + }, + { + SubnetId: aws.String("subnet-secondary-v6"), + AvailableIpAddressCount: aws.Int32(100), + Ipv6CidrBlockAssociationSet: []ec2types.SubnetIpv6CidrBlockAssociation{ + { + Ipv6CidrBlock: aws.String("2001:db8:1::/64"), + }, + }, + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + }, + useSubnetDiscovery: true, + v6Enabled: true, + expectError: false, + }, + { + name: "IPv6 enabled with subnet discovery and mixed IPv4/IPv6 subnets", + subnets: []ec2types.Subnet{ + { + SubnetId: aws.String(subnetID), + AvailableIpAddressCount: aws.Int32(100), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + { + SubnetId: aws.String("subnet-v6-only"), + AvailableIpAddressCount: aws.Int32(100), + Ipv6CidrBlockAssociationSet: []ec2types.SubnetIpv6CidrBlockAssociation{ + { + Ipv6CidrBlock: aws.String("2001:db8:2::/64"), + }, + }, + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + }, + useSubnetDiscovery: true, + v6Enabled: true, + expectError: false, + }, + { + name: "IPv6 enabled with all subnets excluded", + subnets: []ec2types.Subnet{ + { + SubnetId: aws.String(subnetID), + Ipv6CidrBlockAssociationSet: []ec2types.SubnetIpv6CidrBlockAssociation{ + { + Ipv6CidrBlock: aws.String("2001:db8::/64"), + }, + }, + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + }, + }, + useSubnetDiscovery: true, + v6Enabled: true, + expectError: true, + errorContains: "no valid subnets available for ENI creation", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ins := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + imds: TypedIMDS{mockMetadata}, + primaryENImac: primaryMAC, + useSubnetDiscovery: tt.useSubnetDiscovery, + v6Enabled: tt.v6Enabled, + instanceID: instanceID, + vpcID: vpcID, + primaryENI: primaryENI, + clusterName: clusterName, + enablePrefixDelegation: false, + instanceType: "c5n.18xlarge", + subnetID: subnetID, + } + + if tt.useSubnetDiscovery { + subnetResult := &ec2.DescribeSubnetsOutput{Subnets: tt.subnets} + mockEC2.EXPECT().DescribeSubnets(gomock.Any(), gomock.Any(), gomock.Any()).Return(subnetResult, nil) + } + + if !tt.expectError { + // Setup successful ENI creation + mockEC2.EXPECT().CreateNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &ec2.CreateNetworkInterfaceOutput{ + NetworkInterface: &ec2types.NetworkInterface{ + NetworkInterfaceId: aws.String("eni-test-v6"), + TagSet: []ec2types.Tag{ + { + Key: aws.String(eniNodeTagKey), + Value: aws.String(ins.instanceID), + }, + }, + }, + }, nil) + + // Mock DescribeInstances to get available device indices + mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + { + NetworkInterfaces: []ec2types.InstanceNetworkInterface{ + { + Attachment: &ec2types.InstanceNetworkInterfaceAttachment{ + DeviceIndex: aws.Int32(0), + }, + }, + }, + }, + }, + }, + }, + }, nil) + + // Mock AttachNetworkInterface + attachmentID := "eni-attach-123" + mockEC2.EXPECT().AttachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &ec2.AttachNetworkInterfaceOutput{AttachmentId: &attachmentID}, nil) + + // Mock ModifyNetworkInterfaceAttribute + mockEC2.EXPECT().ModifyNetworkInterfaceAttribute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + } + + eniID, err := ins.AllocENI(context.Background(), nil, "", 5, 0) + + if tt.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errorContains) + } else { + assert.NoError(t, err) + assert.Equal(t, "eni-test-v6", eniID) + } + }) + } +} + +func TestAllocENIWithSubnetDiscoveryFailure(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + mockMetadata := testMetadata(nil) + + // Test when DescribeSubnets fails and fallback to primary subnet which is excluded + primarySubnetTags := []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + } + + // First call to DescribeSubnets fails + mockEC2.EXPECT().DescribeSubnets(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("API error")) + + // Then it checks if primary subnet is excluded + primarySubnetResult := &ec2.DescribeSubnetsOutput{ + Subnets: []ec2types.Subnet{ + { + SubnetId: aws.String(subnetID), + Tags: primarySubnetTags, + }, + }, + } + mockEC2.EXPECT().DescribeSubnets(gomock.Any(), gomock.Any()).Return(primarySubnetResult, nil) + + cache := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + imds: TypedIMDS{mockMetadata}, + instanceType: "c5n.18xlarge", + useSubnetDiscovery: true, + subnetID: subnetID, + } + + _, err := cache.AllocENI(context.Background(), nil, "", 5, 0) + assert.Error(t, err) + assert.Contains(t, err.Error(), "primary subnet is tagged with kubernetes.io/role/cni=0") +} + +// TestDiscoverCustomSecurityGroups tests the discoverCustomSecurityGroups method +func TestDiscoverCustomSecurityGroups(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + mockMetadata := testMetadata(nil) + + tests := []struct { + name string + describeOutput *ec2.DescribeSecurityGroupsOutput + describeError error + expectedSGIDs []string + expectError bool + errorContains string + }{ + { + name: "successful discovery with multiple SGs", + describeOutput: &ec2.DescribeSecurityGroupsOutput{ + SecurityGroups: []ec2types.SecurityGroup{ + { + GroupId: aws.String("sg-custom1"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + { + GroupId: aws.String("sg-custom2"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + }, + }, + describeError: nil, + expectedSGIDs: []string{"sg-custom1", "sg-custom2"}, + expectError: false, + }, + { + name: "empty security group list", + describeOutput: &ec2.DescribeSecurityGroupsOutput{ + SecurityGroups: []ec2types.SecurityGroup{}, + }, + describeError: nil, + expectedSGIDs: []string{}, + expectError: false, + }, + { + name: "describe API error", + describeOutput: nil, + describeError: errors.New("API error"), + expectedSGIDs: nil, + expectError: true, + errorContains: "unable to describe security groups", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + imds: TypedIMDS{mockMetadata}, + vpcID: vpcID, + } + + mockEC2.EXPECT().DescribeSecurityGroups( + gomock.Any(), + gomock.Any(), + gomock.Any(), + ).Return(tt.describeOutput, tt.describeError) + + sgIDs, err := cache.discoverCustomSecurityGroups(context.Background()) + + if tt.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errorContains) + } else { + assert.NoError(t, err) + if len(tt.expectedSGIDs) > 0 { + assert.ElementsMatch(t, tt.expectedSGIDs, sgIDs) + } else { + assert.Empty(t, sgIDs) + } + } + }) + } +} + +// TestGetENISubnetID tests the GetENISubnetID helper method +func TestGetENISubnetID(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + mockMetadata := testMetadata(nil) + + tests := []struct { + name string + eniID string + describeOutput *ec2.DescribeNetworkInterfacesOutput + describeError error + expectedSubnetID string + expectError bool + errorContains string + }{ + { + name: "successful subnet lookup", + eniID: "eni-12345678", + describeOutput: &ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []ec2types.NetworkInterface{ + { + NetworkInterfaceId: aws.String("eni-12345678"), + SubnetId: aws.String("subnet-secondary"), + }, + }, + }, + describeError: nil, + expectedSubnetID: "subnet-secondary", + expectError: false, + }, + { + name: "no matching ENI found", + eniID: "eni-nonexistent", + describeOutput: &ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []ec2types.NetworkInterface{}, + }, + describeError: nil, + expectError: true, + errorContains: "no interfaces found", + }, + { + name: "describe API error", + eniID: "eni-12345678", + describeOutput: nil, + describeError: errors.New("API error"), + expectError: true, + errorContains: "unable to describe network interface", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + imds: TypedIMDS{mockMetadata}, + } + + mockEC2.EXPECT().DescribeNetworkInterfaces( + gomock.Any(), + gomock.Any(), + gomock.Any(), + ).Return(tt.describeOutput, tt.describeError) + + subnetID, err := cache.GetENISubnetID(context.Background(), tt.eniID) + + if tt.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errorContains) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedSubnetID, subnetID) + } + }) + } +} + +// TestCreateENIWithCustomSGs tests the custom SG application in createENI +func TestCreateENIWithCustomSGs(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + mockMetadata := testMetadata(nil) + + tests := []struct { + name string + isPrimarySubnet bool + customSGs []string + expectedGroups []string + subnets []ec2types.Subnet + useSubnetDiscovery bool + }{ + { + name: "primary subnet uses primary SGs", + isPrimarySubnet: true, + customSGs: []string{"sg-custom1", "sg-custom2"}, + expectedGroups: []string{sg1, sg2}, // primary ENI security groups + subnets: []ec2types.Subnet{ + { + SubnetId: aws.String(subnetID), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + }, + useSubnetDiscovery: true, + }, + { + name: "secondary subnet with custom SGs", + isPrimarySubnet: false, + customSGs: []string{"sg-custom1", "sg-custom2"}, + expectedGroups: []string{"sg-custom1", "sg-custom2"}, // custom security groups + subnets: []ec2types.Subnet{ + { + SubnetId: aws.String("subnet-secondary"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + }, + useSubnetDiscovery: true, + }, + { + name: "secondary subnet without custom SGs", + isPrimarySubnet: false, + customSGs: []string{}, + expectedGroups: []string{sg1, sg2}, // falls back to primary ENI security groups + subnets: []ec2types.Subnet{ + { + SubnetId: aws.String("subnet-secondary"), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + }, + useSubnetDiscovery: true, + }, + } + + // Define the initial security group IDs + initialSGIDs := []string{sg1, sg2} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + imds: TypedIMDS{mockMetadata}, + useSubnetDiscovery: tt.useSubnetDiscovery, + securityGroups: StringSet{}, // Create a new StringSet to avoid copying mutex + subnetID: subnetID, + } + + // Initialize security groups and custom SG cache + cache.securityGroups.Set(initialSGIDs) + cache.customSecurityGroups.Set(tt.customSGs) + + // Mock the subnet discovery + subnetResult := &ec2.DescribeSubnetsOutput{Subnets: tt.subnets} + mockEC2.EXPECT().DescribeSubnets(gomock.Any(), gomock.Any(), gomock.Any()).Return(subnetResult, nil) + + // Mock free device number detection + ec2ENIs := make([]ec2types.InstanceNetworkInterface, 0) + deviceNum1 := int32(0) + ec2ENI := ec2types.InstanceNetworkInterface{Attachment: &ec2types.InstanceNetworkInterfaceAttachment{DeviceIndex: &deviceNum1}} + ec2ENIs = append(ec2ENIs, ec2ENI) + result := &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}, + } + mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) + + // Mock the CreateNetworkInterface call and capture the input + var capturedInput *ec2.CreateNetworkInterfaceInput + cureniID := eniID + eni := ec2.CreateNetworkInterfaceOutput{NetworkInterface: &ec2types.NetworkInterface{NetworkInterfaceId: &cureniID}} + mockEC2.EXPECT().CreateNetworkInterface( + gomock.Any(), + gomock.Any(), + gomock.Any(), + ).DoAndReturn(func(_ context.Context, input *ec2.CreateNetworkInterfaceInput, _ ...func(*ec2.Options)) (*ec2.CreateNetworkInterfaceOutput, error) { + capturedInput = input + return &eni, nil + }) + + // Mock AttachNetworkInterface + attachmentID := "eni-attach-123" + attachResult := &ec2.AttachNetworkInterfaceOutput{AttachmentId: &attachmentID} + mockEC2.EXPECT().AttachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(attachResult, nil) + mockEC2.EXPECT().ModifyNetworkInterfaceAttribute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + + // Call the function under test + createdENI, err := cache.AllocENI(context.Background(), nil, "", 5, 0) + + // Verify results + assert.NoError(t, err) + assert.NotNil(t, createdENI) + + // Check that the correct security groups were used + assert.NotNil(t, capturedInput) + assert.NotNil(t, capturedInput.Groups) + + // Convert []string to set for easier comparison + expectedGroupSet := StringSet{} + expectedGroupSet.Set(tt.expectedGroups) + + // Convert the actual groups to set + actualGroupSet := StringSet{} + actualGroupSet.Set(capturedInput.Groups) + + // Compare sets (order-independent) + assert.Equal(t, expectedGroupSet.SortedList(), actualGroupSet.SortedList()) + }) + } +} + +// TestRefreshCustomSGIDsWithFallback tests fallback to primary SGs when custom SG discovery fails +func TestRefreshCustomSGIDsWithFallback(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + mockMetadata := testMetadata(nil) + + // Mock primary security groups + primarySGs := []string{sg1, sg2} + + cache := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + imds: TypedIMDS{mockMetadata}, + securityGroups: StringSet{}, + customSecurityGroups: StringSet{}, + subnetID: subnetID, // primary subnet + primaryENI: primaryeniID, + unmanagedENIs: StringSet{}, + useSubnetDiscovery: true, // This function should only be called when subnet discovery is enabled + } + + // Initialize primary security groups + cache.securityGroups.Set(primarySGs) + // Set some custom SGs initially to verify they get cleared + cache.customSecurityGroups.Set([]string{"sg-custom1", "sg-custom2"}) + + tests := []struct { + name string + describeError error + expectedCustomSGs []string + }{ + { + name: "discovery fails - should fallback and clear custom SGs", + describeError: errors.New("AccessDenied: insufficient permissions"), + expectedCustomSGs: []string{}, // empty after fallback + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Mock the failed DescribeSecurityGroups call + mockEC2.EXPECT().DescribeSecurityGroups( + gomock.Any(), + gomock.Any(), + gomock.Any(), + ).Return(nil, tt.describeError) + + // Create a simple datastore - for this test, we just care that + // the function handles the error gracefully and clears the cache + mockDataStore := &datastore.DataStore{} + mockDataStoreAccess := &datastore.DataStoreAccess{ + DataStores: []*datastore.DataStore{mockDataStore}, + } + + // Call RefreshCustomSGIDs + err := cache.RefreshCustomSGIDs(context.Background(), mockDataStoreAccess) + + // Should return (after doing a graceful fallback) + assert.Error(t, err) + + // Custom SGs should be cleared for fallback + assert.Equal(t, tt.expectedCustomSGs, cache.customSecurityGroups.SortedList()) + }) + } +} + +// TestENICreationFallbackLogging tests that ENI creation logs fallback behavior correctly +func TestENICreationFallbackLogging(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + mockMetadata := testMetadata(nil) + + primarySGs := []string{sg1, sg2} + secondarySubnetID := "subnet-secondary" + + tests := []struct { + name string + customSGs []string + targetSubnet string + isPrimarySubnet bool + expectedSGsUsed []string + expectFallbackLog bool + }{ + { + name: "secondary subnet with custom SGs", + customSGs: []string{"sg-custom1", "sg-custom2"}, + targetSubnet: secondarySubnetID, + isPrimarySubnet: false, + expectedSGsUsed: []string{"sg-custom1", "sg-custom2"}, + expectFallbackLog: false, + }, + { + name: "secondary subnet without custom SGs - fallback", + customSGs: []string{}, // no custom SGs available + targetSubnet: secondarySubnetID, + isPrimarySubnet: false, + expectedSGsUsed: primarySGs, // should fallback to primary SGs + expectFallbackLog: true, + }, + { + name: "primary subnet always uses primary SGs", + customSGs: []string{"sg-custom1", "sg-custom2"}, + targetSubnet: subnetID, + isPrimarySubnet: true, + expectedSGsUsed: primarySGs, + expectFallbackLog: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + imds: TypedIMDS{mockMetadata}, + useSubnetDiscovery: true, + securityGroups: StringSet{}, + customSecurityGroups: StringSet{}, + subnetID: subnetID, + } + + // Initialize security groups + cache.securityGroups.Set(primarySGs) + cache.customSecurityGroups.Set(tt.customSGs) + + // Mock subnet discovery + subnets := []ec2types.Subnet{ + { + SubnetId: aws.String(tt.targetSubnet), + AvailableIpAddressCount: aws.Int32(100), + Tags: []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + }, + } + subnetResult := &ec2.DescribeSubnetsOutput{Subnets: subnets} + mockEC2.EXPECT().DescribeSubnets(gomock.Any(), gomock.Any(), gomock.Any()).Return(subnetResult, nil) + + // Mock free device number detection + ec2ENIs := make([]ec2types.InstanceNetworkInterface, 0) + deviceNum1 := int32(0) + ec2ENI := ec2types.InstanceNetworkInterface{Attachment: &ec2types.InstanceNetworkInterfaceAttachment{DeviceIndex: &deviceNum1}} + ec2ENIs = append(ec2ENIs, ec2ENI) + result := &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{{Instances: []ec2types.Instance{{NetworkInterfaces: ec2ENIs}}}}, + } + mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) + + // Mock the CreateNetworkInterface call and capture the input + var capturedInput *ec2.CreateNetworkInterfaceInput + cureniID := eniID + eni := ec2.CreateNetworkInterfaceOutput{NetworkInterface: &ec2types.NetworkInterface{NetworkInterfaceId: &cureniID}} + mockEC2.EXPECT().CreateNetworkInterface( + gomock.Any(), + gomock.Any(), + gomock.Any(), + ).DoAndReturn(func(_ context.Context, input *ec2.CreateNetworkInterfaceInput, _ ...func(*ec2.Options)) (*ec2.CreateNetworkInterfaceOutput, error) { + capturedInput = input + return &eni, nil + }) + + // Mock AttachNetworkInterface + attachmentID := "eni-attach-123" + attachResult := &ec2.AttachNetworkInterfaceOutput{AttachmentId: &attachmentID} + mockEC2.EXPECT().AttachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(attachResult, nil) + mockEC2.EXPECT().ModifyNetworkInterfaceAttribute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + + // Call AllocENI + createdENI, err := cache.AllocENI(context.Background(), nil, "", 5, 0) + + // Verify results + assert.NoError(t, err) + assert.NotNil(t, createdENI) + assert.NotNil(t, capturedInput) + assert.NotNil(t, capturedInput.Groups) + + // Verify correct security groups were used + actualSGs := capturedInput.Groups + assert.ElementsMatch(t, tt.expectedSGsUsed, actualSGs) + }) + } +} + +func TestDeallocPrefixAddresses(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + // Test deallocating IPv4 prefixes only + ipv4Prefixes := []string{"10.0.0.0/28", "10.0.1.0/28"} + inputIPv4 := &ec2.UnassignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv4Prefixes: ipv4Prefixes, + } + mockEC2.EXPECT().UnassignPrivateIpAddresses(gomock.Any(), inputIPv4, gomock.Any()).Return(nil, nil) + + cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", enablePrefixDelegation: true} + err := cache.DeallocPrefixAddresses(context.Background(), eniID, ipv4Prefixes) + assert.NoError(t, err) +} + +func TestDeallocPrefixAddressesIPv6(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + // Test deallocating IPv6 prefixes only + ipv6Prefixes := []string{"2001:db8:1234:5678::/80", "2001:db8:1234:5679::/80"} + inputIPv6 := &ec2.UnassignIpv6AddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv6Prefixes: ipv6Prefixes, + } + mockEC2.EXPECT().UnassignIpv6Addresses(gomock.Any(), inputIPv6, gomock.Any()).Return(nil, nil) + + cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", enablePrefixDelegation: true} + err := cache.DeallocPrefixAddresses(context.Background(), eniID, ipv6Prefixes) + assert.NoError(t, err) +} + +func TestDeallocPrefixAddressesMixed(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + // Test deallocating both IPv4 and IPv6 prefixes + mixedPrefixes := []string{"10.0.0.0/28", "2001:db8:1234:5678::/80", "10.0.1.0/28", "2001:db8:1234:5679::/80"} + + // Expect IPv4 call + inputIPv4 := &ec2.UnassignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv4Prefixes: []string{"10.0.0.0/28", "10.0.1.0/28"}, + } + mockEC2.EXPECT().UnassignPrivateIpAddresses(gomock.Any(), inputIPv4, gomock.Any()).Return(nil, nil) + + // Expect IPv6 call + inputIPv6 := &ec2.UnassignIpv6AddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv6Prefixes: []string{"2001:db8:1234:5678::/80", "2001:db8:1234:5679::/80"}, + } + mockEC2.EXPECT().UnassignIpv6Addresses(gomock.Any(), inputIPv6, gomock.Any()).Return(nil, nil) + + cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", enablePrefixDelegation: true} + err := cache.DeallocPrefixAddresses(context.Background(), eniID, mixedPrefixes) + assert.NoError(t, err) +} + +func TestDeallocPrefixAddressesEmpty(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + // Test with empty prefix list - should do nothing and not call AWS + cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", enablePrefixDelegation: true} + err := cache.DeallocPrefixAddresses(context.Background(), eniID, []string{}) + assert.NoError(t, err) +} + +func TestDeallocPrefixAddressesIPv4Error(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + // Test IPv4 deallocation error + ipv4Prefixes := []string{"10.0.0.0/28"} + inputIPv4 := &ec2.UnassignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv4Prefixes: ipv4Prefixes, + } + + retErr := &smithy.GenericAPIError{Code: "InvalidNetworkInterfaceID.NotFound", Message: "Network interface not found"} + mockEC2.EXPECT().UnassignPrivateIpAddresses(gomock.Any(), inputIPv4, gomock.Any()).Return(nil, retErr) + + cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", enablePrefixDelegation: true} + err := cache.DeallocPrefixAddresses(context.Background(), eniID, ipv4Prefixes) + assert.Error(t, err) + assert.Contains(t, err.Error(), "deallocate IPv4 prefix") +} + +func TestDeallocPrefixAddressesIPv6Error(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + // Test IPv6 deallocation error + ipv6Prefixes := []string{"2001:db8:1234:5678::/80"} + inputIPv6 := &ec2.UnassignIpv6AddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv6Prefixes: ipv6Prefixes, + } + + retErr := &smithy.GenericAPIError{Code: "InvalidNetworkInterfaceID.NotFound", Message: "Network interface not found"} + mockEC2.EXPECT().UnassignIpv6Addresses(gomock.Any(), inputIPv6, gomock.Any()).Return(nil, retErr) + + cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", enablePrefixDelegation: true} + err := cache.DeallocPrefixAddresses(context.Background(), eniID, ipv6Prefixes) + assert.Error(t, err) + assert.Contains(t, err.Error(), "deallocate IPv6 prefix") +} + +func TestDeallocPrefixAddressesInvalidCIDR(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + // Test with invalid CIDR - should skip the invalid one and process valid ones + mixedPrefixes := []string{"10.0.0.0/28", "invalid-cidr", "2001:db8:1234:5678::/80"} + + // Only expect valid prefixes to be processed + inputIPv4 := &ec2.UnassignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv4Prefixes: []string{"10.0.0.0/28"}, + } + mockEC2.EXPECT().UnassignPrivateIpAddresses(gomock.Any(), inputIPv4, gomock.Any()).Return(nil, nil) + + inputIPv6 := &ec2.UnassignIpv6AddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv6Prefixes: []string{"2001:db8:1234:5678::/80"}, + } + mockEC2.EXPECT().UnassignIpv6Addresses(gomock.Any(), inputIPv6, gomock.Any()).Return(nil, nil) + + cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", enablePrefixDelegation: true} + err := cache.DeallocPrefixAddresses(context.Background(), eniID, mixedPrefixes) + assert.NoError(t, err) +} diff --git a/pkg/awsutils/mocks/awsutils_mocks.go b/pkg/awsutils/mocks/awsutils_mocks.go index 6a148b6f23..38039ff29d 100644 --- a/pkg/awsutils/mocks/awsutils_mocks.go +++ b/pkg/awsutils/mocks/awsutils_mocks.go @@ -1,10 +1,25 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. +// + // Code generated by MockGen. DO NOT EDIT. -// Source: awsutils.go +// Source: github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils (interfaces: APIs) -// Package mocks is a generated GoMock package. -package mocks +// Package mock_awsutils is a generated GoMock package. +package mock_awsutils import ( + context "context" net "net" reflect "reflect" @@ -40,133 +55,133 @@ func (m *MockAPIs) EXPECT() *MockAPIsMockRecorder { } // AllocENI mocks base method. -func (m *MockAPIs) AllocENI(sg []*string, eniCfgSubnet string, numIPs, networkCard int) (string, error) { +func (m *MockAPIs) AllocENI(arg0 context.Context, arg1 []*string, arg2 string, arg3, arg4 int) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AllocENI", sg, eniCfgSubnet, numIPs, networkCard) + ret := m.ctrl.Call(m, "AllocENI", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // AllocENI indicates an expected call of AllocENI. -func (mr *MockAPIsMockRecorder) AllocENI(sg, eniCfgSubnet, numIPs, networkCard interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) AllocENI(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocENI", reflect.TypeOf((*MockAPIs)(nil).AllocENI), sg, eniCfgSubnet, numIPs, networkCard) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocENI", reflect.TypeOf((*MockAPIs)(nil).AllocENI), arg0, arg1, arg2, arg3, arg4) } // AllocIPAddress mocks base method. -func (m *MockAPIs) AllocIPAddress(eniID string) error { +func (m *MockAPIs) AllocIPAddress(arg0 context.Context, arg1 string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AllocIPAddress", eniID) + ret := m.ctrl.Call(m, "AllocIPAddress", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // AllocIPAddress indicates an expected call of AllocIPAddress. -func (mr *MockAPIsMockRecorder) AllocIPAddress(eniID interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) AllocIPAddress(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPAddress", reflect.TypeOf((*MockAPIs)(nil).AllocIPAddress), eniID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPAddress", reflect.TypeOf((*MockAPIs)(nil).AllocIPAddress), arg0, arg1) } // AllocIPAddresses mocks base method. -func (m *MockAPIs) AllocIPAddresses(eniID string, numIPs int) (*ec2.AssignPrivateIpAddressesOutput, error) { +func (m *MockAPIs) AllocIPAddresses(arg0 context.Context, arg1 string, arg2 int) (*ec2.AssignPrivateIpAddressesOutput, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AllocIPAddresses", eniID, numIPs) + ret := m.ctrl.Call(m, "AllocIPAddresses", arg0, arg1, arg2) ret0, _ := ret[0].(*ec2.AssignPrivateIpAddressesOutput) ret1, _ := ret[1].(error) return ret0, ret1 } // AllocIPAddresses indicates an expected call of AllocIPAddresses. -func (mr *MockAPIsMockRecorder) AllocIPAddresses(eniID, numIPs interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) AllocIPAddresses(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPAddresses", reflect.TypeOf((*MockAPIs)(nil).AllocIPAddresses), eniID, numIPs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPAddresses", reflect.TypeOf((*MockAPIs)(nil).AllocIPAddresses), arg0, arg1, arg2) } // AllocIPv6Prefixes mocks base method. -func (m *MockAPIs) AllocIPv6Prefixes(eniID string) ([]*string, error) { +func (m *MockAPIs) AllocIPv6Prefixes(arg0 context.Context, arg1 string) ([]*string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AllocIPv6Prefixes", eniID) + ret := m.ctrl.Call(m, "AllocIPv6Prefixes", arg0, arg1) ret0, _ := ret[0].([]*string) ret1, _ := ret[1].(error) return ret0, ret1 } // AllocIPv6Prefixes indicates an expected call of AllocIPv6Prefixes. -func (mr *MockAPIsMockRecorder) AllocIPv6Prefixes(eniID interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) AllocIPv6Prefixes(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPv6Prefixes", reflect.TypeOf((*MockAPIs)(nil).AllocIPv6Prefixes), eniID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPv6Prefixes", reflect.TypeOf((*MockAPIs)(nil).AllocIPv6Prefixes), arg0, arg1) } // DeallocIPAddresses mocks base method. -func (m *MockAPIs) DeallocIPAddresses(eniID string, ips []string) error { +func (m *MockAPIs) DeallocIPAddresses(arg0 context.Context, arg1 string, arg2 []string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeallocIPAddresses", eniID, ips) + ret := m.ctrl.Call(m, "DeallocIPAddresses", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // DeallocIPAddresses indicates an expected call of DeallocIPAddresses. -func (mr *MockAPIsMockRecorder) DeallocIPAddresses(eniID, ips interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) DeallocIPAddresses(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocIPAddresses", reflect.TypeOf((*MockAPIs)(nil).DeallocIPAddresses), eniID, ips) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocIPAddresses", reflect.TypeOf((*MockAPIs)(nil).DeallocIPAddresses), arg0, arg1, arg2) } // DeallocPrefixAddresses mocks base method. -func (m *MockAPIs) DeallocPrefixAddresses(eniID string, ips []string) error { +func (m *MockAPIs) DeallocPrefixAddresses(arg0 context.Context, arg1 string, arg2 []string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeallocPrefixAddresses", eniID, ips) + ret := m.ctrl.Call(m, "DeallocPrefixAddresses", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // DeallocPrefixAddresses indicates an expected call of DeallocPrefixAddresses. -func (mr *MockAPIsMockRecorder) DeallocPrefixAddresses(eniID, ips interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) DeallocPrefixAddresses(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocPrefixAddresses", reflect.TypeOf((*MockAPIs)(nil).DeallocPrefixAddresses), eniID, ips) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocPrefixAddresses", reflect.TypeOf((*MockAPIs)(nil).DeallocPrefixAddresses), arg0, arg1, arg2) } // DescribeAllENIs mocks base method. -func (m *MockAPIs) DescribeAllENIs() (awsutils.DescribeAllENIsResult, error) { +func (m *MockAPIs) DescribeAllENIs(arg0 context.Context) (awsutils.DescribeAllENIsResult, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescribeAllENIs") + ret := m.ctrl.Call(m, "DescribeAllENIs", arg0) ret0, _ := ret[0].(awsutils.DescribeAllENIsResult) ret1, _ := ret[1].(error) return ret0, ret1 } // DescribeAllENIs indicates an expected call of DescribeAllENIs. -func (mr *MockAPIsMockRecorder) DescribeAllENIs() *gomock.Call { +func (mr *MockAPIsMockRecorder) DescribeAllENIs(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeAllENIs", reflect.TypeOf((*MockAPIs)(nil).DescribeAllENIs)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeAllENIs", reflect.TypeOf((*MockAPIs)(nil).DescribeAllENIs), arg0) } // FetchInstanceTypeLimits mocks base method. -func (m *MockAPIs) FetchInstanceTypeLimits() error { +func (m *MockAPIs) FetchInstanceTypeLimits(arg0 context.Context) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FetchInstanceTypeLimits") + ret := m.ctrl.Call(m, "FetchInstanceTypeLimits", arg0) ret0, _ := ret[0].(error) return ret0 } // FetchInstanceTypeLimits indicates an expected call of FetchInstanceTypeLimits. -func (mr *MockAPIsMockRecorder) FetchInstanceTypeLimits() *gomock.Call { +func (mr *MockAPIsMockRecorder) FetchInstanceTypeLimits(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchInstanceTypeLimits", reflect.TypeOf((*MockAPIs)(nil).FetchInstanceTypeLimits)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchInstanceTypeLimits", reflect.TypeOf((*MockAPIs)(nil).FetchInstanceTypeLimits), arg0) } // FreeENI mocks base method. -func (m *MockAPIs) FreeENI(eniName string) error { +func (m *MockAPIs) FreeENI(arg0 context.Context, arg1 string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FreeENI", eniName) + ret := m.ctrl.Call(m, "FreeENI", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // FreeENI indicates an expected call of FreeENI. -func (mr *MockAPIsMockRecorder) FreeENI(eniName interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) FreeENI(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FreeENI", reflect.TypeOf((*MockAPIs)(nil).FreeENI), eniName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FreeENI", reflect.TypeOf((*MockAPIs)(nil).FreeENI), arg0, arg1) } // GetAttachedENIs mocks base method. @@ -212,49 +227,64 @@ func (mr *MockAPIsMockRecorder) GetENILimit() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetENILimit", reflect.TypeOf((*MockAPIs)(nil).GetENILimit)) } +// GetENISubnetID mocks base method. +func (m *MockAPIs) GetENISubnetID(arg0 context.Context, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetENISubnetID", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetENISubnetID indicates an expected call of GetENISubnetID. +func (mr *MockAPIsMockRecorder) GetENISubnetID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetENISubnetID", reflect.TypeOf((*MockAPIs)(nil).GetENISubnetID), arg0, arg1) +} + // GetIPv4PrefixesFromEC2 mocks base method. -func (m *MockAPIs) GetIPv4PrefixesFromEC2(eniID string) ([]types.Ipv4PrefixSpecification, error) { +func (m *MockAPIs) GetIPv4PrefixesFromEC2(arg0 context.Context, arg1 string) ([]types.Ipv4PrefixSpecification, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetIPv4PrefixesFromEC2", eniID) + ret := m.ctrl.Call(m, "GetIPv4PrefixesFromEC2", arg0, arg1) ret0, _ := ret[0].([]types.Ipv4PrefixSpecification) ret1, _ := ret[1].(error) return ret0, ret1 } // GetIPv4PrefixesFromEC2 indicates an expected call of GetIPv4PrefixesFromEC2. -func (mr *MockAPIsMockRecorder) GetIPv4PrefixesFromEC2(eniID interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) GetIPv4PrefixesFromEC2(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv4PrefixesFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv4PrefixesFromEC2), eniID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv4PrefixesFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv4PrefixesFromEC2), arg0, arg1) } // GetIPv4sFromEC2 mocks base method. -func (m *MockAPIs) GetIPv4sFromEC2(eniID string) ([]types.NetworkInterfacePrivateIpAddress, error) { +func (m *MockAPIs) GetIPv4sFromEC2(arg0 context.Context, arg1 string) ([]types.NetworkInterfacePrivateIpAddress, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetIPv4sFromEC2", eniID) + ret := m.ctrl.Call(m, "GetIPv4sFromEC2", arg0, arg1) ret0, _ := ret[0].([]types.NetworkInterfacePrivateIpAddress) ret1, _ := ret[1].(error) return ret0, ret1 } // GetIPv4sFromEC2 indicates an expected call of GetIPv4sFromEC2. -func (mr *MockAPIsMockRecorder) GetIPv4sFromEC2(eniID interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) GetIPv4sFromEC2(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv4sFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv4sFromEC2), eniID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv4sFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv4sFromEC2), arg0, arg1) } // GetIPv6PrefixesFromEC2 mocks base method. -func (m *MockAPIs) GetIPv6PrefixesFromEC2(eniID string) ([]types.Ipv6PrefixSpecification, error) { +func (m *MockAPIs) GetIPv6PrefixesFromEC2(arg0 context.Context, arg1 string) ([]types.Ipv6PrefixSpecification, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetIPv6PrefixesFromEC2", eniID) + ret := m.ctrl.Call(m, "GetIPv6PrefixesFromEC2", arg0, arg1) ret0, _ := ret[0].([]types.Ipv6PrefixSpecification) ret1, _ := ret[1].(error) return ret0, ret1 } // GetIPv6PrefixesFromEC2 indicates an expected call of GetIPv6PrefixesFromEC2. -func (mr *MockAPIsMockRecorder) GetIPv6PrefixesFromEC2(eniID interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) GetIPv6PrefixesFromEC2(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv6PrefixesFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv6PrefixesFromEC2), eniID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv6PrefixesFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv6PrefixesFromEC2), arg0, arg1) } // GetInstanceHypervisorFamily mocks base method. @@ -399,6 +429,21 @@ func (mr *MockAPIsMockRecorder) GetVPCIPv6CIDRs() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVPCIPv6CIDRs", reflect.TypeOf((*MockAPIs)(nil).GetVPCIPv6CIDRs)) } +// GetVpcSubnets mocks base method. +func (m *MockAPIs) GetVpcSubnets(arg0 context.Context) ([]types.Subnet, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVpcSubnets", arg0) + ret0, _ := ret[0].([]types.Subnet) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVpcSubnets indicates an expected call of GetVpcSubnets. +func (mr *MockAPIsMockRecorder) GetVpcSubnets(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVpcSubnets", reflect.TypeOf((*MockAPIs)(nil).GetVpcSubnets), arg0) +} + // InitCachedPrefixDelegation mocks base method. func (m *MockAPIs) InitCachedPrefixDelegation(arg0 bool) { m.ctrl.T.Helper() @@ -412,17 +457,17 @@ func (mr *MockAPIsMockRecorder) InitCachedPrefixDelegation(arg0 interface{}) *go } // IsEfaOnlyENI mocks base method. -func (m *MockAPIs) IsEfaOnlyENI(networkCard int, eni string) bool { +func (m *MockAPIs) IsEfaOnlyENI(arg0 int, arg1 string) bool { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsEfaOnlyENI", networkCard, eni) + ret := m.ctrl.Call(m, "IsEfaOnlyENI", arg0, arg1) ret0, _ := ret[0].(bool) return ret0 } // IsEfaOnlyENI indicates an expected call of IsEfaOnlyENI. -func (mr *MockAPIsMockRecorder) IsEfaOnlyENI(networkCard, eni interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) IsEfaOnlyENI(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsEfaOnlyENI", reflect.TypeOf((*MockAPIs)(nil).IsEfaOnlyENI), networkCard, eni) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsEfaOnlyENI", reflect.TypeOf((*MockAPIs)(nil).IsEfaOnlyENI), arg0, arg1) } // IsPrefixDelegationSupported mocks base method. @@ -440,122 +485,151 @@ func (mr *MockAPIsMockRecorder) IsPrefixDelegationSupported() *gomock.Call { } // IsPrimaryENI mocks base method. -func (m *MockAPIs) IsPrimaryENI(eniID string) bool { +func (m *MockAPIs) IsPrimaryENI(arg0 string) bool { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsPrimaryENI", eniID) + ret := m.ctrl.Call(m, "IsPrimaryENI", arg0) ret0, _ := ret[0].(bool) return ret0 } // IsPrimaryENI indicates an expected call of IsPrimaryENI. -func (mr *MockAPIsMockRecorder) IsPrimaryENI(eniID interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) IsPrimaryENI(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPrimaryENI", reflect.TypeOf((*MockAPIs)(nil).IsPrimaryENI), eniID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPrimaryENI", reflect.TypeOf((*MockAPIs)(nil).IsPrimaryENI), arg0) +} + +// IsSubnetExcluded mocks base method. +func (m *MockAPIs) IsSubnetExcluded(arg0 context.Context, arg1 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsSubnetExcluded", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsSubnetExcluded indicates an expected call of IsSubnetExcluded. +func (mr *MockAPIsMockRecorder) IsSubnetExcluded(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSubnetExcluded", reflect.TypeOf((*MockAPIs)(nil).IsSubnetExcluded), arg0, arg1) } // IsUnmanagedENI mocks base method. -func (m *MockAPIs) IsUnmanagedENI(eniID string) bool { +func (m *MockAPIs) IsUnmanagedENI(arg0 string) bool { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsUnmanagedENI", eniID) + ret := m.ctrl.Call(m, "IsUnmanagedENI", arg0) ret0, _ := ret[0].(bool) return ret0 } // IsUnmanagedENI indicates an expected call of IsUnmanagedENI. -func (mr *MockAPIsMockRecorder) IsUnmanagedENI(eniID interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) IsUnmanagedENI(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUnmanagedENI", reflect.TypeOf((*MockAPIs)(nil).IsUnmanagedENI), eniID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUnmanagedENI", reflect.TypeOf((*MockAPIs)(nil).IsUnmanagedENI), arg0) } // IsUnmanagedNIC mocks base method. -func (m *MockAPIs) IsUnmanagedNIC(networkCard int) bool { +func (m *MockAPIs) IsUnmanagedNIC(arg0 int) bool { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsUnmanagedNIC", networkCard) + ret := m.ctrl.Call(m, "IsUnmanagedNIC", arg0) ret0, _ := ret[0].(bool) return ret0 } // IsUnmanagedNIC indicates an expected call of IsUnmanagedNIC. -func (mr *MockAPIsMockRecorder) IsUnmanagedNIC(networkCard interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) IsUnmanagedNIC(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUnmanagedNIC", reflect.TypeOf((*MockAPIs)(nil).IsUnmanagedNIC), arg0) +} + +// RefreshCustomSGIDs mocks base method. +func (m *MockAPIs) RefreshCustomSGIDs(arg0 context.Context, arg1 *datastore.DataStoreAccess) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RefreshCustomSGIDs", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// RefreshCustomSGIDs indicates an expected call of RefreshCustomSGIDs. +func (mr *MockAPIsMockRecorder) RefreshCustomSGIDs(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUnmanagedNIC", reflect.TypeOf((*MockAPIs)(nil).IsUnmanagedNIC), networkCard) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshCustomSGIDs", reflect.TypeOf((*MockAPIs)(nil).RefreshCustomSGIDs), arg0, arg1) } // RefreshSGIDs mocks base method. -func (m *MockAPIs) RefreshSGIDs(mac string, ds *datastore.DataStoreAccess) error { +func (m *MockAPIs) RefreshSGIDs(arg0 context.Context, arg1 string, arg2 *datastore.DataStoreAccess) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshSGIDs", mac, ds) + ret := m.ctrl.Call(m, "RefreshSGIDs", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // RefreshSGIDs indicates an expected call of RefreshSGIDs. -func (mr *MockAPIsMockRecorder) RefreshSGIDs(mac, ds interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) RefreshSGIDs(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshSGIDs", reflect.TypeOf((*MockAPIs)(nil).RefreshSGIDs), mac, ds) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshSGIDs", reflect.TypeOf((*MockAPIs)(nil).RefreshSGIDs), arg0, arg1, arg2) } // SetEFAOnlyENIs mocks base method. -func (m *MockAPIs) SetEFAOnlyENIs(efaOnlyENIByNetworkCard []string) { +func (m *MockAPIs) SetEFAOnlyENIs(arg0 []string) { m.ctrl.T.Helper() - m.ctrl.Call(m, "SetEFAOnlyENIs", efaOnlyENIByNetworkCard) + m.ctrl.Call(m, "SetEFAOnlyENIs", arg0) } // SetEFAOnlyENIs indicates an expected call of SetEFAOnlyENIs. -func (mr *MockAPIsMockRecorder) SetEFAOnlyENIs(efaOnlyENIByNetworkCard interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) SetEFAOnlyENIs(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetEFAOnlyENIs", reflect.TypeOf((*MockAPIs)(nil).SetEFAOnlyENIs), efaOnlyENIByNetworkCard) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetEFAOnlyENIs", reflect.TypeOf((*MockAPIs)(nil).SetEFAOnlyENIs), arg0) } // SetUnmanagedENIs mocks base method. -func (m *MockAPIs) SetUnmanagedENIs(eniIDs []string) { +func (m *MockAPIs) SetUnmanagedENIs(arg0 []string) { m.ctrl.T.Helper() - m.ctrl.Call(m, "SetUnmanagedENIs", eniIDs) + m.ctrl.Call(m, "SetUnmanagedENIs", arg0) } // SetUnmanagedENIs indicates an expected call of SetUnmanagedENIs. -func (mr *MockAPIsMockRecorder) SetUnmanagedENIs(eniIDs interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) SetUnmanagedENIs(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUnmanagedENIs", reflect.TypeOf((*MockAPIs)(nil).SetUnmanagedENIs), eniIDs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUnmanagedENIs", reflect.TypeOf((*MockAPIs)(nil).SetUnmanagedENIs), arg0) } // SetUnmanagedNetworkCards mocks base method. -func (m *MockAPIs) SetUnmanagedNetworkCards(skipNetworkCards []bool) { +func (m *MockAPIs) SetUnmanagedNetworkCards(arg0 []bool) { m.ctrl.T.Helper() - m.ctrl.Call(m, "SetUnmanagedNetworkCards", skipNetworkCards) + m.ctrl.Call(m, "SetUnmanagedNetworkCards", arg0) } // SetUnmanagedNetworkCards indicates an expected call of SetUnmanagedNetworkCards. -func (mr *MockAPIsMockRecorder) SetUnmanagedNetworkCards(skipNetworkCards interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) SetUnmanagedNetworkCards(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUnmanagedNetworkCards", reflect.TypeOf((*MockAPIs)(nil).SetUnmanagedNetworkCards), skipNetworkCards) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUnmanagedNetworkCards", reflect.TypeOf((*MockAPIs)(nil).SetUnmanagedNetworkCards), arg0) } // TagENI mocks base method. -func (m *MockAPIs) TagENI(eniID string, currentTags map[string]string) error { +func (m *MockAPIs) TagENI(arg0 context.Context, arg1 string, arg2 map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TagENI", eniID, currentTags) + ret := m.ctrl.Call(m, "TagENI", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // TagENI indicates an expected call of TagENI. -func (mr *MockAPIsMockRecorder) TagENI(eniID, currentTags interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) TagENI(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TagENI", reflect.TypeOf((*MockAPIs)(nil).TagENI), eniID, currentTags) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TagENI", reflect.TypeOf((*MockAPIs)(nil).TagENI), arg0, arg1, arg2) } // WaitForENIAndIPsAttached mocks base method. -func (m *MockAPIs) WaitForENIAndIPsAttached(eni string, wantedSecondaryIPs int) (awsutils.ENIMetadata, error) { +func (m *MockAPIs) WaitForENIAndIPsAttached(arg0 string, arg1 int) (awsutils.ENIMetadata, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "WaitForENIAndIPsAttached", eni, wantedSecondaryIPs) + ret := m.ctrl.Call(m, "WaitForENIAndIPsAttached", arg0, arg1) ret0, _ := ret[0].(awsutils.ENIMetadata) ret1, _ := ret[1].(error) return ret0, ret1 } // WaitForENIAndIPsAttached indicates an expected call of WaitForENIAndIPsAttached. -func (mr *MockAPIsMockRecorder) WaitForENIAndIPsAttached(eni, wantedSecondaryIPs interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) WaitForENIAndIPsAttached(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForENIAndIPsAttached", reflect.TypeOf((*MockAPIs)(nil).WaitForENIAndIPsAttached), eni, wantedSecondaryIPs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForENIAndIPsAttached", reflect.TypeOf((*MockAPIs)(nil).WaitForENIAndIPsAttached), arg0, arg1) } diff --git a/pkg/ec2wrapper/client.go b/pkg/ec2wrapper/client.go index eca2c897fa..632559a23d 100644 --- a/pkg/ec2wrapper/client.go +++ b/pkg/ec2wrapper/client.go @@ -36,6 +36,7 @@ type EC2 interface { ModifyNetworkInterfaceAttribute(ctx context.Context, input *ec2.ModifyNetworkInterfaceAttributeInput, opts ...func(*ec2.Options)) (*ec2.ModifyNetworkInterfaceAttributeOutput, error) CreateTags(ctx context.Context, input *ec2.CreateTagsInput, opts ...func(*ec2.Options)) (*ec2.CreateTagsOutput, error) DescribeSubnets(ctx context.Context, input *ec2.DescribeSubnetsInput, opts ...func(*ec2.Options)) (*ec2.DescribeSubnetsOutput, error) + DescribeSecurityGroups(ctx context.Context, input *ec2.DescribeSecurityGroupsInput, opts ...func(*ec2.Options)) (*ec2.DescribeSecurityGroupsOutput, error) } // New creates a new EC2 wrapper diff --git a/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go b/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go index cf8cb72824..11bfafab7b 100644 --- a/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go +++ b/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go @@ -229,6 +229,26 @@ func (mr *MockEC2MockRecorder) DescribeNetworkInterfaces(arg0, arg1 interface{}, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeNetworkInterfaces", reflect.TypeOf((*MockEC2)(nil).DescribeNetworkInterfaces), varargs...) } +// DescribeSecurityGroups mocks base method. +func (m *MockEC2) DescribeSecurityGroups(arg0 context.Context, arg1 *ec2.DescribeSecurityGroupsInput, arg2 ...func(*ec2.Options)) (*ec2.DescribeSecurityGroupsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeSecurityGroups", varargs...) + ret0, _ := ret[0].(*ec2.DescribeSecurityGroupsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeSecurityGroups indicates an expected call of DescribeSecurityGroups. +func (mr *MockEC2MockRecorder) DescribeSecurityGroups(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeSecurityGroups", reflect.TypeOf((*MockEC2)(nil).DescribeSecurityGroups), varargs...) +} + // DescribeSubnets mocks base method. func (m *MockEC2) DescribeSubnets(arg0 context.Context, arg1 *ec2.DescribeSubnetsInput, arg2 ...func(*ec2.Options)) (*ec2.DescribeSubnetsOutput, error) { m.ctrl.T.Helper() diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index 270ec8c888..6a9e929b7e 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -128,6 +128,8 @@ type ENI struct { AvailableIPv4Cidrs map[string]*CidrInfo //IPv6CIDRs contains information tied to IPv6 Prefixes attached to the ENI IPv6Cidrs map[string]*CidrInfo + // IsExcludedForPodIPs indicates whether this ENI should be excluded from pod IP allocation + IsExcludedForPodIPs bool // RouteTableID is the route table ID associated with the ENI on the host RouteTableID int } @@ -190,6 +192,15 @@ func (e *ENI) AssignedIPv4Addresses() int { return count } +// AssignedIPv6Addresses is the number of IPv6 addresses already assigned +func (e *ENI) AssignedIPv6Addresses() int { + count := 0 + for _, availableCidr := range e.IPv6Cidrs { + count += availableCidr.AssignedIPAddressesInCidr() + } + return count +} + // AssignedIPAddressesInCidr is the number of IP addresses already assigned in the IPv4 CIDR func (cidr *CidrInfo) AssignedIPAddressesInCidr() int { count := 0 @@ -480,6 +491,38 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk, return nil } +// SetENIExcludedForPodIPs marks an ENI as excluded from pod IP allocation +func (ds *DataStore) SetENIExcludedForPodIPs(eniID string, excluded bool) error { + ds.lock.Lock() + defer ds.lock.Unlock() + + eni, ok := ds.eniPool[eniID] + if !ok { + return errors.New(UnknownENIError) + } + + eni.IsExcludedForPodIPs = excluded + if excluded { + ds.log.Infof("ENI %s marked as excluded from pod IP allocation", eniID) + } else { + ds.log.Infof("ENI %s marked as available for pod IP allocation", eniID) + } + return nil +} + +// IsENIExcludedForPodIPs returns whether an ENI is excluded from pod IP allocation +func (ds *DataStore) IsENIExcludedForPodIPs(eniID string) bool { + ds.lock.Lock() + defer ds.lock.Unlock() + + eni, ok := ds.eniPool[eniID] + if !ok { + return false + } + + return eni.IsExcludedForPodIPs +} + // AddIPv4AddressToStore adds IPv4 CIDR of an ENI to data store func (ds *DataStore) AddIPv4CidrToStore(eniID string, ipv4Cidr net.IPNet, isPrefix bool) error { ds.lock.Lock() @@ -569,6 +612,55 @@ func (ds *DataStore) DelIPv4CidrFromStore(eniID string, cidr net.IPNet, force bo return nil } +// DelIPv6CidrFromStore deletes IPv6 CIDR from the datastore +func (ds *DataStore) DelIPv6CidrFromStore(eniID string, cidr net.IPNet, force bool) error { + ds.lock.Lock() + defer ds.lock.Unlock() + + curENI, ok := ds.eniPool[eniID] + if !ok { + ds.log.Debugf("Unknown ENI %s while deleting the IPv6 CIDR", eniID) + return errors.New(UnknownENIError) + } + strIPv6Cidr := cidr.String() + + var deletableCidr *CidrInfo + deletableCidr, ok = curENI.IPv6Cidrs[strIPv6Cidr] + if !ok { + ds.log.Debugf("Unknown IPv6 CIDR %s", strIPv6Cidr) + return errors.New(UnknownIPError) + } + + // IPv6 only uses prefix delegation, check for any assigned IPs + updateBackingStore := false + for _, addr := range deletableCidr.IPAddresses { + if addr.Assigned() { + if !force { + return errors.New(IPInUseError) + } + prometheusmetrics.ForceRemovedIPs.Inc() + ds.unassignPodIPAddressUnsafe(addr) + updateBackingStore = true + } + } + if updateBackingStore { + if err := ds.writeBackingStoreUnsafe(); err != nil { + ds.log.Warnf("Unable to update backing store: %v", err) + // Continuing because 'force' + } + } + ds.total -= deletableCidr.Size() + if deletableCidr.IsPrefix { + ds.allocatedPrefix-- + prometheusmetrics.TotalPrefixes.Set(float64(ds.allocatedPrefix)) + } + prometheusmetrics.TotalIPs.Set(float64(ds.total)) + delete(curENI.IPv6Cidrs, strIPv6Cidr) + ds.log.Infof("Deleted ENI(%s)'s IPv6 Prefix %s from datastore", eniID, strIPv6Cidr) + + return nil +} + // AddIPv6AddressToStore adds IPv6 CIDR of an ENI to data store func (ds *DataStore) AddIPv6CidrToStore(eniID string, ipv6Cidr net.IPNet, isPrefix bool) error { ds.lock.Lock() @@ -640,6 +732,12 @@ func (ds *DataStore) AssignPodIPv6Address(ipamKey IPAMKey, ipamMetadata IPAMMeta // In IPv6 Prefix Delegation mode, eniPool will only have Primary ENI. for _, eni := range ds.eniPool { + // Skip ENIs that are excluded from pod IP allocation + if eni.IsExcludedForPodIPs { + ds.log.Debugf("Skipping ENI %s as it is excluded from pod IP allocation", eni.ID) + continue + } + if len(eni.IPv6Cidrs) == 0 { continue } @@ -690,6 +788,12 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey, ipamMetadata IPAMMeta } for _, eni := range ds.eniPool { + // Skip ENIs that are excluded from pod IP allocation + if eni.IsExcludedForPodIPs { + ds.log.Debugf("Skipping ENI %s as it is excluded from pod IP allocation", eni.ID) + continue + } + for _, availableCidr := range eni.AvailableIPv4Cidrs { var addr *AddressInfo var strPrivateIPv4 string @@ -809,6 +913,11 @@ func (ds *DataStore) GetIPStats(addressFamily string) *DataStoreStats { TotalPrefixes: ds.allocatedPrefix, } for _, eni := range ds.eniPool { + // Skip excluded ENIs when calculating available IPs for pod allocation + if eni.IsExcludedForPodIPs { + continue + } + AssignedCIDRs := eni.AvailableIPv4Cidrs if addressFamily == "6" { AssignedCIDRs = eni.IPv6Cidrs @@ -858,6 +967,10 @@ func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENI) bool otherWarmIPs := 0 for _, other := range ds.eniPool { if other.ID != eni.ID { + // Skip excluded ENIs when calculating warm IPs for pod allocation + if other.IsExcludedForPodIPs { + continue + } for _, otherPrefixes := range other.AvailableIPv4Cidrs { if (ds.isPDEnabled && otherPrefixes.IsPrefix) || (!ds.isPDEnabled && !otherPrefixes.IsPrefix) { otherWarmIPs += otherPrefixes.Size() - otherPrefixes.AssignedIPAddressesInCidr() @@ -879,6 +992,10 @@ func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *ENI) otherIPs := 0 for _, other := range ds.eniPool { if other.ID != eni.ID { + // Skip excluded ENIs when calculating total IPs for pod allocation + if other.IsExcludedForPodIPs { + continue + } for _, otherPrefixes := range other.AvailableIPv4Cidrs { if (ds.isPDEnabled && otherPrefixes.IsPrefix) || (!ds.isPDEnabled && !otherPrefixes.IsPrefix) { otherIPs += otherPrefixes.Size() @@ -900,6 +1017,10 @@ func (ds *DataStore) isRequiredForWarmPrefixTarget(warmPrefixTarget int, eni *EN freePrefixes := 0 for _, other := range ds.eniPool { if other.ID != eni.ID { + // Skip excluded ENIs when calculating free prefixes for pod allocation + if other.IsExcludedForPodIPs { + continue + } for _, otherPrefixes := range other.AvailableIPv4Cidrs { if otherPrefixes.AssignedIPAddressesInCidr() == 0 { freePrefixes++ @@ -995,6 +1116,11 @@ func (ds *DataStore) GetAllocatableENIs(maxIPperENI int, skipPrimary bool) []*EN ds.log.Debugf("Skip needs IP check for trunk ENI of primary ENI when Custom Networking is enabled") continue } + // Skip ENIs that are excluded from pod IP allocation + if eni.IsExcludedForPodIPs { + ds.log.Debugf("Skip needs IP check for ENI %s as it is excluded from pod IP allocation", eni.ID) + continue + } if len(eni.AvailableIPv4Cidrs) < maxIPperENI { ds.log.Debugf("Found ENI %s that has less than the maximum number of IP/Prefixes addresses allocated: cur=%d, max=%d", eni.ID, len(eni.AvailableIPv4Cidrs), maxIPperENI) @@ -1144,6 +1270,14 @@ func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, d ipamKey, addr.Address, eni.DeviceNumber) // Decrement ENI IP usage when a pod is deallocated prometheusmetrics.EniIPsInUse.WithLabelValues(eni.ID).Dec() + + // Check if ENI is excluded and CIDR is now empty - cleanup if needed + if eni.IsExcludedForPodIPs && availableCidr.AssignedIPAddressesInCidr() == 0 { + ds.log.Infof("CIDR %s on excluded ENI %s is now empty, scheduling for cleanup", availableCidr.Cidr.String(), eni.ID) + // Schedule async cleanup of the empty CIDR + go ds.deallocateEmptyCIDR(eni.ID, availableCidr) + } + return eni, addr.Address, eni.DeviceNumber, originalIPAMMetadata.InterfacesCount, eni.RouteTableID, nil } @@ -1193,7 +1327,7 @@ func (ds *DataStore) FreeableIPs(eniID string) []net.IPNet { return freeable } -// FreeablePrefixes returns a list of unused and potentially freeable IPs. +// FreeablePrefixes returns a list of unused and potentially freeable prefixes (both IPv4 and IPv6). // Note result may already be stale by the time you look at it. func (ds *DataStore) FreeablePrefixes(eniID string) []net.IPNet { ds.lock.Lock() @@ -1205,12 +1339,22 @@ func (ds *DataStore) FreeablePrefixes(eniID string) []net.IPNet { return nil } - freeable := make([]net.IPNet, 0, len(eni.AvailableIPv4Cidrs)) + freeable := make([]net.IPNet, 0, len(eni.AvailableIPv4Cidrs)+len(eni.IPv6Cidrs)) + + // Check IPv4 prefixes for _, assignedaddr := range eni.AvailableIPv4Cidrs { if assignedaddr.IsPrefix && assignedaddr.AssignedIPAddressesInCidr() == 0 { freeable = append(freeable, assignedaddr.Cidr) } } + + // Check IPv6 prefixes (IPv6 only uses prefix delegation mode) + for _, assignedaddr := range eni.IPv6Cidrs { + if assignedaddr.AssignedIPAddressesInCidr() == 0 { + freeable = append(freeable, assignedaddr.Cidr) + } + } + return freeable } @@ -1609,3 +1753,66 @@ func (ds *DataStoreAccess) ReadAllDataStores(enableIPv6 bool) error { } return nil } + +// deallocateEmptyCIDR asynchronously deallocates an empty CIDR from an excluded ENI +func (ds *DataStore) deallocateEmptyCIDR(eniID string, cidrToCleanup *CidrInfo) { + cidrStr := cidrToCleanup.Cidr.String() + ds.log.Infof("Starting async cleanup for empty CIDR %s on excluded ENI %s", cidrStr, eniID) + + // Add delay to avoid race conditions with pod cleanup + time.Sleep(5 * time.Second) + + ds.lock.Lock() + defer ds.lock.Unlock() + + // Double-check that the CIDR is still empty and ENI is still excluded + eni := ds.eniPool[eniID] + if eni == nil { + ds.log.Warnf("ENI %s not found during CIDR cleanup", eniID) + return + } + + if !eni.IsExcludedForPodIPs { + ds.log.Infof("ENI %s is no longer excluded, skipping CIDR cleanup", eniID) + return + } + + // Find the CIDR in the appropriate map using AddressFamily + var targetCidr *CidrInfo + var isIPv4 bool = cidrToCleanup.AddressFamily == "4" + + if isIPv4 { + targetCidr = eni.AvailableIPv4Cidrs[cidrStr] + } else { + targetCidr = eni.IPv6Cidrs[cidrStr] + } + + if targetCidr == nil { + ds.log.Warnf("CIDR %s not found on ENI %s during cleanup", cidrStr, eniID) + return + } + + // Check if CIDR is still empty + if targetCidr.AssignedIPAddressesInCidr() > 0 { + ds.log.Infof("CIDR %s on ENI %s is no longer empty, skipping cleanup", cidrStr, eniID) + return + } + + // Remove the empty CIDR from the ENI structure + ds.log.Infof("Removing empty CIDR %s from excluded ENI %s in datastore", cidrStr, eniID) + + // Remove the CIDR from the appropriate ENI map + if isIPv4 { + delete(eni.AvailableIPv4Cidrs, cidrStr) + } else { + delete(eni.IPv6Cidrs, cidrStr) + } + + // Update backing store + if err := ds.writeBackingStoreUnsafe(); err != nil { + ds.log.Warnf("Failed to write backing store after removing empty CIDR: %v", err) + // Note: We continue since the CIDR removal from local state was successful + } + + ds.log.Infof("Successfully removed empty CIDR %s from excluded ENI %s", cidrStr, eniID) +} diff --git a/pkg/ipamd/datastore/data_store_test.go b/pkg/ipamd/datastore/data_store_test.go index 3460af9e13..675e5e970b 100644 --- a/pkg/ipamd/datastore/data_store_test.go +++ b/pkg/ipamd/datastore/data_store_test.go @@ -15,6 +15,7 @@ package datastore import ( "errors" + "fmt" "net" "os" "testing" @@ -1084,6 +1085,65 @@ func TestGetIPStatsV6(t *testing.T) { ) } +func TestAssignedIPv6Addresses(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}, true, defaultNetworkCard) + + // Test ENI with no IPv6 CIDRs + _ = ds.AddENI("eni-1", 1, true, false, false, 0) + eniInfo, _ := ds.eniPool["eni-1"] + assert.Equal(t, 0, eniInfo.AssignedIPv6Addresses(), "ENI with no IPv6 CIDRs should have 0 assigned addresses") + + // Add IPv6 CIDR but don't assign any addresses yet + ipv6Addr1 := net.IPNet{IP: net.ParseIP("2001:db8::"), Mask: net.CIDRMask(80, 128)} + _ = ds.AddIPv6CidrToStore("eni-1", ipv6Addr1, true) + assert.Equal(t, 0, eniInfo.AssignedIPv6Addresses(), "ENI with unassigned IPv6 CIDR should have 0 assigned addresses") + + // Assign first IPv6 address to a pod + key1 := IPAMKey{"net0", "sandbox-1", "eth0"} + _, _, _, err := ds.AssignPodIPv6Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "pod-1"}) + assert.NoError(t, err) + assert.Equal(t, 1, eniInfo.AssignedIPv6Addresses(), "Should have 1 assigned IPv6 address after first pod assignment") + + // Assign second IPv6 address to another pod + key2 := IPAMKey{"net0", "sandbox-2", "eth0"} + _, _, _, err = ds.AssignPodIPv6Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "pod-2"}) + assert.NoError(t, err) + assert.Equal(t, 2, eniInfo.AssignedIPv6Addresses(), "Should have 2 assigned IPv6 addresses after second pod assignment") + + // Add another IPv6 CIDR to the same ENI + ipv6Addr2 := net.IPNet{IP: net.ParseIP("2001:db8:1::"), Mask: net.CIDRMask(80, 128)} + _ = ds.AddIPv6CidrToStore("eni-1", ipv6Addr2, true) + assert.Equal(t, 2, eniInfo.AssignedIPv6Addresses(), "Adding new CIDR should not change assigned count") + + // Assign address from the second CIDR + key3 := IPAMKey{"net0", "sandbox-3", "eth0"} + _, _, _, err = ds.AssignPodIPv6Address(key3, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "pod-3"}) + assert.NoError(t, err) + assert.Equal(t, 3, eniInfo.AssignedIPv6Addresses(), "Should have 3 assigned IPv6 addresses across multiple CIDRs") + + // Unassign one address + _, _, _, _, _, err = ds.UnassignPodIPAddress(key1) + assert.NoError(t, err) + assert.Equal(t, 2, eniInfo.AssignedIPv6Addresses(), "Should have 2 assigned IPv6 addresses after unassignment") + + // Test ENI with multiple CIDRs and various assignment states + // In IPv6 PD mode, only primary ENI can have IPv6 addresses assigned, so add CIDRs to eni-1 (primary) + // Add more IPv6 CIDRs to eni-1 for additional testing + for i := 0; i < 3; i++ { + cidr := net.IPNet{IP: net.ParseIP(fmt.Sprintf("2001:db8:%d::", i+10)), Mask: net.CIDRMask(80, 128)} + _ = ds.AddIPv6CidrToStore("eni-1", cidr, true) + } + assert.Equal(t, 2, eniInfo.AssignedIPv6Addresses(), "Should maintain existing assignments after adding new CIDRs") + + // Assign additional addresses from different CIDRs - they will all go to eni-1 (primary) in IPv6 PD mode + for i := 0; i < 3; i++ { + key := IPAMKey{"net0", fmt.Sprintf("sandbox-eni1-%d", i), "eth0"} + _, _, _, err = ds.AssignPodIPv6Address(key, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: fmt.Sprintf("pod-eni1-%d", i)}) + assert.NoError(t, err) + } + assert.Equal(t, 5, eniInfo.AssignedIPv6Addresses(), "Should correctly count assigned addresses across multiple CIDRs on primary ENI") +} + func TestWarmENIInteractions(t *testing.T) { ds := NewDataStore(Testlog, NullCheckpoint{}, false, defaultNetworkCard) @@ -1702,6 +1762,7 @@ func TestInitializeDataStores(t *testing.T) { assert.Equal(t, 2, dsAccess.DataStores[1].GetNetworkCard()) }) } + func TestDataStoreAccess_GetDataStore(t *testing.T) { log := Testlog defaultPath := "/tmp/test-datastore.json" @@ -1719,3 +1780,410 @@ func TestDataStoreAccess_GetDataStore(t *testing.T) { ds := dsAccess.GetDataStore(99) assert.Nil(t, ds) } + +func TestAssignPodIPv4AddressWithExcludedENI(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}, false, defaultNetworkCard) + + // Add primary ENI and mark it as excluded + err := ds.AddENI("eni-1", 0, true, false, false, 0) + assert.NoError(t, err) + err = ds.SetENIExcludedForPodIPs("eni-1", true) + assert.NoError(t, err) + + // Add IPs to the excluded ENI + ipv4Addr := net.IPNet{IP: net.ParseIP("10.0.0.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) + assert.NoError(t, err) + + // Try to assign an IP - should fail as ENI is excluded + _, _, _, err = ds.AssignPodIPv4Address( + IPAMKey{ + NetworkName: "net0", + ContainerID: "container-1", + IfName: "eth0", + }, + IPAMMetadata{ + K8SPodNamespace: "default", + K8SPodName: "pod-1", + }, + ) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no available IP/Prefix addresses") + + // Add a secondary ENI that is not excluded + err = ds.AddENI("eni-2", 1, false, false, false, 0) + assert.NoError(t, err) + + // Add IPs to the secondary ENI + ipv4Addr2 := net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = ds.AddIPv4CidrToStore("eni-2", ipv4Addr2, false) + assert.NoError(t, err) + + // Now assignment should succeed from the non-excluded ENI + ip, deviceNum, _, err := ds.AssignPodIPv4Address( + IPAMKey{ + NetworkName: "net0", + ContainerID: "container-1", + IfName: "eth0", + }, + IPAMMetadata{ + K8SPodNamespace: "default", + K8SPodName: "pod-1", + }, + ) + assert.NoError(t, err) + assert.Equal(t, "10.0.0.2", ip) + assert.Equal(t, 1, deviceNum) +} + +func TestGetAllocatableENIsWithExclusion(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}, false, defaultNetworkCard) + + // Add primary ENI and mark it as excluded + err := ds.AddENI("eni-1", 0, true, false, false, 0) + assert.NoError(t, err) + err = ds.SetENIExcludedForPodIPs("eni-1", true) + assert.NoError(t, err) + + // Add secondary ENI (not excluded) + err = ds.AddENI("eni-2", 1, false, false, false, 0) + assert.NoError(t, err) + + // Get allocatable ENIs - should only return the non-excluded one + allocatable := ds.GetAllocatableENIs(10, false) + assert.Equal(t, 1, len(allocatable)) + assert.Equal(t, "eni-2", allocatable[0].ID) +} + +func TestGetIPStatsWithExcludedENI(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}, false, defaultNetworkCard) + + // Add primary ENI with IPs and mark it as excluded + err := ds.AddENI("eni-1", 0, true, false, false, 0) + assert.NoError(t, err) + err = ds.SetENIExcludedForPodIPs("eni-1", true) + assert.NoError(t, err) + + // Add IPs to excluded ENI + for i := 1; i <= 3; i++ { + ipv4Addr := net.IPNet{IP: net.ParseIP(fmt.Sprintf("10.0.0.%d", i)), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) + assert.NoError(t, err) + } + + // Add secondary ENI with IPs (not excluded) + err = ds.AddENI("eni-2", 1, false, false, false, 0) + assert.NoError(t, err) + + for i := 11; i <= 12; i++ { + ipv4Addr := net.IPNet{IP: net.ParseIP(fmt.Sprintf("10.0.0.%d", i)), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = ds.AddIPv4CidrToStore("eni-2", ipv4Addr, false) + assert.NoError(t, err) + } + + // Get IP stats - should only count IPs from non-excluded ENIs + stats := ds.GetIPStats("4") + assert.Equal(t, 2, stats.TotalIPs) // Only IPs from eni-2 + assert.Equal(t, 0, stats.AssignedIPs) + assert.Equal(t, 2, stats.AvailableAddresses()) +} + +func TestDelIPv6CidrFromStore(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}, true, defaultNetworkCard) + err := ds.AddENI("eni-1", 1, true, false, false, 0) + assert.NoError(t, err) + + // Test 1: Add and delete unassigned IPv6 CIDRs + ipv6Addr1 := net.IPNet{IP: net.ParseIP("2001:db8::"), Mask: net.CIDRMask(80, 128)} + err = ds.AddIPv6CidrToStore("eni-1", ipv6Addr1, true) + assert.NoError(t, err) + assert.Equal(t, 281474976710656, ds.total) // 2^48 addresses in /80 + assert.Equal(t, 1, len(ds.eniPool["eni-1"].IPv6Cidrs)) + + ipv6Addr2 := net.IPNet{IP: net.ParseIP("2001:db8:1::"), Mask: net.CIDRMask(80, 128)} + err = ds.AddIPv6CidrToStore("eni-1", ipv6Addr2, true) + assert.NoError(t, err) + assert.Equal(t, 562949953421312, ds.total) // 2 * 2^48 addresses + assert.Equal(t, 2, len(ds.eniPool["eni-1"].IPv6Cidrs)) + + // Delete one IPv6 CIDR without assigned IPs - should succeed + err = ds.DelIPv6CidrFromStore("eni-1", ipv6Addr2, false) + assert.NoError(t, err) + assert.Equal(t, 281474976710656, ds.total) // Back to 1 CIDR worth + assert.Equal(t, 1, len(ds.eniPool["eni-1"].IPv6Cidrs)) + + // Test 2: Try to delete a non-existent IPv6 CIDR + unknownIPv6Addr := net.IPNet{IP: net.ParseIP("2001:db8:9999::"), Mask: net.CIDRMask(80, 128)} + err = ds.DelIPv6CidrFromStore("eni-1", unknownIPv6Addr, false) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unknown IP") + assert.Equal(t, 281474976710656, ds.total) // Should remain unchanged + assert.Equal(t, 1, len(ds.eniPool["eni-1"].IPv6Cidrs)) + + // Test 3: Assign an IPv6 address to a pod from the remaining CIDR + key := IPAMKey{"net0", "sandbox-1", "eth0"} + ipv6Address, device, _, err := ds.AssignPodIPv6Address(key, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) + assert.NoError(t, err) + assert.NotEmpty(t, ipv6Address) + assert.Equal(t, 1, device) + + // Try to delete the CIDR with assigned IP - should fail without force + err = ds.DelIPv6CidrFromStore("eni-1", ipv6Addr1, false) + assert.Error(t, err) + assert.Contains(t, err.Error(), "IP is used and can not be deleted") + assert.Equal(t, 281474976710656, ds.total) // Should remain unchanged + assert.Equal(t, 1, len(ds.eniPool["eni-1"].IPv6Cidrs)) + + // Test 4: Force delete the CIDR with assigned IP - should succeed + err = ds.DelIPv6CidrFromStore("eni-1", ipv6Addr1, true) + assert.NoError(t, err) + assert.Equal(t, 0, ds.total) // Back to 0 + assert.Equal(t, 0, len(ds.eniPool["eni-1"].IPv6Cidrs)) + + // Test 5: Try to delete from a non-existent ENI + err = ds.DelIPv6CidrFromStore("eni-nonexistent", ipv6Addr1, false) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unknown ENI") +} + +func TestFreeablePrefixesBothIPv4AndIPv6(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}, true, defaultNetworkCard) + err := ds.AddENI("eni-1", 1, true, false, false, 0) + assert.NoError(t, err) + + // Add IPv4 prefixes + ipv4Prefix1 := net.IPNet{IP: net.ParseIP("10.0.0.0"), Mask: net.CIDRMask(28, 32)} + err = ds.AddIPv4CidrToStore("eni-1", ipv4Prefix1, true) + assert.NoError(t, err) + + ipv4Prefix2 := net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: net.CIDRMask(28, 32)} + err = ds.AddIPv4CidrToStore("eni-1", ipv4Prefix2, true) + assert.NoError(t, err) + + // Add IPv4 secondary IP (not a prefix) + ipv4SecondaryIP := net.IPNet{IP: net.ParseIP("10.0.2.1"), Mask: net.CIDRMask(32, 32)} + err = ds.AddIPv4CidrToStore("eni-1", ipv4SecondaryIP, false) + assert.NoError(t, err) + + // Add IPv6 prefixes + ipv6Prefix1 := net.IPNet{IP: net.ParseIP("2001:db8::"), Mask: net.CIDRMask(80, 128)} + err = ds.AddIPv6CidrToStore("eni-1", ipv6Prefix1, true) + assert.NoError(t, err) + + ipv6Prefix2 := net.IPNet{IP: net.ParseIP("2001:db8:1::"), Mask: net.CIDRMask(80, 128)} + err = ds.AddIPv6CidrToStore("eni-1", ipv6Prefix2, true) + assert.NoError(t, err) + + // Get freeable prefixes - should return all unassigned prefixes (both IPv4 and IPv6) + freeablePrefixes := ds.FreeablePrefixes("eni-1") + assert.Equal(t, 4, len(freeablePrefixes)) // 2 IPv4 prefixes + 2 IPv6 prefixes + + // Check that IPv4 secondary IP is not included (it's not a prefix) + ipv4PrefixCount := 0 + ipv6PrefixCount := 0 + for _, prefix := range freeablePrefixes { + if prefix.IP.To4() != nil { + ipv4PrefixCount++ + } else { + ipv6PrefixCount++ + } + } + assert.Equal(t, 2, ipv4PrefixCount, "Should have 2 IPv4 prefixes") + assert.Equal(t, 2, ipv6PrefixCount, "Should have 2 IPv6 prefixes") + + // Assign an IP from IPv4 prefix1 to a pod + key1 := IPAMKey{"net0", "sandbox-1", "eth0"} + ipv4Address, device, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "pod-1"}) + assert.NoError(t, err) + assert.NotEmpty(t, ipv4Address) + assert.Equal(t, 1, device) + + // Assign an IP from IPv6 prefix1 to a pod + key2 := IPAMKey{"net0", "sandbox-2", "eth0"} + ipv6Address, device, _, err := ds.AssignPodIPv6Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "pod-2"}) + assert.NoError(t, err) + assert.NotEmpty(t, ipv6Address) + assert.Equal(t, 1, device) + + // Now get freeable prefixes - should exclude prefixes with assigned IPs + freeablePrefixes = ds.FreeablePrefixes("eni-1") + assert.Equal(t, 2, len(freeablePrefixes)) // Only unassigned prefixes + + // Verify the returned prefixes are the ones without assigned IPs + ipv4PrefixCount = 0 + ipv6PrefixCount = 0 + for _, prefix := range freeablePrefixes { + if prefix.IP.To4() != nil { + ipv4PrefixCount++ + // Should be the second IPv4 prefix (10.0.1.0/28), not the first one with assigned IP + assert.Equal(t, "10.0.1.0", prefix.IP.String()) + } else { + ipv6PrefixCount++ + // Should be the second IPv6 prefix (2001:db8:1::/80), not the first one with assigned IP + assert.Equal(t, "2001:db8:1::", prefix.IP.String()) + } + } + assert.Equal(t, 1, ipv4PrefixCount, "Should have 1 unassigned IPv4 prefix") + assert.Equal(t, 1, ipv6PrefixCount, "Should have 1 unassigned IPv6 prefix") + + // Test with non-existent ENI + freeablePrefixes = ds.FreeablePrefixes("eni-nonexistent") + assert.Nil(t, freeablePrefixes, "Should return nil for non-existent ENI") +} + +func TestDeallocateEmptyCIDR(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}, false, defaultNetworkCard) + + // Setup test ENI + eniID := "eni-test-dealloc" + err := ds.AddENI(eniID, 1, false, false, false, 0) + assert.NoError(t, err) + + // Mark ENI as excluded for pod IPs + err = ds.SetENIExcludedForPodIPs(eniID, true) + assert.NoError(t, err) + + t.Run("IPv4 CIDR cleanup on excluded ENI", func(t *testing.T) { + // Add IPv4 CIDR to ENI + ipv4Cidr := net.IPNet{IP: net.ParseIP("192.168.1.0"), Mask: net.CIDRMask(24, 32)} + err = ds.AddIPv4CidrToStore(eniID, ipv4Cidr, false) + assert.NoError(t, err) + + // Verify CIDR exists + eni := ds.eniPool[eniID] + cidrStr := ipv4Cidr.String() + cidrInfo, exists := eni.AvailableIPv4Cidrs[cidrStr] + assert.True(t, exists, "IPv4 CIDR should exist before cleanup") + + // Call deallocateEmptyCIDR + ds.deallocateEmptyCIDR(eniID, cidrInfo) + + // Verify CIDR is removed + _, exists = eni.AvailableIPv4Cidrs[cidrStr] + assert.False(t, exists, "IPv4 CIDR should be removed after cleanup") + }) + + t.Run("IPv6 CIDR cleanup on excluded ENI", func(t *testing.T) { + // Add IPv6 CIDR to ENI + ipv6Cidr := net.IPNet{IP: net.ParseIP("2001:db8:1::"), Mask: net.CIDRMask(64, 128)} + err = ds.AddIPv6CidrToStore(eniID, ipv6Cidr, false) + assert.NoError(t, err) + + // Verify CIDR exists + eni := ds.eniPool[eniID] + cidrStr := ipv6Cidr.String() + cidrInfo, exists := eni.IPv6Cidrs[cidrStr] + assert.True(t, exists, "IPv6 CIDR should exist before cleanup") + + // Call deallocateEmptyCIDR + ds.deallocateEmptyCIDR(eniID, cidrInfo) + + // Verify CIDR is removed + _, exists = eni.IPv6Cidrs[cidrStr] + assert.False(t, exists, "IPv6 CIDR should be removed after cleanup") + }) + + t.Run("Skip cleanup when ENI is not excluded", func(t *testing.T) { + // Create new non-excluded ENI + nonExcludedENI := "eni-not-excluded" + err := ds.AddENI(nonExcludedENI, 1, false, false, false, 0) + assert.NoError(t, err) + + // Don't mark as excluded (default is false) + + // Add IPv4 CIDR + ipv4Cidr := net.IPNet{IP: net.ParseIP("192.168.2.0"), Mask: net.CIDRMask(24, 32)} + err = ds.AddIPv4CidrToStore(nonExcludedENI, ipv4Cidr, false) + assert.NoError(t, err) + + // Get CIDR info + eni := ds.eniPool[nonExcludedENI] + cidrStr := ipv4Cidr.String() + cidrInfo, exists := eni.AvailableIPv4Cidrs[cidrStr] + assert.True(t, exists, "IPv4 CIDR should exist") + + // Call deallocateEmptyCIDR - should not remove CIDR since ENI is not excluded + ds.deallocateEmptyCIDR(nonExcludedENI, cidrInfo) + + // Verify CIDR still exists + _, exists = eni.AvailableIPv4Cidrs[cidrStr] + assert.True(t, exists, "IPv4 CIDR should remain since ENI is not excluded") + }) + + t.Run("Skip cleanup when CIDR has assigned IPs", func(t *testing.T) { + // Create fresh datastore for isolated test + dsIsolated := NewDataStore(Testlog, NullCheckpoint{}, false, defaultNetworkCard) + + // Create new excluded ENI + excludedENI := "eni-excluded-with-ips" + err := dsIsolated.AddENI(excludedENI, 1, false, false, false, 0) + assert.NoError(t, err) + + err = dsIsolated.SetENIExcludedForPodIPs(excludedENI, true) + assert.NoError(t, err) + + // Add IPv4 CIDR + ipv4Cidr := net.IPNet{IP: net.ParseIP("192.168.10.0"), Mask: net.CIDRMask(24, 32)} + err = dsIsolated.AddIPv4CidrToStore(excludedENI, ipv4Cidr, false) + assert.NoError(t, err) + + // Manually assign an IP to the CIDR to simulate non-empty state + eni := dsIsolated.eniPool[excludedENI] + cidrStr := ipv4Cidr.String() + cidrInfo, exists := eni.AvailableIPv4Cidrs[cidrStr] + assert.True(t, exists, "IPv4 CIDR should exist") + + // Manually mark an IP as assigned in the CIDR + testIP := net.ParseIP("192.168.10.1") + cidrInfo.IPAddresses[testIP.String()] = &AddressInfo{ + Address: testIP.String(), + IPAMKey: IPAMKey{NetworkName: "test", ContainerID: "test-container", IfName: "eth0"}, + IPAMMetadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "test-pod"}, + AssignedTime: time.Now(), + } + + // Verify CIDR has assigned IPs + assert.Greater(t, cidrInfo.AssignedIPAddressesInCidr(), 0, "CIDR should have assigned IPs") + + // Call deallocateEmptyCIDR - should not remove CIDR since it has assigned IPs + dsIsolated.deallocateEmptyCIDR(excludedENI, cidrInfo) + + // Verify CIDR still exists + _, exists = eni.AvailableIPv4Cidrs[cidrStr] + assert.True(t, exists, "IPv4 CIDR should remain since it has assigned IPs") + }) + + t.Run("Handle non-existent ENI gracefully", func(t *testing.T) { + // Create dummy CIDR info + dummyCidr := &CidrInfo{ + Cidr: net.IPNet{IP: net.ParseIP("192.168.4.0"), Mask: net.CIDRMask(24, 32)}, + AddressFamily: "4", + } + + // Should not panic when ENI doesn't exist + ds.deallocateEmptyCIDR("eni-nonexistent", dummyCidr) + }) + + t.Run("Handle non-existent CIDR gracefully", func(t *testing.T) { + // Create ENI + testENI := "eni-cidr-test" + err := ds.AddENI(testENI, 1, false, false, false, 0) + assert.NoError(t, err) + + err = ds.SetENIExcludedForPodIPs(testENI, true) + assert.NoError(t, err) + + // Create CIDR info that doesn't exist in the ENI + nonExistentCidr := &CidrInfo{ + Cidr: net.IPNet{IP: net.ParseIP("192.168.5.0"), Mask: net.CIDRMask(24, 32)}, + AddressFamily: "4", + } + + // Should handle gracefully when CIDR doesn't exist + ds.deallocateEmptyCIDR(testENI, nonExistentCidr) + + // Verify ENI still exists and is unaffected + _, exists := ds.eniPool[testENI] + assert.True(t, exists, "ENI should still exist") + }) +} diff --git a/pkg/ipamd/datastore/secondary_eni_exclusion_focused_test.go b/pkg/ipamd/datastore/secondary_eni_exclusion_focused_test.go new file mode 100644 index 0000000000..a0dad4c495 --- /dev/null +++ b/pkg/ipamd/datastore/secondary_eni_exclusion_focused_test.go @@ -0,0 +1,183 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package datastore + +import ( + "net" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestSecondaryENIExclusionFocused tests the exact scenarios mentioned: +// 1. When secondary ENI is excluded, new pods should avoid it +// 2. When all non-excluded ENIs are full, allocation should fail appropriately +func TestSecondaryENIExclusionFocused(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}, false, defaultNetworkCard) + + // Setup: Primary ENI with one IP + err := ds.AddENI("eni-primary", 0, true, false, false, 0) + assert.NoError(t, err) + + primaryIP := net.IPNet{IP: net.ParseIP("10.0.1.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = ds.AddIPv4CidrToStore("eni-primary", primaryIP, false) + assert.NoError(t, err) + + // Setup: Secondary ENI with one IP + err = ds.AddENI("eni-secondary", 1, false, false, false, 0) + assert.NoError(t, err) + + secondaryIP := net.IPNet{IP: net.ParseIP("10.0.2.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = ds.AddIPv4CidrToStore("eni-secondary", secondaryIP, false) + assert.NoError(t, err) + + // Test 1: Before exclusion - verify we have 2 allocatable ENIs + allocatableENIs := ds.GetAllocatableENIs(10, false) // maxIPperENI=10, skipPrimary=false + assert.Equal(t, 2, len(allocatableENIs), "Should have 2 allocatable ENIs before exclusion") + + // Test 2: Exclude secondary ENI + err = ds.SetENIExcludedForPodIPs("eni-secondary", true) + assert.NoError(t, err) + assert.True(t, ds.eniPool["eni-secondary"].IsExcludedForPodIPs) + + // Test 3: After exclusion - verify only 1 allocatable ENI remains + allocatableENIs = ds.GetAllocatableENIs(10, false) + assert.Equal(t, 1, len(allocatableENIs), "Should have only 1 allocatable ENI after exclusion") + assert.Equal(t, "eni-primary", allocatableENIs[0].ID) + + // Test 4: Assign first pod - should work and go to primary ENI + key1 := IPAMKey{"net0", "pod-1", "eth0"} + ip1, _, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "pod-1"}) + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(ip1, "10.0.1."), "First pod should go to primary ENI, got: %s", ip1) + + // Test 5: Primary ENI should now be full + assert.Equal(t, 1, ds.eniPool["eni-primary"].AssignedIPv4Addresses()) + assert.Equal(t, 0, ds.eniPool["eni-secondary"].AssignedIPv4Addresses()) + + // Test 6: Try to assign second pod - should fail since primary is full and secondary is excluded + key2 := IPAMKey{"net0", "pod-2", "eth0"} + _, _, _, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "pod-2"}) + assert.Error(t, err, "Should fail when only non-excluded ENI is full") + assert.Contains(t, err.Error(), "no available IP/Prefix addresses") + + // Test 7: Verify excluded secondary ENI should be deletable (it has no pods) + ds.eniPool["eni-secondary"].createTime = time.Time{} // Make it old enough + + // Make sure IPs are out of cooldown + for _, cidr := range ds.eniPool["eni-secondary"].AvailableIPv4Cidrs { + for _, addr := range cidr.IPAddresses { + addr.UnassignedTime = time.Time{} + } + } + + removableENI := ds.RemoveUnusedENIFromStore(0, 0, 0) + assert.Equal(t, "eni-secondary", removableENI, "Excluded secondary ENI with no pods should be deletable") + + // Test 8: Verify secondary ENI was removed from pool + _, exists := ds.eniPool["eni-secondary"] + assert.False(t, exists, "Secondary ENI should be removed from pool") + + // Test 9: Primary ENI should remain unaffected + _, exists = ds.eniPool["eni-primary"] + assert.True(t, exists, "Primary ENI should remain in pool") + assert.Equal(t, 1, ds.eniPool["eni-primary"].AssignedIPv4Addresses()) + + // Test 10: Verify allocatable ENIs after secondary removal + allocatableENIs = ds.GetAllocatableENIs(10, false) + assert.Equal(t, 1, len(allocatableENIs), "Should still have 1 allocatable ENI") + assert.Equal(t, "eni-primary", allocatableENIs[0].ID) +} + +// TestSecondaryENIExclusionWithPods tests exclusion when secondary ENI has existing pods +func TestSecondaryENIExclusionWithPods(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}, false, defaultNetworkCard) + + // Setup ENIs + err := ds.AddENI("eni-primary", 0, true, false, false, 0) + assert.NoError(t, err) + + err = ds.AddENI("eni-secondary", 1, false, false, false, 0) + assert.NoError(t, err) + + // Add IPs + primaryIP := net.IPNet{IP: net.ParseIP("10.0.1.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = ds.AddIPv4CidrToStore("eni-primary", primaryIP, false) + assert.NoError(t, err) + + secondaryIP := net.IPNet{IP: net.ParseIP("10.0.2.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = ds.AddIPv4CidrToStore("eni-secondary", secondaryIP, false) + assert.NoError(t, err) + + // Assign pods to both ENIs naturally through regular allocation + key1 := IPAMKey{"net0", "pod-1", "eth0"} + ip1, _, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "pod-1"}) + assert.NoError(t, err) + + key2 := IPAMKey{"net0", "pod-2", "eth0"} + ip2, _, _, err := ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "pod-2"}) + assert.NoError(t, err) + + // Find which key corresponds to secondary ENI assignment + var keyOnSecondary IPAMKey + if strings.HasPrefix(ip1, "10.0.2.") { + keyOnSecondary = key1 + } else if strings.HasPrefix(ip2, "10.0.2.") { + keyOnSecondary = key2 + } else { + t.Fatal("No pod was assigned to secondary ENI") + } + + // Verify we have assignments on both ENIs + totalPods := ds.eniPool["eni-primary"].AssignedIPv4Addresses() + ds.eniPool["eni-secondary"].AssignedIPv4Addresses() + assert.Equal(t, 2, totalPods) + assert.Greater(t, ds.eniPool["eni-secondary"].AssignedIPv4Addresses(), 0) + + // Now exclude secondary ENI + err = ds.SetENIExcludedForPodIPs("eni-secondary", true) + assert.NoError(t, err) + + // Test: Excluded ENI with pods should NOT be deletable + ds.eniPool["eni-secondary"].createTime = time.Time{} // Make old enough + removableENI := ds.RemoveUnusedENIFromStore(0, 0, 0) + assert.Equal(t, "", removableENI, "Excluded ENI with pods should not be deletable") + + // Test: New allocations should skip excluded ENI (if primary has space) + // We'll add another IP to primary to allow new allocation + primaryIP2 := net.IPNet{IP: net.ParseIP("10.0.1.2"), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = ds.AddIPv4CidrToStore("eni-primary", primaryIP2, false) + assert.NoError(t, err) + + key3 := IPAMKey{"net0", "new-pod", "eth0"} + ip3, _, _, err := ds.AssignPodIPv4Address(key3, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "new-pod"}) + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(ip3, "10.0.1."), "New pod should avoid excluded secondary ENI, got: %s", ip3) + + // Test: After removing pod from excluded ENI, it should become deletable + _, _, _, _, _, err = ds.UnassignPodIPAddress(keyOnSecondary) + assert.NoError(t, err) + + // Clear cooldown + for _, cidr := range ds.eniPool["eni-secondary"].AvailableIPv4Cidrs { + for _, addr := range cidr.IPAddresses { + addr.UnassignedTime = time.Time{} + } + } + + // Now it should be deletable + removableENI = ds.RemoveUnusedENIFromStore(0, 0, 0) + assert.Equal(t, "eni-secondary", removableENI, "Excluded ENI should be deletable after pod removal") +} diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 7d3cf8bf33..4b52e3edf7 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -144,17 +144,17 @@ const ( // envNodeName will be used to store Node name envNodeName = "MY_NODE_NAME" - //envEnableIpv4PrefixDelegation is used to allocate /28 prefix instead of secondary IP for an ENI. + // envEnableIpv4PrefixDelegation is used to allocate /28 prefix instead of secondary IP for an ENI. envEnableIpv4PrefixDelegation = "ENABLE_PREFIX_DELEGATION" - //envWarmPrefixTarget is used to keep a /28 prefix in warm pool. + // envWarmPrefixTarget is used to keep a /28 prefix in warm pool. envWarmPrefixTarget = "WARM_PREFIX_TARGET" defaultWarmPrefixTarget = 0 - //envEnableIPv4 - Env variable to enable/disable IPv4 mode + // envEnableIPv4 - Env variable to enable/disable IPv4 mode envEnableIPv4 = "ENABLE_IPv4" - //envEnableIPv6 - Env variable to enable/disable IPv6 mode + // envEnableIPv6 - Env variable to enable/disable IPv6 mode envEnableIPv6 = "ENABLE_IPv6" ipV4AddrFamily = "4" @@ -209,9 +209,7 @@ const ( var log = logger.Get() -var ( - prometheusRegistered = false -) +var prometheusRegistered = false // IPAMContext contains node level control information type IPAMContext struct { @@ -296,7 +294,6 @@ func (c *IPAMContext) setUnmanagedENIs(tagMap map[string]awsutils.TagMap) { // When ENABLE_MULTI_NIC is true, Network Cards which ONLY includes an EFA-only device are marked as unmanaged. // If there is a ENA device on a Network Card along with EFA-only device, CNI manages the Network Card but excludes the EFA-only device func (c *IPAMContext) markUnmanagedNetworkCards(efaOnlyENINetworkCards []string, enisByNetworkCard [][]string) []bool { - skipNetworkCards := make([]bool, c.numNetworkCards) if !c.enableMultiNICSupport { skipNetworkCards = lo.Times(c.numNetworkCards, func(i int) bool { @@ -399,6 +396,7 @@ func (c *IPAMContext) inInsufficientCidrCoolingPeriod() bool { // then initializes IP address pool data store func New(k8sClient client.Client, withApiServer bool) (*IPAMContext, error) { prometheusRegister() + ctx := context.Background() c := &IPAMContext{} c.k8sClient = k8sClient c.networkClient = networkutils.New() @@ -431,7 +429,7 @@ func New(k8sClient client.Client, withApiServer bool) (*IPAMContext, error) { return nil, err } - err = c.awsClient.FetchInstanceTypeLimits() + err = c.awsClient.FetchInstanceTypeLimits(ctx) if err != nil { log.Errorf("Failed to get ENI limits from file:vpc_ip_limits or EC2 for %s", c.awsClient.GetInstanceType()) return nil, err @@ -485,7 +483,7 @@ func (c *IPAMContext) nodeInit() error { // Queries IMDS for all attached ENIs and then compares it against EC2. // Groups the ENIs into different types - metadataResult, err := c.awsClient.DescribeAllENIs() + metadataResult, err := c.awsClient.DescribeAllENIs(ctx) if err != nil { return errors.Wrap(err, "ipamd init: failed to retrieve attached ENIs info") } @@ -505,7 +503,7 @@ func (c *IPAMContext) nodeInit() error { isTrunkENI := eni.ENIID == metadataResult.TrunkENI isEFAENI := metadataResult.EFAENIs[eni.ENIID] if !isTrunkENI && !c.disableENIProvisioning { - if err := c.awsClient.TagENI(eni.ENIID, metadataResult.TagMap[eni.ENIID]); err != nil { + if err := c.awsClient.TagENI(ctx, eni.ENIID, metadataResult.TagMap[eni.ENIID]); err != nil { return errors.Wrapf(err, "ipamd init: failed to tag managed ENI %v", eni.ENIID) } } @@ -514,7 +512,7 @@ func (c *IPAMContext) nodeInit() error { retry := 0 for { retry++ - if err = c.setupENI(eni.ENIID, eni, isTrunkENI, isEFAENI); err == nil { + if err = c.setupENI(ctx, eni.ENIID, eni, isTrunkENI, isEFAENI); err == nil { log.Infof("ENI %s set up.", eni.ENIID) break } @@ -541,15 +539,18 @@ func (c *IPAMContext) nodeInit() error { return err } + // Check all ENIs (primary and secondary) for subnet exclusion + c.checkAndHandleAllENIExclusion(ctx) + if c.enableIPv4 { if c.enablePrefixDelegation { // During upgrade or if prefix delgation knob is disabled to enabled then we // might have secondary IPs attached to ENIs so doing a cleanup if not used before moving on - c.tryUnassignIPsFromENIs() + c.tryUnassignIPsFromENIs(ctx) } else { // When prefix delegation knob is enabled to disabled then we might // have unused prefixes attached to the ENIs so need to cleanup - c.tryUnassignPrefixesFromENIs() + c.tryUnassignPrefixesFromENIs(ctx) } } @@ -566,14 +567,26 @@ func (c *IPAMContext) nodeInit() error { // 1. after managed/unmanaged ENIs have been determined // 2. before any new ENIs are attached if !c.disableENIProvisioning { - if err := c.awsClient.RefreshSGIDs(primaryENIMac, c.dataStoreAccess); err != nil { + if err := c.awsClient.RefreshSGIDs(ctx, primaryENIMac, c.dataStoreAccess); err != nil { return err } + // Also refresh custom security groups for secondary subnets + // Custom security groups are only relevant when subnet discovery is enabled + if c.useSubnetDiscovery { + if err := c.awsClient.RefreshCustomSGIDs(ctx, c.dataStoreAccess); err != nil { + return err + } + } + // Refresh security groups and VPC CIDR blocks in the background // Ignoring errors since we will retry in 30s go wait.Forever(func() { - c.awsClient.RefreshSGIDs(primaryENIMac, c.dataStoreAccess) + c.awsClient.RefreshSGIDs(context.Background(), primaryENIMac, c.dataStoreAccess) + // Also refresh custom security groups for secondary subnets + if c.useSubnetDiscovery { + c.awsClient.RefreshCustomSGIDs(context.Background(), c.dataStoreAccess) + } }, 30*time.Second) } @@ -647,7 +660,6 @@ func (c *IPAMContext) nodeInit() error { } func (c *IPAMContext) handlePreScaling(ctx context.Context) error { - // On node init, check if datastore pool needs to be increased. If so, attach CIDRs from existing ENIs and attach new ENIs. for networkCard, decisions := range c.isDatastorePoolTooLow() { log.Debugf("is datastore pool low for networkCard %d, decision %t", networkCard, decisions.IsLow) @@ -742,11 +754,11 @@ func (c *IPAMContext) updateIPStats(unmanaged int) { } // StartNodeIPPoolManager monitors the IP pool, add or del them when it is required. -func (c *IPAMContext) StartNodeIPPoolManager() { +func (c *IPAMContext) StartNodeIPPoolManager(ctx context.Context) { // For IPv6, if Security Groups for Pods is enabled, wait until trunk ENI is attached and add it to the datastore. if c.enableIPv6 { if c.enablePodENI && c.dataStoreAccess.GetDataStore(DefaultNetworkCardIndex).GetTrunkENI() == "" { - for !c.checkForTrunkENI() { + for !c.checkForTrunkENI(ctx) { time.Sleep(ipPoolMonitorInterval) } } @@ -758,7 +770,7 @@ func (c *IPAMContext) StartNodeIPPoolManager() { log.Infof("IP pool manager - max pods: %d, warm IP target: %d, warm prefix target: %d, warm ENI target: %d, minimum IP target: %d", c.maxPods, c.warmIPTarget, c.warmPrefixTarget, c.warmENITarget, c.minimumIPTarget) sleepDuration := ipPoolMonitorInterval / 2 - ctx := context.Background() + for { if !c.disableENIProvisioning { time.Sleep(sleepDuration) @@ -783,10 +795,10 @@ func (c *IPAMContext) updateIPPoolIfRequired(ctx context.Context) { c.increaseDatastorePool(ctx, networkCard) } else if c.isDatastorePoolTooHigh(decisions.Stats, networkCard) && !c.shouldSkipDataStorePoolDecrease() { isScaleDownExecuted = true - c.decreaseDatastorePool(networkCard) + c.decreaseDatastorePool(ctx, networkCard) } if c.shouldRemoveExtraENIs(decisions.Stats, networkCard) { - c.tryFreeENI(networkCard) + c.tryFreeENI(ctx, networkCard) } } @@ -799,7 +811,6 @@ func (c *IPAMContext) updateIPPoolIfRequired(ctx context.Context) { } func (c *IPAMContext) shouldSkipDataStorePoolDecrease() bool { - timeSinceLastPoolDecrease := time.Since(c.lastDecreaseIPPool) if timeSinceLastPoolDecrease <= decreaseIPPoolInterval { @@ -811,19 +822,19 @@ func (c *IPAMContext) shouldSkipDataStorePoolDecrease() bool { } // decreaseDatastorePool runs every `interval` and attempts to return unused ENIs and IPs -func (c *IPAMContext) decreaseDatastorePool(networkCard int) { +func (c *IPAMContext) decreaseDatastorePool(ctx context.Context, networkCard int) { prometheusmetrics.IpamdActionsInprogress.WithLabelValues("decreaseDatastorePool").Add(float64(1)) defer prometheusmetrics.IpamdActionsInprogress.WithLabelValues("decreaseDatastorePool").Sub(float64(1)) log.Debugf("Starting to decrease pool size for network card %d", networkCard) - c.tryUnassignCidrsFromAll(networkCard) + c.tryUnassignCidrsFromAll(ctx, networkCard) log.Debugf("Successfully decreased IP pool") c.logPoolStats(c.dataStoreAccess.GetDataStore(networkCard).GetIPStats(ipV4AddrFamily), networkCard) } // tryFreeENI always tries to free one ENI -func (c *IPAMContext) tryFreeENI(networkCard int) { +func (c *IPAMContext) tryFreeENI(ctx context.Context, networkCard int) { if c.isTerminating() { log.Debug("AWS CNI is terminating, not detaching any ENIs") return @@ -847,7 +858,7 @@ func (c *IPAMContext) tryFreeENI(networkCard int) { } log.Debugf("Start freeing ENI %s from network card %d", eni, networkCard) - err := c.awsClient.FreeENI(eni) + err := c.awsClient.FreeENI(ctx, eni) if err != nil { ipamdErrInc("decreaseIPPoolFreeENIFailed") log.Errorf("Failed to free ENI %s, err: %v", eni, err) @@ -865,7 +876,7 @@ func (c *IPAMContext) tryFreeENI(networkCard int) { } // When warm IP/prefix targets are defined, free extra IPs -func (c *IPAMContext) tryUnassignCidrsFromAll(networkCard int) { +func (c *IPAMContext) tryUnassignCidrsFromAll(ctx context.Context, networkCard int) { _, over, warmIPTargetsDefined := c.datastoreTargetState(nil, networkCard) // If WARM IP targets are not defined, check if WARM_PREFIX_TARGET is defined. if !warmIPTargetsDefined { @@ -906,7 +917,7 @@ func (c *IPAMContext) tryUnassignCidrsFromAll(networkCard int) { } // Deallocate Cidrs from the instance if they are not used by pods. - c.DeallocCidrs(eniID, deletedCidrs) + c.DeallocCidrs(ctx, eniID, deletedCidrs) // reduce the deallocation target, if the deallocation target is achieved, we can exit if over = over - len(deletedCidrs); over <= 0 { @@ -956,7 +967,7 @@ func (c *IPAMContext) increaseDatastorePool(ctx context.Context, networkCard int return nil } - increasedPool, err := c.tryAssignCidrs(networkCard) + increasedPool, err := c.tryAssignCidrs(ctx, networkCard) if err != nil { if containsInsufficientCIDRsOrSubnetIPs(err) { log.Errorf("Unable to attach IPs/Prefixes for the ENI, subnet doesn't seem to have enough IPs/Prefixes. Consider using new subnet or carve a reserved range using create-subnet-cidr-reservation") @@ -985,7 +996,6 @@ func (c *IPAMContext) increaseDatastorePool(ctx context.Context, networkCard int } func (c *IPAMContext) createSecondaryIPv6ENIs(ctx context.Context) error { - log.Info("Trying to see if new ENIs need to be created") if !c.isENIAttachmentAllowed() { return nil @@ -1045,7 +1055,7 @@ func (c *IPAMContext) tryAllocateENI(ctx context.Context, networkCard int) error resourcesToAllocate := c.GetENIResourcesToAllocate(networkCard) if resourcesToAllocate > 0 { - eni, err := c.awsClient.AllocENI(securityGroups, eniCfgSubnet, resourcesToAllocate, networkCard) + eni, err := c.awsClient.AllocENI(ctx, securityGroups, eniCfgSubnet, resourcesToAllocate, networkCard) if err != nil { log.Errorf("Failed to increase pool size due to not able to allocate ENI %v", err) ipamdErrInc("increaseIPPoolAllocENI") @@ -1065,7 +1075,7 @@ func (c *IPAMContext) tryAllocateENI(ctx context.Context, networkCard int) error return err } // The CNI does not create trunk or EFA ENIs, so they will always be false here - err = c.setupENI(eni, eniMetadata, false, false) + err = c.setupENI(ctx, eni, eniMetadata, false, false) if err != nil { ipamdErrInc("increaseIPPoolsetupENIFailed") log.Errorf("Failed to increase pool size for network card %d: %v", networkCard, err) @@ -1079,18 +1089,17 @@ func (c *IPAMContext) tryAllocateENI(ctx context.Context, networkCard int) error // For an ENI, fill in missing IPs or prefixes. // PRECONDITION: isDatastorePoolTooLow returned true -func (c *IPAMContext) tryAssignCidrs(networkCard int) (increasedPool bool, err error) { +func (c *IPAMContext) tryAssignCidrs(ctx context.Context, networkCard int) (increasedPool bool, err error) { if c.enablePrefixDelegation { - return c.tryAssignPrefixes(networkCard) + return c.tryAssignPrefixes(ctx, networkCard) } else { - return c.tryAssignIPs(networkCard) + return c.tryAssignIPs(ctx, networkCard) } } // For an ENI, try to fill in missing IPs on an existing ENI. // PRECONDITION: isDatastorePoolTooLow returned true -func (c *IPAMContext) tryAssignIPs(networkCard int) (increasedPool bool, err error) { - +func (c *IPAMContext) tryAssignIPs(ctx context.Context, networkCard int) (increasedPool bool, err error) { // If WARM_IP_TARGET is set, only proceed if we are short of target short, _, warmIPTargetsDefined := c.datastoreTargetState(nil, networkCard) if warmIPTargetsDefined && short == 0 { @@ -1110,11 +1119,11 @@ func (c *IPAMContext) tryAssignIPs(networkCard int) (increasedPool bool, err err currentNumberOfAllocatedIPs := len(eni.AvailableIPv4Cidrs) // Try to allocate all available IPs for this ENI resourcesToAllocate := min((c.maxIPsPerENI - currentNumberOfAllocatedIPs), toAllocate) - output, err := c.awsClient.AllocIPAddresses(eni.ID, resourcesToAllocate) + output, err := c.awsClient.AllocIPAddresses(ctx, eni.ID, resourcesToAllocate) if err != nil && !containsPrivateIPAddressLimitExceededError(err) { log.Warnf("failed to allocate all available IP addresses on ENI %s, err: %v", eni.ID, err) // Try to just get one more IP - output, err = c.awsClient.AllocIPAddresses(eni.ID, 1) + output, err = c.awsClient.AllocIPAddresses(ctx, eni.ID, 1) if err != nil && !containsPrivateIPAddressLimitExceededError(err) { ipamdErrInc("increaseIPPoolAllocIPAddressesFailed") if c.useSubnetDiscovery && containsInsufficientCIDRsOrSubnetIPs(err) { @@ -1129,7 +1138,7 @@ func (c *IPAMContext) tryAssignIPs(networkCard int) (increasedPool bool, err err log.Debug("AssignPrivateIpAddresses returned PrivateIpAddressLimitExceeded. This can happen if the data store is out of sync." + "Returning without an error here since we will verify the actual state by calling EC2 to see what addresses have already assigned to this ENI.") // This call to EC2 is needed to verify which IPs got attached to this ENI. - ec2ip4s, err = c.awsClient.GetIPv4sFromEC2(eni.ID) + ec2ip4s, err = c.awsClient.GetIPv4sFromEC2(ctx, eni.ID) if err != nil { ipamdErrInc("increaseIPPoolGetENIaddressesFailed") return true, errors.Wrap(err, "failed to get ENI IP addresses during IP allocation") @@ -1152,28 +1161,28 @@ func (c *IPAMContext) tryAssignIPs(networkCard int) (increasedPool bool, err err return false, nil } -func (c *IPAMContext) assignIPv6Prefix(eniID string, networkCard int) (err error) { +func (c *IPAMContext) assignIPv6Prefix(ctx context.Context, eniID string, networkCard int) (err error) { log.Debugf("Assigning an IPv6Prefix for ENI: %s", eniID) - //Let's make an EC2 API call to get a list of IPv6 prefixes (if any) that are already attached to the - //current ENI. We will make this call only once during boot up/init and doing so will shield us from any - //IMDS out of sync issues. We only need one v6 prefix per ENI/Node. - ec2v6Prefixes, err := c.awsClient.GetIPv6PrefixesFromEC2(eniID) + // Let's make an EC2 API call to get a list of IPv6 prefixes (if any) that are already attached to the + // current ENI. We will make this call only once during boot up/init and doing so will shield us from any + // IMDS out of sync issues. We only need one v6 prefix per ENI/Node. + ec2v6Prefixes, err := c.awsClient.GetIPv6PrefixesFromEC2(ctx, eniID) if err != nil { log.Errorf("assignIPv6Prefix; err: %s", err) return err } log.Debugf("ENI %s has %v prefixe(s) attached", eniID, len(ec2v6Prefixes)) - //Note: If we find more than one v6 prefix attached to the ENI, VPC CNI will not attempt to free it. VPC CNI - //will only attach a single v6 prefix and it will not attempt to free the additional Prefixes. - //We will add all the prefixes to our datastore. TODO - Should we instead pick one of them. If we do, how to track - //that across restarts? + // Note: If we find more than one v6 prefix attached to the ENI, VPC CNI will not attempt to free it. VPC CNI + // will only attach a single v6 prefix and it will not attempt to free the additional Prefixes. + // We will add all the prefixes to our datastore. TODO - Should we instead pick one of them. If we do, how to track + // that across restarts? - //Check if we already have v6 Prefix(es) attached + // Check if we already have v6 Prefix(es) attached if len(ec2v6Prefixes) == 0 { - //Allocate and attach a v6 Prefix to Primary ENI + // Allocate and attach a v6 Prefix to Primary ENI log.Debugf("No IPv6 Prefix(es) found for ENI: %s", eniID) - strPrefixes, err := c.awsClient.AllocIPv6Prefixes(eniID) + strPrefixes, err := c.awsClient.AllocIPv6Prefixes(ctx, eniID) if err != nil { return err } @@ -1182,9 +1191,9 @@ func (c *IPAMContext) assignIPv6Prefix(eniID string, networkCard int) (err error } log.Debugf("Successfully allocated an IPv6Prefix for ENI: %s", eniID) } else if len(ec2v6Prefixes) > 1 { - //Found more than one v6 prefix attached to the ENI. VPC CNI will only attach a single v6 prefix - //and it will not attempt to free any additional Prefixes that are already attached. - //Will use the first IPv6 Prefix attached for IP address allocation. + // Found more than one v6 prefix attached to the ENI. VPC CNI will only attach a single v6 prefix + // and it will not attempt to free any additional Prefixes that are already attached. + // Will use the first IPv6 Prefix attached for IP address allocation. ec2v6Prefixes = []ec2types.Ipv6PrefixSpecification{ec2v6Prefixes[0]} } c.addENIv6prefixesToDataStore(ec2v6Prefixes, eniID, networkCard) @@ -1192,7 +1201,7 @@ func (c *IPAMContext) assignIPv6Prefix(eniID string, networkCard int) (err error } // PRECONDITION: isDatastorePoolTooLow returned true -func (c *IPAMContext) tryAssignPrefixes(networkCard int) (increasedPool bool, err error) { +func (c *IPAMContext) tryAssignPrefixes(ctx context.Context, networkCard int) (increasedPool bool, err error) { toAllocate := c.getPrefixesNeeded(networkCard) // Returns an ENI which has space for more prefixes to be attached, but this // ENI might not suffice the WARM_IP_TARGET/WARM_PREFIX_TARGET @@ -1200,11 +1209,11 @@ func (c *IPAMContext) tryAssignPrefixes(networkCard int) (increasedPool bool, er for _, eni := range enis { currentNumberOfAllocatedPrefixes := len(eni.AvailableIPv4Cidrs) resourcesToAllocate := min((c.maxPrefixesPerENI - currentNumberOfAllocatedPrefixes), toAllocate) - output, err := c.awsClient.AllocIPAddresses(eni.ID, resourcesToAllocate) + output, err := c.awsClient.AllocIPAddresses(ctx, eni.ID, resourcesToAllocate) if err != nil && !containsPrivateIPAddressLimitExceededError(err) { log.Warnf("failed to allocate all available IPv4 Prefixes on ENI %s, err: %v", eni.ID, err) // Try to just get one more prefix - output, err = c.awsClient.AllocIPAddresses(eni.ID, 1) + output, err = c.awsClient.AllocIPAddresses(ctx, eni.ID, 1) if err != nil && !containsPrivateIPAddressLimitExceededError(err) { ipamdErrInc("increaseIPPoolAllocIPAddressesFailed") if c.useSubnetDiscovery && containsInsufficientCIDRsOrSubnetIPs(err) { @@ -1219,7 +1228,7 @@ func (c *IPAMContext) tryAssignPrefixes(networkCard int) (increasedPool bool, er log.Debug("AssignPrivateIpAddresses returned PrivateIpAddressLimitExceeded. This can happen if the data store is out of sync." + "Returning without an error here since we will verify the actual state by calling EC2 to see what addresses have already assigned to this ENI.") // This call to EC2 is needed to verify which IPs got attached to this ENI. - ec2Prefixes, err = c.awsClient.GetIPv4PrefixesFromEC2(eni.ID) + ec2Prefixes, err = c.awsClient.GetIPv4PrefixesFromEC2(ctx, eni.ID) if err != nil { ipamdErrInc("increaseIPPoolGetENIaddressesFailed") return true, errors.Wrap(err, "failed to get ENI IP addresses during IP allocation") @@ -1242,7 +1251,7 @@ func (c *IPAMContext) tryAssignPrefixes(networkCard int) (increasedPool bool, er // 2) add ENI to datastore // 3) set up linux ENI related networking stack. // 4) add all ENI's secondary IP addresses to datastore -func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isTrunkENI, isEFAENI bool) error { +func (c *IPAMContext) setupENI(ctx context.Context, eni string, eniMetadata awsutils.ENIMetadata, isTrunkENI, isEFAENI bool) error { primaryENI := c.awsClient.GetPrimaryENI() var primaryIP string if c.enableIPv6 { @@ -1262,14 +1271,43 @@ func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isT if err != nil && err.Error() != datastore.DuplicatedENIError { return errors.Wrapf(err, "failed to add ENI %s to data store", eni) } - // Keep track of the primary IP for this ENI - c.primaryIP[eni] = primaryIP + + // Check if this ENI (primary or secondary) is in an excluded subnet and mark it for exclusion + if c.useSubnetDiscovery { + subnetID, err := c.awsClient.GetENISubnetID(ctx, eni) + if err != nil { + log.Warnf("setupENI: failed to get subnet ID for ENI %s: %v", eni, err) + } else { + excluded, err := c.awsClient.IsSubnetExcluded(ctx, subnetID) + if err != nil { + log.Warnf("setupENI: failed to check subnet exclusion for ENI %s (subnet %s): %v", eni, subnetID, err) + } else if excluded { + primaryENI := c.awsClient.GetPrimaryENI() + eniType := "secondary" + if eni == primaryENI { + eniType = "primary" + } + log.Infof("Marking %s ENI %s as excluded from pod IP allocation (in excluded subnet %s)", eniType, eni, subnetID) + err := c.dataStoreAccess.GetDataStore(eniMetadata.NetworkCard).SetENIExcludedForPodIPs(eni, true) + if err != nil { + log.Warnf("Failed to mark %s ENI %s as excluded: %v", eniType, eni, err) + } + } + } + } + + // Store the addressable IP for the ENI + if c.enableIPv6 { + c.primaryIP[eni] = eniMetadata.PrimaryIPv6Address() + } else { + c.primaryIP[eni] = eniMetadata.PrimaryIPv4Address() + } if c.enableIPv6 { if !isTrunkENI { // In v6 PD mode, VPC CNI will manage the primary ENI, ENIs on NC > 0 and trunk ENI. Once we start supporting secondary // IP and custom networking modes for IPv6, this restriction can be relaxed. - err := c.assignIPv6Prefix(eni, eniMetadata.NetworkCard) + err := c.assignIPv6Prefix(ctx, eni, eniMetadata.NetworkCard) if err != nil { return errors.Wrapf(err, "Failed to allocate IPv6 Prefixes to ENI") } @@ -1329,7 +1367,7 @@ func (c *IPAMContext) addENIv4prefixesToDataStore(ec2PrefixAddrs []ec2types.Ipv4 strIpv4Prefix := aws.ToString(ec2PrefixAddr.Ipv4Prefix) _, ipnet, err := net.ParseCIDR(strIpv4Prefix) if err != nil { - //Parsing failed, get next prefix + // Parsing failed, get next prefix log.Debugf("Parsing failed, moving on to next prefix") continue } @@ -1364,7 +1402,6 @@ func (c *IPAMContext) addENIv6prefixesToDataStore(ec2PrefixAddrs []ec2types.Ipv6 } c.logPoolStats(c.dataStoreAccess.GetDataStore(networkCard).GetIPStats(ipV6AddrFamily), networkCard) } - } // getMaxENI returns the maximum number of ENIs to attach to this instance. This is calculated as the lesser of @@ -1432,7 +1469,6 @@ func (c *IPAMContext) logPoolStats(dataStoreStats *datastore.DataStoreStats, net } func (c *IPAMContext) tryEnableSecurityGroupsForPods(ctx context.Context) { - // For IPv4, check that there is room for a trunk ENI before patching CNINode CRD. We only check on the Default Network Card if c.enableIPv4 && (c.dataStoreAccess.GetDataStore(DefaultNetworkCardIndex).GetENIs() >= (c.maxENI - c.unmanagedENI[DefaultNetworkCardIndex])) { log.Error("No slot available for a trunk ENI to be attached.") @@ -1504,8 +1540,8 @@ func podENIErrInc(fn string) { } // Used in IPv6 mode to check if trunk ENI has been successfully attached -func (c *IPAMContext) checkForTrunkENI() bool { - metadataResult, err := c.awsClient.DescribeAllENIs() +func (c *IPAMContext) checkForTrunkENI(ctx context.Context) bool { + metadataResult, err := c.awsClient.DescribeAllENIs(ctx) if err != nil { log.Debug("failed to describe attached ENIs") return false @@ -1513,7 +1549,7 @@ func (c *IPAMContext) checkForTrunkENI() bool { if metadataResult.TrunkENI != "" { for _, eni := range metadataResult.ENIMetadata { if eni.ENIID == metadataResult.TrunkENI { - if err := c.setupENI(eni.ENIID, eni, true, false); err == nil { + if err := c.setupENI(ctx, eni.ENIID, eni, true, false); err == nil { log.Infof("ENI %s set up", eni.ENIID) return true } else { @@ -1527,7 +1563,7 @@ func (c *IPAMContext) checkForTrunkENI() bool { } func (c *IPAMContext) getENIsByNetworkCard(allENIs []awsutils.ENIMetadata) map[int][]awsutils.ENIMetadata { - var eniNetworkCardMap = make(map[int][]awsutils.ENIMetadata, c.numNetworkCards) + eniNetworkCardMap := make(map[int][]awsutils.ENIMetadata, c.numNetworkCards) for _, eni := range allENIs { eniNetworkCardMap[eni.NetworkCard] = append(eniNetworkCardMap[eni.NetworkCard], eni) } @@ -1587,7 +1623,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Dur log.Debugf("A new ENI added but not by ipamd, updating tags by calling EC2") // Call describeENIs only once per nodeIPPoolReconcile by checking if metadataResult.ENIMetadata is nil if len(metadataResult.ENIMetadata) == 0 { - metadataResult, err = c.awsClient.DescribeAllENIs() + metadataResult, err = c.awsClient.DescribeAllENIs(ctx) if err != nil { log.Warnf("Failed to call EC2 to describe ENIs, aborting reconcile: %v", err) return @@ -1617,11 +1653,11 @@ func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Dur // If the attached ENI is in the data store log.Debugf("Reconcile existing ENI %s IP pool", attachedENI.ENIID) // Reconcile IP pool - c.eniIPPoolReconcile(eniIPPool, attachedENI, attachedENI.ENIID, networkCard) + c.eniIPPoolReconcile(ctx, eniIPPool, attachedENI, attachedENI.ENIID, networkCard) // If the attached ENI is in the data store log.Debugf("Reconcile existing ENI %s IP prefixes", attachedENI.ENIID) // Reconcile IP pool - c.eniPrefixPoolReconcile(eniPrefixPool, attachedENI, attachedENI.ENIID, networkCard) + c.eniPrefixPoolReconcile(ctx, eniPrefixPool, attachedENI, attachedENI.ENIID, networkCard) // Mark action, remove this ENI from currentENIs map delete(currentENIs, attachedENI.ENIID) continue @@ -1630,7 +1666,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Dur isTrunkENI := attachedENI.ENIID == trunkENI isEFAENI := efaENIs[attachedENI.ENIID] if !isTrunkENI && !c.disableENIProvisioning { - if err := c.awsClient.TagENI(attachedENI.ENIID, eniTagMap[attachedENI.ENIID]); err != nil { + if err := c.awsClient.TagENI(ctx, attachedENI.ENIID, eniTagMap[attachedENI.ENIID]); err != nil { log.Errorf("IP pool reconcile: failed to tag managed ENI %v: %v", attachedENI.ENIID, err) ipamdErrInc("eniReconcileAdd") continue @@ -1639,7 +1675,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Dur // Add new ENI log.Debugf("Reconcile and add a new ENI %s", attachedENI.ENIID) - err = c.setupENI(attachedENI.ENIID, attachedENI, isTrunkENI, isEFAENI) + err = c.setupENI(ctx, attachedENI.ENIID, attachedENI, isTrunkENI, isEFAENI) if err != nil { log.Errorf("IP pool reconcile: Failed to set up ENI %s network: %v", attachedENI.ENIID, err) ipamdErrInc("eniReconcileAdd") @@ -1673,10 +1709,9 @@ func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Dur for eni, primaryIP := range c.primaryIP { log.Debugf("Primary IP for ENI %s is %s", eni, primaryIP) } - } -func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.ENIMetadata, eni string, networkCard int) { +func (c *IPAMContext) eniIPPoolReconcile(ctx context.Context, ipPool []string, attachedENI awsutils.ENIMetadata, eni string, networkCard int) { attachedENIIPs := attachedENI.IPv4Addresses needEC2Reconcile := true // Here we can't trust attachedENI since the IMDS metadata can be stale. We need to check with EC2 API. @@ -1685,7 +1720,7 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.E log.Warnf("Instance metadata does not match data store! ipPool: %v, metadata: %v", ipPool, attachedENIIPs) log.Debugf("We need to check the ENI status by calling the EC2 control plane.") // Call EC2 to verify IPs on this ENI - ec2Addresses, err := c.awsClient.GetIPv4sFromEC2(eni) + ec2Addresses, err := c.awsClient.GetIPv4sFromEC2(ctx, eni) if err != nil { log.Errorf("Failed to fetch ENI IP addresses! Aborting reconcile of ENI %s", eni) return @@ -1695,7 +1730,7 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.E } // Add all known attached IPs to the datastore - seenIPs := c.verifyAndAddIPsToDatastore(eni, attachedENIIPs, needEC2Reconcile, networkCard) + seenIPs := c.verifyAndAddIPsToDatastore(ctx, eni, attachedENIIPs, needEC2Reconcile, networkCard) // Sweep phase, delete remaining IPs since they should not remain in the datastore for _, existingIP := range ipPool { @@ -1717,7 +1752,7 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.E } } -func (c *IPAMContext) eniPrefixPoolReconcile(prefixPool []string, attachedENI awsutils.ENIMetadata, eni string, networkCard int) { +func (c *IPAMContext) eniPrefixPoolReconcile(ctx context.Context, prefixPool []string, attachedENI awsutils.ENIMetadata, eni string, networkCard int) { attachedENIIPs := attachedENI.IPv4Prefixes needEC2Reconcile := true // Here we can't trust attachedENI since the IMDS metadata can be stale. We need to check with EC2 API. @@ -1727,7 +1762,7 @@ func (c *IPAMContext) eniPrefixPoolReconcile(prefixPool []string, attachedENI aw log.Warnf("Instance metadata does not match data store! ipPool: %v, metadata: %v", prefixPool, attachedENIIPs) log.Debugf("We need to check the ENI status by calling the EC2 control plane.") // Call EC2 to verify IPs on this ENI - ec2Addresses, err := c.awsClient.GetIPv4PrefixesFromEC2(eni) + ec2Addresses, err := c.awsClient.GetIPv4PrefixesFromEC2(ctx, eni) if err != nil { log.Errorf("Failed to fetch ENI IP addresses! Aborting reconcile of ENI %s", eni) return @@ -1737,7 +1772,7 @@ func (c *IPAMContext) eniPrefixPoolReconcile(prefixPool []string, attachedENI aw } // Add all known attached IPs to the datastore - seenIPs := c.verifyAndAddPrefixesToDatastore(eni, attachedENIIPs, needEC2Reconcile, networkCard) + seenIPs := c.verifyAndAddPrefixesToDatastore(ctx, eni, attachedENIIPs, needEC2Reconcile, networkCard) // Sweep phase, delete remaining Prefixes since they should not remain in the datastore for _, existingIP := range prefixPool { @@ -1765,7 +1800,7 @@ func (c *IPAMContext) eniPrefixPoolReconcile(prefixPool []string, attachedENI aw // verifyAndAddIPsToDatastore updates the datastore with the known secondary IPs. IPs who are out of cooldown gets added // back to the datastore after being verified against EC2. -func (c *IPAMContext) verifyAndAddIPsToDatastore(eni string, attachedENIIPs []ec2types.NetworkInterfacePrivateIpAddress, needEC2Reconcile bool, networkCard int) map[string]bool { +func (c *IPAMContext) verifyAndAddIPsToDatastore(ctx context.Context, eni string, attachedENIIPs []ec2types.NetworkInterfacePrivateIpAddress, needEC2Reconcile bool, networkCard int) map[string]bool { var ec2VerifiedAddresses []ec2types.NetworkInterfacePrivateIpAddress seenIPs := make(map[string]bool) for _, privateIPv4 := range attachedENIIPs { @@ -1790,7 +1825,7 @@ func (c *IPAMContext) verifyAndAddIPsToDatastore(eni string, attachedENIIPs []ec if ec2VerifiedAddresses == nil { var err error // Call EC2 to verify IPs on this ENI - ec2VerifiedAddresses, err = c.awsClient.GetIPv4sFromEC2(eni) + ec2VerifiedAddresses, err = c.awsClient.GetIPv4sFromEC2(ctx, eni) if err != nil { log.Errorf("Failed to fetch ENI IP addresses from EC2! %v", err) // Do not delete this IP from the datastore or cooldown until we have confirmed with EC2 @@ -1836,7 +1871,7 @@ func (c *IPAMContext) verifyAndAddIPsToDatastore(eni string, attachedENIIPs []ec // verifyAndAddPrefixesToDatastore updates the datastore with the known Prefixes. Prefixes who are out of cooldown gets added // back to the datastore after being verified against EC2. -func (c *IPAMContext) verifyAndAddPrefixesToDatastore(eni string, attachedENIPrefixes []ec2types.Ipv4PrefixSpecification, needEC2Reconcile bool, networkCard int) map[string]bool { +func (c *IPAMContext) verifyAndAddPrefixesToDatastore(ctx context.Context, eni string, attachedENIPrefixes []ec2types.Ipv4PrefixSpecification, needEC2Reconcile bool, networkCard int) map[string]bool { var ec2VerifiedAddresses []ec2types.Ipv4PrefixSpecification seenIPs := make(map[string]bool) for _, privateIPv4Cidr := range attachedENIPrefixes { @@ -1863,7 +1898,7 @@ func (c *IPAMContext) verifyAndAddPrefixesToDatastore(eni string, attachedENIPre if ec2VerifiedAddresses == nil { var err error // Call EC2 to verify Prefixes on this ENI - ec2VerifiedAddresses, err = c.awsClient.GetIPv4PrefixesFromEC2(eni) + ec2VerifiedAddresses, err = c.awsClient.GetIPv4PrefixesFromEC2(ctx, eni) if err != nil { log.Errorf("Failed to fetch ENI IP addresses from EC2! %v", err) // Do not delete this Prefix from the datastore or cooldown until we have confirmed with EC2 @@ -2087,10 +2122,10 @@ func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutil ret := make([]awsutils.ENIMetadata, 0, len(enis)) for _, eni := range enis { - //Filter out any Unmanaged ENIs. VPC CNI will only work with Primary ENI in IPv6 Prefix Delegation mode until - //we open up IPv6 support in Secondary IP and Custom networking modes. Filtering out the ENIs here will - //help us avoid myriad of if/else loops elsewhere in the code. - //We shouldn't need the IsPrimaryENI check as ENIs not created by vpc-cni will be marked unmanaged (including trunk ENI) + // Filter out any Unmanaged ENIs. VPC CNI will only work with Primary ENI in IPv6 Prefix Delegation mode until + // we open up IPv6 support in Secondary IP and Custom networking modes. Filtering out the ENIs here will + // help us avoid myriad of if/else loops elsewhere in the code. + // We shouldn't need the IsPrimaryENI check as ENIs not created by vpc-cni will be marked unmanaged (including trunk ENI) isUnmanagedENI := c.awsClient.IsUnmanagedENI(eni.ENIID) isUnmanagedNIC := c.awsClient.IsUnmanagedNIC(eni.NetworkCard) isEfaOnlyENI := c.awsClient.IsEfaOnlyENI(eni.NetworkCard, eni.ENIID) @@ -2137,7 +2172,6 @@ func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutil // datastoreTargetState determines the number of IPs `short` or `over` our WARM_IP_TARGET, accounting for the MINIMUM_IP_TARGET. // With prefix delegation, this function determines the number of Prefixes `short` or `over` func (c *IPAMContext) datastoreTargetState(stats *datastore.DataStoreStats, networkCard int) (short int, over int, enabled bool) { - warmIPTarget := c.warmIPTarget minimumIPTarget := c.minimumIPTarget @@ -2361,21 +2395,19 @@ func (c *IPAMContext) AnnotatePod(podName string, podNamespace string, key strin return err } -func (c *IPAMContext) tryUnassignIPsFromENIs() { +func (c *IPAMContext) tryUnassignIPsFromENIs(ctx context.Context) { log.Debugf("tryUnassignIPsFromENIs") // From all datastores, get ENIInfos and unassign IPs for _, ds := range c.dataStoreAccess.DataStores { networkCard := ds.GetNetworkCard() eniInfos := ds.GetENIInfos() for eniID := range eniInfos.ENIs { - c.tryUnassignIPFromENI(eniID, networkCard) + c.tryUnassignIPFromENI(ctx, eniID, networkCard) } } - } -func (c *IPAMContext) tryUnassignIPFromENI(eniID string, networkCard int) { - +func (c *IPAMContext) tryUnassignIPFromENI(ctx context.Context, eniID string, networkCard int) { ds := c.dataStoreAccess.GetDataStore(networkCard) freeableIPs := ds.FreeableIPs(eniID) if len(freeableIPs) == 0 { @@ -2399,14 +2431,14 @@ func (c *IPAMContext) tryUnassignIPFromENI(eniID string, networkCard int) { } // Deallocate IPs from the instance if they aren't used by pods. - if err := c.awsClient.DeallocIPAddresses(eniID, deletedIPs); err != nil { + if err := c.awsClient.DeallocIPAddresses(ctx, eniID, deletedIPs); err != nil { log.Warnf("Failed to decrease IP pool by removing IPs %v from ENI %s: %s", deletedIPs, eniID, err) } else { log.Debugf("Successfully decreased IP pool by removing IPs %v from ENI %s", deletedIPs, eniID) } } -func (c *IPAMContext) tryUnassignPrefixesFromENIs() { +func (c *IPAMContext) tryUnassignPrefixesFromENIs(ctx context.Context) { log.Debugf("tryUnassignPrefixesFromENIs") // From all datastores get ENIs and remove prefixes @@ -2414,12 +2446,12 @@ func (c *IPAMContext) tryUnassignPrefixesFromENIs() { networkCard := ds.GetNetworkCard() eniInfos := ds.GetENIInfos() for eniID := range eniInfos.ENIs { - c.tryUnassignPrefixFromENI(eniID, networkCard) + c.tryUnassignPrefixFromENI(ctx, eniID, networkCard) } } } -func (c *IPAMContext) tryUnassignPrefixFromENI(eniID string, networkCard int) { +func (c *IPAMContext) tryUnassignPrefixFromENI(ctx context.Context, eniID string, networkCard int) { ds := c.dataStoreAccess.GetDataStore(networkCard) freeablePrefixes := ds.FreeablePrefixes(eniID) if len(freeablePrefixes) == 0 { @@ -2430,7 +2462,17 @@ func (c *IPAMContext) tryUnassignPrefixFromENI(eniID string, networkCard int) { for _, toDelete := range freeablePrefixes { // Don't force the delete, since a freeable Prefix might have been assigned to a pod // before we get around to deleting it. - err := ds.DelIPv4CidrFromStore(eniID, toDelete, false /* force */) + var err error + + // Determine if this is an IPv4 or IPv6 CIDR and call the appropriate deletion function + if toDelete.IP.To4() != nil { + // IPv4 CIDR + err = ds.DelIPv4CidrFromStore(eniID, toDelete, false /* force */) + } else { + // IPv6 CIDR + err = ds.DelIPv6CidrFromStore(eniID, toDelete, false /* force */) + } + if err != nil { log.Warnf("Failed to delete Prefix %s on ENI %s from datastore: %s", toDelete, eniID, err) ipamdErrInc("decreaseIPPool") @@ -2440,12 +2482,69 @@ func (c *IPAMContext) tryUnassignPrefixFromENI(eniID string, networkCard int) { } } - // Deallocate IPs from the instance if they aren't used by pods. - if err := c.awsClient.DeallocPrefixAddresses(eniID, deletedPrefixes); err != nil { + // Deallocate prefixes from the instance if they aren't used by pods. + // DeallocPrefixAddresses handles both IPv4 and IPv6 prefixes + if err := c.awsClient.DeallocPrefixAddresses(ctx, eniID, deletedPrefixes); err != nil { log.Warnf("Failed to delete prefix %v from ENI %s: %s", deletedPrefixes, eniID, err) } else { - log.Debugf("Successfully prefix removing IPs %v from ENI %s", deletedPrefixes, eniID) + log.Debugf("Successfully removed prefixes %v from ENI %s", deletedPrefixes, eniID) + } +} + +// cleanupExcludedENI cleans up unassigned secondary IPs and prefixes from excluded ENI +func (c *IPAMContext) cleanupExcludedENI(ctx context.Context, eniID string) { + log.Debugf("cleanupExcludedENI: cleaning up resources from excluded ENI %s", eniID) + + // Clean up based on the current IP mode + if c.enableIPv4 { + if c.enablePrefixDelegation { + // In prefix delegation mode, cleanup unassigned IPv4 prefixes + log.Debugf("Cleaning up unassigned IPv4 prefixes from excluded ENI %s", eniID) + c.tryUnassignPrefixFromENI(ctx, eniID, DefaultNetworkCardIndex) + } else { + // In secondary IP mode, cleanup unassigned IPv4 secondary IPs + log.Debugf("Cleaning up unassigned IPv4 secondary IPs from excluded ENI %s", eniID) + c.tryUnassignIPFromENI(ctx, eniID, DefaultNetworkCardIndex) + } } + + // Clean up IPv6 prefixes if IPv6 is enabled (IPv6 only uses prefix delegation) + if c.enableIPv6 { + log.Debugf("Cleaning up unassigned IPv6 prefixes from excluded ENI %s", eniID) + c.tryUnassignPrefixFromENI(ctx, eniID, DefaultNetworkCardIndex) + } + + log.Infof("Completed cleanup of unassigned resources from excluded ENI %s", eniID) +} + +// checkAndHandleAllENIExclusion checks all existing ENIs (primary and secondary) across all network cards for subnet exclusion +func (c *IPAMContext) checkAndHandleAllENIExclusion(ctx context.Context) { + log.Debugf("checkAndHandleAllENIExclusion: checking all existing ENIs for subnet exclusion") + + // Iterate over all datastores (one per network card) + for _, ds := range c.dataStoreAccess.DataStores { + // Get ENI information using the public method + eniInfos := ds.GetENIInfos() + for eniID, eni := range eniInfos.ENIs { + + // Only process ENIs that are already marked as excluded + if !eni.IsExcludedForPodIPs { + continue + } + + eniType := "secondary" + if eni.IsPrimary { + eniType = "primary" + } + + log.Infof("checkAndHandleAllENIExclusion: cleaning up unassigned resources from excluded %s ENI %s", eniType, eniID) + + // Clean up unassigned resources from this excluded ENI + c.cleanupExcludedENI(ctx, eniID) + } + } + + log.Debugf("checkAndHandleAllENIExclusion: completed ENI exclusion check") } func (c *IPAMContext) GetENIResourcesToAllocate(networkCard int) int { @@ -2473,8 +2572,8 @@ func (c *IPAMContext) GetIPv4Limit() (int, int, error) { maxIPsPerENI = c.awsClient.GetENIIPv4Limit() maxPrefixesPerENI = 0 } else if c.enablePrefixDelegation { - //Single PD - allocate one prefix per ENI and new add will be new ENI + prefix - //Multi - allocate one prefix per ENI and new add will be new prefix or new ENI + prefix + // Single PD - allocate one prefix per ENI and new add will be new ENI + prefix + // Multi - allocate one prefix per ENI and new add will be new prefix or new ENI + prefix _, maxIpsPerPrefix, _ = datastore.GetPrefixDelegationDefaults() maxPrefixesPerENI = c.awsClient.GetENIIPv4Limit() maxIPsPerENI = maxPrefixesPerENI * maxIpsPerPrefix @@ -2494,11 +2593,33 @@ func (c *IPAMContext) hasRoomForEni(networkCard int) bool { if c.enablePodENI && networkCard == DefaultNetworkCardIndex && c.dataStoreAccess.GetDataStore(DefaultNetworkCardIndex).GetTrunkENI() == "" { trunkEni = 1 } - return c.dataStoreAccess.GetDataStore(networkCard).GetENIs() < (c.maxENI - c.unmanagedENI[networkCard] - trunkEni) + // Count excluded ENIs to account for them in our calculations + excludedENI := 0 + eniInfos := c.dataStoreAccess.GetDataStore(networkCard).GetENIInfos() + for _, eni := range eniInfos.ENIs { + if eni.IsExcludedForPodIPs { + excludedENI++ + } + } + if excludedENI > 0 { + log.Debugf("Accounting for %d excluded ENIs in hasRoomForEni calculation for network card %d", excludedENI, networkCard) + } + + currentENIs := c.dataStoreAccess.GetDataStore(networkCard).GetENIs() + // Subtract excluded ENIs from current count to match the exclusion from max count + currentUsableENIs := currentENIs - excludedENI + maxUsableENIs := c.maxENI - c.unmanagedENI[networkCard] - trunkEni - excludedENI + + // Check if we have room considering the excluded ENI + hasRoom := currentUsableENIs < maxUsableENIs + + log.Debugf("hasRoomForEni: networkCard=%d, currentENIs=%d, currentUsableENIs=%d, maxENI=%d, unmanagedENI=%d, trunkEni=%d, excludedENI=%d, hasRoom=%v", + networkCard, currentENIs, currentUsableENIs, c.maxENI, c.unmanagedENI[networkCard], trunkEni, excludedENI, hasRoom) + + return hasRoom } func (c *IPAMContext) isDatastorePoolTooLow() map[int]Decisions { - decisions := make(map[int]Decisions, len(c.dataStoreAccess.DataStores)) warmTarget := c.warmENITarget totalIPs := c.maxIPsPerENI @@ -2559,7 +2680,7 @@ func (c *IPAMContext) warmPrefixTargetDefined() bool { } // DeallocCidrs frees IPs and Prefixes from EC2 -func (c *IPAMContext) DeallocCidrs(eniID string, deletableCidrs []datastore.CidrInfo) { +func (c *IPAMContext) DeallocCidrs(ctx context.Context, eniID string, deletableCidrs []datastore.CidrInfo) { var deletableIPs []string var deletablePrefixes []string @@ -2579,11 +2700,11 @@ func (c *IPAMContext) DeallocCidrs(eniID string, deletableCidrs []datastore.Cidr } } - if err := c.awsClient.DeallocPrefixAddresses(eniID, deletablePrefixes); err != nil { + if err := c.awsClient.DeallocPrefixAddresses(ctx, eniID, deletablePrefixes); err != nil { log.Warnf("Failed to free Prefixes %v from ENI %s: %s", deletablePrefixes, eniID, err) } - if err := c.awsClient.DeallocIPAddresses(eniID, deletableIPs); err != nil { + if err := c.awsClient.DeallocIPAddresses(ctx, eniID, deletableIPs); err != nil { log.Warnf("Failed to free IPs %v from ENI %s: %s", deletableIPs, eniID, err) } } @@ -2609,7 +2730,6 @@ func (c *IPAMContext) getPrefixesNeeded(networkCard int) int { } func (c *IPAMContext) initENIAndIPLimits() (err error) { - nodeMaxENI, err := c.getMaxENI() if err != nil { log.Error("Failed to get ENI limit") diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index 6e577690c9..026114cfbb 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -29,6 +29,7 @@ import ( "github.com/aws/smithy-go" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/golang/mock/gomock" "github.com/samber/lo" @@ -52,6 +53,7 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils" mock_networkutils "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils/mocks" + "github.com/aws/amazon-vpc-cni-k8s/pkg/vpc" "github.com/aws/amazon-vpc-cni-k8s/utils/prometheusmetrics" rcscheme "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1" "github.com/prometheus/client_golang/prometheus" @@ -91,6 +93,9 @@ const ( externalEniConfigLabel = "vpc.amazonaws.com/externalEniConfig" defaultNetworkCard = 0 maxENIPerNIC = 4 + primaryENI = primaryENIid + secENI = secENIid + primaryIP = ipaddr01 ) type testMocks struct { @@ -149,20 +154,24 @@ func TestNodeInit(t *testing.T) { enableMultiNICSupport: false, withApiServer: true, } + mockContext.unmanagedENI = make([]int, mockContext.numNetworkCards) eni1, eni2, _ := getDummyENIMetadata() var cidrs []string m.awsutils.EXPECT().GetENILimit().Return(4) m.awsutils.EXPECT().GetENIIPv4Limit().Return(14) - m.awsutils.EXPECT().GetIPv4sFromEC2(eni1.ENIID).AnyTimes().Return(eni1.IPv4Addresses, nil) - m.awsutils.EXPECT().GetIPv4sFromEC2(eni2.ENIID).AnyTimes().Return(eni2.IPv4Addresses, nil) + m.awsutils.EXPECT().GetNetworkCards().Return([]vpc.NetworkCard{{NetworkCardIndex: 0, MaximumNetworkInterfaces: 4}}).AnyTimes() + m.awsutils.EXPECT().SetUnmanagedNetworkCards(gomock.Any()).AnyTimes() + m.awsutils.EXPECT().IsUnmanagedNIC(gomock.Any()).Return(false).AnyTimes() + m.awsutils.EXPECT().IsEfaOnlyENI(gomock.Any(), gomock.Any()).Return(false).AnyTimes() + m.awsutils.EXPECT().GetIPv4sFromEC2(gomock.Any(), eni1.ENIID).AnyTimes().Return(eni1.IPv4Addresses, nil) + m.awsutils.EXPECT().GetIPv4sFromEC2(gomock.Any(), eni2.ENIID).AnyTimes().Return(eni2.IPv4Addresses, nil) m.awsutils.EXPECT().IsUnmanagedENI(eni1.ENIID).Return(false).AnyTimes() m.awsutils.EXPECT().IsUnmanagedENI(eni2.ENIID).Return(false).AnyTimes() - m.awsutils.EXPECT().IsUnmanagedNIC(eni1.NetworkCard).Return(false).AnyTimes() m.awsutils.EXPECT().IsUnmanagedNIC(eni2.NetworkCard).Return(false).AnyTimes() m.awsutils.EXPECT().IsEfaOnlyENI(gomock.Any(), gomock.Any()).Return(false).AnyTimes() - m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() m.network.EXPECT().GetRouteTableNumberForENI(eni1.NetworkCard, gomock.Any(), eni1.DeviceNumber, gomock.Any(), false).Times(1) m.network.EXPECT().GetRouteTableNumberForENI(eni2.NetworkCard, gomock.Any(), eni2.DeviceNumber, gomock.Any(), false).Times(1) @@ -172,8 +181,9 @@ func TestNodeInit(t *testing.T) { m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false, false).Return(nil) m.network.EXPECT().CleanUpStaleAWSChains(true, false).Return(nil) m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) - m.awsutils.EXPECT().RefreshSGIDs(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + m.awsutils.EXPECT().RefreshSGIDs(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) m.awsutils.EXPECT().SetUnmanagedNetworkCards(gomock.Any()).AnyTimes() + m.awsutils.EXPECT().RefreshCustomSGIDs(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) eniMetadataSlice := []awsutils.ENIMetadata{eni1, eni2} resp := awsutils.DescribeAllENIsResult{ ENIMetadata: eniMetadataSlice, @@ -184,17 +194,17 @@ func TestNodeInit(t *testing.T) { ENIsByNetworkCard: [][]string{defaultNetworkCard: {eni1.ENIID, eni2.ENIID}}, } - m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) + m.awsutils.EXPECT().DescribeAllENIs(gomock.Any()).Return(resp, nil) m.awsutils.EXPECT().SetEFAOnlyENIs(resp.EFAOnlyENIByNetworkCard).Times(1) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()).AnyTimes() - m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()) - m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP).AnyTimes() + m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP).AnyTimes().AnyTimes() var rules []netlink.Rule - m.network.EXPECT().GetRuleList(false).Return(rules, nil) - m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any()) - m.network.EXPECT().GetExternalServiceCIDRs().Return(nil) - m.network.EXPECT().UpdateExternalServiceIpRules(gomock.Any(), gomock.Any()) + m.network.EXPECT().GetRuleList(false).Return(rules, nil).AnyTimes() + m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any()).AnyTimes() + m.network.EXPECT().GetExternalServiceCIDRs().Return(nil).AnyTimes() + m.network.EXPECT().UpdateExternalServiceIpRules(gomock.Any(), gomock.Any()).AnyTimes() maxPods, _ := resource.ParseQuantity("500") fakeNode := v1.Node{ @@ -210,7 +220,7 @@ func TestNodeInit(t *testing.T) { m.k8sClient.Create(ctx, &fakeNode) // Add IPs - m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), gomock.Any()) + m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), gomock.Any(), gomock.Any()) os.Setenv("MY_NODE_NAME", myNodeName) err := mockContext.nodeInit() assert.NoError(t, err) @@ -254,13 +264,16 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) { var cidrs []string m.awsutils.EXPECT().GetENILimit().Return(4) m.awsutils.EXPECT().GetENIIPv4Limit().Return(14) - m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(eni1.ENIID).AnyTimes().Return(eni1.IPv4Prefixes, nil) - m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(eni2.ENIID).AnyTimes().Return(eni2.IPv4Prefixes, nil) + m.awsutils.EXPECT().GetNetworkCards().Return([]vpc.NetworkCard{{NetworkCardIndex: 0, MaximumNetworkInterfaces: 4}}).AnyTimes() + m.awsutils.EXPECT().SetUnmanagedNetworkCards(gomock.Any()).AnyTimes() + m.awsutils.EXPECT().IsUnmanagedNIC(gomock.Any()).Return(false).AnyTimes() + m.awsutils.EXPECT().IsEfaOnlyENI(gomock.Any(), gomock.Any()).Return(false).AnyTimes() + m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(gomock.Any(), eni1.ENIID).AnyTimes().Return(eni1.IPv4Prefixes, nil) + m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(gomock.Any(), eni2.ENIID).AnyTimes().Return(eni2.IPv4Prefixes, nil) m.awsutils.EXPECT().IsUnmanagedENI(eni1.ENIID).Return(false).AnyTimes() m.awsutils.EXPECT().IsUnmanagedENI(eni2.ENIID).Return(false).AnyTimes() - m.awsutils.EXPECT().IsUnmanagedNIC(eni1.NetworkCard).Return(false).AnyTimes() m.awsutils.EXPECT().IsUnmanagedNIC(eni2.NetworkCard).Return(false).AnyTimes() - m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() m.awsutils.EXPECT().IsEfaOnlyENI(gomock.Any(), gomock.Any()).Return(false).AnyTimes() m.network.EXPECT().GetRouteTableNumberForENI(eni1.NetworkCard, gomock.Any(), eni1.DeviceNumber, gomock.Any(), false).Times(1) m.network.EXPECT().GetRouteTableNumberForENI(eni2.NetworkCard, gomock.Any(), eni2.DeviceNumber, gomock.Any(), false).Times(1) @@ -271,8 +284,9 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) { m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false, false).Return(nil) m.network.EXPECT().CleanUpStaleAWSChains(true, false).Return(nil) m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) - m.awsutils.EXPECT().RefreshSGIDs(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + m.awsutils.EXPECT().RefreshSGIDs(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) m.awsutils.EXPECT().SetUnmanagedNetworkCards(gomock.Any()).AnyTimes() + m.awsutils.EXPECT().RefreshCustomSGIDs(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) eniMetadataSlice := []awsutils.ENIMetadata{eni1, eni2} resp := awsutils.DescribeAllENIsResult{ @@ -283,18 +297,18 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) { EFAOnlyENIByNetworkCard: []string{""}, ENIsByNetworkCard: [][]string{defaultNetworkCard: {eni1.ENIID, eni2.ENIID}}, } - m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) + m.awsutils.EXPECT().DescribeAllENIs(gomock.Any()).Return(resp, nil) m.awsutils.EXPECT().SetEFAOnlyENIs(resp.EFAOnlyENIByNetworkCard).Times(1) m.awsutils.EXPECT().SetUnmanagedENIs(gomock.Any()).AnyTimes() - m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()).AnyTimes() - m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP).AnyTimes() + m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP).AnyTimes().AnyTimes() var rules []netlink.Rule - m.network.EXPECT().GetRuleList(false).Return(rules, nil) - m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any()) - m.network.EXPECT().GetExternalServiceCIDRs().Return(nil) - m.network.EXPECT().UpdateExternalServiceIpRules(gomock.Any(), gomock.Any()) + m.network.EXPECT().GetRuleList(false).Return(rules, nil).AnyTimes() + m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any()).AnyTimes() + m.network.EXPECT().GetExternalServiceCIDRs().Return(nil).AnyTimes() + m.network.EXPECT().UpdateExternalServiceIpRules(gomock.Any(), gomock.Any()).AnyTimes() maxPods, _ := resource.ParseQuantity("500") fakeNode := v1.Node{ @@ -355,14 +369,14 @@ func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) { m.awsutils.EXPECT().IsUnmanagedNIC(eni1.NetworkCard).Return(false).AnyTimes() m.awsutils.EXPECT().IsEfaOnlyENI(gomock.Any(), gomock.Any()).Return(false).AnyTimes() - m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() // primaryIP := net.ParseIP(ipaddr01) primaryIPv6 := net.ParseIP(v6ipaddr01) m.awsutils.EXPECT().GetVPCIPv6CIDRs().Return(cidrs, nil).AnyTimes() m.network.EXPECT().SetupHostNetwork(cidrs, eni1.MAC, &primaryIPv6, false, true).Return(nil) m.network.EXPECT().CleanUpStaleAWSChains(false, true).Return(nil) - m.awsutils.EXPECT().GetIPv6PrefixesFromEC2(eni1.ENIID).AnyTimes().Return(eni1.IPv6Prefixes, nil) + m.awsutils.EXPECT().GetIPv6PrefixesFromEC2(gomock.Any(), eni1.ENIID).AnyTimes().Return(eni1.IPv6Prefixes, nil) m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) m.awsutils.EXPECT().GetPrimaryENImac().Return(eni1.MAC) m.awsutils.EXPECT().IsPrimaryENI(primaryENIid).Return(true).AnyTimes() @@ -373,7 +387,7 @@ func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) { m.network.EXPECT().GetExternalServiceCIDRs().Return(nil) m.network.EXPECT().UpdateExternalServiceIpRules(gomock.Any(), gomock.Any()) - m.awsutils.EXPECT().RefreshSGIDs(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + m.awsutils.EXPECT().RefreshSGIDs(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) eniMetadataSlice := []awsutils.ENIMetadata{eni1} resp := awsutils.DescribeAllENIsResult{ @@ -386,7 +400,7 @@ func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) { } m.awsutils.EXPECT().GetENILimit().Return(1) - m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) + m.awsutils.EXPECT().DescribeAllENIs(gomock.Any()).Return(resp, nil) m.awsutils.EXPECT().SetEFAOnlyENIs(resp.EFAOnlyENIByNetworkCard).Times(1) m.awsutils.EXPECT().GetLocalIPv6().Return(primaryIPv6).AnyTimes() @@ -583,6 +597,14 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool, unschedulabeNode bool, mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false, networkutils.CalculateRouteTableId(primaryDevice, 0)) } + // Add mock expectations for unified ENI exclusion approach (needed when UseSubnetDiscovery() is true) + if UseSubnetDiscovery() { + m.awsutils.EXPECT().GetENISubnetID(gomock.Any(), primaryENIid).AnyTimes().Return("subnet-primary", nil) + m.awsutils.EXPECT().GetENISubnetID(gomock.Any(), secENIid).AnyTimes().Return("subnet-secondary", nil) + m.awsutils.EXPECT().IsSubnetExcluded(gomock.Any(), "subnet-primary").AnyTimes().Return(false, nil) + m.awsutils.EXPECT().IsSubnetExcluded(gomock.Any(), "subnet-secondary").AnyTimes().Return(false, nil) + } + primary := true notPrimary := false testAddr1 := ipaddr01 @@ -690,27 +712,39 @@ func assertAllocationExternalCalls(shouldCall bool, useENIConfig bool, m *testMo originalErr := errors.New("err") if useENIConfig { - m.awsutils.EXPECT().AllocENI(sg, podENIConfig.Subnet, 14, 0).Times(callCount).Return(eni2, nil) + m.awsutils.EXPECT().AllocENI(gomock.Any(), sg, podENIConfig.Subnet, 14, 0).Times(callCount).Return(eni2, nil) } else if subnetDiscovery { - m.awsutils.EXPECT().AllocIPAddresses(primaryENIid, 14).Times(callCount).Return(nil, &smithy.GenericAPIError{ + m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), primaryENIid, 14).Times(callCount).Return(nil, &smithy.GenericAPIError{ Code: "InsufficientFreeAddressesInSubnet", Message: originalErr.Error(), Fault: smithy.FaultUnknown, }) - m.awsutils.EXPECT().AllocIPAddresses(primaryENIid, 1).Times(callCount).Return(nil, &smithy.GenericAPIError{ + m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), primaryENIid, 1).Times(callCount).Return(nil, &smithy.GenericAPIError{ Code: "InsufficientFreeAddressesInSubnet", Message: originalErr.Error(), Fault: smithy.FaultUnknown, }) - m.awsutils.EXPECT().AllocENI(nil, "", 14, 0).Times(callCount).Return(eni2, nil) + m.awsutils.EXPECT().AllocENI(gomock.Any(), nil, "", 14, 0).Times(callCount).Return(eni2, nil) } else { - m.awsutils.EXPECT().AllocENI(nil, "", 14, 0).Times(callCount).Return(eni2, nil) + m.awsutils.EXPECT().AllocENI(gomock.Any(), nil, "", 14, 0).Times(callCount).Return(eni2, nil) } m.awsutils.EXPECT().GetPrimaryENI().Times(callCount).Return(primaryENIid) m.awsutils.EXPECT().WaitForENIAndIPsAttached(secENIid, 14).Times(callCount).Return(eniMetadata[1], nil) - m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()).Times(callCount) - m.network.EXPECT().GetRouteTableNumberForENI(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Times(callCount) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()).AnyTimes().Times(callCount) + + // Add missing GetRouteTableNumberForENI expectation + m.network.EXPECT().GetRouteTableNumberForENI(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(callCount).Return(0, false, nil) + // Add expectations for unified ENI exclusion approach + if subnetDiscovery { + // Mock GetENISubnetID calls for setupENI (both primary and secondary ENIs) + m.awsutils.EXPECT().GetENISubnetID(gomock.Any(), primaryENIid).AnyTimes().Return("subnet-primary", nil) + m.awsutils.EXPECT().GetENISubnetID(gomock.Any(), secENIid).AnyTimes().Return("subnet-secondary", nil) + + // Mock IsSubnetExcluded calls + m.awsutils.EXPECT().IsSubnetExcluded(gomock.Any(), "subnet-primary").AnyTimes().Return(false, nil) + m.awsutils.EXPECT().IsSubnetExcluded(gomock.Any(), "subnet-secondary").AnyTimes().Return(false, nil) + } } func TestIncreasePrefixPoolDefault(t *testing.T) { @@ -756,6 +790,14 @@ func testIncreasePrefixPool(t *testing.T, useENIConfig, subnetDiscovery bool) { mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false, networkutils.CalculateRouteTableId(primaryDevice, 0)) } + // Add mock expectations for unified ENI exclusion approach (needed when UseSubnetDiscovery() is true) + if UseSubnetDiscovery() { + m.awsutils.EXPECT().GetENISubnetID(gomock.Any(), primaryENIid).AnyTimes().Return("subnet-primary", nil) + m.awsutils.EXPECT().GetENISubnetID(gomock.Any(), secENIid).AnyTimes().Return("subnet-secondary", nil) + m.awsutils.EXPECT().IsSubnetExcluded(gomock.Any(), "subnet-primary").AnyTimes().Return(false, nil) + m.awsutils.EXPECT().IsSubnetExcluded(gomock.Any(), "subnet-secondary").AnyTimes().Return(false, nil) + } + primary := true testAddr1 := ipaddr01 testAddr11 := ipaddr11 @@ -776,21 +818,21 @@ func testIncreasePrefixPool(t *testing.T, useENIConfig, subnetDiscovery bool) { originalErr := errors.New("err") if useENIConfig { - m.awsutils.EXPECT().AllocENI(sg, podENIConfig.Subnet, 1, defaultNetworkCard).Return(eni2, nil) + m.awsutils.EXPECT().AllocENI(gomock.Any(), sg, podENIConfig.Subnet, 1, defaultNetworkCard).Return(eni2, nil) } else if subnetDiscovery { - m.awsutils.EXPECT().AllocIPAddresses(primaryENIid, 1).Return(nil, &smithy.GenericAPIError{ + m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), primaryENIid, 1).Return(nil, &smithy.GenericAPIError{ Code: "InsufficientFreeAddressesInSubnet", Message: originalErr.Error(), Fault: smithy.FaultUnknown, }) - m.awsutils.EXPECT().AllocIPAddresses(primaryENIid, 1).Return(nil, &smithy.GenericAPIError{ + m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), primaryENIid, 1).Return(nil, &smithy.GenericAPIError{ Code: "InsufficientFreeAddressesInSubnet", Message: originalErr.Error(), Fault: smithy.FaultUnknown, }) - m.awsutils.EXPECT().AllocENI(nil, "", 1, defaultNetworkCard).Return(eni2, nil) + m.awsutils.EXPECT().AllocENI(gomock.Any(), nil, "", 1, defaultNetworkCard).Return(eni2, nil) } else { - m.awsutils.EXPECT().AllocENI(nil, "", 1, defaultNetworkCard).Return(eni2, nil) + m.awsutils.EXPECT().AllocENI(gomock.Any(), nil, "", 1, defaultNetworkCard).Return(eni2, nil) } eniMetadata := []awsutils.ENIMetadata{ @@ -830,8 +872,10 @@ func testIncreasePrefixPool(t *testing.T, useENIConfig, subnetDiscovery bool) { m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) m.awsutils.EXPECT().WaitForENIAndIPsAttached(secENIid, 1).Return(eniMetadata[1], nil) - m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()) - m.network.EXPECT().GetRouteTableNumberForENI(defaultNetworkCard, gomock.Any(), secDevice, gomock.Any(), false) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()).AnyTimes() + + // Add missing GetRouteTableNumberForENI expectation + m.network.EXPECT().GetRouteTableNumberForENI(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(0, false, nil) if mockContext.useCustomNetworking { mockContext.myNodeName = myNodeName @@ -895,15 +939,15 @@ func TestDecreaseIPPool(t *testing.T) { mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(secENIid, testAddr12, false) mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AssignPodIPv4Address(datastore.IPAMKey{ContainerID: "container2"}, datastore.IPAMMetadata{K8SPodName: "pod2"}) - m.awsutils.EXPECT().DeallocPrefixAddresses(gomock.Any(), gomock.Any()).Times(1) - m.awsutils.EXPECT().DeallocIPAddresses(gomock.Any(), gomock.Any()).Times(1) + m.awsutils.EXPECT().DeallocPrefixAddresses(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) + m.awsutils.EXPECT().DeallocIPAddresses(gomock.Any(), gomock.Any(), gomock.Any()).Times(1) short, over, enabled := mockContext.datastoreTargetState(nil, defaultNetworkCard) assert.Equal(t, 0, short) // there would not be any shortage assert.Equal(t, 1, over) // out of 4 IPs we have 2 IPs assigned, warm IP target is 1, so over is 1 assert.Equal(t, true, enabled) // there is warm ip target enabled with the value of 1 - mockContext.decreaseDatastorePool(defaultNetworkCard) + mockContext.decreaseDatastorePool(context.Background(), defaultNetworkCard) short, over, enabled = mockContext.datastoreTargetState(nil, defaultNetworkCard) assert.Equal(t, 0, short) // there would not be any shortage @@ -911,7 +955,7 @@ func TestDecreaseIPPool(t *testing.T) { assert.Equal(t, true, enabled) // there is warm ip target enabled with the value of 1 // make another call just to ensure that more deallocations do not happen - mockContext.decreaseDatastorePool(defaultNetworkCard) + mockContext.decreaseDatastorePool(context.Background(), defaultNetworkCard) short, over, enabled = mockContext.datastoreTargetState(nil, defaultNetworkCard) assert.Equal(t, 0, short) // there would not be any shortage @@ -948,7 +992,7 @@ func TestTryAddIPToENI(t *testing.T) { mockContext.dataStoreAccess = testDatastore() - m.awsutils.EXPECT().AllocENI(nil, "", warmIPTarget, defaultNetworkCard).Return(secENIid, nil) + m.awsutils.EXPECT().AllocENI(gomock.Any(), nil, "", warmIPTarget, defaultNetworkCard).Return(secENIid, nil) eniMetadata := []awsutils.ENIMetadata{ { ENIID: primaryENIid, @@ -981,8 +1025,10 @@ func TestTryAddIPToENI(t *testing.T) { } m.awsutils.EXPECT().WaitForENIAndIPsAttached(secENIid, 3).Return(eniMetadata[1], nil) m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) - m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()) - m.network.EXPECT().GetRouteTableNumberForENI(defaultNetworkCard, gomock.Any(), secDevice, mockContext.maxENI, false).Times(1) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()).AnyTimes() + + // Add missing GetRouteTableNumberForENI expectation + m.network.EXPECT().GetRouteTableNumberForENI(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(0, false, nil) mockContext.myNodeName = myNodeName @@ -1020,7 +1066,7 @@ func TestNodeIPPoolReconcile(t *testing.T) { m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) m.awsutils.EXPECT().IsUnmanagedENI(primaryENIid).AnyTimes().Return(false) m.awsutils.EXPECT().IsUnmanagedNIC(primaryENIMetadata.NetworkCard).AnyTimes().Return(false) - m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() eniMetadataList := []awsutils.ENIMetadata{primaryENIMetadata} m.awsutils.EXPECT().GetAttachedENIs().Return(eniMetadataList, nil) resp := awsutils.DescribeAllENIsResult{ @@ -1031,7 +1077,7 @@ func TestNodeIPPoolReconcile(t *testing.T) { EFAOnlyENIByNetworkCard: []string{""}, ENIsByNetworkCard: [][]string{defaultNetworkCard: {primaryENIMetadata.ENIID}}, } - m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) + m.awsutils.EXPECT().DescribeAllENIs(gomock.Any()).Return(resp, nil) m.awsutils.EXPECT().SetUnmanagedNetworkCards(gomock.Any()).AnyTimes() m.awsutils.EXPECT().IsEfaOnlyENI(defaultNetworkCard, primaryENIid).AnyTimes().Return(false) m.network.EXPECT().GetRouteTableNumberForENI(defaultNetworkCard, gomock.Any(), primaryDevice, mockContext.maxENI, false).AnyTimes() @@ -1057,7 +1103,7 @@ func TestNodeIPPoolReconcile(t *testing.T) { } m.awsutils.EXPECT().GetAttachedENIs().Return(oneIPUnassigned, nil) - m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(oneIPUnassigned[0].IPv4Addresses, nil) + m.awsutils.EXPECT().GetIPv4sFromEC2(gomock.Any(), primaryENIid).Return(oneIPUnassigned[0].IPv4Addresses, nil) mockContext.nodeIPPoolReconcile(ctx, 0) curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos() @@ -1083,7 +1129,7 @@ func TestNodeIPPoolReconcile(t *testing.T) { ENIsByNetworkCard: [][]string{defaultNetworkCard: {primaryENIid, newENIMetadata.ENIID}}, } - m.awsutils.EXPECT().DescribeAllENIs().Return(resp2, nil) + m.awsutils.EXPECT().DescribeAllENIs(gomock.Any()).Return(resp2, nil) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()) m.network.EXPECT().GetRouteTableNumberForENI(defaultNetworkCard, gomock.Any(), secDevice, mockContext.maxENI, false).Times(1) mockContext.nodeIPPoolReconcile(ctx, 0) @@ -1128,7 +1174,7 @@ func TestNodePrefixPoolReconcile(t *testing.T) { m.awsutils.EXPECT().IsUnmanagedENI(primaryENIid).AnyTimes().Return(false) m.awsutils.EXPECT().IsUnmanagedNIC(primaryENIMetadata.NetworkCard).AnyTimes().Return(false) m.awsutils.EXPECT().IsEfaOnlyENI(defaultNetworkCard, primaryENIid).AnyTimes().Return(false) - m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() eniMetadataList := []awsutils.ENIMetadata{primaryENIMetadata} m.awsutils.EXPECT().GetAttachedENIs().Return(eniMetadataList, nil) resp := awsutils.DescribeAllENIsResult{ @@ -1139,7 +1185,7 @@ func TestNodePrefixPoolReconcile(t *testing.T) { EFAOnlyENIByNetworkCard: []string{""}, ENIsByNetworkCard: [][]string{defaultNetworkCard: {primaryENIMetadata.ENIID}}, } - m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) + m.awsutils.EXPECT().DescribeAllENIs(gomock.Any()).Return(resp, nil) m.awsutils.EXPECT().SetUnmanagedNetworkCards(gomock.Any()).AnyTimes() m.network.EXPECT().GetRouteTableNumberForENI(defaultNetworkCard, gomock.Any(), primaryDevice, mockContext.maxENI, false).AnyTimes() mockContext.nodeIPPoolReconcile(ctx, 0) @@ -1165,7 +1211,8 @@ func TestNodePrefixPoolReconcile(t *testing.T) { }, } m.awsutils.EXPECT().GetAttachedENIs().Return(oneIPUnassigned, nil) - m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(primaryENIid).Return(oneIPUnassigned[0].IPv4Prefixes, nil) + m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(gomock.Any(), primaryENIid).Return(oneIPUnassigned[0].IPv4Prefixes, nil) + // m.awsutils.EXPECT().GetIPv4sFromEC2(gomock.Any(),primaryENIid).Return(oneIPUnassigned[0].IPv4Addresses, nil) mockContext.nodeIPPoolReconcile(ctx, 0) curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos() @@ -1191,7 +1238,7 @@ func TestNodePrefixPoolReconcile(t *testing.T) { ENIsByNetworkCard: [][]string{defaultNetworkCard: {primaryENIid, newENIMetadata.ENIID}}, } - m.awsutils.EXPECT().DescribeAllENIs().Return(resp2, nil) + m.awsutils.EXPECT().DescribeAllENIs(gomock.Any()).Return(resp2, nil) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()) m.network.EXPECT().GetRouteTableNumberForENI(defaultNetworkCard, gomock.Any(), gomock.Any(), mockContext.maxENI, false).AnyTimes() mockContext.nodeIPPoolReconcile(ctx, 0) @@ -1449,7 +1496,6 @@ func testDatastore() *datastore.DataStoreAccess { } func testDatastorewithPrefix() *datastore.DataStoreAccess { - return &datastore.DataStoreAccess{ DataStores: []*datastore.DataStore{datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion}), true, defaultNetworkCard)}, } @@ -1825,7 +1871,7 @@ func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) { }, nil) // eniIPPoolReconcile() calls EC2 to get the actual count, but that call fails - m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(nil, errors.New("ec2 API call failed")) + m.awsutils.EXPECT().GetIPv4sFromEC2(gomock.Any(), primaryENIid).Return(nil, errors.New("ec2 API call failed")) mockContext.nodeIPPoolReconcile(ctx, 0) curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) @@ -1847,7 +1893,7 @@ func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) { }, nil) // eniIPPoolReconcile() calls EC2 to get the actual count that should still be 2 - m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(primaryENIMetadata.IPv4Addresses, nil) + m.awsutils.EXPECT().GetIPv4sFromEC2(gomock.Any(), primaryENIid).Return(primaryENIMetadata.IPv4Addresses, nil) mockContext.nodeIPPoolReconcile(ctx, 0) curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) @@ -1913,7 +1959,7 @@ func TestNodePrefixPoolReconcileBadIMDSData(t *testing.T) { }, nil) // eniIPPoolReconcile() calls EC2 to get the actual count, but that call fails - m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(primaryENIid).Return(nil, errors.New("ec2 API call failed")) + m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(gomock.Any(), primaryENIid).Return(nil, errors.New("ec2 API call failed")) mockContext.nodeIPPoolReconcile(ctx, 0) curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) @@ -1935,7 +1981,7 @@ func TestNodePrefixPoolReconcileBadIMDSData(t *testing.T) { }, nil) // eniIPPoolReconcile() calls EC2 to get the actual count that should still be 16 - m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(primaryENIid).Return(primaryENIMetadata.IPv4Prefixes, nil) + m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(gomock.Any(), primaryENIid).Return(primaryENIMetadata.IPv4Prefixes, nil) mockContext.nodeIPPoolReconcile(ctx, 0) curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) @@ -2083,7 +2129,7 @@ func TestIPAMContext_setupENI(t *testing.T) { } m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) m.network.EXPECT().GetRouteTableNumberForENI(defaultNetworkCard, gomock.Any(), primaryDevice, mockContext.maxENI, false).Times(1) - err := mockContext.setupENI(primaryENIMetadata.ENIID, primaryENIMetadata, false, false) + err := mockContext.setupENI(context.Background(), primaryENIMetadata.ENIID, primaryENIMetadata, false, false) assert.NoError(t, err) // Primary ENI added assert.Equal(t, 1, len(mockContext.primaryIP)) @@ -2093,7 +2139,7 @@ func TestIPAMContext_setupENI(t *testing.T) { m.network.EXPECT().GetRouteTableNumberForENI(defaultNetworkCard, gomock.Any(), secDevice, mockContext.maxENI, false).Times(1) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()).Return(errors.New("not able to set route 0.0.0.0/0 via 10.10.10.1 table 2")) - err = mockContext.setupENI(newENIMetadata.ENIID, newENIMetadata, false, false) + err = mockContext.setupENI(context.Background(), newENIMetadata.ENIID, newENIMetadata, false, false) assert.Error(t, err) assert.Equal(t, 1, len(mockContext.primaryIP)) } @@ -2133,7 +2179,7 @@ func TestIPAMContext_setupENIwithPDenabled(t *testing.T) { } m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) m.network.EXPECT().GetRouteTableNumberForENI(defaultNetworkCard, gomock.Any(), primaryDevice, mockContext.maxENI, false).Times(1) - err := mockContext.setupENI(primaryENIMetadata.ENIID, primaryENIMetadata, false, false) + err := mockContext.setupENI(context.Background(), primaryENIMetadata.ENIID, primaryENIMetadata, false, false) assert.NoError(t, err) // Primary ENI added assert.Equal(t, 1, len(mockContext.primaryIP)) @@ -2143,7 +2189,7 @@ func TestIPAMContext_setupENIwithPDenabled(t *testing.T) { m.network.EXPECT().GetRouteTableNumberForENI(defaultNetworkCard, gomock.Any(), secDevice, mockContext.maxENI, false).Times(1) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()).Return(errors.New("not able to set route 0.0.0.0/0 via 10.10.10.1 table 2")) - err = mockContext.setupENI(newENIMetadata.ENIID, newENIMetadata, false, false) + err = mockContext.setupENI(context.Background(), newENIMetadata.ENIID, newENIMetadata, false, false) assert.Error(t, err) assert.Equal(t, 1, len(mockContext.primaryIP)) } @@ -2589,7 +2635,7 @@ func TestPodENIErrInc(t *testing.T) { assert.NoError(t, err) // Mock AWS API error - m.awsutils.EXPECT().AllocENI(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + m.awsutils.EXPECT().AllocENI(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return("", errors.New("API error")).Times(2) // Expect 2 calls // Test case 1: First error @@ -2620,7 +2666,7 @@ func TestPodENIErrInc(t *testing.T) { func (c *IPAMContext) tryAssignPodENI(ctx context.Context, pod *corev1.Pod, fnName string) error { // Mock implementation for the test - _, err := c.awsClient.AllocENI(nil, "", 0, defaultNetworkCard) + _, err := c.awsClient.AllocENI(context.Background(), nil, "", 0, defaultNetworkCard) if err != nil { prometheusmetrics.PodENIErr.With(prometheus.Labels{"fn": fnName}).Inc() return err @@ -2696,3 +2742,627 @@ func TestFilterUnmanagedENIs_WithEFAOnlyENIs(t *testing.T) { }) } } + +func TestIPAMContext_PrimarySubnetExclusion(t *testing.T) { + // Test that primary ENI is marked as excluded when primary subnet is excluded + dataStore := datastore.NewDataStore(log, datastore.NullCheckpoint{}, false, 0) + + // Add primary ENI + err := dataStore.AddENI(primaryENIid, primaryDevice, true, false, false, 0) + assert.NoError(t, err) + + // Mark it as excluded + err = dataStore.SetENIExcludedForPodIPs(primaryENIid, true) + assert.NoError(t, err) + + // Verify it's excluded + isExcluded := dataStore.IsENIExcludedForPodIPs(primaryENIid) + assert.True(t, isExcluded, "Primary ENI should be marked as excluded") + + // Test IP allocation skips excluded ENI + _, _, _, err = dataStore.AssignPodIPv4Address( + datastore.IPAMKey{ + NetworkName: "net0", + ContainerID: "test-container", + IfName: "eth0", + }, + datastore.IPAMMetadata{ + K8SPodNamespace: "default", + K8SPodName: "test-pod", + }, + ) + assert.Error(t, err, "Should not be able to assign IP from excluded ENI") + assert.Contains(t, err.Error(), "no available IP/Prefix addresses") +} + +func TestIPAMContext_WarmTargetWithExcludedPrimary(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + // Set env vars + _ = os.Setenv("ENABLE_IPv4", "true") + _ = os.Setenv("ENABLE_IPv6", "false") + _ = os.Setenv(envWarmENITarget, "2") + defer func() { + _ = os.Unsetenv("ENABLE_IPv4") + _ = os.Unsetenv("ENABLE_IPv6") + _ = os.Unsetenv(envWarmENITarget) + }() + + // Create context with excluded primary subnet + ctx := &IPAMContext{ + awsClient: m.awsutils, + dataStoreAccess: testDatastore(), + enableIPv4: true, + maxIPsPerENI: 10, + maxENI: 4, + warmENITarget: 2, + maxPods: 100, // Set a realistic max pods limit + } + + // Add primary ENI (excluded) - no IPs should be allocated to it + err := ctx.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false, 0) + assert.NoError(t, err) + err = ctx.dataStoreAccess.GetDataStore(defaultNetworkCard).SetENIExcludedForPodIPs(primaryENIid, true) + assert.NoError(t, err) + // Note: We don't add IPs to the excluded primary ENI as the real implementation wouldn't allocate them + + // Test that pool is too low when we have only primary ENI + decisions := ctx.isDatastorePoolTooLow() + t.Logf("With only primary ENI - decisions: %v", decisions) + assert.True(t, decisions[defaultNetworkCard].IsLow, "Pool should be too low with only excluded primary ENI") + + // Add secondary ENI with IPs + err = ctx.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(secENIid, secDevice, false, false, false, 0) + assert.NoError(t, err) + + // Add IPs to secondary ENI + for i := 1; i <= 10; i++ { + ipv4Addr := net.IPNet{IP: net.ParseIP(fmt.Sprintf("10.0.1.%d", i)), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = ctx.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(secENIid, ipv4Addr, false) + assert.NoError(t, err) + } + + // Still too low with 1 secondary ENI when warm target is 2 + decisions = ctx.isDatastorePoolTooLow() + t.Logf("With 1 secondary ENI - decisions: %v", decisions) + assert.True(t, decisions[defaultNetworkCard].IsLow, "Pool should still be too low with 1 secondary ENI when warm target is 2") + + // Add another secondary ENI with IPs + err = ctx.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI("eni-3", 2, false, false, false, 0) + assert.NoError(t, err) + + // Add IPs to third ENI + for i := 1; i <= 10; i++ { + ipv4Addr := net.IPNet{IP: net.ParseIP(fmt.Sprintf("10.0.2.%d", i)), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = ctx.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore("eni-3", ipv4Addr, false) + assert.NoError(t, err) + } + + // Now we have 2 secondary ENIs, should meet warm target + decisions = ctx.isDatastorePoolTooLow() + t.Logf("With 2 secondary ENIs - decisions: %v", decisions) + assert.False(t, decisions[defaultNetworkCard].IsLow, "Pool should not be too low with 2 secondary ENIs") +} + +func TestNodeInitPrimarySubnetExclusionWithExistingPodIPs(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + ctx := context.Background() + + // Create a fake checkpoint with an existing pod IP allocation on primary ENI + fakeCheckpoint := datastore.CheckpointData{ + Version: datastore.CheckpointFormatVersion, + Allocations: []datastore.CheckpointEntry{ + {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "existing-pod", IfName: "eth0"}, IPv4: ipaddr02}, + }, + } + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + k8sClient: m.k8sClient, + maxIPsPerENI: 14, + maxENI: 4, + warmENITarget: 1, + warmIPTarget: 3, + primaryIP: make(map[string]string), + terminating: int32(0), + networkClient: m.network, + dataStoreAccess: &datastore.DataStoreAccess{DataStores: []*datastore.DataStore{datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), false, defaultNetworkCard)}}, + myNodeName: myNodeName, + enableIPv4: true, + enableIPv6: false, + withApiServer: true, + useSubnetDiscovery: true, // Enable subnet discovery + numNetworkCards: 1, + } + mockContext.unmanagedENI = make([]int, mockContext.numNetworkCards) + + eni1, eni2, _ := getDummyENIMetadata() + + var cidrs []string + m.awsutils.EXPECT().GetENILimit().Return(4) + m.awsutils.EXPECT().GetENIIPv4Limit().Return(14) + m.awsutils.EXPECT().GetNetworkCards().Return([]vpc.NetworkCard{{NetworkCardIndex: 0, MaximumNetworkInterfaces: 4}}).AnyTimes() + m.awsutils.EXPECT().SetUnmanagedNetworkCards(gomock.Any()).AnyTimes() + m.awsutils.EXPECT().IsUnmanagedNIC(gomock.Any()).Return(false).AnyTimes() + m.awsutils.EXPECT().IsEfaOnlyENI(gomock.Any(), gomock.Any()).Return(false).AnyTimes() + m.awsutils.EXPECT().GetIPv4sFromEC2(gomock.Any(), eni1.ENIID).AnyTimes().Return(eni1.IPv4Addresses, nil) + m.awsutils.EXPECT().GetIPv4sFromEC2(gomock.Any(), eni2.ENIID).AnyTimes().Return(eni2.IPv4Addresses, nil) + m.awsutils.EXPECT().IsUnmanagedENI(eni1.ENIID).Return(false).AnyTimes() + m.awsutils.EXPECT().IsUnmanagedENI(eni2.ENIID).Return(false).AnyTimes() + m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + primaryIP := net.ParseIP(ipaddr01) + m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil) + m.awsutils.EXPECT().GetPrimaryENImac().Return("") + m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false, false).Return(nil) + m.network.EXPECT().CleanUpStaleAWSChains(true, false).Return(nil) + m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) + m.awsutils.EXPECT().RefreshSGIDs(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + m.awsutils.EXPECT().RefreshCustomSGIDs(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + + // Mock expectations for unified ENI exclusion approach in setupENI + m.awsutils.EXPECT().GetENISubnetID(gomock.Any(), eni1.ENIID).AnyTimes().Return("subnet-1", nil) + m.awsutils.EXPECT().GetENISubnetID(gomock.Any(), eni2.ENIID).AnyTimes().Return("subnet-2", nil) + m.awsutils.EXPECT().IsSubnetExcluded(gomock.Any(), "subnet-1").AnyTimes().Return(true, nil) // Primary subnet excluded + m.awsutils.EXPECT().IsSubnetExcluded(gomock.Any(), "subnet-2").AnyTimes().Return(false, nil) // Secondary subnet not excluded + + eniMetadataSlice := []awsutils.ENIMetadata{eni1, eni2} + resp := awsutils.DescribeAllENIsResult{ + ENIMetadata: eniMetadataSlice, + TagMap: map[string]awsutils.TagMap{}, + TrunkENI: "", + EFAENIs: make(map[string]bool), + EFAOnlyENIByNetworkCard: []string{""}, + ENIsByNetworkCard: [][]string{defaultNetworkCard: {eni1.ENIID, eni2.ENIID}}, + } + m.awsutils.EXPECT().DescribeAllENIs(gomock.Any()).Return(resp, nil) + m.awsutils.EXPECT().SetEFAOnlyENIs(resp.EFAOnlyENIByNetworkCard).Times(1) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, defaultNetworkCard, secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()).AnyTimes() + m.network.EXPECT().GetRouteTableNumberForENI(eni1.NetworkCard, gomock.Any(), eni1.DeviceNumber, gomock.Any(), false).Return(0, false, nil).Times(1) + m.network.EXPECT().GetRouteTableNumberForENI(eni2.NetworkCard, gomock.Any(), eni2.DeviceNumber, gomock.Any(), false).Return(0, false, nil).Times(1) + + m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP).AnyTimes() + + var rules []netlink.Rule + m.network.EXPECT().GetRuleList(false).Return(rules, nil).AnyTimes() + m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any()).AnyTimes() + m.network.EXPECT().GetExternalServiceCIDRs().Return(nil).AnyTimes() + m.network.EXPECT().UpdateExternalServiceIpRules(gomock.Any(), gomock.Any()).AnyTimes() + + maxPods, _ := resource.ParseQuantity("500") + fakeNode := v1.Node{ + TypeMeta: metav1.TypeMeta{Kind: "Node"}, + ObjectMeta: metav1.ObjectMeta{Name: myNodeName}, + Spec: v1.NodeSpec{}, + Status: v1.NodeStatus{ + Capacity: v1.ResourceList{ + v1.ResourcePods: maxPods, + }, + }, + } + m.k8sClient.Create(ctx, &fakeNode) + + // Expect cleanup calls for excluded primary ENI (with existing pod IPs) + // The cleanup function will try to unassign any unassigned IPs/prefixes + m.awsutils.EXPECT().DeallocIPAddresses(gomock.Any(), primaryENIid, gomock.Any()).Return(nil).AnyTimes() + m.awsutils.EXPECT().DeallocPrefixAddresses(gomock.Any(), primaryENIid, gomock.Any()).Return(nil).AnyTimes() + + // Add IPs + m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), gomock.Any(), gomock.Any()).Return(&ec2.AssignPrivateIpAddressesOutput{}, nil) + os.Setenv("MY_NODE_NAME", myNodeName) + err := mockContext.nodeInit() + assert.NoError(t, err) + + // Verify that primary ENI exclusion is now always respected + isExcluded := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).IsENIExcludedForPodIPs(primaryENIid) + assert.True(t, isExcluded, "Primary ENI should remain excluded from pod IP allocation despite existing pod IPs") +} + +func TestNodeInitPrimarySubnetExclusionWithoutExistingPodIPs(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + ctx := context.Background() + + // Create a fake checkpoint with NO existing pod IP allocations + fakeCheckpoint := datastore.CheckpointData{ + Version: datastore.CheckpointFormatVersion, + Allocations: []datastore.CheckpointEntry{}, // Empty allocations + } + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + k8sClient: m.k8sClient, + maxIPsPerENI: 14, + maxENI: 4, + warmENITarget: 1, + warmIPTarget: 3, + primaryIP: make(map[string]string), + terminating: int32(0), + networkClient: m.network, + dataStoreAccess: &datastore.DataStoreAccess{DataStores: []*datastore.DataStore{datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), false, defaultNetworkCard)}}, + myNodeName: myNodeName, + enableIPv4: true, + enableIPv6: false, + withApiServer: true, + useSubnetDiscovery: true, // Enable subnet discovery + numNetworkCards: 1, + } + + // Create ENI metadata with NO secondary IPs for primary ENI (simulating fresh node) + primary := true + notPrimary := false + testAddr1 := ipaddr01 + testAddr11 := ipaddr11 + testAddr12 := ipaddr12 + eni1 := awsutils.ENIMetadata{ + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []ec2types.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, // ← Only primary IP, no secondary IPs + }, + }, + } + eni2 := awsutils.ENIMetadata{ + ENIID: secENIid, + MAC: secMAC, + DeviceNumber: secDevice, + SubnetIPv4CIDR: secSubnet, + IPv4Addresses: []ec2types.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr11, Primary: ¬Primary, + }, + { + PrivateIpAddress: &testAddr12, Primary: ¬Primary, + }, + }, + } + + var cidrs []string + m.awsutils.EXPECT().GetENILimit().Return(4) + m.awsutils.EXPECT().GetENIIPv4Limit().Return(14) + m.awsutils.EXPECT().GetNetworkCards().Return([]vpc.NetworkCard{{NetworkCardIndex: 0, MaximumNetworkInterfaces: 4}}).AnyTimes() + m.awsutils.EXPECT().SetUnmanagedNetworkCards(gomock.Any()).AnyTimes() + m.awsutils.EXPECT().IsUnmanagedNIC(gomock.Any()).Return(false).AnyTimes() + m.awsutils.EXPECT().IsEfaOnlyENI(gomock.Any(), gomock.Any()).Return(false).AnyTimes() + m.awsutils.EXPECT().GetIPv4sFromEC2(gomock.Any(), eni1.ENIID).AnyTimes().Return(eni1.IPv4Addresses, nil) + m.awsutils.EXPECT().GetIPv4sFromEC2(gomock.Any(), eni2.ENIID).AnyTimes().Return(eni2.IPv4Addresses, nil) + m.awsutils.EXPECT().IsUnmanagedENI(eni1.ENIID).Return(false).AnyTimes() + m.awsutils.EXPECT().IsUnmanagedENI(eni2.ENIID).Return(false).AnyTimes() + m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + primaryIP := net.ParseIP(ipaddr01) + m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil) + m.awsutils.EXPECT().GetPrimaryENImac().Return("") + m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false, false).Return(nil) + m.network.EXPECT().CleanUpStaleAWSChains(true, false).Return(nil) + m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) + m.awsutils.EXPECT().RefreshSGIDs(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + m.awsutils.EXPECT().RefreshCustomSGIDs(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + + // Mock expectations for unified ENI exclusion approach in setupENI + m.awsutils.EXPECT().GetENISubnetID(gomock.Any(), eni1.ENIID).AnyTimes().Return("subnet-1", nil) + m.awsutils.EXPECT().GetENISubnetID(gomock.Any(), eni2.ENIID).AnyTimes().Return("subnet-2", nil) + m.awsutils.EXPECT().IsSubnetExcluded(gomock.Any(), "subnet-1").AnyTimes().Return(true, nil) // Primary subnet excluded + m.awsutils.EXPECT().IsSubnetExcluded(gomock.Any(), "subnet-2").AnyTimes().Return(false, nil) // Secondary subnet not excluded + + eniMetadataSlice := []awsutils.ENIMetadata{eni1, eni2} + resp := awsutils.DescribeAllENIsResult{ + ENIMetadata: eniMetadataSlice, + TagMap: map[string]awsutils.TagMap{}, + TrunkENI: "", + EFAENIs: make(map[string]bool), + EFAOnlyENIByNetworkCard: []string{""}, + ENIsByNetworkCard: [][]string{defaultNetworkCard: {eni1.ENIID, eni2.ENIID}}, + } + m.awsutils.EXPECT().DescribeAllENIs(gomock.Any()).Return(resp, nil) + m.awsutils.EXPECT().SetEFAOnlyENIs(resp.EFAOnlyENIByNetworkCard).Times(1) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, gomock.Any(), secSubnet, maxENIPerNIC, false, gomock.Any(), gomock.Any()).AnyTimes() + + // Add missing GetRouteTableNumberForENI expectation + m.network.EXPECT().GetRouteTableNumberForENI(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(0, false, nil).AnyTimes() + + m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP).AnyTimes() + + var rules []netlink.Rule + m.network.EXPECT().GetRuleList(false).Return(rules, nil).AnyTimes() + m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any()).AnyTimes() + m.network.EXPECT().GetExternalServiceCIDRs().Return(nil).AnyTimes() + m.network.EXPECT().UpdateExternalServiceIpRules(gomock.Any(), gomock.Any()).AnyTimes() + + maxPods, _ := resource.ParseQuantity("500") + fakeNode := v1.Node{ + TypeMeta: metav1.TypeMeta{Kind: "Node"}, + ObjectMeta: metav1.ObjectMeta{Name: myNodeName}, + Spec: v1.NodeSpec{}, + Status: v1.NodeStatus{ + Capacity: v1.ResourceList{ + v1.ResourcePods: maxPods, + }, + }, + } + m.k8sClient.Create(ctx, &fakeNode) + + // Expect cleanup calls for excluded primary ENI + // The cleanup function will try to unassign any unassigned IPs/prefixes + m.awsutils.EXPECT().DeallocIPAddresses(gomock.Any(), primaryENIid, gomock.Any()).Return(nil).AnyTimes() + m.awsutils.EXPECT().DeallocPrefixAddresses(gomock.Any(), primaryENIid, gomock.Any()).Return(nil).AnyTimes() + + // Add IPs + m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), gomock.Any(), gomock.Any()).Return(&ec2.AssignPrivateIpAddressesOutput{}, nil) + os.Setenv("MY_NODE_NAME", myNodeName) + err := mockContext.nodeInit() + assert.NoError(t, err) + + // Verify that primary ENI remains excluded due to no existing pod IPs + isExcluded := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).IsENIExcludedForPodIPs(primaryENIid) + assert.True(t, isExcluded, "Primary ENI should remain excluded from pod IP allocation with no existing pod IPs") +} + +func TestPrimaryENIWasPreviouslyUsed(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + dataStoreAccess: testDatastore(), + } + + // Test when primary ENI has no allocated CIDRs + // primaryENIWasPreviouslyUsed function was removed in unified approach + + // Add primary ENI to datastore + err := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false, 0) + assert.NoError(t, err) + + // Test when primary ENI exists but has no allocated CIDRs + // primaryENIWasPreviouslyUsed function was removed in unified approach + + // Add IP CIDR to primary ENI (but don't assign to pod yet) + // This simulates launch template IPs or IPs that were freed from pods + ipv4Addr := net.IPNet{IP: net.ParseIP(ipaddr02), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(primaryENIid, ipv4Addr, false) + assert.NoError(t, err) + + // Test when primary ENI has unassigned CIDRs (should NOT be considered previously used) + // This handles launch template IPs that were never assigned to pods + // primaryENIWasPreviouslyUsed function was removed in unified approach + + // Assign IP to a pod to verify it works when pods are actually present + _, _, _, err = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AssignPodIPv4Address( + datastore.IPAMKey{NetworkName: "net0", ContainerID: "test-pod", IfName: "eth0"}, + datastore.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "test-pod"}, + ) + assert.NoError(t, err) + + // Test when primary ENI has IPs assigned to pods + // primaryENIWasPreviouslyUsed function was removed in unified approach +} + +func TestPrimaryENIWasPreviouslyUsedIPv6(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + dataStoreAccess: testDatastorewithPrefix(), + enableIPv6: true, + } + + // Test when primary ENI has no allocated CIDRs + // primaryENIWasPreviouslyUsed function was removed in unified approach + + // Add primary ENI to datastore + err := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false, 0) + assert.NoError(t, err) + + // Test when primary ENI exists but has no allocated CIDRs + // primaryENIWasPreviouslyUsed function was removed in unified approach + + // Add IPv6 CIDR to primary ENI (but don't assign to pod yet) + ipv6Addr := net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(128, 128)} + err = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv6CidrToStore(primaryENIid, ipv6Addr, true) + assert.NoError(t, err) + + // Test when primary ENI has unassigned CIDRs (should NOT be considered previously used) + // m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) // removed - not called anymore + // wasUsed = mockContext.primaryENIWasPreviouslyUsed() // removed + // assert.False(t, wasUsed, // removed - "Primary ENI should NOT be considered previously used when it has only unassigned IPv6 CIDRs") + + // Assign IPv6 to a pod to verify it works when pods are actually present + _, _, _, err = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AssignPodIPv6Address( + datastore.IPAMKey{NetworkName: "net0", ContainerID: "test-pod-v6", IfName: "eth0"}, + datastore.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "test-pod-v6"}, + ) + assert.NoError(t, err) + + // Test when primary ENI has IPv6 addresses assigned to pods + // m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) // removed - not called anymore + // wasUsed = mockContext.primaryENIWasPreviouslyUsed() // removed + // assert.True(t, wasUsed, // removed - "Primary ENI should be considered previously used when it has IPv6 addresses assigned to pods") +} + +func TestPrimaryENICleanupIPv4SecondaryIP(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + dataStoreAccess: testDatastore(), + enableIPv4: true, + enableIPv6: false, + } + + // Add primary ENI to datastore + err := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false, 0) + assert.NoError(t, err) + + // Add unassigned IPv4 secondary IP to primary ENI + ipv4Addr := net.IPNet{IP: net.ParseIP(ipaddr02), Mask: net.IPv4Mask(255, 255, 255, 255)} + err = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(primaryENIid, ipv4Addr, false) + assert.NoError(t, err) + + // Verify the IP exists in datastore before cleanup + ips, prefixes, err := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENICIDRs(primaryENIid) + assert.NoError(t, err) + assert.Equal(t, 1, len(ips), "Should have 1 unassigned IP before cleanup") + assert.Equal(t, 0, len(prefixes), "Should have 0 prefixes before cleanup") + + // Test that primary ENI is not considered previously used (has only unassigned IPs) + // m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) // removed - not called anymore + // wasUsed := mockContext.primaryENIWasPreviouslyUsed() // removed + // assert.False(t, wasUsed, // removed - "Primary ENI should NOT be previously used with only unassigned IPs") + + // Mock the DeallocIPAddresses call that cleanup will make + expectedIPs := []string{ipaddr02} + m.awsutils.EXPECT().DeallocIPAddresses(gomock.Any(), primaryENIid, expectedIPs).Return(nil) + + // Call cleanup function + ctx := context.Background() + mockContext.cleanupExcludedENI(ctx, primaryENIid) + + // Verify the IP was removed from datastore after cleanup + ipsAfter, prefixesAfter, err := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENICIDRs(primaryENIid) + assert.NoError(t, err) + assert.Equal(t, 0, len(ipsAfter), "Should have 0 IPs after cleanup") + assert.Equal(t, 0, len(prefixesAfter), "Should have 0 prefixes after cleanup") +} + +func TestPrimaryENICleanupIPv4Prefixes(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + dataStoreAccess: testDatastorewithPrefix(), + enableIPv4: true, + enableIPv6: false, + enablePrefixDelegation: true, + } + + // Add primary ENI to datastore + err := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false, 0) + assert.NoError(t, err) + + // Add unassigned IPv4 prefix to primary ENI + ipv4Prefix := net.IPNet{IP: net.ParseIP("10.0.0.0"), Mask: net.CIDRMask(28, 32)} + err = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(primaryENIid, ipv4Prefix, true) + assert.NoError(t, err) + + // Verify the prefix exists in datastore before cleanup + freeablePrefixes := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).FreeablePrefixes(primaryENIid) + assert.Equal(t, 1, len(freeablePrefixes), "Should have 1 freeable IPv4 prefix before cleanup") + + // Test that primary ENI is not considered previously used (has only unassigned prefixes) + // m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) // removed - not called anymore + // wasUsed := mockContext.primaryENIWasPreviouslyUsed() // removed + // assert.False(t, wasUsed, // removed - "Primary ENI should NOT be previously used with only unassigned prefixes") + + // Mock the DeallocPrefixAddresses call that cleanup will make + expectedPrefixes := []string{"10.0.0.0/28"} + m.awsutils.EXPECT().DeallocPrefixAddresses(gomock.Any(), primaryENIid, expectedPrefixes).Return(nil) + + // Call cleanup function + ctx := context.Background() + mockContext.cleanupExcludedENI(ctx, primaryENIid) + + // Verify the prefix was removed from datastore after cleanup + freeablePrefixesAfter := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).FreeablePrefixes(primaryENIid) + assert.Equal(t, 0, len(freeablePrefixesAfter), "Should have 0 freeable IPv4 prefixes after cleanup") +} + +func TestPrimaryENICleanupIPv6Prefixes(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + dataStoreAccess: testDatastorewithPrefix(), + enableIPv4: false, + enableIPv6: true, + } + + // Add primary ENI to datastore + err := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false, 0) + assert.NoError(t, err) + + // Add unassigned IPv6 prefix to primary ENI + ipv6Prefix := net.IPNet{IP: net.ParseIP("2001:db8:1234:5678::"), Mask: net.CIDRMask(80, 128)} + err = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv6CidrToStore(primaryENIid, ipv6Prefix, true) + assert.NoError(t, err) + + // Verify the prefix exists in datastore before cleanup + freeablePrefixes := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).FreeablePrefixes(primaryENIid) + assert.Equal(t, 1, len(freeablePrefixes), "Should have 1 freeable IPv6 prefix before cleanup") + + // Test that primary ENI is not considered previously used (has only unassigned IPv6 prefixes) + // m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) // removed - not called anymore + // wasUsed := mockContext.primaryENIWasPreviouslyUsed() // removed + // assert.False(t, wasUsed, // removed - "Primary ENI should NOT be previously used with only unassigned IPv6 prefixes") + + // Mock the DeallocPrefixAddresses call that cleanup will make (IPv6 prefixes use the same function) + expectedPrefixes := []string{"2001:db8:1234:5678::/80"} + m.awsutils.EXPECT().DeallocPrefixAddresses(gomock.Any(), primaryENIid, expectedPrefixes).Return(nil) + + // Call cleanup function + ctx := context.Background() + mockContext.cleanupExcludedENI(ctx, primaryENIid) + + // Verify the prefix was removed from datastore after cleanup + freeablePrefixesAfter := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).FreeablePrefixes(primaryENIid) + assert.Equal(t, 0, len(freeablePrefixesAfter), "Should have 0 freeable IPv6 prefixes after cleanup") +} + +func TestPrimaryENICleanupMixedMode(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + dataStoreAccess: testDatastorewithPrefix(), + enableIPv4: true, + enableIPv6: true, + enablePrefixDelegation: true, + } + + // Add primary ENI to datastore + err := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false, 0) + assert.NoError(t, err) + + // Add unassigned IPv4 prefix to primary ENI + ipv4Prefix := net.IPNet{IP: net.ParseIP("10.0.0.0"), Mask: net.CIDRMask(28, 32)} + err = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(primaryENIid, ipv4Prefix, true) + assert.NoError(t, err) + + // Add unassigned IPv6 prefix to primary ENI + ipv6Prefix := net.IPNet{IP: net.ParseIP("2001:db8:1234:5678::"), Mask: net.CIDRMask(80, 128)} + err = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv6CidrToStore(primaryENIid, ipv6Prefix, true) + assert.NoError(t, err) + + // Verify both prefixes exist in datastore before cleanup + freeablePrefixes := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).FreeablePrefixes(primaryENIid) + assert.Equal(t, 2, len(freeablePrefixes), "Should have 2 freeable prefixes (1 IPv4 + 1 IPv6) before cleanup") + + // Test that primary ENI is not considered previously used + // m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) // removed - not called anymore + // wasUsed := mockContext.primaryENIWasPreviouslyUsed() // removed + // assert.False(t, wasUsed, // removed - "Primary ENI should NOT be previously used with only unassigned prefixes") + + // Mock the DeallocPrefixAddresses call that cleanup will make + // tryUnassignPrefixFromENI will be called twice (IPv4 and IPv6), but the first call + // will process both prefixes and delete them, so the second call will see no freeable prefixes + expectedPrefixes := []string{"10.0.0.0/28", "2001:db8:1234:5678::/80"} + m.awsutils.EXPECT().DeallocPrefixAddresses(gomock.Any(), primaryENIid, expectedPrefixes).Return(nil) + + // Call cleanup function + ctx := context.Background() + mockContext.cleanupExcludedENI(ctx, primaryENIid) + + // Verify both prefixes were removed from datastore after cleanup + freeablePrefixesAfter := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).FreeablePrefixes(primaryENIid) + assert.Equal(t, 0, len(freeablePrefixesAfter), "Should have 0 freeable prefixes after cleanup") +} diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index 34b211462a..771cb9b384 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -367,17 +367,17 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp } if s.ipamContext.enableIPv4 && eni != nil { - //cidrStr will be pod IP i.e, IP/32 for v4 (or) IP/128 for v6. + // cidrStr will be pod IP i.e, IP/32 for v4 (or) IP/128 for v6. // Case 1: PD is enabled but IP/32 key in AvailableIPv4Cidrs[cidrStr] exists, this means it is a secondary IP. Added IsPrefix check just for sanity. // So this IP should be released immediately. // Case 2: PD is disabled then IP/32 key in AvailableIPv4Cidrs[cidrStr] will not exists since key to AvailableIPv4Cidrs will be either /28 prefix or /32 // secondary IP. Hence now see if we need free up a prefix is no other pods are using it. if s.ipamContext.enablePrefixDelegation && eni.AvailableIPv4Cidrs[cidrStr] != nil && eni.AvailableIPv4Cidrs[cidrStr].IsPrefix { log.Debugf("IP belongs to secondary pool with PD enabled so free IP from EC2") - s.ipamContext.tryUnassignIPFromENI(eni.ID, networkCard) + s.ipamContext.tryUnassignIPFromENI(ctx, eni.ID, networkCard) } else if !s.ipamContext.enablePrefixDelegation && eni.AvailableIPv4Cidrs[cidrStr] == nil { log.Debugf("IP belongs to prefix pool with PD disabled so try free prefix from EC2") - s.ipamContext.tryUnassignPrefixFromENI(eni.ID, networkCard) + s.ipamContext.tryUnassignPrefixFromENI(ctx, eni.ID, networkCard) } } @@ -456,7 +456,6 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp } func (s *server) GetNetworkPolicyConfigs(ctx context.Context, e *emptypb.Empty) (*rpc.NetworkPolicyAgentConfigReply, error) { - log.Infof("Received request for Network Policy Agent configs") resp := &rpc.NetworkPolicyAgentConfigReply{ diff --git a/pkg/networkutils/mocks/network_mocks.go b/pkg/networkutils/mocks/network_mocks.go index 84bbee0822..66e9b0384d 100644 --- a/pkg/networkutils/mocks/network_mocks.go +++ b/pkg/networkutils/mocks/network_mocks.go @@ -1,8 +1,22 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. +// + // Code generated by MockGen. DO NOT EDIT. -// Source: network.go +// Source: github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils (interfaces: NetworkAPIs) -// Package mocks is a generated GoMock package. -package mocks +// Package mock_networkutils is a generated GoMock package. +package mock_networkutils import ( net "net" @@ -37,31 +51,31 @@ func (m *MockNetworkAPIs) EXPECT() *MockNetworkAPIsMockRecorder { } // CleanUpStaleAWSChains mocks base method. -func (m *MockNetworkAPIs) CleanUpStaleAWSChains(v4Enabled, v6Enabled bool) error { +func (m *MockNetworkAPIs) CleanUpStaleAWSChains(arg0, arg1 bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CleanUpStaleAWSChains", v4Enabled, v6Enabled) + ret := m.ctrl.Call(m, "CleanUpStaleAWSChains", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // CleanUpStaleAWSChains indicates an expected call of CleanUpStaleAWSChains. -func (mr *MockNetworkAPIsMockRecorder) CleanUpStaleAWSChains(v4Enabled, v6Enabled interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) CleanUpStaleAWSChains(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanUpStaleAWSChains", reflect.TypeOf((*MockNetworkAPIs)(nil).CleanUpStaleAWSChains), v4Enabled, v6Enabled) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanUpStaleAWSChains", reflect.TypeOf((*MockNetworkAPIs)(nil).CleanUpStaleAWSChains), arg0, arg1) } // DeleteRulesBySrc mocks base method. -func (m *MockNetworkAPIs) DeleteRulesBySrc(eniIP string, v6enabled bool) error { +func (m *MockNetworkAPIs) DeleteRulesBySrc(arg0 string, arg1 bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteRulesBySrc", eniIP, v6enabled) + ret := m.ctrl.Call(m, "DeleteRulesBySrc", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // DeleteRulesBySrc indicates an expected call of DeleteRulesBySrc. -func (mr *MockNetworkAPIsMockRecorder) DeleteRulesBySrc(eniIP, v6enabled interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) DeleteRulesBySrc(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRulesBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).DeleteRulesBySrc), eniIP, v6enabled) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRulesBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).DeleteRulesBySrc), arg0, arg1) } // GetExcludeSNATCIDRs mocks base method. @@ -93,134 +107,118 @@ func (mr *MockNetworkAPIsMockRecorder) GetExternalServiceCIDRs() *gomock.Call { } // GetLinkByMac mocks base method. -func (m *MockNetworkAPIs) GetLinkByMac(mac string, retryInterval time.Duration) (netlink.Link, error) { +func (m *MockNetworkAPIs) GetLinkByMac(arg0 string, arg1 time.Duration) (netlink.Link, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLinkByMac", mac, retryInterval) + ret := m.ctrl.Call(m, "GetLinkByMac", arg0, arg1) ret0, _ := ret[0].(netlink.Link) ret1, _ := ret[1].(error) return ret0, ret1 } // GetLinkByMac indicates an expected call of GetLinkByMac. -func (mr *MockNetworkAPIsMockRecorder) GetLinkByMac(mac, retryInterval interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLinkByMac", reflect.TypeOf((*MockNetworkAPIs)(nil).GetLinkByMac), mac, retryInterval) -} - -// GetRouteTableNumberForENI mocks base method. -func (m *MockNetworkAPIs) GetRouteTableNumberForENI(networkCard int, eniIP string, deviceNumber, maxENIsPerNetworkCard int, isV6 bool) (int, bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRouteTableNumberForENI", networkCard, eniIP, deviceNumber, maxENIsPerNetworkCard, isV6) - ret0, _ := ret[0].(int) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetRouteTableNumberForENI indicates an expected call of GetRouteTableNumberForENI. -func (mr *MockNetworkAPIsMockRecorder) GetRouteTableNumberForENI(networkCard, eniIP, deviceNumber, maxENIsPerNetworkCard, isV6 interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) GetLinkByMac(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRouteTableNumberForENI", reflect.TypeOf((*MockNetworkAPIs)(nil).GetRouteTableNumberForENI), networkCard, eniIP, deviceNumber, maxENIsPerNetworkCard, isV6) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLinkByMac", reflect.TypeOf((*MockNetworkAPIs)(nil).GetLinkByMac), arg0, arg1) } // GetRuleList mocks base method. -func (m *MockNetworkAPIs) GetRuleList(v6enabled bool) ([]netlink.Rule, error) { +func (m *MockNetworkAPIs) GetRuleList(arg0 bool) ([]netlink.Rule, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRuleList", v6enabled) + ret := m.ctrl.Call(m, "GetRuleList", arg0) ret0, _ := ret[0].([]netlink.Rule) ret1, _ := ret[1].(error) return ret0, ret1 } // GetRuleList indicates an expected call of GetRuleList. -func (mr *MockNetworkAPIsMockRecorder) GetRuleList(v6enabled interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) GetRuleList(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRuleList", reflect.TypeOf((*MockNetworkAPIs)(nil).GetRuleList), v6enabled) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRuleList", reflect.TypeOf((*MockNetworkAPIs)(nil).GetRuleList), arg0) } // GetRuleListBySrc mocks base method. -func (m *MockNetworkAPIs) GetRuleListBySrc(ruleList []netlink.Rule, src net.IPNet) ([]netlink.Rule, error) { +func (m *MockNetworkAPIs) GetRuleListBySrc(arg0 []netlink.Rule, arg1 net.IPNet) ([]netlink.Rule, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRuleListBySrc", ruleList, src) + ret := m.ctrl.Call(m, "GetRuleListBySrc", arg0, arg1) ret0, _ := ret[0].([]netlink.Rule) ret1, _ := ret[1].(error) return ret0, ret1 } // GetRuleListBySrc indicates an expected call of GetRuleListBySrc. -func (mr *MockNetworkAPIsMockRecorder) GetRuleListBySrc(ruleList, src interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) GetRuleListBySrc(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRuleListBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).GetRuleListBySrc), ruleList, src) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRuleListBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).GetRuleListBySrc), arg0, arg1) } // SetupENINetwork mocks base method. -func (m *MockNetworkAPIs) SetupENINetwork(eniIP, eniMAC string, networkCard int, eniSubnetCIDR string, maxENIPerNIC int, isTrunkENI bool, routeTableID int, isRuleConfigured bool) error { +func (m *MockNetworkAPIs) SetupENINetwork(arg0, arg1 string, arg2 int, arg3 string, arg4 int, arg5 bool, arg6 int, arg7 bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetupENINetwork", eniIP, eniMAC, networkCard, eniSubnetCIDR, maxENIPerNIC, isTrunkENI, routeTableID, isRuleConfigured) + ret := m.ctrl.Call(m, "SetupENINetwork", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) ret0, _ := ret[0].(error) return ret0 } // SetupENINetwork indicates an expected call of SetupENINetwork. -func (mr *MockNetworkAPIsMockRecorder) SetupENINetwork(eniIP, eniMAC, networkCard, eniSubnetCIDR, maxENIPerNIC, isTrunkENI, routeTableID, isRuleConfigured interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) SetupENINetwork(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupENINetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupENINetwork), eniIP, eniMAC, networkCard, eniSubnetCIDR, maxENIPerNIC, isTrunkENI, routeTableID, isRuleConfigured) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupENINetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupENINetwork), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) } // SetupHostNetwork mocks base method. -func (m *MockNetworkAPIs) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, enablePodENI, v6Enabled bool) error { +func (m *MockNetworkAPIs) SetupHostNetwork(arg0 []string, arg1 string, arg2 *net.IP, arg3, arg4 bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetupHostNetwork", vpcCIDRs, primaryMAC, primaryAddr, enablePodENI, v6Enabled) + ret := m.ctrl.Call(m, "SetupHostNetwork", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) return ret0 } // SetupHostNetwork indicates an expected call of SetupHostNetwork. -func (mr *MockNetworkAPIsMockRecorder) SetupHostNetwork(vpcCIDRs, primaryMAC, primaryAddr, enablePodENI, v6Enabled interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) SetupHostNetwork(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupHostNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupHostNetwork), vpcCIDRs, primaryMAC, primaryAddr, enablePodENI, v6Enabled) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupHostNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupHostNetwork), arg0, arg1, arg2, arg3, arg4) } // UpdateExternalServiceIpRules mocks base method. -func (m *MockNetworkAPIs) UpdateExternalServiceIpRules(ruleList []netlink.Rule, externalIPs []string) error { +func (m *MockNetworkAPIs) UpdateExternalServiceIpRules(arg0 []netlink.Rule, arg1 []string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateExternalServiceIpRules", ruleList, externalIPs) + ret := m.ctrl.Call(m, "UpdateExternalServiceIpRules", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // UpdateExternalServiceIpRules indicates an expected call of UpdateExternalServiceIpRules. -func (mr *MockNetworkAPIsMockRecorder) UpdateExternalServiceIpRules(ruleList, externalIPs interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) UpdateExternalServiceIpRules(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateExternalServiceIpRules", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateExternalServiceIpRules), ruleList, externalIPs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateExternalServiceIpRules", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateExternalServiceIpRules), arg0, arg1) } // UpdateHostIptablesRules mocks base method. -func (m *MockNetworkAPIs) UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, v6Enabled bool) error { +func (m *MockNetworkAPIs) UpdateHostIptablesRules(arg0 []string, arg1 string, arg2 *net.IP, arg3 bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateHostIptablesRules", vpcCIDRs, primaryMAC, primaryAddr, v6Enabled) + ret := m.ctrl.Call(m, "UpdateHostIptablesRules", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) return ret0 } // UpdateHostIptablesRules indicates an expected call of UpdateHostIptablesRules. -func (mr *MockNetworkAPIsMockRecorder) UpdateHostIptablesRules(vpcCIDRs, primaryMAC, primaryAddr, v6Enabled interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) UpdateHostIptablesRules(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateHostIptablesRules", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateHostIptablesRules), vpcCIDRs, primaryMAC, primaryAddr, v6Enabled) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateHostIptablesRules", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateHostIptablesRules), arg0, arg1, arg2, arg3) } // UpdateRuleListBySrc mocks base method. -func (m *MockNetworkAPIs) UpdateRuleListBySrc(ruleList []netlink.Rule, src net.IPNet) error { +func (m *MockNetworkAPIs) UpdateRuleListBySrc(arg0 []netlink.Rule, arg1 net.IPNet) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateRuleListBySrc", ruleList, src) + ret := m.ctrl.Call(m, "UpdateRuleListBySrc", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // UpdateRuleListBySrc indicates an expected call of UpdateRuleListBySrc. -func (mr *MockNetworkAPIsMockRecorder) UpdateRuleListBySrc(ruleList, src interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) UpdateRuleListBySrc(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateRuleListBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateRuleListBySrc), ruleList, src) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateRuleListBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateRuleListBySrc), arg0, arg1) } // UseExternalSNAT mocks base method. @@ -236,3 +234,19 @@ func (mr *MockNetworkAPIsMockRecorder) UseExternalSNAT() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UseExternalSNAT", reflect.TypeOf((*MockNetworkAPIs)(nil).UseExternalSNAT)) } + +// GetRouteTableNumberForENI mocks base method. +func (m *MockNetworkAPIs) GetRouteTableNumberForENI(arg0 int, arg1 string, arg2 int, arg3 int, arg4 bool) (int, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRouteTableNumberForENI", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetRouteTableNumberForENI indicates an expected call of GetRouteTableNumberForENI. +func (mr *MockNetworkAPIsMockRecorder) GetRouteTableNumberForENI(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRouteTableNumberForENI", reflect.TypeOf((*MockNetworkAPIs)(nil).GetRouteTableNumberForENI), arg0, arg1, arg2, arg3, arg4) +} diff --git a/pkg/types/networkinterface.go b/pkg/types/networkinterface.go new file mode 100644 index 0000000000..72f2ef147a --- /dev/null +++ b/pkg/types/networkinterface.go @@ -0,0 +1,7 @@ +package types + +type NetworkInterface struct { + ID string + Allocable bool + Primary bool +} diff --git a/test/framework/resources/k8s/resources/pod.go b/test/framework/resources/k8s/resources/pod.go index e2fff6b67a..92c854c4ec 100644 --- a/test/framework/resources/k8s/resources/pod.go +++ b/test/framework/resources/k8s/resources/pod.go @@ -35,6 +35,7 @@ import ( type PodManager interface { PodExec(namespace string, name string, command []string) (string, string, error) + PodExecWithContainer(namespace string, name string, container string, command []string) (string, string, error) PodLogs(namespace string, name string) (string, error) GetPodsWithLabelSelector(labelKey string, labelVal string) (v1.PodList, error) GetPodsWithLabelSelectorMap(labels map[string]string) (v1.PodList, error) @@ -202,6 +203,34 @@ func (d *defaultPodManager) PodExec(namespace string, name string, command []str return stdout.String(), stderr.String(), err } +func (d *defaultPodManager) PodExecWithContainer(namespace string, name string, container string, command []string) (string, string, error) { + execOptions := &v1.PodExecOptions{ + Container: container, + Stdout: true, + Stderr: true, + Command: command, + } + + req := d.k8sClientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(name). + Namespace(namespace). + SubResource("exec"). + VersionedParams(execOptions, runtime.NewParameterCodec(d.k8sSchema)) + + exec, err := remotecommand.NewSPDYExecutor(d.config, http.MethodPost, req.URL()) + if err != nil { + return "", "", err + } + + var stdout, stderr bytes.Buffer + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + return stdout.String(), stderr.String(), err +} + func (d *defaultPodManager) GetPodsWithLabelSelector(labelKey string, labelVal string) (v1.PodList, error) { ctx := context.Background() podList := v1.PodList{} diff --git a/test/integration/eni-subnet-discovery/eni_subnet_discovery_enhanced_test.go b/test/integration/eni-subnet-discovery/eni_subnet_discovery_enhanced_test.go new file mode 100644 index 0000000000..cab37b7515 --- /dev/null +++ b/test/integration/eni-subnet-discovery/eni_subnet_discovery_enhanced_test.go @@ -0,0 +1,610 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package eni_subnet_discovery + +import ( + "context" + "fmt" + "net" + "os" + "time" + + "github.com/apparentlymart/go-cidr/cidr" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/integration/common" + "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/apps/v1" +) + +const ( + enhancedPodLabelKey = "test-type" + enhancedPodLabelVal = "eni-subnet-enhanced" +) + +var customSGID string +var primarySubnetID string + +// This file contains additional tests for the enhanced subnet discovery functionality +// including primary subnet exclusion, custom security groups, and cluster-specific tags + +var _ = Describe("ENI Subnet Discovery Enhanced Tests", func() { + var ( + deployment *v1.Deployment + ) + + Context("when subnet discovery is enabled", func() { + BeforeEach(func() { + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "ENABLE_SUBNET_DISCOVERY": "true", + }) + time.Sleep(utils.PollIntervalMedium) + + // Get primary subnet ID + primarySubnetID = *primaryInstance.SubnetId + }) + + Context("when primary subnet is excluded with tag value 0", func() { + BeforeEach(func() { + By("Tagging primary subnet with kubernetes.io/role/cni=0") + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{primarySubnetID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Tagging secondary subnet with kubernetes.io/role/cni=1") + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{createdSubnet}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + By("Removing tags from primary subnet") + _, err = f.CloudServices.EC2(). + DeleteTags( + context.TODO(), + []string{primarySubnetID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Removing tags from secondary subnet") + _, err = f.CloudServices.EC2(). + DeleteTags( + context.TODO(), + []string{createdSubnet}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should create ENIs only in secondary subnet", func() { + By("creating deployment") + container := manifest.NewNetCatAlpineContainer(f.Options.TestImageRegistry). + Command([]string{"sleep"}). + Args([]string{"3600"}). + Build() + + deploymentBuilder := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). + Container(container). + Replicas(30). // Enough to require secondary ENIs + PodLabel(enhancedPodLabelKey, enhancedPodLabelVal). + NodeName(*primaryInstance.PrivateDnsName). + Build() + + deployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deploymentBuilder, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + // Allow deployment to stabilize + time.Sleep(10 * time.Second) + + By("verifying all secondary ENIs are in the secondary subnet") + instance, err := f.CloudServices.EC2().DescribeInstance(context.TODO(), *primaryInstance.InstanceId) + Expect(err).ToNot(HaveOccurred()) + + secondaryENICount := 0 + for _, nwInterface := range instance.NetworkInterfaces { + if !common.IsPrimaryENI(nwInterface, instance.PrivateIpAddress) { + secondaryENICount++ + // All secondary ENIs should be in the secondary subnet + Expect(*nwInterface.SubnetId).To(Equal(createdSubnet)) + Expect(*nwInterface.SubnetId).ToNot(Equal(primarySubnetID)) + } + } + + By("verifying at least one secondary ENI was created") + Expect(secondaryENICount).To(BeNumerically(">", 0)) + + By("deleting deployment") + err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + + By("sleeping to allow CNI Plugin to delete unused ENIs") + time.Sleep(time.Second * 90) + }) + }) + + Context("when using custom security groups for secondary subnets", func() { + BeforeEach(func() { + By("Creating custom security group") + createSecurityGroupOutput, err := f.CloudServices.EC2(). + CreateSecurityGroup(context.TODO(), "cni-subnet-discovery-test", "custom security group for CNI", f.Options.AWSVPCID) + Expect(err).ToNot(HaveOccurred()) + customSGID = *createSecurityGroupOutput.GroupId + + By("Tagging custom security group with kubernetes.io/role/cni=1") + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{customSGID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Tagging secondary subnet with kubernetes.io/role/cni=1") + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{createdSubnet}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + By("Removing tags from secondary subnet") + _, err = f.CloudServices.EC2(). + DeleteTags( + context.TODO(), + []string{createdSubnet}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Deleting custom security group") + err = f.CloudServices.EC2().DeleteSecurityGroup(context.TODO(), customSGID) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should use custom security group for ENIs in secondary subnet", func() { + By("creating deployment") + container := manifest.NewNetCatAlpineContainer(f.Options.TestImageRegistry). + Command([]string{"sleep"}). + Args([]string{"3600"}). + Build() + + deploymentBuilder := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). + Container(container). + Replicas(30). // Enough to require secondary ENIs + PodLabel(enhancedPodLabelKey, enhancedPodLabelVal). + NodeName(*primaryInstance.PrivateDnsName). + Build() + + deployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deploymentBuilder, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + // Allow deployment to stabilize + time.Sleep(10 * time.Second) + + By("verifying secondary ENIs use custom security group") + instance, err := f.CloudServices.EC2().DescribeInstance(context.TODO(), *primaryInstance.InstanceId) + Expect(err).ToNot(HaveOccurred()) + + // Get primary ENI security groups for comparison + var primaryENISGs []string + for _, nwInterface := range instance.NetworkInterfaces { + if common.IsPrimaryENI(nwInterface, instance.PrivateIpAddress) { + for _, sg := range nwInterface.Groups { + primaryENISGs = append(primaryENISGs, *sg.GroupId) + } + break + } + } + + // Check secondary ENIs + secondaryENICount := 0 + for _, nwInterface := range instance.NetworkInterfaces { + if !common.IsPrimaryENI(nwInterface, instance.PrivateIpAddress) { + secondaryENICount++ + + // Secondary ENIs in secondary subnet should use custom SG + if *nwInterface.SubnetId == createdSubnet { + hasCustomSG := false + for _, sg := range nwInterface.Groups { + if *sg.GroupId == customSGID { + hasCustomSG = true + break + } + } + Expect(hasCustomSG).To(BeTrue(), "Secondary ENI should have custom security group") + } + } + } + + By("verifying at least one secondary ENI was created") + Expect(secondaryENICount).To(BeNumerically(">", 0)) + + By("deleting deployment") + err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + + By("sleeping to allow CNI Plugin to delete unused ENIs") + time.Sleep(time.Second * 90) + }) + }) + + Context("when using cluster-specific subnet tags", func() { + var clusterName string + + BeforeEach(func() { + // Get the cluster name from environment or use a default + clusterName = os.Getenv("CLUSTER_NAME") + if clusterName == "" { + Skip("CLUSTER_NAME environment variable not set, skipping cluster-specific tag test") + } + + By("Tagging secondary subnet with cluster-specific tag") + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{createdSubnet}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/cluster/" + clusterName), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + // Create another subnet that has a different cluster tag + By("Creating a subnet for a different cluster") + var subnetCidr *net.IPNet + if useIPv6 { + // For IPv6, calculate the appropriate number of bits to get a /64 subnet + prefixLen, _ := cidrRange.Mask.Size() + if prefixLen > 64 { + Fail(fmt.Sprintf("IPv6 parent CIDR prefix length must be <= 64, got /%d", prefixLen)) + } + // Calculate how many bits we need to extend to reach /64 + bitsToExtend := 64 - prefixLen + subnetCidr, err = cidr.Subnet(cidrRange, bitsToExtend, 1) // Use index 1 for different subnet + Expect(err).ToNot(HaveOccurred()) + } else { + subnetCidr, err = cidr.Subnet(cidrRange, 2, 1) // Use a different subnet + Expect(err).ToNot(HaveOccurred()) + } + + otherSubnetOutput, err := f.CloudServices.EC2(). + CreateSubnet(context.TODO(), subnetCidr.String(), f.Options.AWSVPCID, *primaryInstance.Placement.AvailabilityZone) + Expect(err).ToNot(HaveOccurred()) + + otherSubnetID := *otherSubnetOutput.Subnet.SubnetId + + By("Tagging other subnet with different cluster tag") + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{otherSubnetID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/cluster/different-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + By("Removing tags from secondary subnet") + _, err = f.CloudServices.EC2(). + DeleteTags( + context.TODO(), + []string{createdSubnet}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/cluster/" + clusterName), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should only use subnets tagged for this cluster", func() { + By("creating deployment") + container := manifest.NewNetCatAlpineContainer(f.Options.TestImageRegistry). + Command([]string{"sleep"}). + Args([]string{"3600"}). + Build() + + deploymentBuilder := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). + Container(container). + Replicas(30). // Enough to require secondary ENIs + PodLabel(enhancedPodLabelKey, enhancedPodLabelVal). + NodeName(*primaryInstance.PrivateDnsName). + Build() + + deployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deploymentBuilder, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + // Allow deployment to stabilize + time.Sleep(10 * time.Second) + + By("verifying secondary ENIs are only in cluster-tagged subnet") + instance, err := f.CloudServices.EC2().DescribeInstance(context.TODO(), *primaryInstance.InstanceId) + Expect(err).ToNot(HaveOccurred()) + + secondaryENICount := 0 + for _, nwInterface := range instance.NetworkInterfaces { + if !common.IsPrimaryENI(nwInterface, instance.PrivateIpAddress) { + secondaryENICount++ + // All secondary ENIs should be in the cluster-tagged subnet + Expect(*nwInterface.SubnetId).To(Equal(createdSubnet)) + } + } + + By("verifying at least one secondary ENI was created") + Expect(secondaryENICount).To(BeNumerically(">", 0)) + + By("deleting deployment") + err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + + By("sleeping to allow CNI Plugin to delete unused ENIs") + time.Sleep(time.Second * 90) + }) + }) + + Context("when security group tags change after ENI creation (automatic refresh)", func() { + var ( + refreshTestSGID string + testENIID string + ) + + BeforeEach(func() { + By("Creating custom security group for refresh testing (initially untagged)") + createSecurityGroupOutput, err := f.CloudServices.EC2(). + CreateSecurityGroup(context.TODO(), "cni-refresh-test-sg", "Test SG for automatic refresh", f.Options.AWSVPCID) + Expect(err).ToNot(HaveOccurred()) + refreshTestSGID = *createSecurityGroupOutput.GroupId + + By("Tagging secondary subnet to enable ENI creation there") + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{createdSubnet}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + By("Cleaning up refresh test security group") + if refreshTestSGID != "" { + err := f.CloudServices.EC2().DeleteSecurityGroup(context.TODO(), refreshTestSGID) + if err != nil { + GinkgoWriter.Printf("Warning: Failed to delete refresh test SG %s: %v\n", refreshTestSGID, err) + } + } + + By("Removing tags from secondary subnet") + _, err = f.CloudServices.EC2(). + DeleteTags( + context.TODO(), + []string{createdSubnet}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should automatically apply newly tagged custom security groups to existing secondary ENIs", func() { + By("creating deployment to force secondary ENI creation") + container := manifest.NewNetCatAlpineContainer(f.Options.TestImageRegistry). + Command([]string{"sleep"}). + Args([]string{"3600"}). + Build() + + deploymentBuilder := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). + Container(container). + Replicas(25). // Enough to require secondary ENIs + PodLabel("refresh-test", "sg-auto-refresh"). + NodeName(*primaryInstance.PrivateDnsName). + Build() + + deployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deploymentBuilder, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + defer func() { + err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + }() + + // Allow deployment to stabilize and ENIs to be created + time.Sleep(15 * time.Second) + + By("finding secondary ENI created in the tagged secondary subnet") + var secondaryENIs []string + Eventually(func() bool { + instance, err := f.CloudServices.EC2().DescribeInstance(context.TODO(), *primaryInstance.InstanceId) + if err != nil { + return false + } + + secondaryENIs = []string{} + for _, nwInterface := range instance.NetworkInterfaces { + if !common.IsPrimaryENI(nwInterface, instance.PrivateIpAddress) && *nwInterface.SubnetId == createdSubnet { + secondaryENIs = append(secondaryENIs, *nwInterface.NetworkInterfaceId) + if testENIID == "" { + testENIID = *nwInterface.NetworkInterfaceId + } + } + } + return len(secondaryENIs) > 0 + }, time.Minute*2, time.Second*10).Should(BeTrue(), "Should create at least one secondary ENI in tagged subnet") + + By("verifying secondary ENI initially uses primary security groups") + var primarySGs []string + instance, err := f.CloudServices.EC2().DescribeInstance(context.TODO(), *primaryInstance.InstanceId) + Expect(err).ToNot(HaveOccurred()) + + // Get primary ENI security groups + for _, nwInterface := range instance.NetworkInterfaces { + if common.IsPrimaryENI(nwInterface, instance.PrivateIpAddress) { + for _, sg := range nwInterface.Groups { + primarySGs = append(primarySGs, *sg.GroupId) + } + break + } + } + + // Verify secondary ENI has primary SGs initially (and not the refresh test SG) + Eventually(func() []string { + eni, err := f.CloudServices.EC2().DescribeNetworkInterface(context.TODO(), []string{testENIID}) + if err != nil || len(eni.NetworkInterfaces) == 0 { + return nil + } + var sgIDs []string + for _, sg := range eni.NetworkInterfaces[0].Groups { + sgIDs = append(sgIDs, *sg.GroupId) + } + return sgIDs + }, time.Second*30, time.Second*5).Should(And( + ContainElements(primarySGs), + Not(ContainElement(refreshTestSGID)), + ), "Secondary ENI should initially have primary security groups") + + By("tagging custom security group with kubernetes.io/role/cni=1 to trigger refresh") + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{refreshTestSGID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("waiting for automatic refresh to detect and apply the new custom security group") + Eventually(func() []string { + eni, err := f.CloudServices.EC2().DescribeNetworkInterface(context.TODO(), []string{testENIID}) + if err != nil || len(eni.NetworkInterfaces) == 0 { + GinkgoWriter.Printf("Error describing ENI %s: %v\n", testENIID, err) + return nil + } + var sgIDs []string + for _, sg := range eni.NetworkInterfaces[0].Groups { + sgIDs = append(sgIDs, *sg.GroupId) + } + GinkgoWriter.Printf("Current ENI %s security groups: %v\n", testENIID, sgIDs) + return sgIDs + }, time.Second*50, time.Second*5).Should(And( + ContainElement(refreshTestSGID), + Not(ContainElements(primarySGs)), + ), "Custom security group should be automatically applied within 50 seconds") + + By("verifying the change persists after another refresh cycle") + time.Sleep(35 * time.Second) + + eni, err := f.CloudServices.EC2().DescribeNetworkInterface(context.TODO(), []string{testENIID}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(eni.NetworkInterfaces)).To(BeNumerically(">", 0)) + + var finalSGIDs []string + for _, sg := range eni.NetworkInterfaces[0].Groups { + finalSGIDs = append(finalSGIDs, *sg.GroupId) + } + + Expect(finalSGIDs).To(ContainElement(refreshTestSGID), "Custom SG should persist after additional refresh cycle") + Expect(finalSGIDs).ToNot(ContainElements(primarySGs), "Primary SGs should remain replaced") + }) + }) + }) +}) diff --git a/test/integration/eni-subnet-discovery/eni_subnet_discovery_primary_exclusion_test.go b/test/integration/eni-subnet-discovery/eni_subnet_discovery_primary_exclusion_test.go new file mode 100644 index 0000000000..62675de36d --- /dev/null +++ b/test/integration/eni-subnet-discovery/eni_subnet_discovery_primary_exclusion_test.go @@ -0,0 +1,500 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package eni_subnet_discovery + +import ( + "context" + "encoding/json" + "fmt" + "net" + "strings" + "time" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/integration/common" + "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +const ( + primaryExclusionPodLabelKey = "test-type" + primaryExclusionPodLabelVal = "primary-eni-exclusion" + awsNodeLabelKey = "k8s-app" +) + +// ENI introspection data structures +type ENIInfo struct { + ID string `json:"id"` + IsPrimary bool `json:"isPrimary"` + IsExcludedForPodIPs bool `json:"isExcludedForPodIPs"` + DeviceNumber int `json:"deviceNumber"` +} + +type IntrospectResponse struct { + ENIs map[string]ENIInfo `json:"enis"` +} + +var _ = Describe("Primary ENI Exclusion Tests", func() { + var ( + initialDeployment *v1.Deployment + primaryENIID string + primarySubnetID string + primarySubnetCIDR *net.IPNet + secondarySubnetCIDR *net.IPNet + ) + + Context("when subnet discovery is enabled", func() { + BeforeEach(func() { + By("Enabling subnet discovery") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "ENABLE_SUBNET_DISCOVERY": "true", + }) + time.Sleep(utils.PollIntervalMedium) + + By("Getting primary ENI information") + primaryENIID, primarySubnetID, primarySubnetCIDR = getPrimaryENIInfo() + + By("Getting secondary subnet CIDR") + secondarySubnetCIDR = getSubnetCIDR(createdSubnet) + }) + + Context("when primary ENI is excluded via subnet tagging", func() { + BeforeEach(func() { + By("Creating initial deployment to populate primary ENI") + container := manifest.NewNetCatAlpineContainer(f.Options.TestImageRegistry). + Command([]string{"sleep"}). + Args([]string{"3600"}). + Build() + + deploymentBuilder := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). + Container(container). + Replicas(3). // Small deployment initially + PodLabel(primaryExclusionPodLabelKey, primaryExclusionPodLabelVal). + NodeName(*primaryInstance.PrivateDnsName). + Build() + + var err error + initialDeployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deploymentBuilder, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + By("Verifying initial pods are using primary ENI subnet") + validatePodsInSubnet(initialDeployment, primarySubnetCIDR, "primary") + + By("Tagging primary subnet as excluded (kubernetes.io/role/cni=0)") + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{primarySubnetID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Tagging secondary subnet as included (kubernetes.io/role/cni=1)") + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{createdSubnet}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Restarting aws-node pods to apply subnet discovery changes") + restartAwsNodePods() + + By("Waiting for configuration to take effect") + time.Sleep(30 * time.Second) + }) + + AfterEach(func() { + By("Cleaning up deployment") + if initialDeployment != nil { + err := f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(initialDeployment) + Expect(err).ToNot(HaveOccurred()) + } + + By("Removing tags from primary subnet") + _, err := f.CloudServices.EC2(). + DeleteTags( + context.TODO(), + []string{primarySubnetID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Removing tags from secondary subnet") + _, err = f.CloudServices.EC2(). + DeleteTags( + context.TODO(), + []string{createdSubnet}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Restarting aws-node pods to restore default configuration") + restartAwsNodePods() + + By("Waiting for cleanup and CNI restart") + time.Sleep(60 * time.Second) + }) + + It("should handle primary ENI exclusion gracefully", func() { + By("Verifying existing pods on primary ENI continue to work") + validateExistingPodsConnectivity(initialDeployment) + + By("Verifying primary ENI is marked as excluded in datastore") + validatePrimaryENIExclusionInDatastore(primaryENIID) + + By("Scaling deployment to force new ENI creation") + scaledDeployment := scaleDeployment(initialDeployment, 20) // Force secondary ENI creation + + By("Waiting for new pods to be scheduled") + time.Sleep(30 * time.Second) + + By("Verifying new pods are only created on secondary ENIs") + validateNewPodsOnSecondarySubnet(scaledDeployment, secondarySubnetCIDR) + + By("Verifying ENI capacity calculations account for exclusion") + validateENICapacityCalculations() + + By("Testing pod deletion and cleanup") + validatePodDeletionAndCleanup(scaledDeployment) + }) + + It("should work with prefix delegation enabled", func() { + By("Enabling prefix delegation") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "ENABLE_PREFIX_DELEGATION": "true", + }) + + By("Waiting for prefix delegation to take effect") + time.Sleep(30 * time.Second) + + By("Scaling deployment to test prefix delegation with exclusion") + scaledDeployment := scaleDeployment(initialDeployment, 15) + + By("Verifying new pods use secondary subnet with prefix delegation") + validateNewPodsOnSecondarySubnet(scaledDeployment, secondarySubnetCIDR) + + By("Verifying prefix cleanup on excluded primary ENI") + validatePrefixCleanupOnExcludedENI(primaryENIID) + + By("Disabling prefix delegation") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "ENABLE_PREFIX_DELEGATION": "false", + }) + }) + }) + }) +}) + +// Helper functions + +func getPrimaryENIInfo() (string, string, *net.IPNet) { + instance, err := f.CloudServices.EC2().DescribeInstance(context.TODO(), *primaryInstance.InstanceId) + Expect(err).ToNot(HaveOccurred()) + + var primaryENIID string + var primarySubnetID string + + for _, nwInterface := range instance.NetworkInterfaces { + if common.IsPrimaryENI(nwInterface, instance.PrivateIpAddress) { + primaryENIID = *nwInterface.NetworkInterfaceId + primarySubnetID = *nwInterface.SubnetId + break + } + } + + Expect(primaryENIID).ToNot(BeEmpty(), "Should find primary ENI") + Expect(primarySubnetID).ToNot(BeEmpty(), "Should find primary subnet ID") + + // Get subnet CIDR + subnetOutput, err := f.CloudServices.EC2().DescribeSubnets(context.TODO(), []string{primarySubnetID}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(subnetOutput.Subnets)).To(Equal(1)) + + _, primarySubnetCIDR, err := net.ParseCIDR(*subnetOutput.Subnets[0].CidrBlock) + Expect(err).ToNot(HaveOccurred()) + + return primaryENIID, primarySubnetID, primarySubnetCIDR +} + +func getSubnetCIDR(subnetID string) *net.IPNet { + subnetOutput, err := f.CloudServices.EC2().DescribeSubnets(context.TODO(), []string{subnetID}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(subnetOutput.Subnets)).To(Equal(1)) + + _, subnetCIDR, err := net.ParseCIDR(*subnetOutput.Subnets[0].CidrBlock) + Expect(err).ToNot(HaveOccurred()) + + return subnetCIDR +} + +func validatePodsInSubnet(deployment *v1.Deployment, expectedSubnet *net.IPNet, subnetType string) { + pods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(primaryExclusionPodLabelKey, primaryExclusionPodLabelVal) + Expect(err).ToNot(HaveOccurred()) + + podCount := 0 + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning && pod.Status.PodIP != "" { + podIP := net.ParseIP(pod.Status.PodIP) + Expect(expectedSubnet.Contains(podIP)).To(BeTrue(), + "Pod %s IP %s should be in %s subnet %s", + pod.Name, pod.Status.PodIP, subnetType, expectedSubnet.String()) + podCount++ + } + } + + Expect(podCount).To(BeNumerically(">", 0), "Should have pods in %s subnet", subnetType) +} + +func validateExistingPodsConnectivity(deployment *v1.Deployment) { + pods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(primaryExclusionPodLabelKey, primaryExclusionPodLabelVal) + Expect(err).ToNot(HaveOccurred()) + + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning { + By(fmt.Sprintf("Testing connectivity for pod %s", pod.Name)) + // Simple connectivity test - ping Google DNS + stdout, stderr, err := f.K8sResourceManagers.PodManager().PodExec(pod.Namespace, pod.Name, []string{"ping", "-c", "1", "8.8.8.8"}) + if err != nil { + GinkgoWriter.Printf("Pod %s connectivity test failed. stdout: %s, stderr: %s, err: %v\n", pod.Name, stdout, stderr, err) + } + // Note: We don't fail the test on connectivity issues as they might be network policy related + // The main goal is to verify the pod is still scheduled and running + Expect(pod.Status.Phase).To(Equal(corev1.PodRunning), "Pod %s should remain running", pod.Name) + } + } +} + +func validatePrimaryENIExclusionInDatastore(primaryENIID string) { + awsNodePods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(awsNodeLabelKey, utils.AwsNodeName) + Expect(err).ToNot(HaveOccurred()) + + for _, pod := range awsNodePods.Items { + if pod.Spec.NodeName == *primaryInstance.PrivateDnsName { + By(fmt.Sprintf("Checking ENI exclusion in aws-node pod %s", pod.Name)) + + // Execute introspection command + stdout, stderr, err := f.K8sResourceManagers.PodManager().PodExec( + pod.Namespace, + pod.Name, + []string{"/app/aws-k8s-agent", "introspect", "eni"}) + + if err != nil { + GinkgoWriter.Printf("Introspect command failed. stdout: %s, stderr: %s, err: %v\n", stdout, stderr, err) + continue + } + + // Parse the introspection output + var introspectData map[string]interface{} + err = json.Unmarshal([]byte(stdout), &introspectData) + if err != nil { + GinkgoWriter.Printf("Failed to parse introspect output: %v\nOutput: %s\n", err, stdout) + continue + } + + // Find primary ENI and check exclusion status + if enis, ok := introspectData["enis"].(map[string]interface{}); ok { + for eniID, eniData := range enis { + if eniInfo, ok := eniData.(map[string]interface{}); ok { + if isPrimary, exists := eniInfo["isPrimary"].(bool); exists && isPrimary { + if excluded, exists := eniInfo["isExcludedForPodIPs"].(bool); exists { + Expect(excluded).To(BeTrue(), + "Primary ENI %s should be marked as excluded for pod IPs", eniID) + return + } + } + } + } + } + + Fail(fmt.Sprintf("Could not find primary ENI exclusion status in introspect output: %s", stdout)) + } + } +} + +func scaleDeployment(deployment *v1.Deployment, replicas int) *v1.Deployment { + // Get the current deployment to update + currentDeployment, err := f.K8sResourceManagers.DeploymentManager().GetDeployment(deployment.Name, deployment.Namespace) + Expect(err).ToNot(HaveOccurred()) + + // Update the replica count + replicasInt32 := int32(replicas) + currentDeployment.Spec.Replicas = &replicasInt32 + + // Update the deployment and wait for it to be ready + err = f.K8sResourceManagers.DeploymentManager().UpdateAndWaitTillDeploymentIsReady(currentDeployment, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + // Get the updated deployment + updatedDeployment, err := f.K8sResourceManagers.DeploymentManager().GetDeployment(deployment.Name, deployment.Namespace) + Expect(err).ToNot(HaveOccurred()) + + // Wait for pods to be scheduled and running + time.Sleep(45 * time.Second) + + return updatedDeployment +} + +func validateNewPodsOnSecondarySubnet(deployment *v1.Deployment, secondarySubnet *net.IPNet) { + pods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(primaryExclusionPodLabelKey, primaryExclusionPodLabelVal) + Expect(err).ToNot(HaveOccurred()) + + // Get pod creation times to identify new pods + baseTime := time.Now().Add(-2 * time.Minute) // Pods created in last 2 minutes are considered "new" + + newPodCount := 0 + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning && pod.Status.PodIP != "" { + // Check if this is a new pod based on creation time + if pod.CreationTimestamp.Time.After(baseTime) { + podIP := net.ParseIP(pod.Status.PodIP) + Expect(secondarySubnet.Contains(podIP)).To(BeTrue(), + "New pod %s IP %s should be in secondary subnet %s", + pod.Name, pod.Status.PodIP, secondarySubnet.String()) + newPodCount++ + } + } + } + + GinkgoWriter.Printf("Found %d new pods in secondary subnet\n", newPodCount) + // Note: We don't enforce a minimum count as it depends on ENI allocation timing +} + +func validateENICapacityCalculations() { + awsNodePods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(awsNodeLabelKey, utils.AwsNodeName) + Expect(err).ToNot(HaveOccurred()) + + for _, pod := range awsNodePods.Items { + if pod.Spec.NodeName == *primaryInstance.PrivateDnsName { + By(fmt.Sprintf("Checking ENI capacity in aws-node pod %s", pod.Name)) + + // Execute pod-limit introspection + stdout, stderr, err := f.K8sResourceManagers.PodManager().PodExec( + pod.Namespace, + pod.Name, + []string{"/app/aws-k8s-agent", "introspect", "pod-limit"}) + + if err != nil { + GinkgoWriter.Printf("Pod-limit introspect failed. stdout: %s, stderr: %s, err: %v\n", stdout, stderr, err) + continue + } + + GinkgoWriter.Printf("Pod limit introspect output: %s\n", stdout) + + // For now, just verify the command succeeds + // In a more comprehensive test, we would parse the output and verify + // that ENI limits properly account for the excluded primary ENI + Expect(err).ToNot(HaveOccurred(), "Pod limit introspection should succeed") + } + } +} + +func validatePodDeletionAndCleanup(deployment *v1.Deployment) { + By("Scaling down deployment to test cleanup") + scaledDownDeployment := scaleDeployment(deployment, 5) + + By("Waiting for pod deletion and resource cleanup") + time.Sleep(60 * time.Second) + + By("Verifying remaining pods are still functional") + validateExistingPodsConnectivity(scaledDownDeployment) +} + +func validatePrefixCleanupOnExcludedENI(primaryENIID string) { + awsNodePods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(awsNodeLabelKey, utils.AwsNodeName) + Expect(err).ToNot(HaveOccurred()) + + for _, pod := range awsNodePods.Items { + if pod.Spec.NodeName == *primaryInstance.PrivateDnsName { + By(fmt.Sprintf("Checking prefix cleanup on primary ENI in pod %s", pod.Name)) + + // Execute ENI introspection to check prefix allocation + stdout, stderr, err := f.K8sResourceManagers.PodManager().PodExec( + pod.Namespace, + pod.Name, + []string{"/app/aws-k8s-agent", "introspect", "eni", "-v"}) + + if err != nil { + GinkgoWriter.Printf("ENI introspect failed. stdout: %s, stderr: %s, err: %v\n", stdout, stderr, err) + continue + } + + GinkgoWriter.Printf("ENI introspect output for prefix validation: %s\n", stdout) + + // Verify that excluded primary ENI doesn't have unnecessary unassigned prefixes + // This is a basic check - in production we'd parse the JSON output more thoroughly + Expect(strings.Contains(stdout, "isExcludedForPodIPs")).To(BeTrue(), + "ENI introspect should show exclusion status") + } + } +} + +func restartAwsNodePods() { + awsNodePods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(awsNodeLabelKey, utils.AwsNodeName) + Expect(err).ToNot(HaveOccurred()) + + for _, pod := range awsNodePods.Items { + err := f.K8sResourceManagers.PodManager().DeleteAndWaitTillPodDeleted(&pod) + Expect(err).ToNot(HaveOccurred()) + } + + // Wait for new aws-node pods to be ready + time.Sleep(30 * time.Second) + + // Verify aws-node pods are running + Eventually(func() bool { + awsNodePods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(awsNodeLabelKey, utils.AwsNodeName) + if err != nil { + return false + } + + runningCount := 0 + for _, pod := range awsNodePods.Items { + if pod.Status.Phase == corev1.PodRunning { + runningCount++ + } + } + + // Expect at least one aws-node pod to be running + return runningCount > 0 + }, 60*time.Second, 5*time.Second).Should(BeTrue(), "AWS node pods should be running after restart") +} diff --git a/test/integration/eni-subnet-discovery/eni_subnet_discovery_secondary_exclusion_test.go b/test/integration/eni-subnet-discovery/eni_subnet_discovery_secondary_exclusion_test.go new file mode 100644 index 0000000000..55d2fab562 --- /dev/null +++ b/test/integration/eni-subnet-discovery/eni_subnet_discovery_secondary_exclusion_test.go @@ -0,0 +1,421 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package eni_subnet_discovery + +import ( + "context" + "encoding/json" + "fmt" + "net" + "strings" + "time" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +const ( + secondaryExclusionPodLabelKey = "test-type" + secondaryExclusionPodLabelVal = "secondary-eni-exclusion" +) + +var _ = Describe("Secondary ENI Exclusion Tests", func() { + var ( + initialDeployment *v1.Deployment + primarySubnetID string + primarySubnetCIDR *net.IPNet + secondarySubnetCIDR *net.IPNet + secondarySubnetID string + ) + + Context("when subnet discovery is enabled", func() { + BeforeEach(func() { + By("Enabling subnet discovery") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "ENABLE_SUBNET_DISCOVERY": "true", + }) + time.Sleep(utils.PollIntervalMedium) + + By("Getting primary ENI information") + _, primarySubnetID, primarySubnetCIDR = getPrimaryENIInfo() + + By("Getting secondary subnet information") + secondarySubnetID = createdSubnet + secondarySubnetCIDR = getSubnetCIDR(createdSubnet) + + By("Ensuring both subnets are initially included (kubernetes.io/role/cni=1)") + // Tag primary subnet as included + _, err := f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{primarySubnetID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + // Tag secondary subnet as included + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{secondarySubnetID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Restarting aws-node pods to apply initial subnet discovery") + restartAwsNodePods() + + By("Waiting for configuration to take effect") + time.Sleep(30 * time.Second) + }) + + Context("when secondary ENI is excluded via subnet tagging", func() { + BeforeEach(func() { + By("Creating initial deployment to populate both primary and secondary ENIs") + container := manifest.NewNetCatAlpineContainer(f.Options.TestImageRegistry). + Command([]string{"sleep"}). + Args([]string{"3600"}). + Build() + + deploymentBuilder := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). + Container(container). + Replicas(15). // Enough to force secondary ENI creation + PodLabel(secondaryExclusionPodLabelKey, secondaryExclusionPodLabelVal). + NodeName(*primaryInstance.PrivateDnsName). + Build() + + var err error + initialDeployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deploymentBuilder, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + By("Verifying pods are distributed across both subnets") + validatePodsInBothSubnets(initialDeployment, primarySubnetCIDR, secondarySubnetCIDR) + + By("Waiting for secondary ENI to be created and populated") + time.Sleep(45 * time.Second) + + By("Tagging secondary subnet as excluded (kubernetes.io/role/cni=0)") + _, err = f.CloudServices.EC2(). + CreateTags( + context.TODO(), + []string{secondarySubnetID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Restarting aws-node pods to apply subnet discovery changes") + restartAwsNodePods() + + By("Waiting for secondary ENI exclusion to take effect") + time.Sleep(30 * time.Second) + }) + + AfterEach(func() { + By("Cleaning up deployment") + if initialDeployment != nil { + err := f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(initialDeployment) + Expect(err).ToNot(HaveOccurred()) + } + + By("Removing tags from primary subnet") + _, err := f.CloudServices.EC2(). + DeleteTags( + context.TODO(), + []string{primarySubnetID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("1"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Removing tags from secondary subnet") + _, err = f.CloudServices.EC2(). + DeleteTags( + context.TODO(), + []string{secondarySubnetID}, + []ec2types.Tag{ + { + Key: aws.String("kubernetes.io/role/cni"), + Value: aws.String("0"), + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Restarting aws-node pods to restore default configuration") + restartAwsNodePods() + + By("Waiting for cleanup and CNI restart") + time.Sleep(60 * time.Second) + }) + + It("should handle secondary ENI exclusion gracefully", func() { + By("Verifying existing pods on secondary ENI continue to work") + validateExistingPodsConnectivity(initialDeployment) + + By("Verifying secondary ENI is marked as excluded in datastore") + validateSecondaryENIExclusionInDatastore() + + By("Scaling deployment to test new pod allocation behavior") + scaledDeployment := scaleDeployment(initialDeployment, 25) // Add more pods + + By("Waiting for new pods to be scheduled") + time.Sleep(45 * time.Second) + + By("Verifying new pods avoid excluded secondary ENI") + validateNewPodsAvoidExcludedSecondaryENI(scaledDeployment, primarySubnetCIDR, secondarySubnetCIDR) + + By("Verifying ENI capacity calculations account for exclusion") + validateENICapacityCalculations() + + By("Testing secondary ENI cleanup after pod removal") + validateSecondaryENICleanupAfterPodRemoval(scaledDeployment) + }) + + It("should work with prefix delegation enabled", func() { + By("Enabling prefix delegation") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "ENABLE_PREFIX_DELEGATION": "true", + }) + + By("Waiting for prefix delegation to take effect") + time.Sleep(30 * time.Second) + + By("Scaling deployment to test prefix delegation with secondary exclusion") + scaledDeployment := scaleDeployment(initialDeployment, 20) + + By("Verifying new pods use primary subnet with prefix delegation") + validateNewPodsAvoidExcludedSecondaryENI(scaledDeployment, primarySubnetCIDR, secondarySubnetCIDR) + + By("Verifying prefix cleanup on excluded secondary ENI") + validatePrefixCleanupOnExcludedSecondaryENI() + + By("Disabling prefix delegation") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "ENABLE_PREFIX_DELEGATION": "false", + }) + }) + }) + }) +}) + +// Helper functions specific to secondary ENI exclusion + +func validatePodsInBothSubnets(deployment *v1.Deployment, primarySubnet, secondarySubnet *net.IPNet) { + pods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(secondaryExclusionPodLabelKey, secondaryExclusionPodLabelVal) + Expect(err).ToNot(HaveOccurred()) + + primaryPodCount := 0 + secondaryPodCount := 0 + + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning && pod.Status.PodIP != "" { + podIP := net.ParseIP(pod.Status.PodIP) + if primarySubnet.Contains(podIP) { + primaryPodCount++ + } else if secondarySubnet.Contains(podIP) { + secondaryPodCount++ + } + } + } + + Expect(primaryPodCount).To(BeNumerically(">", 0), "Should have pods in primary subnet") + Expect(secondaryPodCount).To(BeNumerically(">", 0), "Should have pods in secondary subnet") + GinkgoWriter.Printf("Found %d pods in primary subnet and %d pods in secondary subnet\n", primaryPodCount, secondaryPodCount) +} + +func validateSecondaryENIExclusionInDatastore() { + awsNodePods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(awsNodeLabelKey, utils.AwsNodeName) + Expect(err).ToNot(HaveOccurred()) + + for _, pod := range awsNodePods.Items { + if pod.Spec.NodeName == *primaryInstance.PrivateDnsName { + By(fmt.Sprintf("Checking secondary ENI exclusion in aws-node pod %s", pod.Name)) + + // Execute introspection command in the aws-node container + stdout, stderr, err := f.K8sResourceManagers.PodManager().PodExecWithContainer( + pod.Namespace, + pod.Name, + "aws-node", // Specify the aws-node container + []string{"/app/aws-k8s-agent", "introspect", "eni"}) + + if err != nil { + GinkgoWriter.Printf("Introspect command failed. stdout: %s, stderr: %s, err: %v\n", stdout, stderr, err) + continue + } + + // Parse the introspection output + var introspectData map[string]interface{} + err = json.Unmarshal([]byte(stdout), &introspectData) + if err != nil { + GinkgoWriter.Printf("Failed to parse introspect output: %v\nOutput: %s\n", err, stdout) + continue + } + + // Find secondary ENI and check exclusion status + foundExcludedSecondaryENI := false + if enis, ok := introspectData["enis"].(map[string]interface{}); ok { + for eniID, eniData := range enis { + if eniInfo, ok := eniData.(map[string]interface{}); ok { + if isPrimary, exists := eniInfo["isPrimary"].(bool); exists && !isPrimary { + // This is a secondary ENI + if excluded, exists := eniInfo["isExcludedForPodIPs"].(bool); exists && excluded { + GinkgoWriter.Printf("Found excluded secondary ENI: %s\n", eniID) + foundExcludedSecondaryENI = true + } + } + } + } + } + + if !foundExcludedSecondaryENI { + GinkgoWriter.Printf("Introspect output for debugging: %s\n", stdout) + } + Expect(foundExcludedSecondaryENI).To(BeTrue(), "Should find at least one excluded secondary ENI") + return + } + } +} + +func validateNewPodsAvoidExcludedSecondaryENI(deployment *v1.Deployment, primarySubnet, secondarySubnet *net.IPNet) { + pods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(secondaryExclusionPodLabelKey, secondaryExclusionPodLabelVal) + Expect(err).ToNot(HaveOccurred()) + + // Get pod creation times to identify new pods + baseTime := time.Now().Add(-3 * time.Minute) // Pods created in last 3 minutes are considered "new" + + newPodCount := 0 + newPodsInSecondarySubnet := 0 + + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning && pod.Status.PodIP != "" { + // Check if this is a new pod based on creation time + if pod.CreationTimestamp.Time.After(baseTime) { + podIP := net.ParseIP(pod.Status.PodIP) + newPodCount++ + + if secondarySubnet.Contains(podIP) { + newPodsInSecondarySubnet++ + GinkgoWriter.Printf("WARNING: New pod %s IP %s is in excluded secondary subnet %s\n", + pod.Name, pod.Status.PodIP, secondarySubnet.String()) + } + } + } + } + + GinkgoWriter.Printf("Found %d new pods, %d of which are in secondary subnet\n", newPodCount, newPodsInSecondarySubnet) + + // New pods should avoid the excluded secondary subnet + Expect(newPodsInSecondarySubnet).To(Equal(0), "New pods should not be placed in excluded secondary subnet") +} + +func validateSecondaryENICleanupAfterPodRemoval(deployment *v1.Deployment) { + By("Scaling down deployment to remove pods from secondary ENI") + scaledDownDeployment := scaleDeployment(deployment, 8) // Scale down to force pod removal + + By("Waiting for pod deletion and potential ENI cleanup") + time.Sleep(90 * time.Second) + + By("Verifying excluded secondary ENI becomes deletable") + validateExcludedSecondaryENIDeletable() + + By("Verifying remaining pods are still functional") + validateExistingPodsConnectivity(scaledDownDeployment) +} + +func validateExcludedSecondaryENIDeletable() { + awsNodePods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(awsNodeLabelKey, utils.AwsNodeName) + Expect(err).ToNot(HaveOccurred()) + + for _, pod := range awsNodePods.Items { + if pod.Spec.NodeName == *primaryInstance.PrivateDnsName { + By(fmt.Sprintf("Checking secondary ENI deletability in aws-node pod %s", pod.Name)) + + // Execute introspection command to check ENI status in the aws-node container + stdout, stderr, err := f.K8sResourceManagers.PodManager().PodExecWithContainer( + pod.Namespace, + pod.Name, + "aws-node", // Specify the aws-node container + []string{"/app/aws-k8s-agent", "introspect", "eni", "-v"}) + + if err != nil { + GinkgoWriter.Printf("ENI introspect failed. stdout: %s, stderr: %s, err: %v\n", stdout, stderr, err) + continue + } + + GinkgoWriter.Printf("ENI introspect output for deletability validation: %s\n", stdout) + + // The excluded secondary ENI should either be deleted or be eligible for deletion + // We verify this by checking that excluded ENIs with no pods are marked appropriately + Expect(strings.Contains(stdout, "isExcludedForPodIPs")).To(BeTrue(), + "ENI introspect should show exclusion status") + } + } +} + +func validatePrefixCleanupOnExcludedSecondaryENI() { + awsNodePods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(awsNodeLabelKey, utils.AwsNodeName) + Expect(err).ToNot(HaveOccurred()) + + for _, pod := range awsNodePods.Items { + if pod.Spec.NodeName == *primaryInstance.PrivateDnsName { + By(fmt.Sprintf("Checking prefix cleanup on secondary ENIs in pod %s", pod.Name)) + + // Execute ENI introspection to check prefix allocation in the aws-node container + stdout, stderr, err := f.K8sResourceManagers.PodManager().PodExecWithContainer( + pod.Namespace, + pod.Name, + "aws-node", // Specify the aws-node container + []string{"/app/aws-k8s-agent", "introspect", "eni", "-v"}) + + if err != nil { + GinkgoWriter.Printf("ENI introspect failed. stdout: %s, stderr: %s, err: %v\n", stdout, stderr, err) + continue + } + + GinkgoWriter.Printf("ENI introspect output for prefix validation: %s\n", stdout) + + // Verify that excluded secondary ENIs don't have unnecessary unassigned prefixes + // This is a basic check - in production we'd parse the JSON output more thoroughly + Expect(strings.Contains(stdout, "isExcludedForPodIPs")).To(BeTrue(), + "ENI introspect should show exclusion status for secondary ENIs") + } + } +} diff --git a/test/integration/eni-subnet-discovery/eni_subnet_discovery_suite_test.go b/test/integration/eni-subnet-discovery/eni_subnet_discovery_suite_test.go index e5509ee814..fdfb36c8dc 100644 --- a/test/integration/eni-subnet-discovery/eni_subnet_discovery_suite_test.go +++ b/test/integration/eni-subnet-discovery/eni_subnet_discovery_suite_test.go @@ -47,11 +47,13 @@ var ( cidrBlockAssociationID string createdSubnet string primaryInstance ec2types.Instance + useIPv6 bool ) // Parse test specific variable from flag func init() { flag.StringVar(&cidrRangeString, "secondary-cidr-range", "100.64.0.0/16", "second cidr range to be associated with the VPC") + flag.BoolVar(&useIPv6, "use-ipv6", false, "Use IPv6 for subnet discovery tests") } var _ = BeforeSuite(func() { @@ -79,9 +81,21 @@ var _ = BeforeSuite(func() { primaryInstance, err = f.CloudServices.EC2().DescribeInstance(context.TODO(), instanceID) Expect(err).ToNot(HaveOccurred()) + // Adjust default CIDR if IPv6 is enabled and no custom CIDR provided + if useIPv6 && cidrRangeString == "100.64.0.0/16" { + cidrRangeString = "2600:1f13:000::/56" // AWS IPv6 example range + } + _, cidrRange, err = net.ParseCIDR(cidrRangeString) Expect(err).ToNot(HaveOccurred()) + // Validate CIDR matches IP version + if useIPv6 { + Expect(cidrRange.IP.To4()).To(BeNil(), "IPv6 mode requires IPv6 CIDR") + } else { + Expect(cidrRange.IP.To4()).ToNot(BeNil(), "IPv4 mode requires IPv4 CIDR") + } + By("creating test namespace") _ = f.K8sResourceManagers.NamespaceManager().CreateNamespace(utils.DefaultTestNamespace) @@ -90,15 +104,54 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) By("associating cidr range to the VPC") - association, err := f.CloudServices.EC2().AssociateVPCCIDRBlock(context.TODO(), f.Options.AWSVPCID, cidrRange.String()) - Expect(err).ToNot(HaveOccurred()) - cidrBlockAssociationID = *association.CidrBlockAssociation.AssociationId + if useIPv6 { + // The current framework only supports IPv4 CIDR association + // For IPv6, we assume the VPC already has IPv6 enabled + // In a production environment, you would extend the framework to support IPv6 association + By("IPv6 mode: assuming VPC already has IPv6 enabled") + + // Verify VPC has IPv6 and get an existing association ID for cleanup + vpcInfo, err := f.CloudServices.EC2().DescribeVPC(context.TODO(), f.Options.AWSVPCID) + Expect(err).ToNot(HaveOccurred()) + + hasIPv6 := false + for _, assoc := range vpcInfo.Vpcs[0].Ipv6CidrBlockAssociationSet { + if assoc.Ipv6CidrBlockState != nil && assoc.Ipv6CidrBlockState.State == "associated" { + hasIPv6 = true + // We won't disassociate existing IPv6, so set empty ID + cidrBlockAssociationID = "" + break + } + } + + if !hasIPv6 { + Skip("IPv6 tests require a VPC with IPv6 already enabled") + } + } else { + // IPv4 association works with current framework + association, err := f.CloudServices.EC2().AssociateVPCCIDRBlock(context.TODO(), f.Options.AWSVPCID, cidrRange.String()) + Expect(err).ToNot(HaveOccurred()) + cidrBlockAssociationID = *association.CidrBlockAssociation.AssociationId + } By(fmt.Sprintf("creating the subnet in %s", *primaryInstance.Placement.AvailabilityZone)) - // Subnet must be greater than /19 - subnetCidr, err := cidr.Subnet(cidrRange, 2, 0) - Expect(err).ToNot(HaveOccurred()) + var subnetCidr *net.IPNet + if useIPv6 { + // For IPv6, calculate the appropriate number of bits to get a /64 subnet + prefixLen, _ := cidrRange.Mask.Size() + if prefixLen > 64 { + Fail(fmt.Sprintf("IPv6 parent CIDR prefix length must be <= 64, got /%d", prefixLen)) + } + // Calculate how many bits we need to extend to reach /64 + bitsToExtend := 64 - prefixLen + subnetCidr, err = cidr.Subnet(cidrRange, bitsToExtend, 0) + Expect(err).ToNot(HaveOccurred()) + } else { + // IPv4: Subnet must be greater than /19 + subnetCidr, err = cidr.Subnet(cidrRange, 2, 0) + Expect(err).ToNot(HaveOccurred()) + } createSubnetOutput, err := f.CloudServices.EC2(). CreateSubnet(context.TODO(), subnetCidr.String(), f.Options.AWSVPCID, *primaryInstance.Placement.AvailabilityZone) @@ -125,20 +178,48 @@ var _ = AfterSuite(func() { _ = f.K8sResourceManagers.NamespaceManager(). DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) - var errs prometheus.MultiError - By("sleeping to allow CNI Plugin to delete unused ENIs") time.Sleep(time.Second * 90) - By(fmt.Sprintf("deleting the subnet %s", createdSubnet)) - errs.Append(f.CloudServices.EC2().DeleteSubnet(context.TODO(), createdSubnet)) - - By("disassociating the CIDR range to the VPC") - errs.Append(f.CloudServices.EC2().DisAssociateVPCCIDRBlock(context.TODO(), cidrBlockAssociationID)) - - Expect(errs.MaybeUnwrap()).ToNot(HaveOccurred()) - By("by setting WARM_ENI_TARGET to 1") k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{"WARM_ENI_TARGET": "1"}) + + var errs prometheus.MultiError + + By(fmt.Sprintf("deleting the subnet %s", createdSubnet)) + if err := f.CloudServices.EC2().DeleteSubnet(context.TODO(), createdSubnet); err != nil { + errs.Append(err) + } + + // Wait for subnet deletion to complete before trying to disassociate CIDR + By("waiting for subnet deletion to complete") + time.Sleep(time.Second * 30) + + By("disassociating the CIDR range from the VPC") + if cidrBlockAssociationID != "" { + // Retry CIDR disassociation a few times in case subnet deletion is still in progress + maxRetries := 5 + for i := 0; i < maxRetries; i++ { + if err := f.CloudServices.EC2().DisAssociateVPCCIDRBlock(context.TODO(), cidrBlockAssociationID); err != nil { + if i == maxRetries-1 { + // Last retry failed, append error + errs.Append(err) + } else { + // Wait and retry + By(fmt.Sprintf("CIDR disassociation failed (attempt %d/%d), retrying in 15 seconds", i+1, maxRetries)) + time.Sleep(time.Second * 15) + } + } else { + // Success, break out of retry loop + break + } + } + } + + // Only fail if there were actual errors, not just retry attempts + if errs.MaybeUnwrap() != nil { + GinkgoWriter.Printf("WARNING: Some cleanup operations failed: %v\n", errs.MaybeUnwrap()) + // Don't fail the test suite for cleanup issues, just log them + } }) diff --git a/test/integration/eni-subnet-discovery/eni_subnet_discovery_test.go b/test/integration/eni-subnet-discovery/eni_subnet_discovery_test.go index 8c88697d6d..f238855ab8 100644 --- a/test/integration/eni-subnet-discovery/eni_subnet_discovery_test.go +++ b/test/integration/eni-subnet-discovery/eni_subnet_discovery_test.go @@ -305,16 +305,37 @@ func checkSecondaryENISubnets(expectNewCidr bool) { vpcOutput, err := f.CloudServices.EC2().DescribeVPC(context.TODO(), *primaryInstance.VpcId) Expect(err).ToNot(HaveOccurred()) - expectedCidrRangeString := *vpcOutput.Vpcs[0].CidrBlock - expectedCidrSplit := strings.Split(*vpcOutput.Vpcs[0].CidrBlock, "/") - expectedSuffix, _ := strconv.Atoi(expectedCidrSplit[1]) - _, expectedCIDR, _ := net.ParseCIDR(*vpcOutput.Vpcs[0].CidrBlock) - - if expectNewCidr { - expectedCidrRangeString = cidrRangeString - expectedCidrSplit = strings.Split(cidrRangeString, "/") + var expectedCidrRangeString string + var expectedCIDR *net.IPNet + var expectedSuffix int + + if useIPv6 { + // For IPv6, look for associated IPv6 CIDR blocks + if len(vpcOutput.Vpcs[0].Ipv6CidrBlockAssociationSet) > 0 { + for _, assoc := range vpcOutput.Vpcs[0].Ipv6CidrBlockAssociationSet { + if assoc.Ipv6CidrBlockState != nil && assoc.Ipv6CidrBlockState.State == "associated" { + expectedCidrRangeString = *assoc.Ipv6CidrBlock + break + } + } + } + Expect(expectedCidrRangeString).ToNot(BeEmpty(), "No associated IPv6 CIDR found in VPC") + _, expectedCIDR, _ = net.ParseCIDR(expectedCidrRangeString) + expectedCidrSplit := strings.Split(expectedCidrRangeString, "/") + expectedSuffix, _ = strconv.Atoi(expectedCidrSplit[1]) + } else { + // IPv4 logic + expectedCidrRangeString = *vpcOutput.Vpcs[0].CidrBlock + expectedCidrSplit := strings.Split(*vpcOutput.Vpcs[0].CidrBlock, "/") expectedSuffix, _ = strconv.Atoi(expectedCidrSplit[1]) - _, expectedCIDR, _ = net.ParseCIDR(cidrRangeString) + _, expectedCIDR, _ = net.ParseCIDR(*vpcOutput.Vpcs[0].CidrBlock) + + if expectNewCidr { + expectedCidrRangeString = cidrRangeString + expectedCidrSplit = strings.Split(cidrRangeString, "/") + expectedSuffix, _ = strconv.Atoi(expectedCidrSplit[1]) + _, expectedCIDR, _ = net.ParseCIDR(cidrRangeString) + } } By(fmt.Sprintf("checking the secondary ENI subnets are in the CIDR %s", expectedCidrRangeString)) diff --git a/testdata/amazon-eks-cni-policy-v4.json b/testdata/amazon-eks-cni-policy-v4.json index 42389648ad..a23598ed19 100644 --- a/testdata/amazon-eks-cni-policy-v4.json +++ b/testdata/amazon-eks-cni-policy-v4.json @@ -12,6 +12,8 @@ "ec2:DescribeTags", "ec2:DescribeNetworkInterfaces", "ec2:DescribeInstanceTypes", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", "ec2:DetachNetworkInterface", "ec2:ModifyNetworkInterfaceAttribute", "ec2:UnassignPrivateIpAddresses" diff --git a/testdata/eks-cni-policy.json b/testdata/eks-cni-policy.json new file mode 100644 index 0000000000..86093594c7 --- /dev/null +++ b/testdata/eks-cni-policy.json @@ -0,0 +1,35 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AmazonEKSCNIPolicy", + "Effect": "Allow", + "Action": [ + "ec2:AssignPrivateIpAddresses", + "ec2:AttachNetworkInterface", + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeInstances", + "ec2:DescribeTags", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeInstanceTypes", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DetachNetworkInterface", + "ec2:ModifyNetworkInterfaceAttribute", + "ec2:UnassignPrivateIpAddresses" + ], + "Resource": "*" + }, + { + "Sid": "AmazonEKSCNIPolicyENITag", + "Effect": "Allow", + "Action": [ + "ec2:CreateTags" + ], + "Resource": [ + "arn:aws:ec2:*:*:network-interface/*" + ] + } + ] +}