diff --git a/hack/code/instancetype_testdata_gen.go b/hack/code/instancetype_testdata_gen.go index 6aa43536b7e6..881faa700372 100644 --- a/hack/code/instancetype_testdata_gen.go +++ b/hack/code/instancetype_testdata_gen.go @@ -157,6 +157,21 @@ func getInstanceTypeInfo(info *ec2.InstanceTypeInfo) string { fmt.Fprintf(src, "MaximumNetworkInterfaces: aws.Int64(%d),\n", lo.FromPtr(info.NetworkInfo.MaximumNetworkInterfaces)) fmt.Fprintf(src, "Ipv4AddressesPerInterface: aws.Int64(%d),\n", lo.FromPtr(info.NetworkInfo.Ipv4AddressesPerInterface)) fmt.Fprintf(src, "EncryptionInTransitSupported: aws.Bool(%t),\n", lo.FromPtr(info.NetworkInfo.EncryptionInTransitSupported)) + fmt.Fprintf(src, "DefaultNetworkCardIndex: aws.Int64(%d),\n", lo.FromPtr(info.NetworkInfo.DefaultNetworkCardIndex)) + fmt.Fprintf(src, "NetworkCards: []*ec2.NetworkCardInfo{\n") + for _, networkCard := range info.NetworkInfo.NetworkCards { + fmt.Fprintf(src, getNetworkCardInfo(networkCard)) + } + fmt.Fprintf(src, "},\n") + fmt.Fprintf(src, "},\n") + return src.String() +} + +func getNetworkCardInfo(info *ec2.NetworkCardInfo) string { + src := &bytes.Buffer{} + fmt.Fprintf(src, "{\n") + fmt.Fprintf(src, "NetworkCardIndex: aws.Int64(%d),\n", lo.FromPtr(info.NetworkCardIndex)) + fmt.Fprintf(src, "MaximumNetworkInterfaces: aws.Int64(%d),\n", lo.FromPtr(info.MaximumNetworkInterfaces)) fmt.Fprintf(src, "},\n") return src.String() } diff --git a/hack/codegen.sh b/hack/codegen.sh index 8026edb1cb91..2ca4aadd37a0 100755 --- a/hack/codegen.sh +++ b/hack/codegen.sh @@ -48,7 +48,7 @@ instanceTypeTestData() { SUBJECT="Instance Type Test Data" go run hack/code/instancetype_testdata_gen.go --out-file ${GENERATED_FILE} \ - --instance-types t3.large,m5.large,m5.xlarge,p3.8xlarge,g4dn.8xlarge,c6g.large,inf1.2xlarge,inf1.6xlarge,m5.metal,dl1.24xlarge + --instance-types t3.large,m5.large,m5.xlarge,p3.8xlarge,g4dn.8xlarge,c6g.large,inf1.2xlarge,inf1.6xlarge,m5.metal,dl1.24xlarge,m6idn.32xlarge GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT}" "${GENERATED_FILE}" diff --git a/pkg/fake/zz_generated.describe_instance_types.go b/pkg/fake/zz_generated.describe_instance_types.go index b38f9ef1b888..209d7d79fc80 100644 --- a/pkg/fake/zz_generated.describe_instance_types.go +++ b/pkg/fake/zz_generated.describe_instance_types.go @@ -48,6 +48,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MaximumNetworkInterfaces: aws.Int64(3), Ipv4AddressesPerInterface: aws.Int64(10), EncryptionInTransitSupported: aws.Bool(false), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(3), + }, + }, }, }, { @@ -86,6 +93,25 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MaximumNetworkInterfaces: aws.Int64(60), Ipv4AddressesPerInterface: aws.Int64(50), EncryptionInTransitSupported: aws.Bool(true), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(15), + }, + { + NetworkCardIndex: aws.Int64(1), + MaximumNetworkInterfaces: aws.Int64(15), + }, + { + NetworkCardIndex: aws.Int64(2), + MaximumNetworkInterfaces: aws.Int64(15), + }, + { + NetworkCardIndex: aws.Int64(3), + MaximumNetworkInterfaces: aws.Int64(15), + }, + }, }, }, { @@ -124,6 +150,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MaximumNetworkInterfaces: aws.Int64(4), Ipv4AddressesPerInterface: aws.Int64(15), EncryptionInTransitSupported: aws.Bool(true), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(4), + }, + }, }, }, { @@ -156,6 +189,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MaximumNetworkInterfaces: aws.Int64(4), Ipv4AddressesPerInterface: aws.Int64(10), EncryptionInTransitSupported: aws.Bool(true), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(4), + }, + }, }, }, { @@ -188,6 +228,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MaximumNetworkInterfaces: aws.Int64(8), Ipv4AddressesPerInterface: aws.Int64(30), EncryptionInTransitSupported: aws.Bool(true), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(8), + }, + }, }, }, { @@ -211,6 +258,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MaximumNetworkInterfaces: aws.Int64(3), Ipv4AddressesPerInterface: aws.Int64(10), EncryptionInTransitSupported: aws.Bool(false), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(3), + }, + }, }, }, { @@ -234,6 +288,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MaximumNetworkInterfaces: aws.Int64(15), Ipv4AddressesPerInterface: aws.Int64(50), EncryptionInTransitSupported: aws.Bool(false), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(15), + }, + }, }, }, { @@ -257,6 +318,50 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MaximumNetworkInterfaces: aws.Int64(4), Ipv4AddressesPerInterface: aws.Int64(15), EncryptionInTransitSupported: aws.Bool(false), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(4), + }, + }, + }, + }, + { + InstanceType: aws.String("m6idn.32xlarge"), + SupportedUsageClasses: aws.StringSlice([]string{"on-demand", "spot"}), + SupportedVirtualizationTypes: aws.StringSlice([]string{"hvm"}), + BurstablePerformanceSupported: aws.Bool(false), + BareMetal: aws.Bool(false), + Hypervisor: aws.String("nitro"), + ProcessorInfo: &ec2.ProcessorInfo{ + SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), + }, + VCpuInfo: &ec2.VCpuInfo{ + DefaultCores: aws.Int64(64), + DefaultVCpus: aws.Int64(128), + }, + MemoryInfo: &ec2.MemoryInfo{ + SizeInMiB: aws.Int64(524288), + }, + InstanceStorageInfo: &ec2.InstanceStorageInfo{NvmeSupport: aws.String("required"), + TotalSizeInGB: aws.Int64(7600), + }, + NetworkInfo: &ec2.NetworkInfo{ + MaximumNetworkInterfaces: aws.Int64(14), + Ipv4AddressesPerInterface: aws.Int64(50), + EncryptionInTransitSupported: aws.Bool(true), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(7), + }, + { + NetworkCardIndex: aws.Int64(1), + MaximumNetworkInterfaces: aws.Int64(7), + }, + }, }, }, { @@ -292,6 +397,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MaximumNetworkInterfaces: aws.Int64(8), Ipv4AddressesPerInterface: aws.Int64(30), EncryptionInTransitSupported: aws.Bool(false), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(8), + }, + }, }, }, { @@ -315,6 +427,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ MaximumNetworkInterfaces: aws.Int64(3), Ipv4AddressesPerInterface: aws.Int64(12), EncryptionInTransitSupported: aws.Bool(false), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{ + { + NetworkCardIndex: aws.Int64(0), + MaximumNetworkInterfaces: aws.Int64(3), + }, + }, }, }, }, diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index c7e2c8515e4d..07c6b16a8ad0 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -809,6 +809,21 @@ var _ = Describe("Instance Types", func() { Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) }) + It("should default max pods based off of network interfaces", func() { + instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) + Expect(err).To(BeNil()) + provisioner = test.Provisioner(coretest.ProvisionerOptions{}) + for _, info := range instanceInfo { + if *info.InstanceType == "t3.large" { + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 35)) + } + if *info.InstanceType == "m6idn.32xlarge" { + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 345)) + } + } + }) It("should set max-pods to user-defined value if specified", func() { instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) @@ -856,7 +871,7 @@ var _ = Describe("Instance Types", func() { provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) - limitedPods := resources.Quantity(fmt.Sprint(*info.NetworkInfo.MaximumNetworkInterfaces*(*info.NetworkInfo.Ipv4AddressesPerInterface-1) + 2)) + limitedPods := instancetype.ENILimitedPods(info) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", limitedPods.Value())) } }) @@ -1242,8 +1257,12 @@ func makeFakeInstances() []*ec2.InstanceTypeInfo { SizeInMiB: aws.Int64(8192), }, NetworkInfo: &ec2.NetworkInfo{ - MaximumNetworkInterfaces: aws.Int64(3), Ipv4AddressesPerInterface: aws.Int64(10), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{{ + NetworkCardIndex: lo.ToPtr(int64(0)), + MaximumNetworkInterfaces: aws.Int64(3), + }}, }, SupportedUsageClasses: fake.DefaultSupportedUsageClasses, }) diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index 7287aff97b94..5e938671fce7 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -58,7 +58,7 @@ func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, kc *v1alph Offerings: offerings, Capacity: computeCapacity(ctx, info, amiFamily, nodeTemplate.Spec.BlockDeviceMappings, kc), Overhead: &cloudprovider.InstanceTypeOverhead{ - KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, kc), eniLimitedPods(info), amiFamily, kc), + KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, kc), ENILimitedPods(info), amiFamily, kc), SystemReserved: systemReservedResources(kc), EvictionThreshold: evictionThreshold(memory(ctx, info), ephemeralStorage(amiFamily, nodeTemplate.Spec.BlockDeviceMappings), amiFamily, kc), }, @@ -250,8 +250,12 @@ func habanaGaudis(info *ec2.InstanceTypeInfo) *resource.Quantity { // The number of pods per node is calculated using the formula: // max number of ENIs * (IPv4 Addresses per ENI -1) + 2 // https://github.com/awslabs/amazon-eks-ami/blob/master/files/eni-max-pods.txt#L20 -func eniLimitedPods(info *ec2.InstanceTypeInfo) *resource.Quantity { - return resources.Quantity(fmt.Sprint(*info.NetworkInfo.MaximumNetworkInterfaces*(*info.NetworkInfo.Ipv4AddressesPerInterface-1) + 2)) +func ENILimitedPods(info *ec2.InstanceTypeInfo) *resource.Quantity { + // VPC CNI only uses the default network interface + // https://github.com/aws/amazon-vpc-cni-k8s/blob/3294231c0dce52cfe473bf6c62f47956a3b333b6/scripts/gen_vpc_ip_limits.go#L162 + networkInterfaces := *info.NetworkInfo.NetworkCards[*info.NetworkInfo.DefaultNetworkCardIndex].MaximumNetworkInterfaces + addressesPerInterface := *info.NetworkInfo.Ipv4AddressesPerInterface + return resources.Quantity(fmt.Sprint(networkInterfaces*(addressesPerInterface-1) + 2)) } func systemReservedResources(kc *v1alpha5.KubeletConfiguration) v1.ResourceList { @@ -337,7 +341,7 @@ func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.A case !awssettings.FromContext(ctx).EnableENILimitedPodDensity: count = 110 default: - count = eniLimitedPods(info).Value() + count = ENILimitedPods(info).Value() } if kc != nil && ptr.Int32Value(kc.PodsPerCore) > 0 && amiFamily.FeatureFlags().PodsPerCoreEnabled { count = lo.Min([]int64{int64(ptr.Int32Value(kc.PodsPerCore)) * ptr.Int64Value(info.VCpuInfo.DefaultVCpus), count})