Skip to content

Commit

Permalink
fix: Correctly calculate max pods when multiple network cards are ava…
Browse files Browse the repository at this point in the history 
…ilable (aws#3720)
  • Loading branch information
@ellistarn
ellistarn authored Apr 7, 2023
1 parent 0c4f367 commit 672a832
Show file tree
Hide file tree
Showing 5 changed files with 164 additions and 7 deletions.
15 changes: 15 additions & 0 deletions hack/code/instancetype_testdata_gen.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,21 @@ func getInstanceTypeInfo(info *ec2.InstanceTypeInfo) string {
fmt.Fprintf(src, "MaximumNetworkInterfaces: aws.Int64(%d),\n", lo.FromPtr(info.NetworkInfo.MaximumNetworkInterfaces))
fmt.Fprintf(src, "Ipv4AddressesPerInterface: aws.Int64(%d),\n", lo.FromPtr(info.NetworkInfo.Ipv4AddressesPerInterface))
fmt.Fprintf(src, "EncryptionInTransitSupported: aws.Bool(%t),\n", lo.FromPtr(info.NetworkInfo.EncryptionInTransitSupported))
fmt.Fprintf(src, "DefaultNetworkCardIndex: aws.Int64(%d),\n", lo.FromPtr(info.NetworkInfo.DefaultNetworkCardIndex))
fmt.Fprintf(src, "NetworkCards: []*ec2.NetworkCardInfo{\n")
for _, networkCard := range info.NetworkInfo.NetworkCards {
fmt.Fprintf(src, getNetworkCardInfo(networkCard))
}
fmt.Fprintf(src, "},\n")
fmt.Fprintf(src, "},\n")
return src.String()
}

func getNetworkCardInfo(info *ec2.NetworkCardInfo) string {
src := &bytes.Buffer{}
fmt.Fprintf(src, "{\n")
fmt.Fprintf(src, "NetworkCardIndex: aws.Int64(%d),\n", lo.FromPtr(info.NetworkCardIndex))
fmt.Fprintf(src, "MaximumNetworkInterfaces: aws.Int64(%d),\n", lo.FromPtr(info.MaximumNetworkInterfaces))
fmt.Fprintf(src, "},\n")
return src.String()
}
Expand Down
2 changes: 1 addition & 1 deletion hack/codegen.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ instanceTypeTestData() {
SUBJECT="Instance Type Test Data"

go run hack/code/instancetype_testdata_gen.go --out-file ${GENERATED_FILE} \
--instance-types t3.large,m5.large,m5.xlarge,p3.8xlarge,g4dn.8xlarge,c6g.large,inf1.2xlarge,inf1.6xlarge,m5.metal,dl1.24xlarge
--instance-types t3.large,m5.large,m5.xlarge,p3.8xlarge,g4dn.8xlarge,c6g.large,inf1.2xlarge,inf1.6xlarge,m5.metal,dl1.24xlarge,m6idn.32xlarge

GIT_DIFF=$(git diff --stat "${GENERATED_FILE}")
checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT}" "${GENERATED_FILE}"
Expand Down
119 changes: 119 additions & 0 deletions pkg/fake/zz_generated.describe_instance_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
MaximumNetworkInterfaces: aws.Int64(3),
Ipv4AddressesPerInterface: aws.Int64(10),
EncryptionInTransitSupported: aws.Bool(false),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(3),
},
},
},
},
{
Expand Down Expand Up @@ -86,6 +93,25 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
MaximumNetworkInterfaces: aws.Int64(60),
Ipv4AddressesPerInterface: aws.Int64(50),
EncryptionInTransitSupported: aws.Bool(true),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(15),
},
{
NetworkCardIndex: aws.Int64(1),
MaximumNetworkInterfaces: aws.Int64(15),
},
{
NetworkCardIndex: aws.Int64(2),
MaximumNetworkInterfaces: aws.Int64(15),
},
{
NetworkCardIndex: aws.Int64(3),
MaximumNetworkInterfaces: aws.Int64(15),
},
},
},
},
{
Expand Down Expand Up @@ -124,6 +150,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
MaximumNetworkInterfaces: aws.Int64(4),
Ipv4AddressesPerInterface: aws.Int64(15),
EncryptionInTransitSupported: aws.Bool(true),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(4),
},
},
},
},
{
Expand Down Expand Up @@ -156,6 +189,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
MaximumNetworkInterfaces: aws.Int64(4),
Ipv4AddressesPerInterface: aws.Int64(10),
EncryptionInTransitSupported: aws.Bool(true),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(4),
},
},
},
},
{
Expand Down Expand Up @@ -188,6 +228,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
MaximumNetworkInterfaces: aws.Int64(8),
Ipv4AddressesPerInterface: aws.Int64(30),
EncryptionInTransitSupported: aws.Bool(true),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(8),
},
},
},
},
{
Expand All @@ -211,6 +258,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
MaximumNetworkInterfaces: aws.Int64(3),
Ipv4AddressesPerInterface: aws.Int64(10),
EncryptionInTransitSupported: aws.Bool(false),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(3),
},
},
},
},
{
Expand All @@ -234,6 +288,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
MaximumNetworkInterfaces: aws.Int64(15),
Ipv4AddressesPerInterface: aws.Int64(50),
EncryptionInTransitSupported: aws.Bool(false),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(15),
},
},
},
},
{
Expand All @@ -257,6 +318,50 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
MaximumNetworkInterfaces: aws.Int64(4),
Ipv4AddressesPerInterface: aws.Int64(15),
EncryptionInTransitSupported: aws.Bool(false),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(4),
},
},
},
},
{
InstanceType: aws.String("m6idn.32xlarge"),
SupportedUsageClasses: aws.StringSlice([]string{"on-demand", "spot"}),
SupportedVirtualizationTypes: aws.StringSlice([]string{"hvm"}),
BurstablePerformanceSupported: aws.Bool(false),
BareMetal: aws.Bool(false),
Hypervisor: aws.String("nitro"),
ProcessorInfo: &ec2.ProcessorInfo{
SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
},
VCpuInfo: &ec2.VCpuInfo{
DefaultCores: aws.Int64(64),
DefaultVCpus: aws.Int64(128),
},
MemoryInfo: &ec2.MemoryInfo{
SizeInMiB: aws.Int64(524288),
},
InstanceStorageInfo: &ec2.InstanceStorageInfo{NvmeSupport: aws.String("required"),
TotalSizeInGB: aws.Int64(7600),
},
NetworkInfo: &ec2.NetworkInfo{
MaximumNetworkInterfaces: aws.Int64(14),
Ipv4AddressesPerInterface: aws.Int64(50),
EncryptionInTransitSupported: aws.Bool(true),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(7),
},
{
NetworkCardIndex: aws.Int64(1),
MaximumNetworkInterfaces: aws.Int64(7),
},
},
},
},
{
Expand Down Expand Up @@ -292,6 +397,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
MaximumNetworkInterfaces: aws.Int64(8),
Ipv4AddressesPerInterface: aws.Int64(30),
EncryptionInTransitSupported: aws.Bool(false),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(8),
},
},
},
},
{
Expand All @@ -315,6 +427,13 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
MaximumNetworkInterfaces: aws.Int64(3),
Ipv4AddressesPerInterface: aws.Int64(12),
EncryptionInTransitSupported: aws.Bool(false),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(3),
},
},
},
},
},
Expand Down
23 changes: 21 additions & 2 deletions pkg/providers/instancetype/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -809,6 +809,21 @@ var _ = Describe("Instance Types", func() {
Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10))
})
})
It("should default max pods based off of network interfaces", func() {
instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx)
Expect(err).To(BeNil())
provisioner = test.Provisioner(coretest.ProvisionerOptions{})
for _, info := range instanceInfo {
if *info.InstanceType == "t3.large" {
it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil)
Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 35))
}
if *info.InstanceType == "m6idn.32xlarge" {
it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil)
Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 345))
}
}
})
It("should set max-pods to user-defined value if specified", func() {
instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx)
Expect(err).To(BeNil())
Expand Down Expand Up @@ -856,7 +871,7 @@ var _ = Describe("Instance Types", func() {
provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}})
for _, info := range instanceInfo {
it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil)
limitedPods := resources.Quantity(fmt.Sprint(*info.NetworkInfo.MaximumNetworkInterfaces*(*info.NetworkInfo.Ipv4AddressesPerInterface-1) + 2))
limitedPods := instancetype.ENILimitedPods(info)
Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", limitedPods.Value()))
}
})
Expand Down Expand Up @@ -1242,8 +1257,12 @@ func makeFakeInstances() []*ec2.InstanceTypeInfo {
SizeInMiB: aws.Int64(8192),
},
NetworkInfo: &ec2.NetworkInfo{
MaximumNetworkInterfaces: aws.Int64(3),
Ipv4AddressesPerInterface: aws.Int64(10),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{{
NetworkCardIndex: lo.ToPtr(int64(0)),
MaximumNetworkInterfaces: aws.Int64(3),
}},
},
SupportedUsageClasses: fake.DefaultSupportedUsageClasses,
})
Expand Down
12 changes: 8 additions & 4 deletions pkg/providers/instancetype/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, kc *v1alph
Offerings: offerings,
Capacity: computeCapacity(ctx, info, amiFamily, nodeTemplate.Spec.BlockDeviceMappings, kc),
Overhead: &cloudprovider.InstanceTypeOverhead{
KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, kc), eniLimitedPods(info), amiFamily, kc),
KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, kc), ENILimitedPods(info), amiFamily, kc),
SystemReserved: systemReservedResources(kc),
EvictionThreshold: evictionThreshold(memory(ctx, info), ephemeralStorage(amiFamily, nodeTemplate.Spec.BlockDeviceMappings), amiFamily, kc),
},
Expand Down Expand Up @@ -250,8 +250,12 @@ func habanaGaudis(info *ec2.InstanceTypeInfo) *resource.Quantity {
// The number of pods per node is calculated using the formula:
// max number of ENIs * (IPv4 Addresses per ENI -1) + 2
// https://github.com/awslabs/amazon-eks-ami/blob/master/files/eni-max-pods.txt#L20
func eniLimitedPods(info *ec2.InstanceTypeInfo) *resource.Quantity {
return resources.Quantity(fmt.Sprint(*info.NetworkInfo.MaximumNetworkInterfaces*(*info.NetworkInfo.Ipv4AddressesPerInterface-1) + 2))
func ENILimitedPods(info *ec2.InstanceTypeInfo) *resource.Quantity {
// VPC CNI only uses the default network interface
// https://github.com/aws/amazon-vpc-cni-k8s/blob/3294231c0dce52cfe473bf6c62f47956a3b333b6/scripts/gen_vpc_ip_limits.go#L162
networkInterfaces := *info.NetworkInfo.NetworkCards[*info.NetworkInfo.DefaultNetworkCardIndex].MaximumNetworkInterfaces
addressesPerInterface := *info.NetworkInfo.Ipv4AddressesPerInterface
return resources.Quantity(fmt.Sprint(networkInterfaces*(addressesPerInterface-1) + 2))
}

func systemReservedResources(kc *v1alpha5.KubeletConfiguration) v1.ResourceList {
Expand Down Expand Up @@ -337,7 +341,7 @@ func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.A
case !awssettings.FromContext(ctx).EnableENILimitedPodDensity:
count = 110
default:
count = eniLimitedPods(info).Value()
count = ENILimitedPods(info).Value()
}
if kc != nil && ptr.Int32Value(kc.PodsPerCore) > 0 && amiFamily.FeatureFlags().PodsPerCoreEnabled {
count = lo.Min([]int64{int64(ptr.Int32Value(kc.PodsPerCore)) * ptr.Int64Value(info.VCpuInfo.DefaultVCpus), count})
Expand Down

0 comments on commit 672a832

Please sign in to comment.