From 0115e3b8c1fb08fb91ceb9f5520f1382816abdbe Mon Sep 17 00:00:00 2001 From: Ellis Tarn Date: Tue, 21 Feb 2023 17:28:23 -0800 Subject: [PATCH] feat: Support karpenter.sh/instance-network-bandwidth (#3432) Co-authored-by: Brandon Wagner --- go.mod | 2 + go.sum | 5 + hack/api-code-gen.sh | 16 +- hack/code/bandwidth_gen.go | 100 +++ pkg/apis/v1alpha1/register.go | 2 + pkg/cloudprovider/instancetype.go | 5 + pkg/cloudprovider/instancetypes_test.go | 35 +- pkg/cloudprovider/zz_generated.bandwidth.go | 641 ++++++++++++++++++ test/suites/integration/scheduling_test.go | 3 +- .../content/en/preview/concepts/scheduling.md | 1 + 10 files changed, 798 insertions(+), 12 deletions(-) create mode 100644 hack/code/bandwidth_gen.go create mode 100644 pkg/cloudprovider/zz_generated.bandwidth.go diff --git a/go.mod b/go.mod index 3417865bf9f0..203dffbf7f0f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/Pallinder/go-randomdata v1.2.0 + github.com/PuerkitoBio/goquery v1.8.1 github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.44.195 github.com/aws/karpenter-core v0.25.0 @@ -32,6 +33,7 @@ require ( contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect diff --git a/go.sum b/go.sum index 93a7db593d9f..cba60d59ebff 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= @@ -61,6 +63,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= @@ -499,6 +503,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= diff --git a/hack/api-code-gen.sh b/hack/api-code-gen.sh index ae8548868ba6..07cdfc2fd045 100755 --- a/hack/api-code-gen.sh +++ b/hack/api-code-gen.sh @@ -1,12 +1,23 @@ #!/usr/bin/env bash set -euo pipefail -if [ -z ${ENABLE_GIT_PUSH+x} ];then +if [ -z ${ENABLE_GIT_PUSH+x} ]; then ENABLE_GIT_PUSH=false fi echo "api-code-gen running ENABLE_GIT_PUSH: ${ENABLE_GIT_PUSH}" +bandwidth() { + GENERATED_FILE="pkg/cloudprovider/zz_generated.bandwidth.go" + NO_UPDATE='' + SUBJECT="Bandwidth" + + go run hack/code/bandwidth_gen.go -- "${GENERATED_FILE}" + + GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") + checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT}" "${GENERATED_FILE}" +} + pricing() { GENERATED_FILE="pkg/cloudprovider/zz_generated.pricing.go" NO_UPDATE=$' pkg/cloudprovider/zz_generated.pricing.go | 4 ++--\n 1 file changed, 2 insertions(+), 2 deletions(-)' @@ -82,6 +93,7 @@ if [[ $ENABLE_GIT_PUSH == true ]]; then gitOpenAndPullBranch fi +bandwidth pricing vpcLimits -instanceTypeTestData \ No newline at end of file +instanceTypeTestData diff --git a/hack/code/bandwidth_gen.go b/hack/code/bandwidth_gen.go new file mode 100644 index 000000000000..2582c7cccbef --- /dev/null +++ b/hack/code/bandwidth_gen.go @@ -0,0 +1,100 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "go/format" + "log" + "net/http" + "os" + "sort" + "strconv" + + "github.com/PuerkitoBio/goquery" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/samber/lo" +) + +var uriSelectors = map[string]string{ + "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/general-purpose-instances.html": "#general-purpose-network-performance", + "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/compute-optimized-instances.html": "#compute-network-performance", + "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/memory-optimized-instances.html": "#memory-network-perf", + "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/storage-optimized-instances.html": "#storage-network-performance", + "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/accelerated-computing-instances.html": "#gpu-network-performance", +} + +const fileFormat = ` +package cloudprovider + +// GENERATED FILE. DO NOT EDIT DIRECTLY. +// Update hack/code/bandwidth_gen.go and re-generate to edit +// You can add instance types by adding to the --instance-types CLI flag + +var ( + InstanceTypeBandwidthMegabits = map[string]int64{ + %s + } +) +` +func main() { + flag.Parse() + if flag.NArg() != 1 { + log.Fatalf("Usage: `bandwidth_gen.go pkg/cloudprovider/zz_generated.pricing.go`") + } + + bandwidth := map[string]int64{} + + for uri, selector := range uriSelectors { + response := lo.Must(http.Get(uri)) + defer response.Body.Close() + + doc := lo.Must(goquery.NewDocumentFromReader(response.Body)) + for _, row := range doc.Find(selector).Next().Next().Next().Find("tbody").Find("tr").Nodes { + instanceTypeData := row.FirstChild.NextSibling.FirstChild.FirstChild.Data + bandwidthData := row.FirstChild.NextSibling.NextSibling.NextSibling.FirstChild.Data + bandwidth[instanceTypeData] = int64(lo.Must(strconv.ParseFloat(bandwidthData, 64)) * 1000) + } + } + + sess := session.Must(session.NewSession()) + ec2api := ec2.New(sess) + instanceTypesOutput := lo.Must(ec2api.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{})) + allInstanceTypes := lo.Map(instanceTypesOutput.InstanceTypes, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType }) + + instanceTypes := lo.Keys(bandwidth) + // 2d sort for readability + sort.Strings(instanceTypes) + sort.SliceStable(instanceTypes, func(i, j int) bool { + return bandwidth[instanceTypes[i]] < bandwidth[instanceTypes[j]] + }) + + // Generate body + var body string + for _, instanceType := range lo.Without(allInstanceTypes, instanceTypes...) { + body += fmt.Sprintf("// %s is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html\n", instanceType) + } + for _, instanceType := range instanceTypes { + body += fmt.Sprintf("\t\"%s\": %d,\n", instanceType, bandwidth[instanceType]) + } + + // Format and print to the file + formatted := lo.Must(format.Source([]byte(fmt.Sprintf(fileFormat, body)))) + file := lo.Must(os.Create(flag.Args()[0])) + lo.Must(file.Write(formatted)) + file.Close() +} diff --git a/pkg/apis/v1alpha1/register.go b/pkg/apis/v1alpha1/register.go index ee887fc3fe1d..65ae047415b2 100644 --- a/pkg/apis/v1alpha1/register.go +++ b/pkg/apis/v1alpha1/register.go @@ -68,6 +68,7 @@ var ( LabelInstanceSize = LabelDomain + "/instance-size" LabelInstanceCPU = LabelDomain + "/instance-cpu" LabelInstanceMemory = LabelDomain + "/instance-memory" + LabelInstanceNetworkBandwidth = LabelDomain + "/instance-network-bandwidth" LabelInstancePods = LabelDomain + "/instance-pods" LabelInstanceGPUName = LabelDomain + "/instance-gpu-name" LabelInstanceGPUManufacturer = LabelDomain + "/instance-gpu-manufacturer" @@ -106,6 +107,7 @@ func init() { LabelInstanceLocalNVME, LabelInstanceCPU, LabelInstanceMemory, + LabelInstanceNetworkBandwidth, LabelInstancePods, LabelInstanceGPUName, LabelInstanceGPUManufacturer, diff --git a/pkg/cloudprovider/instancetype.go b/pkg/cloudprovider/instancetype.go index 52f0d6f1dd7c..771e1a14fe3b 100644 --- a/pkg/cloudprovider/instancetype.go +++ b/pkg/cloudprovider/instancetype.go @@ -78,6 +78,7 @@ func computeRequirements(ctx context.Context, info *ec2.InstanceTypeInfo, offeri // Well Known to AWS scheduling.NewRequirement(v1alpha1.LabelInstanceCPU, v1.NodeSelectorOpIn, fmt.Sprint(aws.Int64Value(info.VCpuInfo.DefaultVCpus))), scheduling.NewRequirement(v1alpha1.LabelInstanceMemory, v1.NodeSelectorOpIn, fmt.Sprint(aws.Int64Value(info.MemoryInfo.SizeInMiB))), + scheduling.NewRequirement(v1alpha1.LabelInstanceNetworkBandwidth, v1.NodeSelectorOpDoesNotExist), scheduling.NewRequirement(v1alpha1.LabelInstancePods, v1.NodeSelectorOpIn, fmt.Sprint(pods(ctx, info, amiFamily, kc))), scheduling.NewRequirement(v1alpha1.LabelInstanceCategory, v1.NodeSelectorOpDoesNotExist), scheduling.NewRequirement(v1alpha1.LabelInstanceFamily, v1.NodeSelectorOpDoesNotExist), @@ -105,6 +106,10 @@ func computeRequirements(ctx context.Context, info *ec2.InstanceTypeInfo, offeri if info.InstanceStorageInfo != nil && aws.StringValue(info.InstanceStorageInfo.NvmeSupport) != ec2.EphemeralNvmeSupportUnsupported { requirements[v1alpha1.LabelInstanceLocalNVME].Insert(fmt.Sprint(aws.Int64Value(info.InstanceStorageInfo.TotalSizeInGB))) } + // Network bandwidth + if bandwidth, ok := InstanceTypeBandwidthMegabits[aws.StringValue(info.InstanceType)]; ok { + requirements[v1alpha1.LabelInstanceNetworkBandwidth].Insert(fmt.Sprint(bandwidth)) + } // GPU Labels if info.GpuInfo != nil && len(info.GpuInfo.Gpus) == 1 { gpu := info.GpuInfo.Gpus[0] diff --git a/pkg/cloudprovider/instancetypes_test.go b/pkg/cloudprovider/instancetypes_test.go index 267736961e22..65ca67e47f24 100644 --- a/pkg/cloudprovider/instancetypes_test.go +++ b/pkg/cloudprovider/instancetypes_test.go @@ -46,29 +46,46 @@ import ( var _ = Describe("Instance Types", func() { It("should support instance type labels", func() { ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - var pods []*v1.Pod - for key, value := range map[string]string{ + + nodeSelector := map[string]string{ + // Well known + v1alpha5.ProvisionerNameLabelKey: provisioner.Name, + v1.LabelTopologyRegion: "", + v1.LabelTopologyZone: "test-zone-1a", + v1.LabelInstanceTypeStable: "g4dn.8xlarge", + v1.LabelOSStable: "linux", + v1.LabelArchStable: "amd64", + v1alpha5.LabelCapacityType: "on-demand", + // Well Known to AWS v1alpha1.LabelInstanceHypervisor: "nitro", v1alpha1.LabelInstanceEncryptionInTransitSupported: "true", v1alpha1.LabelInstanceCategory: "g", - v1alpha1.LabelInstanceFamily: "g4dn", v1alpha1.LabelInstanceGeneration: "4", + v1alpha1.LabelInstanceFamily: "g4dn", v1alpha1.LabelInstanceSize: "8xlarge", v1alpha1.LabelInstanceCPU: "32", v1alpha1.LabelInstanceMemory: "131072", + v1alpha1.LabelInstanceNetworkBandwidth: "50000", v1alpha1.LabelInstancePods: "58", v1alpha1.LabelInstanceGPUName: "t4", v1alpha1.LabelInstanceGPUManufacturer: "nvidia", v1alpha1.LabelInstanceGPUCount: "1", v1alpha1.LabelInstanceGPUMemory: "16384", v1alpha1.LabelInstanceLocalNVME: "900", - } { - pods = append(pods, coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{key: value}})) - } - ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) + // Deprecated Labels + v1.LabelFailureDomainBetaRegion: "", + v1.LabelFailureDomainBetaZone: "test-zone-1a", + "beta.kubernetes.io/arch": "amd64", + "beta.kubernetes.io/os": "linux", + v1.LabelInstanceType: "g4dn.8xlarge", + "topology.ebs.csi.aws.com/zone": "test-zone-1a", } + + // Ensure that we're exercising all well known labels + Expect(lo.Keys(nodeSelector)).To(ContainElements(append(v1alpha5.WellKnownLabels.UnsortedList(), lo.Keys(v1alpha5.NormalizedLabels)...))) + pod := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: nodeSelector}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) }) It("should not launch AWS Pod ENI on a t3", func() { ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) diff --git a/pkg/cloudprovider/zz_generated.bandwidth.go b/pkg/cloudprovider/zz_generated.bandwidth.go new file mode 100644 index 000000000000..2a6341490b15 --- /dev/null +++ b/pkg/cloudprovider/zz_generated.bandwidth.go @@ -0,0 +1,641 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +// GENERATED FILE. DO NOT EDIT DIRECTLY. +// Update hack/code/instancetype_testdata_gen.go and re-generate to edit +// You can add instance types by adding to the --instance-types CLI flag + +var ( + InstanceTypeBandwidthMegabits = map[string]int64{ + // g3.16xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html + // f1.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html + "t2.nano": 32, + "t3.nano": 32, + "t3a.nano": 32, + "t4g.nano": 32, + "t2.micro": 64, + "t3.micro": 64, + "t3a.micro": 64, + "t4g.micro": 64, + "t1.micro": 70, + "t2.small": 128, + "t3.small": 128, + "t3a.small": 128, + "t4g.small": 128, + "t2.medium": 256, + "t3.medium": 256, + "t3a.medium": 256, + "t4g.medium": 256, + "c1.medium": 300, + "m1.medium": 300, + "m1.small": 300, + "m2.xlarge": 300, + "m3.medium": 300, + "m4.large": 450, + "a1.medium": 500, + "c3.large": 500, + "c6g.medium": 500, + "c6gd.medium": 500, + "m6g.medium": 500, + "m6gd.medium": 500, + "r3.large": 500, + "r6g.medium": 500, + "r6gd.medium": 500, + "x2gd.medium": 500, + "t2.large": 512, + "t3.large": 512, + "t3a.large": 512, + "t4g.large": 512, + "c7g.medium": 520, + "m7g.medium": 520, + "r7g.medium": 520, + "c4.large": 625, + "x1e.xlarge": 625, + "c3.xlarge": 700, + "i2.xlarge": 700, + "m1.large": 700, + "m2.2xlarge": 700, + "m3.large": 700, + "r3.xlarge": 700, + "a1.large": 750, + "c5.large": 750, + "c5a.large": 750, + "c5ad.large": 750, + "c5d.large": 750, + "c6g.large": 750, + "c6gd.large": 750, + "i3.large": 750, + "m4.xlarge": 750, + "m5.large": 750, + "m5a.large": 750, + "m5ad.large": 750, + "m5d.large": 750, + "m6g.large": 750, + "m6gd.large": 750, + "r4.large": 750, + "r5.large": 750, + "r5a.large": 750, + "r5ad.large": 750, + "r5b.large": 750, + "r5d.large": 750, + "r6g.large": 750, + "r6gd.large": 750, + "t2.xlarge": 750, + "x2gd.large": 750, + "z1d.large": 750, + "c6a.large": 781, + "c6i.large": 781, + "c6id.large": 781, + "i4i.large": 781, + "m6a.large": 781, + "m6i.large": 781, + "m6id.large": 781, + "r6a.large": 781, + "r6i.large": 781, + "r6id.large": 781, + "c7g.large": 937, + "m7g.large": 937, + "r7g.large": 937, + "c1.xlarge": 1000, + "c3.2xlarge": 1000, + "i2.2xlarge": 1000, + "m1.xlarge": 1000, + "m2.4xlarge": 1000, + "m3.2xlarge": 1000, + "m3.xlarge": 1000, + "m4.2xlarge": 1000, + "r3.2xlarge": 1000, + "t2.2xlarge": 1000, + "t3.xlarge": 1024, + "t3a.xlarge": 1024, + "t4g.xlarge": 1024, + "a1.xlarge": 1250, + "c4.xlarge": 1250, + "c5.xlarge": 1250, + "c5a.xlarge": 1250, + "c5ad.xlarge": 1250, + "c5d.xlarge": 1250, + "c6g.xlarge": 1250, + "c6gd.xlarge": 1250, + "d2.xlarge": 1250, + "g5g.xlarge": 1250, + "i3.xlarge": 1250, + "m5.xlarge": 1250, + "m5a.xlarge": 1250, + "m5ad.xlarge": 1250, + "m5d.xlarge": 1250, + "m6g.xlarge": 1250, + "m6gd.xlarge": 1250, + "r4.xlarge": 1250, + "r5.xlarge": 1250, + "r5a.xlarge": 1250, + "r5ad.xlarge": 1250, + "r5b.xlarge": 1250, + "r5d.xlarge": 1250, + "r6g.xlarge": 1250, + "r6gd.xlarge": 1250, + "x1e.2xlarge": 1250, + "x2gd.xlarge": 1250, + "z1d.xlarge": 1250, + "c6a.xlarge": 1562, + "c6i.xlarge": 1562, + "c6id.xlarge": 1562, + "is4gen.medium": 1562, + "m6a.xlarge": 1562, + "m6i.xlarge": 1562, + "m6id.xlarge": 1562, + "r6a.xlarge": 1562, + "r6i.xlarge": 1562, + "r6id.xlarge": 1562, + "c6gn.medium": 1600, + "i4i.xlarge": 1875, + "x2iedn.xlarge": 1875, + "c7g.xlarge": 1876, + "m7g.xlarge": 1876, + "r7g.xlarge": 1876, + "c3.4xlarge": 2000, + "g4ad.xlarge": 2000, + "i2.4xlarge": 2000, + "m4.4xlarge": 2000, + "r3.4xlarge": 2000, + "t3.2xlarge": 2048, + "t3a.2xlarge": 2048, + "t4g.2xlarge": 2048, + "i3en.large": 2100, + "m5dn.large": 2100, + "m5n.large": 2100, + "r5dn.large": 2100, + "r5n.large": 2100, + "a1.2xlarge": 2500, + "c4.2xlarge": 2500, + "c5.2xlarge": 2500, + "c5a.2xlarge": 2500, + "c5ad.2xlarge": 2500, + "c5d.2xlarge": 2500, + "c6g.2xlarge": 2500, + "c6gd.2xlarge": 2500, + "d2.2xlarge": 2500, + "g5.xlarge": 2500, + "g5g.2xlarge": 2500, + "h1.2xlarge": 2500, + "i3.2xlarge": 2500, + "m5.2xlarge": 2500, + "m5a.2xlarge": 2500, + "m5ad.2xlarge": 2500, + "m5d.2xlarge": 2500, + "m6g.2xlarge": 2500, + "m6gd.2xlarge": 2500, + "r4.2xlarge": 2500, + "r5.2xlarge": 2500, + "r5a.2xlarge": 2500, + "r5ad.2xlarge": 2500, + "r5b.2xlarge": 2500, + "r5d.2xlarge": 2500, + "r6g.2xlarge": 2500, + "r6gd.2xlarge": 2500, + "x1e.4xlarge": 2500, + "x2gd.2xlarge": 2500, + "z1d.2xlarge": 2500, + "c5n.large": 3000, + "c6gn.large": 3000, + "d3.xlarge": 3000, + "m5zn.large": 3000, + "c6a.2xlarge": 3125, + "c6i.2xlarge": 3125, + "c6id.2xlarge": 3125, + "c6in.large": 3125, + "im4gn.large": 3125, + "is4gen.large": 3125, + "m6a.2xlarge": 3125, + "m6i.2xlarge": 3125, + "m6id.2xlarge": 3125, + "m6idn.large": 3125, + "m6in.large": 3125, + "r6a.2xlarge": 3125, + "r6i.2xlarge": 3125, + "r6id.2xlarge": 3125, + "r6idn.large": 3125, + "r6in.large": 3125, + "trn1.2xlarge": 3125, + "vt1.3xlarge": 3125, + "c7g.2xlarge": 3750, + "m7g.2xlarge": 3750, + "r7g.2xlarge": 3750, + "m5dn.xlarge": 4100, + "m5n.xlarge": 4100, + "r5dn.xlarge": 4100, + "r5n.xlarge": 4100, + "g4ad.2xlarge": 4167, + "i3en.xlarge": 4200, + "i4i.2xlarge": 4687, + "a1.4xlarge": 5000, + "a1.metal": 5000, + "c3.8xlarge": 5000, + "c4.4xlarge": 5000, + "c4.8xlarge": 5000, + "c5.4xlarge": 5000, + "c5a.4xlarge": 5000, + "c5ad.4xlarge": 5000, + "c5d.4xlarge": 5000, + "c5n.xlarge": 5000, + "c6g.4xlarge": 5000, + "c6gd.4xlarge": 5000, + "d2.4xlarge": 5000, + "d2.8xlarge": 5000, + "g4dn.xlarge": 5000, + "g5.2xlarge": 5000, + "g5g.4xlarge": 5000, + "h1.4xlarge": 5000, + "i2.8xlarge": 5000, + "i3.4xlarge": 5000, + "inf1.2xlarge": 5000, + "inf1.xlarge": 5000, + "m4.10xlarge": 5000, + "m4.16xlarge": 5000, + "m5.4xlarge": 5000, + "m5a.4xlarge": 5000, + "m5ad.4xlarge": 5000, + "m5d.4xlarge": 5000, + "m5zn.xlarge": 5000, + "m6g.4xlarge": 5000, + "m6gd.4xlarge": 5000, + "r3.8xlarge": 5000, + "r4.4xlarge": 5000, + "r5.4xlarge": 5000, + "r5a.4xlarge": 5000, + "r5ad.4xlarge": 5000, + "r5b.4xlarge": 5000, + "r5d.4xlarge": 5000, + "r6g.4xlarge": 5000, + "r6gd.4xlarge": 5000, + "x1e.8xlarge": 5000, + "x2gd.4xlarge": 5000, + "x2iedn.2xlarge": 5000, + "z1d.3xlarge": 5000, + "d3.2xlarge": 6000, + "d3en.xlarge": 6000, + "c6a.4xlarge": 6250, + "c6i.4xlarge": 6250, + "c6id.4xlarge": 6250, + "c6in.xlarge": 6250, + "im4gn.xlarge": 6250, + "is4gen.xlarge": 6250, + "m6a.4xlarge": 6250, + "m6i.4xlarge": 6250, + "m6id.4xlarge": 6250, + "m6idn.xlarge": 6250, + "m6in.xlarge": 6250, + "r6a.4xlarge": 6250, + "r6i.4xlarge": 6250, + "r6id.4xlarge": 6250, + "r6idn.xlarge": 6250, + "r6in.xlarge": 6250, + "vt1.6xlarge": 6250, + "c6gn.xlarge": 6300, + "c7g.4xlarge": 7500, + "m5a.8xlarge": 7500, + "m5ad.8xlarge": 7500, + "m7g.4xlarge": 7500, + "r5a.8xlarge": 7500, + "r5ad.8xlarge": 7500, + "r7g.4xlarge": 7500, + "m5dn.2xlarge": 8125, + "m5n.2xlarge": 8125, + "r5dn.2xlarge": 8125, + "r5n.2xlarge": 8125, + "g4ad.4xlarge": 8333, + "i3en.2xlarge": 8400, + "i4i.4xlarge": 9375, + "c5a.8xlarge": 10000, + "c5ad.8xlarge": 10000, + "c5n.2xlarge": 10000, + "g4dn.2xlarge": 10000, + "g5.4xlarge": 10000, + "m5.8xlarge": 10000, + "m5a.12xlarge": 10000, + "m5ad.12xlarge": 10000, + "m5d.8xlarge": 10000, + "m5zn.2xlarge": 10000, + "mac2.metal": 10000, + "r5.8xlarge": 10000, + "r5a.12xlarge": 10000, + "r5ad.12xlarge": 10000, + "r5b.8xlarge": 10000, + "r5d.8xlarge": 10000, + "c5.12xlarge": 12000, + "c5.9xlarge": 12000, + "c5a.12xlarge": 12000, + "c5ad.12xlarge": 12000, + "c5d.12xlarge": 12000, + "c5d.9xlarge": 12000, + "c6g.8xlarge": 12000, + "c6gd.8xlarge": 12000, + "g5g.8xlarge": 12000, + "h1.8xlarge": 12000, + "i3.8xlarge": 12000, + "m5.12xlarge": 12000, + "m5a.16xlarge": 12000, + "m5ad.16xlarge": 12000, + "m5d.12xlarge": 12000, + "m6g.8xlarge": 12000, + "m6gd.8xlarge": 12000, + "r4.8xlarge": 12000, + "r5.12xlarge": 12000, + "r5a.16xlarge": 12000, + "r5ad.16xlarge": 12000, + "r5b.12xlarge": 12000, + "r5d.12xlarge": 12000, + "r6g.8xlarge": 12000, + "r6gd.8xlarge": 12000, + "x1.16xlarge": 12000, + "x1e.16xlarge": 12000, + "x2gd.8xlarge": 12000, + "z1d.6xlarge": 12000, + "c6a.8xlarge": 12500, + "c6gn.2xlarge": 12500, + "c6i.8xlarge": 12500, + "c6id.8xlarge": 12500, + "c6in.2xlarge": 12500, + "d3.4xlarge": 12500, + "d3en.2xlarge": 12500, + "i3en.3xlarge": 12500, + "im4gn.2xlarge": 12500, + "is4gen.2xlarge": 12500, + "m6a.8xlarge": 12500, + "m6i.8xlarge": 12500, + "m6id.8xlarge": 12500, + "m6idn.2xlarge": 12500, + "m6in.2xlarge": 12500, + "r6a.8xlarge": 12500, + "r6i.8xlarge": 12500, + "r6id.8xlarge": 12500, + "r6idn.2xlarge": 12500, + "r6in.2xlarge": 12500, + "x2iedn.4xlarge": 12500, + "x2iezn.2xlarge": 12500, + "c5n.4xlarge": 15000, + "c6gn.4xlarge": 15000, + "c7g.8xlarge": 15000, + "g4ad.8xlarge": 15000, + "m5zn.3xlarge": 15000, + "m7g.8xlarge": 15000, + "r7g.8xlarge": 15000, + "x2iezn.4xlarge": 15000, + "m5dn.4xlarge": 16250, + "m5n.4xlarge": 16250, + "r5dn.4xlarge": 16250, + "r5n.4xlarge": 16250, + "c6a.12xlarge": 18750, + "c6i.12xlarge": 18750, + "c6id.12xlarge": 18750, + "i4i.8xlarge": 18750, + "m6a.12xlarge": 18750, + "m6i.12xlarge": 18750, + "m6id.12xlarge": 18750, + "r6a.12xlarge": 18750, + "r6i.12xlarge": 18750, + "r6id.12xlarge": 18750, + "c5a.16xlarge": 20000, + "c5a.24xlarge": 20000, + "c5ad.16xlarge": 20000, + "c5ad.24xlarge": 20000, + "c6g.12xlarge": 20000, + "c6gd.12xlarge": 20000, + "g4dn.4xlarge": 20000, + "m5.16xlarge": 20000, + "m5a.24xlarge": 20000, + "m5ad.24xlarge": 20000, + "m5d.16xlarge": 20000, + "m6g.12xlarge": 20000, + "m6gd.12xlarge": 20000, + "r5.16xlarge": 20000, + "r5a.24xlarge": 20000, + "r5ad.24xlarge": 20000, + "r5b.16xlarge": 20000, + "r5d.16xlarge": 20000, + "r6g.12xlarge": 20000, + "r6gd.12xlarge": 20000, + "x2gd.12xlarge": 20000, + "c7g.12xlarge": 22500, + "m7g.12xlarge": 22500, + "r7g.12xlarge": 22500, + "c5.18xlarge": 25000, + "c5.24xlarge": 25000, + "c5.metal": 25000, + "c5d.18xlarge": 25000, + "c5d.24xlarge": 25000, + "c5d.metal": 25000, + "c6a.16xlarge": 25000, + "c6g.16xlarge": 25000, + "c6g.metal": 25000, + "c6gd.16xlarge": 25000, + "c6gd.metal": 25000, + "c6i.16xlarge": 25000, + "c6id.16xlarge": 25000, + "c6in.4xlarge": 25000, + "d3.8xlarge": 25000, + "d3en.4xlarge": 25000, + "g4ad.16xlarge": 25000, + "g5.16xlarge": 25000, + "g5.8xlarge": 25000, + "g5g.16xlarge": 25000, + "g5g.metal": 25000, + "h1.16xlarge": 25000, + "i3.16xlarge": 25000, + "i3.metal": 25000, + "i3en.6xlarge": 25000, + "im4gn.4xlarge": 25000, + "inf1.6xlarge": 25000, + "is4gen.4xlarge": 25000, + "m5.24xlarge": 25000, + "m5.metal": 25000, + "m5d.24xlarge": 25000, + "m5d.metal": 25000, + "m5dn.8xlarge": 25000, + "m5n.8xlarge": 25000, + "m6a.16xlarge": 25000, + "m6g.16xlarge": 25000, + "m6g.metal": 25000, + "m6gd.16xlarge": 25000, + "m6gd.metal": 25000, + "m6i.16xlarge": 25000, + "m6id.16xlarge": 25000, + "m6idn.4xlarge": 25000, + "m6in.4xlarge": 25000, + "mac1.metal": 25000, + "r4.16xlarge": 25000, + "r5.24xlarge": 25000, + "r5.metal": 25000, + "r5b.24xlarge": 25000, + "r5b.metal": 25000, + "r5d.24xlarge": 25000, + "r5d.metal": 25000, + "r5dn.8xlarge": 25000, + "r5n.8xlarge": 25000, + "r6a.16xlarge": 25000, + "r6g.16xlarge": 25000, + "r6g.metal": 25000, + "r6gd.16xlarge": 25000, + "r6gd.metal": 25000, + "r6i.16xlarge": 25000, + "r6id.16xlarge": 25000, + "r6idn.4xlarge": 25000, + "r6in.4xlarge": 25000, + "vt1.24xlarge": 25000, + "x1.32xlarge": 25000, + "x1e.32xlarge": 25000, + "x2gd.16xlarge": 25000, + "x2gd.metal": 25000, + "x2iedn.8xlarge": 25000, + "z1d.12xlarge": 25000, + "z1d.metal": 25000, + "c7g.16xlarge": 30000, + "c7g.metal": 30000, + "m7g.16xlarge": 30000, + "m7g.metal": 30000, + "r7g.16xlarge": 30000, + "r7g.metal": 30000, + "c6a.24xlarge": 37500, + "c6i.24xlarge": 37500, + "c6id.24xlarge": 37500, + "i4i.16xlarge": 37500, + "m6a.24xlarge": 37500, + "m6i.24xlarge": 37500, + "m6id.24xlarge": 37500, + "r6a.24xlarge": 37500, + "r6i.24xlarge": 37500, + "r6id.24xlarge": 37500, + "d3en.6xlarge": 40000, + "g5.12xlarge": 40000, + "c5n.9xlarge": 50000, + "c6a.32xlarge": 50000, + "c6a.48xlarge": 50000, + "c6a.metal": 50000, + "c6gn.8xlarge": 50000, + "c6i.32xlarge": 50000, + "c6i.metal": 50000, + "c6id.32xlarge": 50000, + "c6id.metal": 50000, + "c6in.8xlarge": 50000, + "d3en.8xlarge": 50000, + "g4dn.12xlarge": 50000, + "g4dn.16xlarge": 50000, + "g4dn.8xlarge": 50000, + "g5.24xlarge": 50000, + "i3en.12xlarge": 50000, + "im4gn.8xlarge": 50000, + "is4gen.8xlarge": 50000, + "m5dn.12xlarge": 50000, + "m5n.12xlarge": 50000, + "m5zn.6xlarge": 50000, + "m6a.32xlarge": 50000, + "m6a.48xlarge": 50000, + "m6a.metal": 50000, + "m6i.32xlarge": 50000, + "m6i.metal": 50000, + "m6id.32xlarge": 50000, + "m6id.metal": 50000, + "m6idn.8xlarge": 50000, + "m6in.8xlarge": 50000, + "r5dn.12xlarge": 50000, + "r5n.12xlarge": 50000, + "r6a.32xlarge": 50000, + "r6a.48xlarge": 50000, + "r6a.metal": 50000, + "r6i.32xlarge": 50000, + "r6i.metal": 50000, + "r6id.32xlarge": 50000, + "r6id.metal": 50000, + "r6idn.8xlarge": 50000, + "r6in.8xlarge": 50000, + "u-3tb1.56xlarge": 50000, + "x2idn.16xlarge": 50000, + "x2iedn.16xlarge": 50000, + "x2iezn.6xlarge": 50000, + "c6gn.12xlarge": 75000, + "c6in.12xlarge": 75000, + "d3en.12xlarge": 75000, + "i4i.32xlarge": 75000, + "i4i.metal": 75000, + "m5dn.16xlarge": 75000, + "m5n.16xlarge": 75000, + "m6idn.12xlarge": 75000, + "m6in.12xlarge": 75000, + "r5dn.16xlarge": 75000, + "r5n.16xlarge": 75000, + "r6idn.12xlarge": 75000, + "r6in.12xlarge": 75000, + "x2idn.24xlarge": 75000, + "x2iedn.24xlarge": 75000, + "x2iezn.8xlarge": 75000, + "c5n.18xlarge": 100000, + "c5n.metal": 100000, + "c6gn.16xlarge": 100000, + "c6in.16xlarge": 100000, + "g4dn.metal": 100000, + "g5.48xlarge": 100000, + "hpc6a.48xlarge": 100000, + "i3en.24xlarge": 100000, + "i3en.metal": 100000, + "im4gn.16xlarge": 100000, + "inf1.24xlarge": 100000, + "m5dn.24xlarge": 100000, + "m5dn.metal": 100000, + "m5n.24xlarge": 100000, + "m5n.metal": 100000, + "m5zn.12xlarge": 100000, + "m5zn.metal": 100000, + "m6idn.16xlarge": 100000, + "m6in.16xlarge": 100000, + "p3dn.24xlarge": 100000, + "r5dn.24xlarge": 100000, + "r5dn.metal": 100000, + "r5n.24xlarge": 100000, + "r5n.metal": 100000, + "r6idn.16xlarge": 100000, + "r6in.16xlarge": 100000, + "u-12tb1.112xlarge": 100000, + "u-12tb1.metal": 100000, + "u-18tb1.metal": 100000, + "u-24tb1.metal": 100000, + "u-6tb1.112xlarge": 100000, + "u-6tb1.56xlarge": 100000, + "u-6tb1.metal": 100000, + "u-9tb1.112xlarge": 100000, + "u-9tb1.metal": 100000, + "x2idn.32xlarge": 100000, + "x2idn.metal": 100000, + "x2iedn.32xlarge": 100000, + "x2iedn.metal": 100000, + "x2iezn.12xlarge": 100000, + "x2iezn.metal": 100000, + "c6in.24xlarge": 150000, + "m6idn.24xlarge": 150000, + "m6in.24xlarge": 150000, + "r6idn.24xlarge": 150000, + "r6in.24xlarge": 150000, + "c6in.32xlarge": 400000, + "dl1.24xlarge": 400000, + "hpc6id.32xlarge": 400000, + "m6idn.32xlarge": 400000, + "m6in.32xlarge": 400000, + "p4d.24xlarge": 400000, + "p4de.24xlarge": 400000, + "r6idn.32xlarge": 400000, + "r6in.32xlarge": 400000, + "trn1.32xlarge": 800000, + } +) diff --git a/test/suites/integration/scheduling_test.go b/test/suites/integration/scheduling_test.go index 3acd2f6e0503..ff938b1cf24b 100644 --- a/test/suites/integration/scheduling_test.go +++ b/test/suites/integration/scheduling_test.go @@ -80,6 +80,7 @@ var _ = Describe("Scheduling", func() { v1alpha1.LabelInstanceSize: "8xlarge", v1alpha1.LabelInstanceCPU: "32", v1alpha1.LabelInstanceMemory: "131072", + v1alpha1.LabelInstanceNetworkBandwidth: "50000", v1alpha1.LabelInstancePods: "58", // May vary w/ environment v1alpha1.LabelInstanceGPUName: "t4", v1alpha1.LabelInstanceGPUManufacturer: "nvidia", @@ -87,8 +88,8 @@ var _ = Describe("Scheduling", func() { v1alpha1.LabelInstanceGPUMemory: "16384", v1alpha1.LabelInstanceLocalNVME: "900", // Deprecated Labels - v1.LabelFailureDomainBetaZone: fmt.Sprintf("%sa", env.Region), v1.LabelFailureDomainBetaRegion: env.Region, + v1.LabelFailureDomainBetaZone: fmt.Sprintf("%sa", env.Region), "beta.kubernetes.io/arch": "amd64", "beta.kubernetes.io/os": "linux", v1.LabelInstanceType: "g4dn.8xlarge", diff --git a/website/content/en/preview/concepts/scheduling.md b/website/content/en/preview/concepts/scheduling.md index 346cc6139524..bbad18477355 100755 --- a/website/content/en/preview/concepts/scheduling.md +++ b/website/content/en/preview/concepts/scheduling.md @@ -148,6 +148,7 @@ Take care to ensure the label domains are correct. A well known label like `karp | karpenter.k8s.aws/instance-size | 8xlarge | [AWS Specific] Instance types of similar resource quantities but different properties | | karpenter.k8s.aws/instance-cpu | 32 | [AWS Specific] Number of CPUs on the instance | | karpenter.k8s.aws/instance-memory | 131072 | [AWS Specific] Number of mebibytes of memory on the instance | +| karpenter.k8s.aws/instance-network-bandwidth | 131072 | [AWS Specific] Number of [baseline megabits](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html) available on the instance | | karpenter.k8s.aws/instance-pods | 110 | [AWS Specific] Number of pods the instance supports | | karpenter.k8s.aws/instance-gpu-name | t4 | [AWS Specific] Name of the GPU on the instance, if available | | karpenter.k8s.aws/instance-gpu-manufacturer | nvidia | [AWS Specific] Name of the GPU manufacturer |