From 0c4f367bf305dae1da068917ec8c4db29fe9ca9c Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Fri, 7 Apr 2023 11:59:52 -0700 Subject: [PATCH] fix: Fix `kubeletConfiguration.cpuCFSQuota` to enable for AL2 (#3721) --- go.mod | 5 +++-- go.sum | 10 ++++++---- pkg/providers/amifamily/bootstrap/eksbootstrap.go | 5 +++-- pkg/providers/launchtemplate/suite_test.go | 14 ++++++++++++++ test/go.mod | 3 ++- test/go.sum | 7 ++++--- test/suites/integration/kubelet_config_test.go | 2 ++ .../content/en/preview/concepts/provisioners.md | 9 ++++++++- 8 files changed, 42 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 98943d9756f9..991a0fb2134c 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/PuerkitoBio/goquery v1.8.1 github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.44.195 - github.com/aws/karpenter-core v0.27.1 + github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a github.com/go-playground/validator/v10 v10.11.2 github.com/imdario/mergo v0.3.15 github.com/mitchellh/hashstructure/v2 v2.0.2 @@ -105,8 +105,9 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/component-base v0.25.4 // indirect + k8s.io/csi-translation-lib v0.25.4 // indirect k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54 // indirect - k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect + k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/go.sum b/go.sum index b4dfa31292bc..94f9c3939375 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHS github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.44.195 h1:d5xFL0N83Fpsq2LFiHgtBUHknCRUPGHdOlCWt/jtOJs= github.com/aws/aws-sdk-go v1.44.195/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/karpenter-core v0.27.1 h1:QE81UsHNI+4AaXhvt1fXD7d+GnQ4Xuj8ZplpCYLa9CU= -github.com/aws/karpenter-core v0.27.1/go.mod h1:tYsnBPk7ZFcZFqWB3Fr+ZbIZmUZC5hGAiZa32bJa+O4= +github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a h1:t8sVxSVAnLwNbc+K0uAhE9EPNq7NBoDZTuI/zlLvt4U= +github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a/go.mod h1:NQOEY9xxhPi6xUTknEA0KaY9pPbXzS5N4EKkZ5f85d8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -885,10 +885,12 @@ k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8= k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= k8s.io/component-base v0.25.4 h1:n1bjg9Yt+G1C0WnIDJmg2fo6wbEU1UGMRiQSjmj7hNQ= k8s.io/component-base v0.25.4/go.mod h1:nnZJU8OP13PJEm6/p5V2ztgX2oyteIaAGKGMYb2L2cY= +k8s.io/csi-translation-lib v0.25.4 h1:y8+C2sVIkA9K3Q8yofMc8DJzHUyKdFXlHk563nR1Kgc= +k8s.io/csi-translation-lib v0.25.4/go.mod h1:T8l3i4NTGQ0xgXiBHEr4euaO1kw59sNrH57Fa6SXBCY= k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54 h1:hWRbsoRWt44OEBnYUd4ceLy4ofBoh+p9vauWp/I5Gdg= k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 h1:GfD9OzL11kvZN5iArC6oTS7RTj7oJOIfnislxYlqTj8= k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= knative.dev/pkg v0.0.0-20221123154742-05b694ec4d3a h1:mTDxXL+zRBMz7BcdM3WOgw9FVbmkIN/3cvEj4MeS8zI= diff --git a/pkg/providers/amifamily/bootstrap/eksbootstrap.go b/pkg/providers/amifamily/bootstrap/eksbootstrap.go index 3d4beb6d304c..6f114ae16fb7 100644 --- a/pkg/providers/amifamily/bootstrap/eksbootstrap.go +++ b/pkg/providers/amifamily/bootstrap/eksbootstrap.go @@ -100,14 +100,15 @@ func (e EKS) eksBootstrapScript() string { if e.KubeletConfig.EvictionMaxPodGracePeriod != nil { kubeletExtraArgs.WriteString(fmt.Sprintf(" --eviction-max-pod-grace-period=%d", ptr.Int32Value(e.KubeletConfig.EvictionMaxPodGracePeriod))) } - if e.KubeletConfig.ImageGCHighThresholdPercent != nil { kubeletExtraArgs.WriteString(fmt.Sprintf(" --image-gc-high-threshold=%d", ptr.Int32Value(e.KubeletConfig.ImageGCHighThresholdPercent))) } - if e.KubeletConfig.ImageGCLowThresholdPercent != nil { kubeletExtraArgs.WriteString(fmt.Sprintf(" --image-gc-low-threshold=%d", ptr.Int32Value(e.KubeletConfig.ImageGCLowThresholdPercent))) } + if e.KubeletConfig.CPUCFSQuota != nil { + kubeletExtraArgs.WriteString(fmt.Sprintf(" --cpu-cfs-quota=%t", lo.FromPtr(e.KubeletConfig.CPUCFSQuota))) + } } if e.ContainerRuntime != "" { userData.WriteString(fmt.Sprintf(" \\\n--container-runtime %s", e.ContainerRuntime)) diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 5e62a7dc4cb8..2d7c85fcd30f 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -1208,6 +1208,20 @@ var _ = Describe("LaunchTemplates", func() { Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring("--image-gc-low-threshold=50")) }) + It("should pass --cpu-fs-quota when specified", func() { + provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ + CPUCFSQuota: aws.Bool(false), + } + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() + userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) + Expect(err).To(BeNil()) + Expect(string(userData)).To(ContainSubstring("--cpu-cfs-quota=false")) + }) Context("Bottlerocket", func() { It("should merge in custom user data", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ diff --git a/test/go.mod b/test/go.mod index 5c6b4064a62d..fc1f516003bb 100644 --- a/test/go.mod +++ b/test/go.mod @@ -7,7 +7,7 @@ require ( github.com/aws/aws-sdk-go v1.44.234 github.com/aws/aws-sdk-go-v2/config v1.18.19 github.com/aws/karpenter v0.22.0 - github.com/aws/karpenter-core v0.27.1 + github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a github.com/onsi/ginkgo/v2 v2.9.2 github.com/onsi/gomega v1.27.6 github.com/samber/lo v1.38.1 @@ -101,6 +101,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.25.4 // indirect k8s.io/component-base v0.25.4 // indirect + k8s.io/csi-translation-lib v0.25.4 // indirect k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54 // indirect k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 // indirect diff --git a/test/go.sum b/test/go.sum index 54b2da0de8b8..2a89e9a29c2d 100644 --- a/test/go.sum +++ b/test/go.sum @@ -82,8 +82,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWno github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= -github.com/aws/karpenter-core v0.27.1 h1:QE81UsHNI+4AaXhvt1fXD7d+GnQ4Xuj8ZplpCYLa9CU= -github.com/aws/karpenter-core v0.27.1/go.mod h1:tYsnBPk7ZFcZFqWB3Fr+ZbIZmUZC5hGAiZa32bJa+O4= +github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a h1:t8sVxSVAnLwNbc+K0uAhE9EPNq7NBoDZTuI/zlLvt4U= +github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a/go.mod h1:NQOEY9xxhPi6xUTknEA0KaY9pPbXzS5N4EKkZ5f85d8= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -107,7 +107,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -704,6 +703,8 @@ k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8= k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= k8s.io/component-base v0.25.4 h1:n1bjg9Yt+G1C0WnIDJmg2fo6wbEU1UGMRiQSjmj7hNQ= k8s.io/component-base v0.25.4/go.mod h1:nnZJU8OP13PJEm6/p5V2ztgX2oyteIaAGKGMYb2L2cY= +k8s.io/csi-translation-lib v0.25.4 h1:y8+C2sVIkA9K3Q8yofMc8DJzHUyKdFXlHk563nR1Kgc= +k8s.io/csi-translation-lib v0.25.4/go.mod h1:T8l3i4NTGQ0xgXiBHEr4euaO1kw59sNrH57Fa6SXBCY= k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54 h1:hWRbsoRWt44OEBnYUd4ceLy4ofBoh+p9vauWp/I5Gdg= k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= diff --git a/test/suites/integration/kubelet_config_test.go b/test/suites/integration/kubelet_config_test.go index fd9d3027542e..40b31c4c5cfc 100644 --- a/test/suites/integration/kubelet_config_test.go +++ b/test/suites/integration/kubelet_config_test.go @@ -88,6 +88,7 @@ var _ = Describe("KubeletConfiguration Overrides", func() { EvictionMaxPodGracePeriod: ptr.Int32(120), ImageGCHighThresholdPercent: ptr.Int32(50), ImageGCLowThresholdPercent: ptr.Int32(10), + CPUCFSQuota: ptr.Bool(false), }, }) @@ -147,6 +148,7 @@ var _ = Describe("KubeletConfiguration Overrides", func() { EvictionMaxPodGracePeriod: ptr.Int32(120), ImageGCHighThresholdPercent: ptr.Int32(50), ImageGCLowThresholdPercent: ptr.Int32(10), + CPUCFSQuota: ptr.Bool(false), }, }) diff --git a/website/content/en/preview/concepts/provisioners.md b/website/content/en/preview/concepts/provisioners.md index 6c278b6b56ae..b4385a7beb68 100644 --- a/website/content/en/preview/concepts/provisioners.md +++ b/website/content/en/preview/concepts/provisioners.md @@ -107,8 +107,12 @@ spec: nodefs.available: 1m30s nodefs.inodesFree: 2m evictionMaxPodGracePeriod: 60 + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + cpuCFSQuota: true podsPerCore: 2 maxPods: 20 + # Resource limits constrain the total size of the cluster. # Limits prevent Karpenter from creating new instances once the limit is exceeded. @@ -254,7 +258,7 @@ For more information on weighting Provisioners, see the [Weighting Provisioners ## spec.kubeletConfiguration Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for -additional customization and use cases. Adjust these only if you know you need to do so. +additional customization and use cases. Adjust these only if you know you need to do so. For more details on kubelet configuration arguments, [see the KubeletConfiguration API specification docs](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/). ```yaml spec: @@ -283,6 +287,9 @@ spec: nodefs.available: 1m30s nodefs.inodesFree: 2m evictionMaxPodGracePeriod: 60 + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + cpuCFSQuota: true podsPerCore: 2 maxPods: 20 ```