fix: Fix kubeletConfiguration.cpuCFSQuota to enable for AL2 (aws#3721)

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/aws/aws-sdk-go v1.44.195
-	github.com/aws/karpenter-core v0.27.1
+	github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a
 	github.com/go-playground/validator/v10 v10.11.2
 	github.com/imdario/mergo v0.3.15
 	github.com/mitchellh/hashstructure/v2 v2.0.2
@@ -105,8 +105,9 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/component-base v0.25.4 // indirect
+	k8s.io/csi-translation-lib v0.25.4 // indirect
 	k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54 // indirect
-	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
+	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect

go.sum

@@ -70,8 +70,8 @@ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHS
 github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/aws/aws-sdk-go v1.44.195 h1:d5xFL0N83Fpsq2LFiHgtBUHknCRUPGHdOlCWt/jtOJs=
 github.com/aws/aws-sdk-go v1.44.195/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/karpenter-core v0.27.1 h1:QE81UsHNI+4AaXhvt1fXD7d+GnQ4Xuj8ZplpCYLa9CU=
-github.com/aws/karpenter-core v0.27.1/go.mod h1:tYsnBPk7ZFcZFqWB3Fr+ZbIZmUZC5hGAiZa32bJa+O4=
+github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a h1:t8sVxSVAnLwNbc+K0uAhE9EPNq7NBoDZTuI/zlLvt4U=
+github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a/go.mod h1:NQOEY9xxhPi6xUTknEA0KaY9pPbXzS5N4EKkZ5f85d8=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -885,10 +885,12 @@ k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8=
 k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw=
 k8s.io/component-base v0.25.4 h1:n1bjg9Yt+G1C0WnIDJmg2fo6wbEU1UGMRiQSjmj7hNQ=
 k8s.io/component-base v0.25.4/go.mod h1:nnZJU8OP13PJEm6/p5V2ztgX2oyteIaAGKGMYb2L2cY=
+k8s.io/csi-translation-lib v0.25.4 h1:y8+C2sVIkA9K3Q8yofMc8DJzHUyKdFXlHk563nR1Kgc=
+k8s.io/csi-translation-lib v0.25.4/go.mod h1:T8l3i4NTGQ0xgXiBHEr4euaO1kw59sNrH57Fa6SXBCY=
 k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54 h1:hWRbsoRWt44OEBnYUd4ceLy4ofBoh+p9vauWp/I5Gdg=
 k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA=
-k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU=
+k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
+k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
 k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 h1:GfD9OzL11kvZN5iArC6oTS7RTj7oJOIfnislxYlqTj8=
 k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 knative.dev/pkg v0.0.0-20221123154742-05b694ec4d3a h1:mTDxXL+zRBMz7BcdM3WOgw9FVbmkIN/3cvEj4MeS8zI=

pkg/providers/amifamily/bootstrap/eksbootstrap.go

@@ -100,14 +100,15 @@ func (e EKS) eksBootstrapScript() string {
 		if e.KubeletConfig.EvictionMaxPodGracePeriod != nil {
 			kubeletExtraArgs.WriteString(fmt.Sprintf(" --eviction-max-pod-grace-period=%d", ptr.Int32Value(e.KubeletConfig.EvictionMaxPodGracePeriod)))
 		}
-
 		if e.KubeletConfig.ImageGCHighThresholdPercent != nil {
 			kubeletExtraArgs.WriteString(fmt.Sprintf(" --image-gc-high-threshold=%d", ptr.Int32Value(e.KubeletConfig.ImageGCHighThresholdPercent)))
 		}
-
 		if e.KubeletConfig.ImageGCLowThresholdPercent != nil {
 			kubeletExtraArgs.WriteString(fmt.Sprintf(" --image-gc-low-threshold=%d", ptr.Int32Value(e.KubeletConfig.ImageGCLowThresholdPercent)))
 		}
+		if e.KubeletConfig.CPUCFSQuota != nil {
+			kubeletExtraArgs.WriteString(fmt.Sprintf(" --cpu-cfs-quota=%t", lo.FromPtr(e.KubeletConfig.CPUCFSQuota)))
+		}
 	}
 	if e.ContainerRuntime != "" {
 		userData.WriteString(fmt.Sprintf(" \\\n--container-runtime %s", e.ContainerRuntime))

pkg/providers/launchtemplate/suite_test.go

@@ -1208,6 +1208,20 @@ var _ = Describe("LaunchTemplates", func() {
 			Expect(err).To(BeNil())
 			Expect(string(userData)).To(ContainSubstring("--image-gc-low-threshold=50"))
 		})
+		It("should pass --cpu-fs-quota when specified", func() {
+			provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{
+				CPUCFSQuota: aws.Bool(false),
+			}
+			ExpectApplied(ctx, env.Client, provisioner, nodeTemplate)
+			pod := coretest.UnschedulablePod()
+			ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod)
+			ExpectScheduled(ctx, env.Client, pod)
+			Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1))
+			input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop()
+			userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
+			Expect(err).To(BeNil())
+			Expect(string(userData)).To(ContainSubstring("--cpu-cfs-quota=false"))
+		})
 		Context("Bottlerocket", func() {
 			It("should merge in custom user data", func() {
 				ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{

test/go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/aws/aws-sdk-go v1.44.234
 	github.com/aws/aws-sdk-go-v2/config v1.18.19
 	github.com/aws/karpenter v0.22.0
-	github.com/aws/karpenter-core v0.27.1
+	github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a
 	github.com/onsi/ginkgo/v2 v2.9.2
 	github.com/onsi/gomega v1.27.6
 	github.com/samber/lo v1.38.1
@@ -101,6 +101,7 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.25.4 // indirect
 	k8s.io/component-base v0.25.4 // indirect
+	k8s.io/csi-translation-lib v0.25.4 // indirect
 	k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54 // indirect
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
 	k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 // indirect

test/go.sum

@@ -82,8 +82,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWno
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI=
-github.com/aws/karpenter-core v0.27.1 h1:QE81UsHNI+4AaXhvt1fXD7d+GnQ4Xuj8ZplpCYLa9CU=
-github.com/aws/karpenter-core v0.27.1/go.mod h1:tYsnBPk7ZFcZFqWB3Fr+ZbIZmUZC5hGAiZa32bJa+O4=
+github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a h1:t8sVxSVAnLwNbc+K0uAhE9EPNq7NBoDZTuI/zlLvt4U=
+github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a/go.mod h1:NQOEY9xxhPi6xUTknEA0KaY9pPbXzS5N4EKkZ5f85d8=
 github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
@@ -107,7 +107,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw=
 github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
@@ -704,6 +703,8 @@ k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8=
 k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw=
 k8s.io/component-base v0.25.4 h1:n1bjg9Yt+G1C0WnIDJmg2fo6wbEU1UGMRiQSjmj7hNQ=
 k8s.io/component-base v0.25.4/go.mod h1:nnZJU8OP13PJEm6/p5V2ztgX2oyteIaAGKGMYb2L2cY=
+k8s.io/csi-translation-lib v0.25.4 h1:y8+C2sVIkA9K3Q8yofMc8DJzHUyKdFXlHk563nR1Kgc=
+k8s.io/csi-translation-lib v0.25.4/go.mod h1:T8l3i4NTGQ0xgXiBHEr4euaO1kw59sNrH57Fa6SXBCY=
 k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54 h1:hWRbsoRWt44OEBnYUd4ceLy4ofBoh+p9vauWp/I5Gdg=
 k8s.io/klog/v2 v2.80.2-0.20221028030830-9ae4992afb54/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=

test/suites/integration/kubelet_config_test.go

@@ -88,6 +88,7 @@ var _ = Describe("KubeletConfiguration Overrides", func() {
 				EvictionMaxPodGracePeriod:   ptr.Int32(120),
 				ImageGCHighThresholdPercent: ptr.Int32(50),
 				ImageGCLowThresholdPercent:  ptr.Int32(10),
+				CPUCFSQuota:                 ptr.Bool(false),
 			},
 		})
 
@@ -147,6 +148,7 @@ var _ = Describe("KubeletConfiguration Overrides", func() {
 				EvictionMaxPodGracePeriod:   ptr.Int32(120),
 				ImageGCHighThresholdPercent: ptr.Int32(50),
 				ImageGCLowThresholdPercent:  ptr.Int32(10),
+				CPUCFSQuota:                 ptr.Bool(false),
 			},
 		})
 

website/content/en/preview/concepts/provisioners.md

@@ -107,8 +107,12 @@ spec:
       nodefs.available: 1m30s
       nodefs.inodesFree: 2m
     evictionMaxPodGracePeriod: 60
+    imageGCHighThresholdPercent: 85
+    imageGCLowThresholdPercent: 80
+    cpuCFSQuota: true
     podsPerCore: 2
     maxPods: 20
+
 
   # Resource limits constrain the total size of the cluster.
   # Limits prevent Karpenter from creating new instances once the limit is exceeded.
@@ -254,7 +258,7 @@ For more information on weighting Provisioners, see the [Weighting Provisioners
 ## spec.kubeletConfiguration
 
 Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for
-additional customization and use cases. Adjust these only if you know you need to do so.
+additional customization and use cases. Adjust these only if you know you need to do so. For more details on kubelet configuration arguments, [see the KubeletConfiguration API specification docs](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/).
 
 ```yaml
 spec:
@@ -283,6 +287,9 @@ spec:
       nodefs.available: 1m30s
       nodefs.inodesFree: 2m
     evictionMaxPodGracePeriod: 60
+    imageGCHighThresholdPercent: 85
+    imageGCLowThresholdPercent: 80
+    cpuCFSQuota: true
     podsPerCore: 2
     maxPods: 20
 ```