Add support for overrideBootstrapCommand in AL2023

integration/tests/crud/creategetdelete_test.go

@@ -69,6 +69,8 @@ const (
 	deleteNg        = "ng-delete"
 	taintsNg1       = "ng-taints-1"
 	taintsNg2       = "ng-taints-2"
+	maxPodsMNG1     = "mng-max-pods-1"
+	maxPodsMNG2     = "mng-max-pods-2"
 	scaleSingleNg   = "ng-scale-single"
 	scaleMultipleNg = "ng-scale-multiple"
 
@@ -872,6 +874,8 @@ var _ = Describe("(Integration) Create, Get, Scale & Delete", func() {
 			clientset := makeClientset()
 			nodeListN1 := tests.ListNodes(clientset, taintsNg1)
 			nodeListN2 := tests.ListNodes(clientset, taintsNg2)
+			mngNodeListN1 := tests.ListNodes(clientset, maxPodsMNG1)
+			mngNodeListN2 := tests.ListNodes(clientset, maxPodsMNG2)
 
 			tests.AssertNodeTaints(nodeListN1, []corev1.Taint{
 				{
@@ -896,12 +900,22 @@ var _ = Describe("(Integration) Create, Get, Scale & Delete", func() {
 				},
 			})
 
-			By("asserting that maxPods is set correctly")
+			By("asserting that maxPods is set correctly for AL2 nodegroup")
 			expectedMaxPods := 123
 			for _, node := range nodeListN1.Items {
 				maxPods, _ := node.Status.Allocatable.Pods().AsInt64()
 				Expect(maxPods).To(Equal(int64(expectedMaxPods)))
 			}
+
+			By("asserting that maxPods is set correctly for AL2023 nodegroups")
+			for _, node := range mngNodeListN1.Items {
+				maxPods, _ := node.Status.Allocatable.Pods().AsInt64()
+				Expect(maxPods).To(Equal(int64(expectedMaxPods)))
+			}
+			for _, node := range mngNodeListN2.Items {
+				maxPods, _ := node.Status.Allocatable.Pods().AsInt64()
+				Expect(maxPods).To(Equal(int64(expectedMaxPods)))
+			}
 		})
 
 		It("should be able to create a new GPU nodegroup", func() {

integration/tests/crud/testdata/taints-max-pods.yaml

@@ -6,16 +6,30 @@ metadata:
   name: <generated>
 
 nodeGroups:
-- name: ng-taints-1
-  taints:
-    key1: val1:NoSchedule
-    key2: :NoExecute
-  maxPodsPerNode: 123
-- name: ng-taints-2
-  volumeSize: 35
-  taints:
-  - key: key1
-    value: value1
-    effect: NoSchedule
-  - key: key2
-    effect: NoExecute
+  - name: ng-taints-1
+    taints:
+      key1: val1:NoSchedule
+      key2: :NoExecute
+    maxPodsPerNode: 123
+  - name: ng-taints-2
+    volumeSize: 35
+    taints:
+      - key: key1
+        value: value1
+        effect: NoSchedule
+      - key: key2
+        effect: NoExecute
+
+managedNodeGroups:
+  - name: mng-max-pods-1
+    desiredCapacity: 1
+    maxPodsPerNode: 123
+  - name: mng-max-pods-2
+    desiredCapacity: 1
+    overrideBootstrapCommand: |
+      apiVersion: node.eks.aws/v1alpha1
+      kind: NodeConfig
+      spec:
+        kubelet:
+          config:
+            maxPods: 123

pkg/apis/eksctl.io/v1alpha5/validation.go

@@ -759,19 +759,6 @@ func validateNodeGroupBase(np NodePool, path string, controlPlaneOnOutposts bool
 		}
 	}
 
-	if ng.AMIFamily == NodeImageFamilyAmazonLinux2023 {
-		fieldNotSupported := func(field string) error {
-			return &unsupportedFieldError{
-				ng:    ng,
-				path:  path,
-				field: field,
-			}
-		}
-		if ng.OverrideBootstrapCommand != nil {
-			return fieldNotSupported("overrideBootstrapCommand")
-		}
-	}
-
 	if ng.CapacityReservation != nil {
 		if ng.CapacityReservation.CapacityReservationPreference != nil {
 			if ng.CapacityReservation.CapacityReservationTarget != nil {
@@ -1276,10 +1263,6 @@ func ValidateManagedNodeGroup(index int, ng *ManagedNodeGroup) error {
 		}
 	}
 
-	if ng.AMIFamily == NodeImageFamilyAmazonLinux2023 && ng.MaxPodsPerNode > 0 {
-		return fmt.Errorf("eksctl does not support configuring maxPodsPerNode EKS-managed nodes based on %s", NodeImageFamilyAmazonLinux2023)
-	}
-
 	if ng.AMIFamily == NodeImageFamilyBottlerocket {
 		fieldNotSupported := func(field string) error {
 			return &unsupportedFieldError{
@@ -1362,9 +1345,6 @@ func ValidateManagedNodeGroup(index int, ng *ManagedNodeGroup) error {
 		if ng.OverrideBootstrapCommand == nil && ng.AMIFamily != NodeImageFamilyAmazonLinux2023 {
 			return fmt.Errorf("%[1]s.overrideBootstrapCommand is required when using a custom AMI based on %s (%[1]s.ami)", path, ng.AMIFamily)
 		}
-		if ng.OverrideBootstrapCommand != nil && ng.AMIFamily == NodeImageFamilyAmazonLinux2023 {
-			return fmt.Errorf("%[1]s.overrideBootstrapCommand is not supported when using a custom AMI based on %s (%[1]s.ami)", path, ng.AMIFamily)
-		}
 		notSupportedWithCustomAMIErr := func(field string) error {
 			return fmt.Errorf("%s.%s is not supported when using a custom AMI (%s.ami)", path, field, path)
 		}
@@ -1378,7 +1358,7 @@ func ValidateManagedNodeGroup(index int, ng *ManagedNodeGroup) error {
 			return notSupportedWithCustomAMIErr("releaseVersion")
 		}
 
-	case ng.OverrideBootstrapCommand != nil:
+	case ng.OverrideBootstrapCommand != nil && ng.AMIFamily != NodeImageFamilyAmazonLinux2023:
 		return fmt.Errorf("%s.overrideBootstrapCommand can only be set when a custom AMI (%s.ami) is specified", path, path)
 	}
 

pkg/apis/eksctl.io/v1alpha5/validation_test.go

@@ -2178,33 +2178,6 @@ var _ = Describe("ClusterConfig validation", func() {
 		})
 	})
 
-	Describe("AmazonLinux2023 node groups", func() {
-		It("returns an error when setting maxPodsPerNode for managed nodegroups", func() {
-			ng := api.NewManagedNodeGroup()
-			ng.AMIFamily = api.NodeImageFamilyAmazonLinux2023
-			ng.MaxPodsPerNode = 5
-			err := api.ValidateManagedNodeGroup(0, ng)
-			Expect(err).To(MatchError(ContainSubstring("eksctl does not support configuring maxPodsPerNode EKS-managed nodes")))
-		})
-		It("returns an error when setting overrideBootstrapCommand for self-managed nodegroups", func() {
-			cfg := api.NewClusterConfig()
-			ng := cfg.NewNodeGroup()
-			ng.Name = "node-group"
-			ng.AMI = "ami-1234"
-			ng.AMIFamily = api.NodeImageFamilyAmazonLinux2023
-			ng.OverrideBootstrapCommand = aws.String("echo 'rubarb'")
-			Expect(api.ValidateNodeGroup(0, ng, cfg)).To(MatchError(ContainSubstring(fmt.Sprintf("overrideBootstrapCommand is not supported for %s nodegroups", api.NodeImageFamilyAmazonLinux2023))))
-		})
-		It("returns an error when setting overrideBootstrapCommand for EKS-managed nodegroups", func() {
-			ng := api.NewManagedNodeGroup()
-			ng.Name = "node-group"
-			ng.AMI = "ami-1234"
-			ng.AMIFamily = api.NodeImageFamilyAmazonLinux2023
-			ng.OverrideBootstrapCommand = aws.String("echo 'rubarb'")
-			Expect(api.ValidateManagedNodeGroup(0, ng)).To(MatchError(ContainSubstring(fmt.Sprintf("overrideBootstrapCommand is not supported for %s nodegroups", api.NodeImageFamilyAmazonLinux2023))))
-		})
-	})
-
 	Describe("Windows node groups", func() {
 		It("returns an error with unsupported fields", func() {
 			doc := api.InlineDocument{

pkg/nodebootstrap/al2023.go

@@ -5,7 +5,8 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"strconv"
+
+	"sigs.k8s.io/yaml"
 
 	nodeadmapi "github.com/awslabs/amazon-eks-ami/nodeadm/api"
 	nodeadm "github.com/awslabs/amazon-eks-ami/nodeadm/api/v1alpha1"
@@ -22,8 +23,9 @@ type AL2023 struct {
 	nodePool   api.NodePool
 	clusterDNS string
 
-	scripts   []string
-	cloudboot []string
+	scripts     []string
+	cloudboot   []string
+	nodeConfigs []*nodeadm.NodeConfig
 
 	UserDataMimeBoundary string
 }
@@ -54,27 +56,55 @@ func newAL2023Bootstrapper(cfg *api.ClusterConfig, np api.NodePool, clusterDNS s
 }
 
 func (m *AL2023) UserData() (string, error) {
-	nodeConfig, err := m.createNodeConfig()
+	ng := m.nodePool.BaseNodeGroup()
+
+	minimalNodeConfig, err := m.createMinimalNodeConfig()
 	if err != nil {
-		return "", fmt.Errorf("generating node config: %w", err)
+		return "", fmt.Errorf("generating minimal node config: %w", err)
+	}
+	if minimalNodeConfig != nil {
+		m.nodeConfigs = append(m.nodeConfigs, minimalNodeConfig)
+	}
+
+	if ng.MaxPodsPerNode > 0 {
+		nodeConfig := &nodeadm.NodeConfig{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       nodeadmapi.KindNodeConfig,
+				APIVersion: nodeadm.GroupVersion.String(),
+			},
+		}
+		kubeletConfig, err := ToKubeletConfig(api.InlineDocument{"maxPods": ng.MaxPodsPerNode})
+		if err != nil {
+			return "", err
+		}
+		nodeConfig.Spec.Kubelet.Config = kubeletConfig
+		m.nodeConfigs = append(m.nodeConfigs, nodeConfig)
 	}
 
 	for _, command := range m.nodePool.BaseNodeGroup().PreBootstrapCommands {
 		m.scripts = append(m.scripts, "#!/bin/bash\n"+command)
 	}
 
-	if len(m.scripts) == 0 && len(m.cloudboot) == 0 && nodeConfig == nil {
+	if ng.OverrideBootstrapCommand != nil {
+		nodeConfig, err := stringToNodeConfig(*ng.OverrideBootstrapCommand)
+		if err != nil {
+			return "", err
+		}
+		m.nodeConfigs = append(m.nodeConfigs, nodeConfig)
+	}
+
+	if len(m.scripts) == 0 && len(m.cloudboot) == 0 && len(m.nodeConfigs) == 0 {
 		return "", nil
 	}
 
 	var buf bytes.Buffer
-	if err := createMimeMessage(&buf, m.scripts, m.cloudboot, nodeConfig, m.UserDataMimeBoundary); err != nil {
+	if err := createMimeMessage(&buf, m.scripts, m.cloudboot, m.nodeConfigs, m.UserDataMimeBoundary); err != nil {
 		return "", err
 	}
 	return base64.StdEncoding.EncodeToString(buf.Bytes()), nil
 }
 
-func (m *AL2023) createNodeConfig() (*nodeadm.NodeConfig, error) {
+func (m *AL2023) createMinimalNodeConfig() (*nodeadm.NodeConfig, error) {
 	kubeletConfig := api.InlineDocument{}
 	switch nodeGroup := m.nodePool.(type) {
 	case *api.ManagedNodeGroup:
@@ -89,9 +119,6 @@ func (m *AL2023) createNodeConfig() (*nodeadm.NodeConfig, error) {
 
 	kubeletConfig["clusterDNS"] = []string{m.clusterDNS}
 	ng := m.nodePool.BaseNodeGroup()
-	if ng.MaxPodsPerNode > 0 {
-		kubeletConfig["maxPods"] = strconv.Itoa(ng.MaxPodsPerNode)
-	}
 	nodeKubeletConfig, err := ToKubeletConfig(kubeletConfig)
 	if err != nil {
 		return nil, err
@@ -135,3 +162,12 @@ func ToKubeletConfig(kubeletExtraConfig api.InlineDocument) (map[string]runtime.
 	}
 	return kubeletConfig, nil
 }
+
+func stringToNodeConfig(overrideBootstrapCommand string) (*nodeadm.NodeConfig, error) {
+	var config nodeadm.NodeConfig
+	err := yaml.Unmarshal([]byte(overrideBootstrapCommand), &config)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshalling \"overrideBootstrapCommand\" into \"nodeadm.NodeConfig\": %w", err)
+	}
+	return &config, nil
+}