From d208bab406d679f6d48e0113995ad65a4f119aa6 Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Sat, 6 Sep 2025 10:18:47 +0700 Subject: [PATCH 01/15] Support EKS upgrade policy --- ...ster.x-k8s.io_awsmanagedcontrolplanes.yaml | 11 +++ ...8s.io_awsmanagedcontrolplanetemplates.yaml | 11 +++ controlplane/eks/api/v1beta1/conversion.go | 1 + .../api/v1beta1/zz_generated.conversion.go | 1 + .../v1beta2/awsmanagedcontrolplane_types.go | 9 ++ controlplane/eks/api/v1beta2/types.go | 18 ++++ .../book/src/topics/eks/creating-a-cluster.md | 3 + pkg/cloud/converters/eks.go | 8 ++ pkg/cloud/services/eks/cluster.go | 42 +++++++++ pkg/cloud/services/eks/cluster_test.go | 89 +++++++++++++++++++ 10 files changed, 193 insertions(+) diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index 858dac8a4c..d9769ccbd9 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -3243,6 +3243,17 @@ spec: - iam-authenticator - aws-cli type: string + upgradePolicy: + description: |- + The support policy to use for the cluster. + Extended support indicates that the cluster will not be automatically upgraded + when it leaves the standard support period, and will enter extended support. + Clusters in extended support have higher costs. + If omitted, new clusters use AWS default extended support and existing clusters stay unchanged. + enum: + - extended + - standard + type: string version: description: |- Version defines the desired Kubernetes version. If no version number diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml index ad5c56c54b..bd8e7ea7f9 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml @@ -1044,6 +1044,17 @@ spec: - iam-authenticator - aws-cli type: string + upgradePolicy: + description: |- + The support policy to use for the cluster. + Extended support indicates that the cluster will not be automatically upgraded + when it leaves the standard support period, and will enter extended support. + Clusters in extended support have higher costs. + If omitted, new clusters use AWS default extended support and existing clusters stay unchanged. + enum: + - extended + - standard + type: string version: description: |- Version defines the desired Kubernetes version. If no version number diff --git a/controlplane/eks/api/v1beta1/conversion.go b/controlplane/eks/api/v1beta1/conversion.go index 0985ef66d5..b563a71324 100644 --- a/controlplane/eks/api/v1beta1/conversion.go +++ b/controlplane/eks/api/v1beta1/conversion.go @@ -122,6 +122,7 @@ func (r *AWSManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.RolePermissionsBoundary = restored.Spec.RolePermissionsBoundary dst.Status.Version = restored.Status.Version dst.Spec.BootstrapSelfManagedAddons = restored.Spec.BootstrapSelfManagedAddons + dst.Spec.UpgradePolicy = restored.Spec.UpgradePolicy return nil } diff --git a/controlplane/eks/api/v1beta1/zz_generated.conversion.go b/controlplane/eks/api/v1beta1/zz_generated.conversion.go index 48f326b2dc..95ae9313a6 100644 --- a/controlplane/eks/api/v1beta1/zz_generated.conversion.go +++ b/controlplane/eks/api/v1beta1/zz_generated.conversion.go @@ -384,6 +384,7 @@ func autoConvert_v1beta2_AWSManagedControlPlaneSpec_To_v1beta1_AWSManagedControl if err := Convert_v1beta2_KubeProxy_To_v1beta1_KubeProxy(&in.KubeProxy, &out.KubeProxy, s); err != nil { return err } + // WARNING: in.UpgradePolicy requires manual conversion: does not exist in peer-type return nil } diff --git a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go index 9112863e35..f9a87585cb 100644 --- a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go +++ b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go @@ -212,6 +212,15 @@ type AWSManagedControlPlaneSpec struct { //nolint: maligned // KubeProxy defines managed attributes of the kube-proxy daemonset KubeProxy KubeProxy `json:"kubeProxy,omitempty"` + + // The support policy to use for the cluster. + // Extended support indicates that the cluster will not be automatically upgraded + // when it leaves the standard support period, and will enter extended support. + // Clusters in extended support have higher costs. + // If omitted, new clusters use AWS default extended support and existing clusters stay unchanged. + // +kubebuilder:validation:Enum=extended;standard + // +optional + UpgradePolicy UpgradePolicy `json:"upgradePolicy,omitempty"` } // KubeProxy specifies how the kube-proxy daemonset is managed. diff --git a/controlplane/eks/api/v1beta2/types.go b/controlplane/eks/api/v1beta2/types.go index 79f58f8e77..16b11a1aaf 100644 --- a/controlplane/eks/api/v1beta2/types.go +++ b/controlplane/eks/api/v1beta2/types.go @@ -241,6 +241,24 @@ type AddonIssue struct { ResourceIDs []string `json:"resourceIds,omitempty"` } +// UpgradePolicy defines the support policy to use for the cluster. +type UpgradePolicy string + +var ( + // UpgradePolicyExtended indicates that the cluster will not be automatically upgraded + // when it leaves the standard support period, and will enter extended support. + // Clusters in extended support have higher costs. + UpgradePolicyExtended = UpgradePolicy("extended") + + // UpgradePolicyStandard indicates that the cluster will be automatically upgraded + // when it leaves the standard support period. + UpgradePolicyStandard = UpgradePolicy("standard") +) + +func (e UpgradePolicy) String() string { + return string(e) +} + const ( // SecurityGroupCluster is the security group for communication between EKS // control plane and managed node groups. diff --git a/docs/book/src/topics/eks/creating-a-cluster.md b/docs/book/src/topics/eks/creating-a-cluster.md index 0ef75009c6..5fdea2f6b0 100644 --- a/docs/book/src/topics/eks/creating-a-cluster.md +++ b/docs/book/src/topics/eks/creating-a-cluster.md @@ -14,6 +14,9 @@ clusterctl generate cluster capi-eks-quickstart --flavor eks-managedmachinepool NOTE: When creating an EKS cluster only the **MAJOR.MINOR** of the `-kubernetes-version` is taken into consideration. +By default, EKS cluster uses: +- EXTENDED support. See more info about [cluster upgrade policy](https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html) + ## Kubeconfig When creating an EKS cluster 2 kubeconfigs are generated and stored as secrets in the management cluster. This is different to when you create a non-managed cluster using the AWS provider. diff --git a/pkg/cloud/converters/eks.go b/pkg/cloud/converters/eks.go index fbb35c67c3..56fb464abd 100644 --- a/pkg/cloud/converters/eks.go +++ b/pkg/cloud/converters/eks.go @@ -278,3 +278,11 @@ func AddonConflictResolutionFromSDK(conflict ekstypes.ResolveConflicts) *string } return aws.String(string(ekscontrolplanev1.AddonResolutionOverwrite)) } + +// SupportTypeToSDK converts CAPA upgrade support policy types to SDK types. +func SupportTypeToSDK(input ekscontrolplanev1.UpgradePolicy) ekstypes.SupportType { + if input == ekscontrolplanev1.UpgradePolicyStandard { + return ekstypes.SupportTypeStandard + } + return ekstypes.SupportTypeExtended +} diff --git a/pkg/cloud/services/eks/cluster.go b/pkg/cloud/services/eks/cluster.go index b1b480e0b2..38aa1d78e8 100644 --- a/pkg/cloud/services/eks/cluster.go +++ b/pkg/cloud/services/eks/cluster.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "net" + "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -35,6 +36,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/awserrors" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/converters" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/wait" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/internal/cidr" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/internal/cmp" @@ -478,6 +480,14 @@ func (s *Service) createCluster(ctx context.Context, eksClusterName string) (*ek eksVersion = &v } + var upgradePolicy *ekstypes.UpgradePolicyRequest + + if s.scope.ControlPlane.Spec.UpgradePolicy != "" { + upgradePolicy = &ekstypes.UpgradePolicyRequest{ + SupportType: converters.SupportTypeToSDK(s.scope.ControlPlane.Spec.UpgradePolicy), + } + } + bootstrapAddon := s.scope.BootstrapSelfManagedAddons() input := &eks.CreateClusterInput{ Name: aws.String(eksClusterName), @@ -490,6 +500,7 @@ func (s *Service) createCluster(ctx context.Context, eksClusterName string) (*ek Tags: tags, KubernetesNetworkConfig: netConfig, BootstrapSelfManagedAddons: bootstrapAddon, + UpgradePolicy: upgradePolicy, } var out *eks.CreateClusterOutput @@ -545,6 +556,12 @@ func (s *Service) reconcileClusterConfig(ctx context.Context, cluster *ekstypes. input.ResourcesVpcConfig = updateVpcConfig } + updateUpgradePolicy := s.reconcileUpgradePolicy(cluster.UpgradePolicy) + if updateUpgradePolicy != nil { + needsUpdate = true + input.UpgradePolicy = updateUpgradePolicy + } + if needsUpdate { if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { if _, err := s.EKSClient.UpdateClusterConfig(ctx, input); err != nil { @@ -782,6 +799,31 @@ func (s *Service) reconcileClusterVersion(ctx context.Context, cluster *ekstypes return nil } +func (s *Service) reconcileUpgradePolicy(upgradePolicy *ekstypes.UpgradePolicyResponse) *ekstypes.UpgradePolicyRequest { + s.Info("reconciling upgrade policy") + + if upgradePolicy == nil { + s.Warn("cannot get cluster upgrade policy, no action") + return nil + } + + clusterUpgradePolicy := upgradePolicy.SupportType + + if s.scope.ControlPlane.Spec.UpgradePolicy == "" { + s.Debug("upgrade policy omitted, no action") + return nil + } + + if strings.ToLower(string(clusterUpgradePolicy)) == s.scope.ControlPlane.Spec.UpgradePolicy.String() { + s.Debug("upgrade policy unchanged, no action") + return nil + } + + return &ekstypes.UpgradePolicyRequest{ + SupportType: converters.SupportTypeToSDK(s.scope.ControlPlane.Spec.UpgradePolicy), + } +} + func (s *Service) describeEKSCluster(ctx context.Context, eksClusterName string) (*ekstypes.Cluster, error) { input := &eks.DescribeClusterInput{ Name: aws.String(eksClusterName), diff --git a/pkg/cloud/services/eks/cluster_test.go b/pkg/cloud/services/eks/cluster_test.go index b120226697..25d69aaaf8 100644 --- a/pkg/cloud/services/eks/cluster_test.go +++ b/pkg/cloud/services/eks/cluster_test.go @@ -652,6 +652,7 @@ func TestCreateCluster(t *testing.T) { RoleName: tc.role, NetworkSpec: infrav1.NetworkSpec{Subnets: tc.subnets}, BootstrapSelfManagedAddons: false, + UpgradePolicy: ekscontrolplanev1.UpgradePolicyStandard, }, }, }) @@ -674,6 +675,9 @@ func TestCreateCluster(t *testing.T) { Tags: tc.tags, Version: version, BootstrapSelfManagedAddons: aws.Bool(false), + UpgradePolicy: &ekstypes.UpgradePolicyRequest{ + SupportType: ekstypes.SupportTypeStandard, + }, }).Return(&eks.CreateClusterOutput{}, nil) } s := NewService(scope) @@ -805,6 +809,91 @@ func TestReconcileEKSEncryptionConfig(t *testing.T) { } } +func TestReconcileUpgradePolicy(t *testing.T) { + clusterName := "default.cluster" + tests := []struct { + name string + oldUpgradePolicy *ekstypes.UpgradePolicyResponse + newUpgradePolicy ekscontrolplanev1.UpgradePolicy + expect *ekstypes.UpgradePolicyRequest + expectError bool + }{ + { + name: "no update necessary - upgrade policy omitted", + oldUpgradePolicy: &ekstypes.UpgradePolicyResponse{ + SupportType: ekstypes.SupportTypeStandard, + }, + expect: nil, + expectError: false, + }, + { + name: "no update necessary - cannot get cluster upgrade policy", + newUpgradePolicy: ekscontrolplanev1.UpgradePolicyStandard, + expect: nil, + expectError: false, + }, + { + name: "no update necessary - upgrade policy unchanged", + oldUpgradePolicy: &ekstypes.UpgradePolicyResponse{ + SupportType: ekstypes.SupportTypeStandard, + }, + newUpgradePolicy: ekscontrolplanev1.UpgradePolicyStandard, + expect: nil, + expectError: false, + }, + { + name: "needs update", + oldUpgradePolicy: &ekstypes.UpgradePolicyResponse{ + SupportType: ekstypes.SupportTypeStandard, + }, + newUpgradePolicy: ekscontrolplanev1.UpgradePolicyExtended, + expect: &ekstypes.UpgradePolicyRequest{ + SupportType: ekstypes.SupportTypeExtended, + }, + expectError: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + mockControl := gomock.NewController(t) + defer mockControl.Finish() + + scheme := runtime.NewScheme() + _ = infrav1.AddToScheme(scheme) + _ = ekscontrolplanev1.AddToScheme(scheme) + client := fake.NewClientBuilder().WithScheme(scheme).Build() + scope, err := scope.NewManagedControlPlaneScope(scope.ManagedControlPlaneScopeParams{ + Client: client, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + Name: clusterName, + }, + }, + ControlPlane: &ekscontrolplanev1.AWSManagedControlPlane{ + Spec: ekscontrolplanev1.AWSManagedControlPlaneSpec{ + Version: aws.String("1.16"), + UpgradePolicy: tc.newUpgradePolicy, + }, + }, + }) + g.Expect(err).To(BeNil()) + + s := NewService(scope) + + upgradePolicyRequest := s.reconcileUpgradePolicy(tc.oldUpgradePolicy) + if tc.expectError { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(upgradePolicyRequest).To(Equal(tc.expect)) + }) + } +} + func TestCreateIPv6Cluster(t *testing.T) { g := NewWithT(t) From 80195d4c707961e00a708ee05069e945a14641b8 Mon Sep 17 00:00:00 2001 From: Hung Tran <40334379+phuhung273@users.noreply.github.com> Date: Mon, 8 Sep 2025 04:38:17 -0700 Subject: [PATCH 02/15] Apply suggestions from code review Co-authored-by: Damiano Donati --- .../eks/api/v1beta2/awsmanagedcontrolplane_types.go | 10 +++++----- docs/book/src/topics/eks/creating-a-cluster.md | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go index f9a87585cb..2c95d9bbf9 100644 --- a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go +++ b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go @@ -213,11 +213,11 @@ type AWSManagedControlPlaneSpec struct { //nolint: maligned // KubeProxy defines managed attributes of the kube-proxy daemonset KubeProxy KubeProxy `json:"kubeProxy,omitempty"` - // The support policy to use for the cluster. - // Extended support indicates that the cluster will not be automatically upgraded - // when it leaves the standard support period, and will enter extended support. - // Clusters in extended support have higher costs. - // If omitted, new clusters use AWS default extended support and existing clusters stay unchanged. + // The cluster upgrade policy to use for the cluster. + // (Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html) + // `extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges. + // `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but you EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. + // If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged. // +kubebuilder:validation:Enum=extended;standard // +optional UpgradePolicy UpgradePolicy `json:"upgradePolicy,omitempty"` diff --git a/docs/book/src/topics/eks/creating-a-cluster.md b/docs/book/src/topics/eks/creating-a-cluster.md index 5fdea2f6b0..7ec523837f 100644 --- a/docs/book/src/topics/eks/creating-a-cluster.md +++ b/docs/book/src/topics/eks/creating-a-cluster.md @@ -14,8 +14,8 @@ clusterctl generate cluster capi-eks-quickstart --flavor eks-managedmachinepool NOTE: When creating an EKS cluster only the **MAJOR.MINOR** of the `-kubernetes-version` is taken into consideration. -By default, EKS cluster uses: -- EXTENDED support. See more info about [cluster upgrade policy](https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html) +By default CAPA relies on the default EKS cluster upgrade policy, which at the moment of writing is EXTENDED support. +See more info about [cluster upgrade policy](https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html) ## Kubeconfig From 0086f06d1f624b38530d79734840bfe0e819213b Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Mon, 8 Sep 2025 18:49:43 +0700 Subject: [PATCH 03/15] regenerate --- ...plane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml | 10 +++++----- ...uster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index d9769ccbd9..8cc5f39d9d 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -3245,11 +3245,11 @@ spec: type: string upgradePolicy: description: |- - The support policy to use for the cluster. - Extended support indicates that the cluster will not be automatically upgraded - when it leaves the standard support period, and will enter extended support. - Clusters in extended support have higher costs. - If omitted, new clusters use AWS default extended support and existing clusters stay unchanged. + The cluster upgrade policy to use for the cluster. + (Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html) + `extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges. + `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but you EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. + If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged. enum: - extended - standard diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml index bd8e7ea7f9..4b2bbc9e74 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml @@ -1046,11 +1046,11 @@ spec: type: string upgradePolicy: description: |- - The support policy to use for the cluster. - Extended support indicates that the cluster will not be automatically upgraded - when it leaves the standard support period, and will enter extended support. - Clusters in extended support have higher costs. - If omitted, new clusters use AWS default extended support and existing clusters stay unchanged. + The cluster upgrade policy to use for the cluster. + (Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html) + `extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges. + `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but you EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. + If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged. enum: - extended - standard From 9b841d7111ef2ac2632ec1cdf79f187bda4d2e85 Mon Sep 17 00:00:00 2001 From: Hung Tran <40334379+phuhung273@users.noreply.github.com> Date: Mon, 8 Sep 2025 05:18:58 -0700 Subject: [PATCH 04/15] Apply suggestions from code review Co-authored-by: Damiano Donati --- controlplane/eks/api/v1beta2/types.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/controlplane/eks/api/v1beta2/types.go b/controlplane/eks/api/v1beta2/types.go index 16b11a1aaf..8b090dc3a9 100644 --- a/controlplane/eks/api/v1beta2/types.go +++ b/controlplane/eks/api/v1beta2/types.go @@ -245,13 +245,13 @@ type AddonIssue struct { type UpgradePolicy string var ( - // UpgradePolicyExtended indicates that the cluster will not be automatically upgraded - // when it leaves the standard support period, and will enter extended support. - // Clusters in extended support have higher costs. + // UpgradePolicyExtended indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. + // You will incur extended support charges with this setting. + // You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges. UpgradePolicyExtended = UpgradePolicy("extended") - // UpgradePolicyStandard indicates that the cluster will be automatically upgraded - // when it leaves the standard support period. + // UpgradePolicyStandard indicates that the cluster is eligible for automatic upgrade at the end of standard support. + // You will not incur extended support charges with this setting but you EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. UpgradePolicyStandard = UpgradePolicy("standard") ) From b24d6a7562f1826c65c9a7d0b072d45838ccf4af Mon Sep 17 00:00:00 2001 From: Hung Tran <40334379+phuhung273@users.noreply.github.com> Date: Mon, 8 Sep 2025 07:31:33 -0700 Subject: [PATCH 05/15] Update pkg/cloud/services/eks/cluster.go Co-authored-by: Damiano Donati --- pkg/cloud/services/eks/cluster.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/cloud/services/eks/cluster.go b/pkg/cloud/services/eks/cluster.go index 38aa1d78e8..02a4826ecf 100644 --- a/pkg/cloud/services/eks/cluster.go +++ b/pkg/cloud/services/eks/cluster.go @@ -556,8 +556,7 @@ func (s *Service) reconcileClusterConfig(ctx context.Context, cluster *ekstypes. input.ResourcesVpcConfig = updateVpcConfig } - updateUpgradePolicy := s.reconcileUpgradePolicy(cluster.UpgradePolicy) - if updateUpgradePolicy != nil { + if updateUpgradePolicy := s.reconcileUpgradePolicy(cluster.UpgradePolicy); updateUpgradePolicy != nil { needsUpdate = true input.UpgradePolicy = updateUpgradePolicy } From 7fdf9a3ad8cb629c459878a1b5fcf18df6b79f01 Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Mon, 8 Sep 2025 21:46:05 +0700 Subject: [PATCH 06/15] remove log --- pkg/cloud/services/eks/cluster.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pkg/cloud/services/eks/cluster.go b/pkg/cloud/services/eks/cluster.go index 02a4826ecf..2aa6083ae6 100644 --- a/pkg/cloud/services/eks/cluster.go +++ b/pkg/cloud/services/eks/cluster.go @@ -799,22 +799,17 @@ func (s *Service) reconcileClusterVersion(ctx context.Context, cluster *ekstypes } func (s *Service) reconcileUpgradePolicy(upgradePolicy *ekstypes.UpgradePolicyResponse) *ekstypes.UpgradePolicyRequest { - s.Info("reconciling upgrade policy") - + // Should not update when cluster upgrade policy is unknown if upgradePolicy == nil { - s.Warn("cannot get cluster upgrade policy, no action") return nil } - clusterUpgradePolicy := upgradePolicy.SupportType - + // Cluster stay unchanged when upgrade policy omitted if s.scope.ControlPlane.Spec.UpgradePolicy == "" { - s.Debug("upgrade policy omitted, no action") return nil } - if strings.ToLower(string(clusterUpgradePolicy)) == s.scope.ControlPlane.Spec.UpgradePolicy.String() { - s.Debug("upgrade policy unchanged, no action") + if strings.ToLower(string(upgradePolicy.SupportType)) == s.scope.ControlPlane.Spec.UpgradePolicy.String() { return nil } From 9a68a2dac7e720f6e31f9674d33bd69b2ea5b666 Mon Sep 17 00:00:00 2001 From: Hung Tran <40334379+phuhung273@users.noreply.github.com> Date: Tue, 9 Sep 2025 16:47:37 -0700 Subject: [PATCH 07/15] Update config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml Co-authored-by: Faiq --- .../controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index 8cc5f39d9d..df89cffa49 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -3248,7 +3248,7 @@ spec: The cluster upgrade policy to use for the cluster. (Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html) `extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges. - `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but you EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. + `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged. enum: - extended From 35debd1a3be2b079cff380a7a020845a1953091b Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Wed, 10 Sep 2025 06:51:21 +0700 Subject: [PATCH 08/15] docstring typo --- ...lplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml | 2 +- controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go | 2 +- controlplane/eks/api/v1beta2/types.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml index 4b2bbc9e74..450fd296b0 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml @@ -1049,7 +1049,7 @@ spec: The cluster upgrade policy to use for the cluster. (Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html) `extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges. - `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but you EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. + `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged. enum: - extended diff --git a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go index 2c95d9bbf9..be93930441 100644 --- a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go +++ b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go @@ -216,7 +216,7 @@ type AWSManagedControlPlaneSpec struct { //nolint: maligned // The cluster upgrade policy to use for the cluster. // (Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html) // `extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges. - // `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but you EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. + // `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. // If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged. // +kubebuilder:validation:Enum=extended;standard // +optional diff --git a/controlplane/eks/api/v1beta2/types.go b/controlplane/eks/api/v1beta2/types.go index 8b090dc3a9..60cd4b454d 100644 --- a/controlplane/eks/api/v1beta2/types.go +++ b/controlplane/eks/api/v1beta2/types.go @@ -251,7 +251,7 @@ var ( UpgradePolicyExtended = UpgradePolicy("extended") // UpgradePolicyStandard indicates that the cluster is eligible for automatic upgrade at the end of standard support. - // You will not incur extended support charges with this setting but you EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. + // You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support. UpgradePolicyStandard = UpgradePolicy("standard") ) From f6c6b43ce78af4a0b0308b6294d4c3c727b1d891 Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Wed, 10 Sep 2025 13:09:16 +0700 Subject: [PATCH 09/15] set NotReady if cluster was automatically upgraded --- pkg/cloud/services/eks/cluster.go | 50 ++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/pkg/cloud/services/eks/cluster.go b/pkg/cloud/services/eks/cluster.go index 2aa6083ae6..84a7318695 100644 --- a/pkg/cloud/services/eks/cluster.go +++ b/pkg/cloud/services/eks/cluster.go @@ -149,23 +149,7 @@ func (s *Service) reconcileCluster(ctx context.Context) error { // computeCurrentStatusVersion returns the computed current EKS cluster kubernetes version. // The computation has awareness of the fact that EKS clusters only return a major.minor kubernetes version, // and returns a compatible version for te status according to the one the user specified in the spec. -func computeCurrentStatusVersion(specV *string, clusterV *string) *string { - specVersion := "" - if specV != nil { - specVersion = *specV - } - - clusterVersion := "" - if clusterV != nil { - clusterVersion = *clusterV - } - - // Ignore parsing errors as these are already validated by the kubebuilder validation and the AWS API. - // Also specVersion might not be specified in the spec.Version for AWSManagedControlPlane, this results in a "0.0.0" version. - // Also clusterVersion might not yet be returned by the AWS EKS API, as the cluster might still be initializing, this results in a "0.0.0" version. - specSemverVersion, _ := semver.ParseTolerant(specVersion) - currentSemverVersion, _ := semver.ParseTolerant(clusterVersion) - +func computeCurrentStatusVersion(clusterV *string, specSemverVersion semver.Version, currentSemverVersion semver.Version) *string { // If AWS EKS API is not returning a version, set the status.Version to empty string. if currentSemverVersion.String() == "0.0.0" { return ptr.To("") @@ -189,9 +173,27 @@ func computeCurrentStatusVersion(specV *string, clusterV *string) *string { return clusterV } +// parseClusterVersionString parse a version string to semver version. +// If the string cannot be parsed to semver, returning 0.0.0. +func parseClusterVersionString(str *string) semver.Version { + version := "" + if str != nil { + version = *str + } + + // Ignore parsing errors as these are already validated by the kubebuilder validation and the AWS API. + semverVersion, _ := semver.ParseTolerant(version) + return semverVersion +} + func (s *Service) setStatus(cluster *ekstypes.Cluster) error { + // specSemver might not be specified in the spec.Version for AWSManagedControlPlane, this results in a "0.0.0" version. + specSemver := parseClusterVersionString(s.scope.ControlPlane.Spec.Version) + // clusterSemver might not yet be returned by the AWS EKS API, as the cluster might still be initializing, this results in a "0.0.0" version. + clusterSemver := parseClusterVersionString(cluster.Version) + // Set the current Kubernetes control plane version in the status. - s.scope.ControlPlane.Status.Version = computeCurrentStatusVersion(s.scope.ControlPlane.Spec.Version, cluster.Version) + s.scope.ControlPlane.Status.Version = computeCurrentStatusVersion(cluster.Version, specSemver, clusterSemver) // Set the current cluster status in the control plane status. switch cluster.Status { @@ -213,6 +215,18 @@ func (s *Service) setStatus(cluster *ekstypes.Cluster) error { conditions.MarkFalse(s.scope.ControlPlane, ekscontrolplanev1.EKSControlPlaneUpdatingCondition, "updated", clusterv1.ConditionSeverityInfo, "") record.Eventf(s.scope.ControlPlane, "SuccessfulUpdateEKSControlPlane", "Updated EKS control plane %s", s.scope.KubernetesClusterName()) } + if s.scope.ControlPlane.Spec.UpgradePolicy == ekscontrolplanev1.UpgradePolicyStandard && + (specSemver.Major < clusterSemver.Major || specSemver.Minor < clusterSemver.Minor) { + s.scope.ControlPlane.Status.Ready = false + failureMsg := fmt.Sprintf( + "EKS control plane %s was automatically upgraded to version %s because %s is out of standard support. "+ + "This can be fixed by bumping to the actual version of the cluster", + s.scope.KubernetesClusterName(), + clusterSemver.String(), + specSemver.String(), + ) + s.scope.ControlPlane.Status.FailureMessage = &failureMsg + } // TODO FailureReason case ekstypes.ClusterStatusCreating: s.scope.ControlPlane.Status.Ready = false From d56104b024c2d243bddd757d44362263a9c2b002 Mon Sep 17 00:00:00 2001 From: Hung Tran <40334379+phuhung273@users.noreply.github.com> Date: Wed, 10 Sep 2025 17:16:43 -0700 Subject: [PATCH 10/15] Update pkg/cloud/services/eks/cluster.go Co-authored-by: Faiq --- pkg/cloud/services/eks/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloud/services/eks/cluster.go b/pkg/cloud/services/eks/cluster.go index 84a7318695..37d22e6c9c 100644 --- a/pkg/cloud/services/eks/cluster.go +++ b/pkg/cloud/services/eks/cluster.go @@ -220,7 +220,7 @@ func (s *Service) setStatus(cluster *ekstypes.Cluster) error { s.scope.ControlPlane.Status.Ready = false failureMsg := fmt.Sprintf( "EKS control plane %s was automatically upgraded to version %s because %s is out of standard support. "+ - "This can be fixed by bumping to the actual version of the cluster", + "This can be fixed by changing to the version of the AWSManagedControlPlane to the one reported in the status", s.scope.KubernetesClusterName(), clusterSemver.String(), specSemver.String(), From deedc91910f0951d0b22d6d1882e950bb7b6eadf Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Thu, 11 Sep 2025 07:28:31 +0700 Subject: [PATCH 11/15] fix version compare logic --- pkg/cloud/services/eks/cluster.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/cloud/services/eks/cluster.go b/pkg/cloud/services/eks/cluster.go index 37d22e6c9c..87f68755e6 100644 --- a/pkg/cloud/services/eks/cluster.go +++ b/pkg/cloud/services/eks/cluster.go @@ -216,7 +216,8 @@ func (s *Service) setStatus(cluster *ekstypes.Cluster) error { record.Eventf(s.scope.ControlPlane, "SuccessfulUpdateEKSControlPlane", "Updated EKS control plane %s", s.scope.KubernetesClusterName()) } if s.scope.ControlPlane.Spec.UpgradePolicy == ekscontrolplanev1.UpgradePolicyStandard && - (specSemver.Major < clusterSemver.Major || specSemver.Minor < clusterSemver.Minor) { + (specSemver.Major < clusterSemver.Major || + (specSemver.Major == clusterSemver.Major && specSemver.Minor < clusterSemver.Minor)) { s.scope.ControlPlane.Status.Ready = false failureMsg := fmt.Sprintf( "EKS control plane %s was automatically upgraded to version %s because %s is out of standard support. "+ From 70c8312f0eac999521f3ed92afb3f1df71fe0a41 Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Fri, 19 Sep 2025 21:32:55 +0700 Subject: [PATCH 12/15] e2e --- test/e2e/data/e2e_eks_conf.yaml | 2 + .../cluster-template-eks-upgrade-policy.yaml | 35 ++++++ test/e2e/shared/defaults.go | 1 + .../suites/managed/eks_upgrade_policy_test.go | 119 ++++++++++++++++++ test/e2e/suites/managed/helpers.go | 1 + 5 files changed, 158 insertions(+) create mode 100644 test/e2e/data/eks/cluster-template-eks-upgrade-policy.yaml create mode 100644 test/e2e/suites/managed/eks_upgrade_policy_test.go diff --git a/test/e2e/data/e2e_eks_conf.yaml b/test/e2e/data/e2e_eks_conf.yaml index b8230eace7..8271d1b529 100644 --- a/test/e2e/data/e2e_eks_conf.yaml +++ b/test/e2e/data/e2e_eks_conf.yaml @@ -112,6 +112,8 @@ providers: targetName: "cluster-template-eks-managedmachinepool.yaml" - sourcePath: "./eks/cluster-template-eks-ipv6-cluster.yaml" targetName: "cluster-template-eks-ipv6-cluster.yaml" + - sourcePath: "./eks/cluster-template-eks-upgrade-policy.yaml" + targetName: "cluster-template-eks-upgrade-policy.yaml" - sourcePath: "./eks/cluster-template-eks-control-plane-only-legacy.yaml" targetName: "cluster-template-eks-control-plane-only-legacy.yaml" - sourcePath: "./eks/cluster-template-eks-control-plane-bare-eks.yaml" diff --git a/test/e2e/data/eks/cluster-template-eks-upgrade-policy.yaml b/test/e2e/data/eks/cluster-template-eks-upgrade-policy.yaml new file mode 100644 index 0000000000..16db15c010 --- /dev/null +++ b/test/e2e/data/eks/cluster-template-eks-upgrade-policy.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + kind: AWSManagedCluster + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}" + controlPlaneRef: + kind: AWSManagedControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}-control-plane" +--- +kind: AWSManagedCluster +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_NAME}" +spec: {} +--- +kind: AWSManagedControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + region: "${AWS_REGION}" + version: "${KUBERNETES_VERSION}" + upgradePolicy: "${UPGRADE_POLICY}" + identityRef: + kind: AWSClusterStaticIdentity + name: e2e-account diff --git a/test/e2e/shared/defaults.go b/test/e2e/shared/defaults.go index 4b7adf22c6..48d8417f57 100644 --- a/test/e2e/shared/defaults.go +++ b/test/e2e/shared/defaults.go @@ -69,6 +69,7 @@ const ( MultiTenancy = "MULTI_TENANCY_" EksUpgradeFromVersion = "UPGRADE_FROM_VERSION" EksUpgradeToVersion = "UPGRADE_TO_VERSION" + UpgradePolicy = "UPGRADE_POLICY" ClassicElbTestKubernetesFrom = "CLASSICELB_TEST_KUBERNETES_VERSION_FROM" ClassicElbTestKubernetesTo = "CLASSICELB_TEST_KUBERNETES_VERSION_TO" diff --git a/test/e2e/suites/managed/eks_upgrade_policy_test.go b/test/e2e/suites/managed/eks_upgrade_policy_test.go new file mode 100644 index 0000000000..6f93568c33 --- /dev/null +++ b/test/e2e/suites/managed/eks_upgrade_policy_test.go @@ -0,0 +1,119 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package managed + +import ( + "context" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + + ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/converters" + "sigs.k8s.io/cluster-api-provider-aws/v2/test/e2e/shared" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/util" +) + +// EKS upgrade policy test. +var _ = ginkgo.Describe("EKS upgrade policy test", func() { + var ( + namespace *corev1.Namespace + ctx context.Context + specName = "cluster" + clusterName string + ) + + ginkgo.It("[managed] [upgrade-policy] Able to update cluster upgrade policy from STANDARD to EXTENDED", func() { + ginkgo.By("should have a valid test configuration") + Expect(e2eCtx.Environment.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. BootstrapClusterProxy can't be nil") + Expect(e2eCtx.E2EConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName) + + upgradePolicy := ekscontrolplanev1.UpgradePolicyStandard + shared.SetEnvVar(shared.UpgradePolicy, upgradePolicy.String(), false) + + ctx = context.TODO() + namespace = shared.SetupSpecNamespace(ctx, specName, e2eCtx) + clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + eksClusterName := getEKSClusterName(namespace.Name, clusterName) + + ginkgo.By("default iam role should exist") + VerifyRoleExistsAndOwned(ctx, ekscontrolplanev1.DefaultEKSControlPlaneRole, eksClusterName, false, e2eCtx.AWSSession) + + getManagedClusterSpec := func() ManagedClusterSpecInput { + return ManagedClusterSpecInput{ + E2EConfig: e2eCtx.E2EConfig, + ConfigClusterFn: defaultConfigCluster, + BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, + AWSSession: e2eCtx.BootstrapUserAWSSession, + Namespace: namespace, + ClusterName: clusterName, + Flavour: EKSUpgradePolicyFlavor, + ControlPlaneMachineCount: 1, // NOTE: this cannot be zero as clusterctl returns an error + WorkerMachineCount: 0, + } + } + + ginkgo.By("should create an EKS control plane") + ManagedClusterSpec(ctx, getManagedClusterSpec) + + ginkgo.By(fmt.Sprintf("getting cluster with name %s", clusterName)) + cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{ + Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + Name: clusterName, + }) + Expect(cluster).NotTo(BeNil(), "couldn't find cluster") + + WaitForEKSClusterUpgradePolicy(ctx, e2eCtx.BootstrapUserAWSSession, eksClusterName, upgradePolicy) + + upgradePolicy = ekscontrolplanev1.UpgradePolicyExtended + ginkgo.By(fmt.Sprintf("should update upgrade policy to %s", upgradePolicy)) + shared.SetEnvVar(shared.UpgradePolicy, upgradePolicy.String(), false) + ManagedClusterSpec(ctx, getManagedClusterSpec) + WaitForEKSClusterUpgradePolicy(ctx, e2eCtx.BootstrapUserAWSSession, eksClusterName, upgradePolicy) + + framework.DeleteCluster(ctx, framework.DeleteClusterInput{ + Deleter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + Cluster: cluster, + }) + framework.WaitForClusterDeleted(ctx, framework.WaitForClusterDeletedInput{ + ClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, + Cluster: cluster, + ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath, + ArtifactFolder: e2eCtx.Settings.ArtifactFolder, + }, e2eCtx.E2EConfig.GetIntervals("", "wait-delete-cluster")...) + }) +}) + +func WaitForEKSClusterUpgradePolicy(ctx context.Context, sess *aws.Config, eksClusterName string, upgradePolicy ekscontrolplanev1.UpgradePolicy) { + ginkgo.By(fmt.Sprintf("EKS control plane upgrade policy should be %s", upgradePolicy)) + Eventually(func() (bool, error) { + cluster, err := getEKSCluster(ctx, eksClusterName, sess) + if err != nil { + return false, err + } + return converters.SupportTypeToSDK(upgradePolicy) == cluster.UpgradePolicy.SupportType, nil + }, 5*time.Minute, 10*time.Second).Should(BeTrue(), fmt.Sprintf("eventually failed checking EKS Cluster %q upgrade policy is %s", eksClusterName, upgradePolicy)) +} diff --git a/test/e2e/suites/managed/helpers.go b/test/e2e/suites/managed/helpers.go index 926d914248..2d6a763deb 100644 --- a/test/e2e/suites/managed/helpers.go +++ b/test/e2e/suites/managed/helpers.go @@ -48,6 +48,7 @@ const ( EKSManagedMachinePoolWithLaunchTemplateOnlyFlavor = "eks-managed-machinepool-with-launch-template-only" EKSMachinePoolOnlyFlavor = "eks-machinepool-only" EKSIPv6ClusterFlavor = "eks-ipv6-cluster" + EKSUpgradePolicyFlavor = "eks-upgrade-policy" EKSControlPlaneOnlyLegacyFlavor = "eks-control-plane-only-legacy" EKSClusterClassFlavor = "eks-clusterclass" EKSAuthAPIAndConfigMapFlavor = "eks-auth-api-and-config-map" From 997733c0c25bc54c4651e75102b72e6267656fe6 Mon Sep 17 00:00:00 2001 From: Hung Tran <40334379+phuhung273@users.noreply.github.com> Date: Tue, 23 Sep 2025 17:01:00 -0700 Subject: [PATCH 13/15] Apply suggestions from code review Co-authored-by: Damiano Donati --- test/e2e/suites/managed/eks_upgrade_policy_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/suites/managed/eks_upgrade_policy_test.go b/test/e2e/suites/managed/eks_upgrade_policy_test.go index 6f93568c33..48dd127034 100644 --- a/test/e2e/suites/managed/eks_upgrade_policy_test.go +++ b/test/e2e/suites/managed/eks_upgrade_policy_test.go @@ -88,11 +88,11 @@ var _ = ginkgo.Describe("EKS upgrade policy test", func() { WaitForEKSClusterUpgradePolicy(ctx, e2eCtx.BootstrapUserAWSSession, eksClusterName, upgradePolicy) - upgradePolicy = ekscontrolplanev1.UpgradePolicyExtended - ginkgo.By(fmt.Sprintf("should update upgrade policy to %s", upgradePolicy)) - shared.SetEnvVar(shared.UpgradePolicy, upgradePolicy.String(), false) + changedUpgradePolicy = ekscontrolplanev1.UpgradePolicyExtended + ginkgo.By(fmt.Sprintf("Changing the UpgradePolicy from %s to %s", upgradePolicy, changedUpgradePolicy)) + shared.SetEnvVar(shared.UpgradePolicy, changedUpgradePolicy.String(), false) ManagedClusterSpec(ctx, getManagedClusterSpec) - WaitForEKSClusterUpgradePolicy(ctx, e2eCtx.BootstrapUserAWSSession, eksClusterName, upgradePolicy) + WaitForEKSClusterUpgradePolicy(ctx, e2eCtx.BootstrapUserAWSSession, eksClusterName, changedUpgradePolicy) framework.DeleteCluster(ctx, framework.DeleteClusterInput{ Deleter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), @@ -108,7 +108,7 @@ var _ = ginkgo.Describe("EKS upgrade policy test", func() { }) func WaitForEKSClusterUpgradePolicy(ctx context.Context, sess *aws.Config, eksClusterName string, upgradePolicy ekscontrolplanev1.UpgradePolicy) { - ginkgo.By(fmt.Sprintf("EKS control plane upgrade policy should be %s", upgradePolicy)) + ginkgo.By(fmt.Sprintf("Checking EKS control plane upgrade policy matches %s", upgradePolicy)) Eventually(func() (bool, error) { cluster, err := getEKSCluster(ctx, eksClusterName, sess) if err != nil { From c27451968f394b8094f73cf716e35e8430257362 Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Wed, 24 Sep 2025 07:04:38 +0700 Subject: [PATCH 14/15] syntax --- test/e2e/suites/managed/eks_upgrade_policy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/suites/managed/eks_upgrade_policy_test.go b/test/e2e/suites/managed/eks_upgrade_policy_test.go index 48dd127034..739ef91c8c 100644 --- a/test/e2e/suites/managed/eks_upgrade_policy_test.go +++ b/test/e2e/suites/managed/eks_upgrade_policy_test.go @@ -88,7 +88,7 @@ var _ = ginkgo.Describe("EKS upgrade policy test", func() { WaitForEKSClusterUpgradePolicy(ctx, e2eCtx.BootstrapUserAWSSession, eksClusterName, upgradePolicy) - changedUpgradePolicy = ekscontrolplanev1.UpgradePolicyExtended + changedUpgradePolicy := ekscontrolplanev1.UpgradePolicyExtended ginkgo.By(fmt.Sprintf("Changing the UpgradePolicy from %s to %s", upgradePolicy, changedUpgradePolicy)) shared.SetEnvVar(shared.UpgradePolicy, changedUpgradePolicy.String(), false) ManagedClusterSpec(ctx, getManagedClusterSpec) From c196b5c496312446323e5f2b4e039008a406ffbf Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Wed, 24 Sep 2025 21:41:43 +0700 Subject: [PATCH 15/15] WaitForEKSClusterUpgradePolicy fail early on NotFound --- .../suites/managed/eks_upgrade_policy_test.go | 26 ++++++++++++++++--- test/e2e/suites/managed/helpers.go | 4 +++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/test/e2e/suites/managed/eks_upgrade_policy_test.go b/test/e2e/suites/managed/eks_upgrade_policy_test.go index 739ef91c8c..16208f4b90 100644 --- a/test/e2e/suites/managed/eks_upgrade_policy_test.go +++ b/test/e2e/suites/managed/eks_upgrade_policy_test.go @@ -25,11 +25,13 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" + ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/awserrors" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/converters" "sigs.k8s.io/cluster-api-provider-aws/v2/test/e2e/shared" "sigs.k8s.io/cluster-api/test/framework" @@ -109,11 +111,27 @@ var _ = ginkgo.Describe("EKS upgrade policy test", func() { func WaitForEKSClusterUpgradePolicy(ctx context.Context, sess *aws.Config, eksClusterName string, upgradePolicy ekscontrolplanev1.UpgradePolicy) { ginkgo.By(fmt.Sprintf("Checking EKS control plane upgrade policy matches %s", upgradePolicy)) - Eventually(func() (bool, error) { + Eventually(func() error { cluster, err := getEKSCluster(ctx, eksClusterName, sess) if err != nil { - return false, err + smithyErr := awserrors.ParseSmithyError(err) + notFoundErr := &ekstypes.ResourceNotFoundException{} + if smithyErr.ErrorCode() == notFoundErr.ErrorCode() { + // Unrecoverable error stop trying and fail early. + return StopTrying(fmt.Sprintf("unrecoverable error: cluster %q not found: %s", eksClusterName, smithyErr.ErrorMessage())) + } + return err // For transient errors, retry } - return converters.SupportTypeToSDK(upgradePolicy) == cluster.UpgradePolicy.SupportType, nil - }, 5*time.Minute, 10*time.Second).Should(BeTrue(), fmt.Sprintf("eventually failed checking EKS Cluster %q upgrade policy is %s", eksClusterName, upgradePolicy)) + + expectedPolicy := converters.SupportTypeToSDK(upgradePolicy) + actualPolicy := cluster.UpgradePolicy.SupportType + + if actualPolicy != expectedPolicy { + // The upgrade policy change hasn't been reflected in EKS yet, error and try again. + return fmt.Errorf("upgrade policy mismatch: expected %s, but found %s", expectedPolicy, actualPolicy) + } + + // Success in finding the change has been reflected in EKS. + return nil + }, 5*time.Minute, 10*time.Second).Should(Succeed(), fmt.Sprintf("eventually failed checking EKS Cluster %q upgrade policy is %s", eksClusterName, upgradePolicy)) } diff --git a/test/e2e/suites/managed/helpers.go b/test/e2e/suites/managed/helpers.go index 2d6a763deb..dce878d89a 100644 --- a/test/e2e/suites/managed/helpers.go +++ b/test/e2e/suites/managed/helpers.go @@ -106,6 +106,10 @@ func getEKSCluster(ctx context.Context, eksClusterName string, sess *aws.Config) } result, err := eksClient.DescribeCluster(ctx, input) + if err != nil { + return nil, err + } + return result.Cluster, err }