diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index cb64c61c82..6bc339e9e3 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -2245,6 +2245,28 @@ spec: description: AWSManagedControlPlaneSpec defines the desired state of an Amazon EKS Cluster. properties: + accessConfig: + description: AccessConfig specifies the access configuration information + for the cluster + properties: + authenticationMode: + default: config_map + description: |- + AuthenticationMode specifies the desired authentication mode for the cluster + Defaults to config_map + enum: + - config_map + - api + - api_and_config_map + type: string + bootstrapClusterCreatorAdminPermissions: + default: true + description: |- + BootstrapClusterCreatorAdminPermissions grants cluster admin permissions + to the IAM identity creating the cluster. Only applied during creation, + ignored when updating existing clusters. Defaults to true. + type: boolean + type: object additionalTags: additionalProperties: type: string diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml index 7a0abb3cf8..ad5c56c54b 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml @@ -53,6 +53,28 @@ spec: description: AWSManagedControlPlaneSpec defines the desired state of an Amazon EKS Cluster. properties: + accessConfig: + description: AccessConfig specifies the access configuration + information for the cluster + properties: + authenticationMode: + default: config_map + description: |- + AuthenticationMode specifies the desired authentication mode for the cluster + Defaults to config_map + enum: + - config_map + - api + - api_and_config_map + type: string + bootstrapClusterCreatorAdminPermissions: + default: true + description: |- + BootstrapClusterCreatorAdminPermissions grants cluster admin permissions + to the IAM identity creating the cluster. Only applied during creation, + ignored when updating existing clusters. Defaults to true. + type: boolean + type: object additionalTags: additionalProperties: type: string diff --git a/controlplane/eks/api/v1beta1/conversion.go b/controlplane/eks/api/v1beta1/conversion.go index 4039b113d4..0985ef66d5 100644 --- a/controlplane/eks/api/v1beta1/conversion.go +++ b/controlplane/eks/api/v1beta1/conversion.go @@ -117,6 +117,7 @@ func (r *AWSManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.Partition = restored.Spec.Partition dst.Spec.RestrictPrivateSubnets = restored.Spec.RestrictPrivateSubnets + dst.Spec.AccessConfig = restored.Spec.AccessConfig dst.Spec.RolePath = restored.Spec.RolePath dst.Spec.RolePermissionsBoundary = restored.Spec.RolePermissionsBoundary dst.Status.Version = restored.Status.Version diff --git a/controlplane/eks/api/v1beta1/zz_generated.conversion.go b/controlplane/eks/api/v1beta1/zz_generated.conversion.go index 9fe8517b2f..48f326b2dc 100644 --- a/controlplane/eks/api/v1beta1/zz_generated.conversion.go +++ b/controlplane/eks/api/v1beta1/zz_generated.conversion.go @@ -375,6 +375,7 @@ func autoConvert_v1beta2_AWSManagedControlPlaneSpec_To_v1beta1_AWSManagedControl out.AssociateOIDCProvider = in.AssociateOIDCProvider out.Addons = (*[]Addon)(unsafe.Pointer(in.Addons)) out.OIDCIdentityProviderConfig = (*OIDCIdentityProviderConfig)(unsafe.Pointer(in.OIDCIdentityProviderConfig)) + // WARNING: in.AccessConfig requires manual conversion: does not exist in peer-type if err := Convert_v1beta2_VpcCni_To_v1beta1_VpcCni(&in.VpcCni, &out.VpcCni, s); err != nil { return err } diff --git a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go index a36b35dda3..9112863e35 100644 --- a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go +++ b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go @@ -192,6 +192,10 @@ type AWSManagedControlPlaneSpec struct { //nolint: maligned // +optional OIDCIdentityProviderConfig *OIDCIdentityProviderConfig `json:"oidcIdentityProviderConfig,omitempty"` + // AccessConfig specifies the access configuration information for the cluster + // +optional + AccessConfig *AccessConfig `json:"accessConfig,omitempty"` + // VpcCni is used to set configuration options for the VPC CNI plugin // +optional VpcCni VpcCni `json:"vpcCni,omitempty"` @@ -248,6 +252,21 @@ type EndpointAccess struct { Private *bool `json:"private,omitempty"` } +// AccessConfig represents the access configuration information for the cluster +type AccessConfig struct { + // AuthenticationMode specifies the desired authentication mode for the cluster + // Defaults to config_map + // +kubebuilder:default=config_map + // +kubebuilder:validation:Enum=config_map;api;api_and_config_map + AuthenticationMode EKSAuthenticationMode `json:"authenticationMode,omitempty"` + + // BootstrapClusterCreatorAdminPermissions grants cluster admin permissions + // to the IAM identity creating the cluster. Only applied during creation, + // ignored when updating existing clusters. Defaults to true. + // +kubebuilder:default=true + BootstrapClusterCreatorAdminPermissions *bool `json:"bootstrapClusterCreatorAdminPermissions,omitempty"` +} + // EncryptionConfig specifies the encryption configuration for the EKS clsuter. type EncryptionConfig struct { // Provider specifies the ARN or alias of the CMK (in AWS KMS) diff --git a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook.go b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook.go index ffb4e891b8..5554eff7c1 100644 --- a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook.go +++ b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook.go @@ -107,6 +107,7 @@ func (*awsManagedControlPlaneWebhook) ValidateCreate(_ context.Context, obj runt allErrs = append(allErrs, r.Spec.AdditionalTags.Validate()...) allErrs = append(allErrs, r.validateNetwork()...) allErrs = append(allErrs, r.validatePrivateDNSHostnameTypeOnLaunch()...) + allErrs = append(allErrs, r.validateAccessConfigCreate()...) if len(allErrs) == 0 { return nil, nil @@ -140,6 +141,7 @@ func (*awsManagedControlPlaneWebhook) ValidateUpdate(ctx context.Context, oldObj allErrs = append(allErrs, r.validateEKSClusterNameSame(oldAWSManagedControlplane)...) allErrs = append(allErrs, r.validateEKSVersion(oldAWSManagedControlplane)...) allErrs = append(allErrs, r.Spec.Bastion.Validate()...) + allErrs = append(allErrs, r.validateAccessConfigUpdate(oldAWSManagedControlplane)...) allErrs = append(allErrs, r.validateIAMAuthConfig()...) allErrs = append(allErrs, r.validateSecondaryCIDR()...) allErrs = append(allErrs, r.validateEKSAddons()...) @@ -318,6 +320,53 @@ func validateEKSAddons(eksVersion *string, networkSpec infrav1.NetworkSpec, addo return allErrs } +func (r *AWSManagedControlPlane) validateAccessConfigUpdate(old *AWSManagedControlPlane) field.ErrorList { + var allErrs field.ErrorList + + // If accessConfig is already set, do not allow removal of it. + if old.Spec.AccessConfig != nil && r.Spec.AccessConfig == nil { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "accessConfig"), r.Spec.AccessConfig, "removing AccessConfig is not allowed after it has been enabled"), + ) + } + + // AuthenticationMode is ratcheting - do not allow downgrades + if old.Spec.AccessConfig != nil && r.Spec.AccessConfig != nil && + old.Spec.AccessConfig.AuthenticationMode != r.Spec.AccessConfig.AuthenticationMode && + ((old.Spec.AccessConfig.AuthenticationMode == EKSAuthenticationModeAPIAndConfigMap && r.Spec.AccessConfig.AuthenticationMode == EKSAuthenticationModeConfigMap) || + old.Spec.AccessConfig.AuthenticationMode == EKSAuthenticationModeAPI) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "accessConfig", "authenticationMode"), r.Spec.AccessConfig.AuthenticationMode, "downgrading authentication mode is not allowed after it has been enabled"), + ) + } + + // BootstrapClusterCreatorAdminPermissions only applies on create, but changes should not invalidate updates + if old.Spec.AccessConfig != nil && r.Spec.AccessConfig != nil && + old.Spec.AccessConfig.BootstrapClusterCreatorAdminPermissions != r.Spec.AccessConfig.BootstrapClusterCreatorAdminPermissions { + mcpLog.Info("Ignoring changes to BootstrapClusterCreatorAdminPermissions on cluster update", "old", old.Spec.AccessConfig.BootstrapClusterCreatorAdminPermissions, "new", r.Spec.AccessConfig.BootstrapClusterCreatorAdminPermissions) + } + + return allErrs +} + +func (r *AWSManagedControlPlane) validateAccessConfigCreate() field.ErrorList { + var allErrs field.ErrorList + + if r.Spec.AccessConfig != nil { + if r.Spec.AccessConfig.AuthenticationMode == EKSAuthenticationModeConfigMap && + r.Spec.AccessConfig.BootstrapClusterCreatorAdminPermissions != nil && + !*r.Spec.AccessConfig.BootstrapClusterCreatorAdminPermissions { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "accessConfig", "bootstrapClusterCreatorAdminPermissions"), + *r.Spec.AccessConfig.BootstrapClusterCreatorAdminPermissions, + "bootstrapClusterCreatorAdminPermissions must be true if cluster authentication mode is set to config_map"), + ) + } + } + + return allErrs +} + func (r *AWSManagedControlPlane) validateIAMAuthConfig() field.ErrorList { return validateIAMAuthConfig(r.Spec.IAMAuthenticatorConfig, field.NewPath("spec.iamAuthenticatorConfig")) } diff --git a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go index 276faa5b09..40de7b369b 100644 --- a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go +++ b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go @@ -179,6 +179,7 @@ func TestWebhookCreate(t *testing.T) { secondaryCidr *string secondaryCidrBlocks []infrav1.VpcCidrBlock kubeProxy KubeProxy + accessConfig *AccessConfig }{ { name: "ekscluster specified", @@ -322,6 +323,47 @@ func TestWebhookCreate(t *testing.T) { Disable: true, }, }, + { + name: "BootstrapClusterCreatorAdminPermissions true with EKSAuthenticationModeConfigMap", + eksClusterName: "default_cluster1", + eksVersion: "v1.19", + expectError: false, + accessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeConfigMap, + BootstrapClusterCreatorAdminPermissions: ptr.To(true), + }, + }, + { + name: "BootstrapClusterCreatorAdminPermissions false with EKSAuthenticationModeConfigMap", + eksClusterName: "default_cluster1", + eksVersion: "v1.19", + expectError: true, + expectErrorToContain: "bootstrapClusterCreatorAdminPermissions must be true if cluster authentication mode is set to config_map", + accessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeConfigMap, + BootstrapClusterCreatorAdminPermissions: ptr.To(false), + }, + }, + { + name: "BootstrapClusterCreatorAdminPermissions false with EKSAuthenticationModeAPIAndConfigMap", + eksClusterName: "default_cluster1", + eksVersion: "v1.19", + expectError: false, + accessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeAPIAndConfigMap, + BootstrapClusterCreatorAdminPermissions: ptr.To(false), + }, + }, + { + name: "BootstrapClusterCreatorAdminPermissions false with EKSAuthenticationModeAPI", + eksClusterName: "default_cluster1", + eksVersion: "v1.19", + expectError: false, + accessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeAPI, + BootstrapClusterCreatorAdminPermissions: ptr.To(false), + }, + }, } for _, tc := range tests { @@ -365,6 +407,9 @@ func TestWebhookCreate(t *testing.T) { if tc.secondaryCidr != nil { mcp.Spec.SecondaryCidrBlock = tc.secondaryCidr } + if tc.accessConfig != nil { + mcp.Spec.AccessConfig = tc.accessConfig + } err := testEnv.Create(ctx, mcp) @@ -603,6 +648,112 @@ func TestWebhookUpdate(t *testing.T) { }, expectError: false, }, + { + name: "no change in access config", + oldClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeConfigMap, + }, + }, + newClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeConfigMap, + }, + }, + expectError: false, + }, + { + name: "change in access config to nil", + oldClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeConfigMap, + }, + }, + newClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + }, + expectError: true, + }, + { + name: "change in access config from nil to valid", + oldClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + }, + newClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeConfigMap, + }, + }, + expectError: false, + }, + { + name: "change in access config auth mode from ApiAndConfigMap to API is allowed", + oldClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeAPIAndConfigMap, + }, + }, + newClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeAPI, + }, + }, + expectError: false, + }, + { + name: "change in access config auth mode from API to Config Map is denied", + oldClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeAPI, + }, + }, + newClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeConfigMap, + }, + }, + expectError: true, + }, + { + name: "change in access config auth mode from APIAndConfigMap to Config Map is denied", + oldClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeAPIAndConfigMap, + }, + }, + newClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + AuthenticationMode: EKSAuthenticationModeConfigMap, + }, + }, + expectError: true, + }, + { + name: "change in access config bootstrap admin permissions is ignored", + oldClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + BootstrapClusterCreatorAdminPermissions: ptr.To(true), + }, + }, + newClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + AccessConfig: &AccessConfig{ + BootstrapClusterCreatorAdminPermissions: ptr.To(false), + }, + }, + expectError: false, + }, { name: "change in encryption config to nil", oldClusterSpec: AWSManagedControlPlaneSpec{ diff --git a/controlplane/eks/api/v1beta2/types.go b/controlplane/eks/api/v1beta2/types.go index 622e4b9c3d..79f58f8e77 100644 --- a/controlplane/eks/api/v1beta2/types.go +++ b/controlplane/eks/api/v1beta2/types.go @@ -18,6 +18,7 @@ package v1beta2 import ( "fmt" + "strings" ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -79,6 +80,26 @@ var ( EKSTokenMethodAWSCli = EKSTokenMethod("aws-cli") ) +// EKSAuthenticationMode defines the authentication mode for the cluster +type EKSAuthenticationMode string + +// APIValue returns the corresponding EKS API value for the authentication mode +func (e EKSAuthenticationMode) APIValue() ekstypes.AuthenticationMode { + return ekstypes.AuthenticationMode(strings.ToUpper(string(e))) +} + +var ( + // EKSAuthenticationModeConfigMap indicates that only `aws-auth` ConfigMap will be used for authentication + EKSAuthenticationModeConfigMap = EKSAuthenticationMode("config_map") + + // EKSAuthenticationModeAPI indicates that only AWS Access Entries will be used for authentication + EKSAuthenticationModeAPI = EKSAuthenticationMode("api") + + // EKSAuthenticationModeAPIAndConfigMap indicates that both `aws-auth` ConfigMap and AWS Access Entries will + // be used for authentication + EKSAuthenticationModeAPIAndConfigMap = EKSAuthenticationMode("api_and_config_map") +) + var ( // DefaultEKSControlPlaneRole is the name of the default IAM role to use for the EKS control plane // if no other role is supplied in the spec and if iam role creation is not enabled. The default diff --git a/controlplane/eks/api/v1beta2/zz_generated.deepcopy.go b/controlplane/eks/api/v1beta2/zz_generated.deepcopy.go index 807613dc0d..678a641e9c 100644 --- a/controlplane/eks/api/v1beta2/zz_generated.deepcopy.go +++ b/controlplane/eks/api/v1beta2/zz_generated.deepcopy.go @@ -170,6 +170,11 @@ func (in *AWSManagedControlPlaneSpec) DeepCopyInto(out *AWSManagedControlPlaneSp *out = new(OIDCIdentityProviderConfig) (*in).DeepCopyInto(*out) } + if in.AccessConfig != nil { + in, out := &in.AccessConfig, &out.AccessConfig + *out = new(AccessConfig) + (*in).DeepCopyInto(*out) + } in.VpcCni.DeepCopyInto(&out.VpcCni) out.KubeProxy = in.KubeProxy } @@ -333,6 +338,26 @@ func (in *AWSManagedControlPlaneTemplateSpec) DeepCopy() *AWSManagedControlPlane return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AccessConfig) DeepCopyInto(out *AccessConfig) { + *out = *in + if in.BootstrapClusterCreatorAdminPermissions != nil { + in, out := &in.BootstrapClusterCreatorAdminPermissions, &out.BootstrapClusterCreatorAdminPermissions + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AccessConfig. +func (in *AccessConfig) DeepCopy() *AccessConfig { + if in == nil { + return nil + } + out := new(AccessConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Addon) DeepCopyInto(out *Addon) { *out = *in diff --git a/pkg/cloud/services/eks/cluster.go b/pkg/cloud/services/eks/cluster.go index a16e6d1d34..b1b480e0b2 100644 --- a/pkg/cloud/services/eks/cluster.go +++ b/pkg/cloud/services/eks/cluster.go @@ -121,6 +121,10 @@ func (s *Service) reconcileCluster(ctx context.Context) error { return errors.Wrap(err, "failed reconciling cluster config") } + if err := s.reconcileAccessConfig(ctx, cluster.AccessConfig); err != nil { + return errors.Wrap(err, "failed reconciling access config") + } + if err := s.reconcileLogging(ctx, cluster.Logging); err != nil { return errors.Wrap(err, "failed reconciling logging") } @@ -422,6 +426,20 @@ func (s *Service) createCluster(ctx context.Context, eksClusterName string) (*ek return nil, errors.Wrap(err, "couldn't create vpc config for cluster") } + var accessConfig *ekstypes.CreateAccessConfigRequest + if s.scope.ControlPlane.Spec.AccessConfig != nil && s.scope.ControlPlane.Spec.AccessConfig.AuthenticationMode != "" { + accessConfig = &ekstypes.CreateAccessConfigRequest{ + AuthenticationMode: s.scope.ControlPlane.Spec.AccessConfig.AuthenticationMode.APIValue(), + } + } + + if s.scope.ControlPlane.Spec.AccessConfig != nil && s.scope.ControlPlane.Spec.AccessConfig.BootstrapClusterCreatorAdminPermissions != nil { + if accessConfig == nil { + accessConfig = &ekstypes.CreateAccessConfigRequest{} + } + accessConfig.BootstrapClusterCreatorAdminPermissions = s.scope.ControlPlane.Spec.AccessConfig.BootstrapClusterCreatorAdminPermissions + } + var netConfig *ekstypes.KubernetesNetworkConfigRequest if s.scope.VPC().IsIPv6Enabled() { netConfig = &ekstypes.KubernetesNetworkConfigRequest{ @@ -465,6 +483,7 @@ func (s *Service) createCluster(ctx context.Context, eksClusterName string) (*ek Name: aws.String(eksClusterName), Version: eksVersion, Logging: logging, + AccessConfig: accessConfig, EncryptionConfig: encryptionConfigs, ResourcesVpcConfig: vpcConfig, RoleArn: role.Arn, @@ -542,6 +561,50 @@ func (s *Service) reconcileClusterConfig(ctx context.Context, cluster *ekstypes. return nil } +func (s *Service) reconcileAccessConfig(ctx context.Context, accessConfig *ekstypes.AccessConfigResponse) error { + input := &eks.UpdateClusterConfigInput{Name: aws.String(s.scope.KubernetesClusterName())} + + if s.scope.ControlPlane.Spec.AccessConfig == nil || s.scope.ControlPlane.Spec.AccessConfig.AuthenticationMode == "" { + return nil + } + + expectedAuthenticationMode := s.scope.ControlPlane.Spec.AccessConfig.AuthenticationMode.APIValue() + s.scope.Debug("Reconciling EKS Access Config for cluster", "cluster-name", s.scope.KubernetesClusterName(), "expected", expectedAuthenticationMode, "current", accessConfig.AuthenticationMode) + if expectedAuthenticationMode != accessConfig.AuthenticationMode { + input.AccessConfig = &ekstypes.UpdateAccessConfigRequest{ + AuthenticationMode: expectedAuthenticationMode, + } + } + + if input.AccessConfig != nil { + if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { + if _, err := s.EKSClient.UpdateClusterConfig(ctx, input); err != nil { + return false, err + } + + // Wait until status transitions to UPDATING because there's a short + // window after UpdateClusterConfig returns where the cluster + // status is ACTIVE and the update would be tried again + if err := s.EKSClient.WaitUntilClusterUpdating( + ctx, + &eks.DescribeClusterInput{Name: aws.String(s.scope.KubernetesClusterName())}, + s.scope.MaxWaitActiveUpdateDelete, + ); err != nil { + return false, err + } + + conditions.MarkTrue(s.scope.ControlPlane, ekscontrolplanev1.EKSControlPlaneUpdatingCondition) + record.Eventf(s.scope.ControlPlane, "InitiatedUpdateEKSControlPlane", "Initiated auth config update for EKS control plane %s", s.scope.KubernetesClusterName()) + return true, nil + }); err != nil { + record.Warnf(s.scope.ControlPlane, "FailedUpdateEKSControlPlane", "Failed to update EKS control plane auth config: %v", err) + return errors.Wrapf(err, "failed to update EKS cluster") + } + } + + return nil +} + func (s *Service) reconcileLogging(ctx context.Context, logging *ekstypes.Logging) error { input := &eks.UpdateClusterConfigInput{Name: aws.String(s.scope.KubernetesClusterName())} diff --git a/pkg/cloud/services/eks/cluster_test.go b/pkg/cloud/services/eks/cluster_test.go index 7e397f329e..b120226697 100644 --- a/pkg/cloud/services/eks/cluster_test.go +++ b/pkg/cloud/services/eks/cluster_test.go @@ -474,6 +474,123 @@ func TestReconcileClusterVersion(t *testing.T) { } } +func TestReconcileAccessConfig(t *testing.T) { + clusterName := "default.cluster" + tests := []struct { + name string + expect func(m *mock_eksiface.MockEKSAPIMockRecorder) + expectError bool + }{ + { + name: "no upgrade necessary", + expect: func(m *mock_eksiface.MockEKSAPIMockRecorder) { + m. + DescribeCluster(gomock.Eq(context.TODO()), gomock.AssignableToTypeOf(&eks.DescribeClusterInput{})). + Return(&eks.DescribeClusterOutput{ + Cluster: &ekstypes.Cluster{ + Name: aws.String("default.cluster"), + AccessConfig: &ekstypes.AccessConfigResponse{ + AuthenticationMode: ekstypes.AuthenticationModeApiAndConfigMap, + }, + }, + }, nil) + }, + expectError: false, + }, + { + name: "needs upgrade", + expect: func(m *mock_eksiface.MockEKSAPIMockRecorder) { + m. + DescribeCluster(gomock.Eq(context.TODO()), gomock.AssignableToTypeOf(&eks.DescribeClusterInput{})). + Return(&eks.DescribeClusterOutput{ + Cluster: &ekstypes.Cluster{ + Name: aws.String("default.cluster"), + AccessConfig: &ekstypes.AccessConfigResponse{ + AuthenticationMode: ekstypes.AuthenticationModeConfigMap, + }, + }, + }, nil) + m.WaitUntilClusterUpdating( + gomock.Eq(context.TODO()), + gomock.AssignableToTypeOf(&eks.DescribeClusterInput{}), + gomock.Any(), + ).Return(nil) + m. + UpdateClusterConfig(gomock.Eq(context.TODO()), gomock.AssignableToTypeOf(&eks.UpdateClusterConfigInput{})). + Return(&eks.UpdateClusterConfigOutput{}, nil) + }, + expectError: false, + }, + { + name: "api error", + expect: func(m *mock_eksiface.MockEKSAPIMockRecorder) { + m. + DescribeCluster(gomock.Eq(context.TODO()), gomock.AssignableToTypeOf(&eks.DescribeClusterInput{})). + Return(&eks.DescribeClusterOutput{ + Cluster: &ekstypes.Cluster{ + Name: aws.String("default.cluster"), + AccessConfig: &ekstypes.AccessConfigResponse{ + AuthenticationMode: ekstypes.AuthenticationModeApi, + }, + }, + }, nil) + m. + UpdateClusterConfig(gomock.Eq(context.TODO()), gomock.AssignableToTypeOf(&eks.UpdateClusterConfigInput{})). + Return(&eks.UpdateClusterConfigOutput{}, errors.New("Unsupported authentication mode update")) + }, + expectError: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + mockControl := gomock.NewController(t) + defer mockControl.Finish() + + eksMock := mock_eksiface.NewMockEKSAPI(mockControl) + + scheme := runtime.NewScheme() + _ = infrav1.AddToScheme(scheme) + _ = ekscontrolplanev1.AddToScheme(scheme) + client := fake.NewClientBuilder().WithScheme(scheme).Build() + scope, err := scope.NewManagedControlPlaneScope(scope.ManagedControlPlaneScopeParams{ + Client: client, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + Name: clusterName, + }, + }, + ControlPlane: &ekscontrolplanev1.AWSManagedControlPlane{ + Spec: ekscontrolplanev1.AWSManagedControlPlaneSpec{ + EKSClusterName: clusterName, + AccessConfig: &ekscontrolplanev1.AccessConfig{ + AuthenticationMode: ekscontrolplanev1.EKSAuthenticationModeAPIAndConfigMap, + }, + }, + }, + }) + g.Expect(err).To(BeNil()) + + tc.expect(eksMock.EXPECT()) + s := NewService(scope) + s.EKSClient = eksMock + + cluster, err := s.describeEKSCluster(context.TODO(), clusterName) + g.Expect(err).To(BeNil()) + + err = s.reconcileAccessConfig(context.TODO(), cluster.AccessConfig) + if tc.expectError { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).To(BeNil()) + }) + } +} + func TestCreateCluster(t *testing.T) { clusterName := "cluster.default" version := aws.String("1.24") @@ -787,3 +904,74 @@ func TestCreateIPv6Cluster(t *testing.T) { _, err = s.createCluster(context.TODO(), "cluster-name") g.Expect(err).To(BeNil()) } + +func TestCreateClusterWithBootstrapClusterCreatorAdminPermissions(t *testing.T) { + g := NewWithT(t) + + mockControl := gomock.NewController(t) + defer mockControl.Finish() + + eksMock := mock_eksiface.NewMockEKSAPI(mockControl) + iamMock := mock_iamauth.NewMockIAMAPI(mockControl) + + scheme := runtime.NewScheme() + _ = infrav1.AddToScheme(scheme) + _ = ekscontrolplanev1.AddToScheme(scheme) + client := fake.NewClientBuilder().WithScheme(scheme).Build() + + clusterName := "test-cluster" + scope, err := scope.NewManagedControlPlaneScope(scope.ManagedControlPlaneScopeParams{ + Client: client, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + Name: "capi-name", + }, + }, + ControlPlane: &ekscontrolplanev1.AWSManagedControlPlane{ + Spec: ekscontrolplanev1.AWSManagedControlPlaneSpec{ + EKSClusterName: clusterName, + Version: aws.String("1.24"), + RoleName: aws.String("arn:role"), + NetworkSpec: infrav1.NetworkSpec{ + Subnets: []infrav1.SubnetSpec{ + {ID: "1", AvailabilityZone: "us-west-2a"}, + {ID: "2", AvailabilityZone: "us-west-2b"}, + }, + }, + AccessConfig: &ekscontrolplanev1.AccessConfig{ + BootstrapClusterCreatorAdminPermissions: ptr.To(false), + }, + }, + }, + }) + g.Expect(err).To(BeNil()) + + eksMock.EXPECT().CreateCluster(context.TODO(), &eks.CreateClusterInput{ + Name: aws.String(clusterName), + Version: aws.String("1.24"), + ResourcesVpcConfig: &ekstypes.VpcConfigRequest{ + SubnetIds: []string{"1", "2"}, + }, + RoleArn: aws.String("arn:role"), + Tags: map[string]string{ + "kubernetes.io/cluster/test-cluster": "owned", + }, + AccessConfig: &ekstypes.CreateAccessConfigRequest{ + BootstrapClusterCreatorAdminPermissions: ptr.To(false), + }, + EncryptionConfig: []ekstypes.EncryptionConfig{}, + BootstrapSelfManagedAddons: aws.Bool(false), + }).Return(&eks.CreateClusterOutput{}, nil) + + iamMock.EXPECT().GetRole(gomock.Any(), gomock.Any()).Return(&iam.GetRoleOutput{ + Role: &iamtypes.Role{Arn: aws.String("arn:role")}, + }, nil) + + s := NewService(scope) + s.EKSClient = eksMock + s.IAMClient = iamMock + + _, err = s.createCluster(context.TODO(), clusterName) + g.Expect(err).To(BeNil()) +} diff --git a/test/e2e/data/e2e_eks_conf.yaml b/test/e2e/data/e2e_eks_conf.yaml index fe9fca78e3..b8230eace7 100644 --- a/test/e2e/data/e2e_eks_conf.yaml +++ b/test/e2e/data/e2e_eks_conf.yaml @@ -116,6 +116,10 @@ providers: targetName: "cluster-template-eks-control-plane-only-legacy.yaml" - sourcePath: "./eks/cluster-template-eks-control-plane-bare-eks.yaml" targetName: "cluster-template-eks-control-plane-bare-eks.yaml" + - sourcePath: "./eks/cluster-template-eks-auth-api-and-config-map.yaml" + targetName: "cluster-template-eks-auth-api-and-config-map.yaml" + - sourcePath: "./eks/cluster-template-eks-auth-bootstrap-disabled.yaml" + targetName: "cluster-template-eks-auth-bootstrap-disabled.yaml" - sourcePath: "./infrastructure-aws/withclusterclass/kustomize_sources/eks-clusterclass/clusterclass-eks-e2e.yaml" - sourcePath: "./infrastructure-aws/withclusterclass/generated/cluster-template-eks-clusterclass.yaml" diff --git a/test/e2e/data/eks/cluster-template-eks-auth-api-and-config-map.yaml b/test/e2e/data/eks/cluster-template-eks-auth-api-and-config-map.yaml new file mode 100644 index 0000000000..d9e3c541e6 --- /dev/null +++ b/test/e2e/data/eks/cluster-template-eks-auth-api-and-config-map.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + kind: AWSManagedCluster + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}" + controlPlaneRef: + kind: AWSManagedControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}-control-plane" +--- +kind: AWSManagedCluster +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_NAME}" +spec: {} +--- +kind: AWSManagedControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + region: "${AWS_REGION}" + sshKeyName: "${AWS_SSH_KEY_NAME}" + version: "${KUBERNETES_VERSION}" + accessConfig: + authenticationMode: api_and_config_map + bootstrapClusterCreatorAdminPermissions: true + identityRef: + kind: AWSClusterStaticIdentity + name: e2e-account + diff --git a/test/e2e/data/eks/cluster-template-eks-auth-bootstrap-disabled.yaml b/test/e2e/data/eks/cluster-template-eks-auth-bootstrap-disabled.yaml new file mode 100644 index 0000000000..3655f9c1dc --- /dev/null +++ b/test/e2e/data/eks/cluster-template-eks-auth-bootstrap-disabled.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + kind: AWSManagedCluster + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}" + controlPlaneRef: + kind: AWSManagedControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}-control-plane" +--- +kind: AWSManagedCluster +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_NAME}" +spec: {} +--- +kind: AWSManagedControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + region: "${AWS_REGION}" + sshKeyName: "${AWS_SSH_KEY_NAME}" + version: "${KUBERNETES_VERSION}" + accessConfig: + authenticationMode: api_and_config_map + bootstrapClusterCreatorAdminPermissions: false + identityRef: + kind: AWSClusterStaticIdentity + name: e2e-account + diff --git a/test/e2e/suites/managed/eks_auth_test.go b/test/e2e/suites/managed/eks_auth_test.go new file mode 100644 index 0000000000..af59c31a2c --- /dev/null +++ b/test/e2e/suites/managed/eks_auth_test.go @@ -0,0 +1,180 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package managed + +import ( + "context" + "fmt" + + ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2" + "sigs.k8s.io/cluster-api-provider-aws/v2/test/e2e/shared" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/util" +) + +// EKS authentication mode e2e tests. +var _ = ginkgo.Describe("[managed] [auth] EKS authentication mode tests", func() { + var ( + namespace *corev1.Namespace + ctx context.Context + specName = "auth" + clusterName string + ) + + shared.ConditionalIt(runGeneralTests, "should create a cluster with api_and_config_map authentication mode", func() { + ginkgo.By("should have a valid test configuration") + Expect(e2eCtx.Environment.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. BootstrapClusterProxy can't be nil") + Expect(e2eCtx.E2EConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName) + Expect(e2eCtx.E2EConfig.Variables).To(HaveKey(shared.KubernetesVersion)) + + ctx = context.TODO() + namespace = shared.SetupSpecNamespace(ctx, specName, e2eCtx) + clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + eksClusterName := getEKSClusterName(namespace.Name, clusterName) + + ginkgo.By("should create an EKS control plane with api_and_config_map authentication mode") + ManagedClusterSpec(ctx, func() ManagedClusterSpecInput { + return ManagedClusterSpecInput{ + E2EConfig: e2eCtx.E2EConfig, + ConfigClusterFn: defaultConfigCluster, + BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, + AWSSession: e2eCtx.BootstrapUserAWSSession, + Namespace: namespace, + ClusterName: clusterName, + Flavour: EKSAuthAPIAndConfigMapFlavor, + ControlPlaneMachineCount: 1, + WorkerMachineCount: 0, + } + }) + + ginkgo.By("EKS cluster should be active") + verifyClusterActiveAndOwned(ctx, eksClusterName, e2eCtx.BootstrapUserAWSSession) + + ginkgo.By("verifying cluster has the correct authentication mode") + verifyClusterAuthenticationMode(ctx, eksClusterName, ekstypes.AuthenticationModeApiAndConfigMap, e2eCtx.BootstrapUserAWSSession) + + ginkgo.By("attempting to downgrade from api_and_config_map to config_map should fail") + controlPlaneName := fmt.Sprintf("%s-control-plane", clusterName) + controlPlane := &ekscontrolplanev1.AWSManagedControlPlane{} + err := e2eCtx.Environment.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKey{ + Namespace: namespace.Name, + Name: controlPlaneName, + }, controlPlane) + Expect(err).ToNot(HaveOccurred(), "failed to get control plane") + + controlPlane.Spec.AccessConfig.AuthenticationMode = ekscontrolplanev1.EKSAuthenticationModeConfigMap + err = e2eCtx.Environment.BootstrapClusterProxy.GetClient().Update(ctx, controlPlane) + Expect(err).To(HaveOccurred(), "expected downgrade from api_and_config_map to config_map to fail") + + ginkgo.By("upgrading from api_and_config_map to api should succeed") + err = e2eCtx.Environment.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKey{ + Namespace: namespace.Name, + Name: controlPlaneName, + }, controlPlane) + Expect(err).ToNot(HaveOccurred(), "failed to get control plane for upgrade") + + controlPlane.Spec.AccessConfig.AuthenticationMode = ekscontrolplanev1.EKSAuthenticationModeAPI + err = e2eCtx.Environment.BootstrapClusterProxy.GetClient().Update(ctx, controlPlane) + Expect(err).ToNot(HaveOccurred(), "expected upgrade from api_and_config_map to api to succeed") + + ginkgo.By("attempting to downgrade from api to api_and_config_map should fail") + err = e2eCtx.Environment.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKey{ + Namespace: namespace.Name, + Name: controlPlaneName, + }, controlPlane) + Expect(err).ToNot(HaveOccurred(), "failed to get control plane for downgrade attempt") + + controlPlane.Spec.AccessConfig.AuthenticationMode = ekscontrolplanev1.EKSAuthenticationModeAPIAndConfigMap + err = e2eCtx.Environment.BootstrapClusterProxy.GetClient().Update(ctx, controlPlane) + Expect(err).To(HaveOccurred(), "expected downgrade from api to api_and_config_map to fail") + + cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{ + Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + Name: clusterName, + }) + Expect(cluster).NotTo(BeNil(), "couldn't find CAPI cluster") + + framework.DeleteCluster(ctx, framework.DeleteClusterInput{ + Deleter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + Cluster: cluster, + }) + framework.WaitForClusterDeleted(ctx, framework.WaitForClusterDeletedInput{ + ClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, + Cluster: cluster, + ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath, + ArtifactFolder: e2eCtx.Settings.ArtifactFolder, + }, e2eCtx.E2EConfig.GetIntervals("", "wait-delete-cluster")...) + }) + + shared.ConditionalIt(runGeneralTests, "should create a cluster with bootstrapClusterCreatorAdminPermissions disabled", func() { + ginkgo.By("should have a valid test configuration") + Expect(e2eCtx.Environment.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. BootstrapClusterProxy can't be nil") + Expect(e2eCtx.E2EConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling bootstrap spec") + Expect(e2eCtx.E2EConfig.Variables).To(HaveKey(shared.KubernetesVersion)) + + ctx = context.TODO() + namespace = shared.SetupSpecNamespace(ctx, "bootstrap", e2eCtx) + clusterName = fmt.Sprintf("bootstrap-%s", util.RandomString(6)) + eksClusterName := getEKSClusterName(namespace.Name, clusterName) + + ginkgo.By("should create an EKS control plane with bootstrapClusterCreatorAdminPermissions disabled") + ManagedClusterSpec(ctx, func() ManagedClusterSpecInput { + return ManagedClusterSpecInput{ + E2EConfig: e2eCtx.E2EConfig, + ConfigClusterFn: defaultConfigCluster, + BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, + AWSSession: e2eCtx.BootstrapUserAWSSession, + Namespace: namespace, + ClusterName: clusterName, + Flavour: EKSAuthBootstrapDisabledFlavor, + ControlPlaneMachineCount: 1, + WorkerMachineCount: 0, + } + }) + + ginkgo.By("EKS cluster should be active") + verifyClusterActiveAndOwned(ctx, eksClusterName, e2eCtx.BootstrapUserAWSSession) + + cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{ + Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + Name: clusterName, + }) + Expect(cluster).NotTo(BeNil(), "couldn't find CAPI cluster") + + framework.DeleteCluster(ctx, framework.DeleteClusterInput{ + Deleter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + Cluster: cluster, + }) + framework.WaitForClusterDeleted(ctx, framework.WaitForClusterDeletedInput{ + ClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, + Cluster: cluster, + ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath, + ArtifactFolder: e2eCtx.Settings.ArtifactFolder, + }, e2eCtx.E2EConfig.GetIntervals("", "wait-delete-cluster")...) + }) +}) diff --git a/test/e2e/suites/managed/helpers.go b/test/e2e/suites/managed/helpers.go index 2922c70201..926d914248 100644 --- a/test/e2e/suites/managed/helpers.go +++ b/test/e2e/suites/managed/helpers.go @@ -50,6 +50,8 @@ const ( EKSIPv6ClusterFlavor = "eks-ipv6-cluster" EKSControlPlaneOnlyLegacyFlavor = "eks-control-plane-only-legacy" EKSClusterClassFlavor = "eks-clusterclass" + EKSAuthAPIAndConfigMapFlavor = "eks-auth-api-and-config-map" + EKSAuthBootstrapDisabledFlavor = "eks-auth-bootstrap-disabled" ) const ( @@ -106,6 +108,21 @@ func getEKSCluster(ctx context.Context, eksClusterName string, sess *aws.Config) return result.Cluster, err } +func verifyClusterAuthenticationMode(ctx context.Context, eksClusterName string, expectedAuthMode ekstypes.AuthenticationMode, sess *aws.Config) { + var ( + cluster *ekstypes.Cluster + err error + ) + Eventually(func() error { + cluster, err = getEKSCluster(ctx, eksClusterName, sess) + return err + }, clientRequestTimeout, clientRequestCheckInterval).Should(Succeed(), fmt.Sprintf("eventually failed trying to get EKS Cluster %q", eksClusterName)) + + Expect(cluster.AccessConfig).ToNot(BeNil(), "expecting AccessConfig to be set on the cluster") + Expect(cluster.AccessConfig.AuthenticationMode).To(BeEquivalentTo(expectedAuthMode), + fmt.Sprintf("expecting authentication mode to be %s, got %s", expectedAuthMode, cluster.AccessConfig.AuthenticationMode)) +} + func getEKSClusterAddon(ctx context.Context, eksClusterName, addonName string, sess *aws.Config) (*ekstypes.Addon, error) { eksClient := eks.NewFromConfig(*sess)