feat: add --default-subnets option

Author: 1mwataru

controllers/ingress/group_controller.go

@@ -3,6 +3,7 @@ package ingress
 import (
 	"context"
 	"fmt"
+
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/go-logr/logr"
@@ -59,7 +60,7 @@ func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventReco
 		cloud.EC2(), cloud.ELBV2(), cloud.ACM(),
 		annotationParser, subnetsResolver,
 		authConfigBuilder, enhancedBackendBuilder, trackingProvider, elbv2TaggingManager, controllerConfig.FeatureGates,
-		cloud.VpcID(), controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
+		cloud.VpcID(), controllerConfig.ClusterName, controllerConfig.DefaultSubnets, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
 		controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver,
 		controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), logger)
 	stackMarshaller := deploy.NewDefaultStackMarshaller()

controllers/service/service_controller.go

@@ -3,6 +3,7 @@ package service
 import (
 	"context"
 	"fmt"
+
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/go-logr/logr"
@@ -45,7 +46,7 @@ func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRe
 	trackingProvider := tracking.NewDefaultProvider(serviceTagPrefix, controllerConfig.ClusterName)
 	serviceUtils := service.NewServiceUtils(annotationParser, serviceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates)
 	modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
-		elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
+		elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultSubnets, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
 		controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils,
 		backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger)
 	stackMarshaller := deploy.NewDefaultStackMarshaller()

docs/deploy/configurations.md

@@ -77,6 +77,7 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne
 | backend-security-group                                                          | string                          |                                            | Backend security group id to use for the ingress rules on the worker node SG                                                                   |
 | cluster-name                                                                    | string                          |                                            | Kubernetes cluster name                                                                                                                        |
 | default-ssl-policy                                                              | string                          | ELBSecurityPolicy-2016-08                  | Default SSL Policy that will be applied to all Ingresses or Services that do not have the SSL Policy annotation                                |
+| default-subnets                                                                 | stringList                      | []                                         | Default subnets to be selected when not explicitly specified through annotations or other methods                               |
 | default-tags                                                                    | stringMap                       |                                            | AWS Tags that will be applied to all AWS resources managed by this controller. Specified Tags takes highest priority                           |
 | default-target-type                                                             | string                          | instance                                   | Default target type for Ingresses and Services - ip, instance                                                                                  |
 | default-load-balancer-scheme                                                    | string                          | internal                                   | Default scheme for ELBs - internal,  internet-facing                                                                                  |

docs/deploy/subnet_discovery.md

@@ -4,6 +4,7 @@ set [Feature Gate ALBSingleSubnet](https://kubernetes-sigs.github.io/aws-load-ba
 The subnets must be tagged appropriately for auto-discovery to work. The controller chooses one subnet from each Availability Zone. During auto-discovery, the controller
 considers subnets with at least eight available IP addresses. In the case of multiple qualified tagged subnets in an Availability Zone, the controller chooses the first one in lexicographical 
 order by the subnet IDs.
+However, if subnets are specified using the --default-subnets flag, those will take precedence over the auto-discovered subnets.
 For more information about the subnets for the LBC, see [Application Load Balancers](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/application-load-balancers.html) 
 and [Network Load Balancers](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html).
 If you used `eksctl` or an Amazon EKS AWS CloudFormation template to create your VPC after March 26, 2020, then the subnets are tagged appropriately when they're created. For 

pkg/config/controller_config.go

@@ -15,6 +15,7 @@ import (
 const (
 	flagLogLevel                                     = "log-level"
 	flagK8sClusterName                               = "cluster-name"
+	flagDefaultSubnets                               = "default-subnets"
 	flagDefaultTags                                  = "default-tags"
 	flagDefaultTargetType                            = "default-target-type"
 	flagDefaultLoadBalancerScheme                    = "default-load-balancer-scheme"
@@ -69,6 +70,9 @@ type ControllerConfig struct {
 	// Configurations for the Service controller
 	ServiceConfig ServiceConfig
 
+	// Default subnets that will be used for all AWS resources managed by the networking controller.
+	DefaultSubnets []string
+
 	// Default AWS Tags that will be applied to all AWS resources managed by this controller.
 	DefaultTags map[string]string
 
@@ -119,6 +123,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&cfg.LogLevel, flagLogLevel, defaultLogLevel,
 		"Set the controller log level - info(default), debug")
 	fs.StringVar(&cfg.ClusterName, flagK8sClusterName, "", "Kubernetes cluster name")
+	fs.StringSliceVar(&cfg.DefaultSubnets, flagDefaultSubnets, nil,
+		"Default subnets that will be used for all AWS resources managed by the networking controller")
 	fs.StringToStringVar(&cfg.DefaultTags, flagDefaultTags, nil,
 		"Default AWS Tags that will be applied to all AWS resources managed by this controller")
 	fs.StringVar(&cfg.DefaultTargetType, flagDefaultTargetType, string(elbv2.TargetTypeInstance),
@@ -162,7 +168,9 @@ func (cfg *ControllerConfig) Validate() error {
 	if len(cfg.ClusterName) == 0 {
 		return errors.New("kubernetes cluster name must be specified")
 	}
-
+	if err := cfg.validateDefaultSubnets(); err != nil {
+		return err
+	}
 	if err := cfg.validateDefaultTagsCollisionWithTrackingTags(); err != nil {
 		return err
 	}
@@ -184,6 +192,27 @@ func (cfg *ControllerConfig) Validate() error {
 	return nil
 }
 
+func (cfg *ControllerConfig) validateDefaultSubnets() error {
+	if len(cfg.DefaultSubnets) == 0 {
+		return nil
+	}
+	for _, subnetID := range cfg.DefaultSubnets {
+		if !strings.HasPrefix(subnetID, "subnet-") {
+			return errors.Errorf("invalid value %v for default subnet id", subnetID)
+		}
+	}
+
+	//validate duplicate subnet ids
+	seen := make(map[string]bool)
+	for _, str := range cfg.DefaultSubnets {
+		if seen[str] {
+			return errors.Errorf("duplicate subnet id %v is specified in the --default-subnets flag", str)
+		}
+		seen[str] = true
+	}
+	return nil
+}
+
 func (cfg *ControllerConfig) validateDefaultTagsCollisionWithTrackingTags() error {
 	for tagKey := range cfg.DefaultTags {
 		if trackingTagKeys.Has(tagKey) {

pkg/config/controller_config_test.go

@@ -1,9 +1,10 @@
 package config
 
 import (
+	"testing"
+
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
-	"testing"
 )
 
 func TestControllerConfig_validateDefaultTagsCollisionWithTrackingTags(t *testing.T) {
@@ -164,3 +165,49 @@ func TestControllerConfig_validateExternalManagedTagsCollisionWithDefaultTags(t
 		})
 	}
 }
+
+func TestControllerConfig_validateDefaultSubnets(t *testing.T) {
+	type fields struct {
+		DefaultSubnets []string
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		wantErr error
+	}{
+		{
+			name: "default subnets is empty",
+			fields: fields{
+				DefaultSubnets: nil,
+			},
+			wantErr: nil,
+		},
+		{
+			name: "default subnets is not empty",
+			fields: fields{
+				DefaultSubnets: []string{"subnet-1", "subnet-2"},
+			},
+			wantErr: nil,
+		},
+		{
+			name: "default subnets is not empty and duplicate subnets are specified",
+			fields: fields{
+				DefaultSubnets: []string{"subnet-1", "subnet-2", "subnet-1"},
+			},
+			wantErr: errors.New("duplicate subnet id subnet-1 is specified in the --default-subnets flag"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := &ControllerConfig{
+				DefaultSubnets: tt.fields.DefaultSubnets,
+			}
+			err := cfg.validateDefaultSubnets()
+			if tt.wantErr != nil {
+				assert.EqualError(t, err, tt.wantErr.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

pkg/ingress/model_build_load_balancer.go

@@ -5,12 +5,13 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"regexp"
+
 	awssdk "github.com/aws/aws-sdk-go-v2/aws"
 	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
 	"github.com/google/go-cmp/cmp"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
-	"regexp"
 	"sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/algorithm"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
@@ -260,6 +261,7 @@ func (t *defaultModelBuildTask) buildLoadBalancerSubnetMappings(ctx context.Cont
 			networking.WithSubnetsResolveLBScheme(scheme),
 			networking.WithSubnetsResolveAvailableIPAddressCount(minimalAvailableIPAddressCount),
 			networking.WithSubnetsClusterTagCheck(t.featureGates.Enabled(config.SubnetsClusterTagCheck)),
+			networking.WithDefaultSubnets(t.defaultSubnets),
 		)
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't auto-discover subnets")

pkg/ingress/model_builder.go

@@ -2,10 +2,11 @@ package ingress
 
 import (
 	"context"
-	elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
 	"reflect"
 	"strconv"
 
+	elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
+
 	awssdk "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
@@ -42,7 +43,7 @@ func NewDefaultModelBuilder(k8sClient client.Client, eventRecorder record.EventR
 	annotationParser annotations.Parser, subnetsResolver networkingpkg.SubnetsResolver,
 	authConfigBuilder AuthConfigBuilder, enhancedBackendBuilder EnhancedBackendBuilder,
 	trackingProvider tracking.Provider, elbv2TaggingManager elbv2deploy.TaggingManager, featureGates config.FeatureGates,
-	vpcID string, clusterName string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string,
+	vpcID string, clusterName string, defaultSubnets []string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string,
 	backendSGProvider networkingpkg.BackendSGProvider, sgResolver networkingpkg.SecurityGroupResolver,
 	enableBackendSG bool, disableRestrictedSGRules bool, allowedCAARNs []string, enableIPTargetType bool, logger logr.Logger) *defaultModelBuilder {
 	certDiscovery := NewACMCertDiscovery(acmClient, allowedCAARNs, logger)
@@ -65,6 +66,7 @@ func NewDefaultModelBuilder(k8sClient client.Client, eventRecorder record.EventR
 		trackingProvider:          trackingProvider,
 		elbv2TaggingManager:       elbv2TaggingManager,
 		featureGates:              featureGates,
+		defaultSubnets:            defaultSubnets,
 		defaultTags:               defaultTags,
 		externalManagedTags:       sets.NewString(externalManagedTags...),
 		defaultSSLPolicy:          defaultSSLPolicy,
@@ -100,6 +102,7 @@ type defaultModelBuilder struct {
 	trackingProvider          tracking.Provider
 	elbv2TaggingManager       elbv2deploy.TaggingManager
 	featureGates              config.FeatureGates
+	defaultSubnets            []string
 	defaultTags               map[string]string
 	externalManagedTags       sets.String
 	defaultSSLPolicy          string
@@ -141,6 +144,7 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group) (core.S
 		ingGroup: ingGroup,
 		stack:    stack,
 
+		defaultSubnets:                            b.defaultSubnets,
 		defaultTags:                               b.defaultTags,
 		externalManagedTags:                       b.externalManagedTags,
 		defaultIPAddressType:                      elbv2model.IPAddressTypeIPV4,
@@ -198,6 +202,7 @@ type defaultModelBuildTask struct {
 	disableRestrictedSGRules bool
 	enableIPTargetType       bool
 
+	defaultSubnets                            []string
 	defaultTags                               map[string]string
 	externalManagedTags                       sets.String
 	defaultIPAddressType                      elbv2model.IPAddressType

pkg/networking/subnet_resolver.go

@@ -52,6 +52,8 @@ type SubnetsResolveOptions struct {
 	SubnetsClusterTagCheck bool
 	// whether to allow using only 1 subnet for provisioning ALB, default to false
 	ALBSingleSubnet bool
+	// Subnets specified with --default-subnets
+	DefaultSubnets []string
 }
 
 // ApplyOptions applies slice of SubnetsResolveOption.
@@ -106,6 +108,13 @@ func WithALBSingleSubnet(ALBSingleSubnet bool) SubnetsResolveOption {
 	}
 }
 
+// WithDefaultSubnets generates an option that configures DefaultSubnets.
+func WithDefaultSubnets(defaultSubnets []string) SubnetsResolveOption {
+	return func(opts *SubnetsResolveOptions) {
+		opts.DefaultSubnets = defaultSubnets
+	}
+}
+
 // SubnetsResolver is responsible for resolve EC2 Subnets for Load Balancers.
 type SubnetsResolver interface {
 	// ResolveViaDiscovery resolve subnets by auto discover matching subnets.
@@ -234,6 +243,15 @@ func (r *defaultSubnetsResolver) ResolveViaSelector(ctx context.Context, selecto
 		}
 		subnetsByAZ := mapSDKSubnetsByAZ(filteredSubnets)
 		chosenSubnets = make([]ec2types.Subnet, 0, len(subnetsByAZ))
+
+		prioritySubnetMap := make(map[string]int)
+
+		if len(resolveOpts.DefaultSubnets) > 0 {
+			for i, subnetID := range resolveOpts.DefaultSubnets {
+				prioritySubnetMap[subnetID] = i
+			}
+		}
+
 		for az, subnets := range subnetsByAZ {
 			if len(subnets) == 1 {
 				chosenSubnets = append(chosenSubnets, subnets[0])
@@ -247,6 +265,21 @@ func (r *defaultSubnetsResolver) ResolveViaSelector(ctx context.Context, selecto
 						}
 						return false
 					}
+
+					// When subnets are specified in --default-subnets, the subnets list will be sorted according to this order.
+					// Any subnets not specified in --default-subnets will be sorted in lexicographical order and placed after the prioritized subnets.
+					iVal, iExists := prioritySubnetMap[awssdk.ToString(subnets[i].SubnetId)]
+					jVal, jExists := prioritySubnetMap[awssdk.ToString(subnets[j].SubnetId)]
+
+					if iExists && jExists {
+						return iVal < jVal
+					}
+					if iExists {
+						return true
+					}
+					if jExists {
+						return false
+					}
 					return awssdk.ToString(subnets[i].SubnetId) < awssdk.ToString(subnets[j].SubnetId)
 				})
 				r.logger.Info("multiple subnet in the same AvailabilityZone", "AvailabilityZone", az,