enhance: add block ports config for AlibabaCloud LB network models (#175

) Signed-off-by: ChrisLiu <[email protected]>
openkruise · Nov 1, 2024 · 468b2c7 · 468b2c7
1 parent c114781
commit 468b2c7
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 14 deletions.
diff --git a/cloudprovider/alibabacloud/nlb.go b/cloudprovider/alibabacloud/nlb.go
@@ -70,6 +70,7 @@ const (
 type NlbPlugin struct {
 	maxPort     int32
 	minPort     int32
+	blockPorts  []int32
 	cache       map[string]portAllocated
 	podAllocate map[string]string
 	mutex       sync.RWMutex
@@ -106,14 +107,15 @@ func (n *NlbPlugin) Init(c client.Client, options cloudprovider.CloudProviderOpt
 	slbOptions := options.(provideroptions.AlibabaCloudOptions).NLBOptions
 	n.minPort = slbOptions.MinPort
 	n.maxPort = slbOptions.MaxPort
+	n.blockPorts = slbOptions.BlockPorts
 
 	svcList := &corev1.ServiceList{}
 	err := c.List(ctx, svcList)
 	if err != nil {
 		return err
 	}
 
-	n.cache, n.podAllocate = initLbCache(svcList.Items, n.minPort, n.maxPort)
+	n.cache, n.podAllocate = initLbCache(svcList.Items, n.minPort, n.maxPort, n.blockPorts)
 	log.Infof("[%s] podAllocate cache complete initialization: %v", NlbNetwork, n.podAllocate)
 	return nil
 }
@@ -385,10 +387,15 @@ func (n *NlbPlugin) allocate(lbIds []string, num int, nsName string) (string, []
 	for i := 0; i < num; i++ {
 		var port int32
 		if n.cache[lbId] == nil {
+			// init cache for new lb
 			n.cache[lbId] = make(portAllocated, n.maxPort-n.minPort)
 			for i := n.minPort; i < n.maxPort; i++ {
 				n.cache[lbId][i] = false
 			}
+			// block ports
+			for _, blockPort := range n.blockPorts {
+				n.cache[lbId][blockPort] = true
+			}
 		}
 
 		for p, allocated := range n.cache[lbId] {
@@ -421,6 +428,10 @@ func (n *NlbPlugin) deAllocate(nsName string) {
 	for _, port := range ports {
 		n.cache[lbId][port] = false
 	}
+	// block ports
+	for _, blockPort := range n.blockPorts {
+		n.cache[lbId][blockPort] = true
+	}
 
 	delete(n.podAllocate, nsName)
 	log.Infof("pod %s deallocate nlb %s ports %v", nsName, lbId, ports)

diff --git a/cloudprovider/alibabacloud/slb.go b/cloudprovider/alibabacloud/slb.go
@@ -67,6 +67,7 @@ type portAllocated map[int32]bool
 type SlbPlugin struct {
 	maxPort     int32
 	minPort     int32
+	blockPorts  []int32
 	cache       map[string]portAllocated
 	podAllocate map[string]string
 	mutex       sync.RWMutex
@@ -105,30 +106,39 @@ func (s *SlbPlugin) Init(c client.Client, options cloudprovider.CloudProviderOpt
 	slbOptions := options.(provideroptions.AlibabaCloudOptions).SLBOptions
 	s.minPort = slbOptions.MinPort
 	s.maxPort = slbOptions.MaxPort
+	s.blockPorts = slbOptions.BlockPorts
 
 	svcList := &corev1.ServiceList{}
 	err := c.List(ctx, svcList)
 	if err != nil {
 		return err
 	}
 
-	s.cache, s.podAllocate = initLbCache(svcList.Items, s.minPort, s.maxPort)
+	s.cache, s.podAllocate = initLbCache(svcList.Items, s.minPort, s.maxPort, s.blockPorts)
 	log.Infof("[%s] podAllocate cache complete initialization: %v", SlbNetwork, s.podAllocate)
 	return nil
 }
 
-func initLbCache(svcList []corev1.Service, minPort, maxPort int32) (map[string]portAllocated, map[string]string) {
+func initLbCache(svcList []corev1.Service, minPort, maxPort int32, blockPorts []int32) (map[string]portAllocated, map[string]string) {
 	newCache := make(map[string]portAllocated)
 	newPodAllocate := make(map[string]string)
 	for _, svc := range svcList {
 		lbId := svc.Labels[SlbIdLabelKey]
 		if lbId != "" && svc.Spec.Type == corev1.ServiceTypeLoadBalancer {
+			// init cache for that lb
 			if newCache[lbId] == nil {
 				newCache[lbId] = make(portAllocated, maxPort-minPort)
 				for i := minPort; i < maxPort; i++ {
 					newCache[lbId][i] = false
 				}
 			}
+
+			// block ports
+			for _, blockPort := range blockPorts {
+				newCache[lbId][blockPort] = true
+			}
+
+			// fill in cache for that lb
 			var ports []int32
 			for _, port := range getPorts(svc.Spec.Ports) {
 				if port <= maxPort && port >= minPort {
@@ -335,10 +345,15 @@ func (s *SlbPlugin) allocate(lbIds []string, num int, nsName string) (string, []
 	for i := 0; i < num; i++ {
 		var port int32
 		if s.cache[lbId] == nil {
+			// init cache for new lb
 			s.cache[lbId] = make(portAllocated, s.maxPort-s.minPort)
 			for i := s.minPort; i < s.maxPort; i++ {
 				s.cache[lbId][i] = false
 			}
+			// block ports
+			for _, blockPort := range s.blockPorts {
+				s.cache[lbId][blockPort] = true
+			}
 		}
 
 		for p, allocated := range s.cache[lbId] {
@@ -371,6 +386,10 @@ func (s *SlbPlugin) deAllocate(nsName string) {
 	for _, port := range ports {
 		s.cache[lbId][port] = false
 	}
+	// block ports
+	for _, blockPort := range s.blockPorts {
+		s.cache[lbId][blockPort] = true
+	}
 
 	delete(s.podAllocate, nsName)
 	log.Infof("pod %s deallocate slb %s ports %v", nsName, lbId, ports)

diff --git a/cloudprovider/alibabacloud/slb_test.go b/cloudprovider/alibabacloud/slb_test.go
@@ -201,17 +201,21 @@ func TestInitLbCache(t *testing.T) {
 		svcList     []corev1.Service
 		minPort     int32
 		maxPort     int32
+		blockPorts  []int32
 		cache       map[string]portAllocated
 		podAllocate map[string]string
 	}{
-		minPort: 512,
-		maxPort: 712,
+		minPort:    512,
+		maxPort:    712,
+		blockPorts: []int32{593},
 		cache: map[string]portAllocated{
 			"xxx-A": map[int32]bool{
 				666: true,
+				593: true,
 			},
 			"xxx-B": map[int32]bool{
 				555: true,
+				593: true,
 			},
 		},
 		podAllocate: map[string]string{
@@ -266,7 +270,7 @@ func TestInitLbCache(t *testing.T) {
 		},
 	}
 
-	actualCache, actualPodAllocate := initLbCache(test.svcList, test.minPort, test.maxPort)
+	actualCache, actualPodAllocate := initLbCache(test.svcList, test.minPort, test.maxPort, test.blockPorts)
 	for lb, pa := range test.cache {
 		for port, isAllocated := range pa {
 			if actualCache[lb][port] != isAllocated {

diff --git a/cloudprovider/options/alibabacloud_options.go b/cloudprovider/options/alibabacloud_options.go
@@ -7,27 +7,39 @@ type AlibabaCloudOptions struct {
 }
 
 type SLBOptions struct {
-	MaxPort int32 `toml:"max_port"`
-	MinPort int32 `toml:"min_port"`
+	MaxPort    int32   `toml:"max_port"`
+	MinPort    int32   `toml:"min_port"`
+	BlockPorts []int32 `toml:"block_ports"`
 }
 
 type NLBOptions struct {
-	MaxPort int32 `toml:"max_port"`
-	MinPort int32 `toml:"min_port"`
+	MaxPort    int32   `toml:"max_port"`
+	MinPort    int32   `toml:"min_port"`
+	BlockPorts []int32 `toml:"block_ports"`
 }
 
 func (o AlibabaCloudOptions) Valid() bool {
 	// SLB valid
 	slbOptions := o.SLBOptions
-	if slbOptions.MaxPort-slbOptions.MinPort != 200 {
+	for _, blockPort := range slbOptions.BlockPorts {
+		if blockPort >= slbOptions.MaxPort || blockPort < slbOptions.MinPort {
+			return false
+		}
+	}
+	if int(slbOptions.MaxPort-slbOptions.MinPort)-len(slbOptions.BlockPorts) != 200 {
 		return false
 	}
 	if slbOptions.MinPort <= 0 {
 		return false
 	}
 	// NLB valid
 	nlbOptions := o.NLBOptions
-	if nlbOptions.MaxPort-nlbOptions.MinPort != 500 {
+	for _, blockPort := range nlbOptions.BlockPorts {
+		if blockPort >= nlbOptions.MaxPort || blockPort < nlbOptions.MinPort {
+			return false
+		}
+	}
+	if int(nlbOptions.MaxPort-nlbOptions.MinPort)-len(nlbOptions.BlockPorts) != 500 {
 		return false
 	}
 	if nlbOptions.MinPort <= 0 {

diff --git a/config/manager/config.toml b/config/manager/config.toml
@@ -7,11 +7,13 @@ min_port = 8000
 [alibabacloud]
 enable = true
 [alibabacloud.slb]
-max_port = 700
+max_port = 701
 min_port = 500
+block_ports = [593]
 [alibabacloud.nlb]
-max_port = 1500
+max_port = 1503
 min_port = 1000
+block_ports = [1025, 1434, 1068]
 
 [volcengine]
 enable = true