Static IP allocation via CNI config

Author: antonipp

pkg/alibabacloud/eni/node.go

@@ -306,6 +306,11 @@ func (n *Node) AllocateIPs(ctx context.Context, a *ipam.AllocationAction) error
 	return err
 }
 
+func (n *Node) AllocateStaticIP(ctx context.Context, staticIPTags ipamTypes.Tags) (string, error) {
+	// TODO
+	return "", nil
+}
+
 // PrepareIPRelease prepares the release of ENI IPs.
 func (n *Node) PrepareIPRelease(excessIPs int, scopedLog *logrus.Entry) *ipam.ReleaseAction {
 	r := &ipam.ReleaseAction{}

pkg/aws/ec2/ec2.go

@@ -14,7 +14,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	ec2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
-	log "github.com/sirupsen/logrus"
 
 	"github.com/cilium/cilium/pkg/api/helpers"
 	"github.com/cilium/cilium/pkg/aws/endpoints"
@@ -24,6 +23,8 @@ import (
 	ipPkg "github.com/cilium/cilium/pkg/ip"
 	"github.com/cilium/cilium/pkg/ipam/option"
 	ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
+	"github.com/cilium/cilium/pkg/logging"
+	"github.com/cilium/cilium/pkg/logging/logfields"
 	"github.com/cilium/cilium/pkg/spanstat"
 )
 
@@ -39,6 +40,8 @@ const (
 	InvalidParameterValueStr = "InvalidParameterValue"
 )
 
+var log = logging.DefaultLogger.WithField(logfields.LogSubsys, "ec2")
+
 // Client represents an EC2 API client
 type Client struct {
 	ec2Client           *ec2.Client
@@ -691,6 +694,51 @@ func (c *Client) UnassignENIPrefixes(ctx context.Context, eniID string, prefixes
 	return err
 }
 
+func (c *Client) AssociateEIP(ctx context.Context, instanceID string, eipTags ipamTypes.Tags) (string, error) {
+	filters := make([]ec2_types.Filter, 0, len(eipTags))
+	for k, v := range eipTags {
+		filters = append(filters, ec2_types.Filter{
+			Name:   aws.String(fmt.Sprintf("tag:%s", k)),
+			Values: []string{v},
+		})
+	}
+
+	describeAddressesInput := &ec2.DescribeAddressesInput{
+		Filters: filters,
+	}
+	c.limiter.Limit(ctx, "DescribeAddresses")
+	sinceStart := spanstat.Start()
+	addresses, err := c.ec2Client.DescribeAddresses(ctx, describeAddressesInput)
+	c.metricsAPI.ObserveAPICall("DescribeAddresses", deriveStatus(err), sinceStart.Seconds())
+	if err != nil {
+		return "", err
+	}
+	log.Infof("Found %d EIPs corresponding to tags %v", len(addresses.Addresses), eipTags)
+
+	for _, address := range addresses.Addresses {
+		// Only pick unassociated EIPs
+		if address.AssociationId == nil {
+			associateAddressInput := &ec2.AssociateAddressInput{
+				AllocationId:       address.AllocationId,
+				AllowReassociation: aws.Bool(false),
+				InstanceId:         aws.String(instanceID),
+			}
+			c.limiter.Limit(ctx, "AssociateAddress")
+			sinceStart = spanstat.Start()
+			association, err := c.ec2Client.AssociateAddress(ctx, associateAddressInput)
+			c.metricsAPI.ObserveAPICall("AssociateAddress", deriveStatus(err), sinceStart.Seconds())
+			if err != nil {
+				// TODO some errors can probably be skipped and next EIP can be tried
+				return "", err
+			}
+			log.Infof("Associated EIP %s with instance %s (association ID: %s)", *address.PublicIp, instanceID, *association.AssociationId)
+			return *address.PublicIp, nil
+		}
+	}
+
+	return "", fmt.Errorf("no unassociated EIPs found for tags %v", eipTags)
+}
+
 func createAWSTagSlice(tags map[string]string) []ec2_types.Tag {
 	awsTags := make([]ec2_types.Tag, 0, len(tags))
 	for k, v := range tags {

pkg/aws/ec2/mock/mock.go

@@ -531,6 +531,11 @@ func (e *API) UnassignENIPrefixes(ctx context.Context, eniID string, prefixes []
 	return fmt.Errorf("Unable to find ENI with ID %s", eniID)
 }
 
+func (e *API) AssociateEIP(ctx context.Context, instanceID string, eipTags ipamTypes.Tags) (string, error) {
+	// TODO
+	return "", nil
+}
+
 func (e *API) GetInstances(ctx context.Context, vpcs ipamTypes.VirtualNetworkMap, subnets ipamTypes.SubnetMap) (*ipamTypes.InstanceMap, error) {
 	instances := ipamTypes.NewInstanceMap()
 

pkg/aws/eni/instances.go

@@ -34,6 +34,7 @@ type EC2API interface {
 	UnassignPrivateIpAddresses(ctx context.Context, eniID string, addresses []string) error
 	AssignENIPrefixes(ctx context.Context, eniID string, prefixes int32) error
 	UnassignENIPrefixes(ctx context.Context, eniID string, prefixes []string) error
+	AssociateEIP(ctx context.Context, instanceID string, eipTags ipamTypes.Tags) (string, error)
 }
 
 // InstancesManager maintains the list of instances. It must be kept up to date

pkg/aws/eni/node.go

@@ -295,6 +295,10 @@ func (n *Node) AllocateIPs(ctx context.Context, a *ipam.AllocationAction) error
 	return n.manager.api.AssignPrivateIpAddresses(ctx, a.InterfaceID, int32(a.AvailableForAllocation))
 }
 
+func (n *Node) AllocateStaticIP(ctx context.Context, staticIPTags ipamTypes.Tags) (string, error) {
+	return n.manager.api.AssociateEIP(ctx, n.node.InstanceID(), staticIPTags)
+}
+
 func (n *Node) getSecurityGroupIDs(ctx context.Context, eniSpec eniTypes.ENISpec) ([]string, error) {
 	// 1. check explicit security groups associations via checking Spec.ENI.SecurityGroups
 	// 2. check if Spec.ENI.SecurityGroupTags is passed and if so filter by those

pkg/azure/ipam/node.go

@@ -130,6 +130,11 @@ func (n *Node) AllocateIPs(ctx context.Context, a *ipam.AllocationAction) error
 	}
 }
 
+func (n *Node) AllocateStaticIP(ctx context.Context, staticIPTags ipamTypes.Tags) (string, error) {
+	// TODO
+	return "", nil
+}
+
 // CreateInterface is called to create a new interface. This operation is
 // currently not supported on Azure.
 func (n *Node) CreateInterface(ctx context.Context, allocation *ipam.AllocationAction, scopedLog *logrus.Entry) (int, string, error) {

pkg/ipam/node.go

@@ -131,6 +131,8 @@ type Node struct {
 	// IPAMReleased        : IP released by the operator
 	ipReleaseStatus map[string]string
 
+	assignedStaticIP string
+
 	// logLimiter rate limits potentially repeating warning logs
 	logLimiter logging.Limiter
 }
@@ -264,6 +266,14 @@ func (n *Node) getMaxAllocate() int {
 	return instanceMax
 }
 
+func (n *Node) getStaticIPTags() ipamTypes.Tags {
+	if n.resource.Spec.IPAM.StaticIPTags != nil {
+		return n.resource.Spec.IPAM.StaticIPTags
+	} else {
+		return ipamTypes.Tags{}
+	}
+}
+
 // GetNeededAddresses returns the number of needed addresses that need to be
 // allocated or released. A positive number is returned to indicate allocation.
 // A negative number is returned to indicate release of addresses.
@@ -887,6 +897,16 @@ func (n *Node) maintainIPPool(ctx context.Context) (instanceMutated bool, err er
 		n.removeStaleReleaseIPs()
 	}
 
+	if len(n.getStaticIPTags()) > 1 {
+		if n.assignedStaticIP == "" {
+			ip, err := n.ops.AllocateStaticIP(ctx, n.getStaticIPTags())
+			if err != nil {
+				return false, err
+			}
+			n.assignedStaticIP = ip
+		}
+	}
+
 	a, err := n.determineMaintenanceAction()
 	if err != nil {
 		n.abortNoLongerExcessIPs(nil)
@@ -981,6 +1001,14 @@ func (n *Node) PopulateIPReleaseStatus(node *v2.CiliumNode) {
 	node.Status.IPAM.ReleaseIPs = releaseStatus
 }
 
+func (n *Node) PopulateStaticIPStatus(node *v2.CiliumNode) {
+	n.mutex.Lock()
+	defer n.mutex.Unlock()
+	if n.assignedStaticIP != "" {
+		node.Status.IPAM.AssignedStaticIP = n.assignedStaticIP
+	}
+}
+
 // syncToAPIServer synchronizes the contents of the CiliumNode resource
 // [(*Node).resource)] with the K8s apiserver. This operation occurs on an
 // interval to refresh the CiliumNode resource.
@@ -1029,6 +1057,7 @@ func (n *Node) syncToAPIServer() (err error) {
 
 		n.ops.PopulateStatusFields(node)
 		n.PopulateIPReleaseStatus(node)
+		n.PopulateStaticIPStatus(node)
 
 		err = n.update(origNode, node, retry, true)
 		if err == nil {

pkg/ipam/node_manager.go

@@ -75,6 +75,8 @@ type NodeOperations interface {
 	// to perform the actual allocation.
 	AllocateIPs(ctx context.Context, allocation *AllocationAction) error
 
+	AllocateStaticIP(ctx context.Context, staticIPTags ipamTypes.Tags) (string, error)
+
 	// PrepareIPRelease is called to calculate whether any IP excess needs
 	// to be resolved. It behaves identical to PrepareIPAllocation but
 	// indicates a need to release IPs.

pkg/ipam/node_manager_test.go

@@ -123,6 +123,11 @@ func (n *nodeOperationsMock) AllocateIPs(ctx context.Context, allocation *Alloca
 	return nil
 }
 
+func (n *nodeOperationsMock) AllocateStaticIP(ctx context.Context, staticIPTags ipamTypes.Tags) (string, error) {
+	// TODO
+	return "", nil
+}
+
 func (n *nodeOperationsMock) PrepareIPRelease(excessIPs int, scopedLog *logrus.Entry) *ReleaseAction {
 	n.mutex.RLock()
 	excessIPs = math.IntMin(excessIPs, len(n.allocatedIPs))

pkg/ipam/types/types.go

@@ -176,6 +176,9 @@ type IPAMSpec struct {
 	//
 	// +kubebuilder:validation:Minimum=0
 	PodCIDRReleaseThreshold int `json:"pod-cidr-release-threshold,omitempty"`
+
+	// +optional
+	StaticIPTags map[string]string `json:"static-ip-tags,omitempty"`
 }
 
 // IPReleaseStatus  defines the valid states in IP release handshake
@@ -212,6 +215,9 @@ type IPAMStatus struct {
 	//
 	// +optional
 	ReleaseIPs map[string]IPReleaseStatus `json:"release-ips,omitempty"`
+
+	// +optional
+	AssignedStaticIP string `json:"assigned-static-ip,omitempty"`
 }
 
 // IPAMPoolRequest is a request from the agent to the operator, indicating how

pkg/ipam/types/zz_generated.deepcopy.go

@@ -409,6 +409,10 @@ spec:
                       cilium-operator to get involved.
                     minimum: 0
                     type: integer
+                  static-ip-tags:
+                    additionalProperties:
+                      type: string
+                    type: object
                 type: object
               nodeidentity:
                 description: NodeIdentity is the Cilium numeric identity allocated
@@ -651,6 +655,8 @@ spec:
               ipam:
                 description: IPAM is the IPAM status of the node.
                 properties:
+                  assigned-static-ip:
+                    type: string
                   operator-status:
                     description: Operator is the Operator status of the node
                     properties:

pkg/ipam/types/zz_generated.deepequal.go

@@ -15,5 +15,5 @@ const (
 	//
 	// Maintainers: Run ./Documentation/check-crd-compat-table.sh for each release
 	// Developers: Bump patch for each change in the CRD schema.
-	CustomResourceDefinitionSchemaVersion = "1.27.1"
+	CustomResourceDefinitionSchemaVersion = "1.27.5"
 )

pkg/k8s/apis/cilium.io/client/crds/v2/ciliumnodes.yaml

@@ -571,6 +571,12 @@ func (n *NodeDiscovery) mutateNodeResource(nodeResource *ciliumv2.CiliumNode) er
 	nodeResource.Spec.BootID = n.localNode.BootID
 
 	switch option.Config.IPAM {
+	case ipamOption.IPAMKubernetes:
+		if c := n.NetConf; c != nil {
+			if c.IPAM.StaticIPTags != nil && len(c.IPAM.StaticIPTags) > 0 {
+				nodeResource.Spec.IPAM.StaticIPTags = c.IPAM.StaticIPTags
+			}
+		}
 	case ipamOption.IPAMClusterPoolV2:
 		if c := n.NetConf; c != nil {
 			nodeResource.Spec.IPAM.PodCIDRAllocationThreshold = c.IPAM.PodCIDRAllocationThreshold
@@ -609,6 +615,10 @@ func (n *NodeDiscovery) mutateNodeResource(nodeResource *ciliumv2.CiliumNode) er
 				nodeResource.Spec.IPAM.PreAllocate = c.IPAM.PreAllocate
 			}
 
+			if c.IPAM.StaticIPTags != nil && len(c.IPAM.StaticIPTags) > 0 {
+				nodeResource.Spec.IPAM.StaticIPTags = c.IPAM.StaticIPTags
+			}
+
 			if c.ENI.FirstInterfaceIndex != nil {
 				nodeResource.Spec.ENI.FirstInterfaceIndex = c.ENI.FirstInterfaceIndex
 			}
@@ -674,6 +684,9 @@ func (n *NodeDiscovery) mutateNodeResource(nodeResource *ciliumv2.CiliumNode) er
 			if c.IPAM.PreAllocate != 0 {
 				nodeResource.Spec.IPAM.PreAllocate = c.IPAM.PreAllocate
 			}
+			if c.IPAM.StaticIPTags != nil && len(c.IPAM.StaticIPTags) > 0 {
+				nodeResource.Spec.IPAM.StaticIPTags = c.IPAM.StaticIPTags
+			}
 			if c.Azure.InterfaceName != "" {
 				nodeResource.Spec.Azure.InterfaceName = c.Azure.InterfaceName
 			}
@@ -740,6 +753,10 @@ func (n *NodeDiscovery) mutateNodeResource(nodeResource *ciliumv2.CiliumNode) er
 			if c.IPAM.PreAllocate != 0 {
 				nodeResource.Spec.IPAM.PreAllocate = c.IPAM.PreAllocate
 			}
+
+			if c.IPAM.StaticIPTags != nil && len(c.IPAM.StaticIPTags) > 0 {
+				nodeResource.Spec.IPAM.StaticIPTags = c.IPAM.StaticIPTags
+			}
 		}
 	}