Support Zone-Aware Topologies in the KubeVirt CSI Driver

cmd/kubevirt-csi-driver/kubevirt-csi-driver.go

@@ -105,7 +105,11 @@ func handle() {
 		klog.Fatal(err)
 	}
 
-	var nodeID string
+	var (
+		nodeID            string
+		allowedTopologies = map[string]string{}
+	)
+
 	if *nodeName != "" {
 		node, err := tenantClientSet.CoreV1().Nodes().Get(context.TODO(), *nodeName, v1.GetOptions{})
 		if err != nil {
@@ -121,6 +125,9 @@ func handle() {
 		}
 		nodeID = fmt.Sprintf("%s/%s", vmNamespace, vmName)
 		klog.Infof("Node name: %v, Node ID: %s", *nodeName, nodeID)
+		allowedTopologies[service.WellKnownZoneTopologyKey] = node.Labels[service.WellKnownZoneTopologyKey]
+		allowedTopologies[service.WellKnownRegionTopologyKey] = node.Labels[service.WellKnownRegionTopologyKey]
+		klog.Infof("Node name: %v, Node ID: %s", nodeName, nodeID)
 	}
 
 	identityClientset = tenantClientSet
@@ -138,7 +145,8 @@ func handle() {
 		storageClassEnforcement,
 		nodeID,
 		*runNodeService,
-		*runControllerService)
+		*runControllerService,
+		allowedTopologies)
 
 	driver.Run(*endpoint)
 }

deploy/controller-infra/base/deploy.yaml

@@ -83,6 +83,7 @@ spec:
             - "--v=5"
             - "--timeout=3m"
             - "--retry-interval-max=1m"
+            - "--feature-gates=Topology=true"
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock

deploy/controller-tenant/base/deploy.yaml

@@ -82,6 +82,7 @@ spec:
             - "--v=5"
             - "--timeout=3m"
             - "--retry-interval-max=1m"
+            - "--feature-gates=Topology=true"
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock

hack/cluster-sync.sh

@@ -48,6 +48,12 @@ data:
 END
 }
 
+function tenant::label_all_nodes_with_allowed_topologies() {
+  for node in $(_kubectl get nodes -o name); do
+    _kubectl label $node topology.kubernetes.io/region=eu-central topology.kubernetes.io/zone=az-1 --overwrite
+  done
+}
+
 function cluster::generate_tenant_controller_overlay() {
   cat <<- END > ./deploy/controller-tenant/dev-overlay/controller.yaml
 kind: Deployment
@@ -102,6 +108,7 @@ _kubectl -n $TENANT_CLUSTER_NAMESPACE apply -f ./deploy/infra-cluster-service-ac
 # Generate kustomize overlay for development environment
 # ******************************************************
 tenant::deploy_kubeconfig_secret
+tenant::label_all_nodes_with_allowed_topologies
 cluster::generate_driver_configmap_overlay "tenant"
 cluster::generate_tenant_controller_overlay
 cluster::generate_node_overlay

hack/common.sh

@@ -80,6 +80,14 @@ metadata:
   annotations:
     storageclass.kubernetes.io/is-default-class: "true"
 provisioner: csi.kubevirt.io
+allowedTopologies:
+- matchLabelExpressions:
+  - key: topology.kubernetes.io/zone
+    values:
+    - az-1
+  - key: topology.kubernetes.io/region
+    values:
+    - eu-central
 allowVolumeExpansion: true
 parameters:
   infraStorageClassName: $2

hack/test-driver-rwx.yaml

@@ -5,6 +5,7 @@ SnapshotClass:
   FromName: true
 DriverInfo:
   Name: csi.kubevirt.io
+  TopologyKeys: ["topology.kubernetes.io/region", "topology.kubernetes.io/zone"]
   Capabilities:
     block: true
     controllerExpansion: true
@@ -18,7 +19,7 @@ DriverInfo:
     persistence: true
     singleNodeVolume: false
     snapshotDataSource: true
-    topology: false
+    topology: true
     capacity: false
     RWX: true
   RequiredAccessModes:

hack/test-driver.yaml

@@ -5,6 +5,7 @@ SnapshotClass:
   FromName: true
 DriverInfo:
   Name: csi.kubevirt.io
+  TopologyKeys: ["topology.kubernetes.io/region", "topology.kubernetes.io/zone"]
   Capabilities:
     block: true
     controllerExpansion: true
@@ -19,7 +20,7 @@ DriverInfo:
     singleNodeVolume: false
     snapshotDataSource: true
     pvcDataSource: true
-    topology: false
+    topology: true
     capacity: false
     RWX: false
   SupportedFsType:

pkg/service/controller.go

@@ -218,8 +218,12 @@ func (c *ControllerService) CreateVolume(ctx context.Context, req *csi.CreateVol
 	// Prepare serial for disk
 	serial := string(dv.GetUID())
 
-	// Return response
-	return &csi.CreateVolumeResponse{
+	zone, region, err := c.getAllowedZoneAndRegion(ctx, req.AccessibilityRequirements)
+	if err != nil {
+		return nil, err
+	}
+
+	cvr := &csi.CreateVolumeResponse{
 		Volume: &csi.Volume{
 			CapacityBytes: storageSize,
 			VolumeId:      dvName,
@@ -229,7 +233,27 @@ func (c *ControllerService) CreateVolume(ctx context.Context, req *csi.CreateVol
 			},
 			ContentSource: req.GetVolumeContentSource(),
 		},
-	}, nil
+	}
+
+	var allowedTopologies = map[string]string{}
+	if zone != "" {
+		allowedTopologies[WellKnownZoneTopologyKey] = zone
+	}
+
+	if region != "" {
+		allowedTopologies[WellKnownRegionTopologyKey] = region
+	}
+
+	if len(allowedTopologies) > 0 {
+		cvr.Volume.AccessibleTopology = []*csi.Topology{
+			{
+				Segments: allowedTopologies,
+			},
+		}
+	}
+
+	// Return response
+	return cvr, nil
 }
 
 func (c *ControllerService) determineDvSource(ctx context.Context, req *csi.CreateVolumeRequest) (*cdiv1.DataVolumeSource, error) {
@@ -830,3 +854,38 @@ func (c *ControllerService) ControllerGetCapabilities(context.Context, *csi.Cont
 func (c *ControllerService) ControllerGetVolume(_ context.Context, _ *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) {
 	return nil, status.Error(codes.Unimplemented, "")
 }
+
+func (c *ControllerService) getAllowedZoneAndRegion(_ context.Context, requirement *csi.TopologyRequirement) (string, string, error) {
+	var zone, region string
+
+	if requirement == nil {
+		return "", "", nil
+	}
+
+	for _, topology := range requirement.GetPreferred() {
+		preferredZone, exists := topology.GetSegments()[WellKnownZoneTopologyKey]
+		if exists {
+			zone = preferredZone
+		}
+
+		preferredRegion, exists := topology.GetSegments()[WellKnownRegionTopologyKey]
+		if exists {
+			region = preferredRegion
+		}
+	}
+
+	for _, topology := range requirement.GetRequisite() {
+		requisiteZone, exists := topology.GetSegments()[WellKnownZoneTopologyKey]
+		if exists {
+			zone = requisiteZone
+		}
+
+		requisiteRegion, exists := topology.GetSegments()[WellKnownRegionTopologyKey]
+		if exists {
+			region = requisiteRegion
+		}
+	}
+
+	return zone, region, nil
+
+}

pkg/service/driver.go

@@ -8,6 +8,11 @@ import (
 	"kubevirt.io/csi-driver/pkg/util"
 )
 
+const (
+	WellKnownRegionTopologyKey = "topology.kubernetes.io/region"
+	WellKnownZoneTopologyKey   = "topology.kubernetes.io/zone"
+)
+
 var (
 	// VendorVersion is the vendor version set by ldflags at build time
 	VendorVersion = "0.2.0"
@@ -30,7 +35,8 @@ func NewKubevirtCSIDriver(virtClient kubevirt.Client,
 	storageClassEnforcement util.StorageClassEnforcement,
 	nodeID string,
 	runNodeService bool,
-	runControllerService bool) *KubevirtCSIDriver {
+	runControllerService bool,
+	allowedTopologies map[string]string) *KubevirtCSIDriver {
 	d := KubevirtCSIDriver{
 		IdentityService: NewIdentityService(identityClientset),
 	}
@@ -45,7 +51,7 @@ func NewKubevirtCSIDriver(virtClient kubevirt.Client,
 	}
 
 	if runNodeService {
-		d.NodeService = NewNodeService(nodeID)
+		d.NodeService = NewNodeService(nodeID, allowedTopologies)
 	}
 
 	return &d

pkg/service/identity.go

@@ -69,6 +69,13 @@ func (i *IdentityService) GetPluginCapabilities(context.Context, *csi.GetPluginC
 					},
 				},
 			},
+			{
+				Type: &csi.PluginCapability_Service_{
+					Service: &csi.PluginCapability_Service{
+						Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS,
+					},
+				},
+			},
 		},
 	}, nil
 }

pkg/service/node.go

@@ -39,6 +39,7 @@ type NodeService struct {
 	resizer          ResizerInterface
 	devicePathGetter DevicePathGetter
 	dirMaker         dirMaker
+	allowedTopologies map[string]string
 }
 
 type DeviceLister interface {
@@ -100,14 +101,15 @@ var NewFsMaker = func() FsMaker {
 	})
 }
 
-func NewNodeService(nodeId string) *NodeService {
+func NewNodeService(nodeId string, allowedTopologies map[string]string) *NodeService {
 	return &NodeService{
 		nodeID:           nodeId,
 		deviceLister:     NewDeviceLister(),
 		devicePathGetter: NewDevicePathGetter(),
 		fsMaker:          NewFsMaker(),
 		mounter:          NewMounter(),
 		resizer:          NewResizer(),
+		allowedTopologies: allowedTopologies,
 		dirMaker: dirMakerFunc(func(path string, perm os.FileMode) error {
 			// MkdirAll returns nil if path already exists
 			return os.MkdirAll(path, perm)
@@ -444,7 +446,10 @@ func (n *NodeService) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV
 // NodeGetInfo returns the node ID
 func (n *NodeService) NodeGetInfo(context.Context, *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) {
 	// the nodeID is the VM's ID in kubevirt or VMI.spec.domain.firmware.uuid
-	return &csi.NodeGetInfoResponse{NodeId: n.nodeID}, nil
+
+	return &csi.NodeGetInfoResponse{
+		NodeId:             n.nodeID,
+		AccessibleTopology: &csi.Topology{Segments: n.allowedTopologies}}, nil
 }
 
 // NodeGetCapabilities returns the supported capabilities of the node service

sanity/sanity_suite_test.go

@@ -67,7 +67,11 @@ var _ = ginkgo.BeforeSuite(func() {
 		storagClassEnforcement,
 		getKey(infraClusterNamespace, nodeID),
 		true,
-		true)
+		true,
+		map[string]string{
+			service.WellKnownRegionTopologyKey: "eu-central-1",
+			service.WellKnownZoneTopologyKey:   "eu-central-1a",
+		})
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
 	go func() {