NO-JIRA: Normalize generation of driver metrics RBAC proxy sidecar

assets/common/sidecars/controller_driver_kube_rbac_proxy.yaml

@@ -19,7 +19,7 @@ spec:
           imagePullPolicy: IfNotPresent
           ports:
           - containerPort: ${EXPOSED_METRICS_PORT}
-            name: ${PORT_NAME}
+            name: driver-m
             protocol: TCP
           resources:
             requests:

assets/common/sidecars/node_driver_kube_rbac_proxy.yaml

@@ -19,7 +19,7 @@ spec:
           imagePullPolicy: IfNotPresent
           ports:
           - containerPort: ${EXPOSED_METRICS_PORT}
-            name: ${PORT_NAME}
+            name: driver-m
             protocol: TCP
           resources:
             requests:

assets/overlays/aws-ebs/patches/controller_add_driver.yaml

@@ -65,7 +65,7 @@ spec:
             - --endpoint=$(CSI_ENDPOINT)
             - --k8s-tag-cluster-id=${CLUSTER_ID}
             - --logtostderr
-            - --http-endpoint=localhost:8201
+            - --http-endpoint=localhost:${DRIVER_METRICS_PORT}
             - --v=${LOG_LEVEL}
             - --batching=true
           env:

assets/overlays/azure-disk/patches/controller_add_driver.yaml

@@ -22,7 +22,7 @@ spec:
           args:
             - --endpoint=$(CSI_ENDPOINT)
             - --logtostderr
-            - --metrics-address=localhost:8201
+            - --metrics-address=localhost:${DRIVER_METRICS_PORT}
             - --v=${LOG_LEVEL}
             # Use credentials provided by the azure-inject-credentials container
             - --cloud-config-secret-name=""

assets/overlays/azure-disk/patches/node_add_driver.yaml

@@ -19,7 +19,7 @@ spec:
             - --logtostderr
             - --v=${LOG_LEVEL}
             - --nodeid=$(KUBE_NODE_NAME)
-            - --metrics-address=localhost:8206
+            - --metrics-address=localhost:${DRIVER_METRICS_PORT}
             # Use credentials provided by the azure-inject-credentials container
             - --cloud-config-secret-name=""
             - --cloud-config-secret-namespace=""

assets/overlays/azure-file/patches/controller_add_driver.yaml

@@ -25,7 +25,7 @@ spec:
             - --enable-vhd=false
             - --endpoint=$(CSI_ENDPOINT)
             - --logtostderr
-            - --metrics-address=localhost:8211
+            - --metrics-address=localhost:${DRIVER_METRICS_PORT}
             - --v=${LOG_LEVEL}
             # Use credentials provided by the azure-inject-credentials container
             - --cloud-config-secret-name=""

assets/overlays/openstack-cinder/patches/controller_add_driver.yaml

@@ -40,16 +40,14 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - /bin/cinder-csi-plugin
-            - "--provide-controller-service=true"
-            - "--provide-node-service=false"
-            - "--endpoint=$(CSI_ENDPOINT)"
-            # this is the generated value of the LOCAL_METRICS_PORT variable
-            # we hardcode it because we don't currently support substitution
-            - "--http-endpoint=localhost:8202"
-            - "--cloud-config=$(CLOUD_CONFIG)"
-            - "--cluster=${CLUSTER_ID}"
-            - "--with-topology=$(ENABLE_TOPOLOGY)"
-            - "--v=${LOG_LEVEL}"
+            - --provide-controller-service=true
+            - --provide-node-service=false
+            - --endpoint=$(CSI_ENDPOINT)
+            - --http-endpoint=localhost:${DRIVER_METRICS_PORT}
+            - --cloud-config=$(CLOUD_CONFIG)
+            - --cluster=${CLUSTER_ID}
+            - --with-topology=$(ENABLE_TOPOLOGY)
+            - --v=${LOG_LEVEL}
           env:
             - name: CSI_ENDPOINT
               value: unix://csi/csi.sock

assets/overlays/samba/patches/controller_add_driver.yaml

@@ -17,7 +17,7 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - --endpoint=$(CSI_ENDPOINT)
-            - --metrics-address=localhost:8221
+            - --metrics-address=localhost:${DRIVER_METRICS_PORT}
             - --v=${LOG_LEVEL}
           ports:
             - containerPort: 10307

pkg/driver/aws-ebs/aws_ebs.go

@@ -55,16 +55,10 @@ func GetAWSEBSGeneratorConfig() *generator.CSIDriverGeneratorConfig {
 		OutputDir:        generatedAssetBase,
 
 		ControllerConfig: &generator.ControlPlaneConfig{
-			DeploymentTemplateAssetName: "overlays/aws-ebs/patches/controller_add_driver.yaml",
-			LivenessProbePort:           10301,
-			MetricsPorts: []generator.MetricsPort{
-				{
-					LocalPort:           commongenerator.AWSEBSLoopbackMetricsPortStart,
-					InjectKubeRBACProxy: true,
-					ExposedPort:         commongenerator.AWSEBSExposedMetricsPortStart,
-					Name:                "driver-m",
-				},
-			},
+			DeploymentTemplateAssetName:    "overlays/aws-ebs/patches/controller_add_driver.yaml",
+			LivenessProbePort:              10301,
+			LocalMetricsPort:               commongenerator.AWSEBSLoopbackMetricsPortStart,
+			ExposedMetricsPort:             commongenerator.AWSEBSExposedMetricsPortStart,
 			SidecarLocalMetricsPortStart:   commongenerator.AWSEBSLoopbackMetricsPortStart + 1,
 			SidecarExposedMetricsPortStart: commongenerator.AWSEBSExposedMetricsPortStart + 1,
 			Sidecars: []generator.SidecarConfig{

pkg/driver/azure-disk/azure_disk.go

@@ -72,16 +72,10 @@ func GetAzureDiskGeneratorConfig() *generator.CSIDriverGeneratorConfig {
 		OutputDir:        generatedAssetBase,
 
 		ControllerConfig: &generator.ControlPlaneConfig{
-			DeploymentTemplateAssetName: "overlays/azure-disk/patches/controller_add_driver.yaml",
-			LivenessProbePort:           10301,
-			MetricsPorts: []generator.MetricsPort{
-				{
-					LocalPort:           commongenerator.AzureDiskControllerLoopbackMetricsPortStart,
-					InjectKubeRBACProxy: true,
-					ExposedPort:         commongenerator.AzureDiskControllerExposedMetricsPortStart,
-					Name:                "driver-m",
-				},
-			},
+			DeploymentTemplateAssetName:    "overlays/azure-disk/patches/controller_add_driver.yaml",
+			LivenessProbePort:              10301,
+			LocalMetricsPort:               commongenerator.AzureDiskControllerLoopbackMetricsPortStart,
+			ExposedMetricsPort:             commongenerator.AzureDiskControllerExposedMetricsPortStart,
 			SidecarLocalMetricsPortStart:   commongenerator.AzureDiskControllerLoopbackMetricsPortStart + 1,
 			SidecarExposedMetricsPortStart: commongenerator.AzureDiskControllerExposedMetricsPortStart + 1,
 			Sidecars: []generator.SidecarConfig{
@@ -122,15 +116,9 @@ func GetAzureDiskGeneratorConfig() *generator.CSIDriverGeneratorConfig {
 
 		GuestConfig: &generator.GuestConfig{
 			DaemonSetTemplateAssetName: "overlays/azure-disk/patches/node_add_driver.yaml",
-			MetricsPorts: []generator.MetricsPort{
-				{
-					LocalPort:           commongenerator.AzureDiskNodeLoopbackMetricsPortStart,
-					ExposedPort:         commongenerator.AzureDiskNodeExposedMetricsPortStart,
-					Name:                "driver-m",
-					InjectKubeRBACProxy: true,
-				},
-			},
-			LivenessProbePort: 10300,
+			LocalMetricsPort:           commongenerator.AzureDiskNodeLoopbackMetricsPortStart,
+			ExposedMetricsPort:         commongenerator.AzureDiskNodeExposedMetricsPortStart,
+			LivenessProbePort:          10300,
 			// 10303 port is taken by azurefile stuff and hence we must use 10304 here
 			NodeRegistrarHealthCheckPort: 10304,
 			Sidecars: []generator.SidecarConfig{

pkg/driver/azure-file/azure_file.go

@@ -56,16 +56,10 @@ func GetAzureFileGeneratorConfig() *generator.CSIDriverGeneratorConfig {
 		OutputDir:        generatedAssetBase,
 
 		ControllerConfig: &generator.ControlPlaneConfig{
-			DeploymentTemplateAssetName: "overlays/azure-file/patches/controller_add_driver.yaml",
-			LivenessProbePort:           10303,
-			MetricsPorts: []generator.MetricsPort{
-				{
-					LocalPort:           commongenerator.AzureFileLoopbackMetricsPortStart,
-					InjectKubeRBACProxy: true,
-					ExposedPort:         commongenerator.AzureFileExposedMetricsPortStart,
-					Name:                "driver-m",
-				},
-			},
+			DeploymentTemplateAssetName:    "overlays/azure-file/patches/controller_add_driver.yaml",
+			LivenessProbePort:              10303,
+			LocalMetricsPort:               commongenerator.AzureFileLoopbackMetricsPortStart,
+			ExposedMetricsPort:             commongenerator.AzureFileExposedMetricsPortStart,
 			SidecarLocalMetricsPortStart:   commongenerator.AzureFileLoopbackMetricsPortStart + 1,
 			SidecarExposedMetricsPortStart: commongenerator.AzureFileExposedMetricsPortStart + 1,
 			Sidecars: []generator.SidecarConfig{

pkg/driver/common/operator/test_manifests/aws_ebs_controller_hypershift.yaml

@@ -273,7 +273,7 @@ spec:
           name: metrics-serving-cert
       - args:
         - --csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock
-        - --metrics-address=localhost:8206
+        - --metrics-address=localhost:${DRIVER_METRICS_PORT}
         - --leader-election
         - --leader-election-lease-duration=${LEADER_ELECTION_LEASE_DURATION}
         - --leader-election-renew-deadline=${LEADER_ELECTION_RENEW_DEADLINE}

pkg/driver/openstack-cinder/openstack_cinder.go

@@ -38,16 +38,10 @@ func GetOpenStackCinderGeneratorConfig() *generator.CSIDriverGeneratorConfig {
 		OutputDir:        generatedAssetBase,
 
 		ControllerConfig: &generator.ControlPlaneConfig{
-			DeploymentTemplateAssetName: "overlays/openstack-cinder/patches/controller_add_driver.yaml",
-			LivenessProbePort:           10301,
-			MetricsPorts: []generator.MetricsPort{
-				{
-					LocalPort:           commongenerator.OpenStackCinderLoopbackMetricsPortStart,
-					InjectKubeRBACProxy: true,
-					ExposedPort:         commongenerator.OpenStackCinderExposedMetricsPortStart,
-					Name:                "driver-m",
-				},
-			},
+			DeploymentTemplateAssetName:    "overlays/openstack-cinder/patches/controller_add_driver.yaml",
+			LivenessProbePort:              10301,
+			LocalMetricsPort:               commongenerator.OpenStackCinderLoopbackMetricsPortStart,
+			ExposedMetricsPort:             commongenerator.OpenStackCinderExposedMetricsPortStart,
 			SidecarLocalMetricsPortStart:   commongenerator.OpenStackCinderLoopbackMetricsPortStart + 1,
 			SidecarExposedMetricsPortStart: commongenerator.OpenStackCinderExposedMetricsPortStart + 1,
 			Sidecars: []generator.SidecarConfig{

pkg/driver/samba/samba.go

@@ -28,16 +28,10 @@ func GetSambaGeneratorConfig() *generator.CSIDriverGeneratorConfig {
 		OutputDir:        generatedAssetBase,
 
 		ControllerConfig: &generator.ControlPlaneConfig{
-			DeploymentTemplateAssetName: "overlays/samba/patches/controller_add_driver.yaml",
-			LivenessProbePort:           10307,
-			MetricsPorts: []generator.MetricsPort{
-				{
-					LocalPort:           commongenerator.SambaLoopbackMetricsPortStart,
-					InjectKubeRBACProxy: true,
-					ExposedPort:         commongenerator.SambaExposedMetricsPortStart,
-					Name:                "driver-m",
-				},
-			},
+			DeploymentTemplateAssetName:    "overlays/samba/patches/controller_add_driver.yaml",
+			LivenessProbePort:              10307,
+			LocalMetricsPort:               commongenerator.SambaLoopbackMetricsPortStart,
+			ExposedMetricsPort:             commongenerator.SambaExposedMetricsPortStart,
 			SidecarLocalMetricsPortStart:   commongenerator.SambaLoopbackMetricsPortStart + 1,
 			SidecarExposedMetricsPortStart: commongenerator.SambaExposedMetricsPortStart + 1,
 			Sidecars: []generator.SidecarConfig{

pkg/generator/asset_generator.go

@@ -102,26 +102,21 @@ func (gen *AssetGenerator) patchController() error {
 	return nil
 }
 
-// Inject kube-rbac-proxy container for all metrics ports into yamlFile. The yamlFile can be a Deployment or a
+// Inject kube-rbac-proxy container for the metrics port into yamlFile. The yamlFile can be a Deployment or a
 // DaemonSet. proxyPatchFile is the path to Deployment / DaemonSet patch file that adds the proxy container.
-func (gen *AssetGenerator) addDriverRBACProxyContainers(yamlFile *YAMLWithHistory, proxyPatchFile string, metricPorts []MetricsPort, baseExtraReplacements []string) error {
-	for i := 0; i < len(metricPorts); i++ {
-		port := metricPorts[i]
-		if !port.InjectKubeRBACProxy {
-			continue
-		}
-		extraReplacements := append([]string{}, baseExtraReplacements...) // Poor man's copy of the array.
-		extraReplacements = append(extraReplacements,
-			"${LOCAL_METRICS_PORT}", strconv.Itoa(int(port.LocalPort)),
-			"${EXPOSED_METRICS_PORT}", strconv.Itoa(int(port.ExposedPort)),
-			"${PORT_NAME}", port.Name,
-		)
-		port.LocalPort++
-		port.ExposedPort++
-		err := gen.applyAssetPatch(yamlFile, proxyPatchFile, extraReplacements)
-		if err != nil {
-			return err
-		}
+func (gen *AssetGenerator) addDriverRBACProxyContainers(yamlFile *YAMLWithHistory, proxyPatchFile string, localMetricsPort, exposedMetricsPort uint16, baseExtraReplacements []string) error {
+	if localMetricsPort == 0 {
+		return nil
+	}
+
+	extraReplacements := append([]string{}, baseExtraReplacements...) // Poor man's copy of the array.
+	extraReplacements = append(extraReplacements,
+		"${LOCAL_METRICS_PORT}", strconv.Itoa(int(localMetricsPort)),
+		"${EXPOSED_METRICS_PORT}", strconv.Itoa(int(exposedMetricsPort)),
+	)
+	err := gen.applyAssetPatch(yamlFile, proxyPatchFile, extraReplacements)
+	if err != nil {
+		return err
 	}
 	return nil
 }
@@ -131,7 +126,12 @@ func (gen *AssetGenerator) generateDeployment() error {
 	deploymentYAML := gen.mustReadBaseAsset("base/controller.yaml", nil)
 	var err error
 
-	err = gen.applyAssetPatch(deploymentYAML, ctrlCfg.DeploymentTemplateAssetName, nil)
+	extraReplacements := []string{}
+	if ctrlCfg.LocalMetricsPort != 0 {
+		extraReplacements = append(extraReplacements, "${DRIVER_METRICS_PORT}", strconv.Itoa(int(ctrlCfg.LocalMetricsPort)))
+	}
+
+	err = gen.applyAssetPatch(deploymentYAML, ctrlCfg.DeploymentTemplateAssetName, extraReplacements)
 	if err != nil {
 		return err
 	}
@@ -143,7 +143,7 @@ func (gen *AssetGenerator) generateDeployment() error {
 		baseExtraReplacements = append(baseExtraReplacements, "${LIVENESS_PROBE_PORT}", strconv.Itoa(int(ctrlCfg.LivenessProbePort)))
 	}
 
-	err = gen.addDriverRBACProxyContainers(deploymentYAML, "common/sidecars/controller_driver_kube_rbac_proxy.yaml", ctrlCfg.MetricsPorts, baseExtraReplacements)
+	err = gen.addDriverRBACProxyContainers(deploymentYAML, "common/sidecars/controller_driver_kube_rbac_proxy.yaml", ctrlCfg.LocalMetricsPort, ctrlCfg.ExposedMetricsPort, baseExtraReplacements)
 	if err != nil {
 		return err
 	}
@@ -170,25 +170,26 @@ func (gen *AssetGenerator) generateDeployment() error {
 	return nil
 }
 
-// Add driver's MetricsPorts to the metrics Service and ServiceMonitor.
-func (gen *AssetGenerator) generateDriverMetricsService(serviceYAML, serviceMonitorYAML *YAMLWithHistory, metricsPorts []MetricsPort, servicePrefix string) error {
-	for i := 0; i < len(metricsPorts); i++ {
-		port := metricsPorts[i]
-		extraReplacements := []string{
-			"${EXPOSED_METRICS_PORT}", strconv.Itoa(int(port.ExposedPort)),
-			"${LOCAL_METRICS_PORT}", strconv.Itoa(int(port.LocalPort)),
-			"${PORT_NAME}", port.Name,
-			"${SERVICE_PREFIX}", servicePrefix,
-		}
-		var err error
-		err = gen.applyAssetPatch(serviceYAML, "common/metrics/service_add_port.yaml", extraReplacements)
-		if err != nil {
-			return err
-		}
-		err = gen.applyAssetPatch(serviceMonitorYAML, "common/metrics/service_monitor_add_port.yaml.patch", extraReplacements)
-		if err != nil {
-			return err
-		}
+// Add driver's metrics port to the metrics Service and ServiceMonitor.
+func (gen *AssetGenerator) generateDriverMetricsService(serviceYAML, serviceMonitorYAML *YAMLWithHistory, localMetricsPort, exposedMetricsPort uint16, servicePrefix string) error {
+	if localMetricsPort == 0 {
+		return nil
+	}
+
+	extraReplacements := []string{
+		"${LOCAL_METRICS_PORT}", strconv.Itoa(int(localMetricsPort)),
+		"${EXPOSED_METRICS_PORT}", strconv.Itoa(int(exposedMetricsPort)),
+		"${PORT_NAME}", "driver-m",
+		"${SERVICE_PREFIX}", servicePrefix,
+	}
+	var err error
+	err = gen.applyAssetPatch(serviceYAML, "common/metrics/service_add_port.yaml", extraReplacements)
+	if err != nil {
+		return err
+	}
+	err = gen.applyAssetPatch(serviceMonitorYAML, "common/metrics/service_monitor_add_port.yaml.patch", extraReplacements)
+	if err != nil {
+		return err
 	}
 	return nil
 }
@@ -232,7 +233,7 @@ func (gen *AssetGenerator) generateControllerMonitoringService() error {
 	if err := gen.generateSidecarMetricsServices(serviceYAML, serviceMonitorYAML, int(ctrlCfg.SidecarLocalMetricsPortStart), int(ctrlCfg.SidecarExposedMetricsPortStart), ctrlCfg.Sidecars, "controller"); err != nil {
 		return err
 	}
-	if err := gen.generateDriverMetricsService(serviceYAML, serviceMonitorYAML, ctrlCfg.MetricsPorts, "controller"); err != nil {
+	if err := gen.generateDriverMetricsService(serviceYAML, serviceMonitorYAML, ctrlCfg.LocalMetricsPort, ctrlCfg.ExposedMetricsPort, "controller"); err != nil {
 		return err
 	}
 
@@ -287,12 +288,16 @@ func (gen *AssetGenerator) generateDaemonSet() error {
 		extraReplacements = append(extraReplacements, "${NODE_DRIVER_REGISTRAR_HEALTH_PORT}", strconv.Itoa(int(cfg.NodeRegistrarHealthCheckPort)))
 	}
 
+	if cfg.LocalMetricsPort != 0 {
+		extraReplacements = append(extraReplacements, "${DRIVER_METRICS_PORT}", strconv.Itoa(int(cfg.LocalMetricsPort)))
+	}
+
 	err = gen.applyAssetPatch(dsYAML, cfg.DaemonSetTemplateAssetName, extraReplacements)
 	if err != nil {
 		return err
 	}
 
-	err = gen.addDriverRBACProxyContainers(dsYAML, "common/sidecars/node_driver_kube_rbac_proxy.yaml", cfg.MetricsPorts, extraReplacements)
+	err = gen.addDriverRBACProxyContainers(dsYAML, "common/sidecars/node_driver_kube_rbac_proxy.yaml", cfg.LocalMetricsPort, cfg.ExposedMetricsPort, extraReplacements)
 	if err != nil {
 		return err
 	}
@@ -311,15 +316,15 @@ func (gen *AssetGenerator) generateDaemonSet() error {
 func (gen *AssetGenerator) generateGuestMonitoringService() error {
 	cfg := gen.operatorConfig.GuestConfig
 
-	if len(cfg.MetricsPorts) == 0 {
+	if cfg.LocalMetricsPort == 0 {
 		// Do not add metrics service if driver does not expose any metrics.
 		// There is no node-level sidecar that would export one.
 		return nil
 	}
 	serviceYAML := gen.mustReadBaseAsset("base/node_metrics_service.yaml", nil)
 	serviceMonitorYAML := gen.mustReadBaseAsset("base/node_metrics_servicemonitor.yaml", nil)
 
-	err := gen.generateDriverMetricsService(serviceYAML, serviceMonitorYAML, cfg.MetricsPorts, "node")
+	err := gen.generateDriverMetricsService(serviceYAML, serviceMonitorYAML, cfg.LocalMetricsPort, cfg.ExposedMetricsPort, "node")
 	if err != nil {
 		return err
 	}