Allow the DRA driver for GPUs to be force installed if desired

Signed-off-by: Kevin Klues <[email protected]>

Author: klueska

deployments/helm/nvidia-dra-driver-gpu/templates/clusterrole.yaml

@@ -29,6 +29,9 @@ rules:
 - apiGroups: ["apps"]
   resources: ["daemonsets"]
   verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
 - apiGroups: [""]
   resources: ["nodes"]
   verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]

deployments/helm/nvidia-dra-driver-gpu/templates/deviceclass-compute-domain-daemon.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.resources.computeDomains.enabled }}
 ---
 apiVersion: resource.k8s.io/v1beta1
 kind: DeviceClass
@@ -7,3 +8,4 @@ spec:
   selectors:
   - cel:
       expression: "device.driver == 'compute-domain.nvidia.com' && device.attributes['compute-domain.nvidia.com'].type == 'daemon'"
+{{- end }}

deployments/helm/nvidia-dra-driver-gpu/templates/deviceclass-compute-domain-default-channel.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.resources.computeDomains.enabled }}
 ---
 apiVersion: resource.k8s.io/v1beta1
 kind: DeviceClass
@@ -7,4 +8,4 @@ spec:
   selectors:
   - cel:
       expression: "device.driver == 'compute-domain.nvidia.com' && device.attributes['compute-domain.nvidia.com'].type == 'channel' && device.attributes['compute-domain.nvidia.com'].id == 0"
-
+{{- end }}

deployments/helm/nvidia-dra-driver-gpu/templates/deviceclass-gpu.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.resources.gpus.enabled }}
+---
+apiVersion: resource.k8s.io/v1beta1
+kind: DeviceClass
+metadata:
+  name: gpu.nvidia.com
+spec:
+  selectors:
+  - cel:
+      expression: "device.driver == 'gpu.nvidia.com' && device.attributes['gpu.nvidia.com'].type == 'gpu'"
+{{- end }}

deployments/helm/nvidia-dra-driver-gpu/templates/deviceclass-mig.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.resources.gpus.enabled }}
+---
+apiVersion: resource.k8s.io/v1beta1
+kind: DeviceClass
+metadata:
+  name: mig.nvidia.com
+spec:
+  selectors:
+  - cel:
+      expression: "device.driver == 'gpu.nvidia.com' && device.attributes['gpu.nvidia.com'].type == 'mig'"
+{{- end }}

deployments/helm/nvidia-dra-driver-gpu/templates/kubeletplugin.yaml

@@ -50,15 +50,49 @@ spec:
         {{- toYaml .Values.kubeletPlugin.podSecurityContext | nindent 8 }}
       containers:
       {{- if .Values.resources.computeDomains.enabled }}
-      - name: compute-domain
+      - name: compute-domains
         securityContext:
-          {{- toYaml .Values.kubeletPlugin.containers.computeDomain.securityContext | nindent 10 }}
+          {{- toYaml .Values.kubeletPlugin.containers.computeDomains.securityContext | nindent 10 }}
         image: {{ include "nvidia-dra-driver-gpu.fullimage" . }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         command:
         - compute-domain-kubelet-plugin
         resources:
-          {{- toYaml .Values.kubeletPlugin.containers.computeDomain.resources | nindent 10 }}
+          {{- toYaml .Values.kubeletPlugin.containers.computeDomains.resources | nindent 10 }}
+        env:
+        - name: CDI_ROOT
+          value: /var/run/cdi
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: plugins-registry
+          mountPath: /var/lib/kubelet/plugins_registry
+        - name: plugins
+          mountPath: /var/lib/kubelet/plugins
+          mountPropagation: Bidirectional
+        - name: cdi
+          mountPath: /var/run/cdi
+        # We always mount the driver root at /driver-root in the container.
+        - name: driver-root
+          mountPath: /driver-root
+          readOnly: true
+      {{- end }}
+      {{- if .Values.resources.gpus.enabled }}
+      - name: gpus
+        securityContext:
+          {{- toYaml .Values.kubeletPlugin.containers.gpus.securityContext | nindent 10 }}
+        image: {{ include "nvidia-dra-driver-gpu.fullimage" . }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        command:
+        - gpu-kubelet-plugin
+        resources:
+          {{- toYaml .Values.kubeletPlugin.containers.gpus.resources | nindent 10 }}
         env:
         - name: CDI_ROOT
           value: /var/run/cdi

deployments/helm/nvidia-dra-driver-gpu/templates/validation.yaml

@@ -30,10 +30,11 @@
 {{- fail $error }}
 {{- end }}
 
-{{- if .Values.resources.gpus.enabled }}
+{{- if and .Values.resources.gpus.enabled (not .Values.gpuResourcesEnabledOverride) }}
 {{- $error := "" }}
 {{- $error = printf "%s\nThe default value of 'resources.gpus.enabled=true' is not yet supported." $error }}
 {{- $error = printf "%s\nIt is set to true by default to future proof it as the default once support for it becomes available." $error }}
 {{- $error = printf "%s\nUntil then, please explicitly set 'resources.gpus.enabled=false' when installing this chart." $error }}
+{{- $error = printf "%s\nIf you truly want to force 'resources.gpus.enabled=true' to apply, you can set 'gpuResourcesEnabledOverride=true'." $error }}
 {{- fail $error }}
 {{- end }}

deployments/helm/nvidia-dra-driver-gpu/values.yaml

@@ -31,6 +31,7 @@ nameOverride: ""
 fullnameOverride: ""
 namespaceOverride: ""
 selectorLabelsOverride: {}
+gpuResourcesEnabledOverride: false
 
 allowDefaultNamespace: false
 
@@ -89,7 +90,11 @@ kubeletPlugin:
     init:
       securityContext: {}
       resources: {}
-    computeDomain:
+    computeDomains:
+      securityContext:
+        privileged: true
+      resources: {}
+    gpus:
       securityContext:
         privileged: true
       resources: {}