From 04751ec7ac14bca378d528af3d3f503c7c5716f5 Mon Sep 17 00:00:00 2001 From: Christopher Desiniotis Date: Wed, 27 Nov 2024 08:59:16 -0800 Subject: [PATCH 1/2] Add DRA driver for IMEX Signed-off-by: Christopher Desiniotis --- api/nvidia/v1/clusterpolicy_types.go | 53 ++++++ api/nvidia/v1/zz_generated.deepcopy.go | 31 ++++ .../0100_service_account.yaml | 5 + assets/state-dra-driver/0200_clusterrole.yaml | 15 ++ .../0300_clusterrolebinding.yaml | 12 ++ .../0400_deviceclass-imex.yaml | 8 + assets/state-dra-driver/0500_deployment.yaml | 44 +++++ assets/state-dra-driver/0600_configmap.yaml | 34 ++++ assets/state-dra-driver/0700_daemonset.yaml | 118 ++++++++++++ .../manifests/nvidia.com_clusterpolicies.yaml | 33 ++++ .../crd/bases/nvidia.com_clusterpolicies.yaml | 33 ++++ controllers/object_controls.go | 140 +++++++++++++- controllers/resource_manager.go | 14 +- controllers/state_manager.go | 8 + controllers/transforms_test.go | 171 ++++++++++++++++++ .../crds/nvidia.com_clusterpolicies.yaml | 33 ++++ .../gpu-operator/templates/clusterpolicy.yaml | 20 ++ .../gpu-operator/templates/clusterrole.yaml | 10 + deployments/gpu-operator/values.yaml | 10 + 19 files changed, 788 insertions(+), 4 deletions(-) create mode 100644 assets/state-dra-driver/0100_service_account.yaml create mode 100644 assets/state-dra-driver/0200_clusterrole.yaml create mode 100644 assets/state-dra-driver/0300_clusterrolebinding.yaml create mode 100644 assets/state-dra-driver/0400_deviceclass-imex.yaml create mode 100644 assets/state-dra-driver/0500_deployment.yaml create mode 100644 assets/state-dra-driver/0600_configmap.yaml create mode 100644 assets/state-dra-driver/0700_daemonset.yaml diff --git a/api/nvidia/v1/clusterpolicy_types.go b/api/nvidia/v1/clusterpolicy_types.go index 07e424761..c03d2630c 100644 --- a/api/nvidia/v1/clusterpolicy_types.go +++ b/api/nvidia/v1/clusterpolicy_types.go @@ -53,6 +53,8 @@ type ClusterPolicySpec struct { Toolkit ToolkitSpec `json:"toolkit"` // DevicePlugin component spec DevicePlugin DevicePluginSpec `json:"devicePlugin"` + // DRADriver component spec + DRADriver DRADriverSpec `json:"draDriver"` // DCGMExporter spec DCGMExporter DCGMExporterSpec `json:"dcgmExporter"` // DCGM component spec @@ -841,6 +843,45 @@ type SandboxDevicePluginSpec struct { Env []EnvVar `json:"env,omitempty"` } +// DRADriverSpec defines the properties for the NVIDIA DRA Driver deployment +type DRADriverSpec struct { + // Enabled indicates if the deployment of NVIDIA DRA Driver through the operator is enabled + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Enable NVIDIA DRA Driver deployment through GPU Operator" + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch" + Enabled *bool `json:"enabled,omitempty"` + + // NVIDIA DRA Driver image repository + // +kubebuilder:validation:Optional + Repository string `json:"repository,omitempty"` + + // NVIDIA DRA Driver image name + // +kubebuilder:validation:Pattern=[a-zA-Z0-9\-]+ + Image string `json:"image,omitempty"` + + // NVIDIA DRA Driver image tag + // +kubebuilder:validation:Optional + Version string `json:"version,omitempty"` + + // Image pull policy + // +kubebuilder:validation:Optional + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Image Pull Policy" + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:imagePullPolicy" + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` + + // Image pull secrets + // +kubebuilder:validation:Optional + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Image pull secrets" + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:io.kubernetes:Secret" + ImagePullSecrets []string `json:"imagePullSecrets,omitempty"` + + // DeviceClasses indicates which device classes are enabled in the DRA driver + // +kubebuilder:validation:Optional + DeviceClasses []string `json:"deviceClasses,omitempty"` +} + // DCGMExporterSpec defines the properties for NVIDIA DCGM Exporter deployment type DCGMExporterSpec struct { // Enabled indicates if deployment of NVIDIA DCGM Exporter through operator is enabled @@ -1764,6 +1805,9 @@ func ImagePath(spec interface{}) (string, error) { case *SandboxDevicePluginSpec: config := spec.(*SandboxDevicePluginSpec) return imagePath(config.Repository, config.Image, config.Version, "SANDBOX_DEVICE_PLUGIN_IMAGE") + case *DRADriverSpec: + config := spec.(*DRADriverSpec) + return imagePath(config.Repository, config.Image, config.Version, "DRA_DRIVER_IMAGE") case *DCGMExporterSpec: config := spec.(*DCGMExporterSpec) return imagePath(config.Repository, config.Image, config.Version, "DCGM_EXPORTER_IMAGE") @@ -1872,6 +1916,15 @@ func (p *DevicePluginSpec) IsEnabled() bool { return *p.Enabled } +// IsEnabled returns true if draDriver is enabled through gpu-operator +func (d *DRADriverSpec) IsEnabled() bool { + if d.Enabled == nil { + // default is true if not specified by user + return true + } + return *d.Enabled +} + // IsEnabled returns true if dcgm-exporter is enabled(default) through gpu-operator func (e *DCGMExporterSpec) IsEnabled() bool { if e.Enabled == nil { diff --git a/api/nvidia/v1/zz_generated.deepcopy.go b/api/nvidia/v1/zz_generated.deepcopy.go index 6d876f675..196041a70 100644 --- a/api/nvidia/v1/zz_generated.deepcopy.go +++ b/api/nvidia/v1/zz_generated.deepcopy.go @@ -181,6 +181,7 @@ func (in *ClusterPolicySpec) DeepCopyInto(out *ClusterPolicySpec) { in.Driver.DeepCopyInto(&out.Driver) in.Toolkit.DeepCopyInto(&out.Toolkit) in.DevicePlugin.DeepCopyInto(&out.DevicePlugin) + in.DRADriver.DeepCopyInto(&out.DRADriver) in.DCGMExporter.DeepCopyInto(&out.DCGMExporter) in.DCGM.DeepCopyInto(&out.DCGM) in.NodeStatusExporter.DeepCopyInto(&out.NodeStatusExporter) @@ -406,6 +407,36 @@ func (in *DCGMSpec) DeepCopy() *DCGMSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DRADriverSpec) DeepCopyInto(out *DRADriverSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.ImagePullSecrets != nil { + in, out := &in.ImagePullSecrets, &out.ImagePullSecrets + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DeviceClasses != nil { + in, out := &in.DeviceClasses, &out.DeviceClasses + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DRADriverSpec. +func (in *DRADriverSpec) DeepCopy() *DRADriverSpec { + if in == nil { + return nil + } + out := new(DRADriverSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DaemonsetsSpec) DeepCopyInto(out *DaemonsetsSpec) { *out = *in diff --git a/assets/state-dra-driver/0100_service_account.yaml b/assets/state-dra-driver/0100_service_account.yaml new file mode 100644 index 000000000..76d6d61af --- /dev/null +++ b/assets/state-dra-driver/0100_service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nvidia-dra-driver + namespace: "FILLED BY THE OPERATOR" diff --git a/assets/state-dra-driver/0200_clusterrole.yaml b/assets/state-dra-driver/0200_clusterrole.yaml new file mode 100644 index 000000000..2c8f523e1 --- /dev/null +++ b/assets/state-dra-driver/0200_clusterrole.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nvidia-dra-driver +rules: +# TODO: restrict RBAC for DRA driver +- apiGroups: + - "" + - apps + - resource.k8s.io + - gpu.nvidia.com + resources: + - '*' + verbs: + - '*' diff --git a/assets/state-dra-driver/0300_clusterrolebinding.yaml b/assets/state-dra-driver/0300_clusterrolebinding.yaml new file mode 100644 index 000000000..6cabfb52d --- /dev/null +++ b/assets/state-dra-driver/0300_clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nvidia-dra-driver +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nvidia-dra-driver +subjects: +- kind: ServiceAccount + name: nvidia-dra-driver + namespace: "FILLED BY THE OPERATOR" diff --git a/assets/state-dra-driver/0400_deviceclass-imex.yaml b/assets/state-dra-driver/0400_deviceclass-imex.yaml new file mode 100644 index 000000000..759476c85 --- /dev/null +++ b/assets/state-dra-driver/0400_deviceclass-imex.yaml @@ -0,0 +1,8 @@ +apiVersion: resource.k8s.io/v1alpha3 +kind: DeviceClass +metadata: + name: imex.nvidia.com +spec: + selectors: + - cel: + expression: "device.driver == 'gpu.nvidia.com' && device.attributes['gpu.nvidia.com'].type == 'imex-channel'" diff --git a/assets/state-dra-driver/0500_deployment.yaml b/assets/state-dra-driver/0500_deployment.yaml new file mode 100644 index 000000000..8d8dbf7ee --- /dev/null +++ b/assets/state-dra-driver/0500_deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: nvidia-imex-dra-driver-controller + name: nvidia-imex-dra-driver-controller + namespace: "FILLED BY THE OPERATOR" +spec: + replicas: 1 + selector: + matchLabels: + app: nvidia-imex-dra-driver-controller + template: + metadata: + labels: + app: nvidia-imex-dra-driver-controller + spec: + priorityClassName: system-node-critical + serviceAccountName: nvidia-dra-driver + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - name: controller + image: "FILLED BY THE OPERATOR" + imagePullPolicy: IfNotPresent + command: ["nvidia-dra-controller", "-v", "6"] + env: + - name: DEVICE_CLASSES + value: imex + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace diff --git a/assets/state-dra-driver/0600_configmap.yaml b/assets/state-dra-driver/0600_configmap.yaml new file mode 100644 index 000000000..d910b26e3 --- /dev/null +++ b/assets/state-dra-driver/0600_configmap.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nvidia-dra-driver-kubelet-plugin-entrypoint + namespace: "FILLED BY THE OPERATOR" + labels: + app: nvidia-dra-driver-kubelet-plugin +data: + entrypoint.sh: |- + #!/bin/bash + + until [[ -f /run/nvidia/validations/driver-ready ]] + do + echo "waiting for the driver validations to be ready..." + sleep 5 + done + + set -o allexport + cat /run/nvidia/validations/driver-ready + . /run/nvidia/validations/driver-ready + # TODO: add an alias for DRIVER_ROOT_CTR_PATH in the k8s-dra-driver and remove the below export + export CONTAINER_DRIVER_ROOT=$DRIVER_ROOT_CTR_PATH + + # Conditionally mask the params file to prevent this container from + # recreating any missing GPU device nodes. This is necessary, for + # example, when running under nvkind to limit the set GPUs governed + # by the plugin even though it has cgroup access to all of them. + if [ "${MASK_NVIDIA_DRIVER_PARAMS}" = "true" ]; then + cp /proc/driver/nvidia/params root/gpu-params + sed -i 's/^ModifyDeviceFiles: 1$/ModifyDeviceFiles: 0/' root/gpu-params + mount --bind root/gpu-params /proc/driver/nvidia/params + fi + echo "Starting nvidia-dra-plugin" + exec nvidia-dra-plugin diff --git a/assets/state-dra-driver/0700_daemonset.yaml b/assets/state-dra-driver/0700_daemonset.yaml new file mode 100644 index 000000000..56e7ece66 --- /dev/null +++ b/assets/state-dra-driver/0700_daemonset.yaml @@ -0,0 +1,118 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: nvidia-imex-dra-driver-kubelet-plugin + name: nvidia-imex-dra-driver-kubelet-plugin + namespace: "FILLED BY THE OPERATOR" +spec: + selector: + matchLabels: + app: nvidia-imex-dra-driver-kubelet-plugin + updateStrategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: nvidia-imex-dra-driver-kubelet-plugin + spec: + priorityClassName: system-node-critical + serviceAccountName: nvidia-dra-driver + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.imex-domain + operator: Exists + initContainers: + - image: "FILLED BY THE OPERATOR" + name: driver-validation + command: [ 'sh', '-c' ] + args: [ "until [ -f /run/nvidia/validations/driver-ready ]; do echo waiting for driver to be setup; sleep 5; done" ] + securityContext: + privileged: true + volumeMounts: + - name: run-nvidia-validations + mountPath: /run/nvidia/validations + mountPropagation: HostToContainer + containers: + - name: plugin + image: "FILLED BY THE OPERATOR" + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - /bin/entrypoint.sh + env: + - name: MASK_NVIDIA_DRIVER_PARAMS + value: "false" + - name: NVIDIA_VISIBLE_DEVICES + value: void + - name: CDI_ROOT + value: /var/run/cdi + - name: NVIDIA_MIG_CONFIG_DEVICES + value: all + - name: DEVICE_CLASSES + value: imex + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + securityContext: + privileged: true + volumeMounts: + - name: nvidia-dra-driver-kubelet-plugin-entrypoint + readOnly: true + mountPath: /bin/entrypoint.sh + subPath: entrypoint.sh + - name: run-nvidia-validations + mountPath: /run/nvidia/validations + - name: driver-install-dir + mountPath: /driver-root + mountPropagation: HostToContainer + readOnly: true + - name: host-root + mountPath: /host + readOnly: true + - mountPath: /var/lib/kubelet/plugins_registry + name: plugins-registry + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugins + - mountPath: /var/run/cdi + name: cdi + volumes: + - name: nvidia-dra-driver-kubelet-plugin-entrypoint + configMap: + name: nvidia-dra-driver-kubelet-plugin-entrypoint + defaultMode: 448 + - name: run-nvidia-validations + hostPath: + path: "/run/nvidia/validations" + type: DirectoryOrCreate + - name: driver-install-dir + hostPath: + path: "/run/nvidia/driver" + type: DirectoryOrCreate + - name: host-root + hostPath: + path: / + - name: plugins-registry + hostPath: + path: /var/lib/kubelet/plugins_registry + - name: plugins + hostPath: + path: /var/lib/kubelet/plugins + - name: cdi + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate diff --git a/bundle/manifests/nvidia.com_clusterpolicies.yaml b/bundle/manifests/nvidia.com_clusterpolicies.yaml index 54e4a652b..9e95b49ac 100644 --- a/bundle/manifests/nvidia.com_clusterpolicies.yaml +++ b/bundle/manifests/nvidia.com_clusterpolicies.yaml @@ -596,6 +596,38 @@ spec: description: NVIDIA Device Plugin image tag type: string type: object + draDriver: + description: DRADriver component spec + properties: + deviceClasses: + description: DeviceClasses indicates which device classes are + enabled in the DRA driver + items: + type: string + type: array + enabled: + description: Enabled indicates if the deployment of NVIDIA DRA + Driver through the operator is enabled + type: boolean + image: + description: NVIDIA DRA Driver image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA DRA Driver image repository + type: string + version: + description: NVIDIA DRA Driver image tag + type: string + type: object driver: description: Driver component spec properties: @@ -2296,6 +2328,7 @@ spec: - dcgm - dcgmExporter - devicePlugin + - draDriver - driver - gfd - nodeStatusExporter diff --git a/config/crd/bases/nvidia.com_clusterpolicies.yaml b/config/crd/bases/nvidia.com_clusterpolicies.yaml index 54e4a652b..9e95b49ac 100644 --- a/config/crd/bases/nvidia.com_clusterpolicies.yaml +++ b/config/crd/bases/nvidia.com_clusterpolicies.yaml @@ -596,6 +596,38 @@ spec: description: NVIDIA Device Plugin image tag type: string type: object + draDriver: + description: DRADriver component spec + properties: + deviceClasses: + description: DeviceClasses indicates which device classes are + enabled in the DRA driver + items: + type: string + type: array + enabled: + description: Enabled indicates if the deployment of NVIDIA DRA + Driver through the operator is enabled + type: boolean + image: + description: NVIDIA DRA Driver image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA DRA Driver image repository + type: string + version: + description: NVIDIA DRA Driver image tag + type: string + type: object driver: description: Driver component spec properties: @@ -2296,6 +2328,7 @@ spec: - dcgm - dcgmExporter - devicePlugin + - draDriver - driver - gfd - nodeStatusExporter diff --git a/controllers/object_controls.go b/controllers/object_controls.go index 1b1801391..e9fbe8e6c 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -41,6 +41,7 @@ import ( corev1 "k8s.io/api/core/v1" nodev1 "k8s.io/api/node/v1" nodev1beta1 "k8s.io/api/node/v1beta1" + drav1 "k8s.io/api/resource/v1alpha3" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -147,6 +148,8 @@ const ( NvidiaCtrRuntimeCDIPrefixesEnvName = "NVIDIA_CONTAINER_RUNTIME_MODES_CDI_ANNOTATION_PREFIXES" // CDIEnabledEnvName is the name of the envvar used to enable CDI in the operands CDIEnabledEnvName = "CDI_ENABLED" + // NvidiaCTKPathEnvName is the name of the envvar specifying the path to the 'nvidia-ctk' binary + NvidiaCTKPathEnvName = "NVIDIA_CTK_PATH" // NvidiaCDIHookPathEnvName is the name of the envvar specifying the path to the 'nvidia-cdi-hook' binary NvidiaCDIHookPathEnvName = "NVIDIA_CDI_HOOK_PATH" // CrioConfigModeEnvName is the name of the envvar controlling how the toolkit container updates the cri-o configuration @@ -695,6 +698,7 @@ func preProcessDaemonSet(obj *appsv1.DaemonSet, n ClusterPolicyController) error "nvidia-vgpu-device-manager": TransformVGPUDeviceManager, "nvidia-vfio-manager": TransformVFIOManager, "nvidia-container-toolkit-daemonset": TransformToolkit, + "nvidia-imex-kubelet-plugin": TransformIMEXDRADriverPlugin, "nvidia-device-plugin-daemonset": TransformDevicePlugin, "nvidia-device-plugin-mps-control-daemon": TransformMPSControlDaemon, "nvidia-sandbox-device-plugin-daemonset": TransformSandboxDevicePlugin, @@ -1539,6 +1543,36 @@ func TransformSandboxDevicePlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPo return nil } +// TransformIMEXDRADriverPlugin transforms nvidia-imex-dra-driver-kubelet-plugin daemonset with required config as per ClusterPolicy +func TransformIMEXDRADriverPlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error { + // update validation container + err := transformValidationInitContainer(obj, config) + if err != nil { + return err + } + + // update image + image, err := gpuv1.ImagePath(&config.DRADriver) + if err != nil { + return err + } + obj.Spec.Template.Spec.Containers[0].Image = image + + // update image pull policy + obj.Spec.Template.Spec.Containers[0].ImagePullPolicy = gpuv1.ImagePullPolicy(config.DRADriver.ImagePullPolicy) + + // set image pull secrets + if len(config.DRADriver.ImagePullSecrets) > 0 { + addPullSecrets(&obj.Spec.Template.Spec, config.DRADriver.ImagePullSecrets) + } + + if config.Toolkit.IsEnabled() { + setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), NvidiaCTKPathEnvName, filepath.Join(config.Toolkit.InstallDir, "toolkit/nvidia-ctk")) + } + + return nil +} + // TransformDCGMExporter transforms dcgm exporter daemonset with required config as per ClusterPolicy func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error { // update validation container @@ -3663,17 +3697,50 @@ func getDaemonsetControllerRevisionHash(ctx context.Context, daemonset *appsv1.D return hash, nil } +// TransformIMEXDRADriverController transforms nvidia-imex-dra-driver-controller deployment with required config as per ClusterPolicy +func TransformIMEXDRADriverController(obj *appsv1.Deployment, spec *gpuv1.ClusterPolicySpec) error { + config := spec.DRADriver + // update image + image, err := gpuv1.ImagePath(&config) + if err != nil { + return err + } + obj.Spec.Template.Spec.Containers[0].Image = image + + // update image pull policy + obj.Spec.Template.Spec.Containers[0].ImagePullPolicy = gpuv1.ImagePullPolicy(config.ImagePullPolicy) + + // set image pull secrets + if len(config.ImagePullSecrets) > 0 { + addPullSecrets(&obj.Spec.Template.Spec, config.ImagePullSecrets) + } + + return nil +} + +func transformDeployment(obj *appsv1.Deployment, n ClusterPolicyController) error { + logger := n.logger.WithValues("Deployment", obj.Name, "Namespace", obj.Namespace) + switch obj.Name { + case "nvidia-imex-dra-driver-controller": + return TransformIMEXDRADriverController(obj, &n.singleton.Spec) + default: + logger.Info("No transformation for object") + return nil + } +} + // Deployment creates Deployment resource func Deployment(n ClusterPolicyController) (gpuv1.State, error) { ctx := n.ctx state := n.idx + stateName := n.stateNames[state] obj := n.resources[state].Deployment.DeepCopy() obj.Namespace = n.operatorNamespace logger := n.logger.WithValues("Deployment", obj.Name, "Namespace", obj.Namespace) // Check if state is disabled and cleanup resource if exists - if !n.isStateEnabled(n.stateNames[n.idx]) { + if !n.isStateEnabled(stateName) || (obj.Name == "nvidia-imex-dra-driver-controller" && !n.isDRADeviceClassEnabled("imex")) { err := n.client.Delete(ctx, obj) if err != nil && !apierrors.IsNotFound(err) { logger.Info("Couldn't delete", "Error", err) @@ -3682,6 +3749,11 @@ func Deployment(n ClusterPolicyController) (gpuv1.State, error) { return gpuv1.Disabled, nil } + if err := transformDeployment(obj, n); err != nil { + logger.Info("Failed to transform Deployment", "Error", err) + return gpuv1.NotReady, err + } + if err := controllerutil.SetControllerReference(n.singleton, obj, n.scheme); err != nil { return gpuv1.NotReady, err } @@ -4165,7 +4237,7 @@ func DaemonSet(n ClusterPolicyController) (gpuv1.State, error) { logger := n.logger.WithValues("DaemonSet", obj.Name, "Namespace", obj.Namespace) // Check if state is disabled and cleanup resource if exists - if !n.isStateEnabled(n.stateNames[n.idx]) { + if !n.isStateEnabled(n.stateNames[n.idx]) || (obj.Name == "nvidia-imex-dra-driver-kubelet-plugin" && !n.isDRADeviceClassEnabled("imex")) { err := n.client.Delete(ctx, obj) if err != nil && !apierrors.IsNotFound(err) { logger.Info("Couldn't delete", "Error", err) @@ -4880,3 +4952,67 @@ func PrometheusRule(n ClusterPolicyController) (gpuv1.State, error) { } return gpuv1.Ready, nil } + +func createDeviceClass(n ClusterPolicyController, spec drav1.DeviceClass) (gpuv1.State, error) { + ctx := n.ctx + state := n.idx + obj := spec.DeepCopy() + + logger := n.logger.WithValues("DeviceClass", obj.Name) + + // Check if state is disabled and cleanup resource if exists + if !n.isStateEnabled(n.stateNames[state]) { + err := n.client.Delete(ctx, obj) + if err != nil && !apierrors.IsNotFound(err) { + logger.Info("Couldn't delete", "Error", err) + return gpuv1.NotReady, err + } + return gpuv1.Disabled, nil + } + + if err := controllerutil.SetControllerReference(n.singleton, obj, n.scheme); err != nil { + return gpuv1.NotReady, err + } + + found := &drav1.DeviceClass{} + err := n.client.Get(ctx, types.NamespacedName{Namespace: "", Name: obj.Name}, found) + if err != nil && apierrors.IsNotFound(err) { + logger.Info("Not found, creating...") + err = n.client.Create(ctx, obj) + if err != nil { + logger.Info("Couldn't create", "Error", err) + return gpuv1.NotReady, err + } + return gpuv1.Ready, nil + } else if err != nil { + return gpuv1.NotReady, err + } + + logger.Info("Found Resource, updating...") + obj.ResourceVersion = found.ResourceVersion + + err = n.client.Update(ctx, obj) + if err != nil { + logger.Info("Couldn't update", "Error", err) + return gpuv1.NotReady, err + } + return gpuv1.Ready, nil +} + +// DeviceClasses creates DeviceClass objects +func DeviceClasses(n ClusterPolicyController) (gpuv1.State, error) { + status := gpuv1.Ready + state := n.idx + + for _, obj := range n.resources[state].DeviceClasses { + obj := obj + stat, err := createDeviceClass(n, obj) + if err != nil { + return stat, err + } + if stat != gpuv1.Ready { + status = gpuv1.NotReady + } + } + return status, nil +} diff --git a/controllers/resource_manager.go b/controllers/resource_manager.go index 2789bfe3d..26610300d 100644 --- a/controllers/resource_manager.go +++ b/controllers/resource_manager.go @@ -23,15 +23,15 @@ import ( "sort" "strings" + secv1 "github.com/openshift/api/security/v1" promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" nodev1 "k8s.io/api/node/v1" rbacv1 "k8s.io/api/rbac/v1" + drav1 "k8s.io/api/resource/v1alpha3" schedv1 "k8s.io/api/scheduling/v1beta1" - secv1 "github.com/openshift/api/security/v1" - "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/client-go/kubernetes/scheme" ) @@ -61,6 +61,7 @@ type Resources struct { SecurityContextConstraints secv1.SecurityContextConstraints RuntimeClasses []nodev1.RuntimeClass PrometheusRule promv1.PrometheusRule + DeviceClasses []drav1.DeviceClass } func filePathWalkDir(n *ClusterPolicyController, root string) ([]string, error) { @@ -180,6 +181,15 @@ func addResourcesControls(n *ClusterPolicyController, path string) (Resources, c _, _, err := s.Decode(m, nil, &res.PrometheusRule) panicIfError(err) ctrl = append(ctrl, PrometheusRule) + case "DeviceClass": + deviceClass := drav1.DeviceClass{} + _, _, err := s.Decode(m, nil, &deviceClass) + panicIfError(err) + res.DeviceClasses = append(res.DeviceClasses, deviceClass) + // only add the ctrl function when the first DeviceClass is added + if len(res.DeviceClasses) == 1 { + ctrl = append(ctrl, DeviceClasses) + } default: n.logger.Info("Unknown Resource", "Manifest", m, "Kind", kind) } diff --git a/controllers/state_manager.go b/controllers/state_manager.go index 9c1028ebc..26e705d51 100644 --- a/controllers/state_manager.go +++ b/controllers/state_manager.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "github.com/go-logr/logr" @@ -794,6 +795,7 @@ func (n *ClusterPolicyController) init(ctx context.Context, reconciler *ClusterP addState(n, "/opt/gpu-operator/state-container-toolkit") addState(n, "/opt/gpu-operator/state-operator-validation") addState(n, "/opt/gpu-operator/state-device-plugin") + addState(n, "/opt/gpu-operator/state-dra-driver") addState(n, "/opt/gpu-operator/state-mps-control-daemon") addState(n, "/opt/gpu-operator/state-dcgm") addState(n, "/opt/gpu-operator/state-dcgm-exporter") @@ -1027,8 +1029,14 @@ func (n ClusterPolicyController) isStateEnabled(stateName string) bool { return true case "state-operator-metrics": return true + case "state-dra-driver": + return clusterPolicySpec.DRADriver.IsEnabled() default: n.logger.Error(nil, "invalid state passed", "stateName", stateName) return false } } + +func (n ClusterPolicyController) isDRADeviceClassEnabled(deviceClass string) bool { + return slices.Contains(n.singleton.Spec.DRADriver.DeviceClasses, deviceClass) +} diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index 83b504b7e..240f3fefe 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -137,6 +137,36 @@ func (p Pod) WithRuntimeClassName(name string) Pod { return p } +// deployment is a Deployment wrapper used for testing +type deployment struct { + *appsv1.Deployment +} + +func NewDeployment() deployment { + d := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ds", + Namespace: "test-ns", + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{}, + }, + }, + } + return deployment{d} +} + +func (d deployment) WithContainer(container corev1.Container) deployment { + d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, container) + return d +} + +func (d deployment) WithPullSecret(secret string) deployment { + d.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: secret}} + return d +} + func TestTransformForHostRoot(t *testing.T) { hostRootVolumeName := "host-root" hostDevCharVolumeName := "host-dev-char" @@ -1163,3 +1193,144 @@ func TestTransformSandboxValidator(t *testing.T) { }) } } + +func TestTransformIMEXDRADriverPlugin(t *testing.T) { + testCases := []struct { + description string + ds Daemonset + cpSpec *gpuv1.ClusterPolicySpec + expectedDs Daemonset + errorExpected bool + }{ + { + description: "empty draDriver spec", + ds: NewDaemonset(). + WithInitContainer(corev1.Container{Name: "dummy"}). + WithContainer(corev1.Container{Name: "dummy"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + DRADriver: gpuv1.DRADriverSpec{}, + }, + expectedDs: NewDaemonset(), + errorExpected: true, + }, + { + description: "valid draDriver spec, toolkit disabled", + ds: NewDaemonset(). + WithInitContainer(corev1.Container{Name: "dummy"}). + WithContainer(corev1.Container{Name: "dummy"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + DRADriver: gpuv1.DRADriverSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "k8s-dra-driver", + Version: "v1.0.0", + }, + Toolkit: gpuv1.ToolkitSpec{ + Enabled: newBoolPtr(false), + }, + }, + expectedDs: NewDaemonset(). + WithInitContainer(corev1.Container{Name: "dummy"}). + WithContainer(corev1.Container{ + Name: "dummy", + Image: "nvcr.io/nvidia/cloud-native/k8s-dra-driver:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + }), + }, + { + description: "valid draDriver spec, toolkit enabled", + ds: NewDaemonset(). + WithInitContainer(corev1.Container{Name: "dummy"}). + WithContainer(corev1.Container{Name: "dummy"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + DRADriver: gpuv1.DRADriverSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "k8s-dra-driver", + Version: "v1.0.0", + ImagePullPolicy: "Always", + ImagePullSecrets: []string{"pull-secret"}, + }, + Toolkit: gpuv1.ToolkitSpec{ + Enabled: newBoolPtr(true), + InstallDir: "/usr/local/nvidia", + }, + }, + expectedDs: NewDaemonset(). + WithInitContainer(corev1.Container{Name: "dummy"}). + WithContainer(corev1.Container{ + Name: "dummy", + Image: "nvcr.io/nvidia/cloud-native/k8s-dra-driver:v1.0.0", + ImagePullPolicy: corev1.PullAlways, + Env: []corev1.EnvVar{ + {Name: NvidiaCTKPathEnvName, Value: "/usr/local/nvidia/toolkit/nvidia-ctk"}, + }, + }). + WithPullSecret("pull-secret"), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + err := TransformIMEXDRADriverPlugin(tc.ds.DaemonSet, tc.cpSpec, ClusterPolicyController{logger: ctrl.Log.WithName("test")}) + if tc.errorExpected { + require.Error(t, err) + return + } + require.NoError(t, err) + require.EqualValues(t, tc.expectedDs, tc.ds) + }) + } +} + +func TestTransformIMEXDRADriverController(t *testing.T) { + testCases := []struct { + description string + deployment deployment + cpSpec *gpuv1.ClusterPolicySpec + expectedDeployment deployment + errorExpected bool + }{ + { + description: "empty draDriver spec", + deployment: NewDeployment(). + WithContainer(corev1.Container{Name: "dummy"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + DRADriver: gpuv1.DRADriverSpec{}, + }, + expectedDeployment: NewDeployment(), + errorExpected: true, + }, + { + description: "valid draDriver spec", + deployment: NewDeployment(). + WithContainer(corev1.Container{Name: "dummy"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + DRADriver: gpuv1.DRADriverSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "k8s-dra-driver", + Version: "v1.0.0", + ImagePullPolicy: "Always", + ImagePullSecrets: []string{"pull-secret"}, + }, + }, + expectedDeployment: NewDeployment(). + WithContainer(corev1.Container{ + Name: "dummy", + Image: "nvcr.io/nvidia/cloud-native/k8s-dra-driver:v1.0.0", + ImagePullPolicy: corev1.PullAlways, + }). + WithPullSecret("pull-secret"), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + err := TransformIMEXDRADriverController(tc.deployment.Deployment, tc.cpSpec) + if tc.errorExpected { + require.Error(t, err) + return + } + require.NoError(t, err) + require.EqualValues(t, tc.expectedDeployment, tc.deployment) + }) + } +} diff --git a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml index 54e4a652b..9e95b49ac 100644 --- a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml +++ b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml @@ -596,6 +596,38 @@ spec: description: NVIDIA Device Plugin image tag type: string type: object + draDriver: + description: DRADriver component spec + properties: + deviceClasses: + description: DeviceClasses indicates which device classes are + enabled in the DRA driver + items: + type: string + type: array + enabled: + description: Enabled indicates if the deployment of NVIDIA DRA + Driver through the operator is enabled + type: boolean + image: + description: NVIDIA DRA Driver image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA DRA Driver image repository + type: string + version: + description: NVIDIA DRA Driver image tag + type: string + type: object driver: description: Driver component spec properties: @@ -2296,6 +2328,7 @@ spec: - dcgm - dcgmExporter - devicePlugin + - draDriver - driver - gfd - nodeStatusExporter diff --git a/deployments/gpu-operator/templates/clusterpolicy.yaml b/deployments/gpu-operator/templates/clusterpolicy.yaml index af9e87c38..293dd1509 100644 --- a/deployments/gpu-operator/templates/clusterpolicy.yaml +++ b/deployments/gpu-operator/templates/clusterpolicy.yaml @@ -459,6 +459,26 @@ spec: name: {{ .Values.devicePlugin.config.name }} default: {{ .Values.devicePlugin.config.default }} {{- end }} + draDriver: + enabled: {{ .Values.draDriver.enabled }} + {{- if .Values.draDriver.repository }} + repository: {{ .Values.draDriver.repository }} + {{- end }} + {{- if .Values.draDriver.image }} + image: {{ .Values.draDriver.image }} + {{- end }} + {{- if .Values.draDriver.version }} + version: {{ .Values.draDriver.version | quote }} + {{- end }} + {{- if .Values.draDriver.imagePullPolicy }} + imagePullPolicy: {{ .Values.draDriver.imagePullPolicy }} + {{- end }} + {{- if .Values.draDriver.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.draDriver.imagePullSecrets | nindent 6 }} + {{- end }} + {{- if .Values.draDriver.deviceClasses }} + deviceClasses: {{ toYaml .Values.draDriver.imagePullSecrets | nindent 6 }} + {{- end }} dcgm: enabled: {{ .Values.dcgm.enabled }} {{- if .Values.dcgm.repository }} diff --git a/deployments/gpu-operator/templates/clusterrole.yaml b/deployments/gpu-operator/templates/clusterrole.yaml index 4acbcf29c..f25b5346b 100644 --- a/deployments/gpu-operator/templates/clusterrole.yaml +++ b/deployments/gpu-operator/templates/clusterrole.yaml @@ -144,3 +144,13 @@ rules: {{- if .Values.operator.cleanupCRD }} - delete {{- end }} +# TODO: restrict RBAC for DRA driver +- apiGroups: + - "" + - apps + - resource.k8s.io + - gpu.nvidia.com + resources: + - '*' + verbs: + - '*' diff --git a/deployments/gpu-operator/values.yaml b/deployments/gpu-operator/values.yaml index 5e404f081..50147787e 100644 --- a/deployments/gpu-operator/values.yaml +++ b/deployments/gpu-operator/values.yaml @@ -294,6 +294,16 @@ devicePlugin: # MPS root path on the host root: "/run/nvidia/mps" +draDriver: + enabled: true + repository: nvcr.io/nvidia/cloud-native + image: k8s-dra-driver + version: v0.1.0 + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + deviceClasses: + - imex + # standalone dcgm hostengine dcgm: # disabled by default to use embedded nv-hostengine by exporter From f1372640ebb5be75296e67d172e4162a3c419097 Mon Sep 17 00:00:00 2001 From: Christopher Desiniotis Date: Mon, 16 Dec 2024 12:05:43 -0800 Subject: [PATCH 2/2] Update to use resource v1beta1 API Signed-off-by: Christopher Desiniotis --- assets/state-dra-driver/0400_deviceclass-imex.yaml | 2 +- controllers/object_controls.go | 6 +++--- controllers/resource_manager.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/assets/state-dra-driver/0400_deviceclass-imex.yaml b/assets/state-dra-driver/0400_deviceclass-imex.yaml index 759476c85..77660085d 100644 --- a/assets/state-dra-driver/0400_deviceclass-imex.yaml +++ b/assets/state-dra-driver/0400_deviceclass-imex.yaml @@ -1,4 +1,4 @@ -apiVersion: resource.k8s.io/v1alpha3 +apiVersion: resource.k8s.io/v1beta1 kind: DeviceClass metadata: name: imex.nvidia.com diff --git a/controllers/object_controls.go b/controllers/object_controls.go index e9fbe8e6c..bd8cd6b28 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -41,7 +41,7 @@ import ( corev1 "k8s.io/api/core/v1" nodev1 "k8s.io/api/node/v1" nodev1beta1 "k8s.io/api/node/v1beta1" - drav1 "k8s.io/api/resource/v1alpha3" + resourceapi "k8s.io/api/resource/v1beta1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -4953,7 +4953,7 @@ func PrometheusRule(n ClusterPolicyController) (gpuv1.State, error) { return gpuv1.Ready, nil } -func createDeviceClass(n ClusterPolicyController, spec drav1.DeviceClass) (gpuv1.State, error) { +func createDeviceClass(n ClusterPolicyController, spec resourceapi.DeviceClass) (gpuv1.State, error) { ctx := n.ctx state := n.idx obj := spec.DeepCopy() @@ -4974,7 +4974,7 @@ func createDeviceClass(n ClusterPolicyController, spec drav1.DeviceClass) (gpuv1 return gpuv1.NotReady, err } - found := &drav1.DeviceClass{} + found := &resourceapi.DeviceClass{} err := n.client.Get(ctx, types.NamespacedName{Namespace: "", Name: obj.Name}, found) if err != nil && apierrors.IsNotFound(err) { logger.Info("Not found, creating...") diff --git a/controllers/resource_manager.go b/controllers/resource_manager.go index 26610300d..111de7dc2 100644 --- a/controllers/resource_manager.go +++ b/controllers/resource_manager.go @@ -29,7 +29,7 @@ import ( corev1 "k8s.io/api/core/v1" nodev1 "k8s.io/api/node/v1" rbacv1 "k8s.io/api/rbac/v1" - drav1 "k8s.io/api/resource/v1alpha3" + resourceapi "k8s.io/api/resource/v1beta1" schedv1 "k8s.io/api/scheduling/v1beta1" "k8s.io/apimachinery/pkg/runtime/serializer/json" @@ -61,7 +61,7 @@ type Resources struct { SecurityContextConstraints secv1.SecurityContextConstraints RuntimeClasses []nodev1.RuntimeClass PrometheusRule promv1.PrometheusRule - DeviceClasses []drav1.DeviceClass + DeviceClasses []resourceapi.DeviceClass } func filePathWalkDir(n *ClusterPolicyController, root string) ([]string, error) { @@ -182,7 +182,7 @@ func addResourcesControls(n *ClusterPolicyController, path string) (Resources, c panicIfError(err) ctrl = append(ctrl, PrometheusRule) case "DeviceClass": - deviceClass := drav1.DeviceClass{} + deviceClass := resourceapi.DeviceClass{} _, _, err := s.Decode(m, nil, &deviceClass) panicIfError(err) res.DeviceClasses = append(res.DeviceClasses, deviceClass)