Merge pull request #225 from klueska/add-multi-node-crd

Add ComputeDomain for running multi-node workloads

Author: klueska

.common-ci.yml

@@ -111,8 +111,8 @@ trigger-pipeline:
 .scan-base:
   stage: scan
   variables:
-    IMAGE: "${CI_REGISTRY_IMAGE}/k8s-dra-driver:${CI_COMMIT_SHORT_SHA}-${DIST}"
-    IMAGE_ARCHIVE: "k8s-dra-driver-${CI_COMMIT_SHORT_SHA}-${DIST}-${PLATFORM_ARCH}.tar"
+    IMAGE: "${CI_REGISTRY_IMAGE}/k8s-dra-driver-gpu:${CI_COMMIT_SHORT_SHA}-${DIST}"
+    IMAGE_ARCHIVE: "k8s-dra-driver-gpu-${CI_COMMIT_SHORT_SHA}-${DIST}-${PLATFORM_ARCH}.tar"
   before_script:
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
     - docker pull --platform="${PLATFORM}" "${IMAGE}"
@@ -160,7 +160,7 @@ scan-ubi9-arm64:
   stage: release
   variables:
     # Define the source image for the release
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/k8s-dra-driver"
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/k8s-dra-driver-gpu"
     VERSION: "${CI_COMMIT_SHORT_SHA}"
     # OUT_IMAGE_VERSION is overridden for external releases
     OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}"
@@ -195,7 +195,7 @@ scan-ubi9-arm64:
     OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
     OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
     OUT_REGISTRY: "${CI_REGISTRY}"
-    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/k8s-dra-driver"
+    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/k8s-dra-driver-gpu"
 
 # Define an external release step that pushes an image to an external repository.
 # This includes a devlopment image off main.

.github/workflows/image.yaml

@@ -56,7 +56,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build image
         env:
-          IMAGE_NAME: ghcr.io/${LOWERCASE_REPO_OWNER}/k8s-dra-driver
+          IMAGE_NAME: ghcr.io/${LOWERCASE_REPO_OWNER}/k8s-dra-driver-gpu
           VERSION: ${COMMIT_SHORT_SHA}
         run: |
           echo "${VERSION}"

.gitignore

@@ -1,7 +1,8 @@
 .cache/
 .bash_history
-/nvidia-dra-controller
-/nvidia-dra-plugin
+/compute-domain-controller
+/compute-domain-kubelet-plugin
+/gpu-kubelet-plugin
 .idea
 [._]*.sw[a-p]
 coverage.out

.gitlab-ci.yml

@@ -58,7 +58,7 @@ unit-tests:
 .image-build:
   stage: image-build
   variables:
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/k8s-dra-driver"
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/k8s-dra-driver-gpu"
     VERSION: "${CI_COMMIT_SHORT_SHA}"
     PUSH_ON_BUILD: "true"
   before_script:

.golangci.yaml

@@ -17,4 +17,4 @@ run:
 
 linters-settings:
   goimports:
-    local-prefixes: "github.com/NVIDIA/k8s-dra-driver"
+    local-prefixes: "github.com/NVIDIA/k8s-dra-driver-gpu"

.nvidia-ci.yml

@@ -33,19 +33,19 @@ variables:
   # On the multi-arch builder we don't need the qemu setup.
   SKIP_QEMU_SETUP: "1"
   # Define the public staging registry
-  STAGING_REGISTRY: registry.gitlab.com/nvidia/cloud-native/k8s-dra-driver/staging
+  STAGING_REGISTRY: registry.gitlab.com/nvidia/cloud-native/k8s-dra-driver-gpu/staging
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
 
 .image-pull:
   stage: image-build
   variables:
     IN_REGISTRY: "${STAGING_REGISTRY}"
-    IN_IMAGE_NAME: k8s-dra-driver
+    IN_IMAGE_NAME: k8s-dra-driver-gpu
     IN_VERSION: "${STAGING_VERSION}"
     OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
     OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
     OUT_REGISTRY: "${CI_REGISTRY}"
-    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/k8s-dra-driver"
+    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/k8s-dra-driver-gpu"
     PUSH_MULTIPLE_TAGS: "false"
   # We delay the job start to allow the public pipeline to generate the required images.
   when: delayed

Makefile

@@ -77,7 +77,7 @@ goimports:
 	find . -name \*.go \
 			-not -name "zz_generated.deepcopy.go" \
 			-not -path "./vendor/*" \
-			-not -path "./pkg/nvidia.com/resource/clientset/versioned/*" \
+			-not -path "./$(PKG_BASE)/clientset/versioned/*" \
 		-exec goimports -local $(MODULE) -w {} \;
 
 golangci-lint:
@@ -99,7 +99,19 @@ coverage: test
 	cat $(COVERAGE_FILE) | grep -v "_mock.go" > $(COVERAGE_FILE).no-mocks
 	go tool cover -func=$(COVERAGE_FILE).no-mocks
 
-generate: generate-deepcopy fmt
+generate: generate-crds generate-informers fmt
+
+generate-crds: generate-deepcopy .remove-crds
+	for dir in $(CLIENT_SOURCES); do \
+		controller-gen crd:crdVersions=v1 \
+			paths=$(CURDIR)/$${dir} \
+			output:crd:dir=$(CURDIR)/deployments/helm/tmp_crds; \
+	done
+	mkdir -p $(CURDIR)/deployments/helm/$(HELM_DRIVER_NAME)/crds
+	cp -R $(CURDIR)/deployments/helm/tmp_crds/* \
+		$(CURDIR)/deployments/helm/$(HELM_DRIVER_NAME)/crds
+	rm -rf $(CURDIR)/deployments/helm/tmp_crds
+
 
 generate-deepcopy: .remove-deepcopy
 	for dir in $(DEEPCOPY_SOURCES); do \
@@ -109,14 +121,66 @@ generate-deepcopy: .remove-deepcopy
 			output:object:dir=$(CURDIR)/$${dir}; \
 	done
 
+generate-informers: .remove-informers generate-listers
+	informer-gen \
+		--go-header-file=$(CURDIR)/hack/boilerplate.go.txt \
+		--output-package "$(MODULE)/$(PKG_BASE)/informers" \
+        --input-dirs "$(shell for api in $(CLIENT_APIS); do echo -n "$(MODULE)/$(API_BASE)/$$api,"; done | sed 's/,$$//')" \
+		--output-base "$(CURDIR)/pkg/tmp_informers" \
+		--versioned-clientset-package "$(MODULE)/$(PKG_BASE)/clientset/versioned" \
+		--listers-package "$(MODULE)/$(PKG_BASE)/listers"
+	mkdir -p $(CURDIR)/$(PKG_BASE)
+	mv $(CURDIR)/pkg/tmp_informers/$(MODULE)/$(PKG_BASE)/informers \
+	   $(CURDIR)/$(PKG_BASE)/informers
+	rm -rf $(CURDIR)/pkg/tmp_informers
+
+generate-listers: .remove-listers generate-clientset
+	lister-gen \
+		--go-header-file=$(CURDIR)/hack/boilerplate.go.txt \
+		--output-package "$(MODULE)/$(PKG_BASE)/listers" \
+        --input-dirs "$(shell for api in $(CLIENT_APIS); do echo -n "$(MODULE)/$(API_BASE)/$$api,"; done | sed 's/,$$//')" \
+		--output-base "$(CURDIR)/pkg/tmp_listers"
+	mkdir -p $(CURDIR)/$(PKG_BASE)
+	mv $(CURDIR)/pkg/tmp_listers/$(MODULE)/$(PKG_BASE)/listers \
+	   $(CURDIR)/$(PKG_BASE)/listers
+	rm -rf $(CURDIR)/pkg/tmp_listers
+
+generate-clientset: .remove-clientset
+	client-gen \
+		--go-header-file=$(CURDIR)/hack/boilerplate.go.txt \
+		--clientset-name "versioned" \
+		--build-tag "ignore_autogenerated" \
+		--output-package "$(MODULE)/$(PKG_BASE)/clientset" \
+		--input-base "$(MODULE)/$(API_BASE)" \
+		--output-base "$(CURDIR)/pkg/tmp_clientset" \
+		--input "$(shell echo $(CLIENT_APIS) | tr ' ' ',')" \
+		--plural-exceptions "$(shell echo $(PLURAL_EXCEPTIONS) | tr ' ' ',')"
+	mkdir -p $(CURDIR)/$(PKG_BASE)
+	mv $(CURDIR)/pkg/tmp_clientset/$(MODULE)/$(PKG_BASE)/clientset \
+       $(CURDIR)/$(PKG_BASE)/clientset
+	rm -rf $(CURDIR)/pkg/tmp_clientset
+
+.remove-crds:
+	rm -rf $(CURDIR)/deployments/helm/$(HELM_DRIVER_NAME)/crds
+
 .remove-deepcopy:
 	for dir in $(DEEPCOPY_SOURCES); do \
 		rm -f $(CURDIR)/$${dir}/zz_generated.deepcopy.go; \
 	done
 
+.remove-clientset:
+	rm -rf $(CURDIR)/$(PKG_BASE)/clientset
+
+.remove-listers:
+	rm -rf $(CURDIR)/$(PKG_BASE)/listers
+
+.remove-informers:
+	rm -rf $(CURDIR)/$(PKG_BASE)/informers
+
 # Generate an image for containerized builds
 # Note: This image is local only
-.PHONY: .build-image
+.PHONY: .build-image build-image
+build-image: .build-image
 .build-image:
 	make -f deployments/devel/Makefile .build-image
 

README.md

@@ -46,10 +46,10 @@ subdirectory, so take a moment to browse through the various files and see
 what's available:
 
 ```console
-git clone https://github.com/NVIDIA/k8s-dra-driver.git
+git clone https://github.com/NVIDIA/k8s-dra-driver-gpu.git
 ```
 ```console
-cd k8s-dra-driver
+cd k8s-dra-driver-gpu
 ```
 
 ### Setting up the infrastructure
@@ -76,9 +76,8 @@ This should show two pods running in the `nvidia` namespace:
 kubectl get pods -n nvidia
 ```
 ```
-NAME                                                          READY   STATUS    RESTARTS   AGE
-nvidia-dra-driver-k8s-dra-driver-controller-844fcb94b-ktbkc   1/1     Running   0          69s
-nvidia-dra-driver-k8s-dra-driver-kubelet-plugin-5vfp9         1/1     Running   0          69s
+NAME                                            READY   STATUS    RESTARTS   AGE
+k8s-dra-driver-gpu-kubelet-plugin-5vfp9         1/1     Running   0          69s
 ```
 
 ### Run the examples by following the steps in the demo script
@@ -144,10 +143,10 @@ This may include the following content from the original scripts:
 ```
 set -e
 
-export VERSION=v0.1.0
+export VERSION=v25.2.0
 
 REGISTRY=nvcr.io/nvidia/cloud-native
-IMAGE=k8s-dra-driver
+IMAGE=k8s-dra-driver-gpu
 PLATFORM=ubi9
 
 sudo true

api/nvidia.com/resource/gpu/v1alpha1/imexchannelconfig.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package v1alpha1
+package v1beta1
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -24,12 +24,14 @@ import (
 )
 
 const (
-	GroupName = "gpu.nvidia.com"
-	Version   = "v1alpha1"
+	GroupName = "resource.nvidia.com"
+	Version   = "v1beta1"
 
-	GpuConfigKind         = "GpuConfig"
-	MigDeviceConfigKind   = "MigDeviceConfig"
-	ImexChannelConfigKind = "ImexChannelConfig"
+	GpuConfigKind                  = "GpuConfig"
+	MigDeviceConfigKind            = "MigDeviceConfig"
+	ComputeDomainChannelConfigKind = "ComputeDomainChannelConfig"
+	ComputeDomainDaemonConfigKind  = "ComputeDomainDaemonConfig"
+	ComputeDomainKind              = "ComputeDomain"
 )
 
 // Interface defines the set of common APIs for all configs
@@ -55,7 +57,9 @@ func init() {
 	scheme.AddKnownTypes(schemeGroupVersion,
 		&GpuConfig{},
 		&MigDeviceConfig{},
-		&ImexChannelConfig{},
+		&ComputeDomainChannelConfig{},
+		&ComputeDomainDaemonConfig{},
+		&ComputeDomain{},
 	)
 	metav1.AddToGroupVersion(scheme, schemeGroupVersion)
 

api/nvidia.com/resource/gpu/v1alpha1/api.go api/nvidia.com/resource/v1beta1/api.go

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package v1beta1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	ComputeDomainStatusReady    = "Ready"
+	ComputeDomainStatusNotReady = "NotReady"
+)
+
+// +genclient
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +k8s:openapi-gen=true
+// +kubebuilder:resource:scope=Namespaced
+// +kubebuilder:subresource:status
+
+// ComputeDomain prepares a set of nodes to run a multi-node workload in.
+type ComputeDomain struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   ComputeDomainSpec   `json:"spec,omitempty"`
+	Status ComputeDomainStatus `json:"status,omitempty"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// ComputeDomainList provides a list of ComputeDomains.
+type ComputeDomainList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+
+	Items []ComputeDomain `json:"items"`
+}
+
+// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="A computeDomain.spec is immutable"
+
+// ComputeDomainSpec provides the spec for a ComputeDomain.
+type ComputeDomainSpec struct {
+	NumNodes int                       `json:"numNodes"`
+	Channel  *ComputeDomainChannelSpec `json:"channel"`
+}
+
+// ComputeDomainChannelSpec provides the spec for a channel used to run a workload inside a ComputeDomain.
+type ComputeDomainChannelSpec struct {
+	ResourceClaimTemplate ComputeDomainResourceClaimTemplate `json:"resourceClaimTemplate"`
+}
+
+// ComputeDomainResourceClaimTemplate provides the details of the ResourceClaimTemplate to generate.
+type ComputeDomainResourceClaimTemplate struct {
+	Name string `json:"name"`
+}
+
+// ComputeDomainStatus provides the status for a ComputeDomain.
+type ComputeDomainStatus struct {
+	// +kubebuilder:validation:Enum=Ready;NotReady
+	// +kubebuilder:default=NotReady
+	Status string `json:"status"`
+	// +listType=map
+	// +listMapKey=name
+	Nodes []*ComputeDomainNode `json:"nodes,omitempty"`
+}
+
+// ComputeDomainNode provides information about each node added to a ComputeDomain.
+type ComputeDomainNode struct {
+	Name      string `json:"name"`
+	IPAddress string `json:"ipAddress"`
+	CliqueID  string `json:"cliqueID"`
+}