feat: add managed NVIDIA GPU experience to AKSNodeClass

charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml

@@ -996,7 +996,33 @@ spec:
                     - Driver
                     - None
                     type: string
+                  nvidia:
+                    description: |-
+                      nvidia contains NVIDIA-specific GPU settings, such as the managed GPU
+                      experience. Ignored for non-NVIDIA VM sizes.
+                    properties:
+                      managementMode:
+                        description: |-
+                          managementMode toggles the managed GPU experience for NVIDIA GPUs.
+                          When set to Managed, AKS installs additional components (e.g. DCGM metrics
+                          and the NVIDIA device plugin) on top of the GPU driver. When set to
+                          Unmanaged (or not specified), only the GPU driver is installed and managing
+                          the device plugin/metrics is the user's responsibility.
+                          Managed requires gpu.mode to be Driver (driver installation enabled) and is
+                          only supported on NVIDIA GPU SKUs running Linux.
+                          For more details of what is installed, see aka.ms/aks/managed-gpu.
+                        enum:
+                        - Managed
+                        - Unmanaged
+                        type: string
+                    type: object
                 type: object
+                x-kubernetes-validations:
+                - message: gpu.nvidia.managementMode 'Managed' requires gpu.mode to
+                    be 'Driver' (the managed GPU experience requires driver installation)
+                  rule: '!(has(self.nvidia) && has(self.nvidia.managementMode) &&
+                    self.nvidia.managementMode == ''Managed'' && has(self.mode) &&
+                    self.mode == ''None'')'
               imageFamily:
                 default: Ubuntu
                 description: imageFamily is the image family that instances use.

pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml

@@ -996,7 +996,33 @@ spec:
                     - Driver
                     - None
                     type: string
+                  nvidia:
+                    description: |-
+                      nvidia contains NVIDIA-specific GPU settings, such as the managed GPU
+                      experience. Ignored for non-NVIDIA VM sizes.
+                    properties:
+                      managementMode:
+                        description: |-
+                          managementMode toggles the managed GPU experience for NVIDIA GPUs.
+                          When set to Managed, AKS installs additional components (e.g. DCGM metrics
+                          and the NVIDIA device plugin) on top of the GPU driver. When set to
+                          Unmanaged (or not specified), only the GPU driver is installed and managing
+                          the device plugin/metrics is the user's responsibility.
+                          Managed requires gpu.mode to be Driver (driver installation enabled) and is
+                          only supported on NVIDIA GPU SKUs running Linux.
+                          For more details of what is installed, see aka.ms/aks/managed-gpu.
+                        enum:
+                        - Managed
+                        - Unmanaged
+                        type: string
+                    type: object
                 type: object
+                x-kubernetes-validations:
+                - message: gpu.nvidia.managementMode 'Managed' requires gpu.mode to
+                    be 'Driver' (the managed GPU experience requires driver installation)
+                  rule: '!(has(self.nvidia) && has(self.nvidia.managementMode) &&
+                    self.nvidia.managementMode == ''Managed'' && has(self.mode) &&
+                    self.mode == ''None'')'
               imageFamily:
                 default: Ubuntu
                 description: imageFamily is the image family that instances use.

pkg/apis/v1beta1/aksnodeclass.go

@@ -330,7 +330,38 @@ const (
 	GPUModeNone GPUMode = "None"
 )
 
+// +kubebuilder:validation:Enum:={Managed,Unmanaged}
+type ManagementMode string
+
+const (
+	// ManagementModeManaged enables the managed GPU experience: in addition to the
+	// GPU driver, AKS installs and manages additional components such as the
+	// Data Center GPU Manager (DCGM) metrics exporter and the NVIDIA device plugin.
+	// Requires GPU driver installation (gpu.mode must not be None) and is only
+	// supported on NVIDIA GPU SKUs. For details see aka.ms/aks/managed-gpu.
+	ManagementModeManaged ManagementMode = "Managed"
+	// ManagementModeUnmanaged disables the managed GPU experience. Only the GPU
+	// driver is installed (subject to gpu.mode); device plugin and metrics are
+	// the user's responsibility. This is the default when nvidia is unset.
+	ManagementModeUnmanaged ManagementMode = "Unmanaged"
+)
+
+// NvidiaGPU contains NVIDIA-specific GPU settings.
+type NvidiaGPU struct {
+	// managementMode toggles the managed GPU experience for NVIDIA GPUs.
+	// When set to Managed, AKS installs additional components (e.g. DCGM metrics
+	// and the NVIDIA device plugin) on top of the GPU driver. When set to
+	// Unmanaged (or not specified), only the GPU driver is installed and managing
+	// the device plugin/metrics is the user's responsibility.
+	// Managed requires gpu.mode to be Driver (driver installation enabled) and is
+	// only supported on NVIDIA GPU SKUs running Linux.
+	// For more details of what is installed, see aka.ms/aks/managed-gpu.
+	// +optional
+	ManagementMode *ManagementMode `json:"managementMode,omitempty"`
+}
+
 // GPU contains configuration for GPU-enabled nodes.
+// +kubebuilder:validation:XValidation:message="gpu.nvidia.managementMode 'Managed' requires gpu.mode to be 'Driver' (the managed GPU experience requires driver installation)",rule="!(has(self.nvidia) && has(self.nvidia.managementMode) && self.nvidia.managementMode == 'Managed' && has(self.mode) && self.mode == 'None')"
 type GPU struct {
 	// mode controls GPU driver management on GPU-enabled nodes.
 	// When set to Driver (or not specified), GPU drivers are installed by AKS
@@ -343,6 +374,10 @@ type GPU struct {
 	// +default="Driver"
 	// +optional
 	Mode *GPUMode `json:"mode,omitempty"`
+	// nvidia contains NVIDIA-specific GPU settings, such as the managed GPU
+	// experience. Ignored for non-NVIDIA VM sizes.
+	// +optional
+	Nvidia *NvidiaGPU `json:"nvidia,omitempty"`
 }
 
 // KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes.
@@ -795,3 +830,19 @@ func (in *AKSNodeClass) GetGPUMode() GPUMode {
 func (in *AKSNodeClass) IsGPUDriverInstallationEnabled() bool {
 	return in.GetGPUMode() != GPUModeNone
 }
+
+// GetManagementMode returns the effective NVIDIA GPU management mode.
+// Defaults to Unmanaged when gpu, gpu.nvidia, or gpu.nvidia.managementMode is
+// nil, preserving backward compatibility (existing GPU nodes are unmanaged).
+func (in *AKSNodeClass) GetManagementMode() ManagementMode {
+	if in.Spec.GPU == nil || in.Spec.GPU.Nvidia == nil || in.Spec.GPU.Nvidia.ManagementMode == nil {
+		return ManagementModeUnmanaged
+	}
+	return *in.Spec.GPU.Nvidia.ManagementMode
+}
+
+// IsManagedGPUEnabled returns whether the managed NVIDIA GPU experience is
+// enabled (gpu.nvidia.managementMode == Managed). Returns false when unset.
+func (in *AKSNodeClass) IsManagedGPUEnabled() bool {
+	return in.GetManagementMode() == ManagementModeManaged
+}

pkg/apis/v1beta1/aksnodeclass_hash_test.go

@@ -99,7 +99,18 @@ var _ = Describe("Hash", func() {
 		Entry("LocalDNS.VnetDNSOverrides.CacheDuration", v1beta1.AKSNodeClass{Spec: v1beta1.AKSNodeClassSpec{LocalDNS: &v1beta1.LocalDNS{VnetDNSOverrides: []v1beta1.LocalDNSZoneOverride{{Zone: "example.com", CacheDuration: karpv1.MustParseNillableDuration("2h")}}}}}),
 		Entry("LocalDNS.VnetDNSOverrides.ServeStaleDuration", v1beta1.AKSNodeClass{Spec: v1beta1.AKSNodeClassSpec{LocalDNS: &v1beta1.LocalDNS{VnetDNSOverrides: []v1beta1.LocalDNSZoneOverride{{Zone: "example.com", ServeStaleDuration: karpv1.MustParseNillableDuration("1h")}}}}}),
 		Entry("ArtifactStreaming.Enabled", v1beta1.AKSNodeClass{Spec: v1beta1.AKSNodeClassSpec{ArtifactStreaming: &v1beta1.ArtifactStreaming{Enabled: lo.ToPtr(true)}}}),
+		Entry("GPU.Nvidia.ManagementMode", v1beta1.AKSNodeClass{Spec: v1beta1.AKSNodeClassSpec{GPU: &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeManaged)}}}}),
 	)
+	It("should not change hash when gpu.nvidia is nil (non-breaking field addition)", func() {
+		// Adding the nvidia field must not alter the hash of NodeClasses that do
+		// not set it, so existing GPU nodes do not drift/roll on upgrade.
+		hash := nodeClass.Hash()
+		nodeClass.Spec.GPU = &v1beta1.GPU{Mode: lo.ToPtr(v1beta1.GPUModeDriver)}
+		hashWithGPUNoNvidia := nodeClass.Hash()
+		nodeClass.Spec.GPU.Nvidia = nil
+		Expect(nodeClass.Hash()).To(Equal(hashWithGPUNoNvidia))
+		_ = hash
+	})
 	It("should not change hash when tags are re-ordered", func() {
 		hash := nodeClass.Hash()
 		nodeClass.Spec.Tags = map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"}

pkg/apis/v1beta1/crd_validation_cel_test.go

@@ -765,6 +765,84 @@ var _ = Describe("CEL/Validation", func() {
 			}
 			Expect(env.Client.Create(ctx, nodeClass)).To(Succeed())
 		})
+		It("should accept gpu.nvidia.managementMode set to Managed (default Driver mode)", func() {
+			managed := v1beta1.ManagementModeManaged
+			nodeClass := &v1beta1.AKSNodeClass{
+				ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())},
+				Spec: v1beta1.AKSNodeClassSpec{
+					GPU: &v1beta1.GPU{
+						Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &managed},
+					},
+				},
+			}
+			Expect(env.Client.Create(ctx, nodeClass)).To(Succeed())
+		})
+		It("should accept gpu.nvidia.managementMode set to Unmanaged", func() {
+			unmanaged := v1beta1.ManagementModeUnmanaged
+			nodeClass := &v1beta1.AKSNodeClass{
+				ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())},
+				Spec: v1beta1.AKSNodeClassSpec{
+					GPU: &v1beta1.GPU{
+						Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &unmanaged},
+					},
+				},
+			}
+			Expect(env.Client.Create(ctx, nodeClass)).To(Succeed())
+		})
+		It("should accept gpu.nvidia.managementMode=Managed with mode explicitly Driver", func() {
+			managed := v1beta1.ManagementModeManaged
+			driverMode := v1beta1.GPUModeDriver
+			nodeClass := &v1beta1.AKSNodeClass{
+				ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())},
+				Spec: v1beta1.AKSNodeClassSpec{
+					GPU: &v1beta1.GPU{
+						Mode:   &driverMode,
+						Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &managed},
+					},
+				},
+			}
+			Expect(env.Client.Create(ctx, nodeClass)).To(Succeed())
+		})
+		It("should reject gpu.nvidia.managementMode=Managed when mode is None", func() {
+			managed := v1beta1.ManagementModeManaged
+			noneMode := v1beta1.GPUModeNone
+			nodeClass := &v1beta1.AKSNodeClass{
+				ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())},
+				Spec: v1beta1.AKSNodeClassSpec{
+					GPU: &v1beta1.GPU{
+						Mode:   &noneMode,
+						Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &managed},
+					},
+				},
+			}
+			Expect(env.Client.Create(ctx, nodeClass)).ToNot(Succeed())
+		})
+		It("should accept gpu.nvidia.managementMode=Unmanaged with mode None", func() {
+			unmanaged := v1beta1.ManagementModeUnmanaged
+			noneMode := v1beta1.GPUModeNone
+			nodeClass := &v1beta1.AKSNodeClass{
+				ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())},
+				Spec: v1beta1.AKSNodeClassSpec{
+					GPU: &v1beta1.GPU{
+						Mode:   &noneMode,
+						Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &unmanaged},
+					},
+				},
+			}
+			Expect(env.Client.Create(ctx, nodeClass)).To(Succeed())
+		})
+		It("should reject invalid gpu.nvidia.managementMode value", func() {
+			invalid := v1beta1.ManagementMode("Invalid")
+			nodeClass := &v1beta1.AKSNodeClass{
+				ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())},
+				Spec: v1beta1.AKSNodeClassSpec{
+					GPU: &v1beta1.GPU{
+						Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &invalid},
+					},
+				},
+			}
+			Expect(env.Client.Create(ctx, nodeClass)).ToNot(Succeed())
+		})
 	})
 
 	Context("Requirements", func() {

pkg/cloudprovider/suite_aksmachineapi_features_test.go

@@ -303,6 +303,84 @@ var _ = Describe("CloudProvider", func() {
 			})
 		})
 
+		// Managed NVIDIA GPU experience (gpu.nvidia.managementMode) on the AKS machine API path.
+		Context("Create - Managed NVIDIA GPU", func() {
+			scheduleGPUPod := func() {
+				pod := coretest.UnschedulablePod(coretest.PodOptions{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:   "managed-gpu",
+						Labels: map[string]string{"app": "managed-gpu"},
+					},
+					Image: "mcr.microsoft.com/azuredocs/samples-tf-mnist-demo:gpu",
+					ResourceRequirements: v1.ResourceRequirements{
+						Limits: v1.ResourceList{"nvidia.com/gpu": resource.MustParse("1")},
+					},
+					RestartPolicy: v1.RestartPolicy("OnFailure"),
+					Tolerations: []v1.Toleration{{
+						Key:      "sku",
+						Operator: v1.TolerationOpEqual,
+						Value:    "gpu",
+						Effect:   v1.TaintEffectNoSchedule,
+					}},
+				})
+				ExpectProvisionedAndWaitForPromises(ctx, env.Client, cluster, cloudProvider, coreProvisioner, azureEnv, pod)
+				ExpectScheduled(ctx, env.Client, pod)
+			}
+
+			popCreatedMachine := func() armcontainerservice.Machine {
+				Expect(azureEnv.AKSMachinesAPI.AKSMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1))
+				createInput := azureEnv.AKSMachinesAPI.AKSMachineCreateOrUpdateBehavior.CalledWithInput.Pop()
+				aksMachine := createInput.AKSMachine
+				Expect(aksMachine.Properties).ToNot(BeNil())
+				Expect(aksMachine.Properties.Hardware).ToNot(BeNil())
+				return aksMachine
+			}
+
+			It("should set GpuProfile.Nvidia.ManagementMode=Managed for a Managed NVIDIA GPU NodeClass", func() {
+				nodeClass.Spec.GPU = &v1beta1.GPU{
+					Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeManaged)},
+				}
+				ExpectApplied(ctx, env.Client, nodePool, nodeClass)
+				ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass)
+				scheduleGPUPod()
+
+				aksMachine := popCreatedMachine()
+				Expect(utils.IsNvidiaEnabledSKU(lo.FromPtr(aksMachine.Properties.Hardware.VMSize))).To(BeTrue())
+				gpuProfile := aksMachine.Properties.Hardware.GpuProfile
+				Expect(gpuProfile).ToNot(BeNil())
+				Expect(lo.FromPtr(gpuProfile.Driver)).To(Equal(armcontainerservice.GPUDriverInstall))
+				Expect(gpuProfile.Nvidia).ToNot(BeNil())
+				Expect(lo.FromPtr(gpuProfile.Nvidia.ManagementMode)).To(Equal(armcontainerservice.ManagementModeManaged))
+			})
+
+			It("should not set GpuProfile.Nvidia when management mode is unset (unmanaged default, non-breaking)", func() {
+				ExpectApplied(ctx, env.Client, nodePool, nodeClass)
+				ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass)
+				scheduleGPUPod()
+
+				aksMachine := popCreatedMachine()
+				Expect(utils.IsNvidiaEnabledSKU(lo.FromPtr(aksMachine.Properties.Hardware.VMSize))).To(BeTrue())
+				gpuProfile := aksMachine.Properties.Hardware.GpuProfile
+				Expect(gpuProfile).ToNot(BeNil())
+				Expect(lo.FromPtr(gpuProfile.Driver)).To(Equal(armcontainerservice.GPUDriverInstall))
+				Expect(gpuProfile.Nvidia).To(BeNil())
+			})
+
+			It("should not set GpuProfile.Nvidia when management mode is explicitly Unmanaged", func() {
+				nodeClass.Spec.GPU = &v1beta1.GPU{
+					Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeUnmanaged)},
+				}
+				ExpectApplied(ctx, env.Client, nodePool, nodeClass)
+				ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass)
+				scheduleGPUPod()
+
+				aksMachine := popCreatedMachine()
+				gpuProfile := aksMachine.Properties.Hardware.GpuProfile
+				Expect(gpuProfile).ToNot(BeNil())
+				Expect(gpuProfile.Nvidia).To(BeNil())
+			})
+		})
+
 		// Ported from VM test: Context "additional-tags"
 		Context("Create - Additional Tags", func() {
 			It("should add additional tags to the AKS machine", func() {

pkg/providers/instance/aksmachineinstancehelpers.go

@@ -188,9 +188,25 @@ func configureGPUProfile(instanceType *corecloudprovider.InstanceType, nodeClass
 	if nodeClass.IsGPUDriverInstallationEnabled() {
 		driverSetting = armcontainerservice.GPUDriverInstall
 	}
-	return &armcontainerservice.GPUProfile{
+	gpuProfile := &armcontainerservice.GPUProfile{
 		Driver: lo.ToPtr(driverSetting),
 	}
+	// Managed GPU experience: AKS installs additional components (DCGM metrics,
+	// NVIDIA device plugin) on top of the driver. It is NVIDIA-only and requires
+	// driver installation. Only emit Nvidia settings when the user explicitly
+	// opts in via gpu.nvidia.managementMode=Managed; leaving Nvidia nil keeps the
+	// request unmanaged, which is the non-breaking default (RP treats nil as
+	// Unmanaged). The SKU/driver guards avoid sending Managed for SKUs the RP
+	// would reject (e.g. AMD GPUs or driver=None), which the API-level CEL rule
+	// and instance-type filtering already prevent, but we re-check defensively.
+	if nodeClass.IsManagedGPUEnabled() &&
+		utils.IsNvidiaEnabledSKU(instanceType.Name) &&
+		nodeClass.IsGPUDriverInstallationEnabled() {
+		gpuProfile.Nvidia = &armcontainerservice.NvidiaGPUProfile{
+			ManagementMode: lo.ToPtr(armcontainerservice.ManagementModeManaged),
+		}
+	}
+	return gpuProfile
 }
 
 func configureArtifactStreamingProfile(nodeClass *v1beta1.AKSNodeClass, instanceType *corecloudprovider.InstanceType) *armcontainerservice.AgentPoolArtifactStreamingProfile {