From 2a85e8768864be4971bde72abaae809787aad803 Mon Sep 17 00:00:00 2001 From: muribefalcon Date: Thu, 25 Jun 2026 11:33:23 -0700 Subject: [PATCH 1/5] feat: add managed NVIDIA GPU experience to AKSNodeClass Add gpu.nvidia.managementMode (Managed|Unmanaged) to the v1beta1 AKSNodeClass and forward it to the AKS machine API GPU profile, allowing GPU NodePools to opt into the managed NVIDIA GPU experience (AKS installs and manages the NVIDIA device plugin and DCGM metrics on top of the GPU driver) instead of managing those components themselves. The field is optional and defaults to Unmanaged, so existing GPU NodeClasses are unaffected (no hash/version change). Managed requires driver installation (gpu.mode != None) and applies only to NVIDIA GPU SKUs; both are enforced via a CEL rule and a defensive guard in the machine builder. --- .../karpenter.azure.com_aksnodeclasses.yaml | 26 +++++++ .../karpenter.azure.com_aksnodeclasses.yaml | 26 +++++++ pkg/apis/v1beta1/aksnodeclass.go | 51 ++++++++++++ pkg/apis/v1beta1/aksnodeclass_gpu_test.go | 59 ++++++++++++++ pkg/apis/v1beta1/aksnodeclass_hash_test.go | 11 +++ pkg/apis/v1beta1/crd_validation_cel_test.go | 78 +++++++++++++++++++ pkg/apis/v1beta1/zz_generated.deepcopy.go | 25 ++++++ .../instance/aksmachineinstancehelpers.go | 18 ++++- .../aksmachineinstancehelpers_test.go | 47 +++++++++++ 9 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 pkg/apis/v1beta1/aksnodeclass_gpu_test.go diff --git a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml index fe560ce078..313db7a84b 100644 --- a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml +++ b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml @@ -996,7 +996,33 @@ spec: - Driver - None type: string + nvidia: + description: |- + nvidia contains NVIDIA-specific GPU settings, such as the managed GPU + experience. Ignored for non-NVIDIA VM sizes. + properties: + managementMode: + description: |- + managementMode toggles the managed GPU experience for NVIDIA GPUs. + When set to Managed, AKS installs additional components (e.g. DCGM metrics + and the NVIDIA device plugin) on top of the GPU driver. When set to + Unmanaged (or not specified), only the GPU driver is installed and managing + the device plugin/metrics is the user's responsibility. + Managed requires gpu.mode to be Driver (driver installation enabled) and is + only supported on NVIDIA GPU SKUs running Linux. + For more details of what is installed, see aka.ms/aks/managed-gpu. + enum: + - Managed + - Unmanaged + type: string + type: object type: object + x-kubernetes-validations: + - message: gpu.nvidia.managementMode 'Managed' requires gpu.mode to + be 'Driver' (the managed GPU experience requires driver installation) + rule: '!(has(self.nvidia) && has(self.nvidia.managementMode) && + self.nvidia.managementMode == ''Managed'' && has(self.mode) && + self.mode == ''None'')' imageFamily: default: Ubuntu description: imageFamily is the image family that instances use. diff --git a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml index fe560ce078..313db7a84b 100644 --- a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml +++ b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml @@ -996,7 +996,33 @@ spec: - Driver - None type: string + nvidia: + description: |- + nvidia contains NVIDIA-specific GPU settings, such as the managed GPU + experience. Ignored for non-NVIDIA VM sizes. + properties: + managementMode: + description: |- + managementMode toggles the managed GPU experience for NVIDIA GPUs. + When set to Managed, AKS installs additional components (e.g. DCGM metrics + and the NVIDIA device plugin) on top of the GPU driver. When set to + Unmanaged (or not specified), only the GPU driver is installed and managing + the device plugin/metrics is the user's responsibility. + Managed requires gpu.mode to be Driver (driver installation enabled) and is + only supported on NVIDIA GPU SKUs running Linux. + For more details of what is installed, see aka.ms/aks/managed-gpu. + enum: + - Managed + - Unmanaged + type: string + type: object type: object + x-kubernetes-validations: + - message: gpu.nvidia.managementMode 'Managed' requires gpu.mode to + be 'Driver' (the managed GPU experience requires driver installation) + rule: '!(has(self.nvidia) && has(self.nvidia.managementMode) && + self.nvidia.managementMode == ''Managed'' && has(self.mode) && + self.mode == ''None'')' imageFamily: default: Ubuntu description: imageFamily is the image family that instances use. diff --git a/pkg/apis/v1beta1/aksnodeclass.go b/pkg/apis/v1beta1/aksnodeclass.go index ce299caf16..14410f1b0c 100644 --- a/pkg/apis/v1beta1/aksnodeclass.go +++ b/pkg/apis/v1beta1/aksnodeclass.go @@ -330,7 +330,38 @@ const ( GPUModeNone GPUMode = "None" ) +// +kubebuilder:validation:Enum:={Managed,Unmanaged} +type ManagementMode string + +const ( + // ManagementModeManaged enables the managed GPU experience: in addition to the + // GPU driver, AKS installs and manages additional components such as the + // Data Center GPU Manager (DCGM) metrics exporter and the NVIDIA device plugin. + // Requires GPU driver installation (gpu.mode must not be None) and is only + // supported on NVIDIA GPU SKUs. For details see aka.ms/aks/managed-gpu. + ManagementModeManaged ManagementMode = "Managed" + // ManagementModeUnmanaged disables the managed GPU experience. Only the GPU + // driver is installed (subject to gpu.mode); device plugin and metrics are + // the user's responsibility. This is the default when nvidia is unset. + ManagementModeUnmanaged ManagementMode = "Unmanaged" +) + +// NvidiaGPU contains NVIDIA-specific GPU settings. +type NvidiaGPU struct { + // managementMode toggles the managed GPU experience for NVIDIA GPUs. + // When set to Managed, AKS installs additional components (e.g. DCGM metrics + // and the NVIDIA device plugin) on top of the GPU driver. When set to + // Unmanaged (or not specified), only the GPU driver is installed and managing + // the device plugin/metrics is the user's responsibility. + // Managed requires gpu.mode to be Driver (driver installation enabled) and is + // only supported on NVIDIA GPU SKUs running Linux. + // For more details of what is installed, see aka.ms/aks/managed-gpu. + // +optional + ManagementMode *ManagementMode `json:"managementMode,omitempty"` +} + // GPU contains configuration for GPU-enabled nodes. +// +kubebuilder:validation:XValidation:message="gpu.nvidia.managementMode 'Managed' requires gpu.mode to be 'Driver' (the managed GPU experience requires driver installation)",rule="!(has(self.nvidia) && has(self.nvidia.managementMode) && self.nvidia.managementMode == 'Managed' && has(self.mode) && self.mode == 'None')" type GPU struct { // mode controls GPU driver management on GPU-enabled nodes. // When set to Driver (or not specified), GPU drivers are installed by AKS @@ -343,6 +374,10 @@ type GPU struct { // +default="Driver" // +optional Mode *GPUMode `json:"mode,omitempty"` + // nvidia contains NVIDIA-specific GPU settings, such as the managed GPU + // experience. Ignored for non-NVIDIA VM sizes. + // +optional + Nvidia *NvidiaGPU `json:"nvidia,omitempty"` } // KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes. @@ -795,3 +830,19 @@ func (in *AKSNodeClass) GetGPUMode() GPUMode { func (in *AKSNodeClass) IsGPUDriverInstallationEnabled() bool { return in.GetGPUMode() != GPUModeNone } + +// GetManagementMode returns the effective NVIDIA GPU management mode. +// Defaults to Unmanaged when gpu, gpu.nvidia, or gpu.nvidia.managementMode is +// nil, preserving backward compatibility (existing GPU nodes are unmanaged). +func (in *AKSNodeClass) GetManagementMode() ManagementMode { + if in.Spec.GPU == nil || in.Spec.GPU.Nvidia == nil || in.Spec.GPU.Nvidia.ManagementMode == nil { + return ManagementModeUnmanaged + } + return *in.Spec.GPU.Nvidia.ManagementMode +} + +// IsManagedGPUEnabled returns whether the managed NVIDIA GPU experience is +// enabled (gpu.nvidia.managementMode == Managed). Returns false when unset. +func (in *AKSNodeClass) IsManagedGPUEnabled() bool { + return in.GetManagementMode() == ManagementModeManaged +} diff --git a/pkg/apis/v1beta1/aksnodeclass_gpu_test.go b/pkg/apis/v1beta1/aksnodeclass_gpu_test.go new file mode 100644 index 0000000000..a85538f3f3 --- /dev/null +++ b/pkg/apis/v1beta1/aksnodeclass_gpu_test.go @@ -0,0 +1,59 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1_test + +import ( + "github.com/samber/lo" + + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1beta1" + "github.com/Azure/karpenter-provider-azure/pkg/test" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("GPU managed mode helpers", func() { + var nodeClass *v1beta1.AKSNodeClass + + BeforeEach(func() { + nodeClass = test.AKSNodeClass() + }) + + DescribeTable("GetManagementMode defaults to Unmanaged when unset", + func(gpu *v1beta1.GPU, expected v1beta1.ManagementMode) { + nodeClass.Spec.GPU = gpu + Expect(nodeClass.GetManagementMode()).To(Equal(expected)) + }, + Entry("gpu nil", (*v1beta1.GPU)(nil), v1beta1.ManagementModeUnmanaged), + Entry("gpu set, nvidia nil", &v1beta1.GPU{}, v1beta1.ManagementModeUnmanaged), + Entry("nvidia set, managementMode nil", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{}}, v1beta1.ManagementModeUnmanaged), + Entry("managementMode Unmanaged", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeUnmanaged)}}, v1beta1.ManagementModeUnmanaged), + Entry("managementMode Managed", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeManaged)}}, v1beta1.ManagementModeManaged), + ) + + DescribeTable("IsManagedGPUEnabled", + func(gpu *v1beta1.GPU, expected bool) { + nodeClass.Spec.GPU = gpu + Expect(nodeClass.IsManagedGPUEnabled()).To(Equal(expected)) + }, + Entry("gpu nil", (*v1beta1.GPU)(nil), false), + Entry("nvidia nil", &v1beta1.GPU{}, false), + Entry("managementMode nil", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{}}, false), + Entry("Unmanaged", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeUnmanaged)}}, false), + Entry("Managed", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeManaged)}}, true), + ) +}) diff --git a/pkg/apis/v1beta1/aksnodeclass_hash_test.go b/pkg/apis/v1beta1/aksnodeclass_hash_test.go index be9e6f27d2..a50b1d2d41 100644 --- a/pkg/apis/v1beta1/aksnodeclass_hash_test.go +++ b/pkg/apis/v1beta1/aksnodeclass_hash_test.go @@ -99,7 +99,18 @@ var _ = Describe("Hash", func() { Entry("LocalDNS.VnetDNSOverrides.CacheDuration", v1beta1.AKSNodeClass{Spec: v1beta1.AKSNodeClassSpec{LocalDNS: &v1beta1.LocalDNS{VnetDNSOverrides: []v1beta1.LocalDNSZoneOverride{{Zone: "example.com", CacheDuration: karpv1.MustParseNillableDuration("2h")}}}}}), Entry("LocalDNS.VnetDNSOverrides.ServeStaleDuration", v1beta1.AKSNodeClass{Spec: v1beta1.AKSNodeClassSpec{LocalDNS: &v1beta1.LocalDNS{VnetDNSOverrides: []v1beta1.LocalDNSZoneOverride{{Zone: "example.com", ServeStaleDuration: karpv1.MustParseNillableDuration("1h")}}}}}), Entry("ArtifactStreaming.Enabled", v1beta1.AKSNodeClass{Spec: v1beta1.AKSNodeClassSpec{ArtifactStreaming: &v1beta1.ArtifactStreaming{Enabled: lo.ToPtr(true)}}}), + Entry("GPU.Nvidia.ManagementMode", v1beta1.AKSNodeClass{Spec: v1beta1.AKSNodeClassSpec{GPU: &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeManaged)}}}}), ) + It("should not change hash when gpu.nvidia is nil (non-breaking field addition)", func() { + // Adding the nvidia field must not alter the hash of NodeClasses that do + // not set it, so existing GPU nodes do not drift/roll on upgrade. + hash := nodeClass.Hash() + nodeClass.Spec.GPU = &v1beta1.GPU{Mode: lo.ToPtr(v1beta1.GPUModeDriver)} + hashWithGPUNoNvidia := nodeClass.Hash() + nodeClass.Spec.GPU.Nvidia = nil + Expect(nodeClass.Hash()).To(Equal(hashWithGPUNoNvidia)) + _ = hash + }) It("should not change hash when tags are re-ordered", func() { hash := nodeClass.Hash() nodeClass.Spec.Tags = map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"} diff --git a/pkg/apis/v1beta1/crd_validation_cel_test.go b/pkg/apis/v1beta1/crd_validation_cel_test.go index e7925f876b..95bee88447 100644 --- a/pkg/apis/v1beta1/crd_validation_cel_test.go +++ b/pkg/apis/v1beta1/crd_validation_cel_test.go @@ -765,6 +765,84 @@ var _ = Describe("CEL/Validation", func() { } Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) }) + It("should accept gpu.nvidia.managementMode set to Managed (default Driver mode)", func() { + managed := v1beta1.ManagementModeManaged + nodeClass := &v1beta1.AKSNodeClass{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: v1beta1.AKSNodeClassSpec{ + GPU: &v1beta1.GPU{ + Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &managed}, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should accept gpu.nvidia.managementMode set to Unmanaged", func() { + unmanaged := v1beta1.ManagementModeUnmanaged + nodeClass := &v1beta1.AKSNodeClass{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: v1beta1.AKSNodeClassSpec{ + GPU: &v1beta1.GPU{ + Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &unmanaged}, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should accept gpu.nvidia.managementMode=Managed with mode explicitly Driver", func() { + managed := v1beta1.ManagementModeManaged + driverMode := v1beta1.GPUModeDriver + nodeClass := &v1beta1.AKSNodeClass{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: v1beta1.AKSNodeClassSpec{ + GPU: &v1beta1.GPU{ + Mode: &driverMode, + Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &managed}, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should reject gpu.nvidia.managementMode=Managed when mode is None", func() { + managed := v1beta1.ManagementModeManaged + noneMode := v1beta1.GPUModeNone + nodeClass := &v1beta1.AKSNodeClass{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: v1beta1.AKSNodeClassSpec{ + GPU: &v1beta1.GPU{ + Mode: &noneMode, + Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &managed}, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).ToNot(Succeed()) + }) + It("should accept gpu.nvidia.managementMode=Unmanaged with mode None", func() { + unmanaged := v1beta1.ManagementModeUnmanaged + noneMode := v1beta1.GPUModeNone + nodeClass := &v1beta1.AKSNodeClass{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: v1beta1.AKSNodeClassSpec{ + GPU: &v1beta1.GPU{ + Mode: &noneMode, + Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &unmanaged}, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should reject invalid gpu.nvidia.managementMode value", func() { + invalid := v1beta1.ManagementMode("Invalid") + nodeClass := &v1beta1.AKSNodeClass{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: v1beta1.AKSNodeClassSpec{ + GPU: &v1beta1.GPU{ + Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &invalid}, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).ToNot(Succeed()) + }) }) Context("Requirements", func() { diff --git a/pkg/apis/v1beta1/zz_generated.deepcopy.go b/pkg/apis/v1beta1/zz_generated.deepcopy.go index 4fd113dc71..365248da32 100644 --- a/pkg/apis/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/v1beta1/zz_generated.deepcopy.go @@ -234,6 +234,11 @@ func (in *GPU) DeepCopyInto(out *GPU) { *out = new(GPUMode) **out = **in } + if in.Nvidia != nil { + in, out := &in.Nvidia, &out.Nvidia + *out = new(NvidiaGPU) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPU. @@ -442,6 +447,26 @@ func (in *NodeImage) DeepCopy() *NodeImage { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NvidiaGPU) DeepCopyInto(out *NvidiaGPU) { + *out = *in + if in.ManagementMode != nil { + in, out := &in.ManagementMode, &out.ManagementMode + *out = new(ManagementMode) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NvidiaGPU. +func (in *NvidiaGPU) DeepCopy() *NvidiaGPU { + if in == nil { + return nil + } + out := new(NvidiaGPU) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Security) DeepCopyInto(out *Security) { *out = *in diff --git a/pkg/providers/instance/aksmachineinstancehelpers.go b/pkg/providers/instance/aksmachineinstancehelpers.go index 8af65cc312..21de00d39f 100644 --- a/pkg/providers/instance/aksmachineinstancehelpers.go +++ b/pkg/providers/instance/aksmachineinstancehelpers.go @@ -188,9 +188,25 @@ func configureGPUProfile(instanceType *corecloudprovider.InstanceType, nodeClass if nodeClass.IsGPUDriverInstallationEnabled() { driverSetting = armcontainerservice.GPUDriverInstall } - return &armcontainerservice.GPUProfile{ + gpuProfile := &armcontainerservice.GPUProfile{ Driver: lo.ToPtr(driverSetting), } + // Managed GPU experience: AKS installs additional components (DCGM metrics, + // NVIDIA device plugin) on top of the driver. It is NVIDIA-only and requires + // driver installation. Only emit Nvidia settings when the user explicitly + // opts in via gpu.nvidia.managementMode=Managed; leaving Nvidia nil keeps the + // request unmanaged, which is the non-breaking default (RP treats nil as + // Unmanaged). The SKU/driver guards avoid sending Managed for SKUs the RP + // would reject (e.g. AMD GPUs or driver=None), which the API-level CEL rule + // and instance-type filtering already prevent, but we re-check defensively. + if nodeClass.IsManagedGPUEnabled() && + utils.IsNvidiaEnabledSKU(instanceType.Name) && + nodeClass.IsGPUDriverInstallationEnabled() { + gpuProfile.Nvidia = &armcontainerservice.NvidiaGPUProfile{ + ManagementMode: lo.ToPtr(armcontainerservice.ManagementModeManaged), + } + } + return gpuProfile } func configureArtifactStreamingProfile(nodeClass *v1beta1.AKSNodeClass, instanceType *corecloudprovider.InstanceType) *armcontainerservice.AgentPoolArtifactStreamingProfile { diff --git a/pkg/providers/instance/aksmachineinstancehelpers_test.go b/pkg/providers/instance/aksmachineinstancehelpers_test.go index 498a9bd83d..40f6f7e4d4 100644 --- a/pkg/providers/instance/aksmachineinstancehelpers_test.go +++ b/pkg/providers/instance/aksmachineinstancehelpers_test.go @@ -944,5 +944,52 @@ var _ = Describe("AKSMachineInstance Helper Functions", func() { Expect(profile).ToNot(BeNil()) Expect(*profile.Driver).To(Equal(armcontainerservice.GPUDriverNone)) }) + + It("should not set Nvidia when managementMode is unset (unmanaged default)", func() { + instanceType.Name = "Standard_NC6s_v3" + profile := configureGPUProfile(instanceType, nodeClass) + Expect(profile).ToNot(BeNil()) + Expect(profile.Nvidia).To(BeNil()) + }) + + It("should not set Nvidia when managementMode is Unmanaged", func() { + unmanaged := v1beta1.ManagementModeUnmanaged + nodeClass.Spec.GPU = &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &unmanaged}} + instanceType.Name = "Standard_NC6s_v3" + profile := configureGPUProfile(instanceType, nodeClass) + Expect(profile).ToNot(BeNil()) + Expect(profile.Nvidia).To(BeNil()) + }) + + It("should set Nvidia.ManagementMode=Managed for NVIDIA SKU with Managed mode", func() { + managed := v1beta1.ManagementModeManaged + nodeClass.Spec.GPU = &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &managed}} + instanceType.Name = "Standard_NC6s_v3" + profile := configureGPUProfile(instanceType, nodeClass) + Expect(profile).ToNot(BeNil()) + Expect(*profile.Driver).To(Equal(armcontainerservice.GPUDriverInstall)) + Expect(profile.Nvidia).ToNot(BeNil()) + Expect(*profile.Nvidia.ManagementMode).To(Equal(armcontainerservice.ManagementModeManaged)) + }) + + It("should not set Nvidia for AMD GPU SKU even when Managed is requested (NVIDIA-only guard)", func() { + managed := v1beta1.ManagementModeManaged + nodeClass.Spec.GPU = &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &managed}} + instanceType.Name = "Standard_NV4ads_V710_v5" + profile := configureGPUProfile(instanceType, nodeClass) + Expect(profile).ToNot(BeNil()) + Expect(profile.Nvidia).To(BeNil()) + }) + + It("should not set Nvidia when Managed is requested but driver install is disabled (None mode)", func() { + managed := v1beta1.ManagementModeManaged + noneMode := v1beta1.GPUModeNone + nodeClass.Spec.GPU = &v1beta1.GPU{Mode: &noneMode, Nvidia: &v1beta1.NvidiaGPU{ManagementMode: &managed}} + instanceType.Name = "Standard_NC6s_v3" + profile := configureGPUProfile(instanceType, nodeClass) + Expect(profile).ToNot(BeNil()) + Expect(*profile.Driver).To(Equal(armcontainerservice.GPUDriverNone)) + Expect(profile.Nvidia).To(BeNil()) + }) }) }) From a637770ecd221f0d51ba4a48157cc57a11c02d13 Mon Sep 17 00:00:00 2001 From: muribefalcon Date: Thu, 25 Jun 2026 11:37:33 -0700 Subject: [PATCH 2/5] test: remove aksnodeclass_gpu_test.go --- pkg/apis/v1beta1/aksnodeclass_gpu_test.go | 59 ----------------------- 1 file changed, 59 deletions(-) delete mode 100644 pkg/apis/v1beta1/aksnodeclass_gpu_test.go diff --git a/pkg/apis/v1beta1/aksnodeclass_gpu_test.go b/pkg/apis/v1beta1/aksnodeclass_gpu_test.go deleted file mode 100644 index a85538f3f3..0000000000 --- a/pkg/apis/v1beta1/aksnodeclass_gpu_test.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Portions Copyright (c) Microsoft Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1_test - -import ( - "github.com/samber/lo" - - "github.com/Azure/karpenter-provider-azure/pkg/apis/v1beta1" - "github.com/Azure/karpenter-provider-azure/pkg/test" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("GPU managed mode helpers", func() { - var nodeClass *v1beta1.AKSNodeClass - - BeforeEach(func() { - nodeClass = test.AKSNodeClass() - }) - - DescribeTable("GetManagementMode defaults to Unmanaged when unset", - func(gpu *v1beta1.GPU, expected v1beta1.ManagementMode) { - nodeClass.Spec.GPU = gpu - Expect(nodeClass.GetManagementMode()).To(Equal(expected)) - }, - Entry("gpu nil", (*v1beta1.GPU)(nil), v1beta1.ManagementModeUnmanaged), - Entry("gpu set, nvidia nil", &v1beta1.GPU{}, v1beta1.ManagementModeUnmanaged), - Entry("nvidia set, managementMode nil", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{}}, v1beta1.ManagementModeUnmanaged), - Entry("managementMode Unmanaged", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeUnmanaged)}}, v1beta1.ManagementModeUnmanaged), - Entry("managementMode Managed", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeManaged)}}, v1beta1.ManagementModeManaged), - ) - - DescribeTable("IsManagedGPUEnabled", - func(gpu *v1beta1.GPU, expected bool) { - nodeClass.Spec.GPU = gpu - Expect(nodeClass.IsManagedGPUEnabled()).To(Equal(expected)) - }, - Entry("gpu nil", (*v1beta1.GPU)(nil), false), - Entry("nvidia nil", &v1beta1.GPU{}, false), - Entry("managementMode nil", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{}}, false), - Entry("Unmanaged", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeUnmanaged)}}, false), - Entry("Managed", &v1beta1.GPU{Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeManaged)}}, true), - ) -}) From 73719beb99ba787c9690c073975cc5b9106a8d13 Mon Sep 17 00:00:00 2001 From: MaximilianoUribe Date: Mon, 29 Jun 2026 17:56:53 +0000 Subject: [PATCH 3/5] testing --- .../suite_aksmachineapi_features_test.go | 78 +++++++++++++++++++ test/suites/gpu/suite_test.go | 66 ++++++++++++++++ 2 files changed, 144 insertions(+) diff --git a/pkg/cloudprovider/suite_aksmachineapi_features_test.go b/pkg/cloudprovider/suite_aksmachineapi_features_test.go index ef587d1d35..73e2a691cc 100644 --- a/pkg/cloudprovider/suite_aksmachineapi_features_test.go +++ b/pkg/cloudprovider/suite_aksmachineapi_features_test.go @@ -303,6 +303,84 @@ var _ = Describe("CloudProvider", func() { }) }) + // Managed NVIDIA GPU experience (gpu.nvidia.managementMode) on the AKS machine API path. + Context("Create - Managed NVIDIA GPU", func() { + scheduleGPUPod := func() { + pod := coretest.UnschedulablePod(coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "managed-gpu", + Labels: map[string]string{"app": "managed-gpu"}, + }, + Image: "mcr.microsoft.com/azuredocs/samples-tf-mnist-demo:gpu", + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{"nvidia.com/gpu": resource.MustParse("1")}, + }, + RestartPolicy: v1.RestartPolicy("OnFailure"), + Tolerations: []v1.Toleration{{ + Key: "sku", + Operator: v1.TolerationOpEqual, + Value: "gpu", + Effect: v1.TaintEffectNoSchedule, + }}, + }) + ExpectProvisionedAndWaitForPromises(ctx, env.Client, cluster, cloudProvider, coreProvisioner, azureEnv, pod) + ExpectScheduled(ctx, env.Client, pod) + } + + popCreatedMachine := func() armcontainerservice.Machine { + Expect(azureEnv.AKSMachinesAPI.AKSMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + createInput := azureEnv.AKSMachinesAPI.AKSMachineCreateOrUpdateBehavior.CalledWithInput.Pop() + aksMachine := createInput.AKSMachine + Expect(aksMachine.Properties).ToNot(BeNil()) + Expect(aksMachine.Properties.Hardware).ToNot(BeNil()) + return aksMachine + } + + It("should set GpuProfile.Nvidia.ManagementMode=Managed for a Managed NVIDIA GPU NodeClass", func() { + nodeClass.Spec.GPU = &v1beta1.GPU{ + Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeManaged)}, + } + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + scheduleGPUPod() + + aksMachine := popCreatedMachine() + Expect(utils.IsNvidiaEnabledSKU(lo.FromPtr(aksMachine.Properties.Hardware.VMSize))).To(BeTrue()) + gpuProfile := aksMachine.Properties.Hardware.GpuProfile + Expect(gpuProfile).ToNot(BeNil()) + Expect(lo.FromPtr(gpuProfile.Driver)).To(Equal(armcontainerservice.GPUDriverInstall)) + Expect(gpuProfile.Nvidia).ToNot(BeNil()) + Expect(lo.FromPtr(gpuProfile.Nvidia.ManagementMode)).To(Equal(armcontainerservice.ManagementModeManaged)) + }) + + It("should not set GpuProfile.Nvidia when management mode is unset (unmanaged default, non-breaking)", func() { + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + scheduleGPUPod() + + aksMachine := popCreatedMachine() + Expect(utils.IsNvidiaEnabledSKU(lo.FromPtr(aksMachine.Properties.Hardware.VMSize))).To(BeTrue()) + gpuProfile := aksMachine.Properties.Hardware.GpuProfile + Expect(gpuProfile).ToNot(BeNil()) + Expect(lo.FromPtr(gpuProfile.Driver)).To(Equal(armcontainerservice.GPUDriverInstall)) + Expect(gpuProfile.Nvidia).To(BeNil()) + }) + + It("should not set GpuProfile.Nvidia when management mode is explicitly Unmanaged", func() { + nodeClass.Spec.GPU = &v1beta1.GPU{ + Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeUnmanaged)}, + } + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + scheduleGPUPod() + + aksMachine := popCreatedMachine() + gpuProfile := aksMachine.Properties.Hardware.GpuProfile + Expect(gpuProfile).ToNot(BeNil()) + Expect(gpuProfile.Nvidia).To(BeNil()) + }) + }) + // Ported from VM test: Context "additional-tags" Context("Create - Additional Tags", func() { It("should add additional tags to the AKS machine", func() { diff --git a/test/suites/gpu/suite_test.go b/test/suites/gpu/suite_test.go index 9fb9d1edcd..cbca48e70b 100644 --- a/test/suites/gpu/suite_test.go +++ b/test/suites/gpu/suite_test.go @@ -184,6 +184,72 @@ var _ = Describe("GPU", func() { } }, ) + + It("should provision a GPU node with managed mode (nvidia.managementMode=Managed)", + Label("GPU"), + func() { + // Managed mode: AKS installs and manages the NVIDIA stack (device + // plugin + DCGM metrics) on top of the driver. Unlike the default + // table above, we deliberately do NOT deploy a device plugin + // DaemonSet ourselves — the managed experience must advertise + // nvidia.com/gpu on its own. A healthy pod that requests + // nvidia.com/gpu therefore proves the managed device plugin was + // installed by the platform. + // + // Prerequisite: the cluster subscription must have the managed GPU + // preview feature registered; otherwise the AKS machine API rejects + // managementMode=Managed and the NodeClaim surfaces an error. + nodeClass := env.DefaultAKSNodeClass() + nodeClass.Spec.GPU = &v1beta1.GPU{ + Mode: lo.ToPtr(v1beta1.GPUModeDriver), + Nvidia: &v1beta1.NvidiaGPU{ManagementMode: lo.ToPtr(v1beta1.ManagementModeManaged)}, + } + + nodePool := env.DefaultNodePool(nodeClass) + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + Key: v1beta1.LabelSKUFamily, + Operator: corev1.NodeSelectorOpExists, + }) + nodePool.Spec.Limits = karpv1.Limits{ + corev1.ResourceCPU: resource.MustParse("25"), + corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), + } + + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 1, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "managed-gpu-test", + Labels: map[string]string{"app": "managed-gpu-test"}, + }, + ResourceRequirements: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "nvidia.com/gpu": resource.MustParse("1"), + }, + }, + }, + }) + + // Note: no device plugin DaemonSet is created here on purpose. + env.ExpectCreated(nodeClass, nodePool, deployment) + + env.EventuallyExpectHealthyPodCountWithTimeout( + time.Minute*15, + labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), + int(*deployment.Spec.Replicas), + ) + env.ExpectCreatedNodeCount("==", int(*deployment.Spec.Replicas)) + + // The managed device plugin must have advertised the GPU resource + // without any user-deployed device plugin. + nodes := env.Monitor.CreatedNodes() + Expect(nodes).To(HaveLen(int(*deployment.Spec.Replicas))) + for _, node := range nodes { + _, hasGPUResource := node.Status.Allocatable[corev1.ResourceName("nvidia.com/gpu")] + Expect(hasGPUResource).To(BeTrue(), "node %s should advertise nvidia.com/gpu under managed GPU mode", node.Name) + } + }, + ) }) func createNVIDIADevicePluginDaemonSet() *appsv1.DaemonSet { From 6b436d988c1f40a68b13021ee1c8f031b42a47c9 Mon Sep 17 00:00:00 2001 From: muribefalcon Date: Mon, 6 Jul 2026 20:00:16 -0700 Subject: [PATCH 4/5] docs: clarify gpu.nvidia.managementMode only applies to NAP The managed NVIDIA GPU experience (driver + device plugin + DCGM installed and managed by AKS) is only delivered for node pools managed by AKS Node Auto Provisioning (NAP). On self-hosted Karpenter the setting has no effect and the managed components are not installed. Clarify this in the field/enum godoc and the regenerated AKSNodeClass CRD description. --- .../templates/karpenter.azure.com_aksnodeclasses.yaml | 3 +++ pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml | 3 +++ pkg/apis/v1beta1/aksnodeclass.go | 8 +++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml index 313db7a84b..dcb4329e52 100644 --- a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml +++ b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml @@ -1010,6 +1010,9 @@ spec: the device plugin/metrics is the user's responsibility. Managed requires gpu.mode to be Driver (driver installation enabled) and is only supported on NVIDIA GPU SKUs running Linux. + The managed experience only applies to node pools managed by AKS Node Auto + Provisioning (NAP). On self-hosted Karpenter this setting has no effect and + the managed components are not installed. For more details of what is installed, see aka.ms/aks/managed-gpu. enum: - Managed diff --git a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml index 313db7a84b..dcb4329e52 100644 --- a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml +++ b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml @@ -1010,6 +1010,9 @@ spec: the device plugin/metrics is the user's responsibility. Managed requires gpu.mode to be Driver (driver installation enabled) and is only supported on NVIDIA GPU SKUs running Linux. + The managed experience only applies to node pools managed by AKS Node Auto + Provisioning (NAP). On self-hosted Karpenter this setting has no effect and + the managed components are not installed. For more details of what is installed, see aka.ms/aks/managed-gpu. enum: - Managed diff --git a/pkg/apis/v1beta1/aksnodeclass.go b/pkg/apis/v1beta1/aksnodeclass.go index 14410f1b0c..302cb0a9cf 100644 --- a/pkg/apis/v1beta1/aksnodeclass.go +++ b/pkg/apis/v1beta1/aksnodeclass.go @@ -338,7 +338,10 @@ const ( // GPU driver, AKS installs and manages additional components such as the // Data Center GPU Manager (DCGM) metrics exporter and the NVIDIA device plugin. // Requires GPU driver installation (gpu.mode must not be None) and is only - // supported on NVIDIA GPU SKUs. For details see aka.ms/aks/managed-gpu. + // supported on NVIDIA GPU SKUs. The managed experience only applies to node + // pools managed by AKS Node Auto Provisioning (NAP); on self-hosted Karpenter + // this setting has no effect and the managed components are not installed. + // For details see aka.ms/aks/managed-gpu. ManagementModeManaged ManagementMode = "Managed" // ManagementModeUnmanaged disables the managed GPU experience. Only the GPU // driver is installed (subject to gpu.mode); device plugin and metrics are @@ -355,6 +358,9 @@ type NvidiaGPU struct { // the device plugin/metrics is the user's responsibility. // Managed requires gpu.mode to be Driver (driver installation enabled) and is // only supported on NVIDIA GPU SKUs running Linux. + // The managed experience only applies to node pools managed by AKS Node Auto + // Provisioning (NAP). On self-hosted Karpenter this setting has no effect and + // the managed components are not installed. // For more details of what is installed, see aka.ms/aks/managed-gpu. // +optional ManagementMode *ManagementMode `json:"managementMode,omitempty"` From 6ff6121466595d25cddff3ff76f5d4639fe7a7f5 Mon Sep 17 00:00:00 2001 From: muribefalcon Date: Mon, 6 Jul 2026 20:29:06 -0700 Subject: [PATCH 5/5] test(gpu): skip managed-GPU e2e unless on AKS machine API path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The managed NVIDIA GPU experience (managementMode=Managed) is only delivered via the AKS machine API / NAP provisioning path — the control plane installs the driver, device plugin and DCGM. On self-hosted paths (scriptless / bootstrapping-client) managementMode is not forwarded and there is no control plane to install the managed components, so this spec's assertions could never hold and it would fail every run (e.g. the self-hosted GitHub Actions GPU e2e). Skip when env.IsAKSMachineAPIMode() is false, matching the byok suite pattern. --- test/suites/gpu/suite_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/suites/gpu/suite_test.go b/test/suites/gpu/suite_test.go index cbca48e70b..1951b566d1 100644 --- a/test/suites/gpu/suite_test.go +++ b/test/suites/gpu/suite_test.go @@ -196,6 +196,15 @@ var _ = Describe("GPU", func() { // nvidia.com/gpu therefore proves the managed device plugin was // installed by the platform. // + // The managed GPU experience is only delivered through the AKS + // machine API (NAP) provisioning path: the RP/control plane installs + // the driver, device plugin and DCGM. Self-hosted provisioning paths + // (scriptless / bootstrapping-client) do not forward managementMode + // and have no control plane to install the managed components, so the + // assertions below can never hold there — skip rather than fail. + if !env.IsAKSMachineAPIMode() { + Skip("managed GPU (gpu.nvidia.managementMode) is only supported on the AKS machine API / NAP provisioning path") + } // Prerequisite: the cluster subscription must have the managed GPU // preview feature registered; otherwise the AKS machine API rejects // managementMode=Managed and the NodeClaim surfaces an error.