gpu: add support for additional temperature limits

Use the existing "temp-limit" as the global limit, and introduce GPU
and memory thresholds.

Signed-off-by: Tuomas Katila <[email protected]>

Author: tkatila

cmd/gpu_plugin/gpu_plugin.go

@@ -70,7 +70,9 @@ type cliOptions struct {
 	allowIDs                  string
 	denyIDs                   string
 	sharedDevNum              int
-	temperatureLimit          int
+	globalTempLimit           int
+	memoryTempLimit           int
+	gpuTempLimit              int
 	enableMonitoring          bool
 	wslScan                   bool
 	healthManagement          bool
@@ -402,13 +404,13 @@ func (dp *devicePlugin) healthStatusForCard(cardPath string) string {
 		return health
 	}
 
-	limit := float64(dp.options.temperatureLimit)
-
 	// Temperatures for different areas
-	klog.V(4).Infof("Temperatures: Memory=%.1fC, GPU=%.1fC, Global=%.1fC",
+	klog.V(4).Infof("Temperatures: Memory=%dC, GPU=%dC, Global=%dC",
 		deviceTemps.Memory, deviceTemps.GPU, deviceTemps.Global)
 
-	if deviceTemps.GPU > limit || deviceTemps.Global > limit || deviceTemps.Memory > limit {
+	if deviceTemps.GPU > dp.options.gpuTempLimit ||
+		deviceTemps.Global > dp.options.globalTempLimit ||
+		deviceTemps.Memory > dp.options.memoryTempLimit {
 		health = pluginapi.Unhealthy
 	}
 
@@ -784,7 +786,9 @@ func main() {
 	flag.BoolVar(&opts.healthManagement, "health-management", false, "enable GPU health management")
 	flag.BoolVar(&opts.wslScan, "wsl", false, "scan for / use WSL devices")
 	flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same GPU device")
-	flag.IntVar(&opts.temperatureLimit, "temp-limit", 100, "temperature limit at which device is marked unhealthy")
+	flag.IntVar(&opts.globalTempLimit, "temp-limit", 100, "Global temperature limit at which device is marked unhealthy")
+	flag.IntVar(&opts.gpuTempLimit, "gpu-temp-limit", 100, "GPU temperature limit at which device is marked unhealthy")
+	flag.IntVar(&opts.memoryTempLimit, "memory-temp-limit", 100, "Memory temperature limit at which device is marked unhealthy")
 	flag.StringVar(&opts.preferredAllocationPolicy, "allocation-policy", "none", "modes of allocating GPU devices: balanced, packed and none")
 	flag.StringVar(&opts.allowIDs, "allow-ids", "", "comma-separated list of device IDs to allow (e.g. 0x49c5,0x49c6)")
 	flag.StringVar(&opts.denyIDs, "deny-ids", "", "comma-separated list of device IDs to deny (e.g. 0x49c5,0x49c6)")

cmd/gpu_plugin/gpu_plugin_test.go

@@ -58,10 +58,11 @@ func (n *mockNotifier) Notify(newDeviceTree dpapi.DeviceTree) {
 }
 
 type mockL0Service struct {
-	indices []uint32
-	memSize uint64
-	healthy bool
-	fail    bool
+	indices  []uint32
+	memSize  uint64
+	healthy  bool
+	failTemp bool
+	fail     bool
 }
 
 func (m *mockL0Service) Run(keep bool) {
@@ -83,7 +84,7 @@ func (m *mockL0Service) GetDeviceHealth(bdfAddress string) (levelzeroservice.Dev
 	return levelzeroservice.DeviceHealth{Memory: m.healthy, Bus: m.healthy, SoC: m.healthy}, nil
 }
 func (m *mockL0Service) GetDeviceTemperature(bdfAddress string) (levelzeroservice.DeviceTemperature, error) {
-	if m.fail {
+	if m.fail || m.failTemp {
 		return levelzeroservice.DeviceTemperature{}, errors.Errorf("error, error")
 	}
 
@@ -608,6 +609,24 @@ func TestScanWithHealth(t *testing.T) {
 				healthy: true,
 			},
 		},
+		{
+			name:         "one device with failure on temp reading",
+			pciAddresses: map[string]string{"0000:00:00.0": "card0"},
+			sysfsdirs:    []string{"card0/device/drm/card0", "card0/device/drm/controlD64"},
+			sysfsfiles: map[string][]byte{
+				"card0/device/vendor": []byte("0x8086"),
+			},
+			devfsdirs: []string{
+				"card0",
+				"by-path/pci-0000:00:00.0-card",
+				"by-path/pci-0000:00:00.0-render",
+			},
+			expectedI915Devs: 1,
+			l0mock: &mockL0Service{
+				healthy:  true,
+				failTemp: true,
+			},
+		},
 		{
 			name:         "one unhealthy device with proper symlink",
 			pciAddresses: map[string]string{"0000:00:00.0": "card0"},

cmd/gpu_plugin/levelzeroservice/levelzero_service.go

@@ -39,9 +39,9 @@ type DeviceHealth struct {
 }
 
 type DeviceTemperature struct {
-	Global float64
-	GPU    float64
-	Memory float64
+	Global int
+	GPU    int
+	Memory int
 }
 
 type clientNotReadyErr struct{}
@@ -175,9 +175,9 @@ func (l *levelzero) GetDeviceTemperature(bdfAddress string) (DeviceTemperature,
 	}
 
 	return DeviceTemperature{
-		Global: temps.Global,
-		GPU:    temps.Gpu,
-		Memory: temps.Memory,
+		Global: int(temps.Global),
+		GPU:    int(temps.Gpu),
+		Memory: int(temps.Memory),
 	}, nil
 }