Create device nodes in JIT-CDI mode

Signed-off-by: Evan Lezar <[email protected]>

Author: elezar

Diff for: internal/info/proc/devices/builder.go

@@ -45,7 +45,7 @@ func New(opts ...Option) Devices {
 type Option func(*builder)
 
 // WithDeviceToMajor specifies an explicit device name to major number map.
-func WithDeviceToMajor(deviceToMajor map[string]int) Option {
+func WithDeviceToMajor(deviceToMajor map[string]uint32) Option {
 	return func(b *builder) {
 		b.asMap = make(devices)
 		for name, major := range deviceToMajor {

Diff for: internal/info/proc/devices/devices.go

@@ -45,7 +45,7 @@ const (
 type Name string
 
 // Major represents a device major as specified under /proc/devices
-type Major int
+type Major uint32
 
 // Devices represents the set of devices under /proc/devices
 //
@@ -130,8 +130,8 @@ func nvidiaDeviceFrom(reader io.Reader) (Devices, error) {
 	return nvidiaDevices, nil
 }
 
-func devicesFrom(reader io.Reader) map[string]int {
-	allDevices := make(map[string]int)
+func devicesFrom(reader io.Reader) map[string]uint32 {
+	allDevices := make(map[string]uint32)
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
 		device, major, err := processProcDeviceLine(scanner.Text())
@@ -143,11 +143,11 @@ func devicesFrom(reader io.Reader) map[string]int {
 	return allDevices
 }
 
-func processProcDeviceLine(line string) (string, int, error) {
+func processProcDeviceLine(line string) (string, uint32, error) {
 	trimmed := strings.TrimSpace(line)
 
 	var name string
-	var major int
+	var major uint32
 
 	n, _ := fmt.Sscanf(trimmed, "%d %s", &major, &name)
 	if n == 2 {

Diff for: internal/modifier/cdi.go

@@ -22,12 +22,15 @@ import (
 
 	"tags.cncf.io/container-device-interface/pkg/parser"
 
+	"github.com/NVIDIA/go-nvlib/pkg/nvlib/device"
+
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/root"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier/cdi"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
 )
@@ -198,12 +201,14 @@ func generateAutomaticCDISpec(logger logger.Interface, cfg *config.Config, drive
 		logger.Warningf("Ignoring error(s) loading kernel modules: %v", err)
 	}
 
-	identifiers := []string{}
+	var identifiers []string
 	for _, device := range devices {
 		_, _, id := parser.ParseDevice(device)
 		identifiers = append(identifiers, id)
 	}
 
+	tryCreateDeviceNodes(logger, driver, identifiers...)
+
 	deviceSpecs, err := cdilib.GetDeviceSpecsByID(identifiers...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get CDI device specs: %w", err)
@@ -221,3 +226,27 @@ func generateAutomaticCDISpec(logger logger.Interface, cfg *config.Config, drive
 		spec.WithClass("gpu"),
 	)
 }
+
+func tryCreateDeviceNodes(logger logger.Interface, driver *root.Driver, identifiers ...string) {
+	devices, err := nvdevices.New(
+		nvdevices.WithLogger(logger),
+		nvdevices.WithDevRoot(driver.Root),
+	)
+	if err != nil {
+		logger.Warningf("Failed to create devices library: %v", err)
+		return
+	}
+	if err := devices.CreateNVIDIAControlDevices(); err != nil {
+		logger.Warningf("Failed to create control devices: %v", err)
+	}
+	if err := devices.CreateNVIDIACapsControlDeviceNodes(); err != nil {
+		logger.Warningf("Failed to create nvidia-caps control devices: %v", err)
+	}
+
+	for _, id := range identifiers {
+		identifier := device.Identifier(id)
+		if err := devices.CreateDeviceNodes(identifier); err != nil {
+			logger.Warningf("Error creating device nodes for %v: %v", identifier, err)
+		}
+	}
+}

Diff for: internal/nvcaps/nvcaps.go

@@ -37,7 +37,7 @@ const (
 )
 
 // MigMinor represents the minor number of a MIG device
-type MigMinor int
+type MigMinor uint32
 
 // MigCap represents the path to a MIG cap file.
 // These are listed in /proc/driver/nvidia-caps/mig-minors and have one of the
@@ -144,7 +144,7 @@ func processMigMinorsLine(line string) (MigCap, MigMinor, error) {
 		return "", 0, fmt.Errorf("invalid MIG minors line: '%v'", line)
 	}
 
-	minor, err := strconv.Atoi(parts[1])
+	minor, err := strconv.ParseUint(parts[1], 10, 32)
 	if err != nil {
 		return "", 0, fmt.Errorf("error reading MIG minor from '%v': %v", line, err)
 	}

Diff for: internal/oci/spec_mock.go

@@ -59,7 +59,7 @@ func (m *Interface) CreateNVIDIACapsControlDeviceNodes() error {
 			continue
 		}
 		deviceNodePath := migMinor.DevicePath()
-		if err := m.createDeviceNode(deviceNodePath, int(capsMajor), int(migMinor)); err != nil {
+		if err := m.createDeviceNode(deviceNodePath, capsMajor, uint32(migMinor)); err != nil {
 			errs = errors.Join(errs, fmt.Errorf("failed to create nvidia-caps device node %v: %w", deviceNodePath, err))
 		}
 	}
@@ -82,12 +82,12 @@ func (m *Interface) createControlDeviceNode(node controlDeviceNode) error {
 		return fmt.Errorf("failed to determine minor: %w", err)
 	}
 
-	return m.createDeviceNode(node.path(), int(major), int(minor))
+	return m.createDeviceNode(node.path(), major, minor)
 }
 
 // controlDeviceNodeMajor returns the major number for the specified NVIDIA control device node.
 // If the device node is not supported, an error is returned.
-func (m *Interface) controlDeviceNodeMajor(node controlDeviceNode) (int64, error) {
+func (m *Interface) controlDeviceNodeMajor(node controlDeviceNode) (devices.Major, error) {
 	var valid bool
 	var major devices.Major
 	switch node {
@@ -98,15 +98,15 @@ func (m *Interface) controlDeviceNodeMajor(node controlDeviceNode) (int64, error
 	}
 
 	if valid {
-		return int64(major), nil
+		return major, nil
 	}
 
 	return 0, errInvalidDeviceNode
 }
 
 // controlDeviceNodeMinor returns the minor number for the specified NVIDIA control device node.
 // If the device node is not supported, an error is returned.
-func (m *Interface) controlDeviceNodeMinor(node controlDeviceNode) (int64, error) {
+func (m *Interface) controlDeviceNodeMinor(node controlDeviceNode) (uint32, error) {
 	switch node {
 	case "nvidia-modeset":
 		return devices.NVIDIAModesetMinor, nil

Diff for: internal/system/nvdevices/control-device-nodes.go

@@ -89,25 +89,25 @@ func New(opts ...Option) (*Interface, error) {
 func (m *Interface) CreateDeviceNodes(id device.Identifier) error {
 	switch {
 	case id.IsGpuIndex():
-		index, err := strconv.Atoi(string(id))
+		index, err := strconv.ParseUint(string(id), 10, 32)
 		if err != nil {
 			return fmt.Errorf("invalid GPU index: %v", id)
 		}
-		return m.createGPUDeviceNode(index)
+		return m.createGPUDeviceNode(int(index))
 	case id.IsMigIndex():
 		indices := strings.Split(string(id), ":")
 		if len(indices) != 2 {
 			return fmt.Errorf("invalid MIG index %v", id)
 		}
-		gpuIndex, err := strconv.Atoi(indices[0])
+		gpuIndex, err := strconv.ParseUint(indices[0], 10, 32)
 		if err != nil {
 			return fmt.Errorf("invalid parent index %v: %w", indices[0], err)
 		}
-		if err := m.createGPUDeviceNode(gpuIndex); err != nil {
+		if err := m.createGPUDeviceNode(int(gpuIndex)); err != nil {
 			return fmt.Errorf("failed to create parent device node: %w", err)
 		}
 
-		return m.createMigDeviceNodes(gpuIndex)
+		return m.createMigDeviceNodes(int(gpuIndex))
 	case id.IsGpuUUID(), id.IsMigUUID(), id == "all":
 		return m.createAllGPUDeviceNodes()
 	default:
@@ -117,7 +117,7 @@ func (m *Interface) CreateDeviceNodes(id device.Identifier) error {
 
 // createDeviceNode creates the specified device node with the require major and minor numbers.
 // If a devRoot is configured, this is prepended to the path.
-func (m *Interface) createDeviceNode(path string, major int, minor int) error {
+func (m *Interface) createDeviceNode(path string, major devices.Major, minor uint32) error {
 	path = filepath.Join(m.devRoot, path)
-	return m.Mknode(path, major, minor)
+	return m.Mknode(path, uint32(major), minor)
 }

Diff for: internal/system/nvdevices/devices.go

@@ -33,7 +33,7 @@ func (m *Interface) createGPUDeviceNode(gpuIndex int) error {
 	}
 
 	deviceNodePath := fmt.Sprintf("/dev/nvidia%d", gpuIndex)
-	if err := m.createDeviceNode(deviceNodePath, int(major), gpuIndex); err != nil {
+	if err := m.createDeviceNode(deviceNodePath, major, uint32(gpuIndex)); err != nil {
 		return fmt.Errorf("failed to create device node %v: %w", deviceNodePath, err)
 	}
 	return nil
@@ -47,7 +47,7 @@ func (m *Interface) createMigDeviceNodes(gpuIndex int) error {
 	var errs error
 	for _, capsDeviceMinor := range m.migCaps.FilterForGPU(gpuIndex) {
 		capDevicePath := capsDeviceMinor.DevicePath()
-		err := m.createDeviceNode(capDevicePath, int(capsMajor), int(capsDeviceMinor))
+		err := m.createDeviceNode(capDevicePath, capsMajor, uint32(capsDeviceMinor))
 		errs = errors.Join(errs, fmt.Errorf("failed to create %v: %w", capDevicePath, err))
 	}
 	return errs

Diff for: internal/system/nvdevices/gpu-device-nodes.go

@@ -25,16 +25,18 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 )
 
-//go:generate moq -stub -out mknod_mock.go . mknoder
+type mint uint32
+
+//go:generate moq -fmt=goimports -rm -stub -out mknod_mock.go . mknoder
 type mknoder interface {
-	Mknode(string, int, int) error
+	Mknode(string, uint32, uint32) error
 }
 
 type mknodLogger struct {
 	logger.Interface
 }
 
-func (m *mknodLogger) Mknode(path string, major, minor int) error {
+func (m *mknodLogger) Mknode(path string, major uint32, minor uint32) error {
 	m.Infof("Running: mknod --mode=0666 %s c %d %d", path, major, minor)
 	return nil
 }
@@ -43,7 +45,7 @@ type mknodUnix struct {
 	logger logger.Interface
 }
 
-func (m *mknodUnix) Mknode(path string, major, minor int) error {
+func (m *mknodUnix) Mknode(path string, major uint32, minor uint32) error {
 	// TODO: Ensure that the existing device node has the correct properties.
 	if _, err := os.Stat(path); err == nil {
 		m.logger.Infof("Skipping: %s already exists", path)
@@ -52,7 +54,7 @@ func (m *mknodUnix) Mknode(path string, major, minor int) error {
 		return fmt.Errorf("failed to stat %s: %v", path, err)
 	}
 
-	err := unix.Mknod(path, unix.S_IFCHR, int(unix.Mkdev(uint32(major), uint32(minor))))
+	err := unix.Mknod(path, unix.S_IFCHR, int(unix.Mkdev(major, minor)))
 	if err != nil {
 		return err
 	}