dev: real-GPU kind cluster with DRA passthrough

docs/setup/kind-gpu-cluster.md

@@ -0,0 +1,47 @@
+# A real-GPU kind cluster
+
+One command turns a Linux box with NVIDIA GPUs into a single-node Kubernetes
+cluster where the GPUs are real, allocatable Dynamic Resource Allocation
+devices — the same `gpu.nvidia.com` DeviceClass and ResourceSlices a
+production GKE DRA cluster publishes, without a cloud cluster.
+
+```
+hack/kind/bootstrap.sh    # fresh Ubuntu VM only: installs docker, kind,
+                          # kubectl, helm, nvidia-container-toolkit, uv
+hack/kind/kind-up.sh      # idempotent: cluster + GPU passthrough + DRA driver
+```
+
+After `kind-up.sh`, context `kind-openrl-gpu` has:
+
+- the host's GPUs visible inside the kind node (`docker exec
+  openrl-gpu-control-plane nvidia-smi -L`), injected by
+  nvidia-container-toolkit's volume-mount device mode;
+- the NVIDIA DRA driver (`dra-driver-nvidia-gpu` chart) publishing
+  ResourceSlices, with the `gpu.nvidia.com` DeviceClass registered;
+- Kubernetes 1.34, where DRA is GA — no feature gates.
+
+Anything that consumes DRA can then be developed against it: create a
+ResourceClaim, reference it from a pod, and the allocation is a real GPU.
+
+## How the passthrough works
+
+Three host-side settings, all checked (and offered as fixes) by `kind-up.sh`:
+
+1. docker's default runtime is `nvidia`;
+2. `accept-nvidia-visible-devices-as-volume-mounts = true` in
+   `/etc/nvidia-container-runtime/config.toml`;
+3. the kind node mounts `/var/run/nvidia-container-devices/all`, which the
+   nvidia runtime interprets as "inject every GPU into this container".
+
+The kind node also carries the `feature.node.kubernetes.io/pci-10de.present`
+label statically, because the DRA driver's kubelet plugin selects nodes by
+that node-feature-discovery label and kind runs no NFD.
+
+## Fidelity notes
+
+- One node: everything schedules onto the control plane. Multi-node GPU
+  topologies, real network fabrics, and node failure modes are out of scope.
+- The GPUs are shared with the host: anything else using them (a local
+  training run, another cluster on the same box) will fight over memory.
+- The cluster is meant to persist between runs; `kind delete cluster --name
+  openrl-gpu` removes it.

hack/kind/bootstrap.sh

@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+run_root() {
+  if [ "${EUID}" -eq 0 ]; then
+    "$@"
+  else
+    sudo "$@"
+  fi
+}
+
+apt_update() {
+  run_root env DEBIAN_FRONTEND=noninteractive apt-get -y -qq update
+}
+
+apt_install() {
+  run_root env DEBIAN_FRONTEND=noninteractive apt-get -y -qq install "$@"
+}
+
+ensure_apt_prereqs() {
+  apt_update
+  apt_install ca-certificates curl gnupg make
+}
+
+install_docker() {
+  echo "Installing Docker..."
+  ensure_apt_prereqs
+  run_root install -m 0755 -d /etc/apt/keyrings
+  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | run_root gpg --batch --yes --dearmor -o /etc/apt/keyrings/docker.gpg
+  run_root chmod a+r /etc/apt/keyrings/docker.gpg
+
+  . /etc/os-release
+  arch="$(dpkg --print-architecture)"
+  echo "deb [arch=${arch} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${VERSION_CODENAME} stable" |
+    run_root tee /etc/apt/sources.list.d/docker.list >/dev/null
+
+  apt_update
+  apt_install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+  run_root systemctl enable --now docker
+}
+
+install_nvidia_container_toolkit() {
+  echo "Installing nvidia-container-toolkit..."
+  ensure_apt_prereqs
+  curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey |
+    run_root gpg --batch --yes --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+  curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list |
+    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' |
+    run_root tee /etc/apt/sources.list.d/nvidia-container-toolkit.list >/dev/null
+
+  apt_update
+  apt_install nvidia-container-toolkit
+}
+
+install_kind() {
+  echo "Installing kind..."
+  ensure_apt_prereqs
+  tmp="$(mktemp)"
+  curl -fsSL -o "$tmp" https://kind.sigs.k8s.io/dl/v0.30.0/kind-linux-amd64
+  run_root install -m 0755 "$tmp" /usr/local/bin/kind
+  rm -f "$tmp"
+}
+
+install_kubectl() {
+  echo "Installing kubectl..."
+  ensure_apt_prereqs
+  stable="$(curl -Ls https://dl.k8s.io/release/stable.txt)"
+  tmp="$(mktemp)"
+  curl -fsSL -o "$tmp" "https://dl.k8s.io/release/${stable}/bin/linux/amd64/kubectl"
+  run_root install -m 0755 "$tmp" /usr/local/bin/kubectl
+  rm -f "$tmp"
+}
+
+install_helm() {
+  echo "Installing Helm..."
+  ensure_apt_prereqs
+  curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+}
+
+install_uv() {
+  echo "Installing uv..."
+  curl -LsSf https://astral.sh/uv/install.sh | run_root env UV_INSTALL_DIR=/usr/local/bin sh
+}
+
+if [ ! -r /etc/os-release ]; then
+  echo "This bootstrap script requires Ubuntu."
+  exit 1
+fi
+
+. /etc/os-release
+if [ "${ID:-}" != "ubuntu" ]; then
+  echo "This bootstrap script requires Ubuntu."
+  exit 1
+fi
+
+echo "Checking NVIDIA driver..."
+if ! command -v nvidia-smi >/dev/null 2>&1 || ! nvidia-smi >/dev/null 2>&1; then
+  echo "Install the NVIDIA driver (>= r550) first"
+  exit 1
+fi
+driver_version="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -n 1)"
+echo "Detected NVIDIA driver ${driver_version}"
+
+if command -v docker >/dev/null 2>&1; then
+  echo "Docker already installed."
+else
+  install_docker
+fi
+
+if command -v nvidia-ctk >/dev/null 2>&1; then
+  echo "nvidia-container-toolkit already installed."
+else
+  install_nvidia_container_toolkit
+fi
+
+if command -v kind >/dev/null 2>&1; then
+  echo "kind already installed."
+else
+  install_kind
+fi
+
+if command -v kubectl >/dev/null 2>&1; then
+  echo "kubectl already installed."
+else
+  install_kubectl
+fi
+
+if command -v helm >/dev/null 2>&1; then
+  echo "Helm already installed."
+else
+  install_helm
+fi
+
+if command -v uv >/dev/null 2>&1; then
+  echo "uv already installed."
+else
+  install_uv
+fi
+
+current_user="${USER:-$(id -un)}"
+if [ "${EUID}" -ne 0 ] && ! id -nG "${current_user}" | tr ' ' '\n' | grep -Fxq docker; then
+  echo "Adding ${current_user} to the docker group..."
+  run_root usermod -aG docker "${current_user}"
+  echo "Log out and log back in for docker group membership to take effect."
+fi
+
+echo "Installed versions:"
+docker --version
+nvidia-ctk --version
+kind --version
+kubectl version --client=true
+helm version --short
+
+echo "Next steps:"
+echo "  bash hack/kind/kind-up.sh   (or: make kind-up)"
+echo "  make kind-images && make kind-deploy && make kind-status"

hack/kind/kind-config.yaml

@@ -0,0 +1,18 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+name: openrl-gpu
+nodes:
+  # Kubernetes 1.34 has DRA GA, so no feature gates are needed.
+  - role: control-plane
+    image: kindest/node:v1.34.0
+    labels:
+      # The NVIDIA DRA driver's kubelet plugin schedules only onto nodes with
+      # node-feature-discovery PCI labels (10de = NVIDIA vendor id). kind has
+      # no NFD, so set the label statically.
+      feature.node.kubernetes.io/pci-10de.present: "true"
+    extraMounts:
+      # nvidia-container-toolkit injects all host GPUs into the node container
+      # when it sees this mount; requires the nvidia runtime as docker default
+      # AND accept-nvidia-visible-devices-as-volume-mounts=true.
+      - hostPath: /dev/null
+        containerPath: /var/run/nvidia-container-devices/all

hack/kind/kind-up.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# Bring up (or reuse) the real-GPU kind cluster: host GPUs passed through to
+# the kind node via nvidia-container-toolkit device injection, NVIDIA DRA
+# driver installed, gpu.nvidia.com DeviceClass registered. Idempotent -- safe
+# to run before every smoke. Linux with an NVIDIA driver only.
+#
+# The cluster name is fixed to openrl-gpu (kind reads it from the config).
+set -euo pipefail
+
+install_hint() {
+  case "$1" in
+    docker) echo "Missing docker. Install Docker Engine: https://docs.docker.com/engine/install/" ;;
+    kind) echo "Missing kind. Install kind >= 0.30: https://kind.sigs.k8s.io/docs/user/quick-start/#installation" ;;
+    kubectl) echo "Missing kubectl. Install kubectl: https://kubernetes.io/docs/tasks/tools/" ;;
+    helm) echo "Missing helm. Install Helm 3: https://helm.sh/docs/intro/install/" ;;
+    nvidia-smi) echo "Missing nvidia-smi. Install the NVIDIA Linux driver >= r550." ;;
+    nvidia-ctk) echo "Missing nvidia-ctk. Install nvidia-container-toolkit: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html" ;;
+    *) echo "Missing $1." ;;
+  esac
+}
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo "Checking required tools..."
+missing_tools=0
+for tool in docker kind kubectl helm nvidia-smi nvidia-ctk; do
+  if ! command -v "$tool" >/dev/null 2>&1; then
+    install_hint "$tool"
+    missing_tools=1
+  fi
+done
+if [ "$missing_tools" -ne 0 ]; then
+  exit 1
+fi
+
+echo "Checking docker default runtime..."
+default_runtime="$(docker info --format '{{.DefaultRuntime}}')"
+if [ "$default_runtime" != "nvidia" ]; then
+  echo "Docker default runtime is '$default_runtime', expected 'nvidia'."
+  echo "Fix: sudo nvidia-ctk runtime configure --runtime=docker --set-as-default && sudo systemctl restart docker"
+  read -r -p "Run this fix now? [y/N] " confirm
+  if [ "$confirm" = "y" ] || [ "$confirm" = "Y" ]; then
+    sudo nvidia-ctk runtime configure --runtime=docker --set-as-default
+    sudo systemctl restart docker
+  else
+    exit 1
+  fi
+fi
+
+echo "Checking nvidia-container-runtime config..."
+config_file="/etc/nvidia-container-runtime/config.toml"
+if ! grep -Eq '^[[:space:]]*accept-nvidia-visible-devices-as-volume-mounts[[:space:]]*=[[:space:]]*true[[:space:]]*$' "$config_file"; then
+  echo "NVIDIA runtime config is missing accept-nvidia-visible-devices-as-volume-mounts = true."
+  echo "Fix: sudo nvidia-ctk config --in-place --set accept-nvidia-visible-devices-as-volume-mounts=true"
+  read -r -p "Run this fix now? [y/N] " confirm
+  if [ "$confirm" = "y" ] || [ "$confirm" = "Y" ]; then
+    sudo nvidia-ctk config --in-place --set accept-nvidia-visible-devices-as-volume-mounts=true
+  else
+    exit 1
+  fi
+fi
+
+echo "Ensuring kind cluster openrl-gpu exists..."
+if ! kind get clusters | grep -Fxq "openrl-gpu"; then
+  kind create cluster --config "$script_dir/kind-config.yaml"
+fi
+
+echo "Verifying GPU visibility inside the kind node..."
+if ! docker exec openrl-gpu-control-plane nvidia-smi -L; then
+  echo "GPU is not visible in the kind node. Recheck the runtime config, restart docker, and recreate the cluster."
+  exit 1
+fi
+
+echo "Installing NVIDIA DRA driver..."
+helm upgrade --install dra-driver-nvidia-gpu \
+  oci://registry.k8s.io/dra-driver-nvidia/charts/dra-driver-nvidia-gpu \
+  --version 0.4.1 --create-namespace --namespace dra-driver-nvidia-gpu \
+  --set gpuResourcesEnabledOverride=true \
+  --set nvidiaDriverRoot=/ \
+  --set resources.computeDomains.enabled=false
+
+echo "Waiting for gpu.nvidia.com DeviceClass..."
+deadline=$((SECONDS + 120))
+until kubectl --context kind-openrl-gpu get deviceclass gpu.nvidia.com >/dev/null 2>&1; do
+  if [ "$SECONDS" -ge "$deadline" ]; then
+    echo "Timed out waiting for DeviceClass gpu.nvidia.com."
+    exit 1
+  fi
+  sleep 2
+done
+
+echo "kind GPU cluster is ready: context kind-openrl-gpu."