From 10ff47cb6b7a6f90f4c5c271f4f0fcc05ae9d926 Mon Sep 17 00:00:00 2001 From: ShubyM Date: Thu, 13 Aug 2026 13:05:29 -0400 Subject: [PATCH] dev: real-GPU kind cluster with DRA passthrough One command turns a Linux box with NVIDIA GPUs into a single-node cluster whose GPUs are real, allocatable DRA devices -- the same gpu.nvidia.com DeviceClass and ResourceSlices a production DRA cluster publishes. hack/kind/kind-up.sh is idempotent with preflight checks and offered fixes (nvidia runtime as docker default, volume-mount device injection); bootstrap.sh provisions a fresh Ubuntu VM from nothing. Self-contained: no manifests, no workloads, no opinions about what runs on it. Anything that consumes DRA can be developed against it. --- docs/setup/kind-gpu-cluster.md | 47 ++++++++++ hack/kind/bootstrap.sh | 156 +++++++++++++++++++++++++++++++++ hack/kind/kind-config.yaml | 18 ++++ hack/kind/kind-up.sh | 92 +++++++++++++++++++ 4 files changed, 313 insertions(+) create mode 100644 docs/setup/kind-gpu-cluster.md create mode 100755 hack/kind/bootstrap.sh create mode 100644 hack/kind/kind-config.yaml create mode 100755 hack/kind/kind-up.sh diff --git a/docs/setup/kind-gpu-cluster.md b/docs/setup/kind-gpu-cluster.md new file mode 100644 index 00000000..f30f2e3b --- /dev/null +++ b/docs/setup/kind-gpu-cluster.md @@ -0,0 +1,47 @@ +# A real-GPU kind cluster + +One command turns a Linux box with NVIDIA GPUs into a single-node Kubernetes +cluster where the GPUs are real, allocatable Dynamic Resource Allocation +devices — the same `gpu.nvidia.com` DeviceClass and ResourceSlices a +production GKE DRA cluster publishes, without a cloud cluster. + +``` +hack/kind/bootstrap.sh # fresh Ubuntu VM only: installs docker, kind, + # kubectl, helm, nvidia-container-toolkit, uv +hack/kind/kind-up.sh # idempotent: cluster + GPU passthrough + DRA driver +``` + +After `kind-up.sh`, context `kind-openrl-gpu` has: + +- the host's GPUs visible inside the kind node (`docker exec + openrl-gpu-control-plane nvidia-smi -L`), injected by + nvidia-container-toolkit's volume-mount device mode; +- the NVIDIA DRA driver (`dra-driver-nvidia-gpu` chart) publishing + ResourceSlices, with the `gpu.nvidia.com` DeviceClass registered; +- Kubernetes 1.34, where DRA is GA — no feature gates. + +Anything that consumes DRA can then be developed against it: create a +ResourceClaim, reference it from a pod, and the allocation is a real GPU. + +## How the passthrough works + +Three host-side settings, all checked (and offered as fixes) by `kind-up.sh`: + +1. docker's default runtime is `nvidia`; +2. `accept-nvidia-visible-devices-as-volume-mounts = true` in + `/etc/nvidia-container-runtime/config.toml`; +3. the kind node mounts `/var/run/nvidia-container-devices/all`, which the + nvidia runtime interprets as "inject every GPU into this container". + +The kind node also carries the `feature.node.kubernetes.io/pci-10de.present` +label statically, because the DRA driver's kubelet plugin selects nodes by +that node-feature-discovery label and kind runs no NFD. + +## Fidelity notes + +- One node: everything schedules onto the control plane. Multi-node GPU + topologies, real network fabrics, and node failure modes are out of scope. +- The GPUs are shared with the host: anything else using them (a local + training run, another cluster on the same box) will fight over memory. +- The cluster is meant to persist between runs; `kind delete cluster --name + openrl-gpu` removes it. diff --git a/hack/kind/bootstrap.sh b/hack/kind/bootstrap.sh new file mode 100755 index 00000000..37e51c93 --- /dev/null +++ b/hack/kind/bootstrap.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +set -euo pipefail + +run_root() { + if [ "${EUID}" -eq 0 ]; then + "$@" + else + sudo "$@" + fi +} + +apt_update() { + run_root env DEBIAN_FRONTEND=noninteractive apt-get -y -qq update +} + +apt_install() { + run_root env DEBIAN_FRONTEND=noninteractive apt-get -y -qq install "$@" +} + +ensure_apt_prereqs() { + apt_update + apt_install ca-certificates curl gnupg make +} + +install_docker() { + echo "Installing Docker..." + ensure_apt_prereqs + run_root install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | run_root gpg --batch --yes --dearmor -o /etc/apt/keyrings/docker.gpg + run_root chmod a+r /etc/apt/keyrings/docker.gpg + + . /etc/os-release + arch="$(dpkg --print-architecture)" + echo "deb [arch=${arch} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${VERSION_CODENAME} stable" | + run_root tee /etc/apt/sources.list.d/docker.list >/dev/null + + apt_update + apt_install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + run_root systemctl enable --now docker +} + +install_nvidia_container_toolkit() { + echo "Installing nvidia-container-toolkit..." + ensure_apt_prereqs + curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | + run_root gpg --batch --yes --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg + curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | + sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | + run_root tee /etc/apt/sources.list.d/nvidia-container-toolkit.list >/dev/null + + apt_update + apt_install nvidia-container-toolkit +} + +install_kind() { + echo "Installing kind..." + ensure_apt_prereqs + tmp="$(mktemp)" + curl -fsSL -o "$tmp" https://kind.sigs.k8s.io/dl/v0.30.0/kind-linux-amd64 + run_root install -m 0755 "$tmp" /usr/local/bin/kind + rm -f "$tmp" +} + +install_kubectl() { + echo "Installing kubectl..." + ensure_apt_prereqs + stable="$(curl -Ls https://dl.k8s.io/release/stable.txt)" + tmp="$(mktemp)" + curl -fsSL -o "$tmp" "https://dl.k8s.io/release/${stable}/bin/linux/amd64/kubectl" + run_root install -m 0755 "$tmp" /usr/local/bin/kubectl + rm -f "$tmp" +} + +install_helm() { + echo "Installing Helm..." + ensure_apt_prereqs + curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +} + +install_uv() { + echo "Installing uv..." + curl -LsSf https://astral.sh/uv/install.sh | run_root env UV_INSTALL_DIR=/usr/local/bin sh +} + +if [ ! -r /etc/os-release ]; then + echo "This bootstrap script requires Ubuntu." + exit 1 +fi + +. /etc/os-release +if [ "${ID:-}" != "ubuntu" ]; then + echo "This bootstrap script requires Ubuntu." + exit 1 +fi + +echo "Checking NVIDIA driver..." +if ! command -v nvidia-smi >/dev/null 2>&1 || ! nvidia-smi >/dev/null 2>&1; then + echo "Install the NVIDIA driver (>= r550) first" + exit 1 +fi +driver_version="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -n 1)" +echo "Detected NVIDIA driver ${driver_version}" + +if command -v docker >/dev/null 2>&1; then + echo "Docker already installed." +else + install_docker +fi + +if command -v nvidia-ctk >/dev/null 2>&1; then + echo "nvidia-container-toolkit already installed." +else + install_nvidia_container_toolkit +fi + +if command -v kind >/dev/null 2>&1; then + echo "kind already installed." +else + install_kind +fi + +if command -v kubectl >/dev/null 2>&1; then + echo "kubectl already installed." +else + install_kubectl +fi + +if command -v helm >/dev/null 2>&1; then + echo "Helm already installed." +else + install_helm +fi + +if command -v uv >/dev/null 2>&1; then + echo "uv already installed." +else + install_uv +fi + +current_user="${USER:-$(id -un)}" +if [ "${EUID}" -ne 0 ] && ! id -nG "${current_user}" | tr ' ' '\n' | grep -Fxq docker; then + echo "Adding ${current_user} to the docker group..." + run_root usermod -aG docker "${current_user}" + echo "Log out and log back in for docker group membership to take effect." +fi + +echo "Installed versions:" +docker --version +nvidia-ctk --version +kind --version +kubectl version --client=true +helm version --short + +echo "Next steps:" +echo " bash hack/kind/kind-up.sh (or: make kind-up)" +echo " make kind-images && make kind-deploy && make kind-status" diff --git a/hack/kind/kind-config.yaml b/hack/kind/kind-config.yaml new file mode 100644 index 00000000..6ec68e46 --- /dev/null +++ b/hack/kind/kind-config.yaml @@ -0,0 +1,18 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: openrl-gpu +nodes: + # Kubernetes 1.34 has DRA GA, so no feature gates are needed. + - role: control-plane + image: kindest/node:v1.34.0 + labels: + # The NVIDIA DRA driver's kubelet plugin schedules only onto nodes with + # node-feature-discovery PCI labels (10de = NVIDIA vendor id). kind has + # no NFD, so set the label statically. + feature.node.kubernetes.io/pci-10de.present: "true" + extraMounts: + # nvidia-container-toolkit injects all host GPUs into the node container + # when it sees this mount; requires the nvidia runtime as docker default + # AND accept-nvidia-visible-devices-as-volume-mounts=true. + - hostPath: /dev/null + containerPath: /var/run/nvidia-container-devices/all diff --git a/hack/kind/kind-up.sh b/hack/kind/kind-up.sh new file mode 100755 index 00000000..9e3d1515 --- /dev/null +++ b/hack/kind/kind-up.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Bring up (or reuse) the real-GPU kind cluster: host GPUs passed through to +# the kind node via nvidia-container-toolkit device injection, NVIDIA DRA +# driver installed, gpu.nvidia.com DeviceClass registered. Idempotent -- safe +# to run before every smoke. Linux with an NVIDIA driver only. +# +# The cluster name is fixed to openrl-gpu (kind reads it from the config). +set -euo pipefail + +install_hint() { + case "$1" in + docker) echo "Missing docker. Install Docker Engine: https://docs.docker.com/engine/install/" ;; + kind) echo "Missing kind. Install kind >= 0.30: https://kind.sigs.k8s.io/docs/user/quick-start/#installation" ;; + kubectl) echo "Missing kubectl. Install kubectl: https://kubernetes.io/docs/tasks/tools/" ;; + helm) echo "Missing helm. Install Helm 3: https://helm.sh/docs/intro/install/" ;; + nvidia-smi) echo "Missing nvidia-smi. Install the NVIDIA Linux driver >= r550." ;; + nvidia-ctk) echo "Missing nvidia-ctk. Install nvidia-container-toolkit: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html" ;; + *) echo "Missing $1." ;; + esac +} + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "Checking required tools..." +missing_tools=0 +for tool in docker kind kubectl helm nvidia-smi nvidia-ctk; do + if ! command -v "$tool" >/dev/null 2>&1; then + install_hint "$tool" + missing_tools=1 + fi +done +if [ "$missing_tools" -ne 0 ]; then + exit 1 +fi + +echo "Checking docker default runtime..." +default_runtime="$(docker info --format '{{.DefaultRuntime}}')" +if [ "$default_runtime" != "nvidia" ]; then + echo "Docker default runtime is '$default_runtime', expected 'nvidia'." + echo "Fix: sudo nvidia-ctk runtime configure --runtime=docker --set-as-default && sudo systemctl restart docker" + read -r -p "Run this fix now? [y/N] " confirm + if [ "$confirm" = "y" ] || [ "$confirm" = "Y" ]; then + sudo nvidia-ctk runtime configure --runtime=docker --set-as-default + sudo systemctl restart docker + else + exit 1 + fi +fi + +echo "Checking nvidia-container-runtime config..." +config_file="/etc/nvidia-container-runtime/config.toml" +if ! grep -Eq '^[[:space:]]*accept-nvidia-visible-devices-as-volume-mounts[[:space:]]*=[[:space:]]*true[[:space:]]*$' "$config_file"; then + echo "NVIDIA runtime config is missing accept-nvidia-visible-devices-as-volume-mounts = true." + echo "Fix: sudo nvidia-ctk config --in-place --set accept-nvidia-visible-devices-as-volume-mounts=true" + read -r -p "Run this fix now? [y/N] " confirm + if [ "$confirm" = "y" ] || [ "$confirm" = "Y" ]; then + sudo nvidia-ctk config --in-place --set accept-nvidia-visible-devices-as-volume-mounts=true + else + exit 1 + fi +fi + +echo "Ensuring kind cluster openrl-gpu exists..." +if ! kind get clusters | grep -Fxq "openrl-gpu"; then + kind create cluster --config "$script_dir/kind-config.yaml" +fi + +echo "Verifying GPU visibility inside the kind node..." +if ! docker exec openrl-gpu-control-plane nvidia-smi -L; then + echo "GPU is not visible in the kind node. Recheck the runtime config, restart docker, and recreate the cluster." + exit 1 +fi + +echo "Installing NVIDIA DRA driver..." +helm upgrade --install dra-driver-nvidia-gpu \ + oci://registry.k8s.io/dra-driver-nvidia/charts/dra-driver-nvidia-gpu \ + --version 0.4.1 --create-namespace --namespace dra-driver-nvidia-gpu \ + --set gpuResourcesEnabledOverride=true \ + --set nvidiaDriverRoot=/ \ + --set resources.computeDomains.enabled=false + +echo "Waiting for gpu.nvidia.com DeviceClass..." +deadline=$((SECONDS + 120)) +until kubectl --context kind-openrl-gpu get deviceclass gpu.nvidia.com >/dev/null 2>&1; do + if [ "$SECONDS" -ge "$deadline" ]; then + echo "Timed out waiting for DeviceClass gpu.nvidia.com." + exit 1 + fi + sleep 2 +done + +echo "kind GPU cluster is ready: context kind-openrl-gpu."