♻️ Unify QDMI Slurm integration and the reusable cluster

.agent/plans/qdmi-slurm.md

@@ -0,0 +1,35 @@
+# QDMI workloads on Slurm
+
+Status: implemented and published; hosted checks pending.
+
+## Scope and decisions
+
+- Slurm licenses identify devices and limit allocations. Use ordinary job
+  environments for catalogue and credential configuration; no SPANK module is
+  needed by MQT Core, IQM, or Braket.
+- A privileged site monitor reserves all licenses for an unavailable device.
+  Block before probing; only a successful bounded check reopens scheduling.
+  Running jobs continue. Polling needs supervision and is not a reservation at
+  the remote device service.
+- The checker uses installed Python catalogue metadata when launched through
+  Python and bounded process-tree cleanup on POSIX and Windows.
+- The scalable Docker cluster is shared by tests and demonstrations. Credentials
+  enter at runtime. IQM Emerald mock and Braket SV1 execute small workloads in
+  native and wheel modes; no quantum hardware is in scope.
+- Keep the unreleased Core work consolidated in #2599 and update the two device
+  PRs against upstream main. Preserve credentialed ordinary CI lanes.
+
+## Validation
+
+Run checker CLI/discovery and descendant-cleanup tests, the focused runner
+suite, and a real Slurm availability block/recovery scenario. Exercise both
+credentialed device workloads in native and wheel modes. Run repository lint and
+full-file C++ lint. Record local and hosted results separately; publishing is
+not a request to monitor CI.
+
+Local validation: 48 focused Python cases and seven native tests pass. The
+three-node cluster proves license capacity, outage blocking, and recovery.
+Credentialed Emerald mock and SV1 tests pass in both installation modes.
+Executable documentation and lint pass. Windows execution awaits hosted CI. The
+upstream #2726 portable-CI condition is mirrored until it merges; the
+Cache.cmake workaround is removed.

.github/workflows/slurm.yml

@@ -11,8 +11,11 @@ on:
       - "cmake/**"
       - "include/mqt-core/qdmi/**"
       - "pyproject.toml"
+      - "python/mqt/core/_commands.py"
       - "python/mqt/core/qdmi/**"
       - "src/qdmi/**"
+      - "docker/slurm/**"
+      - "examples/slurm/**"
       - "test/slurm/**"
       - "test/qdmi/**"
       - "test/python/qdmi/**"
@@ -26,8 +29,11 @@ on:
       - "cmake/**"
       - "include/mqt-core/qdmi/**"
       - "pyproject.toml"
+      - "python/mqt/core/_commands.py"
       - "python/mqt/core/qdmi/**"
       - "src/qdmi/**"
+      - "docker/slurm/**"
+      - "examples/slurm/**"
       - "test/slurm/**"
       - "test/qdmi/**"
       - "test/python/qdmi/**"
@@ -44,7 +50,7 @@ permissions:
 
 jobs:
   slurm:
-    name: Slurm 25.11 and QDMI devices
+    name: Slurm scheduling and QDMI execution
     runs-on: ubuntu-26.04
     timeout-minutes: 45
     env:
@@ -66,7 +72,7 @@ jobs:
       - name: Check the Slurm test runner
         run: >-
           uv run --no-project --with 'pytest>=9.0.1' --python 3.14
-          pytest -o addopts= -q test/python/test_slurm_integration.py
+          pytest -o addopts= -q test/python/test_slurm_integration.py test/python/test_slurm_availability.py
 
       - name: Set up MLIR
         uses: munich-quantum-software/setup-mlir@8d3eae73d0f0196fd30c787d91ebd2018cb6c709 # v1.5.0
@@ -80,8 +86,8 @@ jobs:
 
       - name: Build the MQT Core wheel
         timeout-minutes: 15
-        run: uv build --wheel --out-dir test/slurm/dist -Ccmake.define.DEPLOY=ON
+        run: uv build --wheel --out-dir dist -Ccmake.define.DEPLOY=ON
 
       - name: Test Slurm admission and QDMI execution
         timeout-minutes: 15
-        run: uv run --no-project --python 3.14 test/slurm/run_integration.py
+        run: uv run --no-project --python 3.14 test/slurm/run_integration.py --nodes 3

.license-tools-config.json

@@ -18,6 +18,7 @@
     ".proto": "SLASH_STYLE"
   },
   "exclude": [
+    "^docker/slurm/Dockerfile$",
     "^\\.[^/]+",
     "/\\.[^/]+",
     ".*\\.qasm",

CMakeLists.txt

@@ -180,6 +180,9 @@ endif()
 
 if(BUILD_MQT_CORE_BINDINGS)
   set(MQT_CORE_WHEEL_TARGETS mqt-core-bench-bindings mqt-core-dd-bindings mqt-core-qdmi-bindings)
+  if(TARGET mqt-core-qdmi-check)
+    list(APPEND MQT_CORE_WHEEL_TARGETS mqt-core-qdmi-check)
+  endif()
   if(BUILD_MQT_CORE_MLIR)
     list(APPEND MQT_CORE_WHEEL_TARGETS mqt-cc mqt-core-bench mqt-core-mlir-bindings)
   endif()

bindings/qdmi/slurm.cpp

@@ -25,18 +25,18 @@ void registerSlurm(nb::module_& qdmiModule) {
             R"pb(Open the QDMI device named by the Slurm license environment.
 
 ``SLURM_JOB_LICENSES`` must contain one local license whose name equals a stable
-ID visible to the selected QDMI Driver. The optional count must be one. The
-function opens a fresh Client session and accepts device status ``IDLE`` or
+ID visible to the selected QDMI driver. The optional count must be one. The
+function opens a fresh driver session and accepts device status ``IDLE`` or
 ``BUSY``. It does not apply job-specific QDMI configuration or credentials.
 
 Warning:
     ``SLURM_JOB_LICENSES`` is process-mutable. This function uses it only for
     device selection. It does not verify a Slurm allocation, authenticate the
-    caller, or authorize device access. The provider or operating system must
+    caller, or authorize device access. The device implementation or operating system must
     enforce access independently.
 
 Returns:
-    mqt.core.qdmi.Device: The fresh device session.
+    mqt.core.qdmi.Device: The selected device.
 
 Raises:
     RuntimeError: If the license value or named device does not satisfy this

cmake/CompilerOptions.cmake

@@ -33,8 +33,8 @@ function(enable_project_options target_name)
       target_link_libraries(${target_name} INTERFACE --coverage)
     endif()
 
-    if(NOT DEPLOY)
-      # only include machine-specific optimizations when building for the host machine
+    if(NOT DEPLOY AND NOT DEFINED ENV{CI})
+      # CI caches can reuse object files on runners with different CPUs.
       check_cxx_compiler_flag(-mtune=native HAS_MTUNE_NATIVE)
       if(HAS_MTUNE_NATIVE)
         target_compile_options(${target_name} INTERFACE -mtune=native)

docker/slurm/Dockerfile

@@ -0,0 +1,94 @@
+# syntax=docker/dockerfile:1.19
+
+# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM
+# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH
+# All rights reserved.
+#
+# SPDX-License-Identifier: MIT
+#
+# Licensed under the MIT License
+
+FROM ghcr.io/astral-sh/uv:0.12.1 AS uv
+
+FROM ubuntu:26.04 AS base
+
+ENV container=docker
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        ca-certificates \
+        libcurl4t64 \
+        dbus \
+        munge \
+        python3 \
+        slurm-wlm \
+        systemd \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=uv /uv /uvx /bin/
+COPY --chmod=755 start.sh /usr/local/sbin/mqt-slurm-start
+COPY node.conf /etc/systemd/system/slurmd.service.d/mqt-node.conf
+
+RUN useradd --uid 10000 --user-group --create-home mqt-test \
+    && systemctl disable slurmctld.service slurmd.service
+
+RUN --mount=type=bind,from=dist,target=/tmp/dist \
+    --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system --break-system-packages --link-mode=copy /tmp/dist/mqt_core-*.whl
+
+ARG SETUP_SCRIPT=""
+
+FROM scratch AS provider-source
+
+COPY --from=workload \
+    --exclude=.git --exclude=.github --exclude=build --exclude=build-* --exclude=cmake-build-* \
+    --exclude=.cache --exclude=.ccache --exclude=.venv* --exclude=.nox --exclude=CMakeUserPresets.json \
+    --exclude=.ruff_cache --exclude=.rumdl_cache --exclude=.pytest_cache --exclude=.mypy_cache \
+    --exclude=**/__pycache__ --exclude=dist --exclude=test/slurm/runtime --exclude=test/slurm/dist \
+    . /workload
+
+FROM ubuntu:26.04 AS provider-build
+
+COPY --from=uv /uv /uvx /bin/
+
+ARG SETUP_SCRIPT=""
+RUN if [ -n "$SETUP_SCRIPT" ]; then \
+        apt-get update \
+        && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+            build-essential ca-certificates cmake git libcurl4-openssl-dev libssl-dev python3; \
+    fi
+
+ARG PROVIDER_RUNTIME_COMPONENT=""
+RUN --mount=type=bind,from=provider-source,source=/workload,target=/workload \
+    --mount=type=cache,target=/root/.cache/uv \
+    mkdir -p /opt/provider/wheels /opt/provider/native \
+    && if [ -n "$SETUP_SCRIPT" ]; then \
+        CMAKE_BUILD_PARALLEL_LEVEL=2 uv build /workload --wheel --out-dir /opt/provider/wheels \
+            --config-setting=build-dir=/tmp/provider-build \
+        && cmake --install /tmp/provider-build --prefix /opt/provider/native \
+            --component "${PROVIDER_RUNTIME_COMPONENT:?Set the provider Runtime component}"; \
+    fi
+
+FROM base AS common
+
+ARG PROVIDER_INSTALL_MODE="native"
+RUN --mount=type=bind,from=dist,target=/tmp/dist \
+    --mount=type=bind,from=provider-source,source=/workload,target=/workload \
+    --mount=type=bind,from=provider-build,source=/opt/provider,target=/tmp/provider \
+    --mount=type=cache,target=/root/.cache/uv \
+    if [ -n "$SETUP_SCRIPT" ]; then \
+        case "$PROVIDER_INSTALL_MODE" in native|wheel) ;; *) exit 1 ;; esac \
+        && set -- /tmp/provider/wheels/*.whl \
+        && test "$#" -eq 1 && test -f "$1" && provider_wheel="$1[qiskit]" \
+        && set -- /tmp/dist/mqt_core-*.whl \
+        && test "$#" -eq 1 && test -f "$1" \
+        && printf 'mqt-core[qiskit,pennylane] @ file://%s\n' "$1" > /tmp/core-override.txt \
+        && uv pip install --system --break-system-packages --link-mode=copy \
+            --overrides /tmp/core-override.txt "$provider_wheel" "$1[qiskit,pennylane]" \
+        && if [ "$PROVIDER_INSTALL_MODE" = native ]; then cp -a /tmp/provider/native /opt/provider-native; fi \
+        && /bin/sh "/workload/$SETUP_SCRIPT"; \
+    fi
+
+STOPSIGNAL SIGRTMIN+3
+CMD ["/usr/local/sbin/mqt-slurm-start", "node"]

docker/slurm/README.md

@@ -0,0 +1,59 @@
+# Local Slurm cluster
+
+Run a small Slurm cluster for QDMI demonstrations and integration tests. It uses
+one controller and as many compute containers as requested. Each compute node
+offers two CPUs and 512 MiB of scheduled memory.
+
+Use rootful Docker Compose on a disposable Linux host with cgroup v2. The
+containers are privileged and share the host cgroup hierarchy so that Slurm can
+enforce job CPU and memory allocations. This setup is for local use; it does not
+configure a production cluster.
+
+From the MQT Core checkout, build exactly one wheel and prepare the shared
+files:
+
+```console
+uv build --wheel --out-dir dist -Ccmake.define.DEPLOY=ON
+sh docker/slurm/prepare.sh
+docker compose -f docker/slurm/compose.yml up --build -d --wait --scale node=2
+```
+
+Submit jobs as the unprivileged user shared by all nodes:
+
+```console
+docker compose -f docker/slurm/compose.yml exec --user 10000:10000 controller \
+  srun --licenses=mqt.sc.default python3 /workspace/test/slurm/sc_job.py
+docker compose -f docker/slurm/compose.yml exec controller sinfo
+docker compose -f docker/slurm/compose.yml up -d --wait --scale node=4
+```
+
+Slurm registers each compute container dynamically using its unique hostname.
+The default partition accepts up to 128 nodes. Scaling down stops containers;
+Slurm retains their inactive node records until `scontrol delete NodeName=...`
+or a fresh cluster is created. Drain nodes and wait for their jobs before
+scaling down an active cluster.
+
+The shared `/jobs` directory is `build/slurm/jobs` on the host. Edit
+`build/slurm/slurm.conf` to change the license counts or Slurm configuration,
+then run `scontrol reconfigure` in the controller. Stop the cluster with:
+
+```console
+docker compose -f docker/slurm/compose.yml down --volumes
+rm -r build/slurm
+```
+
+The image and Docker build cache remain available. To run isolated clusters, use
+a different Compose `--project-name`, set `MQT_CORE_SLURM_RUNTIME` to an
+absolute directory, and pass that directory to `prepare.sh`.
+
+The integration runner in `test/slurm/run_integration.py` uses these same images
+and services. Its test-only overlay adds daemon environment sentinels; the
+reusable image contains no test configuration. Run it with `--nodes 3` to check
+a different cluster size. `MQT_CORE_SLURM_DIST` selects an existing directory
+containing one MQT Core wheel. Device implementation tests can extend the image
+with `MQT_CORE_SLURM_WORKLOAD`, `MQT_CORE_SLURM_SETUP_SCRIPT`, and the build
+arguments `PROVIDER_RUNTIME_COMPONENT` and `PROVIDER_INSTALL_MODE` (`native` or
+`wheel`). Set build arguments on `controller`; `node` reuses that image. Device
+implementation overlays supply credentials to the submission container at
+runtime. Slurm exports these settings to the workload; credentials are never
+part of the image build.

docker/slurm/compose.yml

@@ -0,0 +1,56 @@
+# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM
+# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH
+# All rights reserved.
+#
+# SPDX-License-Identifier: MIT
+#
+# Licensed under the MIT License
+
+x-cluster: &cluster
+  privileged: true
+  cgroup: host
+  tmpfs:
+    - /run
+    - /run/lock
+  volumes:
+    - /sys/fs/cgroup:/sys/fs/cgroup:rw
+    - ${MQT_CORE_SLURM_CORE:-../..}:/workspace:ro
+    - ${MQT_CORE_SLURM_WORKLOAD:-.}:/workload:ro
+    - ${MQT_CORE_SLURM_RUNTIME:-../../build/slurm}:/runtime
+    - ${MQT_CORE_SLURM_RUNTIME:-../../build/slurm}/jobs:/jobs
+    - ${MQT_CORE_SLURM_RUNTIME:-../../build/slurm}/slurm.conf:/etc/slurm/slurm.conf:ro
+    - ./cgroup.conf:/etc/slurm/cgroup.conf:ro
+
+x-healthcheck: &healthcheck
+  interval: 2s
+  timeout: 2s
+  retries: 30
+  start_period: 5s
+
+services:
+  controller:
+    <<: *cluster
+    build:
+      context: .
+      additional_contexts:
+        workload: ${MQT_CORE_SLURM_WORKLOAD:-.}
+        dist: ${MQT_CORE_SLURM_DIST:-../../dist}
+      args:
+        SETUP_SCRIPT: ${MQT_CORE_SLURM_SETUP_SCRIPT:-}
+    hostname: controller
+    command: [/usr/local/sbin/mqt-slurm-start, controller]
+    healthcheck:
+      <<: *healthcheck
+      test:
+        [CMD, systemctl, is-active, --quiet, munge.service, slurmctld.service]
+
+  node:
+    <<: *cluster
+    image: ${COMPOSE_PROJECT_NAME:-slurm}-controller
+    pull_policy: never
+    depends_on:
+      controller:
+        condition: service_healthy
+    healthcheck:
+      <<: *healthcheck
+      test: [CMD, systemctl, is-active, --quiet, munge.service, slurmd.service]

docker/slurm/node.conf

@@ -0,0 +1,11 @@
+# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM
+# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH
+# All rights reserved.
+#
+# SPDX-License-Identifier: MIT
+#
+# Licensed under the MIT License
+
+[Service]
+ExecStart=
+ExecStart=/usr/sbin/slurmd --systemd -Z --conf "CPUs=2 Boards=1 SocketsPerBoard=1 CoresPerSocket=2 ThreadsPerCore=1 RealMemory=512"

docker/slurm/prepare.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM
+# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH
+# All rights reserved.
+#
+# SPDX-License-Identifier: MIT
+#
+# Licensed under the MIT License
+
+set -eu
+umask 077
+runtime=${1:-"$(dirname "$0")/../../build/slurm"}
+mkdir -p "$(dirname "$runtime")"
+mkdir "$runtime"
+head -c 1024 /dev/urandom > "$runtime/munge.key"
+mkdir "$runtime/jobs"
+chmod 1777 "$runtime/jobs"
+cp "$(dirname "$0")/slurm.conf" "$runtime/slurm.conf"
+chmod 644 "$runtime/slurm.conf"