diff --git a/.agent/plans/qdmi-slurm.md b/.agent/plans/qdmi-slurm.md new file mode 100644 index 0000000000..47ca5aefb1 --- /dev/null +++ b/.agent/plans/qdmi-slurm.md @@ -0,0 +1,35 @@ +# QDMI workloads on Slurm + +Status: implemented and published; hosted checks pending. + +## Scope and decisions + +- Slurm licenses identify devices and limit allocations. Use ordinary job + environments for catalogue and credential configuration; no SPANK module is + needed by MQT Core, IQM, or Braket. +- A privileged site monitor reserves all licenses for an unavailable device. + Block before probing; only a successful bounded check reopens scheduling. + Running jobs continue. Polling needs supervision and is not a reservation at + the remote device service. +- The checker uses installed Python catalogue metadata when launched through + Python and bounded process-tree cleanup on POSIX and Windows. +- The scalable Docker cluster is shared by tests and demonstrations. Credentials + enter at runtime. IQM Emerald mock and Braket SV1 execute small workloads in + native and wheel modes; no quantum hardware is in scope. +- Keep the unreleased Core work consolidated in #2599 and update the two device + PRs against upstream main. Preserve credentialed ordinary CI lanes. + +## Validation + +Run checker CLI/discovery and descendant-cleanup tests, the focused runner +suite, and a real Slurm availability block/recovery scenario. Exercise both +credentialed device workloads in native and wheel modes. Run repository lint and +full-file C++ lint. Record local and hosted results separately; publishing is +not a request to monitor CI. + +Local validation: 48 focused Python cases and seven native tests pass. The +three-node cluster proves license capacity, outage blocking, and recovery. +Credentialed Emerald mock and SV1 tests pass in both installation modes. +Executable documentation and lint pass. Windows execution awaits hosted CI. The +upstream #2726 portable-CI condition is mirrored until it merges; the +Cache.cmake workaround is removed. diff --git a/.github/workflows/slurm.yml b/.github/workflows/slurm.yml index ecd42b8372..b995a9e9fb 100644 --- a/.github/workflows/slurm.yml +++ b/.github/workflows/slurm.yml @@ -11,8 +11,11 @@ on: - "cmake/**" - "include/mqt-core/qdmi/**" - "pyproject.toml" + - "python/mqt/core/_commands.py" - "python/mqt/core/qdmi/**" - "src/qdmi/**" + - "docker/slurm/**" + - "examples/slurm/**" - "test/slurm/**" - "test/qdmi/**" - "test/python/qdmi/**" @@ -26,8 +29,11 @@ on: - "cmake/**" - "include/mqt-core/qdmi/**" - "pyproject.toml" + - "python/mqt/core/_commands.py" - "python/mqt/core/qdmi/**" - "src/qdmi/**" + - "docker/slurm/**" + - "examples/slurm/**" - "test/slurm/**" - "test/qdmi/**" - "test/python/qdmi/**" @@ -44,7 +50,7 @@ permissions: jobs: slurm: - name: Slurm 25.11 and QDMI devices + name: Slurm scheduling and QDMI execution runs-on: ubuntu-26.04 timeout-minutes: 45 env: @@ -66,7 +72,7 @@ jobs: - name: Check the Slurm test runner run: >- uv run --no-project --with 'pytest>=9.0.1' --python 3.14 - pytest -o addopts= -q test/python/test_slurm_integration.py + pytest -o addopts= -q test/python/test_slurm_integration.py test/python/test_slurm_availability.py - name: Set up MLIR uses: munich-quantum-software/setup-mlir@8d3eae73d0f0196fd30c787d91ebd2018cb6c709 # v1.5.0 @@ -80,8 +86,8 @@ jobs: - name: Build the MQT Core wheel timeout-minutes: 15 - run: uv build --wheel --out-dir test/slurm/dist -Ccmake.define.DEPLOY=ON + run: uv build --wheel --out-dir dist -Ccmake.define.DEPLOY=ON - name: Test Slurm admission and QDMI execution timeout-minutes: 15 - run: uv run --no-project --python 3.14 test/slurm/run_integration.py + run: uv run --no-project --python 3.14 test/slurm/run_integration.py --nodes 3 diff --git a/.license-tools-config.json b/.license-tools-config.json index 5080d2adaa..31bd13dbb0 100644 --- a/.license-tools-config.json +++ b/.license-tools-config.json @@ -18,6 +18,7 @@ ".proto": "SLASH_STYLE" }, "exclude": [ + "^docker/slurm/Dockerfile$", "^\\.[^/]+", "/\\.[^/]+", ".*\\.qasm", diff --git a/CMakeLists.txt b/CMakeLists.txt index 654c661958..82254173c7 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,6 +180,9 @@ endif() if(BUILD_MQT_CORE_BINDINGS) set(MQT_CORE_WHEEL_TARGETS mqt-core-bench-bindings mqt-core-dd-bindings mqt-core-qdmi-bindings) + if(TARGET mqt-core-qdmi-check) + list(APPEND MQT_CORE_WHEEL_TARGETS mqt-core-qdmi-check) + endif() if(BUILD_MQT_CORE_MLIR) list(APPEND MQT_CORE_WHEEL_TARGETS mqt-cc mqt-core-bench mqt-core-mlir-bindings) endif() diff --git a/bindings/qdmi/slurm.cpp b/bindings/qdmi/slurm.cpp index 7ca641f184..8ddac54f91 100644 --- a/bindings/qdmi/slurm.cpp +++ b/bindings/qdmi/slurm.cpp @@ -25,18 +25,18 @@ void registerSlurm(nb::module_& qdmiModule) { R"pb(Open the QDMI device named by the Slurm license environment. ``SLURM_JOB_LICENSES`` must contain one local license whose name equals a stable -ID visible to the selected QDMI Driver. The optional count must be one. The -function opens a fresh Client session and accepts device status ``IDLE`` or +ID visible to the selected QDMI driver. The optional count must be one. The +function opens a fresh driver session and accepts device status ``IDLE`` or ``BUSY``. It does not apply job-specific QDMI configuration or credentials. Warning: ``SLURM_JOB_LICENSES`` is process-mutable. This function uses it only for device selection. It does not verify a Slurm allocation, authenticate the - caller, or authorize device access. The provider or operating system must + caller, or authorize device access. The device implementation or operating system must enforce access independently. Returns: - mqt.core.qdmi.Device: The fresh device session. + mqt.core.qdmi.Device: The selected device. Raises: RuntimeError: If the license value or named device does not satisfy this diff --git a/cmake/CompilerOptions.cmake b/cmake/CompilerOptions.cmake index 08df146a25..e258e16f9c 100644 --- a/cmake/CompilerOptions.cmake +++ b/cmake/CompilerOptions.cmake @@ -33,8 +33,8 @@ function(enable_project_options target_name) target_link_libraries(${target_name} INTERFACE --coverage) endif() - if(NOT DEPLOY) - # only include machine-specific optimizations when building for the host machine + if(NOT DEPLOY AND NOT DEFINED ENV{CI}) + # CI caches can reuse object files on runners with different CPUs. check_cxx_compiler_flag(-mtune=native HAS_MTUNE_NATIVE) if(HAS_MTUNE_NATIVE) target_compile_options(${target_name} INTERFACE -mtune=native) diff --git a/docker/slurm/Dockerfile b/docker/slurm/Dockerfile new file mode 100644 index 0000000000..ae66868a70 --- /dev/null +++ b/docker/slurm/Dockerfile @@ -0,0 +1,94 @@ +# syntax=docker/dockerfile:1.19 + +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +FROM ghcr.io/astral-sh/uv:0.12.1 AS uv + +FROM ubuntu:26.04 AS base + +ENV container=docker + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + ca-certificates \ + libcurl4t64 \ + dbus \ + munge \ + python3 \ + slurm-wlm \ + systemd \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=uv /uv /uvx /bin/ +COPY --chmod=755 start.sh /usr/local/sbin/mqt-slurm-start +COPY node.conf /etc/systemd/system/slurmd.service.d/mqt-node.conf + +RUN useradd --uid 10000 --user-group --create-home mqt-test \ + && systemctl disable slurmctld.service slurmd.service + +RUN --mount=type=bind,from=dist,target=/tmp/dist \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system --break-system-packages --link-mode=copy /tmp/dist/mqt_core-*.whl + +ARG SETUP_SCRIPT="" + +FROM scratch AS provider-source + +COPY --from=workload \ + --exclude=.git --exclude=.github --exclude=build --exclude=build-* --exclude=cmake-build-* \ + --exclude=.cache --exclude=.ccache --exclude=.venv* --exclude=.nox --exclude=CMakeUserPresets.json \ + --exclude=.ruff_cache --exclude=.rumdl_cache --exclude=.pytest_cache --exclude=.mypy_cache \ + --exclude=**/__pycache__ --exclude=dist --exclude=test/slurm/runtime --exclude=test/slurm/dist \ + . /workload + +FROM ubuntu:26.04 AS provider-build + +COPY --from=uv /uv /uvx /bin/ + +ARG SETUP_SCRIPT="" +RUN if [ -n "$SETUP_SCRIPT" ]; then \ + apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + build-essential ca-certificates cmake git libcurl4-openssl-dev libssl-dev python3; \ + fi + +ARG PROVIDER_RUNTIME_COMPONENT="" +RUN --mount=type=bind,from=provider-source,source=/workload,target=/workload \ + --mount=type=cache,target=/root/.cache/uv \ + mkdir -p /opt/provider/wheels /opt/provider/native \ + && if [ -n "$SETUP_SCRIPT" ]; then \ + CMAKE_BUILD_PARALLEL_LEVEL=2 uv build /workload --wheel --out-dir /opt/provider/wheels \ + --config-setting=build-dir=/tmp/provider-build \ + && cmake --install /tmp/provider-build --prefix /opt/provider/native \ + --component "${PROVIDER_RUNTIME_COMPONENT:?Set the provider Runtime component}"; \ + fi + +FROM base AS common + +ARG PROVIDER_INSTALL_MODE="native" +RUN --mount=type=bind,from=dist,target=/tmp/dist \ + --mount=type=bind,from=provider-source,source=/workload,target=/workload \ + --mount=type=bind,from=provider-build,source=/opt/provider,target=/tmp/provider \ + --mount=type=cache,target=/root/.cache/uv \ + if [ -n "$SETUP_SCRIPT" ]; then \ + case "$PROVIDER_INSTALL_MODE" in native|wheel) ;; *) exit 1 ;; esac \ + && set -- /tmp/provider/wheels/*.whl \ + && test "$#" -eq 1 && test -f "$1" && provider_wheel="$1[qiskit]" \ + && set -- /tmp/dist/mqt_core-*.whl \ + && test "$#" -eq 1 && test -f "$1" \ + && printf 'mqt-core[qiskit,pennylane] @ file://%s\n' "$1" > /tmp/core-override.txt \ + && uv pip install --system --break-system-packages --link-mode=copy \ + --overrides /tmp/core-override.txt "$provider_wheel" "$1[qiskit,pennylane]" \ + && if [ "$PROVIDER_INSTALL_MODE" = native ]; then cp -a /tmp/provider/native /opt/provider-native; fi \ + && /bin/sh "/workload/$SETUP_SCRIPT"; \ + fi + +STOPSIGNAL SIGRTMIN+3 +CMD ["/usr/local/sbin/mqt-slurm-start", "node"] diff --git a/docker/slurm/README.md b/docker/slurm/README.md new file mode 100644 index 0000000000..a854ff06c5 --- /dev/null +++ b/docker/slurm/README.md @@ -0,0 +1,59 @@ +# Local Slurm cluster + +Run a small Slurm cluster for QDMI demonstrations and integration tests. It uses +one controller and as many compute containers as requested. Each compute node +offers two CPUs and 512 MiB of scheduled memory. + +Use rootful Docker Compose on a disposable Linux host with cgroup v2. The +containers are privileged and share the host cgroup hierarchy so that Slurm can +enforce job CPU and memory allocations. This setup is for local use; it does not +configure a production cluster. + +From the MQT Core checkout, build exactly one wheel and prepare the shared +files: + +```console +uv build --wheel --out-dir dist -Ccmake.define.DEPLOY=ON +sh docker/slurm/prepare.sh +docker compose -f docker/slurm/compose.yml up --build -d --wait --scale node=2 +``` + +Submit jobs as the unprivileged user shared by all nodes: + +```console +docker compose -f docker/slurm/compose.yml exec --user 10000:10000 controller \ + srun --licenses=mqt.sc.default python3 /workspace/test/slurm/sc_job.py +docker compose -f docker/slurm/compose.yml exec controller sinfo +docker compose -f docker/slurm/compose.yml up -d --wait --scale node=4 +``` + +Slurm registers each compute container dynamically using its unique hostname. +The default partition accepts up to 128 nodes. Scaling down stops containers; +Slurm retains their inactive node records until `scontrol delete NodeName=...` +or a fresh cluster is created. Drain nodes and wait for their jobs before +scaling down an active cluster. + +The shared `/jobs` directory is `build/slurm/jobs` on the host. Edit +`build/slurm/slurm.conf` to change the license counts or Slurm configuration, +then run `scontrol reconfigure` in the controller. Stop the cluster with: + +```console +docker compose -f docker/slurm/compose.yml down --volumes +rm -r build/slurm +``` + +The image and Docker build cache remain available. To run isolated clusters, use +a different Compose `--project-name`, set `MQT_CORE_SLURM_RUNTIME` to an +absolute directory, and pass that directory to `prepare.sh`. + +The integration runner in `test/slurm/run_integration.py` uses these same images +and services. Its test-only overlay adds daemon environment sentinels; the +reusable image contains no test configuration. Run it with `--nodes 3` to check +a different cluster size. `MQT_CORE_SLURM_DIST` selects an existing directory +containing one MQT Core wheel. Device implementation tests can extend the image +with `MQT_CORE_SLURM_WORKLOAD`, `MQT_CORE_SLURM_SETUP_SCRIPT`, and the build +arguments `PROVIDER_RUNTIME_COMPONENT` and `PROVIDER_INSTALL_MODE` (`native` or +`wheel`). Set build arguments on `controller`; `node` reuses that image. Device +implementation overlays supply credentials to the submission container at +runtime. Slurm exports these settings to the workload; credentials are never +part of the image build. diff --git a/test/slurm/cgroup.conf b/docker/slurm/cgroup.conf similarity index 100% rename from test/slurm/cgroup.conf rename to docker/slurm/cgroup.conf diff --git a/docker/slurm/compose.yml b/docker/slurm/compose.yml new file mode 100644 index 0000000000..a620cdddb3 --- /dev/null +++ b/docker/slurm/compose.yml @@ -0,0 +1,56 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +x-cluster: &cluster + privileged: true + cgroup: host + tmpfs: + - /run + - /run/lock + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - ${MQT_CORE_SLURM_CORE:-../..}:/workspace:ro + - ${MQT_CORE_SLURM_WORKLOAD:-.}:/workload:ro + - ${MQT_CORE_SLURM_RUNTIME:-../../build/slurm}:/runtime + - ${MQT_CORE_SLURM_RUNTIME:-../../build/slurm}/jobs:/jobs + - ${MQT_CORE_SLURM_RUNTIME:-../../build/slurm}/slurm.conf:/etc/slurm/slurm.conf:ro + - ./cgroup.conf:/etc/slurm/cgroup.conf:ro + +x-healthcheck: &healthcheck + interval: 2s + timeout: 2s + retries: 30 + start_period: 5s + +services: + controller: + <<: *cluster + build: + context: . + additional_contexts: + workload: ${MQT_CORE_SLURM_WORKLOAD:-.} + dist: ${MQT_CORE_SLURM_DIST:-../../dist} + args: + SETUP_SCRIPT: ${MQT_CORE_SLURM_SETUP_SCRIPT:-} + hostname: controller + command: [/usr/local/sbin/mqt-slurm-start, controller] + healthcheck: + <<: *healthcheck + test: + [CMD, systemctl, is-active, --quiet, munge.service, slurmctld.service] + + node: + <<: *cluster + image: ${COMPOSE_PROJECT_NAME:-slurm}-controller + pull_policy: never + depends_on: + controller: + condition: service_healthy + healthcheck: + <<: *healthcheck + test: [CMD, systemctl, is-active, --quiet, munge.service, slurmd.service] diff --git a/docker/slurm/node.conf b/docker/slurm/node.conf new file mode 100644 index 0000000000..0c086180e1 --- /dev/null +++ b/docker/slurm/node.conf @@ -0,0 +1,11 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +[Service] +ExecStart= +ExecStart=/usr/sbin/slurmd --systemd -Z --conf "CPUs=2 Boards=1 SocketsPerBoard=1 CoresPerSocket=2 ThreadsPerCore=1 RealMemory=512" diff --git a/docker/slurm/prepare.sh b/docker/slurm/prepare.sh new file mode 100644 index 0000000000..f369f203c8 --- /dev/null +++ b/docker/slurm/prepare.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +set -eu +umask 077 +runtime=${1:-"$(dirname "$0")/../../build/slurm"} +mkdir -p "$(dirname "$runtime")" +mkdir "$runtime" +head -c 1024 /dev/urandom > "$runtime/munge.key" +mkdir "$runtime/jobs" +chmod 1777 "$runtime/jobs" +cp "$(dirname "$0")/slurm.conf" "$runtime/slurm.conf" +chmod 644 "$runtime/slurm.conf" diff --git a/test/slurm/slurm.conf b/docker/slurm/slurm.conf similarity index 65% rename from test/slurm/slurm.conf rename to docker/slurm/slurm.conf index 8bb1182833..95985254de 100644 --- a/test/slurm/slurm.conf +++ b/docker/slurm/slurm.conf @@ -6,7 +6,7 @@ # # Licensed under the MIT License -ClusterName=mqt-core-test +ClusterName=mqt-core SlurmctldHost=controller SlurmUser=slurm AuthType=auth/munge @@ -19,7 +19,8 @@ ProctrackType=proctrack/cgroup TaskPlugin=task/cgroup,task/affinity JobAcctGatherType=jobacct_gather/cgroup SelectType=select/cons_tres -SelectTypeParameters=CR_CPU +SelectTypeParameters=CR_CPU_Memory +DefMemPerCPU=256 PrologFlags=Contain SchedulerType=sched/backfill @@ -29,6 +30,5 @@ SlurmctldParameters=reconfig_on_restart Licenses=mqt.ddsim.default:2,mqt.sc.default:1 -NodeName=node1 NodeAddr=node1 CPUs=2 Boards=1 SocketsPerBoard=1 CoresPerSocket=2 ThreadsPerCore=1 RealMemory=512 State=UNKNOWN -NodeName=node2 NodeAddr=node2 CPUs=2 Boards=1 SocketsPerBoard=1 CoresPerSocket=2 ThreadsPerCore=1 RealMemory=512 State=UNKNOWN -PartitionName=compute Nodes=node1,node2 Default=YES MaxTime=INFINITE State=UP +MaxNodeCount=128 +PartitionName=compute Nodes=ALL Default=YES MaxTime=INFINITE State=UP diff --git a/test/slurm/mqt-slurm-prepare b/docker/slurm/start.sh similarity index 69% rename from test/slurm/mqt-slurm-prepare rename to docker/slurm/start.sh index a106b55932..5593f4bf4e 100755 --- a/test/slurm/mqt-slurm-prepare +++ b/docker/slurm/start.sh @@ -9,9 +9,17 @@ set -eu +case "${1:-node}" in + controller) daemon=slurmctld ;; + node) daemon=slurmd ;; + *) echo "Expected controller or node" >&2; exit 1 ;; +esac + test -s /runtime/munge.key install -d -o munge -g munge -m 0755 /etc/munge /run/munge install -o munge -g munge -m 0400 /runtime/munge.key /etc/munge/munge.key install -d -o slurm -g slurm -m 0755 /var/spool/slurmctld /var/log/slurm install -d -o root -g root -m 0755 /var/spool/slurmd +systemctl enable munge.service "$daemon.service" +exec /usr/lib/systemd/systemd diff --git a/docs/glossary.md b/docs/glossary.md index fd49176b9f..0492cd0052 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -133,14 +133,14 @@ Quantum Device Management Interface QDMI Client interface **Preferred term:** QDMI Client interface. The standard C interface consumed by - applications to open sessions, query devices, and manage jobs. Core's C++ and + applications to open sessions, query devices, and manage jobs. MQT Core's C++ and Python Client wrappers consume this interface; they are not a driver. QDMI driver **Preferred term:** QDMI driver. An implementation of the QDMI Client interface. The builtin MQT Core QDMI driver loads QDMI device libraries. A replacement driver owns its own discovery, configuration, and device access; applications must not - assume it provides Core's private driver extension. + assume it provides MQT Core's private driver extension. QDMI session **Preferred term:** QDMI session. A connection to a QDMI driver, represented diff --git a/docs/qdmi/configuration.md b/docs/qdmi/configuration.md index c881e7f645..2b7d5614d0 100644 --- a/docs/qdmi/configuration.md +++ b/docs/qdmi/configuration.md @@ -215,10 +215,10 @@ The equivalent C++ function is `qdmi::slurm::openDeviceFromLicense()` from `` or `:1`. They reject remote, compound, and non-unit license values. -The adapter opens a fresh device session from the persistent definition. It does -not replace configuration or inject credentials. Each device implementation -defines its own credential sources. The adapter accepts QDMI device status -`IDLE` and `BUSY`. It rejects all other device states. +The adapter creates a fresh driver session and selects the device with the +licensed ID. It does not replace configuration or inject credentials. Each +device implementation defines its own credential sources. The adapter accepts +QDMI device status `IDLE` and `BUSY`. It rejects all other device states. `SLURM_JOB_LICENSES` is process-mutable. The adapter uses this value only for device selection. It does not verify that Slurm allocated the license. It does diff --git a/docs/qdmi/driver.md b/docs/qdmi/driver.md index 708d9ffdb8..a5235bf9f2 100644 --- a/docs/qdmi/driver.md +++ b/docs/qdmi/driver.md @@ -119,10 +119,37 @@ integration tests. The driver is a shared library. The C++ QDMI library follows the project’s static/shared build setting and is shared in Python wheels. Device-free builds can use another QDMI driver through `driver_path` or `MQT_CORE_QDMI_DRIVER`. The -The builtin MQT Core QDMI driver can load external device libraries through +builtin MQT Core QDMI driver can load external device libraries through [QDMI device configuration](configuration.md). C++ test builds require the bundled devices available in the selected build configuration. +## Probe device availability + +The availability command probes whether a configured QDMI device is operational. +It opens the device, accepts an `IDLE` or `BUSY` status, and exits without +submitting a quantum job: + +```console +mqt-core-qdmi-check --device mqt.sc.default --timeout 10 +``` + +Run it in the workload environment with the required credentials. The Python +console script discovers +[installed device manifests](configuration.md#installed-device-manifests). The +native executable accepts additional manifests through repeated +`--manifest PATH` arguments. Invalid manifests are skipped. These manifests +retain the lowest precedence in [QDMI configuration](configuration.md); +`MQT_CORE_QDMI_CONFIG_FILE` selects an explicit catalogue. + +The timeout covers device initialization, the status query, and worker exit. It +defaults to 30 seconds and accepts whole seconds from 1 to 3600. Exit codes are +0 for availability, 1 for failure, 2 for invalid arguments, and 124 for a +timeout. Device output is suppressed to protect credentials. + +The command is included in Linux, macOS, and Windows builds and wheels (Windows +10 or Windows Server 2016 and newer). Availability is a snapshot; it does not +authorize access or reserve device capacity. + ## Python Bindings The C++ QDMI library adds owning wrappers for driver sessions, devices, sites, diff --git a/docs/qdmi/slurm.md b/docs/qdmi/slurm.md index 0ab6e59132..7eb917655e 100644 --- a/docs/qdmi/slurm.md +++ b/docs/qdmi/slurm.md @@ -1,269 +1,156 @@ # Use QDMI devices with Slurm -This example uses Slurm 25.11 or newer on Ubuntu 26.04. It has one controller, -two compute nodes, and two CPUs on each compute node. +Slurm can limit concurrent access to a quantum device through a cluster-wide +license. Use a QDMI device ID as the license name, then open that device in the +job with MQT Core. The job can compile and submit workloads through the same +QDMI interface it uses outside Slurm. -A Slurm license controls admission to a cluster-wide resource. In this setup, -the license name is a stable QDMI device ID. A license does not show provider -availability. It does not show the device queue. The QDMI provider supplies that -information when its interface supports it. +Slurm schedules jobs; the device implementation authenticates users and submits +quantum work. A license does not grant device access or reserve capacity at a +remote service. Device availability and queues can change after admission. -## Understand the control boundaries +## Run a job -This example is suitable for admission and accounting tests on a cooperative -cluster. It does not make a Slurm license an access-control credential. The -controls are independent: +Install MQT Core and the required QDMI device implementation in the workload +environment. Make its catalogue, libraries, and credentials available on the +compute nodes. See [device configuration](configuration.md) and the device +implementation's installation guide. -- Slurm admits jobs and accounts for the configured license count. -- The MQT Core adapter uses the license environment to select a client-visible - QDMI device. -- The QDMI provider reports device availability and queue data. -- The provider or the operating system authorizes access to the device. - -`SLURM_JOB_LICENSES` is process-mutable. A job can change it before it calls -`slurm.open_device_from_license()`. Thus, the function does not prove that Slurm -allocated the named license. It does not authenticate the user. It does not -authorize access. A lookup through another Slurm interface would not make MQT -Core an access-control boundary because a program can also call -`mqt.core.qdmi.open_device(device_id)` directly. - -## Install the software - -Install the same MQT Core package on each compute node. The package contains the -MQT Core QDMI interface and the bundled QDMI devices. You can use a shared -software environment or install the same wheel on each node. - -Install Slurm, Munge, and systemd. Start Munge before Slurm. Use the same Munge -key on all nodes. Keep this key outside the QDMI device configuration. - -Use the unified cgroup v2 hierarchy. Add these settings to `slurm.conf`: - -```ini -ProctrackType=proctrack/cgroup -TaskPlugin=task/cgroup,task/affinity -JobAcctGatherType=jobacct_gather/cgroup -SelectType=select/cons_tres -SelectTypeParameters=CR_CPU -``` - -Use the cgroup plugin to constrain processors and memory. For example, use this -`cgroup.conf`: - -```ini -CgroupPlugin=autodetect -ConstrainCores=yes -ConstrainRAMSpace=yes -ConstrainSwapSpace=yes -``` - -This fixture has no local device file. For strict isolation of a local device, -configure it as a Slurm GRES with a `File=` entry. Also set -`ConstrainDevices=yes` for the cgroup task plugin. Slurm can then restrict the -device files that a job can open. See the [Slurm GRES configuration] -documentation. A remote QPU has no local device file, so the provider must -enforce its authorization. - -Run `slurmd -C` on each compute node and use its output for the `NodeName` -record. The MQT Core test uses two CPUs on `node1` and `node2`. - -## Register the devices - -MQT Core installs persistent definitions for `mqt.ddsim.default`, -`mqt.sc.default`, `mqt.sc.iqm.garnet`, and `mqt.sc.iqm.emerald`, so these four -devices need no further registry file. Verify their stable IDs before you -configure Slurm: - -```console -python -c "from mqt.core.qdmi import builtin_driver; print(*builtin_driver.registered_device_ids(), sep='\n')" -``` - -For an external provider, install its shared library and QDMI manifest. You can -also add one trusted system registry file at `/etc/mqt-core/qdmi.json`. The `id` -field in that file is the stable device ID. Use the same ID as the local Slurm -license name. Do not put short-lived access tokens in this file. Use the -authentication method that the provider documents for batch jobs. - -The Slurm adapter does not supply credentials. IQM can use its configured token -source. Amazon Braket uses the AWS credential provider chain, such as an -instance role, workload identity, or AWS profile. A job can also export provider -configuration. Use provider-scoped credentials with minimum permissions. Do not -store access tokens or AWS access keys in a persistent device definition. - -Add the licenses to `slurm.conf` on the controller: +An administrator registers each device ID in `slurm.conf`, for example: ```ini -Licenses=mqt.ddsim.default:2,mqt.sc.default:1 -``` - -The DDSIM count is two. Therefore, Slurm can admit two jobs that each request -one DDSIM license. These jobs can run on different nodes. The count is a local -cluster policy. It is not a DDSIM property and it is not a per-node count. - -Restart `slurmctld` after you first add the licenses. Reconfigure the compute -nodes as required by your Slurm installation. Then inspect the configured -resources: - -```console -scontrol show lic +Licenses=mqt.sc.default:1,amazon.braket.sv1:2,iqm.emerald:1 ``` -The initial report must contain these values: - -```text -LicenseName=mqt.ddsim.default Total=2 Used=0 Free=2 Remote=no -LicenseName=mqt.sc.default Total=1 Used=0 Free=1 Remote=no -``` +The counts limit simultaneous Slurm allocations. Choose counts appropriate for +the device and the site's access policy. -## Submit a DDSIM job - -Save this program as `bell.py` in a location that all compute nodes can read: - -```python -from mqt.core.qdmi import ProgramFormat, slurm - -program = """OPENQASM 2.0; -include "qelib1.inc"; -qreg q[2]; -creg c[2]; -h q[0]; -cx q[0], q[1]; -measure q -> c; -""" - -device = slurm.open_device_from_license() -job = device.submit_job(program, ProgramFormat.QASM2, num_shots=256) -if not job.wait(60): - raise RuntimeError("DDSIM did not finish within 60 seconds") - -counts = job.get_counts() -if sum(counts.values()) != 256 or not set(counts) <= {"00", "11"}: - raise RuntimeError(f"Invalid Bell results: {counts}") -print(counts) -``` - -Save this batch script as `bell.sbatch`: +Request one device with `--licenses=ID` or `--licenses=ID:1`: ```bash #!/bin/bash -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=1 -#SBATCH --licenses=mqt.ddsim.default:1 -#SBATCH --output=bell-%j.out - -set -euo pipefail -python bell.py +#SBATCH --licenses=amazon.braket.sv1 +#SBATCH --time=00:05:00 +set -eu + +source /shared/quantum/.venv/bin/activate +export MQT_CORE_QDMI_CONFIG_FILE=/shared/quantum/devices.json +export AWS_PROFILE=research +mqt-core-qdmi-check --device amazon.braket.sv1 --timeout 10 +srun python workload.py ``` -The adapter reads `SLURM_JOB_LICENSES` and selects the persistent device -definition with the same ID. It requires one unambiguous QDMI device license. It -opens a fresh device session and checks the device status. The function accepts -`IDLE` and `BUSY`. This check is not authorization. The provider can still -reject a later submission or put the quantum task in its device queue. +In `workload.py`, select the allocated device: -Submit the job with this command: +```python +from mqt.core.qdmi import slurm -```console -sbatch bell.sbatch +device = slurm.open_device_from_license() ``` -The same open handle works with application adapters. Pass it to -{py:class}`mqt.core.plugins.qiskit.backend.QDMIBackend` or to the PennyLane -{py:class}`mqt.core.plugins.pennylane.device.QDMIDevice`. See the -{doc}`pennylane_device` guide for the PennyLane constructor. - -Open the selected device once per application process and reuse its handle for -subsequent quantum jobs. The adapter validates the license locally, opens only -the selected device, and queries its status once. Device selection uses the -ordinary QDMI driver lookup and needs no controller RPC. Provider initialization -and network requests can still dominate opening time; use provider-supported -timeout settings for network requests. - -## Check concurrent jobs - -For a scheduling test, add a sufficiently long classical post-processing step -after the Python command. For example, add `sleep 120` to `bell.sbatch`. Then -submit two jobs: +Pass `device` to the appropriate Qiskit or PennyLane adapter. MQT Core accepts +one local license with a unit count and requires the device to report `IDLE` or +`BUSY`. Compound license expressions and remote licenses are unsupported. + +The optional [availability command](driver.md#probe-device-availability) gives a +quick indication that the device is operational. Run it after activating the +workload environment and setting credentials. It does not reserve the device. +The workload still opens the device and handles submission errors normally. + +Slurm exports the submission environment by default. Use ordinary environment +variables or Slurm's `--export` option for job-specific settings. Variables set +inside a batch script are inherited by its subsequent `srun` steps. + +## Hold jobs while a device is unavailable + +A check inside a job runs after Slurm has allocated resources. To leave jobs +pending while a device is unavailable, an administrator must update scheduler +state independently of the jobs. + +Slurm supports +[license-only reservations](https://slurm.schedmd.com/reservations.html) for +unavailable shared resources. Reserve the device's entire configured license +count for an administrative account. New jobs requiring it remain pending with +reason `Licenses`; running jobs continue, and other devices remain usable. +Remove the reservation after a successful health check. + +The +[availability monitor example](https://github.com/munich-quantum-toolkit/core/tree/main/examples/slurm) +performs one such update. It blocks the licenses before invoking the bounded +QDMI checker and removes the block only on success. Run it periodically from a +trusted administrative host with Slurm clients, MQT Core, the device +implementation, and site-owned credentials. Initialize the blocks before +admitting workloads. The controller does not need device libraries. + +Use a site account that can assess the shared device's operational state. One +user's expired credentials must not determine availability for every user. +Conversely, a successful site check does not verify each user's authorization. +The monitor is a snapshot: device status can change between checks, and a +stopped monitor leaves its last scheduler state in place. Supervise it and alert +on stale updates. Keep the optional job check for the user's environment. + +This integration uses licenses and Slurm's environment export. Device libraries +run in application or checker processes; no Slurm plugin is needed. The +[QRMI integration paper](https://arxiv.org/abs/2607.19591) describes a separate +acquire/execute/release lifecycle for services that issue allocation tokens. The +QDMI device implementations used here do not require that lifecycle. + +## Configure the cluster + +Use matching Slurm versions across the cluster. This integration requires Slurm +25.11 or newer. + +| Location | Software and configuration | +| ---------------------- | ---------------------------------------------------- | +| Login/submission nodes | Slurm clients and access to the workload environment | +| Controller | `slurmctld`, scheduling policy, and license counts | +| Compute nodes | `slurmd`, cgroup v2, and the workload environment | +| Accounting service | `slurmdbd` when persistent accounting is needed | + +Static local licenses do not require an accounting database. The controller does +not need device libraries or SDKs. Use consistent numeric user/group IDs and +readable catalogue/library paths across compute nodes. A shared versioned +environment and identical per-node installations are both suitable. + +Keep scheduler authentication, such as Munge, separate from device credentials. +For CPU and allocated-memory constraints, use memory-consuming selection with +cgroup enforcement: -```console -sbatch --nodelist=node1 bell.sbatch -sbatch --nodelist=node2 bell.sbatch +```ini +# slurm.conf +ProctrackType=proctrack/cgroup +TaskPlugin=task/cgroup,task/affinity +JobAcctGatherType=jobacct_gather/cgroup +SelectType=select/cons_tres +SelectTypeParameters=CR_CPU_Memory ``` -Both jobs can run because two DDSIM licenses exist. Each job uses one CPU. One -CPU remains free on each node. Submit a third DDSIM job. Slurm keeps it pending -until one DDSIM license becomes free. - -Use these commands to inspect the state: - -```console -squeue --format="%.18i %.9T %.20R %.12N %.20L" -scontrol show lic mqt.ddsim.default -scontrol show node node1 -scontrol show node node2 +```ini +# cgroup.conf +CgroupPlugin=cgroup/v2 +ConstrainCores=yes +ConstrainRAMSpace=yes +ConstrainSwapSpace=yes ``` -The third job must have state `PENDING` and reason `Licenses`. The license -report must show `Total=2 Used=2 Free=0`. The node records must still show one -free CPU on each node. A job that requests `mqt.sc.default:1` can use one of -these CPUs because it uses a different license. - -When one DDSIM job ends, Slurm returns its license. The pending job can then -start without a change to the device registry or `slurm.conf`. +Set node resources, memory defaults, partitions, accounts, and limits for the +site. See the +[Slurm administration guide](https://slurm.schedmd.com/quickstart_admin.html) +and [cgroup configuration](https://slurm.schedmd.com/cgroup.conf.html). -## Diagnose a failure +`SLURM_JOB_LICENSES` is mutable within a process. MQT Core uses it for device +selection, not as proof of allocation or authorization. Device services and +operating-system permissions must enforce access independently. -First, run `scontrol show job `. Check `JobState`, `Reason`, `Licenses`, -and `NodeList`. Use `scontrol show lic` to compare the total, used, and free -counts. Use `scontrol show node` to check CPU allocation. +## Try the Docker cluster -If a node is down, check `systemctl status munge slurmd` and the Slurm journal -on that node. Check that `/sys/fs/cgroup/cgroup.controllers` exists. Check that -all nodes use the same Munge key and the same `slurm.conf`. - -If MQT Core cannot select a device, print `SLURM_JOB_LICENSES` inside the batch -job and list the IDs visible to `Session`. Use this value only to diagnose -selection. It is not proof of the Slurm allocation. The license name and stable -ID must match exactly. Do not add a generic device license. Do not use a Slurm -OR license expression for device selection because the environment does not -identify a single selected device in that case. - -[Slurm GRES configuration]: https://slurm.schedmd.com/gres.conf.html - -## Run the integration tests - -From a source checkout on a Linux Docker host with cgroup v2, build one wheel -and run the fixture: - -```console -uv build --wheel --out-dir test/slurm/dist -Ccmake.define.DEPLOY=ON -uv run --no-project --python 3.14 test/slurm/run_integration.py -``` - -Keep exactly one wheel in `test/slurm/dist`. The fixture installs that wheel in -one controller and two compute containers. It checks admission, license -contention, independent SC execution, Bell results, and terminal job states and -exit codes. It uses privileged containers to exercise real Slurm cgroups; it is -an isolated test cluster, not a production deployment template. - -Each invocation uses its own Docker project, Munge key, and directory below -`test/slurm/runtime`. Successful runs remove their containers, images, and -artifacts; failures retain artifacts and print the project and runtime path. -Docker's build cache remains available to later runs. Commands have 30-second -deadlines, image build/startup has a ten-minute deadline, and diagnostics and -cleanup have separate short deadlines. Batch jobs request a five-minute time -limit. An interrupted command terminates its process group, including a Docker -Compose child. - -The runner prints setup, execution, and total durations. Its inexpensive -failure-path tests can run before building a wheel: - -```console -uv run --no-project --with 'pytest>=9.0.1' --python 3.14 pytest -o addopts= -q test/python/test_slurm_integration.py -``` +The reusable +[Docker Slurm setup](https://github.com/munich-quantum-toolkit/core/tree/main/docker/slurm) +supports local demonstrations and integration tests with a configurable number +of compute containers. Follow its README to build the workload image, start the +cluster, submit jobs, and remove it. -CI enables the existing sccache compiler integration and reports cache counters. -Compare compiler requests, hits, and wall time before attributing a build-time -change to caching. The fixture's bounded polling targets its private controller; -do not copy these loops into production monitoring. See the -[Slurm RPC performance guidance](https://slurm.schedmd.com/squeue.html#SECTION_PERFORMANCE). +Use rootful Docker on a disposable Linux cgroup-v2 host. The containers run +systemd and require privileged access to the host cgroup hierarchy. Jobs run as +an unprivileged user. This setup is intended for development and demonstrations, +not a production security boundary. diff --git a/examples/slurm/README.md b/examples/slurm/README.md new file mode 100644 index 0000000000..f2ec65a5fa --- /dev/null +++ b/examples/slurm/README.md @@ -0,0 +1,99 @@ +# Update Slurm device availability + +`update_availability.py` uses Slurm 25.11+ license-only reservations to keep +jobs pending while a QDMI device is unavailable. It reserves the device's full +local license count before each check and removes that reservation only after a +successful check. Running jobs continue, and unrelated licenses remain usable. +No SPANK plugin or SlurmDBD is required. + +Run one monitor per device, as root or the configured `SlurmUser`, on an +administrative host with Slurm clients and the site's QDMI checker, catalogue, +device implementation, and credentials. The controller itself needs no device +implementation. The monitor's credentials must represent site access; a user's +failed authentication must not change cluster-wide availability. Jobs retain +their own credentials and must handle failures after allocation. + +For `Licenses=mqt.ddsim.default:2`, run: + +```console +python3 update_availability.py --license mqt.ddsim.default:2 --checker /opt/qdmi/bin/mqt-core-qdmi-check +``` + +The count must equal the full configured capacity. The monitor owns +`qdmi-unavailable-mqt.ddsim.default`; reserve this name for it, and do not +create overlapping license reservations. Each invocation renews the block for +one year. Use `--block-only` to close admission without probing. Stop both the +timer and its service before using this option for a manual outage, so an +in-flight or later healthy check cannot reopen it. + +Exit status is `0` for a successful readiness check or `--block-only`, `1` for a +failed or timed-out probe with admission blocked, and `2` for invalid arguments +or a controller operation failure. Alert on controller failures: an unreachable +controller cannot be updated. Checker diagnostics are suppressed because they +can contain credentials. Inspect device failures separately under the site's +logging policy. + +The checker accepts `IDLE` and `BUSY` as operational states. The license count +is the site's concurrency policy; this check does not measure the remote queue +or reserve capacity at the service. Probe failures, including expired monitor +credentials, leave admission closed until a later successful probe. + +## Poll with systemd + +Copy the example to `/opt/mqt-core/examples/slurm/`. Adapt this service for each +device as `/etc/systemd/system/qdmi-availability.service`: + +```ini +[Unit] +Description=Update QDMI device admission in Slurm +Wants=network-online.target +After=network-online.target munge.service + +[Service] +Type=oneshot +User=slurm +Environment=MQT_CORE_QDMI_CONFIG_FILE=/etc/mqt-core/site.qdmi.json +ExecStart=/usr/bin/python3 /opt/mqt-core/examples/slurm/update_availability.py --license mqt.ddsim.default:2 --checker /opt/qdmi/bin/mqt-core-qdmi-check +TimeoutStartSec=45 +``` + +Provide any device-specific credential-file reference through a root-owned +service configuration; do not put tokens or keys in command-line arguments. +Install `/etc/systemd/system/qdmi-availability.timer`: + +```ini +[Unit] +Description=Poll QDMI device readiness + +[Timer] +OnBootSec=1s +OnUnitInactiveSec=30s +AccuracySec=1s + +[Install] +WantedBy=timers.target +``` + +Before opening the queue, establish blocks for every monitored license. For an +existing cluster, this sequence pauses new allocations while retaining running +jobs. Apply it to every partition that can request these licenses: + +```console +scontrol update PartitionName=compute State=DOWN +python3 /opt/mqt-core/examples/slurm/update_availability.py --license mqt.ddsim.default:2 --block-only +systemctl daemon-reload +systemctl enable --now qdmi-availability.timer +scontrol update PartitionName=compute State=UP +``` + +For a new controller, start the affected partitions with `State=DOWN` in +`slurm.conf`, then establish the blocks before setting them `UP`. Incorporate +this ordering into site startup procedures. + +Polling gives a readiness snapshot. A device can fail after a successful probe, +and a stopped timer after success leaves the license open. Supervise the timer +and service; if the site requires a maximum observation age, an independent +watchdog must run `--block-only` when that age is exceeded. A service +`OnFailure` handler can alert or close admission, but cannot detect a stopped +timer by itself. Inspect both `Free` and `Reserved` in `scontrol show licenses`: +reserved tokens can still appear in `Free`. diff --git a/examples/slurm/update_availability.py b/examples/slurm/update_availability.py new file mode 100755 index 0000000000..cee4b29053 --- /dev/null +++ b/examples/slurm/update_availability.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +"""Block a local Slurm device license until a site readiness probe succeeds.""" + +from __future__ import annotations + +import argparse +import json +import logging +import re +import subprocess +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Sequence + +LOGGER = logging.getLogger("qdmi-slurm-availability") + + +def scontrol(*arguments: str) -> str: + """Run a bounded controller operation. + + Returns: + The command's standard output. + """ + # Slurm clients come from the administrator's PATH; no shell is involved. + return subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] + ("scontrol", *arguments), # ruff: ignore[start-process-with-partial-path] + check=True, + stdout=subprocess.PIPE, + text=True, + timeout=10, + ).stdout + + +def main(arguments: Sequence[str] | None = None) -> int: + """Close admission before probing. + + Returns: + Zero for readiness or block-only success, one for a failed probe, or two + for a controller error. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--license", required=True, metavar="ID:COUNT", help="local device ID and full configured capacity" + ) + parser.add_argument("--checker", default="mqt-core-qdmi-check", help="site checker executable") + parser.add_argument("--timeout", type=int, default=10, help="checker deadline in seconds (default: 10)") + parser.add_argument("--block-only", action="store_true", help="close admission without running the checker") + options = parser.parse_args(arguments) + if re.fullmatch(r"[A-Za-z0-9_.-]+:[1-9][0-9]*", options.license) is None: + parser.error("--license must be a local ID:COUNT with a positive count") + if not 1 <= options.timeout <= 3600: + parser.error("--timeout must be between 1 and 3600 seconds") + + device = options.license.rsplit(":", maxsplit=1)[0] + name = f"qdmi-unavailable-{device}" + try: + reservations = json.loads(scontrol("--json", "show", "reservations"))["reservations"] + exists = any(reservation["name"] == name for reservation in reservations) + # Active reservations cannot change their start time. Renew the end time + # because Slurm implements Duration=infinite as one year. + scontrol( + "update" if exists else "create", + f"ReservationName={name}", + *(("StartTime=now",) if not exists else ()), + "EndTime=now+365days", + "Users=root", + "Flags=LICENSE_ONLY,IGNORE_JOBS", + f"Licenses={options.license}", + ) + except (OSError, subprocess.SubprocessError, ValueError, KeyError): + LOGGER.exception("Could not block %s", device) + return 2 + + if options.block_only: + return 0 + try: + # The administrator selects the site executable; arguments use no shell. + result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] + (options.checker, "--device", device, "--timeout", str(options.timeout)), + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=options.timeout + 2, + ) + except (OSError, subprocess.SubprocessError): + LOGGER.exception("Readiness probe failed; %s remains blocked", device) + return 1 + if result.returncode != 0: + LOGGER.error("Readiness probe failed; %s remains blocked", device) + return 1 + + try: + scontrol("delete", f"ReservationName={name}") + except (OSError, subprocess.SubprocessError): + LOGGER.exception("Could not reopen %s", device) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/include/mqt-core/qdmi/Slurm.hpp b/include/mqt-core/qdmi/Slurm.hpp index 559eadd99d..cb72ce748b 100644 --- a/include/mqt-core/qdmi/Slurm.hpp +++ b/include/mqt-core/qdmi/Slurm.hpp @@ -19,16 +19,16 @@ namespace qdmi::slurm { /// Opens the QDMI device named by the Slurm license environment. -/// @return A fresh device session using the registered device definition. +/// @return The selected device. /// /// The @c SLURM_JOB_LICENSES value must contain exactly one local -/// license. Its name must equal a registered QDMI device ID. The optional -/// license count must be one. The device must report @c QDMI_DEVICE_STATUS_IDLE -/// or @c QDMI_DEVICE_STATUS_BUSY. +/// license. Its name must equal an ID visible to the selected QDMI driver. +/// The optional license count must be one. The device must report @c +/// QDMI_DEVICE_STATUS_IDLE or @c QDMI_DEVICE_STATUS_BUSY. /// @warning This function uses process-mutable environment data for device /// selection. It does not verify a Slurm allocation, authenticate the caller, -/// or authorize access to the device. The provider or operating system must -/// enforce access independently. +/// or authorize access to the device. The device implementation or operating +/// system must enforce access independently. /// @throws std::runtime_error If the license value is missing, malformed, /// compound, remote, has a non-unit count, names an unknown device, or names a /// device in another state. diff --git a/pyproject.toml b/pyproject.toml index f4ff5cb3d5..2a6f439765 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,7 @@ PyPI = "https://pypi.org/project/mqt-core/" mqt-cc = "mqt.core._commands:compiler" mqt-core-bench = "mqt.core._commands:benchmark" mqt-core-cli = "mqt.core.__main__:main" +mqt-core-qdmi-check = "mqt.core._commands:qdmi_check" [project.entry-points."pennylane.plugins"] "mqt.ddsim.default" = "mqt.core.plugins.pennylane:DDSIMDevice" diff --git a/python/mqt/core/_commands.py b/python/mqt/core/_commands.py index 98d9223098..c304c376f0 100644 --- a/python/mqt/core/_commands.py +++ b/python/mqt/core/_commands.py @@ -16,6 +16,8 @@ from pathlib import Path from typing import NoReturn +from ._qdmi_discovery import discover_qdmi_manifests + def compiler() -> NoReturn: """Launch the bundled MQT compiler.""" @@ -27,11 +29,19 @@ def benchmark() -> NoReturn: run_tool("mqt-core-bench") -def run_tool(name: str) -> NoReturn: +def qdmi_check() -> NoReturn: + """Probe whether a QDMI device is operational.""" + arguments: list[str] = [] + if sys.argv[1:] != ["--help"]: + discover_qdmi_manifests(lambda path: arguments.extend(("--manifest", str(path)))) + run_tool("mqt-core-qdmi-check", *arguments) + + +def run_tool(name: str, *extra_arguments: str) -> NoReturn: """Replace this process with a bundled native tool.""" suffix = ".exe" if sys.platform == "win32" else "" executable = Path(str(distribution("mqt-core").locate_file(f"mqt/core/bin/{name}{suffix}"))) - os.execv(executable, [str(executable), *sys.argv[1:]]) # ruff: ignore[start-process-with-no-shell] + os.execv(executable, [str(executable), *sys.argv[1:], *extra_arguments]) # ruff: ignore[start-process-with-no-shell] def include_dir() -> Path: diff --git a/python/mqt/core/qdmi/slurm.pyi b/python/mqt/core/qdmi/slurm.pyi index 4c15545394..ba17718d89 100644 --- a/python/mqt/core/qdmi/slurm.pyi +++ b/python/mqt/core/qdmi/slurm.pyi @@ -14,18 +14,18 @@ def open_device_from_license() -> mqt.core.qdmi.Device: """Open the QDMI device named by the Slurm license environment. ``SLURM_JOB_LICENSES`` must contain one local license whose name equals a stable - ID visible to the selected QDMI Driver. The optional count must be one. The - function opens a fresh Client session and accepts device status ``IDLE`` or + ID visible to the selected QDMI driver. The optional count must be one. The + function opens a fresh driver session and accepts device status ``IDLE`` or ``BUSY``. It does not apply job-specific QDMI configuration or credentials. Warning: ``SLURM_JOB_LICENSES`` is process-mutable. This function uses it only for device selection. It does not verify a Slurm allocation, authenticate the - caller, or authorize device access. The provider or operating system must + caller, or authorize device access. The device implementation or operating system must enforce access independently. Returns: - mqt.core.qdmi.Device: The fresh device session. + mqt.core.qdmi.Device: The selected device. Raises: RuntimeError: If the license value or named device does not satisfy this diff --git a/src/qdmi/CMakeLists.txt b/src/qdmi/CMakeLists.txt index aacc1735b4..a908a65355 100644 --- a/src/qdmi/CMakeLists.txt +++ b/src/qdmi/CMakeLists.txt @@ -40,6 +40,23 @@ if(NOT TARGET ${TARGET_NAME}) list(APPEND MQT_CORE_TARGETS ${TARGET_NAME}) endif() +if(UNIX OR WIN32) + add_executable(mqt-core-qdmi-check Check.cpp) + target_link_libraries(mqt-core-qdmi-check PRIVATE MQT::CoreQDMI MQT::ProjectWarnings + MQT::ProjectOptions) + if(WIN32) + target_compile_definitions(mqt-core-qdmi-check PRIVATE NOMINMAX _WIN32_WINNT=0x0A00) + if(MINGW) + target_link_options(mqt-core-qdmi-check PRIVATE -municode) + endif() + endif() + mqt_copy_qdmi_runtime(mqt-core-qdmi-check) + if(MQT_CORE_INSTALL) + install(TARGETS mqt-core-qdmi-check RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" + COMPONENT ${MQT_CORE_TARGET_NAME}_Runtime) + endif() +endif() + set(MQT_CORE_TARGETS ${MQT_CORE_TARGETS} PARENT_SCOPE) diff --git a/src/qdmi/Check.cpp b/src/qdmi/Check.cpp new file mode 100644 index 0000000000..92b2294be0 --- /dev/null +++ b/src/qdmi/Check.cpp @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2023 - 2026 Chair for Design Automation, TUM + * Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH + * All rights reserved. + * + * SPDX-License-Identifier: MIT + * + * Licensed under the MIT License + */ + +#include "qdmi/QDMI.hpp" +#include "qdmi/common/Common.hpp" + +#include "qdmi/constants.h" + +#ifdef _WIN32 +#include +#include +#else +#include +// POSIX signal handling is not part of the C++ interface. +#include // NOLINT(modernize-deprecated-headers) +#include +#include +#include +#include +#endif + +#ifdef __linux__ +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// A signal handler may only communicate through sig_atomic_t. +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +volatile std::sig_atomic_t interrupted = 0; + +extern "C" void handleSignal(const int signal) { interrupted = signal; } + +constexpr std::string_view USAGE = + "Usage: mqt-core-qdmi-check --device ID [--timeout SECONDS] " + "[--manifest PATH]...\n" + "Probe whether a registered QDMI device is operational (default timeout: " + "30 seconds).\n" + "Exit codes: 0 available, 1 failed, 2 invalid arguments, 124 timeout.\n"; + +[[nodiscard]] auto check(const std::string& id, + const std::span manifests) + -> int { + try { + for (const auto& manifest : manifests) { + try { + qdmi::builtin_driver::addManifest(manifest); + } catch (...) { + // Match Python discovery's best-effort handling of installed manifests. + continue; + } + } + const auto device = qdmi::Session::openDevice(id); + const auto status = device.getStatus(); + return status == QDMI_DEVICE_STATUS_IDLE || + status == QDMI_DEVICE_STATUS_BUSY + ? 0 + : 1; + } catch (...) { + return 1; + } +} + +#ifdef _WIN32 +[[nodiscard]] auto supervise(const std::chrono::seconds timeout) -> int { + if (std::signal(SIGTERM, handleSignal) == SIG_ERR || + std::signal(SIGINT, handleSignal) == SIG_ERR) { + return 1; + } + const auto deadline = std::chrono::steady_clock::now() + timeout; + using Handle = std::unique_ptr; + const Handle job(CreateJobObjectW(nullptr, nullptr), CloseHandle); + JOBOBJECT_EXTENDED_LIMIT_INFORMATION limits{}; + limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if (!job || + !SetInformationJobObject(job.get(), JobObjectExtendedLimitInformation, + &limits, sizeof(limits))) { + return 1; + } + + std::wstring executable(MAX_PATH, L'\0'); + while (true) { + const auto length = GetModuleFileNameW( + nullptr, executable.data(), static_cast(executable.size())); + if (length == 0) { + return 1; + } + if (length < executable.size()) { + executable.resize(length); + break; + } + executable.resize(executable.size() * 2); + } + // Preserve the original argument quoting after replacing argv[0]. + std::wstring_view arguments(GetCommandLineW()); + bool quoted = false; + while (!arguments.empty()) { + const auto character = arguments.front(); + if (!quoted && (character == L' ' || character == L'\t')) { + break; + } + if (character == L'"') { + quoted = !quoted; + } + arguments.remove_prefix(1); + } + auto command = L"\"" + executable + L"\" --worker" + std::wstring(arguments); + SECURITY_ATTRIBUTES security{}; + security.nLength = sizeof(security); + security.bInheritHandle = TRUE; + const auto sink = CreateFileW(L"NUL", GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, &security, + OPEN_EXISTING, 0, nullptr); + if (sink == INVALID_HANDLE_VALUE) { + return 1; + } + const Handle output(sink, CloseHandle); + STARTUPINFOEXW startup{}; + startup.StartupInfo.cb = sizeof(startup); + startup.StartupInfo.dwFlags = STARTF_USESTDHANDLES; + startup.StartupInfo.hStdInput = output.get(); + startup.StartupInfo.hStdOutput = output.get(); + startup.StartupInfo.hStdError = output.get(); + SIZE_T attributeBytes = 0; + InitializeProcThreadAttributeList(nullptr, 1, 0, &attributeBytes); + std::vector attributes(attributeBytes); + startup.lpAttributeList = + reinterpret_cast(attributes.data()); + if (!InitializeProcThreadAttributeList(startup.lpAttributeList, 1, 0, + &attributeBytes)) { + return 1; + } + // Assign the job during creation so supervisor death cannot orphan a worker. + auto jobHandle = job.get(); + PROCESS_INFORMATION information{}; + const auto created = + UpdateProcThreadAttribute(startup.lpAttributeList, 0, + PROC_THREAD_ATTRIBUTE_JOB_LIST, &jobHandle, + sizeof(jobHandle), nullptr, nullptr) && + CreateProcessW(executable.c_str(), command.data(), nullptr, nullptr, TRUE, + EXTENDED_STARTUPINFO_PRESENT | CREATE_NO_WINDOW, nullptr, + nullptr, &startup.StartupInfo, &information); + DeleteProcThreadAttributeList(startup.lpAttributeList); + if (!created) { + return 1; + } + const Handle process(information.hProcess, CloseHandle); + const Handle thread(information.hThread, CloseHandle); + int result = 1; + while (interrupted == 0) { + const auto waited = WaitForSingleObject(process.get(), 10); + if (waited == WAIT_OBJECT_0) { + DWORD status = 1; + return GetExitCodeProcess(process.get(), &status) && status == 0 ? 0 : 1; + } + if (waited == WAIT_FAILED) { + break; + } + if (std::chrono::steady_clock::now() >= deadline) { + result = 124; + break; + } + } + TerminateJobObject(job.get(), 1); + WaitForSingleObject(process.get(), INFINITE); + return result; +} +#else +[[nodiscard]] auto +supervise(const std::string& id, const std::chrono::seconds timeout, + const std::span manifests) -> int { + struct sigaction action{}; + action.sa_handler = handleSignal; + sigemptyset(&action.sa_mask); + if (sigaction(SIGTERM, &action, nullptr) != 0 || + sigaction(SIGINT, &action, nullptr) != 0) { + return 1; + } + + const auto deadline = std::chrono::steady_clock::now() + timeout; +#ifdef __linux__ + const auto supervisor = getpid(); +#endif + const auto child = fork(); + if (child < 0) { + return 1; + } + if (child == 0) { +#ifdef __linux__ + // Stop the worker if its supervisor is killed. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0 || getppid() != supervisor) { + _exit(1); + } +#endif + action.sa_handler = SIG_DFL; + const rlimit coreLimit{.rlim_cur = 0, .rlim_max = 0}; + if (setpgid(0, 0) != 0 || sigaction(SIGTERM, &action, nullptr) != 0 || + sigaction(SIGINT, &action, nullptr) != 0 || + setrlimit(RLIMIT_CORE, &coreLimit) != 0) { + _exit(1); + } + // Device diagnostics can contain credentials, URLs, or configuration. + // POSIX open takes no mode argument when it does not create a file. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + const auto sink = open("/dev/null", O_RDWR); + if (sink < 0 || dup2(sink, STDIN_FILENO) < 0 || + dup2(sink, STDOUT_FILENO) < 0 || dup2(sink, STDERR_FILENO) < 0) { + _exit(1); + } + if (sink > STDERR_FILENO) { + close(sink); + } + // Run process-exit handlers within the worker deadline. + std::exit(check(id, manifests)); // NOLINT(concurrency-mt-unsafe) + } + + // Either side can establish the group before the child loads a device. + static_cast(setpgid(child, child)); + int result = 1; + while (interrupted == 0) { + // clang-tidy 23 does not map glibc's wait definitions to . + // NOLINTBEGIN(misc-include-cleaner) + siginfo_t status{}; + const auto waited = waitid(P_PID, static_cast(child), &status, + WEXITED | WNOHANG | WNOWAIT); + if (waited == 0 && status.si_pid == child) { + result = status.si_code == CLD_EXITED && status.si_status == 0 ? 0 : 1; + // NOLINTEND(misc-include-cleaner) + break; + } + if (waited < 0 && errno != EINTR) { + break; + } + if (std::chrono::steady_clock::now() >= deadline) { + result = 124; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + // Keep the worker unreaped until group cleanup so its PID cannot be reused. + static_cast(kill(-child, SIGKILL)); + static_cast(kill(child, SIGKILL)); + while (waitpid(child, nullptr, 0) < 0 && errno == EINTR) { + } + return result; +} +#endif + +} // namespace + +#ifdef _WIN32 +int wmain(const int argc, wchar_t** argv) try { + std::vector utf8Arguments; + for (const auto* argument : std::span(argv, static_cast(argc))) { + utf8Arguments.emplace_back(qdmi::detail::pathToString(argument)); + } + const auto arguments = std::span(utf8Arguments); + bool worker = false; +#else +int main(const int argc, char** argv) try { + const auto arguments = std::span(argv, static_cast(argc)); +#endif + if (argc == 2 && std::string_view(arguments[1]) == "--help") { + std::cout << USAGE; + return 0; + } + std::string id; + std::vector manifests; + int seconds = 30; + bool hasTimeout = false; + for (size_t i = 1; i < arguments.size(); ++i) { + const std::string_view option(arguments[i]); +#ifdef _WIN32 + if (option == "--worker" && !worker) { + worker = true; + continue; + } +#endif + if (++i == arguments.size()) { + std::cerr << USAGE; + return 2; + } + const std::string_view value(arguments[i]); + if (option == "--device" && id.empty() && !value.empty()) { + id = value; + } else if (option == "--manifest" && !value.empty()) { + manifests.emplace_back(qdmi::detail::pathFromString(value)); + } else if (option == "--timeout" && !hasTimeout) { + // from_chars requires a pointer range within the string view. + // NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic) + const auto parsed = + std::from_chars(value.data(), value.data() + value.size(), seconds); + if (parsed.ec != std::errc{} || + parsed.ptr != value.data() + value.size() || seconds < 1 || + seconds > 3600) { + std::cerr << USAGE; + return 2; + } + // NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic) + hasTimeout = true; + } else { + std::cerr << USAGE; + return 2; + } + } + if (id.empty()) { + std::cerr << USAGE; + return 2; + } +#ifdef _WIN32 + if (worker) { + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); + return check(id, manifests); + } + const auto result = supervise(std::chrono::seconds(seconds)); +#else + const auto result = supervise(id, std::chrono::seconds(seconds), manifests); +#endif + if (result == 124) { + std::cerr << "QDMI device check timed out\n"; + } else if (result != 0) { + std::cerr << "QDMI device check failed\n"; + } + return result; +} catch (...) { + std::cerr << "QDMI device check failed\n"; + return 1; +} diff --git a/test/python/qdmi/test_discovery.py b/test/python/qdmi/test_discovery.py index 69a7067dd9..9c17e26839 100644 --- a/test/python/qdmi/test_discovery.py +++ b/test/python/qdmi/test_discovery.py @@ -10,7 +10,9 @@ from __future__ import annotations +import json import sys +from importlib.metadata import distribution from pathlib import Path, PurePosixPath from types import SimpleNamespace from typing import TYPE_CHECKING @@ -22,6 +24,8 @@ if TYPE_CHECKING: from collections.abc import Iterator + from pytest_console_scripts import ScriptRunner + class _Distribution: def __init__(self, root: Path, files: list[str] | None) -> None: @@ -97,3 +101,37 @@ def test_skips_invalid_manifest_metadata( with pytest.warns(RuntimeWarning, match="Skipping QDMI manifest") as warnings: _qdmi_discovery.discover_qdmi_manifests(lambda _: pytest.fail("must skip")) assert len(warnings) == 1 + + +@pytest.mark.script_launch_mode("subprocess") +def test_availability_discovers_installed_provider( + script_runner: ScriptRunner, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Discover installed manifests in the native checker without importing providers.""" + core = distribution("mqt-core") + packaged = next(file for file in core.files or () if file.name == "mqt-core-qdmi-sc-device.qdmi.json") + packaged_path = Path(str(core.locate_file(packaged))) + definition = json.loads(packaged_path.read_text(encoding="utf-8"))["qdmi"]["devices"][0] + definition.update(id="test.installed", library=str(packaged_path.parent / definition["library"])) + + provider = tmp_path / "test_qdmi_provider" + provider.mkdir() + (provider / "__init__.py").write_text("raise RuntimeError('provider must not be imported')\n") + (provider / "device.qdmi.json").write_text(json.dumps({"schema-version": 1, "qdmi": {"devices": [definition]}})) + (provider / "invalid.qdmi.json").write_text("{") + metadata = tmp_path / "test_qdmi_provider-0.0.dist-info" + metadata.mkdir() + (metadata / "METADATA").write_text("Name: test-qdmi-provider\nVersion: 0.0\n") + (metadata / "entry_points.txt").write_text("[mqt.core.qdmi.manifests]\nprobe = test_qdmi_provider\n") + (metadata / "RECORD").write_text("test_qdmi_provider/device.qdmi.json,,\ntest_qdmi_provider/invalid.qdmi.json,,\n") + monkeypatch.setenv("PYTHONPATH", str(tmp_path)) + + result = script_runner.run(["mqt-core-qdmi-check", "--device", "test.installed"]) + assert result.success + assert not result.stdout + + monkeypatch.setenv( + "MQT_CORE_QDMI_CONFIG_JSON", + json.dumps({"schema-version": 1, "qdmi": {"devices": [{"id": "test.installed", "enabled": False}]}}), + ) + assert not script_runner.run(["mqt-core-qdmi-check", "--device", "test.installed"]).success diff --git a/test/python/test_cli.py b/test/python/test_cli.py index 3a1954803f..f431b536ce 100644 --- a/test/python/test_cli.py +++ b/test/python/test_cli.py @@ -138,7 +138,14 @@ def test_benchmark_cli(script_runner: ScriptRunner) -> None: assert '"teleportation"' in ret.stdout -@pytest.mark.parametrize("tool", ["mqt-cc", "mqt-core-bench"]) +@pytest.mark.parametrize( + "tool", + [ + "mqt-cc", + "mqt-core-bench", + "mqt-core-qdmi-check", + ], +) def test_native_tool_entry_point(script_runner: ScriptRunner, tool: str) -> None: """Resolve the console entry point and forward arguments to its native tool.""" with patch("os.execv") as execute: @@ -147,3 +154,11 @@ def test_native_tool_entry_point(script_runner: ScriptRunner, tool: str) -> None executable, arguments = execute.call_args.args assert Path(executable).is_file() assert arguments == [str(executable), "--help"] + + +@pytest.mark.script_launch_mode("subprocess") +def test_qdmi_availability(script_runner: ScriptRunner) -> None: + """Probe a device through the installed command and catalogue.""" + result = script_runner.run(["mqt-core-qdmi-check", "--device", "mqt.sc.default"]) + assert result.success + assert not result.stdout diff --git a/test/python/test_slurm_availability.py b/test/python/test_slurm_availability.py new file mode 100644 index 0000000000..36c0407dc0 --- /dev/null +++ b/test/python/test_slurm_availability.py @@ -0,0 +1,84 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +"""Check that unavailable probes leave scheduler admission closed.""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from types import ModuleType + + +@pytest.fixture +def monitor() -> ModuleType: + """Load the standalone administrative example. + + Returns: + The monitor module. + """ + path = Path(__file__).parents[2] / "examples" / "slurm" / "update_availability.py" + spec = importlib.util.spec_from_file_location("slurm_availability", path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("failure", [1, subprocess.TimeoutExpired("checker", 1), FileNotFoundError("checker")]) +def test_probe_failure_keeps_block_until_recovery( + monitor: ModuleType, monkeypatch: pytest.MonkeyPatch, failure: int | Exception +) -> None: + """Never probe with admission open, and preserve other devices' reservations.""" + reservations = {"another-device": "other:1"} + name = "qdmi-unavailable-example.device" + + def scontrol(*arguments: str) -> str: + if arguments[0] == "--json": + return json.dumps({"reservations": [{"name": key} for key in reservations]}) + fields = dict(argument.split("=", maxsplit=1) for argument in arguments[1:]) + if arguments[0] == "delete": + del reservations[fields["ReservationName"]] + else: + reservations[fields["ReservationName"]] = fields["Licenses"] + return "" + + def probe(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + assert reservations[name] == "example.device:2" + if isinstance(failure, Exception): + raise failure + return subprocess.CompletedProcess("checker", failure) + + monkeypatch.setattr(monitor, "scontrol", scontrol) + monkeypatch.setattr(monitor.subprocess, "run", probe) + arguments = ("--license", "example.device:2") + assert monitor.main(arguments) == 1 + assert reservations[name] == "example.device:2" + failure = 0 + assert monitor.main(arguments) == 0 + assert reservations == {"another-device": "other:1"} + + +def test_controller_failure_never_runs_probe(monitor: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None: + """A failed admission update cannot lead to a healthy, open result.""" + + def unavailable(*_arguments: str) -> str: + msg = "scontrol" + raise subprocess.TimeoutExpired(msg, 10) + + monkeypatch.setattr(monitor, "scontrol", unavailable) + monkeypatch.setattr(monitor.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("probe ran before blocking")) + assert monitor.main(("--license", "example.device:2")) == 2 diff --git a/test/python/test_slurm_integration.py b/test/python/test_slurm_integration.py index 4af0967d6c..437c02e80f 100644 --- a/test/python/test_slurm_integration.py +++ b/test/python/test_slurm_integration.py @@ -44,6 +44,25 @@ def load_runner() -> ModuleType: runner = load_runner() +def test_runner_accepts_scaled_nodes() -> None: + """Run the same cluster with more compute nodes without a new Compose file.""" + assert runner.parse_arguments(("--nodes", "3")).nodes == 3 + + +def test_prepare_keeps_private_key_and_preserves_existing_cluster(tmp_path: Path) -> None: + """Create usable shared files without overwriting a cluster's authentication.""" + runtime = tmp_path / "cluster" + command = ("sh", str(runner.CLUSTER / "prepare.sh"), str(runtime)) + runner.run(command) + key = runtime / "munge.key" + assert len(key.read_bytes()) == 1024 + assert key.stat().st_mode & 0o777 == 0o600 + assert (runtime / "slurm.conf").stat().st_mode & 0o777 == 0o644 + original = key.read_bytes() + assert runner.run(command, check=False).returncode != 0 + assert key.read_bytes() == original + + def test_timeout_kills_children_holding_output_pipes(tmp_path: Path) -> None: """A Compose-like grandchild must not defeat the command deadline.""" marker = tmp_path / "started" @@ -111,7 +130,7 @@ def test_preflight_does_not_touch_runtime(tmp_path: Path, monkeypatch: pytest.Mo def test_diagnostic_failure_still_tears_down(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """An unavailable diagnostic service must not orphan a failed cluster.""" - (tmp_path / "core.whl").touch() + (tmp_path / "mqt_core-0.whl").touch() monkeypatch.setattr(runner, "DIST", tmp_path) monkeypatch.setattr(runner, "RUNTIME", tmp_path / "runtime") monkeypatch.setattr(runner, "run", lambda *args: subprocess.CompletedProcess(args, 0, "2", "")) @@ -150,3 +169,49 @@ def run(command: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProce assert first.RUNTIME != second.RUNTIME assert calls[0][0] != calls[1][0] assert calls[0][1] != calls[1][1] + + +def test_provider_options_preserve_command_arguments(tmp_path: Path) -> None: + """Keep workload arguments separate from fixture options.""" + setup = tmp_path / "setup.sh" + setup.touch() + options = runner.parse_arguments(( + "--workload", + str(tmp_path), + "--setup-script", + "setup.sh", + "--device-license", + "provider.device", + "--", + "python3", + "probe.py", + "--label", + "one argument", + )) + assert options.workload == tmp_path + assert options.command == ["python3", "probe.py", "--label", "one argument"] + + +@pytest.mark.parametrize( + "arguments", + [ + ("--device-license", "provider.device"), + ("--", "python3", "probe.py"), + ("--device-license", "provider.device:2", "--", "/bin/true"), + ], +) +def test_invalid_provider_inputs_fail_before_docker(arguments: tuple[str, ...]) -> None: + """Reject incomplete or unrepresentable fixture inputs at the CLI.""" + with pytest.raises(SystemExit) as error: + runner.parse_arguments(arguments) + assert error.value.code == 2 + + +def test_setup_script_must_stay_inside_workload(tmp_path: Path) -> None: + """Do not accept a setup script that the workload build context cannot supply.""" + workload = tmp_path / "workload" + workload.mkdir() + (tmp_path / "outside.sh").touch() + with pytest.raises(SystemExit) as error: + runner.parse_arguments(("--workload", str(workload), "--setup-script", "../outside.sh")) + assert error.value.code == 2 diff --git a/test/qdmi/CMakeLists.txt b/test/qdmi/CMakeLists.txt index cdb72a97b5..1cb6f61286 100644 --- a/test/qdmi/CMakeLists.txt +++ b/test/qdmi/CMakeLists.txt @@ -60,4 +60,13 @@ if(TARGET MQT::CoreQDMI) endif() add_dependencies(${TARGET_NAME} mqt-core-qdmi-session-device) mqt_copy_qdmi_runtime(${TARGET_NAME}) + if(TARGET mqt-core-qdmi-check) + add_test( + NAME mqt-core-qdmi-check-cli + COMMAND + ${CMAKE_COMMAND} "-DCHECKER=$" + "-DSESSION_DEVICE=$" + "-DWORK_DIR=${CMAKE_CURRENT_BINARY_DIR}/checker" -P + "${CMAKE_CURRENT_SOURCE_DIR}/test_check_cli.cmake") + endif() endif() diff --git a/test/qdmi/driver/session_device.cpp b/test/qdmi/driver/session_device.cpp index 131f4f93f8..435c47749c 100644 --- a/test/qdmi/driver/session_device.cpp +++ b/test/qdmi/driver/session_device.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -292,6 +294,23 @@ TEST_SESSION_QDMI_device_session_init(QDMI_Device_Session session) { if (session->initialized) { return QDMI_ERROR_BADSTATE; } + if (parameter(session, QDMI_DEVICE_SESSION_PARAMETER_CUSTOM4) == + "hang-exit") { + if (std::atexit( + [] { std::this_thread::sleep_for(std::chrono::hours(1)); }) != 0) { + return QDMI_ERROR_FATAL; + } + } + if (parameter(session, QDMI_DEVICE_SESSION_PARAMETER_CUSTOM4) == "hang") { + std::this_thread::sleep_for(std::chrono::hours(1)); + } + if (parameter(session, QDMI_DEVICE_SESSION_PARAMETER_CUSTOM4) == + "hang-child" && + std::system( + parameter(session, QDMI_DEVICE_SESSION_PARAMETER_CUSTOM5).c_str()) != + 0) { + return QDMI_ERROR_FATAL; + } session->initialized = true; return QDMI_SUCCESS; } diff --git a/test/qdmi/test_check_cli.cmake b/test/qdmi/test_check_cli.cmake new file mode 100644 index 0000000000..923c6e3417 --- /dev/null +++ b/test/qdmi/test_check_cli.cmake @@ -0,0 +1,79 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +file(MAKE_DIRECTORY "${WORK_DIR}") +set(configuration "${WORK_DIR}/devices.json") +foreach(status idle busy offline hang hang-exit) + file( + WRITE "${configuration}" + "{\"schema-version\":1,\"qdmi\":{\"devices\":[{\"id\":\"test.check\",\"library\":\"${SESSION_DEVICE}\",\"prefix\":\"TEST_SESSION\",\"session\":{\"custom4\":\"${status}\"}}]}}" + ) + execute_process( + COMMAND + "${CMAKE_COMMAND}" -E env --unset=MQT_CORE_QDMI_DRIVER --unset=MQT_CORE_QDMI_CONFIG_JSON + "MQT_CORE_QDMI_CONFIG_FILE=${configuration}" "${CHECKER}" --device test.check --timeout 1 + RESULT_VARIABLE result + OUTPUT_VARIABLE output + ERROR_VARIABLE error + TIMEOUT 5) + if(status MATCHES "^hang") + set(expected 124) + elseif(status STREQUAL "offline") + set(expected 1) + else() + set(expected 0) + endif() + if(NOT result STREQUAL "${expected}" OR NOT output STREQUAL "") + message( + FATAL_ERROR + "Availability check for ${status}: expected ${expected}, got ${result}: ${output}${error}") + endif() +endforeach() + +file(REMOVE "${WORK_DIR}/child-started" "${WORK_DIR}/child-survived") +file( + WRITE "${WORK_DIR}/child.cmake" + "file(WRITE \"${WORK_DIR}/child-started\" \"\")\n" + "execute_process(COMMAND \"${CMAKE_COMMAND}\" -E sleep 3)\n" + "file(WRITE \"${WORK_DIR}/child-survived\" \"\")\n") +set(child_command "\"${CMAKE_COMMAND}\" -P \"${WORK_DIR}/child.cmake\"") +if(WIN32) + set(child_command "\"${child_command}\"") +endif() +string(REPLACE "\"" "\\\"" child_command "${child_command}") +file( + WRITE "${configuration}" + "{\"schema-version\":1,\"qdmi\":{\"devices\":[{\"id\":\"test.check\",\"library\":\"${SESSION_DEVICE}\",\"prefix\":\"TEST_SESSION\",\"session\":{\"custom4\":\"hang-child\",\"custom5\":\"${child_command}\"}}]}}" +) +execute_process( + COMMAND "${CMAKE_COMMAND}" -E env --unset=MQT_CORE_QDMI_DRIVER --unset=MQT_CORE_QDMI_CONFIG_JSON + "MQT_CORE_QDMI_CONFIG_FILE=${configuration}" "${CHECKER}" --device test.check --timeout 2 + RESULT_VARIABLE result + OUTPUT_QUIET ERROR_QUIET + TIMEOUT 8) +if(NOT result STREQUAL "124" OR NOT EXISTS "${WORK_DIR}/child-started") + message(FATAL_ERROR "The descendant cleanup test did not reach its timeout: ${result}") +endif() +execute_process(COMMAND "${CMAKE_COMMAND}" -E sleep 2) +if(EXISTS "${WORK_DIR}/child-survived") + message(FATAL_ERROR "A device subprocess survived the availability timeout") +endif() + +foreach(arguments IN + ITEMS "0;--help" "2;--device" "2;--timeout;1" "2;--device;test.check;--timeout;0" + "2;--unknown;value" "1;--device;unknown") + list(POP_FRONT arguments expected) + execute_process( + COMMAND "${CHECKER}" ${arguments} + RESULT_VARIABLE result + OUTPUT_QUIET ERROR_QUIET + TIMEOUT 5) + if(NOT result STREQUAL "${expected}") + message(FATAL_ERROR "Availability command ${arguments}: expected ${expected}, got ${result}") + endif() +endforeach() diff --git a/test/slurm/.dockerignore b/test/slurm/.dockerignore deleted file mode 100644 index 712f264ddd..0000000000 --- a/test/slurm/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -runtime/ diff --git a/test/slurm/Dockerfile b/test/slurm/Dockerfile deleted file mode 100644 index cf76b1d637..0000000000 --- a/test/slurm/Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM -# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH -# All rights reserved. -# -# SPDX-License-Identifier: MIT -# -# Licensed under the MIT License - -FROM ghcr.io/astral-sh/uv:0.12.1 AS uv - -FROM ubuntu:26.04 AS common - -ENV container=docker - -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - ca-certificates \ - dbus \ - munge \ - python3 \ - slurm-wlm \ - systemd \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -COPY --from=uv /uv /uvx /bin/ -COPY mqt-slurm-prepare /usr/local/sbin/mqt-slurm-prepare -COPY mqt-slurm-prepare.service /etc/systemd/system/mqt-slurm-prepare.service - -RUN --mount=type=bind,source=dist,target=/tmp/dist \ - --mount=type=cache,target=/root/.cache/uv \ - uv pip install --system --break-system-packages --link-mode=copy /tmp/dist/*.whl \ - && chmod 0755 /usr/local/sbin/mqt-slurm-prepare \ - && systemctl enable mqt-slurm-prepare.service munge.service - -STOPSIGNAL SIGRTMIN+3 - -CMD ["/usr/lib/systemd/systemd"] - -FROM common AS controller - -RUN systemctl enable slurmctld.service - -FROM common AS compute - -RUN systemctl enable slurmd.service diff --git a/test/slurm/bell_job.py b/test/slurm/bell_job.py index ca94ed5b61..07d02adacf 100644 --- a/test/slurm/bell_job.py +++ b/test/slurm/bell_job.py @@ -58,7 +58,7 @@ def main() -> None: "node": os.environ["SLURM_JOB_NODELIST"], "shots": SHOTS, } - runtime = Path("/runtime") + runtime = Path("/jobs") result_path = runtime / f"ddsim-{job_id}.json" temporary = result_path.with_suffix(".tmp") temporary.write_text(json.dumps(result, sort_keys=True), encoding="utf-8") diff --git a/test/slurm/compose.yml b/test/slurm/compose.yml index b933903632..e17ff25ec0 100644 --- a/test/slurm/compose.yml +++ b/test/slurm/compose.yml @@ -7,101 +7,6 @@ # Licensed under the MIT License services: - controller: - build: - context: . - target: controller - hostname: controller - privileged: true - cgroup: host - tmpfs: - - /run - - /run/lock + node: volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:rw - - ../..:/workspace:ro - - ${MQT_CORE_SLURM_RUNTIME:-./runtime}:/runtime - - ./slurm.conf:/etc/slurm/slurm.conf:ro - - ./cgroup.conf:/etc/slurm/cgroup.conf:ro - healthcheck: - test: - [ - "CMD", - "systemctl", - "is-active", - "--quiet", - "munge.service", - "slurmctld.service", - ] - interval: 2s - timeout: 2s - retries: 30 - start_period: 5s - - node1: - build: - context: . - target: compute - hostname: node1 - privileged: true - cgroup: host - depends_on: - controller: - condition: service_healthy - tmpfs: - - /run - - /run/lock - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:rw - - ../..:/workspace:ro - - ${MQT_CORE_SLURM_RUNTIME:-./runtime}:/runtime - - ./slurm.conf:/etc/slurm/slurm.conf:ro - - ./cgroup.conf:/etc/slurm/cgroup.conf:ro - healthcheck: - test: - [ - "CMD", - "systemctl", - "is-active", - "--quiet", - "munge.service", - "slurmd.service", - ] - interval: 2s - timeout: 2s - retries: 30 - start_period: 5s - - node2: - build: - context: . - target: compute - hostname: node2 - privileged: true - cgroup: host - depends_on: - controller: - condition: service_healthy - tmpfs: - - /run - - /run/lock - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:rw - - ../..:/workspace:ro - - ${MQT_CORE_SLURM_RUNTIME:-./runtime}:/runtime - - ./slurm.conf:/etc/slurm/slurm.conf:ro - - ./cgroup.conf:/etc/slurm/cgroup.conf:ro - healthcheck: - test: - [ - "CMD", - "systemctl", - "is-active", - "--quiet", - "munge.service", - "slurmd.service", - ] - interval: 2s - timeout: 2s - retries: 30 - start_period: 5s + - ../../test/slurm/mqt-slurm-test-environment.conf:/etc/systemd/system/slurmd.service.d/mqt-test.conf:ro diff --git a/test/slurm/mqt-slurm-prepare.service b/test/slurm/mqt-slurm-prepare.service deleted file mode 100644 index ea26c9d04e..0000000000 --- a/test/slurm/mqt-slurm-prepare.service +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM -# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH -# All rights reserved. -# -# SPDX-License-Identifier: MIT -# -# Licensed under the MIT License - -[Unit] -Description=Prepare the MQT Core Slurm test node -RequiresMountsFor=/runtime -Before=munge.service slurmctld.service slurmd.service - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/mqt-slurm-prepare -RemainAfterExit=yes - -[Install] -WantedBy=multi-user.target diff --git a/test/slurm/mqt-slurm-test-environment.conf b/test/slurm/mqt-slurm-test-environment.conf new file mode 100644 index 0000000000..90497850f6 --- /dev/null +++ b/test/slurm/mqt-slurm-test-environment.conf @@ -0,0 +1,11 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +[Service] +Environment=MQT_SLURM_TEST_REFERENCE=daemon-only +Environment=MQT_CORE_QDMI_CONFIG_FILE=/daemon-only/qdmi.json diff --git a/test/slurm/provider_probe.py b/test/slurm/provider_probe.py new file mode 100644 index 0000000000..a7c745838f --- /dev/null +++ b/test/slurm/provider_probe.py @@ -0,0 +1,38 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +"""Common assertions for QDMI device workloads in the shared Slurm fixture.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import TYPE_CHECKING + +from mqt.core.qdmi import slurm + +if TYPE_CHECKING: + from mqt.core.qdmi import Device + + +def open_device_from_license() -> Device: + """Open the licensed device and require the catalogue's native library.""" + device = slurm.open_device_from_license() + catalogue = Path(os.environ["MQT_CORE_QDMI_CONFIG_FILE"]) + definitions = json.loads(catalogue.read_text(encoding="utf-8"))["qdmi"]["devices"] + device_id = os.environ["SLURM_JOB_LICENSES"].split(":", maxsplit=1)[0] + definition = next(entry for entry in definitions if entry["id"] == device_id) + library = (catalogue.parent / definition["library"]).resolve() + native_prefix = Path("/opt/provider-native") + if not library.is_relative_to(native_prefix): + assert not native_prefix.exists() + mappings = (line.split(maxsplit=5) for line in Path("/proc/self/maps").read_text(encoding="utf-8").splitlines()) + loaded = {Path(fields[5]).resolve() for fields in mappings if len(fields) == 6 and fields[5].startswith("/")} + assert {path for path in loaded if path.name == library.name} == {library} + return device diff --git a/test/slurm/run_integration.py b/test/slurm/run_integration.py index e936b7b7d5..2b87ad22e6 100644 --- a/test/slurm/run_integration.py +++ b/test/slurm/run_integration.py @@ -10,16 +10,17 @@ from __future__ import annotations +import argparse import contextlib import json import logging import os import re -import secrets import shlex import shutil import signal import subprocess +import sys import time import uuid from pathlib import Path @@ -30,20 +31,26 @@ ROOT = Path(__file__).resolve().parents[2] FIXTURE = ROOT / "test" / "slurm" -DIST = FIXTURE / "dist" -RUNTIME = FIXTURE / "runtime" / uuid.uuid4().hex +CLUSTER = ROOT / "docker" / "slurm" +DIST = ROOT / "dist" +RUNTIME = ROOT / "build" / "slurm-tests" / uuid.uuid4().hex +NODES: list[str] = [] COMPOSE = ( "docker", "compose", "--project-name", f"mqt-core-slurm-{RUNTIME.name}", "--file", + str(CLUSTER / "compose.yml"), + "--file", str(FIXTURE / "compose.yml"), ) TIMEOUT = 120.0 COMMAND_TIMEOUT = 30.0 RESULT_VISIBILITY_GRACE_PERIOD = 5.0 LOGGER = logging.getLogger(__name__) +COMPOSE_FILES: list[str] = [] +COMPOSE_ENV: dict[str, str] = {} def _communicate(process: subprocess.Popen[str], timeout: float) -> tuple[str, str]: @@ -107,11 +114,11 @@ def compose( ) -> subprocess.CompletedProcess[str]: """Run Docker Compose for this invocation's isolated project and artifacts.""" return run( - (*COMPOSE, *arguments), + (*COMPOSE, *COMPOSE_FILES, *arguments), check=check, timeout=timeout, capture_output=capture_output, - env={**os.environ, "MQT_CORE_SLURM_RUNTIME": str(RUNTIME)}, + env={**os.environ, **COMPOSE_ENV, "MQT_CORE_SLURM_RUNTIME": str(RUNTIME)}, ) @@ -122,7 +129,23 @@ def controller(*command: str, check: bool = True, timeout: float = COMMAND_TIMEO def compute(node: str, *command: str, check: bool = True) -> subprocess.CompletedProcess[str]: """Run a diagnostic command in one compute container.""" - return compose("exec", "-T", node, *command, check=check) + return compose("exec", "-T", "--index", str(NODES.index(node) + 1), "node", *command, check=check) + + +def job(*command: str, check: bool = True, timeout: float = COMMAND_TIMEOUT) -> subprocess.CompletedProcess[str]: + """Submit workloads as the same unprivileged user on every node.""" + return compose( + "exec", + "-T", + "--user", + "10000:10000", + "controller", + "env", + "PYTHONPATH=/workspace/test/slurm", + *command, + check=check, + timeout=timeout, + ) def wait_for(description: str, predicate: Callable[[], bool], timeout: float = TIMEOUT) -> None: @@ -195,14 +218,14 @@ def submit(script: str, license_expression: str, *, node: str | None = None, hol "--time=5", f"--licenses={license_expression}", "--chdir=/workspace", - "--output=/runtime/slurm-%j.out", + "--output=/jobs/slurm-%j.out", ] if node is not None: command.append(f"--nodelist={node}") command.append(f"/workspace/test/slurm/{script}") if hold: command.append("--hold") - job_id = controller(*command).stdout.strip().split(";", maxsplit=1)[0] + job_id = job(*command).stdout.strip().split(";", maxsplit=1)[0] if not job_id.isdecimal(): msg = f"sbatch returned an invalid job ID: {job_id!r}" raise RuntimeError(msg) @@ -226,12 +249,12 @@ def assert_license(name: str, *, total: int, used: int, free: int) -> None: def load_result(kind: str, job_id: str) -> dict[str, Any]: """Load one batch-job result from the shared runtime directory.""" - return json.loads((RUNTIME / f"{kind}-{job_id}.json").read_text(encoding="utf-8")) + return json.loads((RUNTIME / "jobs" / f"{kind}-{job_id}.json").read_text(encoding="utf-8")) def wait_for_result(kind: str, job_id: str, description: str) -> None: """Wait for a result and allow bounded shared-file visibility delay.""" - result_path = RUNTIME / f"{kind}-{job_id}.json" + result_path = RUNTIME / "jobs" / f"{kind}-{job_id}.json" left_queue_at: float | None = None def result_exists_or_raise() -> bool: @@ -249,7 +272,7 @@ def result_exists_or_raise() -> bool: if now - left_queue_at < RESULT_VISIBILITY_GRACE_PERIOD: return False - output_path = RUNTIME / f"slurm-{job_id}.out" + output_path = RUNTIME / "jobs" / f"slurm-{job_id}.out" output = output_path.read_text(encoding="utf-8") if output_path.exists() else "" msg = f"Slurm job {job_id} exited before producing {result_path.name}:\n{output.rstrip()}" raise RuntimeError(msg) @@ -259,7 +282,7 @@ def result_exists_or_raise() -> bool: def wait_for_failed_adapter(job_id: str, diagnostic: str) -> None: """Require an adapter diagnostic and a failed batch job without a result.""" - output_path = RUNTIME / f"slurm-{job_id}.out" + output_path = RUNTIME / "jobs" / f"slurm-{job_id}.out" def failed_with_diagnostic() -> bool: if not job_finished(job_id, expected_state="FAILED") or not output_path.exists(): @@ -267,7 +290,7 @@ def failed_with_diagnostic() -> bool: return diagnostic in output_path.read_text(encoding="utf-8") wait_for(f"Slurm job {job_id} to fail with {diagnostic!r}", failed_with_diagnostic) - if (RUNTIME / f"ddsim-{job_id}.json").exists(): + if (RUNTIME / "jobs" / f"ddsim-{job_id}.json").exists(): msg = f"Rejected Slurm job {job_id} unexpectedly produced a DDSIM result" raise AssertionError(msg) @@ -291,10 +314,7 @@ def assert_bell_result(job_id: str, expected_node: str | None = None) -> None: def clean_runtime() -> None: """Create private artifacts and a Munge key for this invocation only.""" - RUNTIME.mkdir(mode=0o700, parents=True, exist_ok=False) - key = RUNTIME / "munge.key" - key.write_bytes(secrets.token_bytes(1024)) - key.chmod(0o600) + run(("sh", str(CLUSTER / "prepare.sh"), str(RUNTIME))) def print_diagnostics() -> None: @@ -310,22 +330,20 @@ def print_diagnostics() -> None: ) controller("scontrol", "show", "node", check=False, timeout=5) controller("scontrol", "show", "lic", check=False, timeout=5) - for output in sorted(RUNTIME.glob("slurm-*.out")): + for output in sorted((RUNTIME / "jobs").glob("slurm-*.out")): LOGGER.info("=== %s ===", output.name) try: LOGGER.info("%s", output.read_text(encoding="utf-8").rstrip()) except OSError as error: LOGGER.info("Could not read %s: %s", output, error) - for service, units in ( - ("controller", ("munge.service", "slurmctld.service")), - ("node1", ("munge.service", "slurmd.service")), - ("node2", ("munge.service", "slurmd.service")), - ): - compose("exec", "-T", service, "systemctl", "status", "--no-pager", *units, check=False, timeout=5) + for service, index, units in [ + ("controller", 1, ("munge.service", "slurmctld.service")), + *(("node", index, ("munge.service", "slurmd.service")) for index in range(1, len(NODES) + 1)), + ]: + prefix = ("exec", "-T", "--index", str(index), service) + compose(*prefix, "systemctl", "status", "--no-pager", *units, check=False, timeout=5) compose( - "exec", - "-T", - service, + *prefix, "journalctl", "--no-pager", "--lines=100", @@ -336,12 +354,199 @@ def print_diagnostics() -> None: compose("logs", "--no-color", check=False, timeout=5) -def main() -> None: - """Build the cluster and verify Slurm admission and DDSIM execution.""" - wheels = tuple(DIST.glob("*.whl")) +def test_core() -> None: + """Verify admission and execution with the bundled DDSIM and SC devices.""" + registry_check = ( + "from pathlib import Path; " + "import mqt.core; " + "from mqt.core.qdmi import device_ids; " + "module_path = Path(mqt.core.__file__).resolve(); " + "assert not any(module_path.is_relative_to(root) for root in ('/workspace', '/runtime')), module_path; " + "ids = set(device_ids()); " + "assert 'mqt.ddsim.default' in ids and 'mqt.sc.default' in ids, ids" + ) + controller("python3", "-c", registry_check) + assert_license("mqt.ddsim.default", total=2, used=0, free=2) + assert_license("mqt.sc.default", total=1, used=0, free=1) + + non_unit = submit("ddsim-job.sh", "mqt.ddsim.default:2") + wait_for_failed_adapter(non_unit, "must request exactly one Slurm license") + compound = submit("ddsim-job.sh", "mqt.ddsim.default:1,mqt.sc.default:1") + wait_for_failed_adapter(compound, "uses a compound AND expression") + alternative = submit("ddsim-job.sh", "mqt.ddsim.default:1|mqt.sc.default:1") + wait_for_failed_adapter(alternative, "uses a compound OR expression") + assert_license("mqt.ddsim.default", total=2, used=0, free=2) + assert_license("mqt.sc.default", total=1, used=0, free=1) + + first = submit("ddsim-job.sh", "mqt.ddsim.default:1", node=NODES[0], hold=True) + second = submit("ddsim-job.sh", "mqt.ddsim.default:1", node=NODES[1], hold=True) + wait_for_result("ddsim", first, "the first DDSIM Bell result") + wait_for_result("ddsim", second, "the second DDSIM Bell result") + wait_for("the first DDSIM job to hold on its node", lambda: job_matches(first, "RUNNING", node=NODES[0])) + wait_for("the second DDSIM job to hold on its node", lambda: job_matches(second, "RUNNING", node=NODES[1])) + if "CPUAlloc=1" not in node_record(NODES[0]) or "CPUAlloc=1" not in node_record(NODES[1]): + msg = "Each held DDSIM job must leave one processor free on its compute node" + raise AssertionError(msg) + + third = submit("ddsim-job.sh", "mqt.ddsim.default:1") + wait_for( + "the third DDSIM job to wait for its license", + lambda: job_matches(third, "PENDING", reason="Licenses"), + ) + assert_license("mqt.ddsim.default", total=2, used=2, free=0) + + sc_job = submit("sc-job.sh", "mqt.sc.default:1") + wait_for_result("sc", sc_job, "the SC job to execute on a free CPU") + wait_for("the SC job to complete", lambda: job_finished(sc_job)) + sc_result = load_result("sc", sc_job) + if sc_result["node"] not in NODES or sc_result["qubits"] <= 0: + msg = f"Unexpected SC job result: {sc_result}" + raise AssertionError(msg) + if not job_matches(first, "RUNNING", node=NODES[0]) or not job_matches(second, "RUNNING", node=NODES[1]): + msg = "The SC job did not complete while both DDSIM licenses remained held" + raise AssertionError(msg) + + configuration = RUNTIME / "jobs" / "availability.json" + configuration.write_text( + json.dumps({"schema-version": 1, "qdmi": {"devices": [{"id": "mqt.ddsim.default", "enabled": False}]}}), + encoding="utf-8", + ) + monitor = ("python3", "/workspace/examples/slurm/update_availability.py", "--license", "mqt.ddsim.default:2") + assert controller("env", "MQT_CORE_QDMI_CONFIG_FILE=/jobs/availability.json", *monitor, check=False).returncode == 1 + assert license_record("mqt.ddsim.default")["Reserved"] == "2" + assert job_matches(first, "RUNNING") + assert job_matches(second, "RUNNING") + for job_id in (first, second): + (RUNTIME / "jobs" / f"release-{job_id}").touch() + wait_for(f"the released DDSIM job {job_id} to finish", lambda job_id=job_id: job_finished(job_id)) + assert_license("mqt.ddsim.default", total=2, used=0, free=2) + controller(*monitor, "--block-only") + job("srun", "--immediate=5", "--time=1", "--licenses=mqt.sc.default", "/bin/true", timeout=60) + assert job_matches(third, "PENDING", node="", reason="Licenses") + assert job("scontrol", "delete", "ReservationName=qdmi-unavailable-mqt.ddsim.default", check=False).returncode != 0 + + controller(*monitor) + wait_for_result("ddsim", third, "the pending DDSIM job to execute after recovery") + wait_for("the third DDSIM job to finish", lambda: job_finished(third)) + assert_bell_result(first, NODES[0]) + assert_bell_result(second, NODES[1]) + assert_bell_result(third) + assert_license("mqt.ddsim.default", total=2, used=0, free=2) + assert license_record("mqt.ddsim.default")["Reserved"] == "0" + + LOGGER.info( + "Slurm enforced license capacity, kept unavailable-device jobs pending without allocating nodes, " + "and resumed the pending Bell job after a successful health check." + ) + + +def parse_arguments(arguments: Sequence[str]) -> argparse.Namespace: + """Read the provider build inputs and the command to run in an allocation.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workload", type=Path, default=CLUSTER) + parser.add_argument("--dist", type=Path, default=DIST) + parser.add_argument("--nodes", type=int, default=2) + parser.add_argument("--setup-script", default="") + parser.add_argument("--compose-file", type=Path) + parser.add_argument("--device-license") + parser.add_argument("--qdmi-config-file") + parser.add_argument("command", nargs=argparse.REMAINDER) + options = parser.parse_args(arguments) + if options.nodes < 2: + parser.error("--nodes must be at least two for the admission tests") + if options.command[:1] == ["--"]: + options.command = options.command[1:] + if bool(options.command) != bool(options.device_license): + parser.error("--device-license and a command after -- must be supplied together") + if options.device_license and re.search(r"[\s,:|@]", options.device_license): + parser.error("--device-license must be one local device ID without a count") + if options.setup_script: + setup = (options.workload / options.setup_script).resolve() + if not setup.is_relative_to(options.workload.resolve()) or not setup.is_file(): + parser.error("--setup-script must name a file inside --workload") + return options + + +def test_provider(options: argparse.Namespace) -> None: + """Execute a device workload with its submission environment.""" + environment = [f"MQT_CORE_QDMI_CONFIG_FILE={options.qdmi_config_file}"] if options.qdmi_config_file else [] + allocation = ("srun", "--immediate=5", "--time=5", "--ntasks=1", f"--licenses={options.device_license}:1") + job("env", *environment, *allocation, *options.command, timeout=300) + + +def test_job_environment() -> None: + """Export job settings through srun and sbatch independently of daemon settings.""" + program = ( + "import os; assert os.geteuid() == 10000; " + "assert os.environ['MQT_CORE_QDMI_CONFIG_FILE'] == '/jobs/devices.json'; " + "assert os.environ['MQT_SLURM_TEST_REFERENCE'] == 'job-value'" + ) + environment = ("env", "MQT_CORE_QDMI_CONFIG_FILE=/jobs/devices.json", "MQT_SLURM_TEST_REFERENCE=job-value") + job(*environment, "srun", "--immediate=5", "--time=1", "python3", "-c", program, timeout=60) + job( + *environment, + "sbatch", + "--wait", + "--time=1", + "--output=/jobs/environment.out", + "--wrap", + shlex.join(("python3", "-c", program)), + timeout=120, + ) + job( + "srun", + "--immediate=5", + "--time=1", + "python3", + "-c", + "import os; assert 'MQT_SLURM_TEST_REFERENCE' not in os.environ; " + "assert 'MQT_CORE_QDMI_CONFIG_FILE' not in os.environ", + timeout=60, + ) + + +def test_explicit_check() -> None: + """Run the payload only after the checker accepts the job's configuration.""" + body = RUNTIME / "jobs" / "checked-body" + (RUNTIME / "jobs" / "checker.qdmi.json").write_text( + json.dumps({"schema-version": 1, "qdmi": {"devices": [{"id": "mqt.sc.default", "enabled": False}]}}), + encoding="utf-8", + ) + for enabled in (True, False): + body.unlink(missing_ok=True) + setup = "unset MQT_CORE_QDMI_CONFIG_FILE MQT_CORE_QDMI_CONFIG_JSON\n" + if not enabled: + setup += "export MQT_CORE_QDMI_CONFIG_FILE=/jobs/checker.qdmi.json\n" + result = job( + "srun", + "--immediate=5", + "--time=1", + "--ntasks=1", + "--licenses=mqt.sc.default", + "sh", + "-ec", + setup + "mqt-core-qdmi-check --device mqt.sc.default --timeout 10\ntouch /jobs/checked-body", + check=False, + timeout=60, + ) + assert (result.returncode == 0) == enabled + assert body.exists() == enabled + + +def main(arguments: Sequence[str] = ()) -> None: + """Build the shared cluster and run its Core or provider workload.""" + options = parse_arguments(arguments) + wheels = tuple(options.dist.glob("mqt_core-*.whl")) if len(wheels) != 1: - msg = f"Build exactly one MQT Core wheel in {DIST}, found {len(wheels)}" + msg = f"Build exactly one MQT Core wheel in {options.dist}, found {len(wheels)}" raise RuntimeError(msg) + COMPOSE_ENV.update({ + "MQT_CORE_SLURM_CORE": str(ROOT), + "MQT_CORE_SLURM_WORKLOAD": str(options.workload.resolve()), + "MQT_CORE_SLURM_DIST": str(options.dist.resolve()), + "MQT_CORE_SLURM_SETUP_SCRIPT": options.setup_script, + }) + COMPOSE_FILES[:] = ("--file", str(options.compose_file.resolve())) if options.compose_file else () started_at = time.monotonic() @@ -354,9 +559,30 @@ def main() -> None: raise RuntimeError(msg) clean_runtime() + if options.device_license: + configuration = (RUNTIME / "slurm.conf").read_text(encoding="utf-8") + configuration = re.sub( + r"^Licenses=(.*)$", rf"Licenses=\1,{options.device_license}:2", configuration, flags=re.MULTILINE + ) + (RUNTIME / "slurm.conf").write_text(configuration, encoding="utf-8") LOGGER.info("Slurm runtime directory: %s", RUNTIME) started = True - compose("up", "--build", "--detach", "--wait", "--wait-timeout", "120", timeout=600, capture_output=False) + compose( + "up", + "--build", + "--detach", + "--scale", + f"node={options.nodes}", + "--wait", + "--wait-timeout", + "120", + timeout=1800, + capture_output=False, + ) + NODES[:] = [ + compose("exec", "-T", "--index", str(index), "node", "hostname").stdout.strip() + for index in range(1, options.nodes + 1) + ] LOGGER.info("Slurm image build and startup: %.2fs", time.monotonic() - started_at) testing_at = time.monotonic() @@ -366,7 +592,7 @@ def main() -> None: msg = f"The fixture requires Slurm 25.11 or newer, got {version_output!r}" raise RuntimeError(msg) - for node in ("node1", "node2"): + for node in NODES: compute(node, "test", "-r", "/sys/fs/cgroup/cgroup.controllers") delegate = compute( node, @@ -381,74 +607,15 @@ def main() -> None: raise RuntimeError(msg) wait_for(f"{node} to become IDLE with two processors", lambda node=node: node_is_idle(node)) - registry_check = ( - "from pathlib import Path; " - "import mqt.core; " - "from mqt.core.qdmi import device_ids; " - "module_path = Path(mqt.core.__file__).resolve(); " - "assert not any(module_path.is_relative_to(root) for root in ('/workspace', '/runtime')), module_path; " - "ids = set(device_ids()); " - "assert 'mqt.ddsim.default' in ids and 'mqt.sc.default' in ids, ids" - ) - controller("python3", "-c", registry_check) - assert_license("mqt.ddsim.default", total=2, used=0, free=2) - assert_license("mqt.sc.default", total=1, used=0, free=1) - - non_unit = submit("ddsim-job.sh", "mqt.ddsim.default:2") - wait_for_failed_adapter(non_unit, "must request exactly one Slurm license") - compound = submit("ddsim-job.sh", "mqt.ddsim.default:1,mqt.sc.default:1") - wait_for_failed_adapter(compound, "uses a compound AND expression") - alternative = submit("ddsim-job.sh", "mqt.ddsim.default:1|mqt.sc.default:1") - wait_for_failed_adapter(alternative, "uses a compound OR expression") - assert_license("mqt.ddsim.default", total=2, used=0, free=2) - assert_license("mqt.sc.default", total=1, used=0, free=1) - - first = submit("ddsim-job.sh", "mqt.ddsim.default:1", node="node1", hold=True) - second = submit("ddsim-job.sh", "mqt.ddsim.default:1", node="node2", hold=True) - wait_for_result("ddsim", first, "the first DDSIM Bell result") - wait_for_result("ddsim", second, "the second DDSIM Bell result") - wait_for("the first DDSIM job to hold on node1", lambda: job_matches(first, "RUNNING", node="node1")) - wait_for("the second DDSIM job to hold on node2", lambda: job_matches(second, "RUNNING", node="node2")) - if "CPUAlloc=1" not in node_record("node1") or "CPUAlloc=1" not in node_record("node2"): - msg = "Each held DDSIM job must leave one processor free on its compute node" - raise AssertionError(msg) - - third = submit("ddsim-job.sh", "mqt.ddsim.default:1") - wait_for( - "the third DDSIM job to wait for its license", - lambda: job_matches(third, "PENDING", reason="Licenses"), - ) - assert_license("mqt.ddsim.default", total=2, used=2, free=0) - - sc_job = submit("sc-job.sh", "mqt.sc.default:1") - wait_for_result("sc", sc_job, "the SC job to execute on a free CPU") - wait_for("the SC job to complete", lambda: job_finished(sc_job)) - sc_result = load_result("sc", sc_job) - if sc_result["node"] not in {"node1", "node2"} or sc_result["qubits"] <= 0: - msg = f"Unexpected SC job result: {sc_result}" - raise AssertionError(msg) - if not job_matches(first, "RUNNING", node="node1") or not job_matches(second, "RUNNING", node="node2"): - msg = "The SC job did not complete while both DDSIM licenses remained held" - raise AssertionError(msg) - - (RUNTIME / f"release-{first}").touch() - wait_for("the released first DDSIM job to finish", lambda: job_finished(first)) - wait_for_result("ddsim", third, "the pending third DDSIM job to execute") - wait_for("the third DDSIM job to finish", lambda: job_finished(third)) - - assert_bell_result(first, "node1") - assert_bell_result(second, "node2") - assert_bell_result(third) - assert_license("mqt.ddsim.default", total=2, used=1, free=1) - - (RUNTIME / f"release-{second}").touch() - wait_for("the released second DDSIM job to finish", lambda: job_finished(second)) - assert_license("mqt.ddsim.default", total=2, used=0, free=2) - - LOGGER.info( - "Slurm 25.11+ admitted two held DDSIM jobs, blocked the third for Licenses, " - "ran the SC job on a free CPU, and executed the third Bell job after release." - ) + registered = set(controller("sinfo", "--Node", "--noheader", "--format=%N").stdout.split()) + assert registered == set(NODES), registered + if options.command: + test_provider(options) + else: + test_core() + test_job_environment() + test_explicit_check() + success = True LOGGER.info("Slurm admission and execution checks: %.2fs", time.monotonic() - testing_at) finally: @@ -471,4 +638,4 @@ def main() -> None: if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format="%(message)s") - main() + main(sys.argv[1:]) diff --git a/test/slurm/sc_job.py b/test/slurm/sc_job.py index 5c5878b3b0..9e2349fc91 100644 --- a/test/slurm/sc_job.py +++ b/test/slurm/sc_job.py @@ -28,7 +28,7 @@ def main() -> None: "node": os.environ["SLURM_JOB_NODELIST"], "qubits": device.qubits_num(), } - result_path = Path(f"/runtime/sc-{job_id}.json") + result_path = Path(f"/jobs/sc-{job_id}.json") temporary = result_path.with_suffix(".tmp") temporary.write_text(json.dumps(result, sort_keys=True), encoding="utf-8") temporary.replace(result_path)