diff --git a/.ci/scripts/tests/test_cuda_workflow.py b/.ci/scripts/tests/test_cuda_workflow.py index 4ab845d8fab..3c3c989ddb3 100644 --- a/.ci/scripts/tests/test_cuda_workflow.py +++ b/.ci/scripts/tests/test_cuda_workflow.py @@ -25,6 +25,45 @@ def _supported_cuda_versions(): return [f"{major}.{minor}" for major, minor in versions] +def _matrix_cuda_versions(job): + """The versions a job's gpu-arch-version takes, read from the key it names.""" + expression = re.fullmatch( + r"\$\{\{ matrix\.([\w-]+) \}\}", job["with"]["gpu-arch-version"] + ) + assert expression, job["with"]["gpu-arch-version"] + key = expression.group(1) + matrix = job["strategy"]["matrix"] + axis = matrix.get(key, []) + assert isinstance(axis, list), axis + # GitHub drops a combination only when it matches every key of an exclude row, and + # applies include after exclude. An exclude that names other keys too could drop + # every row of a version without this seeing it, so it isn't allowed. + excludes = [row for row in matrix.get("exclude", []) if key in row] + assert all(row.keys() == {key} for row in excludes), excludes + excluded = {row[key] for row in excludes} + versions = [version for version in axis if version not in excluded] + return versions + [row[key] for row in matrix.get("include", []) if key in row] + + +def _rendered_names(job): + """The check names GitHub gives a job's include rows, with each row's values filled in.""" + names = [] + for row in job["strategy"]["matrix"]["include"]: + name = job["name"] + for key, value in row.items(): + name = name.replace(f"${{{{ matrix.{key} }}}}", str(value)) + assert "${{" not in name, name + names.append(name) + return names + + +WINDOWS_TRAIN_JOBS = ( + "export-model-cuda-windows-artifact", + "test-model-cuda-windows-e2e", + "unittest-cuda-runtime-windows", +) + + def _all_keys(value): if isinstance(value, dict): for key, child in value.items(): @@ -56,15 +95,81 @@ def test_every_cuda_job_uses_a_supported_version(self): )["jobs"] for name, job in jobs.items(): with_ = job.get("with", {}) - version = with_.get("gpu-arch-version") - if with_.get("gpu-arch-type") != "cuda" or "${{" in str(version): + if with_.get("gpu-arch-type") != "cuda": continue - checked += 1 - with self.subTest(workflow=workflow, job=name): - self.assertIn(version, supported) + if "${{" in str(with_["gpu-arch-version"]): + versions = _matrix_cuda_versions(job) + else: + versions = [with_["gpu-arch-version"]] + for version in versions: + checked += 1 + with self.subTest(workflow=workflow, job=name, version=version): + self.assertIn(version, supported) self.assertGreater(checked, 0) - def test_windows_export_image_matches_the_export_job(self): + def test_windows_jobs_cover_every_supported_train(self): + jobs = yaml.safe_load( + (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() + )["jobs"] + for name in WINDOWS_TRAIN_JOBS: + with self.subTest(job=name): + job = jobs[name] + self.assertEqual( + job["with"]["gpu-arch-version"], "${{ matrix.cuda-version }}" + ) + # GitHub cuts job names at 100 characters, so the train goes first. + self.assertTrue( + job["name"].startswith(f"{name}-${{{{ matrix.cuda-version }}}}"), + job["name"], + ) + # A name that uses a matrix expression stops GitHub appending the matrix + # values, so the name itself has to tell every row apart, within 100 + # characters. + if "include" in job["strategy"]["matrix"]: + rendered = _rendered_names(job) + self.assertEqual(len(set(rendered)), len(rendered), rendered) + for check in rendered: + self.assertLessEqual(len(check), 100, check) + self.assertEqual( + sorted(set(_matrix_cuda_versions(job))), + sorted(_supported_cuda_versions()), + ) + for name in ("test-model-cuda-windows-e2e", "unittest-cuda-runtime-windows"): + with self.subTest(job=name): + script = jobs[name]["with"]["script"] + self.assertIn("CUDA\\v${{ matrix.cuda-version }}'", script) + self.assertNotRegex(script, r"CUDA\\v\d") + # The 13.x Windows toolkits keep their DLLs in bin\x64, so bin\x64 goes + # ahead of bin on PATH. The toolkit has to lead PATH before setup, since + # setup.py builds the CUDA extension with the first nvcc on PATH. + path = script.index( + r"PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;" + ) + self.assertLess(path, script.index(".ci/scripts/setup-windows.ps1")) + export = jobs["export-model-cuda-windows-artifact"]["strategy"]["matrix"] + e2e = jobs["test-model-cuda-windows-e2e"]["strategy"]["matrix"] + self.assertEqual(export["include"], e2e["include"]) + # Only the tile-packed quant lowers to _weight_int4pack_mm, so each train needs it. + for version in _supported_cuda_versions(): + with self.subTest(train=version): + self.assertTrue( + any( + row["quant"] == "quantized-int4-tile-packed" + and row["cuda-version"] == version + for row in export["include"] + ) + ) + + def test_one_failed_export_row_does_not_skip_the_others(self): + jobs = yaml.safe_load( + (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() + )["jobs"] + gate = jobs["test-model-cuda-windows-e2e"]["if"] + self.assertNotIn("export-model-cuda-windows-artifact.result == 'success'", gate) + self.assertIn("!cancelled()", gate) + self.assertIn("export-model-cuda-windows-artifact.result != 'skipped'", gate) + + def test_windows_export_fetches_toolkits_the_image_lacks(self): build = (ROOT / ".ci" / "docker" / "build.sh").read_text() match = re.search( r"executorch-ubuntu-22\.04-cuda-windows\)(.*?);;", build, re.S @@ -75,8 +180,40 @@ def test_windows_export_image_matches_the_export_job(self): jobs = yaml.safe_load( (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() )["jobs"] - export = jobs["export-model-cuda-windows-artifact"]["with"] - self.assertEqual(export["gpu-arch-version"], image_version) + export = jobs["export-model-cuda-windows-artifact"] + rows = export["strategy"]["matrix"]["include"] + # Every model runs on the image's own train, so only the extra rows download. + models = {(row["model_repo"], row["model_name"], row["quant"]) for row in rows} + on_image = { + (row["model_repo"], row["model_name"], row["quant"]) + for row in rows + if row["cuda-version"] == image_version + } + self.assertEqual(on_image, models) + script = export["with"]["script"] + self.assertIn('!= "${{ matrix.cuda-version }}" ]; then', script) + for platform, destination in ( + ("linux-x86_64", "cuda-${{ matrix.cuda-version }}"), + ("windows-x86_64", "cuda-windows-${{ matrix.cuda-version }}"), + ): + with self.subTest(platform=platform): + self.assertIn( + f'--platform {platform} --dest "${{TOOLKITS}}/{destination}"', + script, + ) + self.assertIn( + 'export WINDOWS_CUDA_HOME="${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}"', + script, + ) + # The fetched nvcc decides the torch train install_executorch picks, and the + # e2e runs CPU torch on Windows, so only this assert sees which torch exported. + for line in ( + 'export PATH="${CUDA_HOME}/bin:${PATH}"', + 'export CUDACXX="${CUDA_HOME}/bin/nvcc"', + "assert v == '${{ matrix.cuda-version }}'", + ): + with self.subTest(line=line): + self.assertIn(line, script) def test_cuda_builds_take_the_node_driver_on_an_unpinned_v3(self): # This job used to pin linux_job_v2 to a test-infra SHA because that was diff --git a/.ci/scripts/wheel/install_cuda_redist.py b/.ci/scripts/wheel/install_cuda_redist.py index daf0a92073d..9bfacd634e4 100644 --- a/.ci/scripts/wheel/install_cuda_redist.py +++ b/.ci/scripts/wheel/install_cuda_redist.py @@ -7,11 +7,12 @@ """Assemble part of a CUDA toolkit from NVIDIA's redistributable component archives. -The Windows CUDA wheel end-to-end jobs cover several CUDA trains while the CI images carry -one toolkit, and NVIDIA publishes no full Windows installer for every release. The redist -archives exist for every release on both platforms, need no installer or admin rights, and -are checksummed in a JSON index. Merging the components a job needs into one directory -gives the layout nvcc, CMake's FindCUDAToolkit and torch's cross-compile step expect. +The Windows CUDA wheel end-to-end jobs and cuda-windows.yml cover several CUDA trains +while the CI images carry one toolkit, and NVIDIA publishes no full Windows installer for +every release. The redist archives exist for every release on both platforms, need no +installer or admin rights, and are checksummed in a JSON index. Merging the components a +job needs into one directory gives the layout nvcc, CMake's FindCUDAToolkit and torch's +cross-compile step expect. install_cuda_redist.py --train 13.4 --platform linux-x86_64 --dest /opt/cuda-13.4 install_cuda_redist.py --train 13.4 --platform windows-x86_64 --dest DIR --components cuda_cudart @@ -31,8 +32,9 @@ _REDIST = "https://developer.download.nvidia.com/compute/cuda/redist" # The patch release each train resolves to: one entry per train in filter_cuda_matrix.py's -# SUPPORTED_CUDA_VERSIONS, which the end-to-end jobs take their matrix from. A train missing -# here fails the job at argument parsing rather than testing a different release. +# SUPPORTED_CUDA_VERSIONS, which the wheel end-to-end jobs take their matrix from, and in +# cuda-windows.yml's matrices. A train missing here fails the job at argument parsing rather +# than testing a different release. _RELEASES = {"13.2": "13.2.2", "13.4": "13.4.1"} # What a build of the CUDA backend needs. "a|b" names one component a release publishes diff --git a/.github/workflows/cuda-windows.yml b/.github/workflows/cuda-windows.yml index 09f7251d909..7cdac28dbac 100644 --- a/.github/workflows/cuda-windows.yml +++ b/.github/workflows/cuda-windows.yml @@ -19,6 +19,10 @@ on: - extension/cuda/** - .ci/scripts/export_model_artifact.sh - .ci/scripts/test_model_e2e_windows.ps1 + - .ci/scripts/wheel/install_cuda_redist.py + - install_requirements.py + - install_utils.py + - torch_pin.py # Only unittest-cuda-runtime-windows runs for these alone; the other jobs # filter on their own paths and stay skipped. - .ci/scripts/test_cuda_runtime_windows.ps1 @@ -49,7 +53,7 @@ jobs: uses: ./.github/workflows/_ci-run-decision.yml export-model-cuda-windows-artifact: - name: export-model-cuda-windows-artifact + name: export-model-cuda-windows-artifact-${{ matrix.cuda-version }} (${{ matrix.model_name }}, ${{ matrix.quant }}) # Skip this job if the pull request is from a fork (HuggingFace secrets are not available). # Path-filtered on push: mirrors the workflow-level pull_request `paths:` # filter so push commits that don't touch CUDA-relevant paths skip @@ -65,6 +69,10 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda-windows.yml') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e_windows.ps1') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/wheel/install_cuda_redist.py') || + contains(needs.changed-files.outputs.changed-files, 'install_requirements.py') || + contains(needs.changed-files.outputs.changed-files, 'install_utils.py') || + contains(needs.changed-files.outputs.changed-files, 'torch_pin.py') || needs.run-decision.outputs.is-full-run == 'true' ) uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main @@ -79,15 +87,19 @@ jobs: - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "non-quantized" + cuda-version: "13.2" - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "quantized-int4-weight-only" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "non-quantized" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "quantized-int4-weight-only" + cuda-version: "13.2" # TODO: sortformer produces 0 segments on Windows after D97788666. # Temporarily disabled until root cause is debugged. # - model_repo: "nvidia" @@ -97,18 +109,33 @@ jobs: model_name: "Voxtral-Mini-4B-Realtime-2602" quant: "quantized-int4-tile-packed" mode: "vr-offline" + cuda-version: "13.2" - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" + cuda-version: "13.2" + # Each train installs its own toolkit and, for the export, its own torch, so each + # needs rows of its own. Trains other than the image's (13.2) get only the + # smallest model and the tile-packed int4 model, the only quant here that lowers + # to _weight_int4pack_mm and so runs the int4mm shim that train's nvcc built. + - model_repo: "facebook" + model_name: "dinov2-small-imagenet1k-1-layer" + quant: "non-quantized" + cuda-version: "13.4" + - model_repo: "mistralai" + model_name: "Voxtral-Mini-4B-Realtime-2602" + quant: "quantized-int4-tile-packed" + mode: "vr-offline" + cuda-version: "13.4" with: timeout: 90 secrets-env: EXECUTORCH_HF_TOKEN runner: mt-l-x86aavx2-11-41-a10g gpu-arch-type: cuda - gpu-arch-version: "13.2" + gpu-arch-version: ${{ matrix.cuda-version }} docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-cuda-windows-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive - upload-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }} + upload-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}-cuda${{ matrix.cuda-version }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | set -eux @@ -129,7 +156,27 @@ jobs: ln -sf /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /opt/conda/lib/libstdc++.so.6 echo "::endgroup::" - echo "::group::Verify pre-installed dependencies" + # The image carries one train's toolkits; another train fetches its own Linux nvcc, + # which decides the torch install_executorch picks, and its own Windows runtime to + # link against. + # Under the job's temp directory: the image runs as ci-user, which cannot write to /opt. + if [ "$(nvcc --version | sed -n 's/.*release \([0-9.]*\),.*/\1/p')" != "${{ matrix.cuda-version }}" ]; then + echo "::group::Install the CUDA ${{ matrix.cuda-version }} toolkits" + TOOLKITS="${RUNNER_TEMP:-/tmp}/cuda-toolkits" + mkdir -p "${TOOLKITS}" 2>/dev/null || TOOLKITS=/tmp/cuda-toolkits + mkdir -p "${TOOLKITS}" + python .ci/scripts/wheel/install_cuda_redist.py --train "${{ matrix.cuda-version }}" \ + --platform linux-x86_64 --dest "${TOOLKITS}/cuda-${{ matrix.cuda-version }}" + export CUDA_HOME="${TOOLKITS}/cuda-${{ matrix.cuda-version }}" + export PATH="${CUDA_HOME}/bin:${PATH}" + export CUDACXX="${CUDA_HOME}/bin/nvcc" + python .ci/scripts/wheel/install_cuda_redist.py --train "${{ matrix.cuda-version }}" \ + --platform windows-x86_64 --dest "${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}" --components cuda_cudart + export WINDOWS_CUDA_HOME="${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}" + echo "::endgroup::" + fi + + echo "::group::Verify the toolchain" x86_64-w64-mingw32-g++ --version nvcc --version echo "WINDOWS_CUDA_HOME=${WINDOWS_CUDA_HOME}" @@ -140,6 +187,7 @@ jobs: # Disable MKL to avoid duplicate target error when conda has multiple MKL installations export USE_MKL=OFF PYTHON_EXECUTABLE=python ./install_executorch.sh + python -c "import torch; v = torch.version.cuda; assert v == '${{ matrix.cuda-version }}', v" echo "::endgroup::" # Setup Huggingface only for models that need it (not dinov2) @@ -155,15 +203,16 @@ jobs: source .ci/scripts/export_model_artifact.sh cuda-windows "${{ matrix.model_repo }}/${{ matrix.model_name }}" "${{ matrix.quant }}" "${RUNNER_ARTIFACT_DIR}" "${{ matrix.mode }}" test-model-cuda-windows-e2e: - name: test-model-cuda-windows-e2e - # Same path filter as the export job above. Also explicitly gated - # on the export job succeeding — when needs: jobs are *skipped* - # (e.g. fork PR), GitHub still evaluates this if:, so without the - # explicit success-check this job would run and then fail trying - # to download an artifact that was never produced. + name: test-model-cuda-windows-e2e-${{ matrix.cuda-version }} (${{ matrix.model_name }}, ${{ matrix.quant }}) + # Same path filter as the export job above. Not gated on the export succeeding: that + # result covers the whole matrix, so one row's failed export (say one model's) would + # skip every row here. Each row downloads its own artifact and fails on the download + # if its export failed. The `result != 'skipped'` condition keeps a fork pull request, + # where nothing was exported, from running this and failing on the download. needs: [changed-files, export-model-cuda-windows-artifact, run-decision] if: | - needs.export-model-cuda-windows-artifact.result == 'success' && + !cancelled() && + needs.export-model-cuda-windows-artifact.result != 'skipped' && ( contains(needs.changed-files.outputs.changed-files, 'backends/cuda') || contains(needs.changed-files.outputs.changed-files, 'backends/aoti') || @@ -171,6 +220,10 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda-windows.yml') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e_windows.ps1') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/wheel/install_cuda_redist.py') || + contains(needs.changed-files.outputs.changed-files, 'install_requirements.py') || + contains(needs.changed-files.outputs.changed-files, 'install_utils.py') || + contains(needs.changed-files.outputs.changed-files, 'torch_pin.py') || needs.run-decision.outputs.is-full-run == 'true' ) uses: pytorch/test-infra/.github/workflows/windows_job.yml@main @@ -181,15 +234,19 @@ jobs: - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "non-quantized" + cuda-version: "13.2" - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "quantized-int4-weight-only" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "non-quantized" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "quantized-int4-weight-only" + cuda-version: "13.2" # TODO: sortformer produces 0 segments on Windows after D97788666. # Temporarily disabled until root cause is debugged. # - model_repo: "nvidia" @@ -199,16 +256,28 @@ jobs: model_name: "Voxtral-Mini-4B-Realtime-2602" quant: "quantized-int4-tile-packed" mode: "vr-offline" + cuda-version: "13.2" - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" + cuda-version: "13.2" + # Same rows as the export matrix; see there for why. + - model_repo: "facebook" + model_name: "dinov2-small-imagenet1k-1-layer" + quant: "non-quantized" + cuda-version: "13.4" + - model_repo: "mistralai" + model_name: "Voxtral-Mini-4B-Realtime-2602" + quant: "quantized-int4-tile-packed" + mode: "vr-offline" + cuda-version: "13.4" with: timeout: 240 runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - # The same toolkit the export uses. - gpu-arch-version: "13.2" - download-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }} + # The same CUDA version the export used. + gpu-arch-version: ${{ matrix.cuda-version }} + download-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}-cuda${{ matrix.cuda-version }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | git config --global http.sslBackend openssl @@ -219,9 +288,9 @@ jobs: \$ErrorActionPreference = 'Stop' \$PSNativeCommandUseErrorActionPreference = \$true - \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2' + \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda-version }}' \$env:CUDA_PATH = \$env:CUDA_HOME - \$env:PATH = \"\$env:CUDA_HOME\bin;\$env:PATH\" + \$env:PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;\$env:PATH\" nvcc --version .ci/scripts/setup-windows.ps1 @@ -230,14 +299,14 @@ jobs: throw 'RUNNER_ARTIFACT_DIR is empty. Ensure download-artifact is configured for windows_job.yml.' } - .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '13.2' -Mode '${{ matrix.mode }}' + .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '${{ matrix.cuda-version }}' -Mode '${{ matrix.mode }}' }" # The counterpart of cuda.yml's unittest-cuda-runtime on Windows. The Windows GPU runners # carry data-center GPUs in TCC mode, which have no memory pools, so this also covers the # allocator's synchronous fallback that the Linux job cannot reach. unittest-cuda-runtime-windows: - name: unittest-cuda-runtime-windows + name: unittest-cuda-runtime-windows-${{ matrix.cuda-version }} needs: [changed-files, run-decision] if: | contains(needs.changed-files.outputs.changed-files, 'backends/cuda') || @@ -247,14 +316,20 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_cuda_runtime_windows.ps1') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/install_nvidia_driver_windows.ps1') || contains(needs.changed-files.outputs.changed-files, 'tools/cmake/Test.cmake') || + contains(needs.changed-files.outputs.changed-files, 'install_requirements.py') || + contains(needs.changed-files.outputs.changed-files, 'install_utils.py') || + contains(needs.changed-files.outputs.changed-files, 'torch_pin.py') || needs.run-decision.outputs.is-full-run == 'true' uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + strategy: + fail-fast: false + matrix: + cuda-version: ["13.2", "13.4"] with: timeout: 120 runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - # The same toolkit as test-model-cuda-windows-e2e, for the same reason. - gpu-arch-version: "13.2" + gpu-arch-version: ${{ matrix.cuda-version }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | git config --global http.sslBackend openssl @@ -263,8 +338,9 @@ jobs: powershell -Command "& { Set-PSDebug -Trace 1 \$ErrorActionPreference = 'Stop' - \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2' + \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda-version }}' \$env:CUDA_PATH = \$env:CUDA_HOME + \$env:PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;\$env:PATH\" .ci/scripts/setup-windows.ps1 .ci/scripts/test_cuda_runtime_windows.ps1 }" @@ -272,7 +348,7 @@ jobs: delete-model-cuda-windows-artifacts: name: delete-model-cuda-windows-artifacts # The exports exist only to hand the models to the Windows job above, about - # 18 GB per run, and GitHub bills artifact storage by the hour, so delete + # 21 GB per run, and GitHub bills artifact storage by the hour, so delete # them as soon as the tests have consumed them instead of keeping them for # the repository retention window. They are kept when a test fails so that # "Re-run failed jobs" and debugging still have them; those expire with the