Skip to content

Commit 412c280

Browse files
authored
fix: resovle the CI failures when driver versions mismatch for kernel… (#4638)
1 parent 0fad50d commit 412c280

1 file changed

Lines changed: 71 additions & 3 deletions

File tree

‎tests/py/kernels/conftest.py‎

Lines changed: 71 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,16 +37,84 @@ def _has_cuda_core() -> bool:
3737
)
3838

3939

40+
def _decode_cuda_driver_version(version: int) -> tuple[int, int]:
41+
"""Decode CUDA's integer driver API version into ``(major, minor)``."""
42+
if version <= 0:
43+
raise ValueError(f"invalid CUDA driver version: {version}")
44+
return version // 1000, (version % 1000) // 10
45+
46+
47+
def _cuda_driver_nvrtc_versions() -> tuple[tuple[int, int], tuple[int, int]]:
48+
"""Return the maximum CUDA version supported by the driver and NVRTC's version."""
49+
try:
50+
from cuda.bindings import driver, nvrtc
51+
except ImportError:
52+
# cuda-python < 12.8 exposed the same APIs from these modules.
53+
from cuda import cuda as driver
54+
from cuda import nvrtc
55+
56+
driver_status, encoded_driver_version = driver.cuDriverGetVersion()
57+
if int(driver_status) != 0:
58+
raise RuntimeError(
59+
f"cuDriverGetVersion failed with CUDA error {int(driver_status)}"
60+
)
61+
62+
nvrtc_status, nvrtc_major, nvrtc_minor = nvrtc.nvrtcVersion()
63+
if int(nvrtc_status) != 0:
64+
raise RuntimeError(f"nvrtcVersion failed with CUDA error {int(nvrtc_status)}")
65+
66+
return _decode_cuda_driver_version(encoded_driver_version), (
67+
nvrtc_major,
68+
nvrtc_minor,
69+
)
70+
71+
72+
def _ptx_compatibility_skip_reason(
73+
driver_version: tuple[int, int], nvrtc_version: tuple[int, int]
74+
) -> str | None:
75+
"""Explain why NVRTC PTX cannot be loaded, or return ``None`` if compatible."""
76+
if driver_version >= nvrtc_version:
77+
return None
78+
79+
driver = ".".join(map(str, driver_version))
80+
backend = ".".join(map(str, nvrtc_version))
81+
return (
82+
"CUDA kernel tests require a driver capable of loading the PTX emitted "
83+
f"by NVRTC, but the active driver supports CUDA {driver} and NVRTC is "
84+
f"CUDA {backend}. Upgrade the CI runner driver to >= {backend}, or use "
85+
f"a CUDA toolkit/runtime <= {driver}."
86+
)
87+
88+
4089
def pytest_collection_modifyitems(config, items):
41-
"""Skip the whole kernels suite when cuda-core is missing.
90+
"""Skip the kernels suite when its CUDA PTX toolchain cannot be used.
4291
4392
The QDP kernel tests compile CUDA via the ``cuda.core`` NVRTC API; without
4493
cuda-core they all error on import. Skip (not fail) so a plain test run is
4594
green — `just install-test-ext` pulls cuda-core so they actually run.
95+
96+
NVRTC also emits PTX that must be JIT-loaded by the installed driver. CUDA's
97+
minor-version compatibility does not cover PTX generated by a newer toolkit,
98+
so do not run the suite when the driver API version predates NVRTC.
4699
"""
47-
if _HAS_CUDA_CORE:
100+
if not _HAS_CUDA_CORE:
101+
reason = "cuda-core (cuda.core) not installed"
102+
elif not torch.cuda.is_available():
48103
return
49-
skip = pytest.mark.skip(reason="cuda-core (cuda.core) not installed")
104+
else:
105+
try:
106+
driver_version, nvrtc_version = _cuda_driver_nvrtc_versions()
107+
reason = _ptx_compatibility_skip_reason(driver_version, nvrtc_version)
108+
except Exception as exc:
109+
# Only run kernel tests after positively verifying that their PTX can
110+
# be loaded. This turns an opaque collection of kernel failures into
111+
# one actionable CI skip reason if version discovery itself breaks.
112+
reason = f"could not verify CUDA driver/NVRTC compatibility: {exc}"
113+
114+
if reason is None:
115+
return
116+
117+
skip = pytest.mark.skip(reason=reason)
50118
for item in items:
51119
item.add_marker(skip)
52120

0 commit comments

Comments
 (0)