Skip to content

Commit 364ac26

Browse files
authored
fix(executorch): back device-planned arenas when loading in Python (#4655)
1 parent c86da7c commit 364ac26

9 files changed

Lines changed: 475 additions & 38 deletions

File tree

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/usr/bin/env python3
2+
"""Drop CUDA architectures ExecuTorch's shims cannot compile for.
3+
4+
The shims call ``__dp4a``, which nvcc does not declare before sm_61, so a list carrying
5+
5.0 or 6.0 fails with ``identifier "__dp4a" is undefined`` whichever CUDA is in use. The
6+
cu126 build environment asks for ``5.0;6.0;7.0;7.5;8.0;8.6;9.0`` while the CUDA 13 ones
7+
start at 7.5, which is why only cu126 broke.
8+
9+
Filtering what the environment asked for rather than pinning a list here, so a channel
10+
that adds a newer architecture still builds for it without editing this file.
11+
"""
12+
13+
from __future__ import annotations
14+
15+
import sys
16+
17+
# The first architecture that declares __dp4a. GP100 is 6.0 and does not have it; the
18+
# rest of Pascal is 6.1 and does.
19+
MINIMUM = 6.1
20+
21+
# PyTorch accepts named architectures and expands each to a version list before building
22+
# its gencode flags (torch/utils/cpp_extension.py). Expand them the same way here, or a
23+
# name would pass through untouched and carry its sub-6.1 members into the build: Pascal
24+
# alone expands to 6.0;6.1+PTX, and 6.0 is precisely what this filter exists to drop.
25+
#
26+
# Ordered longest-key-first, and applied by substring replacement, because that is what
27+
# torch does: "Maxwell+Tegra" has to be replaced before "Maxwell", or the suffix is left
28+
# behind as "5.0;5.2+PTX+Tegra".
29+
NAMED_ARCHES = (
30+
("Kepler+Tesla", "3.7"),
31+
("Kepler", "3.5+PTX"),
32+
("Maxwell+Tegra", "5.3"),
33+
("Maxwell", "5.0;5.2+PTX"),
34+
("Pascal", "6.0;6.1+PTX"),
35+
("Volta+Tegra", "7.2"),
36+
("Volta", "7.0+PTX"),
37+
("Turing", "7.5+PTX"),
38+
("Ampere+Tegra", "8.7"),
39+
("Ampere", "8.0;8.6+PTX"),
40+
("Ada", "8.9+PTX"),
41+
("Hopper", "9.0+PTX"),
42+
("Blackwell+Tegra", "11.0"),
43+
("Blackwell", "10.0;10.3;12.0;12.1+PTX"),
44+
("Rubin", "10.7+PTX"),
45+
)
46+
47+
48+
def filter_arches(requested: str) -> list[str]:
49+
for name, versions in NAMED_ARCHES:
50+
requested = requested.replace(name, versions)
51+
kept = []
52+
# A list may be comma, space or semicolon separated. torch normalises spaces the same way
53+
# (torch/utils/cpp_extension.py does _arch_list.replace(' ', ';')), so accept all three rather
54+
# than silently passing a space-separated list through unfiltered.
55+
for entry in requested.replace(",", ";").replace(" ", ";").split(";"):
56+
entry = entry.strip()
57+
if not entry:
58+
continue
59+
# An entry may carry a suffix such as "8.6+PTX", which sorts by its number.
60+
number = entry.split("+", 1)[0]
61+
try:
62+
if float(number) >= MINIMUM:
63+
kept.append(entry)
64+
except ValueError:
65+
# Not a version this understands. Keep it and let the compiler judge,
66+
# rather than silently dropping a target that may be real.
67+
kept.append(entry)
68+
return kept
69+
70+
71+
def main(argv: list[str]) -> int:
72+
requested = argv[1] if len(argv) > 1 else ""
73+
kept = filter_arches(requested)
74+
if requested.strip() and not kept:
75+
# Printing nothing would leave nvcc to pick its own default, which is not what
76+
# the channel asked for and would ship a wheel for the wrong architectures.
77+
print(
78+
f"No CUDA architecture in {requested!r} is at or above sm_61, "
79+
"which ExecuTorch's CUDA shims require.",
80+
file=sys.stderr,
81+
)
82+
return 1
83+
print(";".join(kept))
84+
return 0
85+
86+
87+
if __name__ == "__main__":
88+
raise SystemExit(main(sys.argv))

‎.github/workflows/executorch-build-linux.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,13 @@ jobs:
9696
# setup.py already forwards BAZEL_ARGS, so raise the limit rather than
9797
# debugging blind.
9898
export BAZEL_ARGS="--experimental_ui_max_stdouterr_bytes=33554432"
99+
# ExecuTorch's CUDA shims call __dp4a, which nvcc does not declare before
100+
# sm_61, and the cu126 build environment asks for 5.0;6.0;... while the CUDA
101+
# 13 ones start higher. Building the pinned ExecuTorch for those two
102+
# architectures cannot work, so narrow the list to what it can compile.
103+
TORCH_CUDA_ARCH_LIST="$(python .github/scripts/filter-executorch-cuda-arches.py "${TORCH_CUDA_ARCH_LIST:-}")"
104+
export TORCH_CUDA_ARCH_LIST
105+
echo "TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST}"
99106
100107
# Build the no-compile-for-users Python runtime wheel.
101108
python -m pip wheel --no-build-isolation --no-deps --wheel-dir dist py/torch-tensorrt-executorch-runtime

‎.github/workflows/executorch-test-linux.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,14 @@ jobs:
8686
8787
source tests/py/utils/ci_helpers.sh
8888
trt_tier_executorch
89+
# This job builds the reference runner, whose CMakeLists forces
90+
# EXECUTORCH_BUILD_CUDA=ON, so it compiles the same aoti_cuda_shims the build job
91+
# does and needs the same architecture floor. It sources the channel's build env,
92+
# which on cu126 asks for architectures without __dp4a, so filter here too rather
93+
# than moving that failure from the build job into this one.
94+
TORCH_CUDA_ARCH_LIST="$(python .github/scripts/filter-executorch-cuda-arches.py "${TORCH_CUDA_ARCH_LIST:-}")"
95+
export TORCH_CUDA_ARCH_LIST
96+
echo "TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST}"
8997
9098
bazel build //:libtorchtrt --compilation_mode opt --config=linux
9199
executorch_cmake_location="$(bazel query @executorch//:executorch/CMakeLists.txt --output=location)"

‎py/torch-tensorrt-executorch-runtime/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,12 @@ Consequently, the Python API does not use the backend's device-resident
7171
input/output fast path. Applications that need to keep inputs and outputs on
7272
GPU should use the ExecuTorch C++ runner.
7373

74+
The device copies around the delegate need the program's device-tagged
75+
memory-planned arenas backed by real device memory, which ExecuTorch does only
76+
through its Module API. `Program` therefore loads through
77+
`_load_for_executorch_from_buffer` rather than through `executorch.runtime`,
78+
whose program loader plans every arena on the host.
79+
7480
## Use
7581

7682
```bash

‎py/torch-tensorrt-executorch-runtime/setup.py‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
REPO_ROOT = HERE.parents[1]
2222
BAZEL_TARGET = "//py/torch-tensorrt-executorch-runtime/native:delegate_native"
2323
BUILD_NONCE = os.getenv("TORCH_TENSORRT_EXECUTORCH_BUILD_NONCE", uuid.uuid4().hex)
24-
CUDA_RUNTIME_DISTRIBUTION = "nvidia-cuda-runtime"
2524

2625

2726
def torchtrt_version() -> str:
@@ -50,17 +49,42 @@ def installed_version(distribution: str) -> str:
5049

5150

5251
def tensorrt_distribution() -> str:
53-
"""Return the TensorRT distribution matching the PyTorch CUDA build."""
52+
"""Return the TensorRT distribution matching the PyTorch CUDA build.
53+
54+
Matched on the major. test-infra's matrix carries cu128 alongside cu126, and the filter
55+
drops it rather than the build rejecting it, but a 12.x that is not exactly 12.6 used to
56+
reach the "Unsupported CUDA version" path here even though tensorrt-cu12 is what it needs.
57+
"""
5458
cuda_version = torch.version.cuda
5559
if cuda_version is None:
5660
raise RuntimeError("CUDA-enabled PyTorch is required to build this wheel")
57-
if cuda_version.startswith("12.6"):
61+
if cuda_version.startswith("12."):
5862
return "tensorrt-cu12"
5963
if cuda_version.startswith("13."):
6064
return "tensorrt-cu13"
6165
raise RuntimeError(f"Unsupported CUDA version: {cuda_version}")
6266

6367

68+
def cuda_runtime_distribution() -> str:
69+
"""Return the CUDA runtime distribution matching the PyTorch CUDA build.
70+
71+
NVIDIA splits this one by major: the CUDA 12 wheels are published as
72+
``nvidia-cuda-runtime-cu12``, while the unsuffixed ``nvidia-cuda-runtime``
73+
is the CUDA 13 line. Naming the unsuffixed one unconditionally made every
74+
CUDA 12 row fail with "No package metadata was found for
75+
nvidia-cuda-runtime", because what torch installed there was the suffixed
76+
distribution.
77+
"""
78+
cuda_version = torch.version.cuda
79+
if cuda_version is None:
80+
raise RuntimeError("CUDA-enabled PyTorch is required to build this wheel")
81+
if cuda_version.startswith("12."):
82+
return "nvidia-cuda-runtime-cu12"
83+
if cuda_version.startswith("13."):
84+
return "nvidia-cuda-runtime"
85+
raise RuntimeError(f"Unsupported CUDA version: {cuda_version}")
86+
87+
6488
class BazelExtension(Extension):
6589
def __init__(self, name: str) -> None:
6690
super().__init__(name, sources=[])
@@ -141,6 +165,7 @@ def build_extension(self, ext: Extension) -> None:
141165

142166

143167
TENSORRT_DISTRIBUTION = tensorrt_distribution()
168+
CUDA_RUNTIME_DISTRIBUTION = cuda_runtime_distribution()
144169
executorch_version = installed_version("executorch")
145170
tensorrt_version = installed_version(TENSORRT_DISTRIBUTION)
146171
cuda_runtime_version = installed_version(CUDA_RUNTIME_DISTRIBUTION)

‎py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import ctypes
66
import importlib
7+
import importlib.util
78
import os
89
import sys
910
from types import ModuleType

‎py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,35 @@
33
from __future__ import annotations
44

55
from pathlib import Path
6-
from typing import TYPE_CHECKING, Any, Collection, Sequence, Union, cast
6+
from typing import Any, Collection, Sequence, Union, cast
77

8-
if TYPE_CHECKING:
9-
from torch_tensorrt_executorch_runtime import _Runtime
108

9+
def _load_module(data: bytes) -> Any:
10+
"""Load a program through ExecuTorch's Module API.
1111
12-
def _get_runtime() -> _Runtime:
12+
A TensorRT delegated program is exported with device-tagged memory-planned
13+
arenas, and ExecuTorch backs those with real device memory only through
14+
this API. Its program loader plans every arena on the host, so the device
15+
copy the exporter inserts around the delegate would hand ``cudaMemcpy`` a
16+
host destination and fail with ``invalid argument``.
17+
"""
1318
try:
14-
from torch_tensorrt_executorch_runtime import get_runtime
19+
from torch_tensorrt_executorch_runtime import activate, get_runtime
1520
except ImportError as error:
1621
raise ImportError(
1722
"ExecuTorch Python inference requires the prebuilt delegate. "
1823
'Install it with: pip install "torch-tensorrt[executorch]"'
1924
) from error
20-
return get_runtime()
25+
26+
# get_runtime verifies TensorRTBackend is registered; activate returns the native module it
27+
# installed as the process portable runtime, which is what loads the program.
28+
get_runtime()
29+
native = activate()
30+
# Taken off the native module rather than imported through
31+
# executorch.extension.pybindings.portable_lib. That wrapper's presence in sys.modules is how
32+
# activate() detects that ExecuTorch's stock runtime was imported first, so importing it here
33+
# would make a later activate() in the same process refuse.
34+
return native._load_for_executorch_from_buffer(data)
2135

2236

2337
class Program:
@@ -30,14 +44,14 @@ class Program:
3044
ExecuTorch C++ runner.
3145
"""
3246

33-
def __init__(self, program: Any, data: bytes) -> None:
47+
def __init__(self, module: Any, data: bytes) -> None:
3448
# ExecuTorch's BufferDataLoader references this memory without copying it.
3549
self._data = data
36-
self._program = program
50+
self._module = module
3751

3852
@property
3953
def method_names(self) -> Collection[str]:
40-
return cast(Collection[str], self._program.method_names)
54+
return cast(Collection[str], self._module.method_names())
4155

4256
def run(self, inputs: Sequence[Any], method: str = "forward") -> Sequence[Any]:
4357
"""Run a method using CPU inputs and return CPU outputs.
@@ -56,10 +70,7 @@ def run(self, inputs: Sequence[Any], method: str = "forward") -> Sequence[Any]:
5670
raise ValueError(
5771
f"Unknown method {method!r}; available methods: {sorted(self.method_names)}"
5872
)
59-
loaded = self._program.load_method(method)
60-
if loaded is None:
61-
raise RuntimeError(f"ExecuTorch failed to load method {method!r}")
62-
return cast(Sequence[Any], loaded.execute(inputs))
73+
return cast(Sequence[Any], self._module.run_method(method, inputs))
6374

6475
def forward(self, *inputs: Any) -> Sequence[Any]:
6576
return self.run(inputs, "forward")
@@ -75,7 +86,7 @@ def load(path: Union[str, Path]) -> Program:
7586
if not model_path.is_file():
7687
raise FileNotFoundError(f"ExecuTorch model not found: {model_path}")
7788
data = model_path.read_bytes()
78-
return Program(_get_runtime().load_program(data), data)
89+
return Program(_load_module(data), data)
7990

8091

8192
__all__ = ["Program", "load"]

0 commit comments

Comments
 (0)