33from __future__ import annotations
44
55from pathlib import Path
6- from typing import TYPE_CHECKING , Any , Collection , Sequence , Union , cast
6+ from typing import Any , Collection , Sequence , Union , cast
77
8- if TYPE_CHECKING :
9- from torch_tensorrt_executorch_runtime import _Runtime
108
9+ def _load_module (data : bytes ) -> Any :
10+ """Load a program through ExecuTorch's Module API.
1111
12- def _get_runtime () -> _Runtime :
12+ A TensorRT delegated program is exported with device-tagged memory-planned
13+ arenas, and ExecuTorch backs those with real device memory only through
14+ this API. Its program loader plans every arena on the host, so the device
15+ copy the exporter inserts around the delegate would hand ``cudaMemcpy`` a
16+ host destination and fail with ``invalid argument``.
17+ """
1318 try :
14- from torch_tensorrt_executorch_runtime import get_runtime
19+ from torch_tensorrt_executorch_runtime import activate , get_runtime
1520 except ImportError as error :
1621 raise ImportError (
1722 "ExecuTorch Python inference requires the prebuilt delegate. "
1823 'Install it with: pip install "torch-tensorrt[executorch]"'
1924 ) from error
20- return get_runtime ()
25+
26+ # get_runtime verifies TensorRTBackend is registered; activate returns the native module it
27+ # installed as the process portable runtime, which is what loads the program.
28+ get_runtime ()
29+ native = activate ()
30+ # Taken off the native module rather than imported through
31+ # executorch.extension.pybindings.portable_lib. That wrapper's presence in sys.modules is how
32+ # activate() detects that ExecuTorch's stock runtime was imported first, so importing it here
33+ # would make a later activate() in the same process refuse.
34+ return native ._load_for_executorch_from_buffer (data )
2135
2236
2337class Program :
@@ -30,14 +44,14 @@ class Program:
3044 ExecuTorch C++ runner.
3145 """
3246
33- def __init__ (self , program : Any , data : bytes ) -> None :
47+ def __init__ (self , module : Any , data : bytes ) -> None :
3448 # ExecuTorch's BufferDataLoader references this memory without copying it.
3549 self ._data = data
36- self ._program = program
50+ self ._module = module
3751
3852 @property
3953 def method_names (self ) -> Collection [str ]:
40- return cast (Collection [str ], self ._program .method_names )
54+ return cast (Collection [str ], self ._module .method_names () )
4155
4256 def run (self , inputs : Sequence [Any ], method : str = "forward" ) -> Sequence [Any ]:
4357 """Run a method using CPU inputs and return CPU outputs.
@@ -56,10 +70,7 @@ def run(self, inputs: Sequence[Any], method: str = "forward") -> Sequence[Any]:
5670 raise ValueError (
5771 f"Unknown method { method !r} ; available methods: { sorted (self .method_names )} "
5872 )
59- loaded = self ._program .load_method (method )
60- if loaded is None :
61- raise RuntimeError (f"ExecuTorch failed to load method { method !r} " )
62- return cast (Sequence [Any ], loaded .execute (inputs ))
73+ return cast (Sequence [Any ], self ._module .run_method (method , inputs ))
6374
6475 def forward (self , * inputs : Any ) -> Sequence [Any ]:
6576 return self .run (inputs , "forward" )
@@ -75,7 +86,7 @@ def load(path: Union[str, Path]) -> Program:
7586 if not model_path .is_file ():
7687 raise FileNotFoundError (f"ExecuTorch model not found: { model_path } " )
7788 data = model_path .read_bytes ()
78- return Program (_get_runtime (). load_program (data ), data )
89+ return Program (_load_module (data ), data )
7990
8091
8192__all__ = ["Program" , "load" ]
0 commit comments