Skip to content

Commit e4ee0d9

Browse files
shoumikhinlanluo-nvidia
authored andcommitted
feat(executorch): optional .ptd file for TensorRT engines, refuse engines without weights (#4848)
1 parent 001a01d commit e4ee0d9

13 files changed

Lines changed: 816 additions & 35 deletions

File tree

‎cpp/src/torch_tensorrt/executorch/README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,27 @@ device memory in bytes, or the memory query error if those figures are unavailab
329329
These figures are not the failed allocation size. Loading does not clear a caller's
330330
pending CUDA error to classify TensorRT failures.
331331
332+
An engine serialized without its weights, for example compiled with
333+
`immutable_weights=False, strip_engine_weights=True`, is refused with
334+
`Error::InvalidProgram`, since this backend cannot refit it. A stripped engine
335+
with no weights at all loads, since it has nothing to restore. To tell the two
336+
apart, loading an ordinary engine that has weights asks TensorRT for its stripped
337+
size, and TensorRT logs one warning per engine that it was not built with
338+
`kSTRIP_PLAN`. The backend forwards it at Info level, and it is harmless.
339+
340+
## Engines in a .ptd file
341+
342+
A program exported with `external_engine_data=` keeps its engines in a `.ptd` file
343+
next to the `.pte`. Load it with that file, for example
344+
`Module module("model.pte", "engines.ptd");`. Without the file, or with one that
345+
lacks the engine, loading returns `Error::InvalidExternalData`. A program that shares
346+
`aoti_cuda_blob.ptd` with the CUDA backend can fail first in that backend, with
347+
`Error::NotFound`.
348+
349+
The engine is GPU code, so treat the `.ptd` as trusted input, just like the `.pte`.
350+
Each engine is stored under a key made from its sha256, but the key is only a name:
351+
nothing checks the engine against it at load.
352+
332353
## Shared engines
333354
334355
Loading the same program twice in one process, for example once per robot arm, used to

‎cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,19 @@ Error load_engine(
378378
if (engine == nullptr) {
379379
return report_load_error("deserialize TensorRT engine");
380380
}
381+
// A plan serialized without its weights (built with STRIP_PLAN, or saved with EXCLUDE_WEIGHTS)
382+
// deserializes and runs without an error, but computes with placeholder weights, and this backend
383+
// never refits. TensorRT reports -1 stripped bytes for a plan that carries its weights, and a
384+
// stripped plan can report 0. A plan with no weights has nothing to restore, so the total is read
385+
// first: querying the stripped size of an ordinary engine logs a TensorRT warning, and
386+
// isRefittable() cannot skip it, since a plan built with STRIP_PLAN alone is not refittable.
387+
if (engine->getEngineStat(nvinfer1::EngineStat::kTOTAL_WEIGHTS_SIZE) > 0 &&
388+
engine->getEngineStat(nvinfer1::EngineStat::kSTRIPPED_WEIGHTS_SIZE) >= 0) {
389+
ET_LOG(
390+
Error,
391+
"TensorRTBackend::init: the TensorRT engine was serialized without its weights, and this backend cannot refit them. Compile it with strip_engine_weights=False, or serialize it with its weights, and export again.");
392+
return Error::InvalidProgram;
393+
}
381394
const Error err = apply_weight_streaming_budget(*engine, request);
382395
if (err != Error::Ok) {
383396
return report_load_error("set TensorRT weight streaming budget");
@@ -1231,13 +1244,28 @@ Result<DelegateHandle*> TensorRTBackend::init(
12311244
std::size_t engine_size = header.engine_size;
12321245
std::optional<FreeableBuffer> named_engine;
12331246
if (!header.engine_key.empty()) {
1247+
// A program exported with external_engine_data keeps its engines in a .ptd file, so a
1248+
// missing map or key most likely means that file was not passed at load.
12341249
const auto* named_data = context.get_named_data_map();
12351250
TORCHTRT_ET_CHECK_NOT_NULL(
1236-
named_data, Error::InvalidProgram, "TensorRTBackend::init: engine is named data but the program has none");
1251+
named_data,
1252+
Error::InvalidExternalData,
1253+
"TensorRTBackend::init: the TensorRT engine is named data, but the program has none. If it was exported with external_engine_data, load it together with its .ptd file.");
12371254
auto engine_buffer = named_data->get_data(header.engine_key);
12381255
if (!engine_buffer.ok()) {
1239-
ET_LOG(Error, "TensorRTBackend::init: named engine '%s' not found", header.engine_key.c_str());
1240-
return engine_buffer.error();
1256+
if (engine_buffer.error() != Error::NotFound) {
1257+
ET_LOG(
1258+
Error,
1259+
"TensorRTBackend::init: failed to read TensorRT engine '%s' from named data: %s",
1260+
header.engine_key.c_str(),
1261+
::executorch::runtime::to_string(engine_buffer.error()));
1262+
return engine_buffer.error();
1263+
}
1264+
ET_LOG(
1265+
Error,
1266+
"TensorRTBackend::init: the program's data has no TensorRT engine with key '%s'. If it was exported with external_engine_data, load it together with the .ptd file written with it.",
1267+
header.engine_key.c_str());
1268+
return Error::InvalidExternalData;
12411269
}
12421270
named_engine.emplace(std::move(engine_buffer.get()));
12431271
engine_data = named_engine->data();

‎py/torch_tensorrt/_compile.py‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -725,6 +725,7 @@ def load(
725725
"weight_streaming_budget_per_engine": None,
726726
"use_cuda_graphs": None,
727727
"zero_copy_kv": False,
728+
"external_engine_data": None,
728729
}
729730

730731

@@ -835,7 +836,8 @@ def save(
835836
kwargs: Additional format-specific kwargs. ``partitioners=``,
836837
``compile_specs=``, ``backend_config=``, ``constant_methods=``,
837838
``transform_passes=``, ``compile_config=``, ``generate_etrecord=``,
838-
``weight_streaming_budget_per_engine=`` and ``zero_copy_kv=`` are
839+
``weight_streaming_budget_per_engine=``, ``zero_copy_kv=`` and
840+
``external_engine_data=`` are
839841
only used with ``output_format="executorch"``; otherwise they are
840842
ignored with a warning. Pass ``compile_specs=[CompileSpec("target_device",
841843
b"cuda:<i>")]`` to override the default target device (``cuda:0``).
@@ -879,6 +881,11 @@ def save(
879881
:func:`torch_tensorrt.executorch.check_zero_copy_kv` on the
880882
finalized program before writing the file, so a single
881883
``zero_copy_kv=True`` is enough.
884+
``external_engine_data=`` (default ``None``) takes a file name without
885+
the ``.ptd`` extension and
886+
writes every TensorRT engine to ``<name>.ptd`` next to the ``.pte``
887+
instead of into it. Pass that file as the program's data file when
888+
loading. See :func:`torch_tensorrt.executorch.export`.
882889
"""
883890
if isinstance(module, CudaGraphsTorchTensorRTModule):
884891
module = module.compiled_module
@@ -922,6 +929,7 @@ def save(
922929
"weight_streaming_budget_per_engine"
923930
]
924931
executorch_zero_copy_kv = executorch_options["zero_copy_kv"]
932+
executorch_external_engine_data = executorch_options["external_engine_data"]
925933

926934
if output_format not in accepted_formats:
927935
raise ValueError(
@@ -945,6 +953,7 @@ def save(
945953
)
946954
# Validate options before input and model-shape checks can obscure type errors.
947955
from torch_tensorrt.executorch.partitioner import (
956+
normalize_external_engine_data,
948957
normalize_use_cuda_graphs,
949958
normalize_weight_streaming_budget_per_engine,
950959
)
@@ -953,6 +962,7 @@ def save(
953962
executorch_weight_streaming_budget_per_engine
954963
)
955964
normalize_use_cuda_graphs(use_cuda_graphs)
965+
normalize_external_engine_data(executorch_external_engine_data)
956966
# For the same reason, the one refusal zero-copy makes on the config
957967
# alone. _save_as_executorch reaches it only through
958968
# zero_copy_backend_config, which it calls after export() has partitioned
@@ -1116,6 +1126,11 @@ def _extract_tensor(obj: Any) -> Any:
11161126
"use_cuda_graphs= is only used with output_format='executorch' and will "
11171127
f"be ignored for output_format='{output_format}'."
11181128
)
1129+
if executorch_external_engine_data is not None and output_format != "executorch":
1130+
logger.warning(
1131+
"external_engine_data= is only used with output_format='executorch' and "
1132+
f"will be ignored for output_format='{output_format}'."
1133+
)
11191134
if executorch_zero_copy_kv and output_format != "executorch":
11201135
logger.warning(
11211136
"zero_copy_kv= is only used with output_format='executorch' and will "
@@ -1218,6 +1233,7 @@ def _extract_tensor(obj: Any) -> Any:
12181233
weight_streaming_budget_per_engine=executorch_weight_streaming_budget_per_engine,
12191234
use_cuda_graphs=use_cuda_graphs,
12201235
zero_copy_kv=executorch_zero_copy_kv,
1236+
external_engine_data=executorch_external_engine_data,
12211237
)
12221238
else:
12231239
raise RuntimeError(
@@ -1337,6 +1353,7 @@ def _extract_tensor(obj: Any) -> Any:
13371353
weight_streaming_budget_per_engine=executorch_weight_streaming_budget_per_engine,
13381354
use_cuda_graphs=use_cuda_graphs,
13391355
zero_copy_kv=executorch_zero_copy_kv,
1356+
external_engine_data=executorch_external_engine_data,
13401357
)
13411358
else:
13421359
raise RuntimeError(
@@ -1465,6 +1482,7 @@ def _extract_tensor(obj: Any) -> Any:
14651482
weight_streaming_budget_per_engine=executorch_weight_streaming_budget_per_engine,
14661483
use_cuda_graphs=use_cuda_graphs,
14671484
zero_copy_kv=executorch_zero_copy_kv,
1485+
external_engine_data=executorch_external_engine_data,
14681486
)
14691487
else:
14701488
raise RuntimeError(
@@ -1476,20 +1494,21 @@ def _write_external_tensor_data(executorch_program: Any, file_path: str) -> None
14761494
"""Write an ExecuTorch program's external named tensor data (``.ptd``) next to the ``.pte``.
14771495
14781496
The CUDA (AOTInductor) backend emits its weights as external named data
1479-
(``save_data_externally``), which ExecuTorch serializes only via
1497+
(``save_data_externally``), and so does the TensorRT backend for its engines
1498+
when saved with ``external_engine_data=``. ExecuTorch serializes that data only via
14801499
``write_tensor_data_to_file`` -- ``ExecutorchProgram.write_to_file`` does not
1481-
persist it. So a partition that carries external weights (e.g. a
1482-
``CudaPartitioner`` delegate) would lose its blob and the ``.pte`` could not
1500+
persist it. So a partition that carries external data (e.g. a
1501+
``CudaPartitioner`` delegate) would lose it and the ``.pte`` could not
14831502
load. This writes the ``.ptd`` data file(s) into the ``.pte``'s directory when
14841503
the program has any; the runtime must then be given those data file(s) (e.g.
1485-
via the ExecuTorch ``Module`` data-files argument) to load the weights.
1504+
via the ExecuTorch ``Module`` data-files argument) to load the program.
14861505
"""
14871506
# Direct attribute access (no getattr default) so a future rename fails loud.
14881507
if executorch_program._tensor_data:
14891508
out_dir = os.path.dirname(os.path.abspath(file_path))
14901509
executorch_program.write_tensor_data_to_file(out_dir)
14911510
logger.info(
1492-
"Wrote external delegate weights (.ptd) to %s; point the runtime's "
1511+
"Wrote external delegate data (.ptd) to %s; point the runtime's "
14931512
"data-files at this directory to load them.",
14941513
out_dir,
14951514
)
@@ -1538,6 +1557,7 @@ def _save_as_executorch(exp_program: Any, file_path: str, **kwargs: Any) -> None
15381557
),
15391558
use_cuda_graphs=kwargs.get("use_cuda_graphs"),
15401559
zero_copy_kv=zero_copy_kv,
1560+
external_engine_data=kwargs.get("external_engine_data"),
15411561
)
15421562
# Unlike the direct export()+to_executorch() path -- where the two steps
15431563
# belong to different owners and pairing them is the caller's job -- save()

‎py/torch_tensorrt/executorch/_export.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,32 @@ def _apply_use_cuda_graphs(
326326
specs.append(CompileSpec(CUDA_GRAPHS_COMPILE_SPEC_KEY, spec_value))
327327

328328

329+
def _apply_external_engine_data(
330+
method_compile_specs: dict[str, list[Any]],
331+
external_engine_data: str | None,
332+
) -> None:
333+
"""Point every method's TensorRT engines at the named .ptd file, rejecting raw specs."""
334+
from executorch.exir.backend.compile_spec_schema import CompileSpec
335+
from torch_tensorrt.executorch.backend import EXTERNAL_ENGINE_DATA_COMPILE_SPEC_KEY
336+
from torch_tensorrt.executorch.partitioner import normalize_external_engine_data
337+
338+
for name, specs in method_compile_specs.items():
339+
if any(
340+
getattr(spec, "key", None) == EXTERNAL_ENGINE_DATA_COMPILE_SPEC_KEY
341+
for spec in specs
342+
):
343+
raise ValueError(
344+
f"compile_specs for {name!r} carries a "
345+
f"CompileSpec({EXTERNAL_ENGINE_DATA_COMPILE_SPEC_KEY!r}, ...). Pass "
346+
"external_engine_data= instead."
347+
)
348+
spec_value = normalize_external_engine_data(external_engine_data)
349+
if spec_value is None:
350+
return
351+
for specs in method_compile_specs.values():
352+
specs.append(CompileSpec(EXTERNAL_ENGINE_DATA_COMPILE_SPEC_KEY, spec_value))
353+
354+
329355
def _apply_zero_copy_kv(
330356
program_map: dict[str, ExportedProgram],
331357
) -> dict[str, list[str]]:
@@ -456,6 +482,7 @@ def export(
456482
generate_etrecord: bool = False,
457483
weight_streaming_budget_per_engine: int | None = None,
458484
use_cuda_graphs: bool | None = None,
485+
external_engine_data: str | None = None,
459486
) -> "EdgeProgramManager":
460487
"""Prepare TensorRT-compiled programs for composable ExecuTorch lowering.
461488
@@ -584,6 +611,18 @@ def export(
584611
unset so the runtime default applies. A load-time option overrides this
585612
value. Non-booleans and raw ``CompileSpec("use_cuda_graphs", ...)`` entries
586613
are rejected; use this keyword instead.
614+
external_engine_data (Optional[str]): Write every TensorRT engine to an
615+
ExecuTorch data file named ``<external_engine_data>.ptd`` instead of into
616+
the ``.pte``, which then holds only a key per engine. Pass the name without
617+
the ``.ptd`` extension. The engine is the same bytes either way. Load the
618+
program with that file as its data file,
619+
for example ``Runtime.get().load_program(pte, data_path=ptd)``; without it
620+
the method fails to load. That loader takes one data file. If another
621+
backend in the program writes its own ``.ptd``, pass that backend's name
622+
here so both share one file. For the CUDA backend that name is
623+
``aoti_cuda_blob``. After ``to_executorch()``, write the ``.ptd`` with
624+
``write_tensor_data_to_file(directory)``; ``torch_tensorrt.save`` does this
625+
next to the ``.pte``. None, the default, keeps every engine in the ``.pte``.
587626
588627
Returns:
589628
executorch.exir.EdgeProgramManager: The Edge program, ready for inspection,
@@ -665,6 +704,7 @@ def export(
665704
method_compile_specs, weight_streaming_budget_per_engine
666705
)
667706
_apply_use_cuda_graphs(method_compile_specs, use_cuda_graphs)
707+
_apply_external_engine_data(method_compile_specs, external_engine_data)
668708

669709
if constant_methods is not None:
670710
invalid = [

‎py/torch_tensorrt/executorch/backend.py‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,11 @@ def _warn_engine_copy() -> None:
7777
# error.
7878
ZERO_COPY_KV_COMPILE_SPEC_KEY = "zero_copy_kv"
7979

80+
# CompileSpec key naming the ExecuTorch data file (.ptd) that receives each engine of the
81+
# method, so the .pte keeps only the engine's key. The value is the file name without the
82+
# .ptd extension. Absent, every engine is stored in the .pte.
83+
EXTERNAL_ENGINE_DATA_COMPILE_SPEC_KEY = "external_engine_data"
84+
8085

8186
def _schema_name(target: Any) -> str:
8287
"""Return the qualified op schema name for an OpOverload or EdgeOpOverload."""
@@ -508,6 +513,40 @@ def _elided_output_names(compile_specs: List[CompileSpec]) -> Optional[Set[str]]
508513
return None
509514

510515

516+
def _external_engine_tag(compile_specs: List[CompileSpec]) -> Optional[str]:
517+
"""The .ptd file name, without the extension, that engines go to, or ``None``."""
518+
from torch_tensorrt.executorch.partitioner import normalize_external_engine_data
519+
520+
tags = [
521+
spec.value
522+
for spec in compile_specs
523+
if getattr(spec, "key", None) == EXTERNAL_ENGINE_DATA_COMPILE_SPEC_KEY
524+
]
525+
if not tags:
526+
return None
527+
if len(tags) > 1:
528+
raise ValueError(
529+
"TensorRT ExecuTorch backend: compile spec "
530+
f"'{EXTERNAL_ENGINE_DATA_COMPILE_SPEC_KEY}' must appear at most once."
531+
)
532+
value = tags[0]
533+
if not isinstance(value, (bytes, bytearray)):
534+
raise ValueError(
535+
"TensorRT ExecuTorch backend: compile spec "
536+
f"'{EXTERNAL_ENGINE_DATA_COMPILE_SPEC_KEY}' must hold the file name as UTF-8 "
537+
f"bytes, not {type(value).__name__}."
538+
)
539+
try:
540+
tag = bytes(value).decode("utf-8")
541+
except UnicodeDecodeError as e:
542+
raise ValueError(
543+
"TensorRT ExecuTorch backend: compile spec "
544+
f"'{EXTERNAL_ENGINE_DATA_COMPILE_SPEC_KEY}' is not valid UTF-8: {value!r}."
545+
) from e
546+
normalize_external_engine_data(tag)
547+
return tag
548+
549+
511550
def _get_str(engine_info: List[Any], index: int, default: str = "") -> str:
512551
if index < 0 or index >= len(engine_info):
513552
return default
@@ -524,7 +563,9 @@ class TensorRTBackend(BackendDetails): # type: ignore[misc]
524563
"""Backend that serializes TensorRT engines for the native ExecuTorch runtime.
525564
526565
The partition contains a single execute_engine node; we extract the engine
527-
and metadata and encode them as a standalone blob. The C++ runtime
566+
and metadata, store the engine as named data (in the .pte, or with
567+
external_engine_data in a .ptd file), and encode its key and the metadata as a
568+
standalone blob. The C++ runtime
528569
backend parses that blob directly without the legacy Torch-TensorRT C++ runtime.
529570
"""
530571

@@ -625,10 +666,16 @@ def preprocess(
625666
# entry and different engines can never collide on a key.
626667
engine_key="tensorrt_engine_" + hashlib.sha256(engine_bytes).hexdigest(),
627668
)
628-
# The engine goes to named data by reference and is streamed into the .pte,
629-
# so the delegate blob holds only the header and metadata.
669+
# The engine goes to named data by reference and is streamed into the .pte, or
670+
# into <external_engine_data>.ptd when set, so the delegate blob holds only the
671+
# header and metadata.
630672
named_data = NamedDataStore()
631-
named_data.add_named_data(metadata.engine_key, engine_bytes, alignment=16)
673+
named_data.add_named_data(
674+
metadata.engine_key,
675+
engine_bytes,
676+
alignment=16,
677+
external_tag=_external_engine_tag(compile_specs),
678+
)
632679
return PreprocessResult(
633680
processed_bytes=serialize_engine(b"", metadata),
634681
data_store_output=named_data.get_named_data_store_output(),

0 commit comments

Comments
 (0)