@@ -725,6 +725,7 @@ def load(
725725 "weight_streaming_budget_per_engine" : None ,
726726 "use_cuda_graphs" : None ,
727727 "zero_copy_kv" : False ,
728+ "external_engine_data" : None ,
728729}
729730
730731
@@ -835,7 +836,8 @@ def save(
835836 kwargs: Additional format-specific kwargs. ``partitioners=``,
836837 ``compile_specs=``, ``backend_config=``, ``constant_methods=``,
837838 ``transform_passes=``, ``compile_config=``, ``generate_etrecord=``,
838- ``weight_streaming_budget_per_engine=`` and ``zero_copy_kv=`` are
839+ ``weight_streaming_budget_per_engine=``, ``zero_copy_kv=`` and
840+ ``external_engine_data=`` are
839841 only used with ``output_format="executorch"``; otherwise they are
840842 ignored with a warning. Pass ``compile_specs=[CompileSpec("target_device",
841843 b"cuda:<i>")]`` to override the default target device (``cuda:0``).
@@ -879,6 +881,11 @@ def save(
879881 :func:`torch_tensorrt.executorch.check_zero_copy_kv` on the
880882 finalized program before writing the file, so a single
881883 ``zero_copy_kv=True`` is enough.
884+ ``external_engine_data=`` (default ``None``) takes a file name without
885+ the ``.ptd`` extension and
886+ writes every TensorRT engine to ``<name>.ptd`` next to the ``.pte``
887+ instead of into it. Pass that file as the program's data file when
888+ loading. See :func:`torch_tensorrt.executorch.export`.
882889 """
883890 if isinstance (module , CudaGraphsTorchTensorRTModule ):
884891 module = module .compiled_module
@@ -922,6 +929,7 @@ def save(
922929 "weight_streaming_budget_per_engine"
923930 ]
924931 executorch_zero_copy_kv = executorch_options ["zero_copy_kv" ]
932+ executorch_external_engine_data = executorch_options ["external_engine_data" ]
925933
926934 if output_format not in accepted_formats :
927935 raise ValueError (
@@ -945,6 +953,7 @@ def save(
945953 )
946954 # Validate options before input and model-shape checks can obscure type errors.
947955 from torch_tensorrt .executorch .partitioner import (
956+ normalize_external_engine_data ,
948957 normalize_use_cuda_graphs ,
949958 normalize_weight_streaming_budget_per_engine ,
950959 )
@@ -953,6 +962,7 @@ def save(
953962 executorch_weight_streaming_budget_per_engine
954963 )
955964 normalize_use_cuda_graphs (use_cuda_graphs )
965+ normalize_external_engine_data (executorch_external_engine_data )
956966 # For the same reason, the one refusal zero-copy makes on the config
957967 # alone. _save_as_executorch reaches it only through
958968 # zero_copy_backend_config, which it calls after export() has partitioned
@@ -1116,6 +1126,11 @@ def _extract_tensor(obj: Any) -> Any:
11161126 "use_cuda_graphs= is only used with output_format='executorch' and will "
11171127 f"be ignored for output_format='{ output_format } '."
11181128 )
1129+ if executorch_external_engine_data is not None and output_format != "executorch" :
1130+ logger .warning (
1131+ "external_engine_data= is only used with output_format='executorch' and "
1132+ f"will be ignored for output_format='{ output_format } '."
1133+ )
11191134 if executorch_zero_copy_kv and output_format != "executorch" :
11201135 logger .warning (
11211136 "zero_copy_kv= is only used with output_format='executorch' and will "
@@ -1218,6 +1233,7 @@ def _extract_tensor(obj: Any) -> Any:
12181233 weight_streaming_budget_per_engine = executorch_weight_streaming_budget_per_engine ,
12191234 use_cuda_graphs = use_cuda_graphs ,
12201235 zero_copy_kv = executorch_zero_copy_kv ,
1236+ external_engine_data = executorch_external_engine_data ,
12211237 )
12221238 else :
12231239 raise RuntimeError (
@@ -1337,6 +1353,7 @@ def _extract_tensor(obj: Any) -> Any:
13371353 weight_streaming_budget_per_engine = executorch_weight_streaming_budget_per_engine ,
13381354 use_cuda_graphs = use_cuda_graphs ,
13391355 zero_copy_kv = executorch_zero_copy_kv ,
1356+ external_engine_data = executorch_external_engine_data ,
13401357 )
13411358 else :
13421359 raise RuntimeError (
@@ -1465,6 +1482,7 @@ def _extract_tensor(obj: Any) -> Any:
14651482 weight_streaming_budget_per_engine = executorch_weight_streaming_budget_per_engine ,
14661483 use_cuda_graphs = use_cuda_graphs ,
14671484 zero_copy_kv = executorch_zero_copy_kv ,
1485+ external_engine_data = executorch_external_engine_data ,
14681486 )
14691487 else :
14701488 raise RuntimeError (
@@ -1476,20 +1494,21 @@ def _write_external_tensor_data(executorch_program: Any, file_path: str) -> None
14761494 """Write an ExecuTorch program's external named tensor data (``.ptd``) next to the ``.pte``.
14771495
14781496 The CUDA (AOTInductor) backend emits its weights as external named data
1479- (``save_data_externally``), which ExecuTorch serializes only via
1497+ (``save_data_externally``), and so does the TensorRT backend for its engines
1498+ when saved with ``external_engine_data=``. ExecuTorch serializes that data only via
14801499 ``write_tensor_data_to_file`` -- ``ExecutorchProgram.write_to_file`` does not
1481- persist it. So a partition that carries external weights (e.g. a
1482- ``CudaPartitioner`` delegate) would lose its blob and the ``.pte`` could not
1500+ persist it. So a partition that carries external data (e.g. a
1501+ ``CudaPartitioner`` delegate) would lose it and the ``.pte`` could not
14831502 load. This writes the ``.ptd`` data file(s) into the ``.pte``'s directory when
14841503 the program has any; the runtime must then be given those data file(s) (e.g.
1485- via the ExecuTorch ``Module`` data-files argument) to load the weights .
1504+ via the ExecuTorch ``Module`` data-files argument) to load the program .
14861505 """
14871506 # Direct attribute access (no getattr default) so a future rename fails loud.
14881507 if executorch_program ._tensor_data :
14891508 out_dir = os .path .dirname (os .path .abspath (file_path ))
14901509 executorch_program .write_tensor_data_to_file (out_dir )
14911510 logger .info (
1492- "Wrote external delegate weights (.ptd) to %s; point the runtime's "
1511+ "Wrote external delegate data (.ptd) to %s; point the runtime's "
14931512 "data-files at this directory to load them." ,
14941513 out_dir ,
14951514 )
@@ -1538,6 +1557,7 @@ def _save_as_executorch(exp_program: Any, file_path: str, **kwargs: Any) -> None
15381557 ),
15391558 use_cuda_graphs = kwargs .get ("use_cuda_graphs" ),
15401559 zero_copy_kv = zero_copy_kv ,
1560+ external_engine_data = kwargs .get ("external_engine_data" ),
15411561 )
15421562 # Unlike the direct export()+to_executorch() path -- where the two steps
15431563 # belong to different owners and pairing them is the caller's job -- save()
0 commit comments