Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions docsrc/user_guide/runtime_performance/saving_models.rst
Original file line number Diff line number Diff line change
Expand Up @@ -280,8 +280,11 @@ warning and are ignored. With this format, an unknown keyword raises a ``TypeErr
* ``zero_copy_kv``: let an engine update its KV cache in place. See
:ref:`Zero-copy KV cache <executorch_zero_copy_kv>`.
* ``generate_etrecord``: also write an ETRecord, a debug record for the ExecuTorch Developer Tools
``Inspector``, as ``<name>_etrecord.bin``. It does not work for a program with TensorRT
engines: writing the record fails with a ``SerializeError``.
``Inspector``, as ``<name>_etrecord.bin``. With this save it does not work for a program with
TensorRT engines: writing the record fails with a ``SerializeError``.
:func:`torch_tensorrt.compile_and_save` can write one, because it keeps each engine as a byte
buffer read from its file. The record then holds copies of every engine, and writing it reads
them into memory.

:ref:`executorch_deployment` explains the run time side of ``use_cuda_graphs``,
``weight_streaming_budget_per_engine`` and ``external_engine_data``.
Expand Down
118 changes: 103 additions & 15 deletions py/torch_tensorrt/_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@
from __future__ import annotations

import collections.abc
import contextlib
import gc
import importlib
import inspect
import logging
import os
import platform
import shutil
import warnings
from enum import Enum
from typing import (
Expand Down Expand Up @@ -77,6 +79,8 @@
from torch_tensorrt.dynamo._tracer import trace as dynamo_trace
from torch_tensorrt.dynamo.runtime import TorchTensorRTModule
from torch_tensorrt.dynamo.runtime._offloaded_engines import (
defer_engine_setup_to_load,
moving_engines_into,
offload_engines_to_disk,
)
from torch_tensorrt.dynamo.utils import get_torch_inputs
Expand Down Expand Up @@ -871,8 +875,12 @@ def save(
TRT engines.
``generate_etrecord=True`` writes an ETRecord to
``<model>_etrecord.bin`` next to the ``.pte`` for use with the
ExecuTorch Inspector. It does not work for a program with
TensorRT engines: writing the record raises a ``SerializeError``.
ExecuTorch Inspector. With this save it does not work for a program
with TensorRT engines: writing the record raises a ``SerializeError``.
:func:`torch_tensorrt.compile_and_save` can write one, because it
keeps each engine as a byte buffer read from its file. The record
then holds copies of every engine, and writing it reads them into
memory.
``weight_streaming_budget_per_engine=`` takes an ``Optional[int]``
number of bytes of engine weights that may stay resident in GPU
memory, with the rest streamed from host memory. It applies to
Expand Down Expand Up @@ -1550,6 +1558,17 @@ def compile_and_save(
but each engine is written to a file as soon as it is built, so no built engine is held on the GPU
or in host memory while the next ones build. With ``output_format="executorch"`` no engine is ever
loaded: the save streams each file into the ``.pte`` (or the ``.ptd`` with ``external_engine_data``).
When the outputs would not fit in the free space of ``file_path``'s file system, with a tenth to
spare (an output already there counts as free), the save moves each engine file into them instead of
copying it, so the engines are on disk once (plus one chunk) instead of twice, on a file system with
sparse files. ExecuTorch moves other data that owns its file the same way, such as the CUDA
backend's weights. The move syncs each chunk to disk before the engine file gives it up, so it takes
2.5 to 5.5 times as long as a copy (measured on btrfs), which is why it is used only then. It needs
an ExecuTorch whose save takes ``consume``, and for the ``.pte`` a ``file_path`` that ends with
``.pte``. If a move fails, the call also deletes the partial output and any data file it moved only
in part, since neither can be finished; export again. With ``generate_etrecord``, the save copies
the engines, because the ETRecord, written after the program, reads them. The ETRecord holds copies
of every engine, and writing it reads them into memory.
The other formats save each engine from a loaded one, so their engines are loaded once all of them
are built, and their save needs as much memory as after ``compile``.

Expand Down Expand Up @@ -1600,7 +1619,46 @@ def compile_and_save(
save(trt_gm, os.fspath(file_path), output_format=output_format, **save_kwargs)


def _write_external_tensor_data(executorch_program: Any, file_path: str) -> None:
def _takes_consume(method: Callable[..., Any]) -> bool:
# ExecuTorch releases before pytorch/executorch#25099 have no consume argument; they copy.
return "consume" in inspect.signature(method).parameters


def _external_tensor_data_paths(executorch_program: Any, file_path: str) -> List[str]:
"""The ``.ptd`` files ``_write_external_tensor_data`` writes, next to ``file_path``."""
out_dir = os.path.dirname(os.path.abspath(file_path))
# Direct attribute access (no getattr default) so a future rename fails loud.
# write_tensor_data_to_file adds .ptd unless the name already ends with it.
return [
os.path.join(out_dir, name if name.endswith(".ptd") else f"{name}.ptd")
for name in executorch_program._tensor_data or ()
]


def _cords(executorch_program: Any) -> List[Any]:
"""The serialized program and its ``.ptd`` data, as ExecuTorch writes them."""
# Direct attribute access (no getattr default) so a future rename fails loud.
return [
executorch_program._pte_data,
*(executorch_program._tensor_data or {}).values(),
]


def _copy_fits(executorch_program: Any, outputs: Sequence[str]) -> bool:
"""Whether a copy of the outputs fits in their file system's free space, with a tenth to spare.

An output already there counts as free, since both a copy and a move replace it.
"""
size = sum(len(cord) for cord in _cords(executorch_program))
free = shutil.disk_usage(os.path.dirname(outputs[0])).free + sum(
os.path.getsize(path) for path in outputs if os.path.isfile(path)
)
return size + size // 10 <= free


def _write_external_tensor_data(
executorch_program: Any, file_path: str, consume: bool = False
) -> None:
"""Write an ExecuTorch program's external named tensor data (``.ptd``) next to the ``.pte``.

The CUDA (AOTInductor) backend emits its weights as external named data
Expand All @@ -1613,15 +1671,14 @@ def _write_external_tensor_data(executorch_program: Any, file_path: str) -> None
the program has any; the runtime must then be given those data file(s) (e.g.
via the ExecuTorch ``Module`` data-files argument) to load the program.
"""
# Direct attribute access (no getattr default) so a future rename fails loud.
if executorch_program._tensor_data:
out_dir = os.path.dirname(os.path.abspath(file_path))
executorch_program.write_tensor_data_to_file(out_dir)
# write_tensor_data_to_file adds .ptd unless the name already ends with it.
paths = [
os.path.join(out_dir, name if name.endswith(".ptd") else f"{name}.ptd")
for name in executorch_program._tensor_data
]
paths = _external_tensor_data_paths(executorch_program, file_path)
if paths:
out_dir = os.path.dirname(paths[0])
write = executorch_program.write_tensor_data_to_file
if consume and _takes_consume(write):
write(out_dir, consume=True)
else:
write(out_dir)
logger.info(
"Wrote external delegate data to %s. Pass %s, not the directory, "
"when loading the program.",
Expand Down Expand Up @@ -1752,9 +1809,40 @@ def _lower_and_write_executorch(
# the same call is the caller's to make, because finalization happens on
# ExecuTorch's own manager and out of this library's sight.
check_zero_copy_kv(executorch_program)
with open(file_path, "wb") as f:
executorch_program.write_to_file(f)
_write_external_tensor_data(executorch_program, file_path)
file_path = os.fspath(file_path)
# Inside compile_and_save the engine files are deleted after this save, so when a copy would not
# fit it moves them into the program instead. The move is slower, so it is used only then. The
# ETRecord reads the engines after the program is written, so with one the save copies them.
outputs = [
os.path.abspath(file_path),
*_external_tensor_data_paths(executorch_program, file_path),
]
consume = (
defer_engine_setup_to_load()
and not generate_etrecord
and _takes_consume(executorch_program.save)
and not _copy_fits(executorch_program, outputs)
)
moving = (
moving_engines_into(
outputs,
[
(data.path, len(data))
for cord in _cords(executorch_program)
for data in cord.file_backed_buffers()
],
)
if consume
else contextlib.nullcontext()
)
with moving:
if consume and file_path.endswith(".pte"):
# ExecuTorch's save takes only a .pte path; its .ptd write moves data for any name.
executorch_program.save(file_path, consume=True)
else:
with open(file_path, "wb") as f:
executorch_program.write_to_file(f)
_write_external_tensor_data(executorch_program, file_path, consume)
if generate_etrecord:
# Follows ExecuTorch's example convention (e.g. examples/cuda/scripts/export.py):
# persist the ETRecord as "<pte_base>_etrecord.bin" next to the .pte.
Expand Down
77 changes: 75 additions & 2 deletions py/torch_tensorrt/dynamo/runtime/_offloaded_engines.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@
"""

import contextlib
import glob
import logging
import os
import tempfile
from contextvars import ContextVar
from typing import Any, Dict, Iterator, Optional
from typing import Any, Dict, Iterator, Optional, Sequence, Set, Tuple

import torch
from torch_tensorrt._utils import _cache_root
Expand All @@ -41,8 +42,80 @@ def offload_engines_to_disk() -> Iterator[None]:
paths = list(files)
files.clear()
for path in paths:
with contextlib.suppress(FileNotFoundError):
try:
os.remove(path)
except FileNotFoundError:
pass # ExecuTorch deleted it, as the owner of a file it copied or moved.
except OSError as error:
# Logged, so it can't replace an error that ended the block.
logger.warning("Could not delete the engine file %s: %s", path, error)


@contextlib.contextmanager
def moving_engines_into(
outputs: Sequence[str], sources: Sequence[Tuple[str, int]]
) -> Iterator[None]:
"""Around an ExecuTorch save inside the block that moves the engines into ``outputs``.

``sources`` names each file the save may move data from, with the size of that data. If the save
fails, this deletes each partial output (``<output>.<random>.partial``) it left, and each source
other than an engine file that it moved in part, such as a file of CUDA weights. Neither can be
finished: the program is gone and the block deletes the engine files. Afterwards the plan of each
engine the save moved, in part or in full, is emptied, so a later read of it through its module
raises instead of killing the process with a bus error.
"""
files = _offloaded_engine_files.get()
assert files is not None, "engines move only inside offload_engines_to_disk()"

def partials() -> Set[str]:
return {
p for path in outputs for p in glob.glob(f"{glob.escape(path)}.*.partial")
}

earlier = partials()
try:
yield
except BaseException as e:
moved_in_part = [
path
for path, size in sources
if path not in files and 0 < _size(path) < size
]
deleted = []
for path in [*sorted(partials() - earlier), *moved_in_part]:
with contextlib.suppress(OSError):
os.remove(path)
deleted.append(path)
if deleted:
e.add_note(
f"Deleted {', '.join(deleted)}: what this save moved can't be finished, since "
"the files that hold the rest of it are deleted. Export again."
)
raise
finally:
for path, (plan, size) in _engine_files(files).items():
if size < plan.numel():
plan.set_() # in place, so every module holding the plan sees it empty
files[path] = None


def _size(path: str) -> int:
try:
return os.path.getsize(path)
except OSError:
return 0


def _engine_files(
files: Dict[str, Optional[torch.Tensor]],
) -> Dict[str, Tuple[torch.Tensor, int]]:
"""Each mapped engine file whose size can be read, with its mapping and its size now."""
found = {}
for path, plan in files.items():
if plan is not None:
with contextlib.suppress(OSError):
found[path] = plan, os.path.getsize(path)
return found


def defer_engine_setup_to_load() -> bool:
Expand Down
5 changes: 4 additions & 1 deletion py/torch_tensorrt/executorch/_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -598,7 +598,10 @@ def export(
zero_copy_kv=True)`` owns both steps and needs only the one flag.
generate_etrecord (bool): Ask ExecuTorch for an ETRecord for later debugging.
This copies the whole program, engines included. Saving the record does not
work for a program with TensorRT engines: it raises a ``SerializeError``.
work for a program from ``torch_tensorrt.dynamo.compile`` with TensorRT engines:
it raises a ``SerializeError``. ``torch_tensorrt.compile_and_save`` can write one,
because it keeps each engine as a byte buffer read from its file. The record
then holds copies of every engine, and writing it reads them into memory.
weight_streaming_budget_per_engine (Optional[int]): Bytes of engine weights that
may stay resident in GPU memory, with the rest streamed from host memory.
It applies to **each** TensorRT engine separately, not as a total for the
Expand Down
4 changes: 3 additions & 1 deletion py/torch_tensorrt/executorch/backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -699,7 +699,9 @@ def preprocess(

engine_data: Any
if engine_file is not None:
engine_data = FileBackedData(engine_file)
# compile_and_save deletes the file after the save, so the data can own it, which lets
# a consuming write move the engine into the program instead of copying it.
engine_data = FileBackedData(engine_file, cleanup=True)
engine_data_sha256 = engine_data.sha256()
else:
assert isinstance(engine_bytes, bytes)
Expand Down
Loading
Loading