Skip to content

Commit 61129f4

Browse files
committed
Serialize a constant left on the GPU from a host copy
When a module is still on the GPU at export, as in flows that compile it there first such as Torch-TensorRT, a constant that no delegate takes stays in the program with its storage on the GPU. That happens when forward() returns a buffer, alone or next to a computed result, or when a delegate reads a constant from outside it. to_executorch() then raised an InternalError (and crashed before #24202). A program stores constants as host bytes whatever device they came from. So when only delegates, copies between host and device memory, or the method output read such a constant, the emitter now reads it from a host copy instead of raising. The constant keeps its place and device information in the program, so the runtime treats it as it would the same constant exported from the CPU. The InternalError stays when an operator outside any delegate reads the constant. That operator was traced on the GPU but runs on a CPU kernel, which may not match (#21545): a GPU batch norm lowered with no backend, for example, gets statistics of a shape the CPU kernel rejects, and fails on every call. Storage with no data, such as "meta", still raises too. Test plan: CUDA tests export a GPU module that returns (x * 2, buffer) with the CudaPartitioner, and a GPU module whose constant a delegate reads from outside it. Both run, match eager, and fail without the change. Another test checks that a GPU Linear lowered with no backend still raises, and the "meta" test now also covers a buffer that only the output reads.
1 parent aa83485 commit 61129f4

4 files changed

Lines changed: 112 additions & 23 deletions

File tree

‎backends/cuda/tests/test_cuda_export.py‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,14 @@
1919
from executorch.backends.cuda.cuda_partitioner import CudaPartitioner
2020
from executorch.backends.cuda.passes.remove_assert_async import RemoveAssertAsyncPass
2121
from executorch.examples.models.toy_model import SdpaModule
22-
from executorch.exir import EdgeCompileConfig, schema, to_edge_transform_and_lower
22+
from executorch.exir import (
23+
EdgeCompileConfig,
24+
schema,
25+
to_edge,
26+
to_edge_transform_and_lower,
27+
)
2328
from executorch.exir.backend.compile_spec_schema import CompileSpec
29+
from executorch.exir.error import InternalError
2430
from executorch.exir.passes import replace_complex_with_real_pass
2531
from torch.export import export
2632
from torch.nn.attention.varlen import varlen_attn
@@ -1088,6 +1094,43 @@ def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, ...]:
10881094
for output, expected in zip(outputs, model(x), strict=True):
10891095
torch.testing.assert_close(output.cpu(), expected)
10901096

1097+
@unittest.skipIf(
1098+
torch.version.hip is not None,
1099+
"loads through the Python runtime, which the ROCm build does not include",
1100+
)
1101+
def test_buffer_returned_outside_the_delegate(self):
1102+
"""A module on the GPU returns a buffer that no delegate reads. The
1103+
program stores the buffer's bytes and returns it from host memory."""
1104+
from executorch.runtime import Runtime
1105+
1106+
class ReturnsBuffer(torch.nn.Module):
1107+
def __init__(self):
1108+
super().__init__()
1109+
self.register_buffer("table", torch.arange(6.0).reshape(2, 3))
1110+
1111+
def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, ...]:
1112+
return x * 2, self.table
1113+
1114+
model = ReturnsBuffer().cuda()
1115+
x = torch.randn(2, 3)
1116+
edge = self._export_to_cuda_with_lower(model, (x.cuda(),))
1117+
with tempfile.TemporaryDirectory() as directory:
1118+
path = os.path.join(directory, "model.pte")
1119+
with open(path, "wb") as f:
1120+
edge.to_executorch().write_to_file(f)
1121+
method = Runtime.get().load_program(path).load_method("forward")
1122+
outputs = method.execute([x])
1123+
for output, expected in zip(outputs, model(x.cuda()), strict=True):
1124+
torch.testing.assert_close(output, expected.cpu())
1125+
1126+
def test_constant_read_outside_the_delegate_raises(self):
1127+
"""With no backend, a host operator reads the GPU weight. That operator
1128+
was traced on the GPU, so the export still refuses the weight."""
1129+
model = torch.nn.Linear(3, 2).cuda()
1130+
edge = to_edge(export(model, (torch.randn(1, 3, device="cuda"),)))
1131+
with self.assertRaisesRegex(InternalError, "on device 'cuda:0'"):
1132+
edge.to_executorch()
1133+
10911134
@unittest.skipIf(
10921135
torch.version.hip is not None,
10931136
"loads through the Python runtime, which the ROCm build does not include",

‎backends/cuda/tests/test_device_placement.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,26 @@ def forward(self, x):
122122
for output, expected in zip(run(model, (x,)), model(x), strict=True):
123123
torch.testing.assert_close(output, expected)
124124

125+
def test_gpu_constant_read_by_a_delegate(self):
126+
"""The module is still on the GPU at export, and a delegate reads its
127+
constant from outside it. The program stores the constant from a host
128+
copy and copies it to the device on every call."""
129+
130+
class Model(torch.nn.Module):
131+
def __init__(self):
132+
super().__init__()
133+
self.register_buffer("table", torch.arange(6.0).reshape(2, 3))
134+
135+
def forward(self, x):
136+
return torch.add(x, self.table)
137+
138+
model = Model().cuda()
139+
x = torch.arange(6.0).reshape(2, 3) * 10
140+
with tempfile.TemporaryDirectory() as directory:
141+
method = load(model, (x.cuda(),), directory)
142+
(output,) = method.execute([x])
143+
torch.testing.assert_close(output.cpu(), model(x.cuda()).cpu())
144+
125145
def test_input_on_device_also_read_on_host(self):
126146
"""With skip_h2d_for_method_inputs, the caller passes the input on the
127147
device, and the default memory planning leaves it unplanned. A delegate

‎exir/emit/_emitter.py‎

Lines changed: 31 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,17 @@ class _AbstractValue:
214214
]
215215

216216

217+
def _read_on_host(node: Node) -> bool:
218+
"""Whether an operator outside any delegate reads node, other than a copy
219+
between host and device memory."""
220+
return any(
221+
user.op == "call_function"
222+
and user.target is not executorch_call_delegate
223+
and getattr(user.target, "namespace", None) != "et_copy"
224+
for user in node.users
225+
)
226+
227+
217228
class _Emitter(torch.fx.Interpreter):
218229
"""An abstract interpreter (https://wiki.mozilla.org/Abstract_Interpretation) used to emit the
219230
given traced torch.fx.GraphModule to the flatbuffer schema."""
@@ -467,29 +478,31 @@ def _tensor_spec_to_evalue(
467478
if spec.const:
468479
# Tensor with a blob we need to serialize. May not actually be constant at runtime
469480
# if it's a weight with an associated gradient.
470-
spec_array_type = (
471-
ctypes.c_char * typing.cast(torch.UntypedStorage, spec.storage).nbytes()
472-
)
473-
474-
# data_ptr() is dereferenced from the host below; for non-CPU storage
475-
# (e.g. CUDA) that read can segfault instead of raising (#21545).
476-
storage_device = typing.cast(torch.UntypedStorage, spec.storage).device
477-
if spec.allocated_memory != 0 and storage_device.type != "cpu":
478-
raise InternalError(
479-
self._emit_node_specific_error(
480-
self.node,
481-
f"Constant tensor storage is on device '{storage_device}', "
482-
"but constants serialized into the program must be in CPU "
483-
"memory. Move them to CPU before export (e.g. model.cpu()), "
484-
"or delegate the ops that use them to a backend that "
485-
"accepts device tensors.",
481+
storage = typing.cast(torch.UntypedStorage, spec.storage)
482+
spec_array_type = ctypes.c_char * storage.nbytes()
483+
484+
if spec.allocated_memory != 0 and storage.device.type != "cpu":
485+
# An operator outside any delegate that reads a device constant
486+
# was traced on that device, so its host kernel may not match
487+
# (#21545), and meta storage holds no data. Any other constant
488+
# is serialized from a host copy.
489+
if storage.device.type == "meta" or _read_on_host(self.node):
490+
raise InternalError(
491+
self._emit_node_specific_error(
492+
self.node,
493+
f"Constant tensor storage is on device '{storage.device}', "
494+
"but constants serialized into the program must be in CPU "
495+
"memory. Move them to CPU before export (e.g. model.cpu()), "
496+
"or delegate the ops that use them to a backend that "
497+
"accepts device tensors.",
498+
)
486499
)
487-
)
500+
storage = storage.cpu()
488501

489502
buffer_data = (
490503
bytes(
491504
ctypes.cast(
492-
typing.cast(torch.UntypedStorage, spec.storage).data_ptr(),
505+
storage.data_ptr(),
493506
ctypes.POINTER(spec_array_type),
494507
).contents
495508
)

‎exir/emit/test/test_emit.py‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1858,10 +1858,23 @@ def test_emit_non_cpu_constant_raises(self) -> None:
18581858
# Like CUDA storage, "meta" storage is not host-readable, but needs no
18591859
# accelerator. Unguarded, it already fails with a wrapped "NULL pointer
18601860
# access" InternalError, so match on the device name.
1861-
model = torch.nn.Linear(4, 3).to("meta")
1862-
edge = to_edge(export(model, (torch.ones(2, 4, device="meta"),), strict=True))
1863-
with self.assertRaisesRegex(InternalError, "on device 'meta'"):
1864-
edge.to_executorch()
1861+
class ReturnsBuffer(torch.nn.Module):
1862+
def __init__(self) -> None:
1863+
super().__init__()
1864+
self.register_buffer("b", torch.ones(3, device="meta"))
1865+
1866+
def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
1867+
return x + 1, self.b
1868+
1869+
# An operator reads the Linear weight, while only the output reads the
1870+
# buffer, which has no data to copy either.
1871+
for model, x in (
1872+
(torch.nn.Linear(4, 3).to("meta"), torch.ones(2, 4, device="meta")),
1873+
(ReturnsBuffer(), torch.ones(3, device="meta")),
1874+
):
1875+
edge = to_edge(export(model, (x,), strict=True))
1876+
with self.assertRaisesRegex(InternalError, "on device 'meta'"):
1877+
edge.to_executorch()
18651878

18661879
def test_non_persistent_buffer(self) -> None:
18671880
class NonPersistentBuffer(nn.Module):

0 commit comments

Comments
 (0)