Skip to content

Commit 9fab878

Browse files
committed
Move file-backed data into the output file instead of copying it
`FileBackedData` keeps a large blob, such as a delegate's compiled engine, in a file until the program is written. Writing it copies the file into the `.pte`. So until the caller deletes the file afterwards, the blob is on disk twice. For an engine of tens of gigabytes, that copy alone can make an export run out of disk. This adds `consume_on_write` for a caller that deletes the file after the write anyway. The first write into a seekable file then moves the data instead of copying it: - It copies chunks from the end of the data back to the start, each to its final offset in the output, and truncates the source after each chunk. - The part of the output not yet written stays a hole (a sparse file), so the disk holds the data once, plus at most one chunk. - Afterwards the source is empty, and a later read of the data raises an error instead of returning a short file. - A stream that cannot seek, such as a pipe, or a file opened to append, still gets a copy. The default is unchanged. Test plan: in `exir/_serialize/test/test_cord.py`, a new test counts the bytes on disk after every write. It fails if the data is copied (twice its size on disk at the end) or if the source is only truncated after the whole copy. Other new tests cover a pipe, a file opened to append, and a second buffer on the same file, which now raises instead of writing a hole. `test_cord.py`, `test_named_data_store.py` and `exir/tests/test_serde.py` pass.
1 parent a532d10 commit 9fab878

2 files changed

Lines changed: 127 additions & 6 deletions

File tree

‎exir/_serialize/_cord.py‎

Lines changed: 51 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,18 @@
1010
import shutil
1111
import tempfile
1212
import weakref
13-
from typing import List, Optional, Union
13+
from typing import IO, List, Optional, Union
1414

1515

1616
class FileBackedData:
17-
"""A byte buffer that stays on disk until explicitly closed."""
17+
"""A byte buffer that stays on disk until explicitly closed.
18+
19+
With ``consume_on_write``, the first write into a seekable file moves the data there
20+
instead of copying it, so the data is never on disk twice: chunks are copied from
21+
the end back to the start, and ``path`` is truncated after each one, while the part
22+
of the output not yet written stays a hole. Afterwards ``path`` is empty and the
23+
data can't be read again. For a caller that deletes ``path`` after the write.
24+
"""
1825

1926
_COPY_CHUNK_SIZE = 8 * 1024 * 1024
2027

@@ -23,10 +30,13 @@ def __init__(
2330
path: str,
2431
cleanup: bool = False,
2532
sha256: Optional[bytes] = None,
33+
consume_on_write: bool = False,
2634
) -> None:
2735
self._path = path
2836
self._size = os.path.getsize(path)
2937
self._sha256 = sha256
38+
self._consume_on_write = consume_on_write
39+
self._moved = False
3040
self._finalizer = (
3141
weakref.finalize(self, self._remove, path) if cleanup else None
3242
)
@@ -56,27 +66,62 @@ def move_from(cls, path: str, sha256: Optional[bytes] = None) -> "FileBackedData
5666
def __len__(self) -> int:
5767
return self._size
5868

69+
def _open(self, mode: str = "rb") -> IO[bytes]:
70+
if self._moved:
71+
raise RuntimeError(
72+
f"The data of {self._path} was moved into the file it was written to"
73+
)
74+
return open(self._path, mode)
75+
5976
def prefix(self, size: int) -> bytes:
60-
with open(self._path, "rb") as f:
77+
with self._open() as f:
6178
return f.read(size)
6279

6380
def sha256(self) -> bytes:
6481
if self._sha256 is None:
6582
digest = hashlib.sha256()
66-
with open(self._path, "rb") as f:
83+
with self._open() as f:
6784
while chunk := f.read(self._COPY_CHUNK_SIZE):
6885
digest.update(chunk)
6986
self._sha256 = digest.digest()
7087
return self._sha256
7188

7289
def to_bytes(self) -> bytes:
73-
with open(self._path, "rb") as f:
90+
with self._open() as f:
7491
return f.read()
7592

7693
def write_to_file(self, outfile: io.BufferedIOBase) -> None:
77-
with open(self._path, "rb") as f:
94+
# In append mode every write lands at the end, not at the chunk's offset.
95+
if (
96+
self._consume_on_write
97+
and outfile.seekable()
98+
and "a" not in getattr(outfile, "mode", "")
99+
):
100+
self._move_to_file(outfile)
101+
return
102+
with self._open() as f:
78103
shutil.copyfileobj(f, outfile, length=self._COPY_CHUNK_SIZE)
79104

105+
def _move_to_file(self, outfile: io.BufferedIOBase) -> None:
106+
start = outfile.tell()
107+
with self._open("r+b") as source:
108+
# Set before any truncation, so a partly moved file is never read as whole.
109+
self._moved = True
110+
end = self._size
111+
while end > 0:
112+
begin = max(end - self._COPY_CHUNK_SIZE, 0)
113+
source.seek(begin)
114+
chunk = source.read(end - begin)
115+
if len(chunk) != end - begin:
116+
raise RuntimeError(
117+
f"{self._path} no longer holds the {self._size} bytes to move"
118+
)
119+
outfile.seek(start + begin)
120+
outfile.write(chunk)
121+
source.truncate(begin)
122+
end = begin
123+
outfile.seek(start + self._size)
124+
80125
def close(self) -> None:
81126
if self._finalizer is not None:
82127
self._finalizer()

‎exir/_serialize/test/test_cord.py‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import os
1010
import tempfile
1111
import unittest
12+
from unittest import mock
1213

1314
from executorch.exir._serialize._cord import Cord, FileBackedData
1415

@@ -82,3 +83,78 @@ def test_file_backed_data(self) -> None:
8283
self.assertEqual(b"PrefixFileBacked", bytes(cord))
8384

8485
self.assertEqual([], os.listdir(directory))
86+
87+
def test_file_backed_data_moves_into_a_file_without_a_second_copy(self) -> None:
88+
payload = bytes(range(256)) * 41 # 10 chunks and part of one
89+
chunk = 1000
90+
with tempfile.TemporaryDirectory() as directory:
91+
source_path = os.path.join(directory, "source.bin")
92+
with open(source_path, "wb") as f:
93+
f.write(payload)
94+
written, on_disk = [0], []
95+
96+
class Recording(io.FileIO):
97+
# Bytes on disk after each write: the source left plus all written.
98+
def write(self, b) -> int:
99+
size = super().write(b)
100+
written[0] += size
101+
on_disk.append(os.path.getsize(source_path) + written[0])
102+
return size
103+
104+
data = FileBackedData(source_path, consume_on_write=True)
105+
cord = Cord(b"Prefix")
106+
cord.append(data)
107+
cord.append(b"Suffix")
108+
out_path = os.path.join(directory, "out.bin")
109+
with mock.patch.object(FileBackedData, "_COPY_CHUNK_SIZE", chunk):
110+
with io.BufferedWriter(Recording(out_path, "wb")) as outfile:
111+
cord.write_to_file(outfile)
112+
113+
with open(out_path, "rb") as result:
114+
self.assertEqual(b"Prefix" + payload + b"Suffix", result.read())
115+
self.assertEqual(0, os.path.getsize(source_path))
116+
self.assertLessEqual(
117+
max(on_disk), len(b"PrefixSuffix") + len(payload) + chunk
118+
)
119+
with self.assertRaises(RuntimeError):
120+
data.to_bytes()
121+
122+
def test_file_backed_data_is_copied_into_a_stream_it_cannot_seek(self) -> None:
123+
with tempfile.TemporaryDirectory() as directory:
124+
source_path = os.path.join(directory, "source.bin")
125+
with open(source_path, "wb") as f:
126+
f.write(b"FileBacked")
127+
data = FileBackedData(source_path, consume_on_write=True)
128+
read, write = os.pipe()
129+
with open(write, "wb") as outfile:
130+
data.write_to_file(outfile)
131+
with open(read, "rb") as infile:
132+
self.assertEqual(b"FileBacked", infile.read())
133+
self.assertEqual(b"FileBacked", data.to_bytes())
134+
135+
def test_file_backed_data_is_copied_into_a_file_opened_to_append(self) -> None:
136+
with tempfile.TemporaryDirectory() as directory:
137+
source_path = os.path.join(directory, "source.bin")
138+
with open(source_path, "wb") as f:
139+
f.write(b"FileBacked")
140+
out_path = os.path.join(directory, "out.bin")
141+
with open(out_path, "wb") as f:
142+
f.write(b"Prefix")
143+
data = FileBackedData(source_path, consume_on_write=True)
144+
with mock.patch.object(FileBackedData, "_COPY_CHUNK_SIZE", 4):
145+
with open(out_path, "ab") as outfile:
146+
data.write_to_file(outfile)
147+
with open(out_path, "rb") as result:
148+
self.assertEqual(b"PrefixFileBacked", result.read())
149+
150+
def test_file_backed_data_refuses_to_move_a_file_that_shrank(self) -> None:
151+
with tempfile.TemporaryDirectory() as directory:
152+
source_path = os.path.join(directory, "source.bin")
153+
with open(source_path, "wb") as f:
154+
f.write(b"FileBacked")
155+
first = FileBackedData(source_path, consume_on_write=True)
156+
second = FileBackedData(source_path, consume_on_write=True)
157+
with open(os.path.join(directory, "out.bin"), "wb") as outfile:
158+
first.write_to_file(outfile)
159+
with self.assertRaises(RuntimeError):
160+
second.write_to_file(outfile)

0 commit comments

Comments
 (0)