Skip to content

Commit fcfc152

Browse files
committed
Let a write move file-backed data into the output instead of copying it
`FileBackedData` keeps a large blob, such as a delegate's compiled engine, in a file until the program is written. Writing it copies the file into the `.pte`. So until the caller deletes the file afterwards, the blob is on disk twice. For an engine of tens of gigabytes, that copy alone can make an export run out of disk. This adds `consume=True` to `FileBackedData.write_to_file`, `Cord.write_to_file` (whose new `keep` names buffers to copy anyway), and `ExecutorchProgramManager.write_to_file`, `write_tensor_data_to_file` and `save`. With it, data that owns its file (`cleanup=True`, as from `move_from`) is moved into the output: - Chunks are copied from the end of the data back to the start, each to its final offset. The source is truncated after each chunk, once the write returned the full length and was flushed. - The part of the output not yet written stays a hole (a sparse file), so the disk holds the data once, plus at most one chunk. - Afterwards the program can't be written or read again. Any `FileBackedData` on an emptied file now raises instead of reading a short file, because the check is on the file's size, not a flag. - A buffer that deduplication put in both the `.pte` and a `.ptd` (or in two `.ptd` files) is copied, because no single write knows it is the last one to need it. Everything else gets a copy, as before: data that does not own its file, a file with another hard link or that is not writable, an output that is not a regular file opened through `io.FileIO` or a buffered writer, an output opened with `O_APPEND`, and Windows, where files are not sparse by default. A consuming write that fails before its first truncation leaves the data intact, so it can be written again. One that fails later adds a note to the error that says which bytes are still in the source and where the rest is in the output, and leaves that source file in place. `ExecutorchProgramManager.save` now re-raises a failed write instead of only logging it. Before, a failure returned normally and left a broken `.pte` behind. The default is unchanged: without `consume`, every write copies. Test plan: `exir/_serialize/test/test_cord.py` and `exir/program/test/test_program.py`. New tests check: - the source still holds each range when it reaches the output, and allocated blocks never exceed the data plus one chunk; - only a consuming write consumes; - the copy fallbacks: unowned data, a second hard link, Windows, `BytesIO`, `/dev/null`, `O_APPEND`, gzip, a write-only object, and a pipe that reports it can seek, as pipes do on Windows; - a short write, and a failure on a later chunk; - a second buffer on an emptied file, a read-only file, and buffers listed in `keep`; - a buffer shared by the `.pte` and a `.ptd` ends up whole in both; - a save without `consume` can be repeated, and `buffer` still works after it; - `save` re-raises, and `save` and `write_tensor_data_to_file` pass `consume` on. `test_named_data_store.py` and `exir/tests/test_serde.py` also pass.
1 parent a532d10 commit fcfc152

4 files changed

Lines changed: 423 additions & 19 deletions

File tree

‎exir/_serialize/_cord.py‎

Lines changed: 108 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,28 @@
88
import io
99
import os
1010
import shutil
11+
import stat
1112
import tempfile
1213
import weakref
13-
from typing import List, Optional, Union
14+
from typing import Container, IO, List, Optional, Union
1415

1516

1617
class FileBackedData:
17-
"""A byte buffer that stays on disk until explicitly closed."""
18+
"""A byte buffer that stays on disk until explicitly closed or consumed.
19+
20+
``write_to_file(outfile, consume=True)`` moves data that owns its file
21+
(``cleanup=True``, as from ``move_from``) into ``outfile`` instead of copying it,
22+
so the data is never on disk twice. Chunks are copied from the end back to the
23+
start, and the file is truncated after each chunk reaches ``outfile``; the part of
24+
``outfile`` not yet written stays a hole. That needs a regular, sparse output that
25+
writes at the offsets it seeks to, so any other output, and Windows (where files
26+
are not sparse by default), gets a copy. Afterwards the file is empty, and neither
27+
this nor any other ``FileBackedData`` on it can be read again.
28+
29+
A consuming write that fails before its first truncation leaves the data intact,
30+
so it can be written again. One that fails later says, in a note on the error,
31+
which bytes are still in the file and where the rest is in ``outfile``.
32+
"""
1833

1934
_COPY_CHUNK_SIZE = 8 * 1024 * 1024
2035

@@ -56,27 +71,96 @@ def move_from(cls, path: str, sha256: Optional[bytes] = None) -> "FileBackedData
5671
def __len__(self) -> int:
5772
return self._size
5873

74+
def _open(self, mode: str = "rb") -> IO[bytes]:
75+
# The size, not a flag, shows a consumed file, so every object on it sees it.
76+
f = open(self._path, mode)
77+
size = os.fstat(f.fileno()).st_size
78+
if size != self._size:
79+
f.close()
80+
raise RuntimeError(
81+
f"{self._path} holds {size} of its {self._size} bytes; "
82+
"a consuming write may have taken the rest"
83+
)
84+
return f
85+
5986
def prefix(self, size: int) -> bytes:
60-
with open(self._path, "rb") as f:
87+
with self._open() as f:
6188
return f.read(size)
6289

6390
def sha256(self) -> bytes:
6491
if self._sha256 is None:
6592
digest = hashlib.sha256()
66-
with open(self._path, "rb") as f:
93+
with self._open() as f:
6794
while chunk := f.read(self._COPY_CHUNK_SIZE):
6895
digest.update(chunk)
6996
self._sha256 = digest.digest()
7097
return self._sha256
7198

7299
def to_bytes(self) -> bytes:
73-
with open(self._path, "rb") as f:
100+
with self._open() as f:
74101
return f.read()
75102

76-
def write_to_file(self, outfile: io.BufferedIOBase) -> None:
77-
with open(self._path, "rb") as f:
103+
def write_to_file(self, outfile: io.BufferedIOBase, consume: bool = False) -> None:
104+
"""Write the data to ``outfile``; with ``consume``, move it there when it can
105+
(see the class docstring)."""
106+
if consume and self._can_consume_into(outfile):
107+
self._consume_into(outfile)
108+
return
109+
with self._open() as f:
78110
shutil.copyfileobj(f, outfile, length=self._COPY_CHUNK_SIZE)
79111

112+
def _can_consume_into(self, outfile: io.BufferedIOBase) -> bool:
113+
owned = self._finalizer is not None and self._finalizer.alive
114+
if not owned or os.name == "nt":
115+
return False
116+
# Truncating changes every name of the file, unlike deleting it.
117+
source = os.lstat(self._path)
118+
if source.st_nlink != 1 or not os.access(self._path, os.W_OK):
119+
return False
120+
if not isinstance(outfile, (io.BufferedWriter, io.BufferedRandom, io.FileIO)):
121+
return False
122+
fd = outfile.fileno()
123+
if not stat.S_ISREG(os.fstat(fd).st_mode):
124+
return False
125+
import fcntl # POSIX only; Windows returned above.
126+
127+
# With O_APPEND every write lands at the end, not at the chunk's offset.
128+
return not fcntl.fcntl(fd, fcntl.F_GETFL) & os.O_APPEND
129+
130+
def _consume_into(self, outfile: io.BufferedIOBase) -> None:
131+
start = outfile.tell()
132+
end = self._size
133+
with self._open("r+b") as source:
134+
try:
135+
while end > 0:
136+
begin = max(end - self._COPY_CHUNK_SIZE, 0)
137+
source.seek(begin)
138+
chunk = source.read(end - begin)
139+
outfile.seek(start + begin)
140+
written = outfile.write(chunk)
141+
if written != len(chunk):
142+
raise OSError(
143+
f"wrote {written} of {len(chunk)} bytes at offset "
144+
f"{start + begin}"
145+
)
146+
# The chunk has to be in the output before the source gives it up.
147+
outfile.flush()
148+
source.truncate(begin)
149+
end = begin
150+
except BaseException as e:
151+
if end < self._size:
152+
# The note sends the caller to the rest of the data, so it must
153+
# outlive this object.
154+
if self._finalizer is not None:
155+
self._finalizer.detach()
156+
e.add_note(
157+
f"Bytes [0, {end}) of {self._path} are still in it; bytes "
158+
f"[{end}, {self._size}) are at offset {start + end} of the "
159+
"output."
160+
)
161+
raise
162+
outfile.seek(start + self._size)
163+
80164
def close(self) -> None:
81165
if self._finalizer is not None:
82166
self._finalizer()
@@ -131,10 +215,24 @@ def append(self, data: Union[CordBuffer, "Cord"]) -> None:
131215
f"Can only append bytes, FileBackedData, or Cords, received {type(data)}"
132216
)
133217

134-
def write_to_file(self, outfile: io.BufferedIOBase) -> None:
135-
"""Write the Cord to a file."""
218+
def file_backed(self) -> List[FileBackedData]:
219+
"""The file-backed buffers of the Cord."""
220+
return [item for item in self._buffers if isinstance(item, FileBackedData)]
221+
222+
def write_to_file(
223+
self,
224+
outfile: io.BufferedIOBase,
225+
consume: bool = False,
226+
keep: Container[FileBackedData] = (),
227+
) -> None:
228+
"""Write the Cord to a file.
229+
230+
With ``consume``, file-backed buffers that own their files, except those in
231+
``keep``, are moved into ``outfile`` rather than copied (see
232+
``FileBackedData``), so the Cord can't be written or read again afterwards.
233+
"""
136234
for item in self._buffers:
137235
if isinstance(item, bytes):
138236
outfile.write(item)
139237
else:
140-
item.write_to_file(outfile)
238+
item.write_to_file(outfile, consume=consume and item not in keep)

‎exir/_serialize/test/test_cord.py‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,13 @@
55
# LICENSE file in the root directory of this source tree.
66

77

8+
import gzip
89
import io
910
import os
1011
import tempfile
12+
import threading
1113
import unittest
14+
from unittest import mock
1215

1316
from executorch.exir._serialize._cord import Cord, FileBackedData
1417

@@ -82,3 +85,211 @@ def test_file_backed_data(self) -> None:
8285
self.assertEqual(b"PrefixFileBacked", bytes(cord))
8386

8487
self.assertEqual([], os.listdir(directory))
88+
89+
# 10 full chunks and part of one; the part is written last and fits in a buffer.
90+
_CHUNK = 8192
91+
_PAYLOAD = bytes(range(256)) * (10 * _CHUNK // 256) + b"x" * 1000
92+
93+
def _owned(self, directory: str) -> FileBackedData:
94+
path = os.path.join(directory, "source.bin")
95+
with open(path, "wb") as f:
96+
f.write(self._PAYLOAD)
97+
return FileBackedData.move_from(path)
98+
99+
def _assert_intact(self, data: FileBackedData) -> None:
100+
self.assertEqual(self._PAYLOAD, data.to_bytes())
101+
102+
def test_consume_moves_owned_data_without_a_second_copy(self) -> None:
103+
payload, chunk = self._PAYLOAD, self._CHUNK
104+
with tempfile.TemporaryDirectory() as directory:
105+
data = self._owned(directory)
106+
source = data._path
107+
out_path = os.path.join(directory, "out.bin")
108+
on_disk = []
109+
110+
class Recording(io.FileIO):
111+
def write(self, b) -> int:
112+
offset = self.tell() - len(b"Prefix")
113+
if 0 <= offset < len(payload):
114+
# The source must still hold the range that reaches the output.
115+
assert os.path.getsize(source) >= offset + len(b), offset
116+
size = super().write(b)
117+
on_disk.append(
118+
(os.stat(source).st_blocks + os.fstat(self.fileno()).st_blocks)
119+
* 512
120+
)
121+
return size
122+
123+
cord = Cord(b"Prefix")
124+
cord.append(data)
125+
cord.append(b"Suffix")
126+
with mock.patch.object(FileBackedData, "_COPY_CHUNK_SIZE", chunk):
127+
with io.BufferedWriter(Recording(out_path, "wb")) as outfile:
128+
cord.write_to_file(outfile, consume=True)
129+
130+
with open(out_path, "rb") as result:
131+
self.assertEqual(b"Prefix" + payload + b"Suffix", result.read())
132+
self.assertEqual(0, os.path.getsize(source))
133+
# Allocated blocks, so the unwritten part of the output must stay a hole.
134+
self.assertLessEqual(max(on_disk), len(payload) + chunk + 4 * 4096)
135+
with self.assertRaises(RuntimeError):
136+
data.to_bytes()
137+
138+
def test_only_a_consuming_write_consumes(self) -> None:
139+
with tempfile.TemporaryDirectory() as directory:
140+
data = self._owned(directory)
141+
cord = Cord(data)
142+
for _ in range(2):
143+
with open(os.path.join(directory, "out.bin"), "wb") as outfile:
144+
cord.write_to_file(outfile)
145+
self.assertEqual(self._PAYLOAD, bytes(cord))
146+
with open(os.path.join(directory, "out.bin"), "wb") as outfile:
147+
cord.write_to_file(outfile, consume=True)
148+
with self.assertRaises(RuntimeError):
149+
bytes(cord)
150+
151+
def test_consume_copies_data_that_does_not_own_its_file(self) -> None:
152+
with tempfile.TemporaryDirectory() as directory:
153+
path = os.path.join(directory, "source.bin")
154+
with open(path, "wb") as f:
155+
f.write(self._PAYLOAD)
156+
data = FileBackedData(path)
157+
with open(os.path.join(directory, "out.bin"), "wb") as outfile:
158+
data.write_to_file(outfile, consume=True)
159+
self._assert_intact(data)
160+
161+
def test_consume_copies_into_outputs_it_cannot_move_into(self) -> None:
162+
class WriteOnly:
163+
def __init__(self) -> None:
164+
self.data = b""
165+
166+
def write(self, b) -> int:
167+
self.data += bytes(b)
168+
return len(b)
169+
170+
with tempfile.TemporaryDirectory() as directory:
171+
data = self._owned(directory)
172+
out_path = os.path.join(directory, "out.bin")
173+
appending = os.open(out_path, os.O_WRONLY | os.O_CREAT | os.O_APPEND)
174+
outputs = {
175+
"BytesIO": lambda: io.BytesIO(),
176+
"devnull": lambda: open(os.devnull, "wb"),
177+
"O_APPEND opened as wb": lambda: os.fdopen(appending, "wb"),
178+
"gzip": lambda: gzip.GzipFile(out_path + ".gz", "wb"),
179+
"write-only": WriteOnly,
180+
}
181+
for name, make in outputs.items():
182+
with self.subTest(name):
183+
outfile = make()
184+
with mock.patch.object(FileBackedData, "_COPY_CHUNK_SIZE", 4096):
185+
data.write_to_file(outfile, consume=True)
186+
if hasattr(outfile, "close"):
187+
outfile.close()
188+
self._assert_intact(data)
189+
with open(out_path, "rb") as result:
190+
self.assertEqual(self._PAYLOAD, result.read())
191+
192+
def test_consume_copies_into_a_pipe_that_reports_it_can_seek(self) -> None:
193+
class ReportsSeekable(io.BufferedWriter):
194+
# On Windows a pipe reports seekable(), so the move can't rely on it.
195+
def seekable(self) -> bool:
196+
return True
197+
198+
with tempfile.TemporaryDirectory() as directory:
199+
data = self._owned(directory)
200+
read, write = os.pipe()
201+
# The data is bigger than a pipe's buffer, so read it while it is written.
202+
received = []
203+
with open(read, "rb") as infile:
204+
reader = threading.Thread(target=lambda: received.append(infile.read()))
205+
reader.start()
206+
with ReportsSeekable(io.FileIO(write, "wb")) as outfile:
207+
data.write_to_file(outfile, consume=True)
208+
reader.join()
209+
self.assertEqual([self._PAYLOAD], received)
210+
self._assert_intact(data)
211+
212+
def test_consume_copies_a_file_with_another_name(self) -> None:
213+
with tempfile.TemporaryDirectory() as directory:
214+
data = self._owned(directory)
215+
os.link(data._path, os.path.join(directory, "other_name.bin"))
216+
with open(os.path.join(directory, "out.bin"), "wb") as outfile:
217+
data.write_to_file(outfile, consume=True)
218+
self._assert_intact(data)
219+
220+
def test_consume_copies_on_windows(self) -> None:
221+
with tempfile.TemporaryDirectory() as directory:
222+
data = self._owned(directory)
223+
with open(os.path.join(directory, "out.bin"), "wb") as outfile:
224+
with mock.patch.object(os, "name", "nt"):
225+
data.write_to_file(outfile, consume=True)
226+
self._assert_intact(data)
227+
228+
def test_a_consuming_write_that_fails_first_keeps_the_data(self) -> None:
229+
class Short(io.FileIO):
230+
def write(self, b) -> int:
231+
return super().write(bytes(b)[:-1])
232+
233+
with tempfile.TemporaryDirectory() as directory:
234+
data = self._owned(directory)
235+
out_path = os.path.join(directory, "out.bin")
236+
with Short(out_path, "wb") as outfile:
237+
with self.assertRaises(OSError):
238+
data.write_to_file(outfile, consume=True)
239+
self._assert_intact(data)
240+
with open(out_path, "wb") as outfile:
241+
data.write_to_file(outfile, consume=True)
242+
with open(out_path, "rb") as result:
243+
self.assertEqual(self._PAYLOAD, result.read())
244+
245+
def test_a_consuming_write_that_fails_later_says_where_the_bytes_are(self) -> None:
246+
class FailsSecond(io.FileIO):
247+
writes = 0
248+
249+
def write(self, b) -> int:
250+
FailsSecond.writes += 1
251+
if FailsSecond.writes == 2:
252+
raise OSError("disk full")
253+
return super().write(b)
254+
255+
with tempfile.TemporaryDirectory() as directory:
256+
data = self._owned(directory)
257+
with mock.patch.object(FileBackedData, "_COPY_CHUNK_SIZE", self._CHUNK):
258+
with FailsSecond(os.path.join(directory, "out.bin"), "wb") as outfile:
259+
with self.assertRaises(OSError) as raised:
260+
data.write_to_file(outfile, consume=True)
261+
# The first chunk moved is the last one, a full chunk.
262+
left = len(self._PAYLOAD) - self._CHUNK
263+
path = data._path
264+
del data # The bytes the note points to outlive the object.
265+
self.assertEqual(left, os.path.getsize(path))
266+
self.assertIn(f"Bytes [0, {left}) of {path}", raised.exception.__notes__[0])
267+
self.assertIn(
268+
f"at offset {left} of the output", raised.exception.__notes__[0]
269+
)
270+
271+
def test_another_buffer_on_a_consumed_file_refuses_to_copy_it(self) -> None:
272+
with tempfile.TemporaryDirectory() as directory:
273+
data = self._owned(directory)
274+
second = FileBackedData(data._path)
275+
with open(os.path.join(directory, "out.bin"), "wb") as outfile:
276+
data.write_to_file(outfile, consume=True)
277+
with self.assertRaises(RuntimeError):
278+
second.write_to_file(outfile)
279+
280+
def test_consume_copies_data_it_cannot_write_to(self) -> None:
281+
if os.geteuid() == 0:
282+
self.skipTest("root can write to any file")
283+
with tempfile.TemporaryDirectory() as directory:
284+
data = self._owned(directory)
285+
os.chmod(data._path, 0o444)
286+
with open(os.path.join(directory, "out.bin"), "wb") as outfile:
287+
data.write_to_file(outfile, consume=True)
288+
self._assert_intact(data)
289+
290+
def test_consume_copies_buffers_listed_to_keep(self) -> None:
291+
with tempfile.TemporaryDirectory() as directory:
292+
data = self._owned(directory)
293+
with open(os.path.join(directory, "out.bin"), "wb") as outfile:
294+
Cord(data).write_to_file(outfile, consume=True, keep={data})
295+
self._assert_intact(data)

0 commit comments

Comments
 (0)