Skip to content

Commit f3d4581

Browse files
committed
Copy instead of failing when a planned move can't start
A consuming save decides which buffers move before it writes anything, and checks each source again when its turn comes. Until now, a refusal at that point raised, after the rest of the output was already written, even though the data was intact and a copy would have worked. It also took the file lock only at that point, so a file system without `flock` support passed every check while planning and then failed the save. Now: - The lock is tried while planning, before the other checks. A file system without locks is a reason to copy, shown in the warning. A file that another save is moving still raises. - If a buffer can't move any more when its turn comes, for example because it gained a second name, it is copied into its place, counted in the warning, and synced before the rename. A source whose size changed still raises. - The writer reports which buffers it has started to move, and the cleanup uses that list instead of guessing from the sizes of the source files. A save that loses a shared buffer to another save no longer keeps its temporary file or claims that it holds the data. - The output directory is synced after the temporary file is created and after the rename, so both names survive a crash. This is skipped where a directory can't be opened. - The output directory is found with `os.path.dirname`, as `move_from` does, so a path through `link/..` is checked against the directory the file is really created in. - Cadence's own `save_pte_program` now calls `prog.save`, so it raises on a failed write like the shared helper does. Test plan: `exir/_serialize/test/test_cord.py` and `exir/program/test/test_program.py` on Linux x86_64. New tests cover a file system without locks, a source that gains a second name after planning, two saves that share a buffer, a first truncation that fails, and the directory syncs; each fails when its fix is removed. Another new test covers a source on another file system at the save level, a case that already copied. The `os.path.dirname` change has no test of its own.
1 parent 905c4bc commit f3d4581

5 files changed

Lines changed: 311 additions & 72 deletions

File tree

‎backends/cadence/aot/utils.py‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -290,12 +290,7 @@ def save_pte_program(
290290
else:
291291
filename = os.path.join(output_dir, f"{model_name}.pte")
292292

293-
try:
294-
with open(filename, "wb") as file:
295-
prog.write_to_file(file)
296-
logging.info(f"Saved exported program to {filename}")
297-
except Exception as e:
298-
logging.error(f"Error while saving to {filename}: {e}")
293+
prog.save(filename)
299294

300295

301296
def save_bpte_program(

‎exir/_serialize/_cord.py‎

Lines changed: 75 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import sys
1313
import tempfile
1414
import weakref
15+
from collections import Counter
1516
from typing import Collection, Dict, IO, Iterator, List, Optional, Union
1617

1718

@@ -162,8 +163,8 @@ def write_to_file(self, outfile: io.BufferedIOBase) -> None:
162163

163164
def _open_to_move(self, output_device: int) -> Union[int, str]:
164165
"""Open the file to move the data into a file on ``output_device`` (see the class
165-
docstring). Returns the open file descriptor, or why the data has to be copied
166-
instead."""
166+
docstring), locked. Returns the open file descriptor, or why the data has to be
167+
copied instead. Raises if another save is moving it."""
167168
if self._finalizer is None or not self._finalizer.alive:
168169
return "not owned"
169170
if sys.platform == "win32":
@@ -174,11 +175,27 @@ def _open_to_move(self, output_device: int) -> Union[int, str]:
174175
source = os.open(self._path, os.O_RDWR | os.O_NOFOLLOW | os.O_NONBLOCK)
175176
except OSError as e:
176177
return "a link" if e.errno == errno.ELOOP else "read-only or unreadable"
178+
if not stat.S_ISREG(os.fstat(source).st_mode):
179+
# Checked before the lock, since macOS can't lock a pipe.
180+
os.close(source)
181+
return "not a regular file"
182+
import fcntl
183+
184+
try:
185+
# Taken before the checks, so no other move changes the file after them; two
186+
# moves of one file would each copy what the other already cut off.
187+
fcntl.flock(source, fcntl.LOCK_EX | fcntl.LOCK_NB)
188+
except OSError as e:
189+
os.close(source)
190+
if isinstance(e, BlockingIOError):
191+
raise RuntimeError(
192+
f"{self._path} is being moved by another save"
193+
) from None
194+
return "on a file system without file locks"
177195
opened = os.fstat(source)
178196
# Truncating changes every name of the file, and on another file system it frees
179197
# nothing for the output.
180198
checks = (
181-
(not stat.S_ISREG(opened.st_mode), "not a regular file"),
182199
(opened.st_nlink != 1, "with another hard link"),
183200
(opened.st_dev != output_device, "on another file system"),
184201
(opened.st_size != self._size, "changed in size"),
@@ -197,32 +214,34 @@ def _move_reason(self, output_device: int) -> Optional[str]:
197214
os.close(opened)
198215
return None
199216

200-
def _move_to(self, outfile: io.BufferedIOBase, offset: int) -> None:
201-
"""Move the data into ``outfile`` at ``offset``. ``outfile`` is a regular file its
202-
caller created for this write, and ``_move_reason`` found the data movable."""
217+
def _move_to(
218+
self, outfile: io.BufferedIOBase, offset: int, moved: List["FileBackedData"]
219+
) -> Optional[str]:
220+
"""Move the data into ``outfile`` at ``offset``, adding this object to ``moved``
221+
at the first truncation. ``outfile`` is a regular file its caller created for this
222+
write. Returns why the data has to be copied instead if it can't move any more,
223+
having changed nothing."""
203224
source = self._open_to_move(os.fstat(outfile.fileno()).st_dev)
204225
if isinstance(source, str):
205-
raise RuntimeError(f"{self._path} can't be moved any more ({source})")
226+
return source
206227
try:
207-
import fcntl
208-
209-
try:
210-
# Two moves of one file would each copy what the other already cut off.
211-
fcntl.flock(source, fcntl.LOCK_EX | fcntl.LOCK_NB)
212-
except BlockingIOError:
213-
raise RuntimeError(
214-
f"{self._path} is being moved by another save"
215-
) from None
216-
self._move_chunks(outfile, source, offset)
228+
self._move_chunks(outfile, source, offset, moved)
217229
finally:
218230
os.close(source)
231+
return None
219232

220233
def _own_file(self) -> None:
221234
# A move detaches the finalizer while it shrinks the file; this takes the file back.
222235
if self._finalizer is not None and not self._finalizer.alive:
223236
self._finalizer = weakref.finalize(self, self._remove, self._path)
224237

225-
def _move_chunks(self, outfile: io.BufferedIOBase, source: int, start: int) -> None:
238+
def _move_chunks(
239+
self,
240+
outfile: io.BufferedIOBase,
241+
source: int,
242+
start: int,
243+
moved: List["FileBackedData"],
244+
) -> None:
226245
end = self._size
227246
try:
228247
while end > 0:
@@ -243,10 +262,13 @@ def _move_chunks(self, outfile: io.BufferedIOBase, source: int, start: int) -> N
243262
# The chunk has to be on the output's disk before the source gives it up.
244263
outfile.flush()
245264
_sync(outfile.fileno())
246-
if end == self._size and self._finalizer is not None:
247-
# From the first truncation, part of the data is found only through
248-
# the error note, so no interrupt may let the finalizer delete it.
249-
self._finalizer.detach()
265+
if end == self._size:
266+
moved.append(self)
267+
if self._finalizer is not None:
268+
# From the first truncation, part of the data is found only
269+
# through the error note, so no interrupt may let the finalizer
270+
# delete it.
271+
self._finalizer.detach()
250272
end = begin
251273
os.ftruncate(source, begin)
252274
self._own_file()
@@ -257,8 +279,11 @@ def _move_chunks(self, outfile: io.BufferedIOBase, source: int, start: int) -> N
257279
except OSError:
258280
pass
259281
if end == self._size:
260-
# Nothing left the file, so this object still owns it.
282+
# Nothing left the file, so this object still owns it and the output
283+
# holds nothing that exists only there.
261284
self._own_file()
285+
if self in moved:
286+
moved.remove(self)
262287
else:
263288
e.add_note(
264289
f"Bytes [0, {end}) are still in {self._path}; bytes "
@@ -329,16 +354,24 @@ def write_to_file(self, outfile: io.BufferedIOBase) -> None:
329354
self._write_to_file(outfile)
330355

331356
def _write_to_file(
332-
self, outfile: io.BufferedIOBase, move: Collection[FileBackedData] = ()
357+
self,
358+
outfile: io.BufferedIOBase,
359+
move: Collection[FileBackedData] = (),
360+
moved: Optional[List[FileBackedData]] = None,
361+
copied: Optional["Counter[str]"] = None,
333362
) -> None:
334363
"""Write the Cord to ``outfile``, moving the buffers in ``move`` into it (see
335364
``FileBackedData``). Each of them must appear once, and ``_move_reason`` must have
336365
found it movable into ``outfile``, a regular file created for this write.
337366
Everything else is written first, around holes left for them, and the sync
338367
before the first truncation reports any error from it, so running out of space
339-
shows before any source shrinks. If the write fails, a note
340-
on the error lists the buffers already moved, whose only copy is now in
341-
``outfile``, and where they are."""
368+
shows before any source shrinks. Adds each buffer to ``moved`` once its source
369+
starts to shrink, even if the write then fails, since ``outfile`` then holds its
370+
only copy. A buffer that can't move any more is copied, and its reason counted in
371+
``copied``. If the write fails, a note on the error lists the buffers already
372+
moved in full, and where they are."""
373+
moved = [] if moved is None else moved
374+
copied = Counter() if copied is None else copied
342375
if not move:
343376
for item in self._buffers:
344377
_write_buffer(item, outfile)
@@ -350,7 +383,7 @@ def _write_to_file(
350383
moving = [
351384
isinstance(item, FileBackedData) and item in move for item in self._buffers
352385
]
353-
moved: List[str] = []
386+
done: List[str] = []
354387
try:
355388
for item, offset, skip in zip(self._buffers, offsets, moving):
356389
if not skip:
@@ -359,13 +392,23 @@ def _write_to_file(
359392
for item, offset, skip in zip(self._buffers, offsets, moving):
360393
if skip:
361394
assert isinstance(item, FileBackedData)
362-
item._move_to(outfile, offset)
363-
moved.append(f"{item.path} ({len(item)} bytes) at offset {offset}")
395+
reason = item._move_to(outfile, offset, moved)
396+
if reason is None:
397+
done.append(
398+
f"{item.path} ({len(item)} bytes) at offset {offset}"
399+
)
400+
else:
401+
copied[reason] += 1
402+
outfile.seek(offset)
403+
_write_buffer(item, outfile)
364404
outfile.seek(end)
405+
# A buffer copied after the last move would not be on disk yet.
406+
outfile.flush()
407+
_sync(outfile.fileno())
365408
except BaseException as e:
366-
if moved:
409+
if done:
367410
e.add_note(
368-
"Moved into the output before the failure: " + "; ".join(moved)
411+
"Moved into the output before the failure: " + "; ".join(done)
369412
)
370413
raise
371414

‎exir/_serialize/test/test_cord.py‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import tempfile
1616
import unittest
1717
import weakref
18+
from collections import Counter
1819
from typing import Iterable
1920
from unittest import mock
2021

@@ -624,15 +625,32 @@ def test_a_move_raises_if_another_save_is_moving_the_file(self) -> None:
624625
self.assertEqual(self._PAYLOAD, data.to_bytes())
625626

626627
@_needs_move
627-
def test_a_move_raises_if_the_file_changed_after_it_was_checked(self) -> None:
628+
def test_a_move_copies_a_file_that_changed_after_it_was_checked(self) -> None:
628629
with tempfile.TemporaryDirectory() as directory:
629630
data = self._owned(directory)
630-
with open(os.path.join(directory, "out.bin"), "wb") as outfile:
631+
out_path = os.path.join(directory, "out.bin")
632+
moved, copied, calls = [], Counter(), []
633+
sync = _cord._sync
634+
635+
class Recording(io.FileIO):
636+
def write(self, b) -> int:
637+
calls.append("write")
638+
return super().write(b)
639+
640+
def recording_sync(fd: int) -> None:
641+
sync(fd)
642+
calls.append("sync")
643+
644+
with Recording(out_path, "wb") as outfile:
631645
self.assertIsNone(data._move_reason(os.fstat(outfile.fileno()).st_dev))
632646
os.link(data.path, os.path.join(directory, "other_name.bin"))
633-
with self.assertRaisesRegex(RuntimeError, "with another hard link"):
634-
Cord(data)._write_to_file(outfile, {data})
635-
self.assertEqual(self._PAYLOAD, data.to_bytes())
647+
with mock.patch.object(_cord, "_sync", recording_sync):
648+
Cord(data)._write_to_file(outfile, {data}, moved, copied)
649+
self.assertEqual([], moved)
650+
self.assertEqual({"with another hard link": 1}, dict(copied))
651+
self._assert_copied(data, out_path)
652+
# The copy is on disk before the caller renames the output into place.
653+
self.assertEqual(["write", "sync"], calls[-2:])
636654

637655
@unittest.skipIf(sys.platform == "win32", "no fcntl on Windows")
638656
def test_a_sync_on_macos_flushes_the_drive_cache(self) -> None:

0 commit comments

Comments
 (0)