Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

## Unreleased

- Use faster gzip compression for private checkpoints, trading larger transfers for less packing CPU while preserving full authenticated database validation.

- Avoid redundant checkpoint packing reads by hashing the existing byte streams, and separate transfer and restore verification timings while retaining full authenticated disk and SQLite checks.

- Pin the reviewed private acquisition policy for manual runs and prepare member refresh within the existing shared acquisition deadline; automatic cadence remains disabled pending measured publication qualification.
Expand Down
8 changes: 5 additions & 3 deletions docs/guides/private-checkpoints.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,11 @@ readback use that deadline. The intermediate backup is not scanned again: its
hash and pinned source facts are both checked against the downloaded image.
Packing carries the database hash from its raw input traversal and compressed
part hashes from successful output writes, including the gzip trailer. It does
not reread those local files only to hash them. The authenticated restored image
still receives an independent disk hash and full SQLite checks. Fixed phase
names and elapsed times separate download, reconstruction, disk verification
not reread those local files only to hash them. Gzip level 1 favors compression
speed over payload size; existing higher-compression checkpoints remain readable.
The authenticated restored image still receives an independent disk hash and
full SQLite checks. Fixed phase names and elapsed times separate download,
reconstruction, disk verification
and SQLite verification without exposing data. These reductions do not establish
publication headroom until a complete run is measured.
Promotion requires at least one minute remaining. Deadline checks are cooperative
Expand Down
2 changes: 1 addition & 1 deletion scripts/private_discord_checkpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ def pack(database, directory, deadline=None):
raw_bytes = 0
with target.open("xb") as raw:
sink = HashingWriter(raw, deadline)
with gzip.GzipFile(filename="", mode="wb", fileobj=sink, mtime=0) as compressed:
with gzip.GzipFile(filename="", mode="wb", fileobj=sink, mtime=0, compresslevel=1) as compressed:
while raw_bytes < RAW_PART:
check_deadline(deadline)
chunk = source.read(min(BLOCK, RAW_PART - raw_bytes))
Expand Down
33 changes: 31 additions & 2 deletions scripts/test_private_discord_checkpoint_save.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def test_phase_output_contains_only_fixed_labels_and_elapsed_time(self):
with checkpoint.checkpoint_phase("private-secret"):
pass

def test_pack_carries_exact_disk_hashes_and_unchanged_gzip_bytes_without_rereads(self):
def test_fast_pack_carries_exact_disk_hashes_and_chunk_bytes_without_rereads(self):
for size in (0, 1, 17, 18, 34, 39):
with self.subTest(size=size):
directory = self.root / str(size)
Expand All @@ -132,7 +132,7 @@ def test_pack_carries_exact_disk_hashes_and_unchanged_gzip_bytes_without_rereads
for index, part in enumerate(parts):
expected = io.BytesIO()
chunk = data[index * 17:(index + 1) * 17]
with gzip.GzipFile(filename="", mode="wb", fileobj=expected, mtime=0) as compressed:
with gzip.GzipFile(filename="", mode="wb", fileobj=expected, mtime=0, compresslevel=1) as compressed:
for start in range(0, len(chunk), 5):
compressed.write(chunk[start:start + 5])
actual = (directory / part["name"]).read_bytes()
Expand All @@ -141,6 +141,35 @@ def test_pack_carries_exact_disk_hashes_and_unchanged_gzip_bytes_without_rereads
self.assertEqual(part, {"name": f"state-{index:03d}.sqlite.gz", "bytes": len(actual),
"raw_bytes": len(chunk), "sha256": checkpoint.digest(actual)})

def test_authenticated_restore_accepts_old_and_fast_gzip_with_identical_native_state(self):
self.seed()
release = self.github.releases[self.github.latest]
assets = {asset["name"]: asset for asset in release["assets"]}
manifest = checkpoint.strict_json(self.github.payloads[assets[checkpoint.MANIFEST]["id"]])
expected = manifest["database"].copy()
for level in (9, 1):
with self.subTest(level=level):
for part in manifest["parts"]:
asset = assets[part["name"]]
raw = gzip.decompress(self.github.payloads[asset["id"]])
output = io.BytesIO()
with gzip.GzipFile(filename="", mode="wb", fileobj=output, mtime=0, compresslevel=level) as compressed:
compressed.write(raw)
body = output.getvalue()
self.github.payloads[asset["id"]] = body
part.update(bytes=len(body), sha256=checkpoint.digest(body))
asset.update(size=len(body), digest="sha256:" + part["sha256"])
body = checkpoint.canonical(manifest)
asset = assets[checkpoint.MANIFEST]
self.github.payloads[asset["id"]] = body
asset.update(size=len(body), digest="sha256:" + checkpoint.digest(body))
target = self.root / f"level-{level}.sqlite"
checkpoint.restore(self.github, target, self.receipt)
self.assertEqual(checkpoint.file_digest(target), expected["sha256"])
with contextlib.closing(checkpoint.readonly(target)) as db:
self.assertEqual(checkpoint.database_facts(db),
{key: expected[key] for key in ("user_version", "schema_sha256", "tables")})

def test_short_writes_include_header_flush_and_trailer_in_carried_hash(self):
class ShortWriter(io.BytesIO):
def write(self, data):
Expand Down
Loading