diff --git a/CHANGELOG.md b/CHANGELOG.md index abd95b0f..ff18e14b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Use faster gzip compression for private checkpoints, trading larger transfers for less packing CPU while preserving full authenticated database validation. + - Avoid redundant checkpoint packing reads by hashing the existing byte streams, and separate transfer and restore verification timings while retaining full authenticated disk and SQLite checks. - Pin the reviewed private acquisition policy for manual runs and prepare member refresh within the existing shared acquisition deadline; automatic cadence remains disabled pending measured publication qualification. diff --git a/docs/guides/private-checkpoints.md b/docs/guides/private-checkpoints.md index 92de4ffe..3d65df21 100644 --- a/docs/guides/private-checkpoints.md +++ b/docs/guides/private-checkpoints.md @@ -41,9 +41,11 @@ readback use that deadline. The intermediate backup is not scanned again: its hash and pinned source facts are both checked against the downloaded image. Packing carries the database hash from its raw input traversal and compressed part hashes from successful output writes, including the gzip trailer. It does -not reread those local files only to hash them. The authenticated restored image -still receives an independent disk hash and full SQLite checks. Fixed phase -names and elapsed times separate download, reconstruction, disk verification +not reread those local files only to hash them. Gzip level 1 favors compression +speed over payload size; existing higher-compression checkpoints remain readable. +The authenticated restored image still receives an independent disk hash and +full SQLite checks. Fixed phase names and elapsed times separate download, +reconstruction, disk verification and SQLite verification without exposing data. These reductions do not establish publication headroom until a complete run is measured. Promotion requires at least one minute remaining. Deadline checks are cooperative diff --git a/scripts/private_discord_checkpoint.py b/scripts/private_discord_checkpoint.py index 9a66308b..928fc4ef 100644 --- a/scripts/private_discord_checkpoint.py +++ b/scripts/private_discord_checkpoint.py @@ -216,7 +216,7 @@ def pack(database, directory, deadline=None): raw_bytes = 0 with target.open("xb") as raw: sink = HashingWriter(raw, deadline) - with gzip.GzipFile(filename="", mode="wb", fileobj=sink, mtime=0) as compressed: + with gzip.GzipFile(filename="", mode="wb", fileobj=sink, mtime=0, compresslevel=1) as compressed: while raw_bytes < RAW_PART: check_deadline(deadline) chunk = source.read(min(BLOCK, RAW_PART - raw_bytes)) diff --git a/scripts/test_private_discord_checkpoint_save.py b/scripts/test_private_discord_checkpoint_save.py index b2af0407..27a699f4 100644 --- a/scripts/test_private_discord_checkpoint_save.py +++ b/scripts/test_private_discord_checkpoint_save.py @@ -116,7 +116,7 @@ def test_phase_output_contains_only_fixed_labels_and_elapsed_time(self): with checkpoint.checkpoint_phase("private-secret"): pass - def test_pack_carries_exact_disk_hashes_and_unchanged_gzip_bytes_without_rereads(self): + def test_fast_pack_carries_exact_disk_hashes_and_chunk_bytes_without_rereads(self): for size in (0, 1, 17, 18, 34, 39): with self.subTest(size=size): directory = self.root / str(size) @@ -132,7 +132,7 @@ def test_pack_carries_exact_disk_hashes_and_unchanged_gzip_bytes_without_rereads for index, part in enumerate(parts): expected = io.BytesIO() chunk = data[index * 17:(index + 1) * 17] - with gzip.GzipFile(filename="", mode="wb", fileobj=expected, mtime=0) as compressed: + with gzip.GzipFile(filename="", mode="wb", fileobj=expected, mtime=0, compresslevel=1) as compressed: for start in range(0, len(chunk), 5): compressed.write(chunk[start:start + 5]) actual = (directory / part["name"]).read_bytes() @@ -141,6 +141,35 @@ def test_pack_carries_exact_disk_hashes_and_unchanged_gzip_bytes_without_rereads self.assertEqual(part, {"name": f"state-{index:03d}.sqlite.gz", "bytes": len(actual), "raw_bytes": len(chunk), "sha256": checkpoint.digest(actual)}) + def test_authenticated_restore_accepts_old_and_fast_gzip_with_identical_native_state(self): + self.seed() + release = self.github.releases[self.github.latest] + assets = {asset["name"]: asset for asset in release["assets"]} + manifest = checkpoint.strict_json(self.github.payloads[assets[checkpoint.MANIFEST]["id"]]) + expected = manifest["database"].copy() + for level in (9, 1): + with self.subTest(level=level): + for part in manifest["parts"]: + asset = assets[part["name"]] + raw = gzip.decompress(self.github.payloads[asset["id"]]) + output = io.BytesIO() + with gzip.GzipFile(filename="", mode="wb", fileobj=output, mtime=0, compresslevel=level) as compressed: + compressed.write(raw) + body = output.getvalue() + self.github.payloads[asset["id"]] = body + part.update(bytes=len(body), sha256=checkpoint.digest(body)) + asset.update(size=len(body), digest="sha256:" + part["sha256"]) + body = checkpoint.canonical(manifest) + asset = assets[checkpoint.MANIFEST] + self.github.payloads[asset["id"]] = body + asset.update(size=len(body), digest="sha256:" + checkpoint.digest(body)) + target = self.root / f"level-{level}.sqlite" + checkpoint.restore(self.github, target, self.receipt) + self.assertEqual(checkpoint.file_digest(target), expected["sha256"]) + with contextlib.closing(checkpoint.readonly(target)) as db: + self.assertEqual(checkpoint.database_facts(db), + {key: expected[key] for key in ("user_version", "schema_sha256", "tables")}) + def test_short_writes_include_header_flush_and_trailer_in_carried_hash(self): class ShortWriter(io.BytesIO): def write(self, data):