From cce528b55261c5664f823dcc37bbf1851613dd81 Mon Sep 17 00:00:00 2001 From: AIwork4me Date: Sat, 18 Jul 2026 10:31:30 +0000 Subject: [PATCH] feat(cli): hunyuan-ocr benchmark + report subcommands (P1) benchmark: print the verified results from the lock (read-only), sharing the renderer with scripts/render_benchmark_tables.py (single source). report: assemble a benchmark release-artifact bundle (run_manifest + environment + commands + lock + checksums) per docs/release-artifact.md. Both are pure package code, CPU-tested. Co-Authored-By: Claude --- scripts/render_benchmark_tables.py | 42 ++------------- src/hunyuan_ocr/cli.py | 41 +++++++++++++++ src/hunyuan_ocr/report.py | 76 +++++++++++++++++++++++++++ src/hunyuan_ocr/results.py | 49 +++++++++++++++++ tests/test_cli_extras.py | 84 ++++++++++++++++++++++++++++++ 5 files changed, 255 insertions(+), 37 deletions(-) create mode 100644 src/hunyuan_ocr/report.py create mode 100644 src/hunyuan_ocr/results.py create mode 100644 tests/test_cli_extras.py diff --git a/scripts/render_benchmark_tables.py b/scripts/render_benchmark_tables.py index bfce2ad..f10167b 100644 --- a/scripts/render_benchmark_tables.py +++ b/scripts/render_benchmark_tables.py @@ -19,48 +19,16 @@ import yaml +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from hunyuan_ocr.results import render_results_block # noqa: E402 + REPO = Path(__file__).resolve().parents[1] LOCK_PATH = REPO / "reproducibility.lock.yaml" README_PATH = REPO / "README.md" REGION_RE = re.compile(r".*?", re.DOTALL) -BACKEND_DISPLAY = {"llamacpp": "llama.cpp", "transformers": "transformers", "vllm": "vLLM"} - - -def _fmt_value(v) -> str: - if isinstance(v, str) and v.strip().lower() == "invalid": - return "invalid (excluded; see reproducibility.lock.yaml)" - return str(v) - - -def render_block(lock: dict) -> str: - """Render the BEGIN..END block (without surrounding README text) from the lock.""" - bench = (lock or {}).get("benchmark", {}) or {} - lines = [ - "", - "", - "", - "| Page set | Backend | Overall | Source |", - "|---|---|---|---|", - ] - for page_key, label in (("canary_148", "canary 148"), ("full_1651", "full 1651")): - section = bench.get(page_key, {}) or {} - rows = [] - for k, v in section.items(): - if not k.endswith("_overall"): - continue - backend = k[: -len("_overall")] - rows.append((BACKEND_DISPLAY.get(backend, backend), _fmt_value(v))) - for display, value in sorted(rows): - lines.append(f"| {label} | {display} | {value} | reproducibility.lock.yaml |") - official = bench.get("official_reference", {}) or {} - if official: - engine = official.get("inference_engine", "official") - overall = _fmt_value(official.get("omnidocbench_overall")) - lines.append(f"| official | {engine} | {overall} | official HunyuanOCR table |") - lines.append("") - lines.append("") - return "\n".join(lines) +# Shared with the `hunyuan-ocr benchmark` CLI via hunyuan_ocr.results. +render_block = render_results_block def current_region(readme: str) -> str | None: diff --git a/src/hunyuan_ocr/cli.py b/src/hunyuan_ocr/cli.py index 89712f0..4ed3abc 100644 --- a/src/hunyuan_ocr/cli.py +++ b/src/hunyuan_ocr/cli.py @@ -9,6 +9,8 @@ canary materialize— rebuild the 148-page canary from the full GT predict — multi-server predict via hunyuan_ocr.driver (llamacpp/vllm/openai) score — OmniDocBench scoring via hunyuan_ocr.scoring (needs the scorer venv) + benchmark — print the verified results from reproducibility.lock.yaml (read-only) + report — assemble a benchmark release-artifact bundle from a run_manifest.json ``predict --backend transformers`` is the one exception: it still delegates to the repo-only ``scripts/run_phase1_transformers.py`` driver and needs a ROCm torch. @@ -387,6 +389,35 @@ def _clean_extra(extra): return extra +def _benchmark(args) -> int: + """Print the verified benchmark results from reproducibility.lock.yaml (read-only).""" + import yaml + + from hunyuan_ocr.results import render_results_block + + lock_path = Path(args.lock) if getattr(args, "lock", None) else Path.cwd() / "reproducibility.lock.yaml" + if not lock_path.is_file(): + print(f"[error] lock not found: {lock_path}", file=sys.stderr) + return 2 + lock = yaml.safe_load(lock_path.read_text(encoding="utf-8")) + print(render_results_block(lock)) + return 0 + + +def _report(args) -> int: + """Assemble a benchmark release-artifact bundle (see docs/release-artifact.md).""" + from hunyuan_ocr.report import assemble_release_artifact + + repo_root = Path(args.repo_root) if getattr(args, "repo_root", None) else Path(__file__).resolve().parents[2] + try: + out = assemble_release_artifact(args.pred_dir, args.out, repo_root) + except FileNotFoundError as exc: + print(f"[error] {exc}", file=sys.stderr) + return 2 + print(f"[OK] wrote release artifact -> {out} (run_manifest, environment, commands, lock, checksums)") + return 0 + + def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( prog="hunyuan-ocr", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -437,6 +468,14 @@ def build_parser() -> argparse.ArgumentParser: sc.add_argument("--omnidocbench-repo", default=scoring.DEFAULT_OMNIDOCBENCH_REPO) sc.add_argument("--venv-python", default=scoring.DEFAULT_VENV_PYTHON) sc.add_argument("--skip-validation", action="store_true", help="DANGEROUS: bypass pre-score validation") + + bm = sub.add_parser("benchmark", help="print the verified results from the lock (read-only)") + bm.add_argument("--lock", help="path to reproducibility.lock.yaml (default: ./reproducibility.lock.yaml)") + + rep = sub.add_parser("report", help="assemble a benchmark release-artifact bundle") + rep.add_argument("--pred-dir", required=True, help="prediction dir containing run_manifest.json") + rep.add_argument("--out", required=True, help="output artifact directory") + rep.add_argument("--repo-root", help="repo root (to copy reproducibility.lock.yaml); default: this package's repo") return p @@ -449,6 +488,8 @@ def main(argv=None) -> int: "canary": lambda a: _canary_materialize(a) if a.ccmd == "materialize" else 2, "predict": _predict, "score": _score, + "benchmark": _benchmark, + "report": _report, } handler = dispatch[args.cmd] rc = handler(args) diff --git a/src/hunyuan_ocr/report.py b/src/hunyuan_ocr/report.py new file mode 100644 index 0000000..0fb20f1 --- /dev/null +++ b/src/hunyuan_ocr/report.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 AIwork4me +"""Assemble a benchmark release-artifact bundle from a run manifest + the lock. + +See docs/release-artifact.md for the layout. This module is pure filesystem + +stdlib (no GPU, no scorer); it packages the reproducibility evidence for a run: +the manifest, its environment + command, the lock, and a tamper-evident checksum +file. Metrics from the scorer are intentionally NOT produced here — run the +scorer separately and drop its output into the bundle. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + + +def _sha256(path: Path) -> str: + h = hashlib.sha256() + h.update(path.read_bytes()) + return h.hexdigest() + + +def assemble_release_artifact(pred_dir, out_dir, repo_root) -> Path: + """Build the bundle at ``out_dir`` from ``pred_dir/run_manifest.json`` + the + repo's ``reproducibility.lock.yaml``. Returns the output directory.""" + pred_dir = Path(pred_dir) + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + manifest_path = pred_dir / "run_manifest.json" + if not manifest_path.is_file(): + raise FileNotFoundError(f"no run_manifest.json in {pred_dir}") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + # run_manifest.json (verbatim copy) + (out_dir / "run_manifest.json").write_text(manifest_path.read_text(encoding="utf-8"), encoding="utf-8") + + # environment.json (best-effort env + platform from the manifest) + (out_dir / "environment.json").write_text( + json.dumps({"env": manifest.get("env", {}), "platform": manifest.get("platform", {})}, indent=2), + encoding="utf-8", + ) + + # commands.txt (redacted argv already stored in the manifest) + cmd = manifest.get("command") + cmd_text = " ".join(cmd) + "\n" if isinstance(cmd, list) else str(cmd) + "\n" + (out_dir / "commands.txt").write_text(cmd_text, encoding="utf-8") + + # reproducibility.lock.yaml (copy from the repo) + lock_src = Path(repo_root) / "reproducibility.lock.yaml" + if lock_src.is_file(): + (out_dir / "reproducibility.lock.yaml").write_text(lock_src.read_text(encoding="utf-8"), encoding="utf-8") + + # README.md describing the bundle + backend = manifest.get("backend", "?") + status = manifest.get("status", "?") + sha = manifest.get("repo_commit") or "?" + (out_dir / "README.md").write_text( + "# Benchmark release artifact\n\n" + f"- backend: `{backend}`\n- status: `{status}`\n- repo commit: `{sha}`\n\n" + "Reproduce by checking out the pinned commits + weights in " + "`reproducibility.lock.yaml`, then running the commands in `commands.txt`.\n" + "Verify integrity with `sha256sum -c checksums.sha256`.\n", + encoding="utf-8", + ) + + # checksums.sha256 (over every other file in the bundle) + lines = [] + for f in sorted(out_dir.iterdir()): + if f.name == "checksums.sha256": + continue + lines.append(f"{_sha256(f)} {f.name}") + (out_dir / "checksums.sha256").write_text("\n".join(lines) + "\n", encoding="utf-8") + return out_dir diff --git a/src/hunyuan_ocr/results.py b/src/hunyuan_ocr/results.py new file mode 100644 index 0000000..6052d0e --- /dev/null +++ b/src/hunyuan_ocr/results.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 AIwork4me +"""Render the verified benchmark results from ``reproducibility.lock.yaml``. + +Single source of truth for the headline numbers: the lock. Both the README +generator (``scripts/render_benchmark_tables.py``) and the ``hunyuan-ocr +benchmark`` CLI render through this module, so they can never disagree. No number +is invented — only values present in the lock are emitted. +""" + +from __future__ import annotations + +BACKEND_DISPLAY = {"llamacpp": "llama.cpp", "transformers": "transformers", "vllm": "vLLM"} + + +def _fmt_value(v) -> str: + if isinstance(v, str) and v.strip().lower() == "invalid": + return "invalid (excluded; see reproducibility.lock.yaml)" + return str(v) + + +def render_results_block(lock: dict) -> str: + """Render the BEGIN..END GENERATED RESULTS block (markdown) from the lock.""" + bench = (lock or {}).get("benchmark", {}) or {} + lines = [ + "", + "", + "", + "| Page set | Backend | Overall | Source |", + "|---|---|---|---|", + ] + for page_key, label in (("canary_148", "canary 148"), ("full_1651", "full 1651")): + section = bench.get(page_key, {}) or {} + rows = [] + for k, v in section.items(): + if not k.endswith("_overall"): + continue + backend = k[: -len("_overall")] + rows.append((BACKEND_DISPLAY.get(backend, backend), _fmt_value(v))) + for display, value in sorted(rows): + lines.append(f"| {label} | {display} | {value} | reproducibility.lock.yaml |") + official = bench.get("official_reference", {}) or {} + if official: + engine = official.get("inference_engine", "official") + overall = _fmt_value(official.get("omnidocbench_overall")) + lines.append(f"| official | {engine} | {overall} | official HunyuanOCR table |") + lines.append("") + lines.append("") + return "\n".join(lines) diff --git a/tests/test_cli_extras.py b/tests/test_cli_extras.py new file mode 100644 index 0000000..46b566a --- /dev/null +++ b/tests/test_cli_extras.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 AIwork4me +"""CPU tests for the hunyuan-ocr benchmark + report subcommands (no network/GPU).""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from hunyuan_ocr import cli + + +# --- benchmark --------------------------------------------------------------- + + +def test_benchmark_prints_lock_results(tmp_path, capsys): + lock = tmp_path / "reproducibility.lock.yaml" + lock.write_text( + "benchmark:\n canary_148:\n vllm_overall: 94.81\n llamacpp_overall: 93.33\n", + encoding="utf-8", + ) + rc = cli.main(["benchmark", "--lock", str(lock)]) + assert rc == 0 + out = capsys.readouterr().out + assert "94.81" in out and "93.33" in out and "BEGIN GENERATED RESULTS" in out + + +def test_benchmark_missing_lock(tmp_path): + rc = cli.main(["benchmark", "--lock", str(tmp_path / "nope.yaml")]) + assert rc == 2 + + +# --- report ------------------------------------------------------------------ + + +def _write_manifest(pred_dir: Path): + manifest = { + "schema_version": 2, + "repo_commit": "abc123", + "backend": "llamacpp", + "model": "HYVL", + "timestamp_iso": "2026-07-18T03:00:00Z", + "status": "ok", + "run_counts": {"attempted": 1, "succeeded": 1, "failed": 0, "skipped": 0, "interrupted": 0}, + "final_state": {"expected": 1, "complete": 1, "failed": 0, "pending": 0}, + "command": ["run_inference.py", "--backend-name", "llamacpp"], + "env": {"torch": "2.9.1"}, + "platform": {"python": "3.12.3"}, + } + pred_dir.mkdir(parents=True, exist_ok=True) + (pred_dir / "run_manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + return manifest + + +def test_report_assembles_bundle_with_checksums(tmp_path): + pred = tmp_path / "pred" + _write_manifest(pred) + repo_root = tmp_path / "repo" + repo_root.mkdir() + (repo_root / "reproducibility.lock.yaml").write_text("hunyuanocr_rocm:\n commit: x\n", encoding="utf-8") + out = tmp_path / "artifact" + + rc = cli.main(["report", "--pred-dir", str(pred), "--out", str(out), "--repo-root", str(repo_root)]) + assert rc == 0 + + assert (out / "run_manifest.json").is_file() + assert (out / "environment.json").is_file() + assert (out / "commands.txt").is_file() + assert (out / "reproducibility.lock.yaml").is_file() + assert (out / "README.md").is_file() + # checksums cover every other file and verify + sums = (out / "checksums.sha256").read_text(encoding="utf-8").strip().splitlines() + assert len(sums) == 5 + for line in sums: + digest, name = line.split(" ", 1) + assert hashlib.sha256((out / name).read_bytes()).hexdigest() == digest + + +def test_report_missing_manifest(tmp_path): + pred = tmp_path / "empty" + pred.mkdir() + rc = cli.main(["report", "--pred-dir", str(pred), "--out", str(tmp_path / "o"), "--repo-root", str(tmp_path)]) + assert rc == 2