feat(cli): hunyuan-ocr benchmark + report (P1)

scripts/render_benchmark_tables.py

@@ -19,48 +19,16 @@
 
 import yaml
 
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+from hunyuan_ocr.results import render_results_block  # noqa: E402
+
 REPO = Path(__file__).resolve().parents[1]
 LOCK_PATH = REPO / "reproducibility.lock.yaml"
 README_PATH = REPO / "README.md"
 REGION_RE = re.compile(r"<!-- BEGIN GENERATED RESULTS -->.*?<!-- END GENERATED RESULTS -->", re.DOTALL)
 
-BACKEND_DISPLAY = {"llamacpp": "llama.cpp", "transformers": "transformers", "vllm": "vLLM"}
-
-
-def _fmt_value(v) -> str:
-    if isinstance(v, str) and v.strip().lower() == "invalid":
-        return "invalid (excluded; see reproducibility.lock.yaml)"
-    return str(v)
-
-
-def render_block(lock: dict) -> str:
-    """Render the BEGIN..END block (without surrounding README text) from the lock."""
-    bench = (lock or {}).get("benchmark", {}) or {}
-    lines = [
-        "<!-- BEGIN GENERATED RESULTS -->",
-        "<!-- Auto-generated from reproducibility.lock.yaml by scripts/render_benchmark_tables.py (do not edit by hand). -->",
-        "",
-        "| Page set | Backend | Overall | Source |",
-        "|---|---|---|---|",
-    ]
-    for page_key, label in (("canary_148", "canary 148"), ("full_1651", "full 1651")):
-        section = bench.get(page_key, {}) or {}
-        rows = []
-        for k, v in section.items():
-            if not k.endswith("_overall"):
-                continue
-            backend = k[: -len("_overall")]
-            rows.append((BACKEND_DISPLAY.get(backend, backend), _fmt_value(v)))
-        for display, value in sorted(rows):
-            lines.append(f"| {label} | {display} | {value} | reproducibility.lock.yaml |")
-    official = bench.get("official_reference", {}) or {}
-    if official:
-        engine = official.get("inference_engine", "official")
-        overall = _fmt_value(official.get("omnidocbench_overall"))
-        lines.append(f"| official | {engine} | {overall} | official HunyuanOCR table |")
-    lines.append("")
-    lines.append("<!-- END GENERATED RESULTS -->")
-    return "\n".join(lines)
+# Shared with the `hunyuan-ocr benchmark` CLI via hunyuan_ocr.results.
+render_block = render_results_block
 
 
 def current_region(readme: str) -> str | None:

src/hunyuan_ocr/cli.py

@@ -9,6 +9,8 @@
   canary materialize— rebuild the 148-page canary from the full GT
   predict           — multi-server predict via hunyuan_ocr.driver (llamacpp/vllm/openai)
   score             — OmniDocBench scoring via hunyuan_ocr.scoring (needs the scorer venv)
+  benchmark         — print the verified results from reproducibility.lock.yaml (read-only)
+  report            — assemble a benchmark release-artifact bundle from a run_manifest.json
 
 ``predict --backend transformers`` is the one exception: it still delegates to the
 repo-only ``scripts/run_phase1_transformers.py`` driver and needs a ROCm torch.
@@ -387,6 +389,35 @@ def _clean_extra(extra):
     return extra
 
 
+def _benchmark(args) -> int:
+    """Print the verified benchmark results from reproducibility.lock.yaml (read-only)."""
+    import yaml
+
+    from hunyuan_ocr.results import render_results_block
+
+    lock_path = Path(args.lock) if getattr(args, "lock", None) else Path.cwd() / "reproducibility.lock.yaml"
+    if not lock_path.is_file():
+        print(f"[error] lock not found: {lock_path}", file=sys.stderr)
+        return 2
+    lock = yaml.safe_load(lock_path.read_text(encoding="utf-8"))
+    print(render_results_block(lock))
+    return 0
+
+
+def _report(args) -> int:
+    """Assemble a benchmark release-artifact bundle (see docs/release-artifact.md)."""
+    from hunyuan_ocr.report import assemble_release_artifact
+
+    repo_root = Path(args.repo_root) if getattr(args, "repo_root", None) else Path(__file__).resolve().parents[2]
+    try:
+        out = assemble_release_artifact(args.pred_dir, args.out, repo_root)
+    except FileNotFoundError as exc:
+        print(f"[error] {exc}", file=sys.stderr)
+        return 2
+    print(f"[OK] wrote release artifact -> {out} (run_manifest, environment, commands, lock, checksums)")
+    return 0
+
+
 def build_parser() -> argparse.ArgumentParser:
     p = argparse.ArgumentParser(
         prog="hunyuan-ocr", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
@@ -437,6 +468,14 @@ def build_parser() -> argparse.ArgumentParser:
     sc.add_argument("--omnidocbench-repo", default=scoring.DEFAULT_OMNIDOCBENCH_REPO)
     sc.add_argument("--venv-python", default=scoring.DEFAULT_VENV_PYTHON)
     sc.add_argument("--skip-validation", action="store_true", help="DANGEROUS: bypass pre-score validation")
+
+    bm = sub.add_parser("benchmark", help="print the verified results from the lock (read-only)")
+    bm.add_argument("--lock", help="path to reproducibility.lock.yaml (default: ./reproducibility.lock.yaml)")
+
+    rep = sub.add_parser("report", help="assemble a benchmark release-artifact bundle")
+    rep.add_argument("--pred-dir", required=True, help="prediction dir containing run_manifest.json")
+    rep.add_argument("--out", required=True, help="output artifact directory")
+    rep.add_argument("--repo-root", help="repo root (to copy reproducibility.lock.yaml); default: this package's repo")
     return p
 
 
@@ -449,6 +488,8 @@ def main(argv=None) -> int:
         "canary": lambda a: _canary_materialize(a) if a.ccmd == "materialize" else 2,
         "predict": _predict,
         "score": _score,
+        "benchmark": _benchmark,
+        "report": _report,
     }
     handler = dispatch[args.cmd]
     rc = handler(args)

src/hunyuan_ocr/report.py

@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2026 AIwork4me
+"""Assemble a benchmark release-artifact bundle from a run manifest + the lock.
+
+See docs/release-artifact.md for the layout. This module is pure filesystem +
+stdlib (no GPU, no scorer); it packages the reproducibility evidence for a run:
+the manifest, its environment + command, the lock, and a tamper-evident checksum
+file. Metrics from the scorer are intentionally NOT produced here — run the
+scorer separately and drop its output into the bundle.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+
+def _sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    h.update(path.read_bytes())
+    return h.hexdigest()
+
+
+def assemble_release_artifact(pred_dir, out_dir, repo_root) -> Path:
+    """Build the bundle at ``out_dir`` from ``pred_dir/run_manifest.json`` + the
+    repo's ``reproducibility.lock.yaml``. Returns the output directory."""
+    pred_dir = Path(pred_dir)
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    manifest_path = pred_dir / "run_manifest.json"
+    if not manifest_path.is_file():
+        raise FileNotFoundError(f"no run_manifest.json in {pred_dir}")
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+
+    # run_manifest.json (verbatim copy)
+    (out_dir / "run_manifest.json").write_text(manifest_path.read_text(encoding="utf-8"), encoding="utf-8")
+
+    # environment.json (best-effort env + platform from the manifest)
+    (out_dir / "environment.json").write_text(
+        json.dumps({"env": manifest.get("env", {}), "platform": manifest.get("platform", {})}, indent=2),
+        encoding="utf-8",
+    )
+
+    # commands.txt (redacted argv already stored in the manifest)
+    cmd = manifest.get("command")
+    cmd_text = " ".join(cmd) + "\n" if isinstance(cmd, list) else str(cmd) + "\n"
+    (out_dir / "commands.txt").write_text(cmd_text, encoding="utf-8")
+
+    # reproducibility.lock.yaml (copy from the repo)
+    lock_src = Path(repo_root) / "reproducibility.lock.yaml"
+    if lock_src.is_file():
+        (out_dir / "reproducibility.lock.yaml").write_text(lock_src.read_text(encoding="utf-8"), encoding="utf-8")
+
+    # README.md describing the bundle
+    backend = manifest.get("backend", "?")
+    status = manifest.get("status", "?")
+    sha = manifest.get("repo_commit") or "?"
+    (out_dir / "README.md").write_text(
+        "# Benchmark release artifact\n\n"
+        f"- backend: `{backend}`\n- status: `{status}`\n- repo commit: `{sha}`\n\n"
+        "Reproduce by checking out the pinned commits + weights in "
+        "`reproducibility.lock.yaml`, then running the commands in `commands.txt`.\n"
+        "Verify integrity with `sha256sum -c checksums.sha256`.\n",
+        encoding="utf-8",
+    )
+
+    # checksums.sha256 (over every other file in the bundle)
+    lines = []
+    for f in sorted(out_dir.iterdir()):
+        if f.name == "checksums.sha256":
+            continue
+        lines.append(f"{_sha256(f)}  {f.name}")
+    (out_dir / "checksums.sha256").write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return out_dir

src/hunyuan_ocr/results.py

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2026 AIwork4me
+"""Render the verified benchmark results from ``reproducibility.lock.yaml``.
+
+Single source of truth for the headline numbers: the lock. Both the README
+generator (``scripts/render_benchmark_tables.py``) and the ``hunyuan-ocr
+benchmark`` CLI render through this module, so they can never disagree. No number
+is invented — only values present in the lock are emitted.
+"""
+
+from __future__ import annotations
+
+BACKEND_DISPLAY = {"llamacpp": "llama.cpp", "transformers": "transformers", "vllm": "vLLM"}
+
+
+def _fmt_value(v) -> str:
+    if isinstance(v, str) and v.strip().lower() == "invalid":
+        return "invalid (excluded; see reproducibility.lock.yaml)"
+    return str(v)
+
+
+def render_results_block(lock: dict) -> str:
+    """Render the BEGIN..END GENERATED RESULTS block (markdown) from the lock."""
+    bench = (lock or {}).get("benchmark", {}) or {}
+    lines = [
+        "<!-- BEGIN GENERATED RESULTS -->",
+        "<!-- Auto-generated from reproducibility.lock.yaml by scripts/render_benchmark_tables.py (do not edit by hand). -->",
+        "",
+        "| Page set | Backend | Overall | Source |",
+        "|---|---|---|---|",
+    ]
+    for page_key, label in (("canary_148", "canary 148"), ("full_1651", "full 1651")):
+        section = bench.get(page_key, {}) or {}
+        rows = []
+        for k, v in section.items():
+            if not k.endswith("_overall"):
+                continue
+            backend = k[: -len("_overall")]
+            rows.append((BACKEND_DISPLAY.get(backend, backend), _fmt_value(v)))
+        for display, value in sorted(rows):
+            lines.append(f"| {label} | {display} | {value} | reproducibility.lock.yaml |")
+    official = bench.get("official_reference", {}) or {}
+    if official:
+        engine = official.get("inference_engine", "official")
+        overall = _fmt_value(official.get("omnidocbench_overall"))
+        lines.append(f"| official | {engine} | {overall} | official HunyuanOCR table |")
+    lines.append("")
+    lines.append("<!-- END GENERATED RESULTS -->")
+    return "\n".join(lines)

tests/test_cli_extras.py

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2026 AIwork4me
+"""CPU tests for the hunyuan-ocr benchmark + report subcommands (no network/GPU)."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+from hunyuan_ocr import cli
+
+
+# --- benchmark ---------------------------------------------------------------
+
+
+def test_benchmark_prints_lock_results(tmp_path, capsys):
+    lock = tmp_path / "reproducibility.lock.yaml"
+    lock.write_text(
+        "benchmark:\n  canary_148:\n    vllm_overall: 94.81\n    llamacpp_overall: 93.33\n",
+        encoding="utf-8",
+    )
+    rc = cli.main(["benchmark", "--lock", str(lock)])
+    assert rc == 0
+    out = capsys.readouterr().out
+    assert "94.81" in out and "93.33" in out and "BEGIN GENERATED RESULTS" in out
+
+
+def test_benchmark_missing_lock(tmp_path):
+    rc = cli.main(["benchmark", "--lock", str(tmp_path / "nope.yaml")])
+    assert rc == 2
+
+
+# --- report ------------------------------------------------------------------
+
+
+def _write_manifest(pred_dir: Path):
+    manifest = {
+        "schema_version": 2,
+        "repo_commit": "abc123",
+        "backend": "llamacpp",
+        "model": "HYVL",
+        "timestamp_iso": "2026-07-18T03:00:00Z",
+        "status": "ok",
+        "run_counts": {"attempted": 1, "succeeded": 1, "failed": 0, "skipped": 0, "interrupted": 0},
+        "final_state": {"expected": 1, "complete": 1, "failed": 0, "pending": 0},
+        "command": ["run_inference.py", "--backend-name", "llamacpp"],
+        "env": {"torch": "2.9.1"},
+        "platform": {"python": "3.12.3"},
+    }
+    pred_dir.mkdir(parents=True, exist_ok=True)
+    (pred_dir / "run_manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+    return manifest
+
+
+def test_report_assembles_bundle_with_checksums(tmp_path):
+    pred = tmp_path / "pred"
+    _write_manifest(pred)
+    repo_root = tmp_path / "repo"
+    repo_root.mkdir()
+    (repo_root / "reproducibility.lock.yaml").write_text("hunyuanocr_rocm:\n  commit: x\n", encoding="utf-8")
+    out = tmp_path / "artifact"
+
+    rc = cli.main(["report", "--pred-dir", str(pred), "--out", str(out), "--repo-root", str(repo_root)])
+    assert rc == 0
+
+    assert (out / "run_manifest.json").is_file()
+    assert (out / "environment.json").is_file()
+    assert (out / "commands.txt").is_file()
+    assert (out / "reproducibility.lock.yaml").is_file()
+    assert (out / "README.md").is_file()
+    # checksums cover every other file and verify
+    sums = (out / "checksums.sha256").read_text(encoding="utf-8").strip().splitlines()
+    assert len(sums) == 5
+    for line in sums:
+        digest, name = line.split("  ", 1)
+        assert hashlib.sha256((out / name).read_bytes()).hexdigest() == digest
+
+
+def test_report_missing_manifest(tmp_path):
+    pred = tmp_path / "empty"
+    pred.mkdir()
+    rc = cli.main(["report", "--pred-dir", str(pred), "--out", str(tmp_path / "o"), "--repo-root", str(tmp_path)])
+    assert rc == 2