-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathREPRO.yaml
More file actions
107 lines (101 loc) · 7.48 KB
/
Copy pathREPRO.yaml
File metadata and controls
107 lines (101 loc) · 7.48 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
# reproducibility.lock.yaml — machine-readable reproducibility snapshot for the
# published HunyuanOCR-ROCm results. Verified fields carry a source in a comment;
# unverified fields are `not_recorded` with a fill command. Do not invent values.
#
# Verification status legend:
# # (verified) = recomputed in this repo session, value recorded here
# # (not_recorded) = could not be verified from this env; fill command provided
hunyuanocr_rocm:
repo: https://github.com/AIwork4me/HunyuanOCR-ROCm
# The published benchmark results were finalized at the handoff commit. It is
# the reproducibility snapshot anchor. (The branch carrying this lock file has
# a later tip; the published *results* correspond to the commit below.)
commit: e17fab1d3c2586599b9ee0c845784e4b000e2101 # (verified) handoff/published-results commit
llama_cpp:
repo: https://github.com/ggml-org/llama.cpp
commit: a320cbfcb7056b7b81fb854d97fe01d0ea77c4b5 # (verified) GitHub API 2026-07-16
ggml_version: "0.16.0" # (reported in HANDOFF; not re-derived)
contains_hunyuan_multimodal: true # (verified) tools/mtmd/models/hunyuanvl.cpp + src/models/hunyuan-vl.cpp at this commit
model:
hf_repo: tencent/HunyuanOCR
gguf_repo: ggml-org/HunyuanOCR-GGUF
gguf_files: [HunyuanOCR-bf16.gguf, mmproj-HunyuanOCR-bf16.gguf]
# The local files actually used to produce the published results (sha256sum,
# recomputed this session). These are the benchmark artifacts. They are
# cross-checked against the official HF remote in `current_remote_artifact`
# below — all four match byte-for-byte.
benchmark_artifact:
hunyuanocr_bf16_gguf_sha256: a160215620dbd0ab43ec6faa28259654fd24c929953aa97c765176f7c0363217 # (verified) 1,083,218,528 B
mmproj_hunyuanocr_bf16_gguf_sha256: 46401739a91d0778d86369bb952db685b215512d61a941c3b859f337f6014fcd # (verified) 997,235,840 B
hunyuanocr_safetensors_sha256: 632a1e082c4dd5a3284cf1ffcdba2fdaa06f435762c58c2f34aff0f3bd6c0249 # (verified) 2,239,932,512 B
hunyuanocr_config_sha256: cc34ab90d0b873a1832c06e0f3fe127b47d7f390e8fda19445e4144068ed2af9 # (verified)
# Current remote state of the official HF repos. Cross-checked 2026-07-17 via
# hf-mirror.com (a transparent HuggingFace mirror; huggingface.co itself is
# unreachable from the bench env, so the official API was queried through the
# mirror). All four benchmark artifacts above match the official repos
# BYTE-FOR-BYTE (LFS oid == content sha256; config.json content sha matched).
current_remote_artifact:
hf_repo_revision: de8f10ad2f00a0cefd790b526de8a65dcfdb3205 # (verified) tencent/HunyuanOCR @ main
gguf_repo_revision: 8e070c9ad79e4ca97a9b4daa2f1ce17e8759afb1 # (verified) ggml-org/HunyuanOCR-GGUF @ main
hunyuanocr_bf16_gguf_lfs_oid: a160215620dbd0ab43ec6faa28259654fd24c929953aa97c765176f7c0363217 # (verified) == benchmark_artifact
mmproj_hunyuanocr_bf16_gguf_lfs_oid: 46401739a91d0778d86369bb952db685b215512d61a941c3b859f337f6014fcd # (verified) == benchmark_artifact
hunyuanocr_safetensors_lfs_oid: 632a1e082c4dd5a3284cf1ffcdba2fdaa06f435762c58c2f34aff0f3bd6c0249 # (verified) == benchmark_artifact
hunyuanocr_config_sha256: cc34ab90d0b873a1832c06e0f3fe127b47d7f390e8fda19445e4144068ed2af9 # (verified) == benchmark_artifact (non-LFS; content sha)
cross_check_source: "https://hf-mirror.com (HuggingFace mirror; official API data)"
omnidocbench:
version: v1.6
# Reproducibility ID is the repo URL + commit (not the local checkout path).
scorer_repo_url: https://github.com/opendatalab/OmniDocBench
scorer_commit: 2b161d010d2e3aff77a0edef359ea3a6411d23cd # (verified) git rev-parse HEAD of the local checkout
scorer_local_path: /root/ocr-eval/OmniDocBench # (observation) machine-local; not a portable repro id
repo_default_branch: main
# repo commit (fill, in case the default branch moved): curl -sL https://api.github.com/repos/opendatalab/OmniDocBench/commits/main | jq -r .sha
# Canonical name is OmniDocBench_canary_148.json (materialized from the full GT
# via `hunyuan-ocr canary materialize`). This is a rename of the historical
# OmniDocBench_150.json — byte-identical, same SHA256 (the canary has always
# been 148 pages; the old "_150" filename was a legacy misnomer).
gt_json_canary: OmniDocBench_canary_148.json
gt_json_canary_legacy_name: OmniDocBench_150.json # (historical) content-identical to gt_json_canary
gt_json_canary_sha256: 3e3fbea07702084d9466e231260ad92141848a32631c9895d8e55b24e2c2f7b5 # (verified) sha256sum, 148 pages
gt_json_full: OmniDocBench.json
gt_json_full_sha256: a45cd84b04ad8b793e775089640e6b681209abea33ead54c1828ddca35fae496 # (verified) sha256sum, 1651 pages
eval_config_sha256: a65a5b39df0e36a42ee940f9ef78cc71d793e7fb5885bc89004194515f3cc264 # (verified) src/hunyuan_ocr/data/eval_config.yaml
canary_manifest_sha256: 9839dbdf0424a1b2d1bb42bf0344cd9359847665a1cc04f6fdf0e8e447fa269f # (verified) eval/canary_148.manifest.json (manifest_sha256 field)
metric:
overall_formula: "((1 - text_EditDist)*100 + formula_CDM*100 + table_TEDS*100) / 3"
note: "reading_order EditDist is reported separately and is NOT part of Overall"
match_method: quick_match
environment:
python: "3.12.3" # (verified) sys.version
rocm_hip: "7.2.53211-e1a6bc5663" # (verified) torch.version.hip
torch: "2.9.1+gitff65f5b" # (verified) torch.__version__
transformers:
benchmark_venv: "5.13.0" # (reported in HANDOFF) the published benchmark used this isolated venv; not re-verified this session
current_opt_venv: "4.57.6" # (verified) transformers.__version__ in /opt/venv
note: "The published benchmark was produced in the isolated 5.13.0 venv, NOT this /opt/venv."
vllm: "0.16.1.dev0+g89a77b108.d20260317" # (verified) vllm.__version__
gpu_arch: gfx1100 # (per repo docs) RDNA3
rocm_smi_device_id: "0x744b" # (verified) rocm-smi
benchmark:
date: "2026-07-16" # (per reports)
hardware: "4x AMD gfx1100 (RDNA3, 48GB), ROCm 7.2"
# SAME 148-page canary, three ROCm backends (see docs/benchmark-methodology.md).
canary_148:
vllm_overall: 94.81
transformers_overall: 94.11
llamacpp_overall: 93.33
# Full 1651-page set; llama.cpp (92.09) and vLLM (91.31, validated 2026-07-25) both valid.
full_1651:
llamacpp_overall: 92.09
vllm_overall: 91.31 # validated 2026-07-25 (capped 3.4M, sequential): text 95.48 / CDM 92.46 / TEDS 86.00; 0 errors
transformers_overall: 94.05 # validated 2026-07-31 (GA-7.14 torch 2.11.0+rocm7.14.0, cap OFF, full-res ≤15360 ViT tok/page, attn sdpa, 4-GPU sharded, GPU3 excluded-faulty): text 96.22 / CDM 92.60 / TEDS 93.33; reading_order EditDist 0.1315; 0 errors, 1651/1651
# Official reference, taken from the HunyuanOCR GitHub benchmark table. NOTE:
# measured with TensorRT (a different engine) on an unlabeled OmniDocBench
# version — not a same-engine, same-page-set comparison. See docs/benchmark-methodology.md.
official_reference:
source: https://github.com/Tencent-Hunyuan/HunyuanOCR
omnidocbench_overall: "94.10" # quoted to preserve the exact figure (YAML would parse 94.10 as 94.1)
text_edit: 0.042
formula_cdm: 0.9473
table_teds: 0.9181
inference_engine: TensorRT