From 157074d4cac1f625118d7398ea35f8fa5546c5d6 Mon Sep 17 00:00:00 2001
From: ShushengYang <shusheng.yss@alibaba-inc.com>
Date: Mon, 11 Sep 2023 21:18:18 +0800
Subject: [PATCH] add docvqa eval scripts

---
 eval_mm/EVALUATION.md | 44 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/eval_mm/EVALUATION.md b/eval_mm/EVALUATION.md
index f814ed8..4a55488 100644
--- a/eval_mm/EVALUATION.md
+++ b/eval_mm/EVALUATION.md
@@ -306,6 +306,50 @@ python -m torch.distributed.launch --use-env \
 
 </details>
 
+### [DocVQA](https://www.docvqa.org/datasets)
+
+<details>
+<summary>Data Preparation</summary>
+
+```bash
+mkdir -p data/docvqa && cd data/docvqa
+
+# download images and annotations from https://www.docvqa.org/datasets
+
+# download converted files
+# train
+wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/docvqa/train.jsonl
+# val
+wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/docvqa/val.jsonl
+# test
+wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/docvqa/test.jsonl
+cd ../..
+```
+
+</details>
+
+<details>
+<summary>Evaluation</summary>
+
+```bash
+# evaluate vqa score on docvqa val split
+ds="docvqa_val"
+checkpoint=/PATH/TO/CHECKPOINT
+python -m torch.distributed.launch --use-env \
+    --nproc_per_node ${NPROC_PER_NODE:-8} \
+    --nnodes ${WORLD_SIZE:-1} \
+    --node_rank ${RANK:-0} \
+    --master_addr ${MASTER_ADDR:-127.0.0.1} \
+    --master_port ${MASTER_PORT:-12345} \
+    evaluate_vqa.py \
+    --checkpoint $checkpoint \
+    --dataset $ds \
+    --batch-size 8 \
+    --num-workers 2
+```
+
+</details>
+
 ### [ChartQA](https://aclanthology.org/2022.findings-acl.177/)
 
 <details>