[Task] Add STARE Task

lmms_eval/tasks/stare/_default_template_yaml

@@ -0,0 +1,28 @@
+dataset_path: pangyyyyy/STARE
+
+generation_kwargs:
+  max_new_tokens: 8192
+  temperature: 0.0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+
+output_type: generate_until
+doc_to_visual: !function utils.stare_doc_to_visual
+doc_to_text: !function utils.stare_doc_to_text
+doc_to_target: utils.stare_doc_to_target
+process_results: !function utils.stare_process_results
+
+metric_list:
+  - metric: stare_score
+    aggregation: !function utils.stare_aggregate_results
+    higher_is_better: true
+
+dataset_kwargs:
+  token: True
+  cache_dir: STARE
+  force_download: true
+
+metadata:
+  strategy: CoT # ['Direct', 'CoT']
+  use_lmms_judge: False

lmms_eval/tasks/stare/stare.yaml

@@ -0,0 +1,5 @@
+# Official STARE paper use this configuration
+dataset_name: All
+test_split: train
+task: "stare_full"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_2d_text_instruct.yaml

@@ -0,0 +1,4 @@
+dataset_name: 2d_text_instruct
+test_split: test
+task: "stare_2d_text_instruct"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_2d_text_instruct_vsim.yaml

@@ -0,0 +1,4 @@
+dataset_name: 2d_text_instruct_vsim
+test_split: test
+task: "stare_2d_text_instruct_vsim"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_2d_va.yaml

@@ -0,0 +1,4 @@
+dataset_name: 2d_va
+test_split: test
+task: "stare_2d_va"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_2d_va_vsim.yaml

@@ -0,0 +1,4 @@
+dataset_name: 2d_va_vsim
+test_split: test
+task: "stare_2d_va_vsim"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_3d_text_instruct.yaml

@@ -0,0 +1,4 @@
+dataset_name: 3d_text_instruct
+test_split: test
+task: "stare_3d_text_instruct"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_3d_text_instruct_vsim.yaml

@@ -0,0 +1,4 @@
+dataset_name: 3d_text_instruct_vsim
+test_split: test
+task: "stare_3d_text_instruct_vsim"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_3d_va.yaml

@@ -0,0 +1,4 @@
+dataset_name: 3d_va
+test_split: test
+task: "stare_3d_va"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_3d_va_vsim.yaml

@@ -0,0 +1,4 @@
+dataset_name: 3d_va_vsim
+test_split: test
+task: "stare_3d_va_vsim"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_folding_nets.yaml

@@ -0,0 +1,4 @@
+dataset_name: folding_nets
+test_split: test
+task: "stare_folding_nets"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_folding_nets_vsim.yaml

@@ -0,0 +1,4 @@
+dataset_name: folding_nets_vsim
+test_split: test
+task: "stare_folding_nets_vsim"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_perspective.yaml

@@ -0,0 +1,4 @@
+dataset_name: perspective
+test_split: test
+task: "stare_perspective"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_tangram_puzzle.yaml

@@ -0,0 +1,4 @@
+dataset_name: tangram_puzzle
+test_split: test
+task: "stare_tangram_puzzle"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_tangram_puzzle_vsim.yaml

@@ -0,0 +1,4 @@
+dataset_name: tangram_puzzle_vsim
+test_split: test
+task: "stare_tangram_puzzle_vsim"
+include: _default_template_yaml

lmms_eval/tasks/stare/stare_temporal.yaml

@@ -0,0 +1,4 @@
+dataset_name: temporal
+test_split: test
+task: "stare_temporal"
+include: _default_template_yaml