From 0d6a0969db68312edf10367cbe88a1abe6bc3718 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Mon, 25 Jul 2022 14:23:59 +0200 Subject: [PATCH 1/8] Add MLQA --- promptsource/mlqa/mlqa.en.en/templates.yaml | 104 ++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 promptsource/mlqa/mlqa.en.en/templates.yaml diff --git a/promptsource/mlqa/mlqa.en.en/templates.yaml b/promptsource/mlqa/mlqa.en.en/templates.yaml new file mode 100644 index 000000000..58bd1e87a --- /dev/null +++ b/promptsource/mlqa/mlqa.en.en/templates.yaml @@ -0,0 +1,104 @@ +dataset: mlqa +subset: mlqa.en.en +templates: + 320fffe0-b752-43f8-bf50-ecf009703ef0: !Template + answer_choices: null + id: 320fffe0-b752-43f8-bf50-ecf009703ef0 + jinja: '{{context}} + + Q: {{question}} + + Referring to the passage above, the correct answer to the given question is + ||| {{answers["text"][0]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Squad + original_task: true + name: answer_given_context_and_question + reference: '' + 52a9896f-34d5-4bde-8843-6d01d4621016: !Template + answer_choices: null + id: 52a9896f-34d5-4bde-8843-6d01d4621016 + jinja: "{{context}}\n\nWith reference to the above context, {{question}} ||| \n\ + \n{{answers.text[0]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Squad + original_task: true + name: answer_the_question + reference: '' + 20b53380-5c3b-4884-8cd1-9b4316da7993: !Template + answer_choices: null + id: 20b53380-5c3b-4884-8cd1-9b4316da7993 + jinja: 'Refer to the passage below and then answer the question afterwards: + + + Passage: {{context}} + + + Question: {{question}} + + ||| + + {{answers["text"][0]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Squad + original_task: true + name: answer_question_given_context + reference: '' + 2cff064e-97e0-4026-94bc-3f7987856ec7: !Template + answer_choices: null + id: 2cff064e-97e0-4026-94bc-3f7987856ec7 + jinja: 'D: {{context}} + + + Q: {{question}} + + + A: ||| {{answers["text"][0]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Squad + original_task: true + name: given_context_answer_question_variation + reference: '' + g3d9ac66-1188-40d4-9ac9-17e0af50b788: !Template + answer_choices: null + id: g3d9ac66-1188-40d4-9ac9-17e0af50b788 + jinja: "Read the following passage and then answer the question that follows: \n{{context}}\ + \ \n{{question}}\n||| {{answers.text[0]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - Squad + original_task: true + name: read_passage + reference: '' + g4d9ac66-1188-40d4-9ac9-17e0af50b788: !Template + answer_choices: null + id: g4d9ac66-1188-40d4-9ac9-17e0af50b788 + jinja: "Question: {{question}}\n + Context: {{context}}\n + Answer: ||| {{answers.text[0]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - Squad + original_task: true + name: read_passage + reference: '' From c629956138dd4f0072d1823eaa5c17eee29d5ed6 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Mon, 25 Jul 2022 14:35:05 +0200 Subject: [PATCH 2/8] Specify language to use --- promptsource/mlqa/mlqa.en.en/templates.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/promptsource/mlqa/mlqa.en.en/templates.yaml b/promptsource/mlqa/mlqa.en.en/templates.yaml index 58bd1e87a..29645f4a7 100644 --- a/promptsource/mlqa/mlqa.en.en/templates.yaml +++ b/promptsource/mlqa/mlqa.en.en/templates.yaml @@ -8,7 +8,7 @@ templates: Q: {{question}} - Referring to the passage above, the correct answer to the given question is + Referring to the passage above, the correct answer to the given question in the language of the passage is ||| {{answers["text"][0]}}' metadata: !TemplateMetadata choices_in_prompt: false @@ -36,7 +36,7 @@ templates: 20b53380-5c3b-4884-8cd1-9b4316da7993: !Template answer_choices: null id: 20b53380-5c3b-4884-8cd1-9b4316da7993 - jinja: 'Refer to the passage below and then answer the question afterwards: + jinja: 'Refer to the passage below and then answer the question afterwards in the same language as the passage: Passage: {{context}} @@ -78,7 +78,7 @@ templates: g3d9ac66-1188-40d4-9ac9-17e0af50b788: !Template answer_choices: null id: g3d9ac66-1188-40d4-9ac9-17e0af50b788 - jinja: "Read the following passage and then answer the question that follows: \n{{context}}\ + jinja: "Read the following passage and then answer the question that follows by extracting the correct part from the passage: \n{{context}}\ \ \n{{question}}\n||| {{answers.text[0]}}" metadata: !TemplateMetadata choices_in_prompt: false @@ -93,7 +93,7 @@ templates: id: g4d9ac66-1188-40d4-9ac9-17e0af50b788 jinja: "Question: {{question}}\n Context: {{context}}\n - Answer: ||| {{answers.text[0]}}" + Answer from the Context: ||| {{answers.text[0]}}" metadata: !TemplateMetadata choices_in_prompt: false languages: [] From 4a854212a120ee24f8b9598d9d0ec729f6ee483d Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Mon, 25 Jul 2022 14:43:41 +0200 Subject: [PATCH 3/8] Move MLQA --- promptsource/{mlqa => templates}/mlqa.en.en/templates.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename promptsource/{mlqa => templates}/mlqa.en.en/templates.yaml (100%) diff --git a/promptsource/mlqa/mlqa.en.en/templates.yaml b/promptsource/templates/mlqa.en.en/templates.yaml similarity index 100% rename from promptsource/mlqa/mlqa.en.en/templates.yaml rename to promptsource/templates/mlqa.en.en/templates.yaml From 09b368458b056516f95df88d743ec7a36cc73c85 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Mon, 25 Jul 2022 14:47:00 +0200 Subject: [PATCH 4/8] Move all of mlqa --- promptsource/templates/{ => mlqa}/mlqa.en.en/templates.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename promptsource/templates/{ => mlqa}/mlqa.en.en/templates.yaml (100%) diff --git a/promptsource/templates/mlqa.en.en/templates.yaml b/promptsource/templates/mlqa/mlqa.en.en/templates.yaml similarity index 100% rename from promptsource/templates/mlqa.en.en/templates.yaml rename to promptsource/templates/mlqa/mlqa.en.en/templates.yaml From b8ee4c49096af155f379371171f085160bc4d085 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Mon, 25 Jul 2022 14:59:22 +0200 Subject: [PATCH 5/8] Adapt ID --- promptsource/templates/mlqa/mlqa.en.en/templates.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/promptsource/templates/mlqa/mlqa.en.en/templates.yaml b/promptsource/templates/mlqa/mlqa.en.en/templates.yaml index 29645f4a7..c231ba54b 100644 --- a/promptsource/templates/mlqa/mlqa.en.en/templates.yaml +++ b/promptsource/templates/mlqa/mlqa.en.en/templates.yaml @@ -88,9 +88,9 @@ templates: original_task: true name: read_passage reference: '' - g4d9ac66-1188-40d4-9ac9-17e0af50b788: !Template + 320fffe1-b752-43f8-bf50-ecf009703ef0: !Template answer_choices: null - id: g4d9ac66-1188-40d4-9ac9-17e0af50b788 + id: 320fffe1-b752-43f8-bf50-ecf009703ef0 jinja: "Question: {{question}}\n Context: {{context}}\n Answer from the Context: ||| {{answers.text[0]}}" From 9a33cee6cc74a1665ca64e46b7a9f22bacb37625 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Mon, 25 Jul 2022 15:07:36 +0200 Subject: [PATCH 6/8] Adapt UUID --- promptsource/templates/mlqa/mlqa.en.en/templates.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/promptsource/templates/mlqa/mlqa.en.en/templates.yaml b/promptsource/templates/mlqa/mlqa.en.en/templates.yaml index c231ba54b..8a8e883da 100644 --- a/promptsource/templates/mlqa/mlqa.en.en/templates.yaml +++ b/promptsource/templates/mlqa/mlqa.en.en/templates.yaml @@ -88,9 +88,9 @@ templates: original_task: true name: read_passage reference: '' - 320fffe1-b752-43f8-bf50-ecf009703ef0: !Template + 420fffe1-b752-43f8-bf50-ecf009703ef0: !Template answer_choices: null - id: 320fffe1-b752-43f8-bf50-ecf009703ef0 + id: 420fffe1-b752-43f8-bf50-ecf009703ef0 jinja: "Question: {{question}}\n Context: {{context}}\n Answer from the Context: ||| {{answers.text[0]}}" From 5432cd5a30afc22c5a552129db7b0d197a720868 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Mon, 25 Jul 2022 15:26:13 +0200 Subject: [PATCH 7/8] Invent names --- .../templates/mlqa/mlqa.en.en/templates.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/promptsource/templates/mlqa/mlqa.en.en/templates.yaml b/promptsource/templates/mlqa/mlqa.en.en/templates.yaml index 8a8e883da..7880c4eaf 100644 --- a/promptsource/templates/mlqa/mlqa.en.en/templates.yaml +++ b/promptsource/templates/mlqa/mlqa.en.en/templates.yaml @@ -17,7 +17,7 @@ templates: metrics: - Squad original_task: true - name: answer_given_context_and_question + name: cqreferringa reference: '' 52a9896f-34d5-4bde-8843-6d01d4621016: !Template answer_choices: null @@ -31,7 +31,7 @@ templates: metrics: - Squad original_task: true - name: answer_the_question + name: creferenceqa reference: '' 20b53380-5c3b-4884-8cd1-9b4316da7993: !Template answer_choices: null @@ -54,7 +54,7 @@ templates: metrics: - Squad original_task: true - name: answer_question_given_context + name: refercqa reference: '' 2cff064e-97e0-4026-94bc-3f7987856ec7: !Template answer_choices: null @@ -73,7 +73,7 @@ templates: metrics: - Squad original_task: true - name: given_context_answer_question_variation + name: dqa reference: '' g3d9ac66-1188-40d4-9ac9-17e0af50b788: !Template answer_choices: null @@ -86,7 +86,7 @@ templates: metrics: - Squad original_task: true - name: read_passage + name: exractingcqa reference: '' 420fffe1-b752-43f8-bf50-ecf009703ef0: !Template answer_choices: null @@ -100,5 +100,5 @@ templates: metrics: - Squad original_task: true - name: read_passage + name: qaanswera reference: '' From b82904456fedf2c9a5302f2cf3101692202f64dd Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Mon, 25 Jul 2022 15:44:43 +0200 Subject: [PATCH 8/8] Adapt UUID --- promptsource/templates/mlqa/mlqa.en.en/templates.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/promptsource/templates/mlqa/mlqa.en.en/templates.yaml b/promptsource/templates/mlqa/mlqa.en.en/templates.yaml index 7880c4eaf..44a2def13 100644 --- a/promptsource/templates/mlqa/mlqa.en.en/templates.yaml +++ b/promptsource/templates/mlqa/mlqa.en.en/templates.yaml @@ -75,9 +75,9 @@ templates: original_task: true name: dqa reference: '' - g3d9ac66-1188-40d4-9ac9-17e0af50b788: !Template + 530fffe1-b752-43f8-bf50-ecf009703ef0: !Template answer_choices: null - id: g3d9ac66-1188-40d4-9ac9-17e0af50b788 + id: 530fffe1-b752-43f8-bf50-ecf009703ef0 jinja: "Read the following passage and then answer the question that follows by extracting the correct part from the passage: \n{{context}}\ \ \n{{question}}\n||| {{answers.text[0]}}" metadata: !TemplateMetadata