diff --git a/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml b/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml index 695678191..3cce7f0e9 100644 --- a/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml +++ b/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml @@ -17,7 +17,7 @@ templates: name: bigbio_sts_seem_binary reference: '' 9c87c669-7be0-42f3-bd38-32ff149f2722: !Template - answer_choices: null + answer_choices: 0 ||| 1 ||| 2 ||| 3 ||| 4 id: 9c87c669-7be0-42f3-bd38-32ff149f2722 jinja: 'from {{"0.0"}} to {{"4.0"}}, how similar are "{{text_1}}" and "{{text_2}}"?||| @@ -25,12 +25,12 @@ templates: metadata: !TemplateMetadata choices_in_prompt: false metrics: - - Pearson Correlation + - Accuracy original_task: false name: bigbio_sts_similarity_scale reference: '' abc04725-8f0d-4653-a813-b9b41a45f253: !Template - answer_choices: null + answer_choices: 0 ||| 1 ||| 2 ||| 3 ||| 4 id: abc04725-8f0d-4653-a813-b9b41a45f253 jinja: 'How similar are "{{text_1}}" and "{{text_2}}"? Give a score between {{"0.0"}} and {{"4.0"}}.||| @@ -38,7 +38,8 @@ templates: {{label}}' metadata: !TemplateMetadata choices_in_prompt: false - metrics: [] + metrics: + - Accuracy original_task: false name: bigbio_sts_similarity_how reference: '' @@ -56,7 +57,7 @@ templates: name: bigbio_sts_express_binary reference: '' e0b0f20b-47a8-459d-898e-d9a428706e20: !Template - answer_choices: null + answer_choices: 0 ||| 1 ||| 2 ||| 3 ||| 4 id: e0b0f20b-47a8-459d-898e-d9a428706e20 jinja: 'Rate the similarity of these two sentences ({{"0.0"}} being the lowest and {{"4.0"}} the highest): @@ -67,7 +68,7 @@ templates: metadata: !TemplateMetadata choices_in_prompt: false metrics: - - Pearson Correlation + - Accuracy original_task: false name: bigbio_sts_similarity_rate reference: '' diff --git a/promptsource/templates/gad/gad_blurb_bigbio_text/templates.yaml b/promptsource/templates/gad/gad_blurb_bigbio_text/templates.yaml new file mode 100644 index 000000000..c122eb1b0 --- /dev/null +++ b/promptsource/templates/gad/gad_blurb_bigbio_text/templates.yaml @@ -0,0 +1,103 @@ +dataset: gad +subset: gad_fold1_bigbio_text +templates: + 079471a6-8eb1-468c-ad98-b40d9e290caf: !Template + answer_choices: No ||| Yes + id: 079471a6-8eb1-468c-ad98-b40d9e290caf + jinja: 'Is there an association between the gene @GENE$ and the disease @DISEASE$ + expressed in this passage? + + + {{ text }} + + ||| + + {{ answer_choices[labels[0] | int] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + - Other + original_task: true + name: Is there an association expressed? (passage last) + reference: '"Other" metric refers to the micro F1 used in BLURB.' + 8d5b31ab-3eb3-4f6d-a653-3ec1c4704f39: !Template + answer_choices: No ||| Yes + id: 8d5b31ab-3eb3-4f6d-a653-3ec1c4704f39 + jinja: 'I''m a doctor. Can you tell me, is there an association between the gene + @GENE$ and the disease @DISEASE$ expressed in this passage? + + + {{ text }} + + ||| + + {{ answer_choices[labels[0] | int] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + - Other + original_task: true + name: I'm a doctor + reference: '"Other" metric refers to the micro F1 used in BLURB.' + 9ecf58f5-e6dd-486e-8234-f050fa806d85: !Template + answer_choices: No ||| Yes + id: 9ecf58f5-e6dd-486e-8234-f050fa806d85 + jinja: '{{ text }} + + + Does this passage indicate that there is an association between the gene @GENE$ + and the disease @DISEASE$ ? + + ||| + + {{ answer_choices[labels[0] | int] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + - Other + original_task: true + name: Does this passage (passage first) + reference: '"Other" metric refers to the micro F1 used in BLURB.' + b26032d2-3edd-4dd9-b1d3-379db0d7f145: !Template + answer_choices: No ||| Yes + id: b26032d2-3edd-4dd9-b1d3-379db0d7f145 + jinja: 'Does the following passage indicate that there is an association between + the gene @GENE$ and the disease @DISEASE$ ? + + + {{ text }} + + ||| + + {{ answer_choices[labels[0] | int] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + - Other + original_task: true + name: Does this passage (passage last) + reference: '"Other" metric refers to the micro F1 used in BLURB.' + edf792c0-6598-4e41-aa38-8841561b203f: !Template + answer_choices: No ||| Yes + id: edf792c0-6598-4e41-aa38-8841561b203f + jinja: '{{ text }} + + + Is there an association between the gene @GENE$ and the disease @DISEASE$ expressed + in this passage? + + ||| + + {{ answer_choices[labels[0] | int] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + - Other + original_task: true + name: Is there an association expressed? (passage first) + reference: '"Other" metric refers to the micro F1 used in BLURB.'