Switched to all Jinja.

Author: stephenbach

promptsource/promptsource.py

@@ -1,5 +1,3 @@
-import random
-
 import datasets
 import streamlit as st
 from templates import Template, TemplateCollection
@@ -90,14 +88,10 @@ def save_data(message="Done!"):
 #
 dataset_list = datasets.list_datasets(with_community_datasets=False)
 
-#
-# Initializes state
-#
+
 #
 # Select a dataset
 #
-# TODO: Currently raises an error if you select a dataset that requires a
-# TODO: configuration. Not clear how to query for these options.
 dataset_key = st.sidebar.selectbox(
     "Dataset",
     dataset_list,
@@ -184,7 +178,7 @@ def save_data(message="Done!"):
                     elif new_template_name == "":
                         st.error(f"Need to provide a template name.")
                     else:
-                        template = Template(new_template_name, "", 'return ""', 'return ""', "")
+                        template = Template(new_template_name, "", "")
                         templates.add_template(template_key, template)
                         save_data()
                 else:
@@ -209,42 +203,17 @@ def save_data(message="Done!"):
             #
             # If template is selected, displays template editor
             #
-            editor = st.radio("Editor Type", ["Code", "Jinja"], 1 if template.jinja_tpl else 0)
-
-            if editor == "Code":
-                with st.form("edit_template_form"):
+            with st.form("edit_template_form"):
+                input_template = st.text_area("Template", height=40, value=template.jinja)
 
-                    code_height = 40
-                    prompt_fn_code = st.text_area("Prompt Function", height=code_height, value=template.prompt_fn)
-                    output_fn_code = st.text_area("Output Function", height=code_height, value=template.output_fn)
-
-                    reference = st.text_area(
-                        "Template Reference", help="Your name and/or paper reference.", value=template.reference
-                    )
+                reference = st.text_area(
+                    "Template Reference", help="Your name and/or paper reference.", value=template.reference
+                )
 
-                    if st.form_submit_button("Save"):
-                        template.jinja = ""
-                        template.prompt_fn = prompt_fn_code
-                        template.output_fn = output_fn_code
-                        template.reference = reference
-                        save_data()
-            if editor == "Jinja":
-                with st.form("edit_template_form"):
-                    st.write("Jinja2 Templates.")
-
-                    code_height = 40
-                    input_template = st.text_area("Template", height=code_height, value=template.jinja_tpl)
-
-                    reference = st.text_area(
-                        "Template Reference", help="Your name and/or paper reference.", value=template.reference
-                    )
-
-                    if st.form_submit_button("Save"):
-                        template.jinja = input_template
-                        template.prompt_fn = ""
-                        template.output_fn = ""
-                        template.reference = reference
-                        save_data()
+                if st.form_submit_button("Save"):
+                    template.jinja = input_template
+                    template.reference = reference
+                    save_data()
     #
     # Displays template output on current example if a template is selected
     # (in second column)
@@ -256,4 +225,5 @@ def save_data(message="Done!"):
             template = dataset_templates[template_name]
             prompt = template.apply(example)
             st.write(prompt[0])
-            st.write(prompt[1])
+            if len(prompt) > 1:
+                st.write(prompt[1])

promptsource/templates.py

@@ -1,45 +1,9 @@
 import yaml
-from jinja2 import BaseLoader, Environment, PackageLoader, select_autoescape
+from jinja2 import BaseLoader, Environment
 
 env = Environment(loader=BaseLoader)
 
 
-def get_sample_template_data():
-    data = TemplateCollection()
-
-    ag_news_template = Template(
-        "basic",
-        "Example template.",
-        'return example["text"] + "\n\nIs this an example of news about world politics, sports, business, or technology?"',
-        "label_map = {\n"
-        '    0: "World politics",\n'
-        '    1: "Sports",\n'
-        '    2: "Business",\n'
-        '    3: "Technology"}\n'
-        'return label_map[example["label"]]',
-    )
-
-    data.add_template("ag_news", ag_news_template)
-
-    trec_template = Template(
-        "basic",
-        "Example template.",
-        'return example["text"] + "\n\nIs this asking about a description, an entity, '
-        'an abbreviation, a person, or a quantity?"',
-        "label_map = {\n"
-        '    0: "A description",\n'
-        '    1: "An entity",\n'
-        '    2: "An abbreviation",\n'
-        '    3: "A person",\n'
-        '    4: "A quantity"}\n'
-        'return label_map[example["label-coarse"]]',
-    )
-
-    data.add_template("trec", trec_template)
-
-    return data
-
-
 class TemplateCollection:
     """
     Collection of prompt templates.
@@ -133,34 +97,25 @@ class Template(yaml.YAMLObject):
 
     yaml_tag = "!Template"
 
-    def __init__(self, name, reference, prompt_fn=None, output_fn=None, jinja_tpl=None):
+    def __init__(self, name, jinja, reference):
         """
         Creates a prompt template.
 
-        A prompt template is made up three main pieces: strings that define
-        three functions, one each for generating the input, the prompt, and the
-        output given an example. These strings should not include the function
-        signature, but should assume that there is an input called "example".
-        Each function should return a string.
+        A prompt template is expressed in Jinja. It is rendered using an example
+        from the corresponding Hugging Face datasets library (a dictionary). The
+        separator ||| should appear once to divide the template into prompt and
+        output. Generally, the prompt should provide information on the desired
+        behavior, e.g., text passage and instructions, and the output should be
+        a desired response.
 
         :param name: unique name (per dataset) for template
+        :param jinja: template expressed in Jinja
         :param reference: string metadata describing author or paper reference
                           for template
-        :param prompt_fn: string defining function that creates prompt from example
-        :param output_fn: string defining function that creates output from example
         """
         self.name = name
-        self.prompt_fn = prompt_fn
-        self.output_fn = output_fn
+        self.jinja = jinja
         self.reference = reference
-        self.jinja = jinja_tpl
-
-    @property
-    def jinja_tpl(self):
-        if hasattr(self, "jinja"):
-            return self.jinja
-        else:
-            return ""
 
     def get_name(self):
         """
@@ -183,29 +138,7 @@ def apply(self, example):
         Creates a prompt by applying this template to an example
 
         :param example: the dataset example to create a prompt for
-        :return: tuple of 3 strings, for input, prompt, and output
-        """
-        if self.jinja_tpl:
-            rtemplate = env.from_string(self.jinja_tpl)
-            return rtemplate.render(**example).split("|||")
-
-        else:
-            fns = {}
-            exec(self._make_fun_str("prompt_fn", ["example"], self.prompt_fn), fns)
-            exec(self._make_fun_str("output_fn", ["example"], self.output_fn), fns)
-            return (fns["prompt_fn"](example), fns["output_fn"](example))
-
-    @classmethod
-    def _make_fun_str(cls, name, args, body):
-        """
-        Creates a string representation of a Python function.
-
-        :param name: the name of the function
-        :param args: iterable of strings naming function arguments
-        :param body: the function definition. The outermost context should be unindented.
-        :return: full function definition that can be parsed by exec
+        :return: tuple of 2 strings, for prompt and output
         """
-        arg_str = ", ".join(args)
-        signature = f"def {name}({arg_str}):\n"
-        body = "\n".join([("    " + line) for line in body.split("\n")])
-        return signature + body
+        rtemplate = env.from_string(self.jinja)
+        return rtemplate.render(**example).split("|||")

templates.yaml

@@ -4,35 +4,12 @@ ag_news:
       \ or technology? ||| \n{{[\"World politics\", \"Sport\", \"Business\", \"Technology\"\
       ][label] }}"
     name: test1
-    output_fn: ''
-    prompt_fn: ''
     reference: example template_1
   basic: !Template
     jinja: "{{text}} \nIs this text an example of news about world politics, sports,\
       \ business, or technology? ||| \n{{[\"World politics\", \"Sport\", \"Business\"\
       , \"Technology\"][label] }}"
     name: basic
-    output_fn: ''
-    prompt_fn: ''
-    reference: Example template.
-  jinja_example: !Template
-    jinja: ''
-    name: jinja_example
-    output_fn: return ""
-    prompt_fn: return ""
-    reference: ''
-trec:
-  basic: !Template
-    input_fn: return example["text"]
-    name: basic
-    output_fn: "label_map = {\n    0: \"A description\",\n    1: \"An entity\",\n\
-      \    2: \"An abbreviation\",\n    3: \"A person\",\n    4: \"A quantity\"}\n\
-      return label_map[example[\"label-coarse\"]]"
-    prompt_fn: 'return example["text"] + "
-
-
-      Is this asking about a description, an entity, an abbreviation, a person, or
-      a quantity?'
     reference: Example template.
 ? !!python/tuple
 - glue
@@ -41,17 +18,13 @@ trec:
     jinja: '{{sentence}} \nIs this example grammatically correct? ||| {{ ["No", "Yes"][label]
       }}'
     name: jinja_example
-    output_fn: ''
-    prompt_fn: ''
     reference: A sample glue template
 ? !!python/tuple
 - adversarial_qa
 - adversarialQA
 : test1: !Template
     jinja: "{{context}} \n{{question}}||| \n{{answers.text}}"
     name: test1
-    output_fn: ''
-    prompt_fn: ''
     reference: test template
 ? !!python/tuple
 - scitail
@@ -60,8 +33,6 @@ trec:
     jinja: "{{sentence1}} \n{{sentence2}}\nAre these sentences neutral or entailment\
       \ to one another?|||\n{{gold_label}}"
     name: Test1
-    output_fn: ''
-    prompt_fn: ''
     reference: ''
 ? !!python/tuple
 - ade_corpus_v2
@@ -73,6 +44,4 @@ trec:
 
       {{["Not-Related", "Related"][label]}}'
     name: Test1
-    output_fn: ''
-    prompt_fn: ''
     reference: ''