From 55239843691f81c10f97c17e4f1ed3ae2b883b49 Mon Sep 17 00:00:00 2001 From: Alexandre Payot Date: Tue, 20 Feb 2024 10:09:59 +0000 Subject: [PATCH 1/5] Remove whitespace --- README.md | 14 ++++---- bigcode_eval/evaluator.py | 6 ++-- bigcode_eval/generation.py | 6 ++-- bigcode_eval/tasks/humanevalpack.py | 52 ++++++++++++++--------------- main.py | 6 ++-- 5 files changed, 42 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 3aefd1344..843e1f919 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ ## Features -This is a framework for the evaluation of code generation models. This work is inspired from [EleutherAI/lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) for evaluating language models in general. We welcome contributions to fix issues, enhance features and add new benchmarks. You can find contribution guides in [`docs/guide.md`](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/docs/guide.md) and [`CONTRIBUTING.md`](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/CONTRIBUTING.md) and more documentation in [`docs/README.md`](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/docs/README.md). +This is a framework for the evaluation of code generation models. This work is inspired from [EleutherAI/lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) for evaluating language models in general. We welcome contributions to fix issues, enhance features and add new benchmarks. You can find contribution guides in [`docs/guide.md`](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/docs/guide.md) and [`CONTRIBUTING.md`](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/CONTRIBUTING.md) and more documentation in [`docs/README.md`](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/docs/README.md). Below are the features and tasks of this framework: @@ -57,7 +57,7 @@ pip install -e ".[ds1000]" # installs all additional dependencies except PyTorch # torch==1.12.1 required. Download version with relevant GPU support etc., e.g., pip install torch==1.12.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 -# to suppress any tensorflow optimization warnings, +# to suppress any tensorflow optimization warnings, # precede call to "accelerate launch" with "TF_CPP_MIN_LOG_LEVEL=3" # on some systems, tensorflow will attempt to allocate all GPU memory @@ -82,7 +82,7 @@ The evaluation part (solutions execution) for [MultiPL-E](https://github.com/nup ## Usage You can use this evaluation harness to generate text solutions to code benchmarks with your model, to evaluate (and execute) the solutions or to do both. While it is better to use GPUs for the generation, the evaluation only requires CPUs. So it might be beneficial to separate these two steps. By default both generation and evaluation are performed. -For more details on how to evaluate on the tasks, please refer to the documentation in [`docs/README.md`](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/docs/README.md). +For more details on how to evaluate on the tasks, please refer to the documentation in [`docs/README.md`](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/docs/README.md). ### Generation and evaluation Below is an example to generate and evaluate on a task. @@ -101,8 +101,8 @@ accelerate launch main.py \ --allow_code_execution \ --save_generations ``` -* `limit` represents the number of problems to solve, if it's not provided all problems in the benchmark are selected. -* `allow_code_execution` is for executing the generated code: it is off by default, read the displayed warning before calling it to enable execution. +* `limit` represents the number of problems to solve, if it's not provided all problems in the benchmark are selected. +* `allow_code_execution` is for executing the generated code: it is off by default, read the displayed warning before calling it to enable execution. * Some models with custom code on the HF hub like [SantaCoder](https://huggingface.co/bigcode/santacoder) require calling `--trust_remote_code`, for private models add `--use_auth_token`. * `save_generations` saves the post-processed generations in a json file at `save_generations_path` (by default `generations.json`). You can also save references by calling `--save_references` * `max_length_generation` is the maximum token length of generation including the input token length. The default is 512, but for some tasks like GSM8K and GSM-Hard, the complete prompt with 8 shot examples (as used in [PAL](https://github.com/reasoning-machines/pal)) take up `~1500` tokens, hence the value should be greater than that and the recommended value of `max_length_generation` is `2048` for these tasks. @@ -113,7 +113,7 @@ Some tasks don't require code execution such as ### Generation only -If you want to generate solutions without executing and evaluating the code, call `--generation_only`, in addition to the instructions above. This will save the solutions in a json file provided in `save_generation_path` in the working directory. +If you want to generate solutions without executing and evaluating the code, call `--generation_only`, in addition to the instructions above. This will save the solutions in a json file provided in `save_generation_path` in the working directory. This can be useful if you don't want to execute code in the machine you're using for generations for security or efficiency reasons. For instance, you can do the generations on multiple GPUs, but switch to a multiple workers CPU machine or docker container for the execution. @@ -193,7 +193,7 @@ To implement a new task in this evaluation harness, see the guide in [`docs/guid We provide documentation for the existing benchmarks and how to run the evaluation in [`docs/README.md`](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/docs/README.md). ## Remarks -* Currenltly, we use data parallel evaluation across multiple GPUs using `accelerate`, this assumes that you can fit the model in one GPU. +* Currenltly, we use data parallel evaluation across multiple GPUs using `accelerate`, this assumes that you can fit the model in one GPU. ## Acknowledgements We thank EleutherAI for their work on the [lm-evaluation harness](https://github.com/EleutherAI/lm-evaluation-harness) from which this repository is inspired. diff --git a/bigcode_eval/evaluator.py b/bigcode_eval/evaluator.py index fa70000b9..be8650632 100644 --- a/bigcode_eval/evaluator.py +++ b/bigcode_eval/evaluator.py @@ -13,7 +13,7 @@ ################################################################################ !!!WARNING!!! ################################################################################ -The "code_eval"/"apps_metric" you are about to use, execute untrusted +The "code_eval"/"apps_metric" you are about to use, execute untrusted model-generated code in Python. Although it is highly unlikely that model-generated code will do something overtly malicious in response to this test suite, model-generated code may act @@ -22,7 +22,7 @@ does not perform destructive actions on their host or network. For more information on how OpenAI sandboxes its code, see the paper "Evaluating Large Language Models Trained on Code" (https://arxiv.org/abs/2107.03374). -Once you have read this disclaimer and taken appropriate precautions, set the argument +Once you have read this disclaimer and taken appropriate precautions, set the argument "allow_code_execution" to True. ################################################################################\ """ @@ -47,7 +47,7 @@ def generate_text(self, task_name, intermediate_generations=None): # if args.limit is used, make sure args.limit_start + args.limit <= len(dataset) n_tasks = min(self.args.limit, len(dataset) - self.args.limit_start) if self.args.limit else len(dataset) # when args.limit is None - # adjust n_tasks by args.limit_start to prevent out of bounds issues + # adjust n_tasks by args.limit_start to prevent out of bounds issues if not self.args.limit: n_tasks -= self.args.limit_start references = [task.get_reference(dataset[i]) for i in range(self.args.limit_start, self.args.limit_start+n_tasks)] diff --git a/bigcode_eval/generation.py b/bigcode_eval/generation.py index 64f7e6984..0bbce09fb 100644 --- a/bigcode_eval/generation.py +++ b/bigcode_eval/generation.py @@ -74,7 +74,7 @@ def parallel_generations( prompts = [task.get_prompt(doc) for doc in dataset] awaitables = [client.completions.create( model=args.model, - prompt=prompt, + prompt=prompt, n=args.batch_size, max_tokens=args.max_length_generation, temperature=args.temperature, @@ -101,7 +101,7 @@ def parallel_generations( # The input_length / start_length set to 0 for now will be adjusted later # Check if the task has a custom check_fn method for the stopping criteria if task.stop_words and tokenizer.eos_token: - task.stop_words.append(tokenizer.eos_token) + task.stop_words.append(tokenizer.eos_token) if hasattr(task, "check_fn"): stopping_criteria.append( EndOfFunctionCriteria(0, task.stop_words, tokenizer, task.check_fn) @@ -114,7 +114,7 @@ def parallel_generations( stopping_criteria.append( TooLongFunctionCriteria(0, task.max_length_multiplier) ) - + if stopping_criteria: gen_kwargs["stopping_criteria"] = StoppingCriteriaList(stopping_criteria) diff --git a/bigcode_eval/tasks/humanevalpack.py b/bigcode_eval/tasks/humanevalpack.py index c1cdaa642..5efc87062 100644 --- a/bigcode_eval/tasks/humanevalpack.py +++ b/bigcode_eval/tasks/humanevalpack.py @@ -6,7 +6,7 @@ _CITATION = """ @article{muennighoff2023octopack, - title={OctoPack: Instruction Tuning Code Large Language Models}, + title={OctoPack: Instruction Tuning Code Large Language Models}, author={Niklas Muennighoff and Qian Liu and Armel Zebaze and Qinkai Zheng and Binyuan Hui and Terry Yue Zhuo and Swayam Singh and Xiangru Tang and Leandro von Werra and Shayne Longpre}, journal={arXiv preprint arXiv:2308.07124}, year={2023} @@ -100,7 +100,7 @@ "crypto/md5", ], "cpp": [ - "using namespace std;", + "using namespace std;", "#include", "#include", "#include", @@ -136,18 +136,18 @@ class HumanEvalFixTests(HumanEvalFixBase): def __init__(self, language=language, prompt="instruct"): super().__init__(language=language, prompt=prompt, with_docs=False) class HumanEvalFixDocs(HumanEvalFixBase): - def __init__(self, language=language, prompt="instruct"): + def __init__(self, language=language, prompt="instruct"): super().__init__(language=language, prompt=prompt, with_docs=True) class HumanEvalExplainDescribe(HumanEvalExplainDescribeBase): def __init__(self, language=language, prompt="instruct"): - super().__init__(language=language, prompt=prompt, with_docs=False) + super().__init__(language=language, prompt=prompt, with_docs=False) class HumanEvalExplainSynthesize(HumanEvalExplainSynthesizeBase): def __init__(self, language=language, prompt="instruct", load_data_path=None): super().__init__(language=language, prompt=prompt, with_docs=False, load_data_path=load_data_path) class HumanEvalSynthesize(HumanEvalSynthesizeBase): def __init__(self, language=language, prompt="instruct"): super().__init__(language=language, prompt=prompt, with_docs=True) - + if name == "fixtests": return HumanEvalFixTests elif name == "fixdocs": return HumanEvalFixDocs elif name == "explaindescribe": return HumanEvalExplainDescribe @@ -161,9 +161,9 @@ class HumanEvalPack(Task): DATASET_NAME = None def __init__(self, prompt="instruct", language="python", with_docs=True): - + self.DATASET_NAME = language - self.prompt = prompt + self.prompt = prompt stop_words = LANGUAGE_TO_STOP_WORDS[language] if self.prompt.startswith("edit"): stop_words.extend([ @@ -176,7 +176,7 @@ def __init__(self, prompt="instruct", language="python", with_docs=True): elif self.prompt == "diff": stop_words = ["", "", ""] elif self.prompt == "diff-carper": - stop_words = ["", "", "", "\ No newline at end of file"] + stop_words = ["", "", "", "\ No newline at end of file"] stop_words.append("<|endoftext|>") self.with_docs = with_docs super().__init__(stop_words=stop_words, requires_execution=True) @@ -188,7 +188,7 @@ def get_prompt_base(self, doc): if self.with_docs: return doc["prompt"] # Already includes fn main for rust else: if self.DATASET_NAME == "rust": - # See + # See # https://github.com/roG0d/CodeGeeX/blob/f66205b5f615a4eead9c26d7ec297e14738ea18d/codegeex/benchmark/evaluate_humaneval_x.py#L78 # https://github.com/THUDM/CodeGeeX/pull/76#issuecomment-1500653190 return "fn main(){}\n" + doc["declaration"] @@ -202,7 +202,7 @@ def get_prompt(self, prompt_base, instruction, context=None): inp = context + "\n" + instruction else: inp = instruction + "\n" + context - + if self.prompt == "continue": assert context is None, "The `continue` prompt should only be used for HumanEvalSynthesize. Use `instruct` for HumanEvalFix and HumanEvalExplain." prompt = prompt_base @@ -211,7 +211,7 @@ def get_prompt(self, prompt_base, instruction, context=None): elif self.prompt == "octocoder": prompt = f'Question: {inp}\n\nAnswer:\n{prompt_base}' elif self.prompt == "octogeex": - prompt = f'Question: {inp.strip()}\n\nAnswer:\n{prompt_base}' + prompt = f'Question: {inp.strip()}\n\nAnswer:\n{prompt_base}' elif self.prompt == "starchat": # https://huggingface.co/HuggingFaceH4/starchat-beta prompt = f'<|system|>\n<|end|>\n<|user|>\n{inp}<|end|>\n<|assistant|>\n{prompt_base}' @@ -219,7 +219,7 @@ def get_prompt(self, prompt_base, instruction, context=None): prompt = f'{inp}{prompt_base}' elif self.prompt == "instructcodet5p": # https://github.com/salesforce/CodeT5/blob/main/CodeT5%2B/humaneval/generate_codet5p.py#L89 - prompt = f'Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{inp}\n\n### Response:{prompt_base}' + prompt = f'Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{inp}\n\n### Response:{prompt_base}' elif self.prompt == "wizardcoder": # https://github.com/nlpxucan/WizardLM/blob/main/WizardCoder/src/humaneval_gen.py#L37 prompt = f'Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{inp}\n\n### Response:\n{prompt_base}' @@ -241,7 +241,7 @@ def get_prompt(self, prompt_base, instruction, context=None): # i.e. the model has never seen the token sequence of ['()', 'Ċ', 'ĠĠ'], but only ['()', 'ĊĠĠ'] # The same holds for Java, JS, Go, Rust, C++ tho the start sequences are slightly different return prompt.strip() - + def get_reference(self, doc, get_solution=False): if get_solution: return doc["prompt"] + doc["canonical_solution"] @@ -270,7 +270,7 @@ def check_fn(self, code): open_brackets = 2 if self.DATASET_NAME == "java" else 1 if code.count("{") + open_brackets == code.count("}"): return True - return False + return False def remove_last_block(self, code): """ @@ -323,7 +323,7 @@ def postprocess_generation(self, generation, idx): gen = self.remove_last_block(generation[len(prompt):].rstrip()) # Strip to maintain same behavior as with get_prompt return doc["prompt"].rstrip() + gen - + def process_results(self, generations, references): """Takes the list of LM generations and evaluates them against ground truth references. @@ -347,7 +347,7 @@ def process_results(self, generations, references): for gen, doc in zip(generations, ds): prompt_base = self.get_prompt_base(doc) old_code = prompt_base + doc["buggy_solution"] - for i, diff in enumerate(gen): + for i, diff in enumerate(gen): try: # Strip away anything to the left such as \n patches = dmp.patch_fromText(diff.lstrip()) @@ -366,7 +366,7 @@ def process_results(self, generations, references): if not(diff_hunk): gen[i] = "" continue - res: str = apply_diff(old_code, diff_hunk) + res: str = apply_diff(old_code, diff_hunk) gen[i] = res ### CUSTOM PROG LANGUAGE CHANGES ### @@ -457,7 +457,7 @@ def process_results(self, generations, references): """Debugging help for i, (gen, ref) in enumerate(zip(generations, references)): import time - starttime = time.time() + starttime = time.time() results, log = code_metric.compute( references=[ref], predictions=[gen], @@ -485,7 +485,7 @@ def get_filename_with_extension(self, input_file): """Returns the synthetic filename for different datasets""" file_name = input_file if input_file is not None else "solution" return file_name + "." + LANGUAGE_TO_EXTENSION[self.DATASET_NAME] - + def get_prompt(self, doc): """Builds the prompt for the LM to generate from.""" prompt_base = self.get_prompt_base(doc) @@ -517,7 +517,7 @@ def postprocess_generation(self, generation, idx): (not used for Humaneval-Task) """ doc = self.get_dataset()[idx] - prompt = self.get_prompt(doc) + prompt = self.get_prompt(doc) if self.prompt == "diff-carper": # Only remove final stopwords like generation = self.remove_last_block(generation[len(prompt):].rstrip()) @@ -559,13 +559,13 @@ def get_prompt_encoder(self, doc): context = prompt_base + doc["canonical_solution"] return super().get_prompt("", instruction, context) # No prompt base as not generating - + def get_prompt(self, doc): """Builds the prompt for the LM to generate from.""" prompt_base = self.get_prompt_base(doc) instruction = f"Provide a concise natural language description of the code using at most {len(doc['docstring'])} characters." context = prompt_base + doc["canonical_solution"] - + return super().get_prompt("", instruction, context) def remove_last_block(self, text): @@ -580,7 +580,7 @@ def remove_code(self, text, canonical_solution): if len(line) > 20 and line in text: text = text.replace(line, "") return text - + def postprocess_generation(self, generation, idx): """Defines the postprocessing for a LM generation. :param generation: str @@ -610,7 +610,7 @@ def __init__(self, load_data_path=None, **kwargs): assert load_data_path is not None, "load_data_path must be specified to load the descriptions." with open(load_data_path) as fp: self.descriptions = json.load(fp) - print(f"{len(self.descriptions)} descriptions with {len(self.descriptions[0])} description candidates loaded.") + print(f"{len(self.descriptions)} descriptions with {len(self.descriptions[0])} description candidates loaded.") super().__init__(**kwargs) @@ -630,7 +630,7 @@ def get_prompt_encoder(self, doc): context = doc["description"] return super().get_prompt(prompt_base, instruction, context) - + def get_prompt(self, doc): """Builds the prompt for the LM to generate from.""" prompt_base = self.get_prompt_base(doc) @@ -648,7 +648,7 @@ def get_prompt_encoder(self, doc): instruction = doc["instruction"].strip() return super().get_prompt(prompt_base, instruction) - + def get_prompt(self, doc): """Builds the prompt for the LM to generate from.""" prompt_base = self.get_prompt_base(doc) diff --git a/main.py b/main.py index 013752e8f..fadc6a0ae 100644 --- a/main.py +++ b/main.py @@ -318,7 +318,7 @@ def get_evaluator(accelerator: Accelerator, args: Namespace): revision=args.revision, trust_remote_code=args.trust_remote_code, use_auth_token=args.use_auth_token, - padding_side="left", + padding_side="left", ) else: # used by default for most models @@ -328,7 +328,7 @@ def get_evaluator(accelerator: Accelerator, args: Namespace): trust_remote_code=args.trust_remote_code, use_auth_token=args.use_auth_token, truncation_side="left", - padding_side="right", + padding_side="right", ) if not tokenizer.eos_token: if tokenizer.bos_token: @@ -338,7 +338,7 @@ def get_evaluator(accelerator: Accelerator, args: Namespace): raise ValueError("No eos_token or bos_token found") try: tokenizer.pad_token = tokenizer.eos_token - + # Some models like CodeGeeX2 have pad_token as a read-only property except AttributeError: print("Not setting pad_token to eos_token") From 2b441dfc341e20accf0de944a9d08cecc48c93d3 Mon Sep 17 00:00:00 2001 From: Alexandre Payot Date: Tue, 20 Feb 2024 10:17:49 +0000 Subject: [PATCH 2/5] Save all the stages of data transformation when evaluating the API --- bigcode_eval/evaluator.py | 7 +++++++ bigcode_eval/generation.py | 26 ++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/bigcode_eval/evaluator.py b/bigcode_eval/evaluator.py index be8650632..658578e1b 100644 --- a/bigcode_eval/evaluator.py +++ b/bigcode_eval/evaluator.py @@ -118,6 +118,13 @@ def save_json_files( with open(save_generations_path, "w") as fp: json.dump(generations, fp) print(f"generations were saved at {save_generations_path}") + + if hasattr(generations, "to_table"): + full_gen = f"{os.path.splitext(save_generations_path)[0]}_full.json" + with open(full_gen, "w") as fp: + json.dump(generations.to_table(), fp) + print(f"Full generations were saved at {full_gen}") + if self.args.save_references: with open(save_references_path, "w") as fp: json.dump(references, fp) diff --git a/bigcode_eval/generation.py b/bigcode_eval/generation.py index 0bbce09fb..31bb20396 100644 --- a/bigcode_eval/generation.py +++ b/bigcode_eval/generation.py @@ -37,7 +37,26 @@ def __init__(self, input_length, multiplier): def __call__(self, input_ids, scores, **kwargs): """Returns true if generated sequence is too long.""" return input_ids.shape[1] > int(self.input_length * self.multiplier) - + + +class Generations(list): + """A list which can carry extra data in a dictionary and can turn it all in a table. + + TODO: This is a terrible hack, do something better. + """ + @property + def sneaky_extra_data(self): + if not hasattr(self, "_extra_data"): + self._extra_data = {} + return self._extra_data + + def to_table(self): + headers = [*self.sneaky_extra_data.keys(), "generations"] + return [ + {k:v[0] if isinstance(v, list) and len(v)==1 else v for k, v in zip(headers, row)} + for row in zip(*self.sneaky_extra_data.values(), self) + ] + def parallel_generations( task, @@ -81,10 +100,13 @@ def parallel_generations( top_p=args.top_p ) for prompt in prompts] responses = asyncio.run(tqdm.gather(*awaitables)) - generations = [] + generations = Generations() + generations.sneaky_extra_data["prompts"] = prompts + generations.sneaky_extra_data["raw_response"] = [] for i, (prompt, response) in enumerate(zip(prompts, responses)): texts = [prompt + choice.text for choice in response.choices] generations.append([task.postprocess_generation(text, i) for text in texts]) + generations.sneaky_extra_data["raw_response"].append([choice.text for choice in response.choices]) return generations set_seed(args.seed, device_specific=True) From 0d30df6da2a85443c60b8408cfa5e624d3a444fc Mon Sep 17 00:00:00 2001 From: Alexandre Payot Date: Tue, 20 Feb 2024 10:18:22 +0000 Subject: [PATCH 3/5] Support arbitrary prompt formats for HumanEvalPack --- bigcode_eval/tasks/humanevalpack.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bigcode_eval/tasks/humanevalpack.py b/bigcode_eval/tasks/humanevalpack.py index 5efc87062..a7ff15376 100644 --- a/bigcode_eval/tasks/humanevalpack.py +++ b/bigcode_eval/tasks/humanevalpack.py @@ -1,5 +1,6 @@ import json import re +import pathlib from evaluate import load from bigcode_eval.base import Task @@ -225,8 +226,14 @@ def get_prompt(self, prompt_base, instruction, context=None): prompt = f'Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{inp}\n\n### Response:\n{prompt_base}' elif self.prompt == "codellama": prompt = f"[INST] {inp.strip()} [/INST] {prompt_base}" + elif pathlib.Path(self.prompt).exists(): + prompt = pathlib.Path(self.prompt).read_text().format(inp, prompt_base) else: - raise ValueError(f"The --prompt argument {self.prompt} wasn't provided or isn't supported") + raise ValueError( + f"The --prompt argument {self.prompt} wasn't provided or isn't supported." + " It can be one of the pre-defined options or a text file defining the prompt, " + "the string in that file should accept two arguments eg. 'this is a prompt: {} {}' ." + ) # Strip off the final \n to make the tokens more natural # Essentially, we want to make sure that if there was no distinction between # input & output, the tokens would be the same From ab794aff23824289dbf7b8878cd7a3fee06699b9 Mon Sep 17 00:00:00 2001 From: Alexandre Payot Date: Tue, 20 Feb 2024 10:19:07 +0000 Subject: [PATCH 4/5] Cache requirements for the docker build --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 262194fe0..37ff7e7ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,14 @@ FROM ubuntu:22.04 RUN apt-get update && apt-get install -y python3 python3-pip -COPY . /app +COPY ./requirements.txt /app/requirements.txt WORKDIR /app +RUN pip3 install -r requirements.txt + +COPY . /app + RUN test -f /app/generations.json && rm /app/generations.json || true RUN pip3 install . From d349ff6c2cd001492cac06ac803026cddc78f518 Mon Sep 17 00:00:00 2001 From: Alexandre Payot Date: Tue, 20 Feb 2024 15:09:28 +0000 Subject: [PATCH 5/5] Output individual sample results in the result output --- bigcode_eval/evaluator.py | 2 +- bigcode_eval/tasks/__init__.py | 3 ++- bigcode_eval/tasks/humanevalpack.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bigcode_eval/evaluator.py b/bigcode_eval/evaluator.py index 658578e1b..18fa96bfb 100644 --- a/bigcode_eval/evaluator.py +++ b/bigcode_eval/evaluator.py @@ -97,7 +97,7 @@ def evaluate(self, task_name, intermediate_generations=None): if self.accelerator.is_main_process: if not self.args.load_generations_path: save_generations_path = f"{os.path.splitext(self.args.save_generations_path)[0]}_{task_name}.json" - self.save_json_files(generations, references, save_generations_path, f"references_{task_name}.json") + self.save_json_files(generations, references, save_generations_path, self.args.save_references_path) # make sure tokenizer plays nice with multiprocessing os.environ["TOKENIZERS_PARALLELISM"] = "false" diff --git a/bigcode_eval/tasks/__init__.py b/bigcode_eval/tasks/__init__.py index a9545d473..a969e5920 100644 --- a/bigcode_eval/tasks/__init__.py +++ b/bigcode_eval/tasks/__init__.py @@ -1,4 +1,5 @@ import inspect +from .. import base from pprint import pprint from . import (apps, codexglue_code_to_text, codexglue_text_to_text, conala, @@ -32,7 +33,7 @@ ALL_TASKS = sorted(list(TASK_REGISTRY)) -def get_task(task_name, args=None): +def get_task(task_name, args=None)-> base.Task: try: kwargs = {} if "prompt" in inspect.signature(TASK_REGISTRY[task_name]).parameters: diff --git a/bigcode_eval/tasks/humanevalpack.py b/bigcode_eval/tasks/humanevalpack.py index a7ff15376..29af967ee 100644 --- a/bigcode_eval/tasks/humanevalpack.py +++ b/bigcode_eval/tasks/humanevalpack.py @@ -457,6 +457,7 @@ def process_results(self, generations, references): timeout=timeout, num_workers=num_workers, ) + results["details"] = logs # Write logs to json with open("logs.json", "w") as f: json.dump(logs, f, indent=4, ensure_ascii=False)