Adding automatically generating "score" and "evaluate" commands for each metric (#7)

Author: danieldeutsch

sacrerouge/__main__.py

@@ -1,6 +1,6 @@
 import argparse
 
-from sacrerouge.commands import correlate, evaluate, score, setup_dataset, setup_metric
+from sacrerouge.commands import correlate, evaluate, metric_command, score, setup_dataset, setup_metric
 
 
 def main():
@@ -17,6 +17,9 @@ def main():
     for subcommand in subcommands:
         subcommand.add_subparser(subparsers)
 
+    # Add a command for each individual metric
+    metric_command.add_metric_subcommands(subparsers)
+
     args = parser.parse_args()
     if 'func' in dir(args):
         args.func(args)

sacrerouge/commands/evaluate.py

@@ -55,46 +55,72 @@ def evaluate_instances(instances: List[EvalInstance], metrics: List[Metric]) ->
     return macro, micro_list
 
 
-class EvaluateSubcommand(Subcommand):
-    @overrides
-    def add_subparser(self, parser: argparse._SubParsersAction):
-        description = 'Evaluate a summarization model'
-        self.parser = parser.add_parser('evaluate', description=description, help=description)
-        self.parser.add_argument(
+def save_evaluation_results(macro_results: MetricsDict,
+                            micro_results_list: List[Metrics],
+                            macro_output_json: str,
+                            micro_output_jsonl: str,
+                            silent: bool) -> None:
+    dirname = os.path.dirname(macro_output_json)
+    if dirname:
+        os.makedirs(dirname, exist_ok=True)
+
+    serialized_macro = jsons.dumps({'metrics': macro_results}, jdkwargs={'indent': 2})
+    with open(macro_output_json, 'w') as out:
+        out.write(serialized_macro)
+    if not silent:
+        logger.info(serialized_macro)
+
+    with JsonlWriter(micro_output_jsonl) as out:
+        for metrics_dict in micro_results_list:
+            out.write(metrics_dict)
+
+
+def add_evaluate_arguments(parser: argparse.ArgumentParser, include_config_arguments: bool) -> None:
+    if include_config_arguments:
+        parser.add_argument(
             'config',
             type=str,
             help='The config file that specifies the dataset reader and metrics'
         )
-        self.parser.add_argument(
-            'macro_output_json',
-            type=str,
-            help='The path to where the system-level metrics should be written'
-        )
-        self.parser.add_argument(
-            'micro_output_jsonl',
-            type=str,
-            help='The path to where the input-level metrics should be written'
-        )
-        self.parser.add_argument(
-            '--log-file',
-            type=str,
-            help='The file where the log should be written'
-        )
-        self.parser.add_argument(
-            '--silent',
-            action='store_true',
-            help='Controls whether the log should be written to stdout'
-        )
-        self.parser.add_argument(
+        parser.add_argument(
             '--overrides',
             type=str,
             help='A serialized json that will override the parameters passed in "config"'
         )
-        self.parser.add_argument(
-            '--include-packages',
-            nargs='+',
-            help='A list of additional packages to include'
-        )
+
+    parser.add_argument(
+        'macro_output_json',
+        type=str,
+        help='The path to where the system-level metrics should be written'
+    )
+    parser.add_argument(
+        'micro_output_jsonl',
+        type=str,
+        help='The path to where the input-level metrics should be written'
+    )
+    parser.add_argument(
+        '--log-file',
+        type=str,
+        help='The file where the log should be written'
+    )
+    parser.add_argument(
+        '--silent',
+        action='store_true',
+        help='Controls whether the log should be written to stdout'
+    )
+    parser.add_argument(
+        '--include-packages',
+        nargs='+',
+        help='A list of additional packages to include'
+    )
+
+
+class EvaluateSubcommand(Subcommand):
+    @overrides
+    def add_subparser(self, parser: argparse._SubParsersAction):
+        description = 'Evaluate a summarization model'
+        self.parser = parser.add_parser('evaluate', description=description, help=description)
+        add_evaluate_arguments(self.parser, True)
         self.parser.set_defaults(func=self.run)
 
     @overrides
@@ -117,16 +143,4 @@ def run(self, args):
         instances = dataset_reader.read(*input_files)
         macro, micro_list = evaluate_instances(instances, metrics)
 
-        dirname = os.path.dirname(args.macro_output_json)
-        if dirname:
-            os.makedirs(dirname, exist_ok=True)
-
-        serialized_macro = jsons.dumps({'metrics': macro}, jdkwargs={'indent': 2})
-        with open(args.macro_output_json, 'w') as out:
-            out.write(serialized_macro)
-        if not args.silent:
-            logger.info(serialized_macro)
-
-        with JsonlWriter(args.micro_output_jsonl) as out:
-            for metrics_dict in micro_list:
-                out.write(metrics_dict)
+        save_evaluation_results(macro, micro_list, args.macro_output_json, args.micro_output_jsonl, args.silent)

sacrerouge/commands/metric_command.py

@@ -0,0 +1,84 @@
+import argparse
+from overrides import overrides
+from typing import Type
+
+from sacrerouge.commands import Subcommand
+from sacrerouge.commands.evaluate import add_evaluate_arguments, evaluate_instances, save_evaluation_results
+from sacrerouge.commands.score import add_score_arguments, save_score_results, score_instances
+from sacrerouge.common import Registrable
+from sacrerouge.common.arguments import add_metric_arguments, get_dataset_reader_from_argument, get_metric_from_arguments
+from sacrerouge.common.logging import prepare_global_logging
+from sacrerouge.metrics import Metric
+
+
+def add_metric_subcommands(subparsers: argparse._SubParsersAction) -> None:
+    """Adds a MetricSubcommand for every registered metric."""
+    for name, (metric, _) in Registrable._registry[Metric].items():
+        command = MetricSubcommand(name, metric)
+        command.add_subparser(subparsers)
+
+
+def add_dataset_reader_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        '--dataset-reader',
+        type=str,
+        required=True,
+        help='The name or the parameters as a serialized json for the dataset reader'
+    )
+    parser.add_argument(
+        '--input-files',
+        nargs='+',
+        required=True,
+        help='The input files to be passed to the dataset reader'
+    )
+
+
+class MetricSubcommand(Subcommand):
+    def __init__(self, name: str, metric_type: Type) -> None:
+        super().__init__()
+        self.name = name
+        self.metric_type = metric_type
+
+    @overrides
+    def add_subparser(self, parser: argparse._SubParsersAction):
+        description = f'Run "evaluate" or "score" with the "{self.name}" metric.'
+        self.parser = parser.add_parser(self.name, description=description, help=description)
+        subparsers = self.parser.add_subparsers()
+
+        description = f'Run "evaluate" with the "{self.name}" metric.'
+        self.evaluate_parser = subparsers.add_parser('evaluate', description=description, help=description)
+        add_evaluate_arguments(self.evaluate_parser, False)
+        add_metric_arguments(self.evaluate_parser, self.metric_type)
+        add_dataset_reader_arguments(self.evaluate_parser)
+        self.evaluate_parser.set_defaults(func=self.run_evaluate)
+
+        description = f'Run "score" with the "{self.name}" metric.'
+        self.score_parser = subparsers.add_parser('score', description=description, help=description)
+        add_score_arguments(self.score_parser, False)
+        add_metric_arguments(self.score_parser, self.metric_type)
+        add_dataset_reader_arguments(self.score_parser)
+        self.score_parser.set_defaults(func=self.run_score)
+
+    def run_evaluate(self, args: argparse.Namespace) -> None:
+        prepare_global_logging(file_path=args.log_file, silent=args.silent)
+
+        dataset_reader = get_dataset_reader_from_argument(args.dataset_reader)
+        metric = get_metric_from_arguments(self.metric_type, args)
+        input_files = args.input_files
+
+        instances = dataset_reader.read(*input_files)
+        macro, micro_list = evaluate_instances(instances, [metric])
+
+        save_evaluation_results(macro, micro_list, args.macro_output_json, args.micro_output_jsonl, args.silent)
+
+    def run_score(self, args: argparse.Namespace) -> None:
+        prepare_global_logging(file_path=args.log_file, silent=args.silent)
+
+        dataset_reader = get_dataset_reader_from_argument(args.dataset_reader)
+        metric = get_metric_from_arguments(self.metric_type, args)
+        input_files = args.input_files
+
+        instances = dataset_reader.read(*input_files)
+        metrics_dicts = score_instances(instances, [metric])
+
+        save_score_results(metrics_dicts, args.output_jsonl, args.silent)

sacrerouge/commands/score.py

@@ -16,6 +16,41 @@
 logger = logging.getLogger(__name__)
 
 
+def add_score_arguments(parser: argparse.ArgumentParser, include_config_arguments: bool) -> None:
+    if include_config_arguments:
+        parser.add_argument(
+            'config',
+            type=str,
+            help='The config file that specifies the dataset reader and metrics'
+        )
+        parser.add_argument(
+            '--overrides',
+            type=str,
+            help='A serialized json that will override the parameters passed in "config"'
+        )
+
+    parser.add_argument(
+        'output_jsonl',
+        type=str,
+        help='The path to where the input-level metrics should be written'
+    )
+    parser.add_argument(
+        '--log-file',
+        type=str,
+        help='The file where the log should be written'
+    )
+    parser.add_argument(
+        '--silent',
+        action='store_true',
+        help='Controls whether the log should be written to stdout'
+    )
+    parser.add_argument(
+        '--include-packages',
+        nargs='+',
+        help='A list of additional packages to include'
+    )
+
+
 def _load_metrics(params: Params) -> List[Metric]:
     metrics = []
     for metric_params in params.pop('metrics'):
@@ -106,41 +141,19 @@ def score_instances(instances: List[EvalInstance], metrics: List[Metric]) -> Dic
     return metrics_dicts
 
 
+def save_score_results(metrics_dicts: Dict[str, Dict[str, Metrics]], output_file: str, silent: bool) -> None:
+    with JsonlWriter(output_file) as out:
+        for instance_id in sorted(metrics_dicts.keys()):
+            for summarizer_id in sorted(metrics_dicts[instance_id].keys()):
+                out.write(metrics_dicts[instance_id][summarizer_id])
+
+
 class ScoreSubcommand(Subcommand):
     @overrides
     def add_subparser(self, parser: argparse._SubParsersAction):
         description = 'Score all of the inputs to evaluate a metric'
         self.parser = parser.add_parser('score', description=description, help=description)
-        self.parser.add_argument(
-            'config',
-            type=str,
-            help='The config file that specifies the dataset reader and metrics'
-        )
-        self.parser.add_argument(
-            'output_jsonl',
-            type=str,
-            help='The path to where the input-level metrics should be written'
-        )
-        self.parser.add_argument(
-            '--log-file',
-            type=str,
-            help='The file where the log should be written'
-        )
-        self.parser.add_argument(
-            '--silent',
-            action='store_true',
-            help='Controls whether the log should be written to stdout'
-        )
-        self.parser.add_argument(
-            '--overrides',
-            type=str,
-            help='A serialized json that will override the parameters passed in "config"'
-        )
-        self.parser.add_argument(
-            '--include-packages',
-            nargs='+',
-            help='A list of additional packages to include'
-        )
+        add_score_arguments(self.parser, True)
         self.parser.set_defaults(func=self.run)
 
     @overrides
@@ -163,8 +176,4 @@ def run(self, args):
         instances = dataset_reader.read(*input_files)
         metrics_dicts = score_instances(instances, metrics)
 
-        # Save the results to the output file
-        with JsonlWriter(args.output_jsonl) as out:
-            for instance_id in sorted(metrics_dicts.keys()):
-                for summarizer_id in sorted(metrics_dicts[instance_id].keys()):
-                    out.write(metrics_dicts[instance_id][summarizer_id])
+        save_score_results(metrics_dicts, args.output_jsonl, args.silent)