Enhance SacreBLEU integration (#125)

Author: jvamvas

doc/metrics/bleu.md

@@ -1,7 +1,7 @@
 # BLEU
 Our BLEU implementation is a wrapper around [SacreBLEU](https://github.com/mjpost/sacrebleu).
 Although BLEU was intended to be a corpus-level metric, we have only implemented the sentence-level version.
-See `sacrebleu.sentence_bleu` for details.
+See `sacrebleu.BLEU` for details.
 The metric is registered under the name `sent-bleu`.
 
 ## Setting Up

doc/metrics/chrf.md

@@ -0,0 +1,7 @@
+# chrF/chrF++
+We use the chrF/chrF++ implementation of [SacreBLEU](https://github.com/mjpost/sacrebleu).
+See `sacrebleu.CHRF` for details.
+The metric is registered under the name `chrf`.
+
+## Setting Up
+No setup is required.

requirements.txt

@@ -9,4 +9,4 @@ pytest
 repro==0.1.1
 requests
 scipy>=1.5.2
-sacrebleu==1.5.1
+sacrebleu==2.0.0

sacrerouge/metrics/__init__.py

@@ -6,6 +6,7 @@
 from sacrerouge.metrics.blanc import Blanc
 from sacrerouge.metrics.bleu import SentBleu
 from sacrerouge.metrics.bleurt import Bleurt
+from sacrerouge.metrics.chrf import ChrF
 from sacrerouge.metrics.meteor import Meteor
 from sacrerouge.metrics.moverscore import MoverScore
 from sacrerouge.metrics.pyramid_score import PyramidScore

sacrerouge/metrics/bleu.py

@@ -1,5 +1,5 @@
 import logging
-from sacrebleu import sentence_bleu
+from sacrebleu import BLEU
 from typing import List
 
 from sacrerouge.common.util import flatten
@@ -16,11 +16,13 @@ class SentBleu(ReferenceBasedMetric):
     def __init__(self, **kwargs) -> None:
         """
         Args:
-            **kwargs: The kwargs that will be passed to `sacrebleu.sentence_bleu`. See
+            **kwargs: The kwargs that will be passed to `sacrebleu.BLEU`. See
                 the `sacrebleu` documentation for details.
         """
         super().__init__()
-        self.kwargs = kwargs
+        if "effective_order" not in kwargs:
+            kwargs["effective_order"] = True
+        self.bleu = BLEU(**kwargs)
 
     def score_multi_all(
         self,
@@ -34,6 +36,6 @@ def score_multi_all(
             scores_list.append([])
             for summary in summaries:
                 summary = flatten(summary)
-                score = sentence_bleu(summary, references, **self.kwargs)
+                score = self.bleu.sentence_score(summary, references)
                 scores_list[-1].append(MetricsDict({'sent-bleu': score.score}))
-        return scores_list
+        return scores_list

sacrerouge/metrics/chrf.py

@@ -0,0 +1,40 @@
+import logging
+from typing import List
+
+import sacrebleu
+
+from sacrerouge.common.util import flatten
+from sacrerouge.data import MetricsDict
+from sacrerouge.data.types import ReferenceType
+from sacrerouge.data.types import SummaryType
+from sacrerouge.metrics import Metric, ReferenceBasedMetric
+
+logger = logging.getLogger(__name__)
+
+
+@Metric.register('chrf')
+class ChrF(ReferenceBasedMetric):
+    def __init__(self, **kwargs) -> None:
+        """
+        Args:
+            **kwargs: The kwargs that will be passed to `sacrebleu.CHRF`. See
+                the `sacrebleu` documentation for details.
+        """
+        super().__init__()
+        self.chrf = sacrebleu.CHRF(**kwargs)
+
+    def score_multi_all(
+        self,
+        summaries_list: List[List[SummaryType]],
+        references_list: List[List[ReferenceType]],
+        **kwargs,
+    ) -> List[List[MetricsDict]]:
+        scores_list = []
+        for summaries, references in zip(summaries_list, references_list):
+            references = [flatten(reference) for reference in references]
+            scores_list.append([])
+            for summary in summaries:
+                summary = flatten(summary)
+                score = self.chrf.sentence_score(summary, references)
+                scores_list[-1].append(MetricsDict({'chrf': score.score}))
+        return scores_list

sacrerouge/tests/metrics/chrf_test.py

@@ -0,0 +1,52 @@
+from sacrerouge.common.testing.metric_test_cases import ReferenceBasedMetricTestCase
+from sacrerouge.common.testing.util import sacrerouge_command_exists
+from sacrerouge.metrics.chrf import ChrF
+
+
+class TestChrF(ReferenceBasedMetricTestCase):
+    def test_chrf(self):
+        # This is a regression test, not necessarily a test for correctness
+        metric = ChrF()
+        expected_output = [
+            {'chrf': 43.85388187268078},
+            {'chrf': 52.60988684061151},
+            {'chrf': 49.65855270200033},
+            {'chrf': 41.43422564802467},
+            {'chrf': 42.53477752245672},
+            {'chrf': 41.91433502501302},
+            {'chrf': 49.91729287293663},
+            {'chrf': 49.63089847829706},
+            {'chrf': 43.484464138558174},
+            {'chrf': 56.073961048238665},
+            {'chrf': 48.94445553665466},
+            {'chrf': 46.85719853900437}
+        ]
+        super().assert_expected_output(metric, expected_output)
+
+    def test_chrf_examples(self):
+        chrf = ChrF()
+
+        hypothesis = 'The dog bit the man.'
+        references = ['The dog bit the man.', 'The dog had bit the man.']
+        expected = 100.0
+        actual = chrf.score(hypothesis, references)['chrf']
+        self.assertAlmostEqual(expected, actual)
+
+        hypothesis = 'It wasn\'t surprising.'
+        references = ['It was not unexpected.', 'No one was surprised.']
+        expected = 35.34635301425291
+        actual = chrf.score(hypothesis, references)['chrf']
+        self.assertAlmostEqual(expected, actual)
+
+        hypothesis = 'The man had just bitten him.'
+        references = ['The man bit him first.', 'The man had bitten the dog.']
+        expected = 51.87740799504779
+        actual = chrf.score(hypothesis, references)['chrf']
+        self.assertAlmostEqual(expected, actual)
+
+    def test_chrf_order_invariant(self):
+        metric = ChrF()
+        self.assert_order_invariant(metric)
+
+    def test_command_exists(self):
+        assert sacrerouge_command_exists(['chrf'])