Merge pull request #94 from boeddeker/main

add more normalizers, "lower,rm([^a-z0-9 ])" and chime6/7/8

Author: boeddeker

meeteval/wer/__main__.py

@@ -5,6 +5,7 @@
 import os
 import re
 import decimal
+import textwrap
 from pathlib import Path
 
 import meeteval.io
@@ -608,13 +609,11 @@ def add_argument(self, command_parser, name, p):
                 nargs='+', action=self.extend_action,
             )
         elif name == 'normalizer':
+            from meeteval.wer.api import normalizers
             command_parser.add_argument(
                 '--normalizer',
-                help='A normalizer that is applied to the transcript.\n'
-                     'Choices:\n'
-                     '- None: Do nothing (default)\n'
-                     '- lower,rm(.?!,): Lowercase the transcript and remove punctuations (.,?!).',
-                choices=[None, 'lower,rm(.?!,)'],
+                help=textwrap.dedent(normalizers.__doc__),
+                choices=[None, *normalizers.keys()],
             )
         elif name == 'partial':
             command_parser.add_argument(

meeteval/wer/api.py

@@ -39,6 +39,87 @@ def _maybe_load(paths, file_format) -> meeteval.io.SegLST:
             raise TypeError(type(paths), paths) from e
 
 
+class _Normalizers:
+    """
+    A normalizer that is applied to the transcript.
+    Choices:
+    - None: Do nothing (default)
+    - lower,rm(.?!,): Lowercase the transcript and remove punctuations (.,?!).
+    - lower,rm([^a-z0-9 ]): Lowercase the transcript and remove all characters that are not a-z, 0-9, or space.
+    - chime6: Normalize the transcript according to the CHiME-6 challenge.
+    - chime7: Normalize the transcript according to the CHiME-7 challenge.
+              In contrast to chime6, words like "hmm" are normalized,
+              e.g. "hm", "hmm" and "hmmm" get "hmmm".
+    - chime8: Normalize the transcript according to the CHiME-8 challenge.
+              Removes words like "hmm", converts integers to words,
+              and much more.
+    """
+    # Note: The text above is used as CLI help text.
+    #       So the None is not a real option for this class, but the default
+    #       value for the normalizer argument.
+
+    def keys(self):
+        return [
+            'lower,rm(.?!,)',
+            'lower,rm([^a-z0-9 ])',
+            'chime6',
+            'chime7',
+            'chime8',
+        ]
+
+    def __getitem__(self, normalizer):
+        """
+        >>> def test(normalizer):
+        ...     seg = {'words': 'Hello, World! äèア😊 hm hmm hmmm [noise] [inaudible] wi-fi word-word 1.1 11 Mr. then,\" said'}
+        ...     return _Normalizers()[normalizer](seg)['words']
+        >>> test('lower,rm(.?!,)')
+        'hello world äèア😊 hm hmm hmmm [noise] [inaudible] wi-fi word-word 11 11 mr then" said'
+        >>> test('lower,rm([^a-z0-9 ])')
+        'hello world hm hmm hmmm noise inaudible wifi wordword 11 11 mr then said'
+        >>> test('chime8')
+        'hello world aeア wifi word word 1.1 eleven mister then said'
+        >>> test('chime7')
+        'hello world äèア😊 hmmm hmmm hmmm wi-fi word-word 11 11 mr then said'
+        >>> test('chime6')
+        'hello world äèア😊 hm hmm hmmm wi-fi word-word 11 11 mr then said'
+        """
+        if normalizer == 'lower,rm(.?!,)':
+            def normalizer(seg):
+                seg['words'] = seg['words'].lower().replace('.', '').replace('?', '').replace('!', '').replace(',', '')
+                return seg
+        elif normalizer == 'lower,rm([^a-z0-9 ])':
+            r = re.compile('[^a-z0-9 ]')
+            r2 = re.compile(r'\s{2,}')
+            def normalizer(seg):
+                seg['words'] = r2.sub(' ', r.sub('', seg['words'].lower()).strip())
+                return seg
+        elif normalizer in ['chime8', 'chime7', 'chime6']:
+            from chime_utils.text_norm import get_txt_norm  # pip install git+https://github.com/chimechallenge/chime-utils
+            chime_utils_normalizer = get_txt_norm(normalizer)
+
+            # Note:
+            #     chime6 and chime7 remove all words, when [redacted] is
+            #     present.
+
+            def normalizer(seg):
+                words = chime_utils_normalizer(seg['words'])
+                if words != chime_utils_normalizer(seg['words']):
+                    raise RuntimeError(
+                        f'You discovered an idempotence bug in the chime_utils normalizer: {normalizer!r}.\n'
+                        f'Original:         {seg["words"]}\n'
+                        f'Normalized:       {words}\n'
+                        f'Normalized again: {chime_utils_normalizer(words)}'
+                    )
+                seg['words'] = words
+                return seg
+        else:
+            raise ValueError(f'Unknown normalizer: {normalizer}. Available normalizers: {self.keys()}')
+        return normalizer
+
+
+normalizers = _Normalizers()
+
+
 def _load_texts(
         reference_paths: 'list[str]',
         hypothesis_paths: 'list[str]',
@@ -89,12 +170,7 @@ def filter(s):
         hypothesis = hypothesis.filter_by_uem(uem)
 
     if normalizer is not None:
-        if normalizer == 'lower,rm(.?!,)':
-            def normalizer(seg):
-                seg['words'] = seg['words'].lower().replace('.', '').replace('?', '').replace('!', '').replace(',', '')
-                return seg
-        else:
-            raise NotImplementedError(normalizer)
+        normalizer = normalizers[normalizer]
         reference = reference.map(normalizer)
         hypothesis = hypothesis.map(normalizer)
 

requirements.txt

@@ -1,3 +1,4 @@
 editdistance
 pytest
-hypothesis
+hypothesis
+chime-utils @ git+https://github.com/chimechallenge/chime-utils