List Unlisted Pronunciation

Author: graphemecluster

scripts/list_unlist_pron.py

@@ -1,6 +1,5 @@
 from glob import iglob
 from operator import itemgetter
-from itertools import accumulate
 from collections import defaultdict
 
 non_han = {*'，： '}
@@ -18,9 +17,9 @@ def column_start(column):
 for filename in ['char.csv', 'scripts/ignore.csv']:
     with open(filename, encoding='utf-8') as f:
         assert next(f).startswith('char,jyutping')
-        listed |= {tuple(line.rstrip('\n').split(',')[:1]) for line in f}
+        listed |= {tuple(line.rstrip('\n').split(',')[:2]) for line in f}
 
-unlisted = defaultdict(list)
+unlisted = defaultdict(lambda: defaultdict(list))
 for filename in iglob("*.csv"):
     if filename == 'char.csv':
         continue
@@ -32,58 +31,49 @@ def column_start(column):
         romans_column_index = column_names.index("jyutping")
 
         for line_num, line in enumerate(f, 2):
-            def yield_chars(chars, start):
+            def yield_chars(chars):
                 s = ''
                 is_punctuation = True
-                for i, char in enumerate(chars, start):
+                for char in chars:
                     if (char in non_han) != is_punctuation:
                         if s:
                             if not is_punctuation:
-                                yield (i, s)
+                                yield s
                             s = ''
                         is_punctuation = not is_punctuation
                     if not is_punctuation and s and not (is_ascii_lowercase_letter(char) and is_ascii_letter(s[-1])):
-                        yield (i, s)
+                        yield s
                         s = ''
                     s += char
-                if s:
-                    i += 1
-                    if not is_punctuation:
-                        yield (i, s)
+                if s and not is_punctuation:
+                    yield s
 
-            def yield_romans(romans, start):
+            def yield_romans(romans):
                 s = ''
                 is_space = True
-                for i, roman in enumerate(romans, start):
+                for roman in romans:
                     if (roman == ' ') != is_space:
                         if s:
                             if not is_space:
-                                yield (i, s)
+                                yield s
                             s = ''
                         is_space = not is_space
                     s += roman
-                if s:
-                    i += 1
-                    if not is_space:
-                        yield (i, s)
+                if s and not is_space:
+                    yield s
 
             columns = line.rstrip('\n').split(',')
-            columns_start = list(accumulate(map(column_start, columns), initial=0))
 
             if chars_column_index < len(columns) > romans_column_index:
                 chars_orig = columns[chars_column_index]
                 romans_orig = columns[romans_column_index]
-                chars = yield_chars(chars_orig, columns_start[chars_column_index])
-                romans = yield_romans(romans_orig, columns_start[romans_column_index])
-                for char_i, char in chars:
-                    try:
-                        roman_i, roman = next(romans)
-                    except StopIteration:
-                        break
-                    if (char, roman) not in listed:
-                        unlisted[(char, roman)].append(f'{filename}:{line_num}')
+                chars = yield_chars(chars_orig)
+                romans = yield_romans(romans_orig)
+                for pair in zip(chars, romans):
+                    if pair not in listed:
+                        unlisted[pair][filename].append(f'{line_num} {chars_orig} {romans_orig}')
 
 with open('scripts/unlisted.csv', 'w', encoding='utf-8') as f:
     print('char', 'jyutping', 'location', sep=',', file=f)
     for (char, roman), locations in sorted(unlisted.items(), key=itemgetter(0)):
-        print(char, roman, ';'.join(locations), sep=',', file=f)
+        print(char, roman, ';'.join(f'{filename}:[{"|".join(dict.fromkeys(lines))}]' for filename, lines in locations.items()), sep=',', file=f)