Skip to content

Allow library to be imported and used as a module #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yeoji wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Allow library to be imported and used as a module #37

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b3d4060
WIP: Separating evaluation results from print output
Aug 31, 2020
dcd285b
WIP: Remove counters from being global variables
Sep 1, 2020
7b47d9d
Remove word tables from being global variables
Sep 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 97 additions & 63 deletions asr_evaluation/asr_evaluation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,29 +33,22 @@
min_count = 0
wer_vs_length_p = True

# For keeping track of the total number of tokens, errors, and matches
ref_token_count = 0
error_count = 0
match_count = 0
counter = 0
sent_error_count = 0

# For keeping track of word error rates by sentence length
# this is so we can see if performance is better/worse for longer
# and/or shorter sentences
lengths = []
error_rates = []
wer_bins = defaultdict(list)
wer_vs_length = defaultdict(list)
# Tables for keeping track of which words get confused with one another
insertion_table = defaultdict(int)
deletion_table = defaultdict(int)
substitution_table = defaultdict(int)
# These are the editdistance opcodes that are condsidered 'errors'
error_codes = ['replace', 'delete', 'insert']


# TODO - rename this function. Move some of it into evaluate.py?
class AsrEvaluationResults(dict):
def __getattr__(self, name):
return self[name]


def main(args):
"""Main method - this reads the hyp and ref files, and creates
editdistance.SequenceMatcher objects to compute the edit distance.
Expand All @@ -66,20 +59,47 @@ def main(args):
hypothesis file have the same number of lines. It will stop after the
shortest one runs out of lines. This should be easy to fix...
"""
global counter
results = evaluate(args)

if confusions:
print_confusions(results.insertions, results.deletions, results.substitutions)
if wer_vs_length_p:
print_wer_vs_length()

print('Sentence count: {}'.format(results.sentence_count))
print('WER: {:10.3%} ({:10d} / {:10d})'.format(results.wer, results.error_count, results.ref_token_count))
print('WRR: {:10.3%} ({:10d} / {:10d})'.format(results.wrr, results.match_count, results.ref_token_count))
print('SER: {:10.3%} ({:10d} / {:10d})'.format(results.ser, results.sent_error_count, results.sentence_count))


def evaluate(args):
set_global_variables(args)

counter = 0
ref_token_count = 0
match_count = 0
error_count = 0
sent_error_count = 0

insertion_table = defaultdict(int)
deletion_table = defaultdict(int)
substitution_table = defaultdict(int)

line_count = 0
# Loop through each line of the reference and hyp file
for ref_line, hyp_line in zip(args.ref, args.hyp):
processed_p = process_line_pair(ref_line, hyp_line, case_insensitive=args.case_insensitive,
remove_empty_refs=args.remove_empty_refs)
processed_p, ref_length, matches, errors = process_line_pair(ref_line, hyp_line, line_count, insertion_table,
deletion_table, substitution_table,
case_insensitive=args.case_insensitive,
remove_empty_refs=args.remove_empty_refs)
if processed_p:
counter += 1
if confusions:
print_confusions()
if wer_vs_length_p:
print_wer_vs_length()
line_count += 1
ref_token_count += ref_length
match_count += matches
error_count += errors

if errors != 0:
sent_error_count += 1

# Compute WER and WRR
if ref_token_count > 0:
wrr = match_count / ref_token_count
Expand All @@ -88,28 +108,38 @@ def main(args):
wrr = 0.0
wer = 0.0
# Compute SER
if counter > 0:
ser = sent_error_count / counter
if line_count > 0:
ser = sent_error_count / line_count
else:
ser = 0.0
print('Sentence count: {}'.format(counter))
print('WER: {:10.3%} ({:10d} / {:10d})'.format(wer, error_count, ref_token_count))
print('WRR: {:10.3%} ({:10d} / {:10d})'.format(wrr, match_count, ref_token_count))
print('SER: {:10.3%} ({:10d} / {:10d})'.format(ser, sent_error_count, counter))


def process_line_pair(ref_line, hyp_line, case_insensitive=False, remove_empty_refs=False):
substitutions = map(lambda item: {'ref': item[0][0], 'hyp': item[0][1], 'count': item[1]},
substitution_table.items())
return AsrEvaluationResults({
'sentence_count': line_count,
'ref_token_count': ref_token_count,
'match_count': match_count,
'error_count': error_count,
'sent_error_count': sent_error_count,
'wer': wer,
'wrr': wrr,
'ser': ser,
'insertions': insertion_table,
'deletions': deletion_table,
'substitutions': list(substitutions)
})


def process_line_pair(ref_line, hyp_line, line_count, insertion_table, deletion_table, substitution_table,
case_insensitive=False, remove_empty_refs=False):
"""Given a pair of strings corresponding to a reference and hypothesis,
compute the edit distance, print if desired, and keep track of results
in global variables.

Return true if the pair was counted, false if the pair was not counted due
to an empty reference string."""
# I don't believe these all need to be global. In any case, they shouldn't be.
global error_count
global match_count
global ref_token_count
global sent_error_count
to an empty reference string.

Also returns the ref token count, match count, and error count for this line pair."""

# Split into tokens by whitespace
ref = ref_line.split()
Expand All @@ -128,7 +158,7 @@ def process_line_pair(ref_line, hyp_line, case_insensitive=False, remove_empty_r
ref = list(map(str.lower, ref))
hyp = list(map(str.lower, hyp))
if remove_empty_refs and len(ref) == 0:
return False
return False, 0, 0, 0

# Create an object to get the edit distance, and then retrieve the
# relevant counts that we need.
Expand All @@ -137,21 +167,13 @@ def process_line_pair(ref_line, hyp_line, case_insensitive=False, remove_empty_r
matches = get_match_count(sm)
ref_length = len(ref)

# Increment the total counts we're tracking
error_count += errors
match_count += matches
ref_token_count += ref_length

if errors != 0:
sent_error_count += 1

# If we're keeping track of which words get mixed up with which others, call track_confusions
if confusions:
track_confusions(sm, ref, hyp)
track_confusions(sm, ref, hyp, insertion_table, deletion_table, substitution_table)

# If we're printing instances, do it here (in roughly the align.c format)
if print_instances_p or (print_errors_p and errors != 0):
print_instances(ref, hyp, sm, id_=id_)
print_instances(ref, hyp, sm, line_count, id_=id_)

# Keep track of the individual error rates, and reference lengths, so we
# can compute average WERs by sentence length
Expand All @@ -162,7 +184,8 @@ def process_line_pair(ref_line, hyp_line, case_insensitive=False, remove_empty_r
error_rate = float("inf")
error_rates.append(error_rate)
wer_bins[len(ref)].append(error_rate)
return True
return True, ref_length, matches, errors


def set_global_variables(args):
"""Copy argparse args into global variables."""
Expand All @@ -182,6 +205,7 @@ def set_global_variables(args):
min_count = args.min_word_count
wer_vs_length_p = args.print_wer_vs_length


def remove_head_id(ref, hyp):
"""Assumes that the ID is the begin token of the string which is common
in Kaldi but not in Sphinx."""
Expand All @@ -196,6 +220,7 @@ def remove_head_id(ref, hyp):
hyp = hyp[1:]
return ref, hyp


def remove_tail_id(ref, hyp):
"""Assumes that the ID is the final token of the string which is common
in Sphinx but not in Kaldi."""
Expand All @@ -210,13 +235,17 @@ def remove_tail_id(ref, hyp):
hyp = hyp[:-1]
return ref, hyp

def print_instances(ref, hyp, sm, id_=None):

def print_instances(ref, hyp, sm, line_count, id_=None):
"""Print a single instance of a ref/hyp pair."""
print_diff(sm, ref, hyp)
ref_sentence, hyp_sentence = get_diffs(sm, ref, hyp)
print(ref_sentence)
print(hyp_sentence)

if id_:
print(('SENTENCE {0:d} {1!s}'.format(counter + 1, id_)))
print(('SENTENCE {0:d} {1!s}'.format(line_count + 1, id_)))
else:
print('SENTENCE {0:d}'.format(counter + 1))
print('SENTENCE {0:d}'.format(line_count + 1))
# Handle cases where the reference is empty without dying
if len(ref) != 0:
correct_rate = sm.matches() / len(ref)
Expand All @@ -230,7 +259,8 @@ def print_instances(ref, hyp, sm, id_=None):
print('Correct = {0:6.1%} {1:3d} ({2:6d})'.format(correct_rate, sm.matches(), len(ref)))
print('Errors = {0:6.1%} {1:3d} ({2:6d})'.format(error_rate, sm.distance(), len(ref)))

def track_confusions(sm, seq1, seq2):

def track_confusions(sm, seq1, seq2, insertion_table, deletion_table, substitution_table):
"""Keep track of the errors in a global variable, given a sequence matcher."""
opcodes = sm.get_opcodes()
for tag, i1, i2, j1, j2 in opcodes:
Expand All @@ -248,7 +278,8 @@ def track_confusions(sm, seq1, seq2):
key = (w1, w2)
substitution_table[key] += 1

def print_confusions():

def print_confusions(insertion_table, deletion_table, substitutions_list):
"""Print the confused words that we found... grouped by insertions, deletions
and substitutions."""
if len(insertion_table) > 0:
Expand All @@ -261,11 +292,12 @@ def print_confusions():
for item in sorted(list(deletion_table.items()), key=lambda x: x[1], reverse=True):
if item[1] >= min_count:
print('{0:20s} {1:10d}'.format(*item))
if len(substitution_table) > 0:
if len(substitutions_list) > 0:
print('SUBSTITUTIONS:')
for [w1, w2], count in sorted(list(substitution_table.items()), key=lambda x: x[1], reverse=True):
if count >= min_count:
print('{0:20s} -> {1:20s} {2:10d}'.format(w1, w2, count))
for substitution in substitutions_list:
if substitution['count'] >= min_count:
print('{0:20s} -> {1:20s} {2:10d}'.format(substitution['ref'], substitution['hyp'], substitution['count']))


# TODO - For some reason I was getting two different counts depending on how I count the matches,
# so do an assertion in this code to make sure we're getting matching counts.
Expand All @@ -289,11 +321,11 @@ def get_error_count(sm):
error_lengths = [max(x[2] - x[1], x[4] - x[3]) for x in errors]
return reduce(lambda x, y: x + y, error_lengths, 0)


# TODO - This is long and ugly. Perhaps we can break it up?
# It would make more sense for this to just return the two strings...
def print_diff(sm, seq1, seq2, prefix1='REF:', prefix2='HYP:', suffix1=None, suffix2=None):
"""Given a sequence matcher and the two sequences, print a Sphinx-style
'diff' off the two."""
def get_diffs(sm, seq1, seq2, prefix1='REF:', prefix2='HYP:', suffix1=None, suffix2=None):
"""Given a sequence matcher and the two sequences, returns the Sphinx-style
'diff' of the two strings."""
ref_tokens = []
hyp_tokens = []
opcodes = sm.get_opcodes()
Expand Down Expand Up @@ -358,13 +390,15 @@ def print_diff(sm, seq1, seq2, prefix1='REF:', prefix2='HYP:', suffix1=None, suf
if prefix2: hyp_tokens.insert(0, prefix2)
if suffix1: ref_tokens.append(suffix1)
if suffix2: hyp_tokens.append(suffix2)
print(' '.join(ref_tokens))
print(' '.join(hyp_tokens))

return ' '.join(ref_tokens), ' '.join(hyp_tokens)


def mean(seq):
"""Return the average of the elements of a sequence."""
return float(sum(seq)) / len(seq) if len(seq) > 0 else float('nan')


def print_wer_vs_length():
"""Print the average word error rate for each length sentence."""
avg_wers = {length: mean(wers) for length, wers in wer_bins.items()}
Expand Down