|
|
|
@ -10,47 +10,52 @@ import numpy as np
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _levenshtein_distance(ref, hyp):
|
|
|
|
|
"""Levenshtein distance is a string metric for measuring the difference between
|
|
|
|
|
two sequences. Informally, the levenshtein disctance is defined as the minimum
|
|
|
|
|
number of single-character edits (substitutions, insertions or deletions)
|
|
|
|
|
required to change one word into the other. We can naturally extend the edits to
|
|
|
|
|
word level when calculate levenshtein disctance for two sentences.
|
|
|
|
|
"""Levenshtein distance is a string metric for measuring the difference
|
|
|
|
|
between two sequences. Informally, the levenshtein disctance is defined as
|
|
|
|
|
the minimum number of single-character edits (substitutions, insertions or
|
|
|
|
|
deletions) required to change one word into the other. We can naturally
|
|
|
|
|
extend the edits to word level when calculate levenshtein disctance for
|
|
|
|
|
two sentences.
|
|
|
|
|
"""
|
|
|
|
|
ref_len = len(ref)
|
|
|
|
|
hyp_len = len(hyp)
|
|
|
|
|
m = len(ref)
|
|
|
|
|
n = len(hyp)
|
|
|
|
|
|
|
|
|
|
# special case
|
|
|
|
|
if ref == hyp:
|
|
|
|
|
return 0
|
|
|
|
|
if ref_len == 0:
|
|
|
|
|
return hyp_len
|
|
|
|
|
if hyp_len == 0:
|
|
|
|
|
return ref_len
|
|
|
|
|
if m == 0:
|
|
|
|
|
return n
|
|
|
|
|
if n == 0:
|
|
|
|
|
return m
|
|
|
|
|
|
|
|
|
|
distance = np.zeros((ref_len + 1, hyp_len + 1), dtype=np.int32)
|
|
|
|
|
if m < n:
|
|
|
|
|
ref, hyp = hyp, ref
|
|
|
|
|
m, n = n, m
|
|
|
|
|
|
|
|
|
|
# use O(min(m, n)) space
|
|
|
|
|
distance = np.zeros((2, n + 1), dtype=np.int32)
|
|
|
|
|
|
|
|
|
|
# initialize distance matrix
|
|
|
|
|
for j in xrange(hyp_len + 1):
|
|
|
|
|
for j in xrange(n + 1):
|
|
|
|
|
distance[0][j] = j
|
|
|
|
|
for i in xrange(ref_len + 1):
|
|
|
|
|
distance[i][0] = i
|
|
|
|
|
|
|
|
|
|
# calculate levenshtein distance
|
|
|
|
|
for i in xrange(1, ref_len + 1):
|
|
|
|
|
for j in xrange(1, hyp_len + 1):
|
|
|
|
|
for i in xrange(1, m + 1):
|
|
|
|
|
distance[i % 2][0] = i
|
|
|
|
|
for j in xrange(1, n + 1):
|
|
|
|
|
if ref[i - 1] == hyp[j - 1]:
|
|
|
|
|
distance[i][j] = distance[i - 1][j - 1]
|
|
|
|
|
distance[i % 2][j] = distance[(i - 1) % 2][j - 1]
|
|
|
|
|
else:
|
|
|
|
|
s_num = distance[i - 1][j - 1] + 1
|
|
|
|
|
i_num = distance[i][j - 1] + 1
|
|
|
|
|
d_num = distance[i - 1][j] + 1
|
|
|
|
|
distance[i][j] = min(s_num, i_num, d_num)
|
|
|
|
|
s_num = distance[(i - 1) % 2][j - 1] + 1
|
|
|
|
|
i_num = distance[i % 2][j - 1] + 1
|
|
|
|
|
d_num = distance[(i - 1) % 2][j] + 1
|
|
|
|
|
distance[i % 2][j] = min(s_num, i_num, d_num)
|
|
|
|
|
|
|
|
|
|
return distance[ref_len][hyp_len]
|
|
|
|
|
return distance[m % 2][n]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def wer(reference, hypothesis, ignore_case=False, delimiter=' '):
|
|
|
|
|
"""Calculate word error rate (WER). WER compares reference text and
|
|
|
|
|
"""Calculate word error rate (WER). WER compares reference text and
|
|
|
|
|
hypothesis text in word-level. WER is defined as:
|
|
|
|
|
|
|
|
|
|
.. math::
|
|
|
|
@ -65,8 +70,8 @@ def wer(reference, hypothesis, ignore_case=False, delimiter=' '):
|
|
|
|
|
Iw is the number of words inserted,
|
|
|
|
|
Nw is the number of words in the reference
|
|
|
|
|
|
|
|
|
|
We can use levenshtein distance to calculate WER. Please draw an attention that
|
|
|
|
|
empty items will be removed when splitting sentences by delimiter.
|
|
|
|
|
We can use levenshtein distance to calculate WER. Please draw an attention
|
|
|
|
|
that empty items will be removed when splitting sentences by delimiter.
|
|
|
|
|
|
|
|
|
|
:param reference: The reference sentence.
|
|
|
|
|
:type reference: basestring
|
|
|
|
@ -95,7 +100,7 @@ def wer(reference, hypothesis, ignore_case=False, delimiter=' '):
|
|
|
|
|
return wer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def cer(reference, hypothesis, ignore_case=False):
|
|
|
|
|
def cer(reference, hypothesis, ignore_case=False, remove_space=False):
|
|
|
|
|
"""Calculate charactor error rate (CER). CER compares reference text and
|
|
|
|
|
hypothesis text in char-level. CER is defined as:
|
|
|
|
|
|
|
|
|
@ -111,10 +116,10 @@ def cer(reference, hypothesis, ignore_case=False):
|
|
|
|
|
Ic is the number of characters inserted
|
|
|
|
|
Nc is the number of characters in the reference
|
|
|
|
|
|
|
|
|
|
We can use levenshtein distance to calculate CER. Chinese input should be
|
|
|
|
|
encoded to unicode. Please draw an attention that the leading and tailing
|
|
|
|
|
white space characters will be truncated and multiple consecutive white
|
|
|
|
|
space characters in a sentence will be replaced by one white space character.
|
|
|
|
|
We can use levenshtein distance to calculate CER. Chinese input should be
|
|
|
|
|
encoded to unicode. Please draw an attention that the leading and tailing
|
|
|
|
|
space characters will be truncated and multiple consecutive space
|
|
|
|
|
characters in a sentence will be replaced by one space character.
|
|
|
|
|
|
|
|
|
|
:param reference: The reference sentence.
|
|
|
|
|
:type reference: basestring
|
|
|
|
@ -122,6 +127,8 @@ def cer(reference, hypothesis, ignore_case=False):
|
|
|
|
|
:type hypothesis: basestring
|
|
|
|
|
:param ignore_case: Whether case-sensitive or not.
|
|
|
|
|
:type ignore_case: bool
|
|
|
|
|
:param remove_space: Whether remove internal space characters
|
|
|
|
|
:type remove_space: bool
|
|
|
|
|
:return: Character error rate.
|
|
|
|
|
:rtype: float
|
|
|
|
|
:raises ValueError: If the reference length is zero.
|
|
|
|
@ -130,8 +137,12 @@ def cer(reference, hypothesis, ignore_case=False):
|
|
|
|
|
reference = reference.lower()
|
|
|
|
|
hypothesis = hypothesis.lower()
|
|
|
|
|
|
|
|
|
|
reference = ' '.join(filter(None, reference.split(' ')))
|
|
|
|
|
hypothesis = ' '.join(filter(None, hypothesis.split(' ')))
|
|
|
|
|
join_char = ' '
|
|
|
|
|
if remove_space == True:
|
|
|
|
|
join_char = ''
|
|
|
|
|
|
|
|
|
|
reference = join_char.join(filter(None, reference.split(' ')))
|
|
|
|
|
hypothesis = join_char.join(filter(None, hypothesis.split(' ')))
|
|
|
|
|
|
|
|
|
|
if len(reference) == 0:
|
|
|
|
|
raise ValueError("Length of reference should be greater than 0.")
|
|
|
|
|