You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
142 lines
4.6 KiB
142 lines
4.6 KiB
# -*- coding: utf-8 -*-
|
|
"""This module provides functions to calculate error rate in different level.
|
|
e.g. wer for word-level, cer for char-level.
|
|
"""
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import numpy as np
|
|
|
|
|
|
def _levenshtein_distance(ref, hyp):
|
|
"""Levenshtein distance is a string metric for measuring the difference between
|
|
two sequences. Informally, the levenshtein disctance is defined as the minimum
|
|
number of single-character edits (substitutions, insertions or deletions)
|
|
required to change one word into the other. We can naturally extend the edits to
|
|
word level when calculate levenshtein disctance for two sentences.
|
|
"""
|
|
ref_len = len(ref)
|
|
hyp_len = len(hyp)
|
|
|
|
# special case
|
|
if ref == hyp:
|
|
return 0
|
|
if ref_len == 0:
|
|
return hyp_len
|
|
if hyp_len == 0:
|
|
return ref_len
|
|
|
|
distance = np.zeros((ref_len + 1, hyp_len + 1), dtype=np.int32)
|
|
|
|
# initialize distance matrix
|
|
for j in xrange(hyp_len + 1):
|
|
distance[0][j] = j
|
|
for i in xrange(ref_len + 1):
|
|
distance[i][0] = i
|
|
|
|
# calculate levenshtein distance
|
|
for i in xrange(1, ref_len + 1):
|
|
for j in xrange(1, hyp_len + 1):
|
|
if ref[i - 1] == hyp[j - 1]:
|
|
distance[i][j] = distance[i - 1][j - 1]
|
|
else:
|
|
s_num = distance[i - 1][j - 1] + 1
|
|
i_num = distance[i][j - 1] + 1
|
|
d_num = distance[i - 1][j] + 1
|
|
distance[i][j] = min(s_num, i_num, d_num)
|
|
|
|
return distance[ref_len][hyp_len]
|
|
|
|
|
|
def wer(reference, hypothesis, ignore_case=False, delimiter=' '):
|
|
"""Calculate word error rate (WER). WER compares reference text and
|
|
hypothesis text in word-level. WER is defined as:
|
|
|
|
.. math::
|
|
WER = (Sw + Dw + Iw) / Nw
|
|
|
|
where
|
|
|
|
.. code-block:: text
|
|
|
|
Sw is the number of words subsituted,
|
|
Dw is the number of words deleted,
|
|
Iw is the number of words inserted,
|
|
Nw is the number of words in the reference
|
|
|
|
We can use levenshtein distance to calculate WER. Please draw an attention that
|
|
empty items will be removed when splitting sentences by delimiter.
|
|
|
|
:param reference: The reference sentence.
|
|
:type reference: basestring
|
|
:param hypothesis: The hypothesis sentence.
|
|
:type hypothesis: basestring
|
|
:param ignore_case: Whether case-sensitive or not.
|
|
:type ignore_case: bool
|
|
:param delimiter: Delimiter of input sentences.
|
|
:type delimiter: char
|
|
:return: Word error rate.
|
|
:rtype: float
|
|
:raises ValueError: If the reference length is zero.
|
|
"""
|
|
if ignore_case == True:
|
|
reference = reference.lower()
|
|
hypothesis = hypothesis.lower()
|
|
|
|
ref_words = filter(None, reference.split(delimiter))
|
|
hyp_words = filter(None, hypothesis.split(delimiter))
|
|
|
|
if len(ref_words) == 0:
|
|
raise ValueError("Reference's word number should be greater than 0.")
|
|
|
|
edit_distance = _levenshtein_distance(ref_words, hyp_words)
|
|
wer = float(edit_distance) / len(ref_words)
|
|
return wer
|
|
|
|
|
|
def cer(reference, hypothesis, ignore_case=False):
|
|
"""Calculate charactor error rate (CER). CER compares reference text and
|
|
hypothesis text in char-level. CER is defined as:
|
|
|
|
.. math::
|
|
CER = (Sc + Dc + Ic) / Nc
|
|
|
|
where
|
|
|
|
.. code-block:: text
|
|
|
|
Sc is the number of characters substituted,
|
|
Dc is the number of characters deleted,
|
|
Ic is the number of characters inserted
|
|
Nc is the number of characters in the reference
|
|
|
|
We can use levenshtein distance to calculate CER. Chinese input should be
|
|
encoded to unicode. Please draw an attention that the leading and tailing
|
|
white space characters will be truncated and multiple consecutive white
|
|
space characters in a sentence will be replaced by one white space character.
|
|
|
|
:param reference: The reference sentence.
|
|
:type reference: basestring
|
|
:param hypothesis: The hypothesis sentence.
|
|
:type hypothesis: basestring
|
|
:param ignore_case: Whether case-sensitive or not.
|
|
:type ignore_case: bool
|
|
:return: Character error rate.
|
|
:rtype: float
|
|
:raises ValueError: If the reference length is zero.
|
|
"""
|
|
if ignore_case == True:
|
|
reference = reference.lower()
|
|
hypothesis = hypothesis.lower()
|
|
|
|
reference = ' '.join(filter(None, reference.split(' ')))
|
|
hypothesis = ' '.join(filter(None, hypothesis.split(' ')))
|
|
|
|
if len(reference) == 0:
|
|
raise ValueError("Length of reference should be greater than 0.")
|
|
|
|
edit_distance = _levenshtein_distance(reference, hypothesis)
|
|
cer = float(edit_distance) / len(reference)
|
|
return cer
|