You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
PaddleSpeech/utils/fst/prepare_dict.py

112 lines
3.5 KiB

3 years ago
#!/usr/bin/env python3
import argparse
def main(args):
3 years ago
# load vocab file
# line: token
3 years ago
unit_table = set()
with open(args.unit_file, 'r') as fin:
for line in fin:
unit = line.strip()
unit_table.add(unit)
3 years ago
def contain_oov(units):
3 years ago
"""token not in vocab
Args:
units (str): token
Returns:
bool: True token in voca, else False.
"""
3 years ago
for unit in units:
if unit not in unit_table:
return True
return False
3 years ago
# load spm model, for English
3 years ago
bpemode = args.bpemodel
if bpemode:
import sentencepiece as spm
sp = spm.SentencePieceProcessor()
sp.Load(sys.bpemodel)
3 years ago
# used to filter polyphone and invalid word
3 years ago
lexicon_table = set()
3 years ago
in_n = 0 # in lexicon word count
3 years ago
out_n = 0 # out lexicon word cout
3 years ago
with open(args.in_lexicon, 'r') as fin, \
open(args.out_lexicon, 'w') as fout:
3 years ago
for line in fin:
word = line.split()[0]
3 years ago
in_n += 1
3 years ago
if word == 'SIL' and not bpemode: # `sil` might be a valid piece in bpemodel
3 years ago
# filter 'SIL' for mandarin, keep it in English
3 years ago
continue
elif word == '<SPOKEN_NOISE>':
3 years ago
# filter <SPOKEN_NOISE>
3 years ago
continue
else:
# each word only has one pronunciation for e2e system
if word in lexicon_table:
continue
if bpemode:
3 years ago
# for english
3 years ago
pieces = sp.EncodeAsPieces(word)
if contain_oov(pieces):
print('Ignoring words {}, which contains oov unit'.
format(''.join(word).strip('')))
continue
3 years ago
# word is piece list, which not have <unk> piece, filter out by `contain_oov(pieces)`
3 years ago
chars = ' '.join(
[p if p in unit_table else '<unk>' for p in pieces])
else:
# ignore words with OOV
if contain_oov(word):
print('Ignoring words {}, which contains oov unit'.
format(word))
continue
# Optional, append ▁ in front of english word
# we assume the model unit of our e2e system is char now.
if word.encode('utf8').isalpha() and '' in unit_table:
word = '' + word
3 years ago
3 years ago
chars = ' '.join(word) # word is a char list
fout.write('{} {}\n'.format(word, chars))
lexicon_table.add(word)
3 years ago
out_n += 1
3 years ago
3 years ago
print(
f"Filter lexicon by unit table: filter out {in_n - out_n}, {out_n}/{in_n}"
)
3 years ago
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='FST: preprae e2e(char/spm) dict')
parser.add_argument(
'--unit_file',
required=True,
help='e2e model unit file(lang_char.txt/vocab.txt). line: char/spm_pices'
)
parser.add_argument(
'--in_lexicon',
required=True,
help='raw lexicon file. line: word ph0 ... phn')
parser.add_argument(
'--out_lexicon',
required=True,
help='output lexicon file. line: word char0 ... charn')
parser.add_argument('--bpemodel', default=None, help='bpemodel')
args = parser.parse_args()
print(args)
main(args)