Hui Zhang
5 years ago
parent
12c01f39aa
commit
64f0bad5ca
@ -0,0 +1,127 @@
|
|||||||
|
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
"""format manifest with more metadata."""
|
||||||
|
import argparse
|
||||||
|
import functools
|
||||||
|
import json
|
||||||
|
from collections import Counter
|
||||||
|
import os
|
||||||
|
import copy
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from deepspeech.frontend.utility import read_manifest
|
||||||
|
from deepspeech.frontend.utility import UNK
|
||||||
|
from deepspeech.frontend.utility import BLANK
|
||||||
|
from deepspeech.frontend.utility import SOS
|
||||||
|
from deepspeech.utils.utility import add_arguments
|
||||||
|
from deepspeech.utils.utility import print_arguments
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
add_arg = functools.partial(add_arguments, argparser=parser)
|
||||||
|
# yapf: disable
|
||||||
|
add_arg('feat_type', str, "raw", "speech feature type, e.g. raw(wav, flac), kaldi")
|
||||||
|
add_arg('unit_type', str, "character", "Unit type, e.g. character, word, bpe")
|
||||||
|
add_arg('vocab_path', str,
|
||||||
|
'examples/librispeech/data/vocab.txt',
|
||||||
|
"Filepath to write the vocabulary.")
|
||||||
|
add_arg('manifest_paths', str,
|
||||||
|
None,
|
||||||
|
"Filepaths of manifests for building vocabulary. "
|
||||||
|
"You can provide multiple manifest files.",
|
||||||
|
nargs='+',
|
||||||
|
required=True)
|
||||||
|
# bpe
|
||||||
|
add_arg('bpe_model_prefix', str, "bpe_model_%(bpe_mode)_%(count_threshold)", "bpe model prefix, only need when `unit_type` is bpe")
|
||||||
|
add_arg('output_path', str, None, "filepath of formated manifest.", required=True)
|
||||||
|
# yapf: disable
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print_arguments(args)
|
||||||
|
|
||||||
|
# read vocab
|
||||||
|
vocab = dict()
|
||||||
|
with open(args.vocab_path, 'r', encoding='utf-8') as fin:
|
||||||
|
for line in fin:
|
||||||
|
token = line.strip()
|
||||||
|
vocab[token] = len(vocab)
|
||||||
|
vocab_size = len(vocab)
|
||||||
|
|
||||||
|
fout = open(args.output_path, 'w', encoding='utf-8')
|
||||||
|
|
||||||
|
if args.unit_type != 'bpe':
|
||||||
|
for manifest_path in args.manifest_paths:
|
||||||
|
manifest_jsons = read_manifest(manifest_path)
|
||||||
|
for line_json in manifest_jsons:
|
||||||
|
tokens = []
|
||||||
|
tokenids = []
|
||||||
|
if args.unit_type == 'character':
|
||||||
|
for char in line_json['text']:
|
||||||
|
tokens.append(char)
|
||||||
|
tokenids.append(vocab[char])
|
||||||
|
elif args.unit_type == 'word':
|
||||||
|
for word in line_json['text'].split():
|
||||||
|
tokens.append(word)
|
||||||
|
tokenids.append(vocab[word])
|
||||||
|
line_json['token'] = tokens
|
||||||
|
line_json['token_id'] = tokenids
|
||||||
|
line_json['token_shape'] = (len(tokenids), vocab_size)
|
||||||
|
fout.write(json.dumps(line_json) + '\n')
|
||||||
|
else:
|
||||||
|
import sentencepiece as spm
|
||||||
|
|
||||||
|
# encode
|
||||||
|
sp = spm.SentencePieceProcessor()
|
||||||
|
sp.Load(args.bpe_model_prefix + '.model')
|
||||||
|
|
||||||
|
def valid(line):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def encode(l):
|
||||||
|
return sp.EncodeAsPieces(l)
|
||||||
|
|
||||||
|
def encode_line(line):
|
||||||
|
line = line.strip()
|
||||||
|
if len(line) > 0:
|
||||||
|
line = encode(line)
|
||||||
|
if valid(line):
|
||||||
|
return line
|
||||||
|
else:
|
||||||
|
stats["num_filtered"] += 1
|
||||||
|
else:
|
||||||
|
stats["num_empty"] += 1
|
||||||
|
return None
|
||||||
|
|
||||||
|
for manifest_path in args.manifest_paths:
|
||||||
|
manifest_jsons = read_manifest(manifest_path)
|
||||||
|
for line_json in manifest_jsons:
|
||||||
|
line = line_json['text']
|
||||||
|
tokens = []
|
||||||
|
tokenids = []
|
||||||
|
enc_line = encode_line(line)
|
||||||
|
for code in enc_line:
|
||||||
|
tokens.append(code)
|
||||||
|
tokenids.append(vocab[code])
|
||||||
|
#print(code, vocab[code])
|
||||||
|
line_json['token'] = tokens
|
||||||
|
line_json['token_id'] = tokenids
|
||||||
|
line_json['token_shape'] = (len(tokenids), vocab_size)
|
||||||
|
fout.write(json.dumps(line_json) + '\n')
|
||||||
|
|
||||||
|
fout.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Loading…
Reference in new issue