From 794c77dd380dd40c4db20eb0bcb40b0c0d3efef4 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 19 May 2021 17:04:24 +0800 Subject: [PATCH] add tarball utils (#626) --- deepspeech/exps/u2/bin/alignment.py | 48 +++++++++++++++++++++++++++++ deepspeech/frontend/normalizer.py | 3 +- env.sh | 2 +- examples/aishell/s0/path.sh | 4 +-- examples/aishell/s1/path.sh | 2 +- examples/librispeech/s0/path.sh | 2 +- examples/librispeech/s1/path.sh | 2 +- examples/spm/path.sh | 2 +- examples/tiny/s0/path.sh | 2 +- examples/tiny/s1/path.sh | 2 +- requirements.txt | 1 + utils/avg_model.py | 0 utils/build_vocab.py | 0 utils/compute_mean_std.py | 0 utils/format_data.py | 0 utils/ngram_train.sh | 0 utils/tarball.sh | 26 ++++++++++++++++ utils/zh_tn.py | 0 18 files changed, 85 insertions(+), 11 deletions(-) create mode 100644 deepspeech/exps/u2/bin/alignment.py mode change 100644 => 100755 utils/avg_model.py mode change 100644 => 100755 utils/build_vocab.py mode change 100644 => 100755 utils/compute_mean_std.py mode change 100644 => 100755 utils/format_data.py mode change 100644 => 100755 utils/ngram_train.sh create mode 100755 utils/tarball.sh mode change 100644 => 100755 utils/zh_tn.py diff --git a/deepspeech/exps/u2/bin/alignment.py b/deepspeech/exps/u2/bin/alignment.py new file mode 100644 index 000000000..c1c9582f8 --- /dev/null +++ b/deepspeech/exps/u2/bin/alignment.py @@ -0,0 +1,48 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Alignment for U2 model.""" +from deepspeech.exps.u2.config import get_cfg_defaults +from deepspeech.exps.u2.model import U2Tester as Tester +from deepspeech.training.cli import default_argument_parser +from deepspeech.utils.utility import print_arguments + + +def main_sp(config, args): + exp = Tester(config, args) + exp.setup() + exp.run_align() + + +def main(config, args): + main_sp(config, args) + + +if __name__ == "__main__": + parser = default_argument_parser() + args = parser.parse_args() + print_arguments(args, globals()) + + # https://yaml.org/type/float.html + config = get_cfg_defaults() + if args.config: + config.merge_from_file(args.config) + if args.opts: + config.merge_from_list(args.opts) + config.freeze() + print(config) + if args.dump_config: + with open(args.dump_config, 'w') as f: + print(config, file=f) + + main(config, args) diff --git a/deepspeech/frontend/normalizer.py b/deepspeech/frontend/normalizer.py index deaa87c8c..6b224080b 100644 --- a/deepspeech/frontend/normalizer.py +++ b/deepspeech/frontend/normalizer.py @@ -179,8 +179,7 @@ class FeatureNormalizer(object): wav_number += batch_size if wav_number % 1000 == 0: - logger.info('process {} wavs,{} frames'.format(wav_number, - all_number)) + logger.info(f'process {wav_number} wavs,{all_number} frames.') self.cmvn_info = { 'mean_stat': list(all_mean_stat.tolist()), diff --git a/env.sh b/env.sh index 5486a1da3..c5acd0112 100644 --- a/env.sh +++ b/env.sh @@ -1,6 +1,6 @@ export MAIN_ROOT=${PWD} -export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH} export LC_ALL=C # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C diff --git a/examples/aishell/s0/path.sh b/examples/aishell/s0/path.sh index f67893266..552b96783 100644 --- a/examples/aishell/s0/path.sh +++ b/examples/aishell/s0/path.sh @@ -1,10 +1,10 @@ export MAIN_ROOT=${PWD}/../../../ -export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH} export LC_ALL=C # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C -export PYTHONIOENCODING=UTF-8 +export PYTHONIOENCODING=UTF-8 export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH} export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib/ diff --git a/examples/aishell/s1/path.sh b/examples/aishell/s1/path.sh index 14e4fc3ec..30adb6ca0 100644 --- a/examples/aishell/s1/path.sh +++ b/examples/aishell/s1/path.sh @@ -1,6 +1,6 @@ export MAIN_ROOT=${PWD}/../../../ -export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH} export LC_ALL=C # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C diff --git a/examples/librispeech/s0/path.sh b/examples/librispeech/s0/path.sh index 44b178383..777da29ef 100644 --- a/examples/librispeech/s0/path.sh +++ b/examples/librispeech/s0/path.sh @@ -1,6 +1,6 @@ export MAIN_ROOT=${PWD}/../../../ -export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH} export LC_ALL=C # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C diff --git a/examples/librispeech/s1/path.sh b/examples/librispeech/s1/path.sh index 14e4fc3ec..30adb6ca0 100644 --- a/examples/librispeech/s1/path.sh +++ b/examples/librispeech/s1/path.sh @@ -1,6 +1,6 @@ export MAIN_ROOT=${PWD}/../../../ -export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH} export LC_ALL=C # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C diff --git a/examples/spm/path.sh b/examples/spm/path.sh index 1617b592c..9da641e19 100644 --- a/examples/spm/path.sh +++ b/examples/spm/path.sh @@ -1,6 +1,6 @@ export MAIN_ROOT=${PWD}/../../ -export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH} export LC_ALL=C # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C diff --git a/examples/tiny/s0/path.sh b/examples/tiny/s0/path.sh index 44b178383..777da29ef 100644 --- a/examples/tiny/s0/path.sh +++ b/examples/tiny/s0/path.sh @@ -1,6 +1,6 @@ export MAIN_ROOT=${PWD}/../../../ -export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH} export LC_ALL=C # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C diff --git a/examples/tiny/s1/path.sh b/examples/tiny/s1/path.sh index 14e4fc3ec..30adb6ca0 100644 --- a/examples/tiny/s1/path.sh +++ b/examples/tiny/s1/path.sh @@ -1,6 +1,6 @@ export MAIN_ROOT=${PWD}/../../../ -export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH} export LC_ALL=C # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C diff --git a/requirements.txt b/requirements.txt index 315bb69ec..a6facb6cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ tensorboardX typeguard yacs pybind11 +textgrid diff --git a/utils/avg_model.py b/utils/avg_model.py old mode 100644 new mode 100755 diff --git a/utils/build_vocab.py b/utils/build_vocab.py old mode 100644 new mode 100755 diff --git a/utils/compute_mean_std.py b/utils/compute_mean_std.py old mode 100644 new mode 100755 diff --git a/utils/format_data.py b/utils/format_data.py old mode 100644 new mode 100755 diff --git a/utils/ngram_train.sh b/utils/ngram_train.sh old mode 100644 new mode 100755 diff --git a/utils/tarball.sh b/utils/tarball.sh new file mode 100755 index 000000000..100b4719e --- /dev/null +++ b/utils/tarball.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +if [ $# != 4 ];then + echo "usage: $0 ckpt_prefix model_config mean_std vocab" + exit -1 +fi + +ckpt_prefix=$1 +model_config=$2 +mean_std=$3 +vocab=$4 + +output=release + +mkdir -p ${output} +function clean() { + rm -rf ${output} +} +trap clean EXIT + +cp ${ckpt_prefix}.* ${output} +cp ${model_config} ${mean_std} ${vocab} ${output} + +tar zcvf release.tar.gz ${output} + +echo "tarball done!" diff --git a/utils/zh_tn.py b/utils/zh_tn.py old mode 100644 new mode 100755