Merge pull request #1958 from SmileGoat/refactor_file_struct
[Speechx] refactor example dir && add aishell build TLG scriptpull/1961/head
Hui Zhang
3 years ago
committed by
GitHub
commit
d871e4c74c
@ -1,3 +0,0 @@
|
||||
cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
|
||||
|
||||
add_subdirectory(glog)
|
||||
@ -1,8 +0,0 @@
|
||||
cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
|
||||
|
||||
add_executable(glog_test ${CMAKE_CURRENT_SOURCE_DIR}/glog_test.cc)
|
||||
target_link_libraries(glog_test glog)
|
||||
|
||||
|
||||
add_executable(glog_logtostderr_test ${CMAKE_CURRENT_SOURCE_DIR}/glog_logtostderr_test.cc)
|
||||
target_link_libraries(glog_logtostderr_test glog)
|
||||
@ -1,25 +0,0 @@
|
||||
# [GLOG](https://rpg.ifi.uzh.ch/docs/glog.html)
|
||||
|
||||
Unless otherwise specified, glog writes to the filename `/tmp/<program name>.<hostname>.<user name>.log.<severity level>.<date>.<time>.<pid>` (e.g., "/tmp/hello_world.example.com.hamaji.log.INFO.20080709-222411.10474"). By default, glog copies the log messages of severity level ERROR or FATAL to standard error (stderr) in addition to log files.
|
||||
|
||||
Several flags influence glog's output behavior. If the Google gflags library is installed on your machine, the configure script (see the INSTALL file in the package for detail of this script) will automatically detect and use it, allowing you to pass flags on the command line. For example, if you want to turn the flag --logtostderr on, you can start your application with the following command line:
|
||||
|
||||
`./your_application --logtostderr=1`
|
||||
|
||||
If the Google gflags library isn't installed, you set flags via environment variables, prefixing the flag name with "GLOG_", e.g.
|
||||
|
||||
`GLOG_logtostderr=1 ./your_application`
|
||||
|
||||
You can also modify flag values in your program by modifying global variables `FLAGS_*` . Most settings start working immediately after you update `FLAGS_*` . The exceptions are the flags related to destination files. For example, you might want to set `FLAGS_log_dir` before calling `google::InitGoogleLogging` . Here is an example:
|
||||
∂∂
|
||||
```c++
|
||||
LOG(INFO) << "file";
|
||||
// Most flags work immediately after updating values.
|
||||
FLAGS_logtostderr = 1;
|
||||
LOG(INFO) << "stderr";
|
||||
FLAGS_logtostderr = 0;
|
||||
// This won't change the log destination. If you want to set this
|
||||
// value, you should do this before google::InitGoogleLogging .
|
||||
FLAGS_log_dir = "/some/log/directory";
|
||||
LOG(INFO) << "the same file";
|
||||
```
|
||||
@ -1,15 +0,0 @@
|
||||
# This contains the locations of binarys build required for running the examples.
|
||||
|
||||
SPEECHX_ROOT=$PWD/../../../
|
||||
|
||||
SPEECHX_TOOLS=$SPEECHX_ROOT/tools
|
||||
TOOLS_BIN=$SPEECHX_TOOLS/valgrind/install/bin
|
||||
|
||||
|
||||
SPEECHX_EXAMPLES=$SPEECHX_ROOT/build/examples
|
||||
[ -d $SPEECHX_EXAMPLES ] || { echo "Error: 'build/examples' directory not found. please ensure that the project build successfully"; }
|
||||
|
||||
SPEECHX_BIN=$SPEECHX_EXAMPLES/dev/glog
|
||||
export PATH=$PATH:$SPEECHX_BIN:$TOOLS_BIN
|
||||
|
||||
export LC_AL=C
|
||||
@ -1,22 +0,0 @@
|
||||
#!/bin/bash
|
||||
set +x
|
||||
set -e
|
||||
|
||||
. ./path.sh
|
||||
|
||||
# 1. compile
|
||||
if [ ! -d ${SPEECHX_EXAMPLES} ]; then
|
||||
pushd ${SPEECHX_ROOT}
|
||||
bash build.sh
|
||||
popd
|
||||
fi
|
||||
|
||||
# 2. run
|
||||
glog_test
|
||||
|
||||
echo "------"
|
||||
export FLAGS_logtostderr=1
|
||||
glog_test
|
||||
|
||||
echo "------"
|
||||
glog_logtostderr_test
|
||||
@ -0,0 +1,141 @@
|
||||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
. path.sh
|
||||
|
||||
# attention, please replace the vocab is only for this script.
|
||||
# different acustic model has different vocab
|
||||
ckpt_dir=data/fbank_model
|
||||
unit=$ckpt_dir/data/lang_char/vocab.txt # vocab file, line: char/spm_pice
|
||||
model_dir=$ckpt_dir/exp/deepspeech2_online/checkpoints/
|
||||
|
||||
stage=-1
|
||||
stop_stage=100
|
||||
corpus=aishell
|
||||
lexicon=data/lexicon.txt # line: word ph0 ... phn, aishell/resource_aishell/lexicon.txt
|
||||
text=data/text # line: utt text, aishell/data_aishell/transcript/aishell_transcript_v0.8.txt
|
||||
|
||||
. utils/parse_options.sh
|
||||
|
||||
data=$PWD/data
|
||||
mkdir -p $data
|
||||
|
||||
if [ $stage -le -1 ] && [ $stop_stage -ge -1 ]; then
|
||||
if [ ! -f $data/speech.ngram.zh.tar.gz ];then
|
||||
pushd $data
|
||||
wget -c http://paddlespeech.bj.bcebos.com/speechx/examples/ngram/zh/speech.ngram.zh.tar.gz
|
||||
tar xvzf speech.ngram.zh.tar.gz
|
||||
popd
|
||||
fi
|
||||
|
||||
if [ ! -f $ckpt_dir/data/mean_std.json ]; then
|
||||
mkdir -p $ckpt_dir
|
||||
pushd $ckpt_dir
|
||||
wget -c https://paddlespeech.bj.bcebos.com/s2t/wenetspeech/asr0/WIP1_asr0_deepspeech2_online_wenetspeech_ckpt_1.0.0a.model.tar.gz
|
||||
tar xzfv WIP1_asr0_deepspeech2_online_wenetspeech_ckpt_1.0.0a.model.tar.gz
|
||||
popd
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f $unit ]; then
|
||||
echo "$0: No such file $unit"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
if ! which ngram-count; then
|
||||
pushd $MAIN_ROOT/tools
|
||||
make srilm.done
|
||||
popd
|
||||
fi
|
||||
|
||||
mkdir -p data/local/dict
|
||||
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||
# Prepare dict
|
||||
# line: char/spm_pices
|
||||
cp $unit data/local/dict/units.txt
|
||||
|
||||
if [ ! -f $lexicon ];then
|
||||
utils/text_to_lexicon.py --has_key true --text $text --lexicon $lexicon
|
||||
echo "Generate $lexicon from $text"
|
||||
fi
|
||||
|
||||
# filter by vocab
|
||||
# line: word ph0 ... phn -> line: word char0 ... charn
|
||||
utils/fst/prepare_dict.py \
|
||||
--unit_file $unit \
|
||||
--in_lexicon ${lexicon} \
|
||||
--out_lexicon data/local/dict/lexicon.txt
|
||||
fi
|
||||
|
||||
lm=data/local/lm
|
||||
mkdir -p $lm
|
||||
|
||||
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||
# Train lm
|
||||
cp $text $lm/text
|
||||
local/aishell_train_lms.sh
|
||||
echo "build LM done."
|
||||
fi
|
||||
|
||||
# build TLG
|
||||
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
||||
# build T & L
|
||||
utils/fst/compile_lexicon_token_fst.sh \
|
||||
data/local/dict data/local/tmp data/local/lang
|
||||
|
||||
# build G & TLG
|
||||
utils/fst/make_tlg.sh data/local/lm data/local/lang data/lang_test || exit 1;
|
||||
|
||||
fi
|
||||
|
||||
aishell_wav_scp=aishell_test.scp
|
||||
nj=40
|
||||
cmvn=$data/cmvn_fbank.ark
|
||||
wfst=$data/lang_test
|
||||
|
||||
if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
|
||||
|
||||
if [ ! -d $data/test ]; then
|
||||
pushd $data
|
||||
wget -c https://paddlespeech.bj.bcebos.com/s2t/paddle_asr_online/aishell_test.zip
|
||||
unzip aishell_test.zip
|
||||
popd
|
||||
|
||||
realpath $data/test/*/*.wav > $data/wavlist
|
||||
awk -F '/' '{ print $(NF) }' $data/wavlist | awk -F '.' '{ print $1 }' > $data/utt_id
|
||||
paste $data/utt_id $data/wavlist > $data/$aishell_wav_scp
|
||||
fi
|
||||
|
||||
./local/split_data.sh $data $data/$aishell_wav_scp $aishell_wav_scp $nj
|
||||
|
||||
cmvn-json2kaldi --json_file=$ckpt_dir/data/mean_std.json --cmvn_write_path=$cmvn
|
||||
fi
|
||||
|
||||
wer=aishell_wer
|
||||
label_file=aishell_result
|
||||
export GLOG_logtostderr=1
|
||||
|
||||
if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
|
||||
# TLG decoder
|
||||
utils/run.pl JOB=1:$nj $data/split${nj}/JOB/check_tlg.log \
|
||||
recognizer_test_main \
|
||||
--wav_rspecifier=scp:$data/split${nj}/JOB/${aishell_wav_scp} \
|
||||
--cmvn_file=$cmvn \
|
||||
--model_path=$model_dir/avg_5.jit.pdmodel \
|
||||
--streaming_chunk=30 \
|
||||
--use_fbank=true \
|
||||
--param_path=$model_dir/avg_5.jit.pdiparams \
|
||||
--word_symbol_table=$wfst/words.txt \
|
||||
--model_output_names=softmax_0.tmp_0,tmp_5,concat_0.tmp_0,concat_1.tmp_0 \
|
||||
--model_cache_shapes="5-1-2048,5-1-2048" \
|
||||
--graph_path=$wfst/TLG.fst --max_active=7500 \
|
||||
--acoustic_scale=1.2 \
|
||||
--result_wspecifier=ark,t:$data/split${nj}/JOB/result_check_tlg
|
||||
|
||||
cat $data/split${nj}/*/result_check_tlg > $exp/${label_file}_check_tlg
|
||||
utils/compute-wer.py --char=1 --v=1 $text $exp/${label_file}_check_tlg > $exp/${wer}.check_tlg
|
||||
echo "recognizer test have finished!!!"
|
||||
echo "please checkout in ${exp}/${wer}.check_tlg"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@ -1,2 +0,0 @@
|
||||
data
|
||||
exp
|
||||
@ -1,101 +0,0 @@
|
||||
# ngram train for mandarin
|
||||
|
||||
Quick run:
|
||||
```
|
||||
bash run.sh --stage -1
|
||||
```
|
||||
|
||||
## input
|
||||
|
||||
input files:
|
||||
```
|
||||
data/
|
||||
├── lexicon.txt
|
||||
├── text
|
||||
└── vocab.txt
|
||||
```
|
||||
|
||||
```
|
||||
==> data/text <==
|
||||
BAC009S0002W0122 而 对 楼市 成交 抑制 作用 最 大 的 限 购
|
||||
BAC009S0002W0123 也 成为 地方 政府 的 眼中 钉
|
||||
BAC009S0002W0124 自 六月 底 呼和浩特 市 率先 宣布 取消 限 购 后
|
||||
BAC009S0002W0125 各地 政府 便 纷纷 跟进
|
||||
BAC009S0002W0126 仅 一 个 多 月 的 时间 里
|
||||
BAC009S0002W0127 除了 北京 上海 广州 深圳 四 个 一 线 城市 和 三亚 之外
|
||||
BAC009S0002W0128 四十六 个 限 购 城市 当中
|
||||
BAC009S0002W0129 四十一 个 已 正式 取消 或 变相 放松 了 限 购
|
||||
BAC009S0002W0130 财政 金融 政策 紧随 其后 而来
|
||||
BAC009S0002W0131 显示 出 了 极 强 的 威力
|
||||
|
||||
==> data/lexicon.txt <==
|
||||
SIL sil
|
||||
<SPOKEN_NOISE> sil
|
||||
啊 aa a1
|
||||
啊 aa a2
|
||||
啊 aa a4
|
||||
啊 aa a5
|
||||
啊啊啊 aa a2 aa a2 aa a2
|
||||
啊啊啊 aa a5 aa a5 aa a5
|
||||
坐地 z uo4 d i4
|
||||
坐实 z uo4 sh ix2
|
||||
坐视 z uo4 sh ix4
|
||||
坐稳 z uo4 uu un3
|
||||
坐拥 z uo4 ii iong1
|
||||
坐诊 z uo4 zh en3
|
||||
坐庄 z uo4 zh uang1
|
||||
坐姿 z uo4 z iy1
|
||||
|
||||
==> data/vocab.txt <==
|
||||
<blank>
|
||||
<unk>
|
||||
A
|
||||
B
|
||||
C
|
||||
D
|
||||
E
|
||||
龙
|
||||
龚
|
||||
龛
|
||||
<eos>
|
||||
```
|
||||
|
||||
## output
|
||||
|
||||
```
|
||||
data/
|
||||
├── local
|
||||
│ ├── dict
|
||||
│ │ ├── lexicon.txt
|
||||
│ │ └── units.txt
|
||||
│ └── lm
|
||||
│ ├── heldout
|
||||
│ ├── lm.arpa
|
||||
│ ├── text
|
||||
│ ├── text.no_oov
|
||||
│ ├── train
|
||||
│ ├── unigram.counts
|
||||
│ ├── word.counts
|
||||
│ └── wordlist
|
||||
```
|
||||
|
||||
```
|
||||
/workspace/srilm/bin/i686-m64/ngram-count
|
||||
Namespace(bpemodel=None, in_lexicon='data/lexicon.txt', out_lexicon='data/local/dict/lexicon.txt', unit_file='data/vocab.txt')
|
||||
Ignoring words 矽, which contains oov unit
|
||||
Ignoring words 傩, which contains oov unit
|
||||
Ignoring words 堀, which contains oov unit
|
||||
Ignoring words 莼, which contains oov unit
|
||||
Ignoring words 菰, which contains oov unit
|
||||
Ignoring words 摭, which contains oov unit
|
||||
Ignoring words 帙, which contains oov unit
|
||||
Ignoring words 迨, which contains oov unit
|
||||
Ignoring words 孥, which contains oov unit
|
||||
Ignoring words 瑗, which contains oov unit
|
||||
...
|
||||
...
|
||||
...
|
||||
file data/local/lm/heldout: 10000 sentences, 89496 words, 0 OOVs
|
||||
0 zeroprobs, logprob= -270337.9 ppl= 521.2819 ppl1= 1048.745
|
||||
build LM done.
|
||||
```
|
||||
@ -1,30 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
data=$1
|
||||
scp=$2
|
||||
split_name=$3
|
||||
numsplit=$4
|
||||
|
||||
# save in $data/split{n}
|
||||
# $scp to split
|
||||
#
|
||||
|
||||
if [[ ! $numsplit -gt 0 ]]; then
|
||||
echo "Invalid num-split argument";
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
directories=$(for n in `seq $numsplit`; do echo $data/split${numsplit}/$n; done)
|
||||
scp_splits=$(for n in `seq $numsplit`; do echo $data/split${numsplit}/$n/${split_name}; done)
|
||||
|
||||
# if this mkdir fails due to argument-list being too long, iterate.
|
||||
if ! mkdir -p $directories >&/dev/null; then
|
||||
for n in `seq $numsplit`; do
|
||||
mkdir -p $data/split${numsplit}/$n
|
||||
done
|
||||
fi
|
||||
|
||||
echo "utils/split_scp.pl $scp $scp_splits"
|
||||
utils/split_scp.pl $scp $scp_splits
|
||||
@ -1,12 +0,0 @@
|
||||
# This contains the locations of binarys build required for running the examples.
|
||||
|
||||
MAIN_ROOT=`realpath $PWD/../../../../`
|
||||
SPEECHX_ROOT=`realpath $MAIN_ROOT/speechx`
|
||||
|
||||
export LC_AL=C
|
||||
|
||||
# srilm
|
||||
export LIBLBFGS=${MAIN_ROOT}/tools/liblbfgs-1.10
|
||||
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${LIBLBFGS}/lib/.libs
|
||||
export SRILM=${MAIN_ROOT}/tools/srilm
|
||||
export PATH=${PATH}:${SRILM}/bin:${SRILM}/bin/i686-m64
|
||||
@ -1,68 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
. path.sh
|
||||
|
||||
stage=-1
|
||||
stop_stage=100
|
||||
corpus=aishell
|
||||
|
||||
unit=data/vocab.txt # vocab file, line: char/spm_pice
|
||||
lexicon=data/lexicon.txt # line: word ph0 ... phn, aishell/resource_aishell/lexicon.txt
|
||||
text=data/text # line: utt text, aishell/data_aishell/transcript/aishell_transcript_v0.8.txt
|
||||
|
||||
. utils/parse_options.sh
|
||||
|
||||
data=$PWD/data
|
||||
mkdir -p $data
|
||||
|
||||
if [ $stage -le -1 ] && [ $stop_stage -ge -1 ]; then
|
||||
if [ ! -f $data/speech.ngram.zh.tar.gz ];then
|
||||
pushd $data
|
||||
wget -c http://paddlespeech.bj.bcebos.com/speechx/examples/ngram/zh/speech.ngram.zh.tar.gz
|
||||
tar xvzf speech.ngram.zh.tar.gz
|
||||
popd
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f $unit ]; then
|
||||
echo "$0: No such file $unit"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
if ! which ngram-count; then
|
||||
pushd $MAIN_ROOT/tools
|
||||
make srilm.done
|
||||
popd
|
||||
fi
|
||||
|
||||
mkdir -p data/local/dict
|
||||
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||
# 7.1 Prepare dict
|
||||
# line: char/spm_pices
|
||||
cp $unit data/local/dict/units.txt
|
||||
|
||||
if [ ! -f $lexicon ];then
|
||||
local/text_to_lexicon.py --has_key true --text $text --lexicon $lexicon
|
||||
echo "Generate $lexicon from $text"
|
||||
fi
|
||||
|
||||
# filter by vocab
|
||||
# line: word ph0 ... phn -> line: word char0 ... charn
|
||||
utils/fst/prepare_dict.py \
|
||||
--unit_file $unit \
|
||||
--in_lexicon ${lexicon} \
|
||||
--out_lexicon data/local/dict/lexicon.txt
|
||||
fi
|
||||
|
||||
lm=data/local/lm
|
||||
mkdir -p $lm
|
||||
|
||||
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||
# 7.2 Train lm
|
||||
cp $text $lm/text
|
||||
local/aishell_train_lms.sh
|
||||
fi
|
||||
|
||||
echo "build LM done."
|
||||
exit 0
|
||||
@ -1 +0,0 @@
|
||||
../../../../utils/
|
||||
@ -1 +0,0 @@
|
||||
data
|
||||
@ -1,19 +0,0 @@
|
||||
# This contains the locations of binarys build required for running the examples.
|
||||
|
||||
MAIN_ROOT=`realpath $PWD/../../../`
|
||||
SPEECHX_ROOT=`realpath $MAIN_ROOT/speechx`
|
||||
|
||||
export LC_AL=C
|
||||
|
||||
# srilm
|
||||
export LIBLBFGS=${MAIN_ROOT}/tools/liblbfgs-1.10
|
||||
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${LIBLBFGS}/lib/.libs
|
||||
export SRILM=${MAIN_ROOT}/tools/srilm
|
||||
export PATH=${PATH}:${SRILM}/bin:${SRILM}/bin/i686-m64
|
||||
|
||||
# Kaldi
|
||||
export KALDI_ROOT=${MAIN_ROOT}/tools/kaldi
|
||||
[ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh
|
||||
export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$PWD:$PATH
|
||||
[ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present, can not using Kaldi!"
|
||||
[ -f $KALDI_ROOT/tools/config/common_path.sh ] && . $KALDI_ROOT/tools/config/common_path.sh
|
||||
@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
. path.sh
|
||||
|
||||
stage=-1
|
||||
stop_stage=100
|
||||
|
||||
. utils/parse_options.sh
|
||||
|
||||
if ! which fstprint ; then
|
||||
pushd $MAIN_ROOT/tools
|
||||
make kaldi.done
|
||||
popd
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||
# build T & L
|
||||
# utils/fst/compile_lexicon_token_fst.sh <dict-src-dir> <tmp-dir> <lang-dir>
|
||||
utils/fst/compile_lexicon_token_fst.sh \
|
||||
data/local/dict data/local/tmp data/local/lang
|
||||
|
||||
# build G & LG & TLG
|
||||
# utils/fst/make_tlg.sh <lm_dir> <src_lang> <tgt_lang>
|
||||
utils/fst/make_tlg.sh data/local/lm data/local/lang data/lang_test || exit 1;
|
||||
fi
|
||||
|
||||
echo "build TLG done."
|
||||
exit 0
|
||||
@ -1 +0,0 @@
|
||||
../../../utils/
|
||||
Loading…
Reference in new issue