Hui Zhang
3 years ago
parent
cf1a395e15
commit
37d9c08da5
@ -0,0 +1,27 @@
|
|||||||
|
# This contains the locations of binarys build required for running the examples.
|
||||||
|
|
||||||
|
SPEECHX_ROOT=$PWD/../../../
|
||||||
|
MAIN_ROOT=$SPEECHX_ROOT/../
|
||||||
|
SPEECHX_EXAMPLES=$SPEECHX_ROOT/build/examples
|
||||||
|
|
||||||
|
SPEECHX_TOOLS=$SPEECHX_ROOT/tools
|
||||||
|
TOOLS_BIN=$SPEECHX_TOOLS/valgrind/install/bin
|
||||||
|
|
||||||
|
[ -d $SPEECHX_EXAMPLES ] || { echo "Error: 'build/examples' directory not found. please ensure that the project build successfully"; }
|
||||||
|
|
||||||
|
export LC_AL=C
|
||||||
|
|
||||||
|
export PATH=$PATH:$TOOLS_BIN
|
||||||
|
|
||||||
|
# srilm
|
||||||
|
export LIBLBFGS=${MAIN_ROOT}/tools/liblbfgs-1.10
|
||||||
|
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${LIBLBFGS}/lib/.libs
|
||||||
|
export SRILM=${MAIN_ROOT}/tools/srilm
|
||||||
|
export PATH=${PATH}:${SRILM}/bin:${SRILM}/bin/i686-m64
|
||||||
|
|
||||||
|
# Kaldi
|
||||||
|
export KALDI_ROOT=${MAIN_ROOT}/tools/kaldi
|
||||||
|
[ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh
|
||||||
|
export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$PWD:$PATH
|
||||||
|
[ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present, can not using Kaldi!"
|
||||||
|
[ -f $KALDI_ROOT/tools/config/common_path.sh ] && . $KALDI_ROOT/tools/config/common_path.sh
|
||||||
@ -0,0 +1,64 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
set -eo pipefail
|
||||||
|
|
||||||
|
. path.sh
|
||||||
|
|
||||||
|
stage=-1
|
||||||
|
stop_stage=100
|
||||||
|
corpus=aishell
|
||||||
|
lmtype=srilm
|
||||||
|
|
||||||
|
lexicon= # aishell/resource_aishell/lexicon.txt
|
||||||
|
text= # aishell/data_aishell/transcript/aishell_transcript_v0.8.txt
|
||||||
|
|
||||||
|
source parse_options.sh
|
||||||
|
|
||||||
|
if [ ! which ngram-count ]; then
|
||||||
|
pushd $MAIN_ROOT/tools
|
||||||
|
make srilm.done
|
||||||
|
popd
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! which fstprint ]; then
|
||||||
|
pushd $MAIN_ROOT/tools
|
||||||
|
make kaldi.done
|
||||||
|
popd
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||||
|
# 7.1 Prepare dict
|
||||||
|
unit_file=data/vocab.txt
|
||||||
|
mkdir -p data/local/dict
|
||||||
|
cp $unit_file data/local/dict/units.txt
|
||||||
|
utils/fst/prepare_dict.py \
|
||||||
|
--unit_file $unit_file \
|
||||||
|
--in_lexicon ${lexicon} \
|
||||||
|
--out_lexicon data/local/dict/lexicon.txt
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||||
|
# 7.2 Train lm
|
||||||
|
lm=data/local/lm
|
||||||
|
mkdir -p data/train
|
||||||
|
mkdir -p $lm
|
||||||
|
utils/manifest_key_value.py \
|
||||||
|
--manifest_path data/manifest.train \
|
||||||
|
--output_path data/train
|
||||||
|
utils/filter_scp.pl data/train/text \
|
||||||
|
$text > $lm/text
|
||||||
|
if [ $lmtype == 'srilm' ];then
|
||||||
|
local/aishell_train_lms.sh
|
||||||
|
else
|
||||||
|
utils/ngram_train.sh --order 3 $lm/text $lm/lm.arpa
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
||||||
|
# 7.3 Build decoding TLG
|
||||||
|
utils/fst/compile_lexicon_token_fst.sh \
|
||||||
|
data/local/dict data/local/tmp data/local/lang
|
||||||
|
utils/fst/make_tlg.sh data/local/lm data/local/lang data/lang_test || exit 1;
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Aishell build TLG done."
|
||||||
|
exit 0
|
||||||
@ -0,0 +1,57 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# To be run from one directory above this script.
|
||||||
|
. ./path.sh
|
||||||
|
|
||||||
|
text=data/local/lm/text
|
||||||
|
lexicon=data/local/dict/lexicon.txt
|
||||||
|
|
||||||
|
for f in "$text" "$lexicon"; do
|
||||||
|
[ ! -f $x ] && echo "$0: No such file $f" && exit 1;
|
||||||
|
done
|
||||||
|
|
||||||
|
# Check SRILM tools
|
||||||
|
if ! which ngram-count > /dev/null; then
|
||||||
|
echo "srilm tools are not found, please download it and install it from: "
|
||||||
|
echo "http://www.speech.sri.com/projects/srilm/download.html"
|
||||||
|
echo "Then add the tools to your PATH"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# This script takes no arguments. It assumes you have already run
|
||||||
|
# aishell_data_prep.sh.
|
||||||
|
# It takes as input the files
|
||||||
|
# data/local/lm/text
|
||||||
|
# data/local/dict/lexicon.txt
|
||||||
|
dir=data/local/lm
|
||||||
|
mkdir -p $dir
|
||||||
|
|
||||||
|
cleantext=$dir/text.no_oov
|
||||||
|
|
||||||
|
cat $text | awk -v lex=$lexicon 'BEGIN{while((getline<lex) >0){ seen[$1]=1; } }
|
||||||
|
{for(n=1; n<=NF;n++) { if (seen[$n]) { printf("%s ", $n); } else {printf("<SPOKEN_NOISE> ");} } printf("\n");}' \
|
||||||
|
> $cleantext || exit 1;
|
||||||
|
|
||||||
|
cat $cleantext | awk '{for(n=2;n<=NF;n++) print $n; }' | sort | uniq -c | \
|
||||||
|
sort -nr > $dir/word.counts || exit 1;
|
||||||
|
|
||||||
|
# Get counts from acoustic training transcripts, and add one-count
|
||||||
|
# for each word in the lexicon (but not silence, we don't want it
|
||||||
|
# in the LM-- we'll add it optionally later).
|
||||||
|
cat $cleantext | awk '{for(n=2;n<=NF;n++) print $n; }' | \
|
||||||
|
cat - <(grep -w -v '!SIL' $lexicon | awk '{print $1}') | \
|
||||||
|
sort | uniq -c | sort -nr > $dir/unigram.counts || exit 1;
|
||||||
|
|
||||||
|
cat $dir/unigram.counts | awk '{print $2}' | cat - <(echo "<s>"; echo "</s>" ) > $dir/wordlist
|
||||||
|
|
||||||
|
heldout_sent=10000 # Don't change this if you want result to be comparable with
|
||||||
|
# kaldi_lm results
|
||||||
|
mkdir -p $dir
|
||||||
|
cat $cleantext | awk '{for(n=2;n<=NF;n++){ printf $n; if(n<NF) printf " "; else print ""; }}' | \
|
||||||
|
head -$heldout_sent > $dir/heldout
|
||||||
|
cat $cleantext | awk '{for(n=2;n<=NF;n++){ printf $n; if(n<NF) printf " "; else print ""; }}' | \
|
||||||
|
tail -n +$heldout_sent > $dir/train
|
||||||
|
|
||||||
|
ngram-count -text $dir/train -order 3 -limit-vocab -vocab $dir/wordlist -unk \
|
||||||
|
-map-unk "<UNK>" -kndiscount -interpolate -lm $dir/lm.arpa
|
||||||
|
ngram -lm $dir/lm.arpa -ppl $dir/heldout
|
||||||
@ -0,0 +1,20 @@
|
|||||||
|
# This contains the locations of binarys build required for running the examples.
|
||||||
|
|
||||||
|
SPEECHX_ROOT=$PWD/../../../
|
||||||
|
MAIN_ROOT=$SPEECHX_ROOT/../
|
||||||
|
SPEECHX_EXAMPLES=$SPEECHX_ROOT/build/examples
|
||||||
|
|
||||||
|
SPEECHX_TOOLS=$SPEECHX_ROOT/tools
|
||||||
|
TOOLS_BIN=$SPEECHX_TOOLS/valgrind/install/bin
|
||||||
|
|
||||||
|
[ -d $SPEECHX_EXAMPLES ] || { echo "Error: 'build/examples' directory not found. please ensure that the project build successfully"; }
|
||||||
|
|
||||||
|
export LC_AL=C
|
||||||
|
|
||||||
|
export PATH=$PATH:$TOOLS_BIN
|
||||||
|
|
||||||
|
# srilm
|
||||||
|
export LIBLBFGS=${MAIN_ROOT}/tools/liblbfgs-1.10
|
||||||
|
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${LIBLBFGS}/lib/.libs
|
||||||
|
export SRILM=${MAIN_ROOT}/tools/srilm
|
||||||
|
export PATH=${PATH}:${SRILM}/bin:${SRILM}/bin/i686-m64
|
||||||
@ -0,0 +1,61 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
set -eo pipefail
|
||||||
|
|
||||||
|
. path.sh
|
||||||
|
|
||||||
|
stage=-1
|
||||||
|
stop_stage=100
|
||||||
|
corpus=aishell
|
||||||
|
|
||||||
|
unit=data/vocab.txt # vocab
|
||||||
|
lexicon= # aishell/resource_aishell/lexicon.txt
|
||||||
|
text= # aishell/data_aishell/transcript/aishell_transcript_v0.8.txt
|
||||||
|
|
||||||
|
. parse_options.sh
|
||||||
|
|
||||||
|
data=$PWD/data
|
||||||
|
mkdir -p $data
|
||||||
|
|
||||||
|
if [ ! -f $unit ]; then
|
||||||
|
echo "$0: No such file $unit"
|
||||||
|
exit 1;
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! which ngram-count ]; then
|
||||||
|
pushd $MAIN_ROOT/tools
|
||||||
|
make srilm.done
|
||||||
|
popd
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! which fstaddselfloops ]; then
|
||||||
|
pushd $MAIN_ROOT/tools
|
||||||
|
make kaldi.done
|
||||||
|
popd
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p data/local/dict
|
||||||
|
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||||
|
# 7.1 Prepare dict
|
||||||
|
cp $unit data/local/dict/units.txt
|
||||||
|
utils/fst/prepare_dict.py \
|
||||||
|
--unit_file $unit \
|
||||||
|
--in_lexicon ${lexicon} \
|
||||||
|
--out_lexicon data/local/dict/lexicon.txt
|
||||||
|
fi
|
||||||
|
|
||||||
|
lm=data/local/lm
|
||||||
|
mkdir -p data/train
|
||||||
|
mkdir -p $lm
|
||||||
|
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||||
|
# 7.2 Train lm
|
||||||
|
utils/manifest_key_value.py \
|
||||||
|
--manifest_path data/manifest.train \
|
||||||
|
--output_path data/train
|
||||||
|
utils/filter_scp.pl data/train/text \
|
||||||
|
$text > $lm/text
|
||||||
|
|
||||||
|
local/aishell_train_lms.sh
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "build LM done."
|
||||||
|
exit 0
|
||||||
@ -0,0 +1 @@
|
|||||||
|
../../../utils/
|
||||||
@ -1,97 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
current_path=`pwd`
|
|
||||||
current_dir=`basename "$current_path"`
|
|
||||||
|
|
||||||
if [ "tools" != "$current_dir" ]; then
|
|
||||||
echo "You should run this script in tools/ directory!!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ! -d liblbfgs-1.10 ]; then
|
|
||||||
echo Installing libLBFGS library to support MaxEnt LMs
|
|
||||||
bash extras/install_liblbfgs.sh || exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
! command -v gawk > /dev/null && \
|
|
||||||
echo "GNU awk is not installed so SRILM will probably not work correctly: refusing to install" && exit 1;
|
|
||||||
|
|
||||||
if [ $# -ne 3 ]; then
|
|
||||||
echo "SRILM download requires some information about you"
|
|
||||||
echo
|
|
||||||
echo "Usage: $0 <name> <organization> <email>"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
srilm_url="http://www.speech.sri.com/projects/srilm/srilm_download.php"
|
|
||||||
post_data="WWW_file=srilm-1.7.3.tar.gz&WWW_name=$1&WWW_org=$2&WWW_email=$3"
|
|
||||||
|
|
||||||
if ! wget --post-data "$post_data" -O ./srilm.tar.gz "$srilm_url"; then
|
|
||||||
echo 'There was a problem downloading the file.'
|
|
||||||
echo 'Check you internet connection and try again.'
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
mkdir -p srilm
|
|
||||||
cd srilm
|
|
||||||
|
|
||||||
|
|
||||||
if [ -f ../srilm.tgz ]; then
|
|
||||||
tar -xvzf ../srilm.tgz # Old SRILM format
|
|
||||||
elif [ -f ../srilm.tar.gz ]; then
|
|
||||||
tar -xvzf ../srilm.tar.gz # Changed format type from tgz to tar.gz
|
|
||||||
fi
|
|
||||||
|
|
||||||
major=`gawk -F. '{ print $1 }' RELEASE`
|
|
||||||
minor=`gawk -F. '{ print $2 }' RELEASE`
|
|
||||||
micro=`gawk -F. '{ print $3 }' RELEASE`
|
|
||||||
|
|
||||||
if [ $major -le 1 ] && [ $minor -le 7 ] && [ $micro -le 1 ]; then
|
|
||||||
echo "Detected version 1.7.1 or earlier. Applying patch."
|
|
||||||
patch -p0 < ../extras/srilm.patch
|
|
||||||
fi
|
|
||||||
|
|
||||||
# set the SRILM variable in the top-level Makefile to this directory.
|
|
||||||
cp Makefile tmpf
|
|
||||||
|
|
||||||
cat tmpf | gawk -v pwd=`pwd` '/SRILM =/{printf("SRILM = %s\n", pwd); next;} {print;}' \
|
|
||||||
> Makefile || exit 1
|
|
||||||
rm tmpf
|
|
||||||
|
|
||||||
mtype=`sbin/machine-type`
|
|
||||||
|
|
||||||
echo HAVE_LIBLBFGS=1 >> common/Makefile.machine.$mtype
|
|
||||||
grep ADDITIONAL_INCLUDES common/Makefile.machine.$mtype | \
|
|
||||||
sed 's|$| -I$(SRILM)/../liblbfgs-1.10/include|' \
|
|
||||||
>> common/Makefile.machine.$mtype
|
|
||||||
|
|
||||||
grep ADDITIONAL_LDFLAGS common/Makefile.machine.$mtype | \
|
|
||||||
sed 's|$| -L$(SRILM)/../liblbfgs-1.10/lib/ -Wl,-rpath -Wl,$(SRILM)/../liblbfgs-1.10/lib/|' \
|
|
||||||
>> common/Makefile.machine.$mtype
|
|
||||||
|
|
||||||
make || exit
|
|
||||||
|
|
||||||
cd ..
|
|
||||||
(
|
|
||||||
[ ! -z "${SRILM}" ] && \
|
|
||||||
echo >&2 "SRILM variable is aleady defined. Undefining..." && \
|
|
||||||
unset SRILM
|
|
||||||
|
|
||||||
[ -f ./env.sh ] && . ./env.sh
|
|
||||||
|
|
||||||
[ ! -z "${SRILM}" ] && \
|
|
||||||
echo >&2 "SRILM config is already in env.sh" && exit
|
|
||||||
|
|
||||||
wd=`pwd`
|
|
||||||
wd=`readlink -f $wd || pwd`
|
|
||||||
|
|
||||||
echo "export SRILM=$wd/srilm"
|
|
||||||
dirs="\${PATH}"
|
|
||||||
for directory in $(cd srilm && find bin -type d ) ; do
|
|
||||||
dirs="$dirs:\${SRILM}/$directory"
|
|
||||||
done
|
|
||||||
echo "export PATH=$dirs"
|
|
||||||
) >> env.sh
|
|
||||||
|
|
||||||
echo >&2 "Installation of SRILM finished successfully"
|
|
||||||
echo >&2 "Please source the tools/env.sh in your path.sh to enable it"
|
|
||||||
Loading…
Reference in new issue