|
|
@ -4,19 +4,22 @@ set -e
|
|
|
|
|
|
|
|
|
|
|
|
stage=-1
|
|
|
|
stage=-1
|
|
|
|
stop_stage=100
|
|
|
|
stop_stage=100
|
|
|
|
|
|
|
|
dict_dir=data/lang_char
|
|
|
|
|
|
|
|
|
|
|
|
# bpemode (unigram or bpe)
|
|
|
|
# bpemode (unigram or bpe)
|
|
|
|
nbpe=8000
|
|
|
|
nbpe=8000
|
|
|
|
bpemode=unigram
|
|
|
|
bpemode=unigram
|
|
|
|
bpeprefix="data/bpe_${bpemode}_${nbpe}"
|
|
|
|
bpeprefix="${dict_dir}/bpe_${bpemode}_${nbpe}"
|
|
|
|
data_dir=./TED-En-Zh
|
|
|
|
data_dir=./TED-En-Zh
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
source ${MAIN_ROOT}/utils/parse_options.sh
|
|
|
|
. ${MAIN_ROOT}/utils/parse_options.sh || exit -1;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
TARGET_DIR=${MAIN_ROOT}/examples/dataset
|
|
|
|
TARGET_DIR=${MAIN_ROOT}/examples/dataset
|
|
|
|
mkdir -p ${TARGET_DIR}
|
|
|
|
mkdir -p ${TARGET_DIR}
|
|
|
|
mkdir -p data
|
|
|
|
mkdir -p data
|
|
|
|
|
|
|
|
mkdir -p ${dict_dir}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
|
|
|
|
if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
|
|
|
@ -73,11 +76,10 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
|
|
|
--spm_vocab_size=${nbpe} \
|
|
|
|
--spm_vocab_size=${nbpe} \
|
|
|
|
--spm_mode ${bpemode} \
|
|
|
|
--spm_mode ${bpemode} \
|
|
|
|
--spm_model_prefix ${bpeprefix} \
|
|
|
|
--spm_model_prefix ${bpeprefix} \
|
|
|
|
--vocab_path="data/vocab.txt" \
|
|
|
|
--vocab_path="${dict_dir}/vocab.txt" \
|
|
|
|
--text_keys 'text' 'text1' \
|
|
|
|
--text_keys 'text' 'text1' \
|
|
|
|
--manifest_paths="data/manifest.train.raw"
|
|
|
|
--manifest_paths="data/manifest.train.raw"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if [ $? -ne 0 ]; then
|
|
|
|
if [ $? -ne 0 ]; then
|
|
|
|
echo "Build vocabulary failed. Terminated."
|
|
|
|
echo "Build vocabulary failed. Terminated."
|
|
|
|
exit 1
|
|
|
|
exit 1
|
|
|
@ -92,7 +94,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
|
|
|
--cmvn_path "data/mean_std.json" \
|
|
|
|
--cmvn_path "data/mean_std.json" \
|
|
|
|
--unit_type "spm" \
|
|
|
|
--unit_type "spm" \
|
|
|
|
--spm_model_prefix ${bpeprefix} \
|
|
|
|
--spm_model_prefix ${bpeprefix} \
|
|
|
|
--vocab_path="data/vocab.txt" \
|
|
|
|
--vocab_path="${dict_dir}/vocab.txt" \
|
|
|
|
--manifest_path="data/manifest.${set}.raw" \
|
|
|
|
--manifest_path="data/manifest.${set}.raw" \
|
|
|
|
--output_path="data/manifest.${set}"
|
|
|
|
--output_path="data/manifest.${set}"
|
|
|
|
|
|
|
|
|
|
|
|