diff --git a/examples/aishell/s0/local/train.sh b/examples/aishell/s0/local/train.sh index 85d1d42c..668ad0ea 100755 --- a/examples/aishell/s0/local/train.sh +++ b/examples/aishell/s0/local/train.sh @@ -19,6 +19,7 @@ fi mkdir -p exp +# seed may break model convergence seed=10086 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True diff --git a/examples/aishell/s1/local/train.sh b/examples/aishell/s1/local/train.sh index 2861e11e..f905b766 100755 --- a/examples/aishell/s1/local/train.sh +++ b/examples/aishell/s1/local/train.sh @@ -19,8 +19,9 @@ echo "using ${device}..." mkdir -p exp -seed=10086 -if [ ${seed} != 0]; then +# seed may break model convergence +seed=0 +if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/callcenter/s1/local/train.sh b/examples/callcenter/s1/local/train.sh index 6e63df83..d5dc15b0 100755 --- a/examples/callcenter/s1/local/train.sh +++ b/examples/callcenter/s1/local/train.sh @@ -19,8 +19,9 @@ echo "using ${device}..." mkdir -p exp -seed=10086 -if [ ${seed} != 0]; then +# seed may break model convergence +seed=0 +if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/librispeech/s0/local/train.sh b/examples/librispeech/s0/local/train.sh index c95659ac..6aee372a 100755 --- a/examples/librispeech/s0/local/train.sh +++ b/examples/librispeech/s0/local/train.sh @@ -20,7 +20,8 @@ echo "using ${device}..." mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/librispeech/s1/local/train.sh b/examples/librispeech/s1/local/train.sh index 17a9e28d..f905b766 100755 --- a/examples/librispeech/s1/local/train.sh +++ b/examples/librispeech/s1/local/train.sh @@ -19,7 +19,8 @@ echo "using ${device}..." mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi @@ -31,7 +32,7 @@ python3 -u ${BIN_DIR}/train.py \ --output exp/${ckpt_name} \ --seed ${seed} -if [ ${seed} != 0]; then +if [ ${seed} != 0 ]; then unset FLAGS_cudnn_deterministic fi diff --git a/examples/librispeech/s2/local/train.sh b/examples/librispeech/s2/local/train.sh index a75e2bb2..66754201 100755 --- a/examples/librispeech/s2/local/train.sh +++ b/examples/librispeech/s2/local/train.sh @@ -19,7 +19,8 @@ echo "using ${device}..." mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/ted_en_zh/t0/local/train.sh b/examples/ted_en_zh/t0/local/train.sh index 928356f9..f905b766 100755 --- a/examples/ted_en_zh/t0/local/train.sh +++ b/examples/ted_en_zh/t0/local/train.sh @@ -19,7 +19,8 @@ echo "using ${device}..." mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi @@ -31,7 +32,7 @@ python3 -u ${BIN_DIR}/train.py \ --output exp/${ckpt_name} \ --seed ${seed} -if [ ${seed} != 0 ]; then +if [ ${seed} != 0 ]; then unset FLAGS_cudnn_deterministic fi diff --git a/examples/timit/s1/local/train.sh b/examples/timit/s1/local/train.sh index 3e2e4522..180d8b5a 100755 --- a/examples/timit/s1/local/train.sh +++ b/examples/timit/s1/local/train.sh @@ -19,7 +19,8 @@ echo "using ${device}..." mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/tiny/s0/local/train.sh b/examples/tiny/s0/local/train.sh index bf4766ee..ea29b7fc 100755 --- a/examples/tiny/s0/local/train.sh +++ b/examples/tiny/s0/local/train.sh @@ -19,7 +19,8 @@ fi mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/tiny/s0/local/tune.sh b/examples/tiny/s0/local/tune.sh deleted file mode 100755 index c344e77e..00000000 --- a/examples/tiny/s0/local/tune.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -if [ $# != 1 ];then - echo "usage: tune ckpt_path" - exit 1 -fi - -# grid-search for hyper-parameters in language model -python3 -u ${BIN_DIR}/tune.py \ ---device 'gpu' \ ---nproc 1 \ ---config conf/deepspeech2.yaml \ ---num_batches=-1 \ ---batch_size=128 \ ---beam_size=500 \ ---num_proc_bsearch=12 \ ---num_alphas=45 \ ---num_betas=8 \ ---alpha_from=1.0 \ ---alpha_to=3.2 \ ---beta_from=0.1 \ ---beta_to=0.45 \ ---cutoff_prob=1.0 \ ---cutoff_top_n=40 \ ---checkpoint_path ${1} - -if [ $? -ne 0 ]; then - echo "Failed in tuning!" - exit 1 -fi - - -exit 0 diff --git a/examples/tiny/s1/local/train.sh b/examples/tiny/s1/local/train.sh index 48968f63..374608fd 100755 --- a/examples/tiny/s1/local/train.sh +++ b/examples/tiny/s1/local/train.sh @@ -18,7 +18,8 @@ fi mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi