From 8d44aa481323291fd78f951a62d83c15eff9d31f Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 15 Sep 2021 06:08:14 +0000 Subject: [PATCH] do not set seed since break model covergence, aishell s0 seed 10086 test ok --- examples/aishell/s0/local/train.sh | 1 + examples/aishell/s1/local/train.sh | 5 ++-- examples/callcenter/s1/local/train.sh | 5 ++-- examples/librispeech/s0/local/train.sh | 3 ++- examples/librispeech/s1/local/train.sh | 5 ++-- examples/librispeech/s2/local/train.sh | 3 ++- examples/ted_en_zh/t0/local/train.sh | 5 ++-- examples/timit/s1/local/train.sh | 3 ++- examples/tiny/s0/local/train.sh | 3 ++- examples/tiny/s0/local/tune.sh | 33 -------------------------- examples/tiny/s1/local/train.sh | 3 ++- 11 files changed, 23 insertions(+), 46 deletions(-) delete mode 100755 examples/tiny/s0/local/tune.sh diff --git a/examples/aishell/s0/local/train.sh b/examples/aishell/s0/local/train.sh index 85d1d42c3..668ad0ead 100755 --- a/examples/aishell/s0/local/train.sh +++ b/examples/aishell/s0/local/train.sh @@ -19,6 +19,7 @@ fi mkdir -p exp +# seed may break model convergence seed=10086 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True diff --git a/examples/aishell/s1/local/train.sh b/examples/aishell/s1/local/train.sh index 2861e11ec..f905b766e 100755 --- a/examples/aishell/s1/local/train.sh +++ b/examples/aishell/s1/local/train.sh @@ -19,8 +19,9 @@ echo "using ${device}..." mkdir -p exp -seed=10086 -if [ ${seed} != 0]; then +# seed may break model convergence +seed=0 +if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/callcenter/s1/local/train.sh b/examples/callcenter/s1/local/train.sh index 6e63df83a..d5dc15b03 100755 --- a/examples/callcenter/s1/local/train.sh +++ b/examples/callcenter/s1/local/train.sh @@ -19,8 +19,9 @@ echo "using ${device}..." mkdir -p exp -seed=10086 -if [ ${seed} != 0]; then +# seed may break model convergence +seed=0 +if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/librispeech/s0/local/train.sh b/examples/librispeech/s0/local/train.sh index c95659acf..6aee372a4 100755 --- a/examples/librispeech/s0/local/train.sh +++ b/examples/librispeech/s0/local/train.sh @@ -20,7 +20,8 @@ echo "using ${device}..." mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/librispeech/s1/local/train.sh b/examples/librispeech/s1/local/train.sh index 17a9e28df..f905b766e 100755 --- a/examples/librispeech/s1/local/train.sh +++ b/examples/librispeech/s1/local/train.sh @@ -19,7 +19,8 @@ echo "using ${device}..." mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi @@ -31,7 +32,7 @@ python3 -u ${BIN_DIR}/train.py \ --output exp/${ckpt_name} \ --seed ${seed} -if [ ${seed} != 0]; then +if [ ${seed} != 0 ]; then unset FLAGS_cudnn_deterministic fi diff --git a/examples/librispeech/s2/local/train.sh b/examples/librispeech/s2/local/train.sh index a75e2bb26..66754201f 100755 --- a/examples/librispeech/s2/local/train.sh +++ b/examples/librispeech/s2/local/train.sh @@ -19,7 +19,8 @@ echo "using ${device}..." mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/ted_en_zh/t0/local/train.sh b/examples/ted_en_zh/t0/local/train.sh index 928356f96..f905b766e 100755 --- a/examples/ted_en_zh/t0/local/train.sh +++ b/examples/ted_en_zh/t0/local/train.sh @@ -19,7 +19,8 @@ echo "using ${device}..." mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi @@ -31,7 +32,7 @@ python3 -u ${BIN_DIR}/train.py \ --output exp/${ckpt_name} \ --seed ${seed} -if [ ${seed} != 0 ]; then +if [ ${seed} != 0 ]; then unset FLAGS_cudnn_deterministic fi diff --git a/examples/timit/s1/local/train.sh b/examples/timit/s1/local/train.sh index 3e2e4522d..180d8b5a7 100755 --- a/examples/timit/s1/local/train.sh +++ b/examples/timit/s1/local/train.sh @@ -19,7 +19,8 @@ echo "using ${device}..." mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/tiny/s0/local/train.sh b/examples/tiny/s0/local/train.sh index bf4766ee3..ea29b7fcf 100755 --- a/examples/tiny/s0/local/train.sh +++ b/examples/tiny/s0/local/train.sh @@ -19,7 +19,8 @@ fi mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi diff --git a/examples/tiny/s0/local/tune.sh b/examples/tiny/s0/local/tune.sh deleted file mode 100755 index c344e77e5..000000000 --- a/examples/tiny/s0/local/tune.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -if [ $# != 1 ];then - echo "usage: tune ckpt_path" - exit 1 -fi - -# grid-search for hyper-parameters in language model -python3 -u ${BIN_DIR}/tune.py \ ---device 'gpu' \ ---nproc 1 \ ---config conf/deepspeech2.yaml \ ---num_batches=-1 \ ---batch_size=128 \ ---beam_size=500 \ ---num_proc_bsearch=12 \ ---num_alphas=45 \ ---num_betas=8 \ ---alpha_from=1.0 \ ---alpha_to=3.2 \ ---beta_from=0.1 \ ---beta_to=0.45 \ ---cutoff_prob=1.0 \ ---cutoff_top_n=40 \ ---checkpoint_path ${1} - -if [ $? -ne 0 ]; then - echo "Failed in tuning!" - exit 1 -fi - - -exit 0 diff --git a/examples/tiny/s1/local/train.sh b/examples/tiny/s1/local/train.sh index 48968f63c..374608fd1 100755 --- a/examples/tiny/s1/local/train.sh +++ b/examples/tiny/s1/local/train.sh @@ -18,7 +18,8 @@ fi mkdir -p exp -seed=10086 +# seed may break model convergence +seed=0 if [ ${seed} != 0 ]; then export FLAGS_cudnn_deterministic=True fi