Merge pull request #783 from Jackwaterveg/ds2_online

修正了seed功能

deepspeech/exps/deepspeech2/bin/train.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Trainer for DeepSpeech2 model."""
-import os
-
 from paddle import distributed as dist
 
 from deepspeech.exps.deepspeech2.config import get_cfg_defaults
@@ -55,7 +53,5 @@ if __name__ == "__main__":
     if args.dump_config:
         with open(args.dump_config, 'w') as f:
             print(config, file=f)
-    if config.training.seed is not None:
-        os.environ.setdefault('FLAGS_cudnn_deterministic', 'True')
 
     main(config, args)

deepspeech/exps/deepspeech2/model.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Contains DeepSpeech2 and DeepSpeech2Online model."""
-import random
 import time
 from collections import defaultdict
 from pathlib import Path
@@ -54,7 +53,6 @@ class DeepSpeech2Trainer(Trainer):
                 weight_decay=1e-6,  # the coeff of weight decay
                 global_grad_clip=5.0,  # the global norm clip
                 n_epoch=50,  # train epochs
-                seed=1024,  #train seed
             ))
 
         if config is not None:
@@ -63,13 +61,6 @@ class DeepSpeech2Trainer(Trainer):
 
     def __init__(self, config, args):
         super().__init__(config, args)
-        if config.training.seed is not None:
-            self.set_seed(config.training.seed)
-
-    def set_seed(self, seed):
-        np.random.seed(seed)
-        random.seed(seed)
-        paddle.seed(seed)
 
     def train_batch(self, batch_index, batch_data, msg):
         start = time.time()

deepspeech/training/cli.py

@@ -62,6 +62,9 @@ def default_argument_parser():
     # help="options to overwrite --config file and the default config, passing in KEY VALUE pairs")
     parser.add_argument("--opts", type=str, default=[], nargs='+',
                         help="options to overwrite --config file and the default config, passing in KEY VALUE pairs")
+
+    parser.add_argument("--seed", type=int, default=None,
+                        help="seed to use for paddle, np and random. The default value is None")
     # yapd: enable
 
     return parser

deepspeech/training/trainer.py

@@ -11,9 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import random
 import time
 from pathlib import Path
 
+import numpy as np
 import paddle
 from paddle import distributed as dist
 from tensorboardX import SummaryWriter
@@ -93,6 +95,13 @@ class Trainer():
         self.checkpoint_dir = None
         self.iteration = 0
         self.epoch = 0
+        if args.seed is not None:
+            self.set_seed(args.seed)
+
+    def set_seed(self, seed):
+        np.random.seed(seed)
+        random.seed(seed)
+        paddle.seed(seed)
 
     def setup(self):
         """Setup the experiment.

examples/aishell/s0/local/train.sh

@@ -19,12 +19,22 @@ fi
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
---model_type ${model_type}
+--model_type ${model_type} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/aishell/s1/local/train.sh

@@ -19,11 +19,21 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/callcenter/s1/local/train.sh

@@ -19,11 +19,21 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/librispeech/s0/local/train.sh

@@ -20,12 +20,22 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
---model_type ${model_type}
+--model_type ${model_type} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/librispeech/s1/local/train.sh

@@ -19,11 +19,21 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/librispeech/s2/local/train.sh

@@ -19,12 +19,22 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --model-name u2_kaldi \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/ted_en_zh/t0/local/train.sh

@@ -19,11 +19,21 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/timit/s1/local/train.sh

@@ -19,11 +19,21 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/tiny/s0/local/train.sh

@@ -19,12 +19,22 @@ fi
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
---model_type ${model_type}
+--model_type ${model_type} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/tiny/s1/local/train.sh

@@ -18,11 +18,21 @@ fi
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"