Merge pull request #783 from Jackwaterveg/ds2_online

修正了seed功能

deepspeech/exps/deepspeech2/bin/train.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Trainer for DeepSpeech2 model."""
-import os
-
 from paddle import distributed as dist
 
 from deepspeech.exps.deepspeech2.config import get_cfg_defaults
@@ -55,7 +53,5 @@ if __name__ == "__main__":
     if args.dump_config:
         with open(args.dump_config, 'w') as f:
             print(config, file=f)
-    if config.training.seed is not None:
-        os.environ.setdefault('FLAGS_cudnn_deterministic', 'True')
 
     main(config, args)

deepspeech/exps/deepspeech2/model.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Contains DeepSpeech2 and DeepSpeech2Online model."""
-import random
 import time
 from collections import defaultdict
 from pathlib import Path
@@ -54,7 +53,6 @@ class DeepSpeech2Trainer(Trainer):
                 weight_decay=1e-6,  # the coeff of weight decay
                 global_grad_clip=5.0,  # the global norm clip
                 n_epoch=50,  # train epochs
-                seed=1024,  #train seed
             ))
 
         if config is not None:
@@ -63,13 +61,6 @@ class DeepSpeech2Trainer(Trainer):
 
     def __init__(self, config, args):
         super().__init__(config, args)
-        if config.training.seed is not None:
-            self.set_seed(config.training.seed)
-
-    def set_seed(self, seed):
-        np.random.seed(seed)
-        random.seed(seed)
-        paddle.seed(seed)
 
     def train_batch(self, batch_index, batch_data, msg):
         start = time.time()

deepspeech/training/cli.py

@@ -16,23 +16,23 @@ import argparse
 
 def default_argument_parser():
     r"""A simple yet genral argument parser for experiments with parakeet.
-    
+
-    This is used in examples with parakeet. And it is intended to be used by 
+    This is used in examples with parakeet. And it is intended to be used by
-    other experiments with parakeet. It requires a minimal set of command line 
+    other experiments with parakeet. It requires a minimal set of command line
     arguments to start a training script.
-    
+
-    The ``--config`` and ``--opts`` are used for overwrite the deault 
+    The ``--config`` and ``--opts`` are used for overwrite the deault
     configuration.
-    
+
-    The ``--data`` and ``--output`` specifies the data path and output path. 
+    The ``--data`` and ``--output`` specifies the data path and output path.
-    Resuming training from existing progress at the output directory is the 
+    Resuming training from existing progress at the output directory is the
     intended default behavior.
-    
+
     The ``--checkpoint_path`` specifies the checkpoint to load from.
-    
+
     The ``--device`` and ``--nprocs`` specifies how to run the training.
-    
+
-    
+
     See Also
     --------
     parakeet.training.experiment
@@ -58,10 +58,13 @@ def default_argument_parser():
     parser.add_argument("--nprocs", type=int, default=1, help="number of parallel processes to use.")
 
     # overwrite extra config and default config
-    # parser.add_argument("--opts", nargs=argparse.REMAINDER, 
+    # parser.add_argument("--opts", nargs=argparse.REMAINDER,
     # help="options to overwrite --config file and the default config, passing in KEY VALUE pairs")
     parser.add_argument("--opts", type=str, default=[], nargs='+',
                         help="options to overwrite --config file and the default config, passing in KEY VALUE pairs")
+
+    parser.add_argument("--seed", type=int, default=None,
+                        help="seed to use for paddle, np and random. The default value is None")
     # yapd: enable
 
     return parser

deepspeech/training/trainer.py

@@ -11,9 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import random
 import time
 from pathlib import Path
 
+import numpy as np
 import paddle
 from paddle import distributed as dist
 from tensorboardX import SummaryWriter
@@ -93,6 +95,13 @@ class Trainer():
         self.checkpoint_dir = None
         self.iteration = 0
         self.epoch = 0
+        if args.seed is not None:
+            self.set_seed(args.seed)
+
+    def set_seed(self, seed):
+        np.random.seed(seed)
+        random.seed(seed)
+        paddle.seed(seed)
 
     def setup(self):
         """Setup the experiment.

examples/aishell/s0/local/train.sh

@@ -19,12 +19,22 @@ fi
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
---model_type ${model_type}
+--model_type ${model_type} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/aishell/s1/local/train.sh

@@ -19,11 +19,21 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/callcenter/s1/local/train.sh

@@ -19,11 +19,21 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/librispeech/s0/local/train.sh

@@ -20,12 +20,22 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
---model_type ${model_type}
+--model_type ${model_type} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/librispeech/s1/local/train.sh

@@ -19,11 +19,21 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/librispeech/s2/local/train.sh

@@ -19,12 +19,22 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --model-name u2_kaldi \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/ted_en_zh/t0/local/train.sh

@@ -19,11 +19,21 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/timit/s1/local/train.sh

@@ -19,11 +19,21 @@ echo "using ${device}..."
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/tiny/s0/local/train.sh

@@ -19,12 +19,22 @@ fi
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
---model_type ${model_type}
+--model_type ${model_type} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"

examples/tiny/s1/local/train.sh

@@ -18,11 +18,21 @@ fi
 
 mkdir -p exp
 
+seed=1024
+if [ ${seed} ]; then
+    export FLAGS_cudnn_deterministic=True
+fi
+
 python3 -u ${BIN_DIR}/train.py \
 --device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
---output exp/${ckpt_name}
+--output exp/${ckpt_name} \
+--seed ${seed}
+
+if [ ${seed} ]; then
+    unset FLAGS_cudnn_deterministic
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"