diff --git a/deepspeech/exps/deepspeech2/model.py b/deepspeech/exps/deepspeech2/model.py index 8272d72ee..7bf029300 100644 --- a/deepspeech/exps/deepspeech2/model.py +++ b/deepspeech/exps/deepspeech2/model.py @@ -35,8 +35,8 @@ from deepspeech.models.ds2 import DeepSpeech2Model from deepspeech.models.ds2_online import DeepSpeech2InferModelOnline from deepspeech.models.ds2_online import DeepSpeech2ModelOnline from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog -from deepspeech.training.trainer import Trainer from deepspeech.training.reporter import report +from deepspeech.training.trainer import Trainer from deepspeech.utils import error_rate from deepspeech.utils import layer_tools from deepspeech.utils import mp_tools @@ -108,7 +108,7 @@ class DeepSpeech2Trainer(Trainer): report("batch_size", batch_size) report("accum", accum_grad) report("step_cost", iteration_time) - + if dist.get_rank() == 0 and self.visualizer: for k, v in losses_np.items(): # `step -1` since we update `step` after optimizer.step(). diff --git a/deepspeech/exps/u2/model.py b/deepspeech/exps/u2/model.py index 68b001ca6..2e512ef1e 100644 --- a/deepspeech/exps/u2/model.py +++ b/deepspeech/exps/u2/model.py @@ -34,11 +34,11 @@ from deepspeech.io.sampler import SortagradBatchSampler from deepspeech.io.sampler import SortagradDistributedBatchSampler from deepspeech.models.u2 import U2Model from deepspeech.training.optimizer import OptimizerFactory +from deepspeech.training.reporter import ObsScope +from deepspeech.training.reporter import report from deepspeech.training.scheduler import LRSchedulerFactory from deepspeech.training.timer import Timer from deepspeech.training.trainer import Trainer -from deepspeech.training.reporter import report -from deepspeech.training.reporter import ObsScope from deepspeech.utils import ctc_utils from deepspeech.utils import error_rate from deepspeech.utils import layer_tools @@ -207,17 +207,21 @@ class U2Trainer(Trainer): report("Rank", dist.get_rank()) report("epoch", self.epoch) report('step', self.iteration) - report('step/total', (batch_index + 1) / len(self.train_loader)) + report('step/total', + (batch_index + 1) / len(self.train_loader)) report("lr", self.lr_scheduler()) self.train_batch(batch_index, batch, msg) self.after_train_batch() report('reader_cost', dataload_time) - observation['batch_cost'] = observation['reader_cost']+observation['step_cost'] + observation['batch_cost'] = observation[ + 'reader_cost'] + observation['step_cost'] observation['samples'] = observation['batch_size'] - observation['ips[sent./sec]'] = observation['batch_size'] / observation['batch_cost'] + observation['ips[sent./sec]'] = observation[ + 'batch_size'] / observation['batch_cost'] for k, v in observation.items(): msg += f" {k}: " - msg += f"{v:>.8f}" if isinstance(v, float) else f"{v}" + msg += f"{v:>.8f}" if isinstance(v, + float) else f"{v}" msg += "," logger.info(msg) data_start_time = time.time() diff --git a/deepspeech/training/extensions/evaluator.py b/deepspeech/training/extensions/evaluator.py index 5137dbdde..1026a4ec3 100644 --- a/deepspeech/training/extensions/evaluator.py +++ b/deepspeech/training/extensions/evaluator.py @@ -20,8 +20,8 @@ from paddle.nn import Layer from . import extension from ..reporter import DictSummary -from ..reporter import report from ..reporter import ObsScope +from ..reporter import report from ..timer import Timer from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/training/trainer.py b/deepspeech/training/trainer.py index 18578b429..a5efdd541 100644 --- a/deepspeech/training/trainer.py +++ b/deepspeech/training/trainer.py @@ -13,16 +13,16 @@ # limitations under the License. import sys import time -from pathlib import Path from collections import OrderedDict +from pathlib import Path import paddle from paddle import distributed as dist from tensorboardX import SummaryWriter -from deepspeech.training.timer import Timer -from deepspeech.training.reporter import report from deepspeech.training.reporter import ObsScope +from deepspeech.training.reporter import report +from deepspeech.training.timer import Timer from deepspeech.utils import mp_tools from deepspeech.utils import profiler from deepspeech.utils.checkpoint import Checkpoint @@ -30,7 +30,6 @@ from deepspeech.utils.log import Log from deepspeech.utils.utility import seed_all from deepspeech.utils.utility import UpdateConfig - __all__ = ["Trainer"] logger = Log(__name__).getlog() @@ -236,17 +235,21 @@ class Trainer(): report("Rank", dist.get_rank()) report("epoch", self.epoch) report('step', self.iteration) - report('step/total', (batch_index + 1) / len(self.train_loader)) + report('step/total', + (batch_index + 1) / len(self.train_loader)) report("lr", self.lr_scheduler()) self.train_batch(batch_index, batch, msg) self.after_train_batch() report('reader_cost', dataload_time) - observation['batch_cost'] = observation['reader_cost']+observation['step_cost'] + observation['batch_cost'] = observation[ + 'reader_cost'] + observation['step_cost'] observation['samples'] = observation['batch_size'] - observation['ips[sent./sec]'] = observation['batch_size'] / observation['batch_cost'] + observation['ips[sent./sec]'] = observation[ + 'batch_size'] / observation['batch_cost'] for k, v in observation.items(): msg += f" {k}: " - msg += f"{v:>.8f}" if isinstance(v, float) else f"{v}" + msg += f"{v:>.8f}" if isinstance(v, + float) else f"{v}" msg += "," logger.info(msg) data_start_time = time.time()