add timer

deepspeech/training/timer.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import datetime
+import time
+
+from deepspeech.utils.log import Log
+
+__all__ = ["Timer"]
+
+logger = Log(__name__).getlog()
+
+
+class Timer():
+    """To be used like this: 
+        with Timer("Message") as value:
+            do some thing
+    """
+
+    def __init__(self, message):
+        self.message = message
+
+    def duration(self) -> str:
+        elapsed_time = time.time() - self.start
+        time_str = str(datetime.timedelta(seconds=elapsed_time))
+        return time_str
+
+    def __enter__(self):
+        self.start = time.time()
+        return self
+
+    def __exit__(self, type, value, traceback):
+        logger.info(self.message.format(self.duration()))
+
+    def __call__(self) -> float:
+        return time.time() - self.start
+
+    def __str__(self):
+        return self.duration()

deepspeech/training/trainer.py

@@ -18,6 +18,7 @@ import paddle
 from paddle import distributed as dist
 from tensorboardX import SummaryWriter
 
+from deepspeech.training.timer import Timer
 from deepspeech.utils import mp_tools
 from deepspeech.utils.checkpoint import Checkpoint
 from deepspeech.utils.log import Log
@@ -184,6 +185,7 @@ class Trainer():
 
     def train(self):
         """The training process control by epoch."""
+        with Timer("Load/Init Model: {}"):
             from_scratch = self.resume_or_scratch()
             if from_scratch:
                 # save init model, i.e. 0 epoch
@@ -240,6 +242,7 @@ class Trainer():
         """The routine of the experiment after setup. This method is intended
         to be used by the user.
         """
+        with Timer("Training Done: {}"):
             try:
                 self.train()
             except KeyboardInterrupt:
@@ -247,7 +250,6 @@ class Trainer():
                 exit(-1)
             finally:
                 self.destory()
-        logger.info("Training Done.")
 
     def setup_output_dir(self):
         """Create a directory used for output.