From f0470e05294db5ddc23b24ff457b9436b047e4e7 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Fri, 10 Sep 2021 04:45:26 +0000 Subject: [PATCH] not dump all grad info, since slow down train porcess --- deepspeech/training/gradclip.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/deepspeech/training/gradclip.py b/deepspeech/training/gradclip.py index 6393197a..87b36aca 100644 --- a/deepspeech/training/gradclip.py +++ b/deepspeech/training/gradclip.py @@ -47,9 +47,10 @@ class ClipGradByGlobalNormWithLog(paddle.nn.ClipGradByGlobalNorm): sum_square = layers.reduce_sum(square) sum_square_list.append(sum_square) - # debug log - logger.debug( - f"Grad Before Clip: {p.name}: {float(sum_square.sqrt()) }") + # debug log, not dump all since slow down train process + if i < 10: + logger.debug( + f"Grad Before Clip: {p.name}: {float(sum_square.sqrt()) }") # all parameters have been filterd out if len(sum_square_list) == 0: @@ -75,9 +76,10 @@ class ClipGradByGlobalNormWithLog(paddle.nn.ClipGradByGlobalNorm): new_grad = layers.elementwise_mul(x=g, y=clip_var) params_and_grads.append((p, new_grad)) - # debug log - logger.debug( - f"Grad After Clip: {p.name}: {float(new_grad.square().sum().sqrt())}" - ) + # debug log, not dump all since slow down train process + if i < 10: + logger.debug( + f"Grad After Clip: {p.name}: {float(new_grad.square().sum().sqrt())}" + ) return params_and_grads