From 5fb62ea9132bfd29c43e81707fbd8f39933dfc98 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Thu, 22 Apr 2021 03:39:38 +0000 Subject: [PATCH] fix layer tools --- .notebook/Linear_test.ipynb | 346 ++++++++++++++++++++++++++------ deepspeech/utils/layer_tools.py | 5 +- 2 files changed, 291 insertions(+), 60 deletions(-) diff --git a/.notebook/Linear_test.ipynb b/.notebook/Linear_test.ipynb index a11da386f..5c7370cf3 100644 --- a/.notebook/Linear_test.ipynb +++ b/.notebook/Linear_test.ipynb @@ -5,17 +5,7 @@ "execution_count": 1, "id": "academic-surname", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/workspace/DeepSpeech-2.x/tools/venv/lib/python3.7/site-packages/paddle/fluid/layers/utils.py:26: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n", - "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n", - " def convert_to_list(value, n, name, dtype=np.int):\n" - ] - } - ], + "outputs": [], "source": [ "import paddle\n", "from paddle import nn" @@ -31,7 +21,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/workspace/DeepSpeech-2.x/tools/venv/lib/python3.7/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n", + "/workspace/DeepSpeech-2.x/tools/venv-dev/lib/python3.7/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n", " and should_run_async(code)\n" ] } @@ -64,43 +54,43 @@ "text": [ "float64\n", "Tensor(shape=[2, 51, 256], dtype=float32, place=CUDAPlace(0), stop_gradient=True,\n", - " [[[-0.03137276, 0.75036579, -0.62955737, ..., -0.39516482, 2.41965628, 0.19466873],\n", - " [ 0.55916852, 1.13357353, 0.28754908, ..., 0.28860641, 0.48257691, -1.07664418],\n", - " [-0.27433595, -0.05911482, 0.04942252, ..., 0.46596146, 1.24395037, -1.98374581],\n", + " [[[-1.54171216, -2.61531472, -1.79881978, ..., -0.31395876, 0.56513089, -0.44516513],\n", + " [-0.79492962, 1.91157901, 0.66567147, ..., 0.54825783, -1.01471853, -0.84924090],\n", + " [-1.22556651, -0.36225814, 0.65063190, ..., 0.65726501, 0.05563191, 0.09009409],\n", " ...,\n", - " [-0.45322138, 0.51459873, 0.28475651, ..., -0.90797561, -0.80436397, -2.30388594],\n", - " [ 0.20310247, 1.90435207, -1.02483511, ..., -1.59850407, -0.30733466, 0.49769276],\n", - " [-2.63085651, -0.52244109, 0.32019949, ..., 1.10662329, -0.55995786, -0.36770794]],\n", + " [ 0.38615900, -0.77905393, 0.99732304, ..., -1.38463700, -3.32365036, -1.31089687],\n", + " [ 0.05579993, 0.06885809, -1.66662002, ..., -0.23346378, -3.29372883, 1.30561364],\n", + " [ 1.90676069, 1.95093191, -0.28849599, ..., -0.06860496, 0.95347673, 1.00475824]],\n", "\n", - " [[-1.78831303, 2.24759626, 0.41386250, ..., -0.30020580, -0.16084948, 0.93251175],\n", - " [ 0.03264519, -0.92942363, 1.58523536, ..., 1.23681784, -0.94711000, 0.63553023],\n", - " [-0.19725564, -2.38587499, -0.29334834, ..., 0.83498263, -0.58492625, 0.58732986],\n", + " [[-0.91453546, 0.55298805, -1.06146812, ..., -0.86378336, 1.00454640, 1.26062179],\n", + " [ 0.10223761, 0.81301165, 2.36865163, ..., 0.16821407, 0.29240361, 1.05408621],\n", + " [-1.33196676, 1.94433689, 0.01934209, ..., 0.48036841, 0.51585966, 1.22893548],\n", " ...,\n", - " [-0.61646742, -1.02978027, 0.45410269, ..., 0.87052751, -0.20801133, 2.17943859],\n", - " [-0.67230755, -0.79410625, -0.13054833, ..., -1.18138039, -0.47578079, -0.22610545],\n", - " [ 2.57333422, 0.63872230, 0.70852041, ..., -0.44040251, -0.33339104, -0.24722832]]])\n", - "tensor([[[-0.0314, 0.7504, -0.6296, ..., -0.3952, 2.4197, 0.1947],\n", - " [ 0.5592, 1.1336, 0.2875, ..., 0.2886, 0.4826, -1.0766],\n", - " [-0.2743, -0.0591, 0.0494, ..., 0.4660, 1.2440, -1.9837],\n", + " [-0.19558455, -0.47075930, 0.90796155, ..., -1.28598249, -0.24321797, 0.17734711],\n", + " [ 0.89819717, -1.39516675, 0.17138045, ..., 2.39761519, 1.76364994, -0.52177650],\n", + " [ 0.94122332, -0.18581429, 1.36099780, ..., 0.67647684, -0.04699665, 1.51205540]]])\n", + "tensor([[[-1.5417, -2.6153, -1.7988, ..., -0.3140, 0.5651, -0.4452],\n", + " [-0.7949, 1.9116, 0.6657, ..., 0.5483, -1.0147, -0.8492],\n", + " [-1.2256, -0.3623, 0.6506, ..., 0.6573, 0.0556, 0.0901],\n", " ...,\n", - " [-0.4532, 0.5146, 0.2848, ..., -0.9080, -0.8044, -2.3039],\n", - " [ 0.2031, 1.9044, -1.0248, ..., -1.5985, -0.3073, 0.4977],\n", - " [-2.6309, -0.5224, 0.3202, ..., 1.1066, -0.5600, -0.3677]],\n", + " [ 0.3862, -0.7791, 0.9973, ..., -1.3846, -3.3237, -1.3109],\n", + " [ 0.0558, 0.0689, -1.6666, ..., -0.2335, -3.2937, 1.3056],\n", + " [ 1.9068, 1.9509, -0.2885, ..., -0.0686, 0.9535, 1.0048]],\n", "\n", - " [[-1.7883, 2.2476, 0.4139, ..., -0.3002, -0.1608, 0.9325],\n", - " [ 0.0326, -0.9294, 1.5852, ..., 1.2368, -0.9471, 0.6355],\n", - " [-0.1973, -2.3859, -0.2933, ..., 0.8350, -0.5849, 0.5873],\n", + " [[-0.9145, 0.5530, -1.0615, ..., -0.8638, 1.0045, 1.2606],\n", + " [ 0.1022, 0.8130, 2.3687, ..., 0.1682, 0.2924, 1.0541],\n", + " [-1.3320, 1.9443, 0.0193, ..., 0.4804, 0.5159, 1.2289],\n", " ...,\n", - " [-0.6165, -1.0298, 0.4541, ..., 0.8705, -0.2080, 2.1794],\n", - " [-0.6723, -0.7941, -0.1305, ..., -1.1814, -0.4758, -0.2261],\n", - " [ 2.5733, 0.6387, 0.7085, ..., -0.4404, -0.3334, -0.2472]]])\n" + " [-0.1956, -0.4708, 0.9080, ..., -1.2860, -0.2432, 0.1773],\n", + " [ 0.8982, -1.3952, 0.1714, ..., 2.3976, 1.7636, -0.5218],\n", + " [ 0.9412, -0.1858, 1.3610, ..., 0.6765, -0.0470, 1.5121]]])\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/workspace/DeepSpeech-2.x/tools/venv/lib/python3.7/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n", + "/workspace/DeepSpeech-2.x/tools/venv-dev/lib/python3.7/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n", " and should_run_async(code)\n" ] } @@ -148,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "assured-zambia", "metadata": {}, "outputs": [ @@ -204,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 7, "id": "viral-indian", "metadata": {}, "outputs": [ @@ -250,7 +240,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 8, "id": "skilled-vietnamese", "metadata": {}, "outputs": [ @@ -258,14 +248,60 @@ "name": "stdout", "output_type": "stream", "text": [ - "[[-0.25528666 -0.9090747 0.12996 ... 0.02552819 0.37376517\n", - " -0.558986 ]\n", - " [-0.45657372 0.23811203 0.33472425 ... 1.0797666 -0.7263612\n", - " 0.31549692]]\n", - "[[-0.25528657 -0.9090746 0.12996009 ... 0.02552832 0.37376505\n", - " -0.5589858 ]\n", - " [-0.45657367 0.23811209 0.33472428 ... 1.0797666 -0.7263612\n", - " 0.31549698]]\n", + "[[[ 0.67277956 0.08313607 -0.62761104 ... -0.17480263 0.42718208\n", + " -0.5787626 ]\n", + " [ 0.91516656 0.5393416 1.7159258 ... 0.06144593 0.06486575\n", + " -0.03350811]\n", + " [ 0.438351 0.6227843 0.24096036 ... 1.0912522 -0.90929437\n", + " -1.012989 ]\n", + " ...\n", + " [ 0.68631977 0.14240924 0.10763275 ... -0.11513516 0.48065388\n", + " 0.04070369]\n", + " [-0.9525228 0.23197874 0.31264272 ... 0.5312439 0.18773697\n", + " -0.8450228 ]\n", + " [ 0.42024016 -0.04561988 0.54541194 ... -0.41933843 -0.00436018\n", + " -0.06663495]]\n", + "\n", + " [[-0.11638781 -0.33566502 -0.20887226 ... 0.17423287 -0.9195841\n", + " -0.8161046 ]\n", + " [-0.3469874 0.88269687 -0.11887559 ... -0.15566081 0.16357468\n", + " -0.20766167]\n", + " [-0.3847657 0.3984318 -0.06963477 ... -0.00360622 1.2360432\n", + " -0.26811332]\n", + " ...\n", + " [ 0.08230796 -0.46158582 0.54582864 ... 0.15747628 -0.44790155\n", + " 0.06020184]\n", + " [-0.8095085 0.43163058 -0.42837143 ... 0.8627463 0.90656304\n", + " 0.15847842]\n", + " [-1.485811 -0.18216592 -0.8882585 ... 0.32596245 0.7822631\n", + " -0.6460344 ]]]\n", + "[[[ 0.67278004 0.08313602 -0.6276114 ... -0.17480245 0.42718196\n", + " -0.5787625 ]\n", + " [ 0.91516703 0.5393413 1.7159253 ... 0.06144581 0.06486579\n", + " -0.03350812]\n", + " [ 0.43835106 0.62278455 0.24096027 ... 1.0912521 -0.9092943\n", + " -1.0129892 ]\n", + " ...\n", + " [ 0.6863195 0.14240888 0.10763284 ... -0.11513527 0.48065376\n", + " 0.04070365]\n", + " [-0.9525231 0.23197863 0.31264275 ... 0.53124386 0.18773702\n", + " -0.84502304]\n", + " [ 0.42024007 -0.04561983 0.545412 ... -0.41933888 -0.00436005\n", + " -0.066635 ]]\n", + "\n", + " [[-0.11638767 -0.33566508 -0.20887226 ... 0.17423296 -0.9195838\n", + " -0.8161046 ]\n", + " [-0.34698725 0.88269705 -0.11887549 ... -0.15566081 0.16357464\n", + " -0.20766166]\n", + " [-0.3847657 0.3984319 -0.06963488 ... -0.00360619 1.2360426\n", + " -0.26811326]\n", + " ...\n", + " [ 0.08230786 -0.4615857 0.5458287 ... 0.15747619 -0.44790167\n", + " 0.06020182]\n", + " [-0.8095083 0.4316307 -0.42837155 ... 0.862746 0.9065631\n", + " 0.15847899]\n", + " [-1.485811 -0.18216613 -0.8882584 ... 0.32596254 0.7822631\n", + " -0.6460344 ]]]\n", "True\n", "False\n" ] @@ -299,7 +335,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 9, "id": "governmental-surge", "metadata": {}, "outputs": [ @@ -307,14 +343,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "[[ 0.07453135 0.0698561 0.6273111 ... 0.5845924 -0.65527105\n", - " 0.5881643 ]\n", - " [ 0.3902049 -0.17455879 -1.1802813 ... -0.36912322 0.55681896\n", - " -0.11917676]]\n", - "[[ 0.07453132 0.06985616 0.62731117 ... 0.5845925 -0.65527105\n", - " 0.5881642 ]\n", - " [ 0.39020485 -0.17455864 -1.1802814 ... -0.3691232 0.556819\n", - " -0.11917675]]\n", + "[[ 0.04476918 0.554463 -0.3027508 ... -0.49600336 0.3751858\n", + " 0.8254095 ]\n", + " [ 0.95594174 -0.29528382 -1.2899452 ... 0.43718258 0.05584608\n", + " -0.06974669]]\n", + "[[ 0.04476918 0.5544631 -0.3027507 ... -0.49600336 0.37518573\n", + " 0.8254096 ]\n", + " [ 0.95594174 -0.29528376 -1.2899454 ... 0.4371827 0.05584623\n", + " -0.0697467 ]]\n", "True\n", "False\n", "True\n" @@ -344,9 +380,203 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "improved-civilization", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5e7e7c9fde8350084abf1898cf52651cfc84b17a\n" + ] + } + ], + "source": [ + "print(paddle.version.commit)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "d1e2d3b4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['__builtins__',\n", + " '__cached__',\n", + " '__doc__',\n", + " '__file__',\n", + " '__loader__',\n", + " '__name__',\n", + " '__package__',\n", + " '__spec__',\n", + " 'commit',\n", + " 'full_version',\n", + " 'istaged',\n", + " 'major',\n", + " 'minor',\n", + " 'mkl',\n", + " 'patch',\n", + " 'rc',\n", + " 'show',\n", + " 'with_mkl']" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dir(paddle.version)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "c880c719", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2.1.0\n" + ] + } + ], + "source": [ + "print(paddle.version.full_version)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "f26977bf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "commit: 5e7e7c9fde8350084abf1898cf52651cfc84b17a\n", + "None\n" + ] + } + ], + "source": [ + "print(paddle.version.show())" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "04ad47f6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.6.0\n" + ] + } + ], + "source": [ + "print(torch.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "e1e03830", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['__builtins__',\n", + " '__cached__',\n", + " '__doc__',\n", + " '__file__',\n", + " '__loader__',\n", + " '__name__',\n", + " '__package__',\n", + " '__spec__',\n", + " '__version__',\n", + " 'cuda',\n", + " 'debug',\n", + " 'git_version',\n", + " 'hip']" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dir(torch.version)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "4ad0389b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'b31f58de6fa8bbda5353b3c77d9be4914399724d'" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.version.git_version" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "7870ea10", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'10.2'" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.version.cuda" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "db8ee5a7", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6321ec2a", + "metadata": {}, "outputs": [], "source": [] } diff --git a/deepspeech/utils/layer_tools.py b/deepspeech/utils/layer_tools.py index f1fd50aa1..1c3105605 100644 --- a/deepspeech/utils/layer_tools.py +++ b/deepspeech/utils/layer_tools.py @@ -31,8 +31,9 @@ def summary(layer: nn.Layer, print_func=print): num_elements += np.prod(param.shape) num_params += 1 if print_func: - num_elements = num_elements / 1024**3 - print_func(f"Total parameters: {num_params}, {num_elements}G elements.") + num_elements = num_elements / 1024**2 + print_func( + f"Total parameters: {num_params}, {num_elements:.4f}M elements.") def print_grads(model, print_func=print):