Hui Zhang
4 years ago
commit
05d2e3c6e6
@ -0,0 +1,15 @@
|
||||
unset GREP_OPTIONS
|
||||
|
||||
# https://zhuanlan.zhihu.com/p/33050965
|
||||
alias nvs='nvidia-smi'
|
||||
alias his='history'
|
||||
alias jobs='jobs -l'
|
||||
alias ports='netstat -tulanp'
|
||||
alias wget='wget -c'
|
||||
|
||||
## Colorize the grep command output for ease of use (good for log files)##
|
||||
alias grep='grep --color=auto'
|
||||
alias egrep='egrep --color=auto'
|
||||
alias fgrep='fgrep --color=auto'
|
||||
|
||||
|
||||
@ -1,605 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "academic-surname",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import paddle\n",
|
||||
"from paddle import nn"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "fundamental-treasure",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/workspace/DeepSpeech-2.x/tools/venv-dev/lib/python3.7/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
|
||||
" and should_run_async(code)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"L = nn.Linear(256, 2048)\n",
|
||||
"L2 = nn.Linear(2048, 256)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "consolidated-elephant",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import torch\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "moderate-noise",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"float64\n",
|
||||
"Tensor(shape=[2, 51, 256], dtype=float32, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[[-1.54171216, -2.61531472, -1.79881978, ..., -0.31395876, 0.56513089, -0.44516513],\n",
|
||||
" [-0.79492962, 1.91157901, 0.66567147, ..., 0.54825783, -1.01471853, -0.84924090],\n",
|
||||
" [-1.22556651, -0.36225814, 0.65063190, ..., 0.65726501, 0.05563191, 0.09009409],\n",
|
||||
" ...,\n",
|
||||
" [ 0.38615900, -0.77905393, 0.99732304, ..., -1.38463700, -3.32365036, -1.31089687],\n",
|
||||
" [ 0.05579993, 0.06885809, -1.66662002, ..., -0.23346378, -3.29372883, 1.30561364],\n",
|
||||
" [ 1.90676069, 1.95093191, -0.28849599, ..., -0.06860496, 0.95347673, 1.00475824]],\n",
|
||||
"\n",
|
||||
" [[-0.91453546, 0.55298805, -1.06146812, ..., -0.86378336, 1.00454640, 1.26062179],\n",
|
||||
" [ 0.10223761, 0.81301165, 2.36865163, ..., 0.16821407, 0.29240361, 1.05408621],\n",
|
||||
" [-1.33196676, 1.94433689, 0.01934209, ..., 0.48036841, 0.51585966, 1.22893548],\n",
|
||||
" ...,\n",
|
||||
" [-0.19558455, -0.47075930, 0.90796155, ..., -1.28598249, -0.24321797, 0.17734711],\n",
|
||||
" [ 0.89819717, -1.39516675, 0.17138045, ..., 2.39761519, 1.76364994, -0.52177650],\n",
|
||||
" [ 0.94122332, -0.18581429, 1.36099780, ..., 0.67647684, -0.04699665, 1.51205540]]])\n",
|
||||
"tensor([[[-1.5417, -2.6153, -1.7988, ..., -0.3140, 0.5651, -0.4452],\n",
|
||||
" [-0.7949, 1.9116, 0.6657, ..., 0.5483, -1.0147, -0.8492],\n",
|
||||
" [-1.2256, -0.3623, 0.6506, ..., 0.6573, 0.0556, 0.0901],\n",
|
||||
" ...,\n",
|
||||
" [ 0.3862, -0.7791, 0.9973, ..., -1.3846, -3.3237, -1.3109],\n",
|
||||
" [ 0.0558, 0.0689, -1.6666, ..., -0.2335, -3.2937, 1.3056],\n",
|
||||
" [ 1.9068, 1.9509, -0.2885, ..., -0.0686, 0.9535, 1.0048]],\n",
|
||||
"\n",
|
||||
" [[-0.9145, 0.5530, -1.0615, ..., -0.8638, 1.0045, 1.2606],\n",
|
||||
" [ 0.1022, 0.8130, 2.3687, ..., 0.1682, 0.2924, 1.0541],\n",
|
||||
" [-1.3320, 1.9443, 0.0193, ..., 0.4804, 0.5159, 1.2289],\n",
|
||||
" ...,\n",
|
||||
" [-0.1956, -0.4708, 0.9080, ..., -1.2860, -0.2432, 0.1773],\n",
|
||||
" [ 0.8982, -1.3952, 0.1714, ..., 2.3976, 1.7636, -0.5218],\n",
|
||||
" [ 0.9412, -0.1858, 1.3610, ..., 0.6765, -0.0470, 1.5121]]])\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/workspace/DeepSpeech-2.x/tools/venv-dev/lib/python3.7/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
|
||||
" and should_run_async(code)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"x = np.random.randn(2, 51, 256)\n",
|
||||
"print(x.dtype)\n",
|
||||
"px = paddle.to_tensor(x, dtype='float32')\n",
|
||||
"tx = torch.tensor(x, dtype=torch.float32)\n",
|
||||
"print(px)\n",
|
||||
"print(tx)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cooked-progressive",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "mechanical-prisoner",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"data = np.load('enc_0_ff_out.npz', allow_pickle=True)\n",
|
||||
"t_norm_ff = data['norm_ff']\n",
|
||||
"t_ff_out = data['ff_out']\n",
|
||||
"t_ff_l_x = data['ff_l_x']\n",
|
||||
"t_ff_l_a_x = data['ff_l_a_x']\n",
|
||||
"t_ff_l_a_l_x = data['ff_l_a_l_x']\n",
|
||||
"t_ps = data['ps']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "indie-marriage",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "assured-zambia",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"True\n",
|
||||
"True\n",
|
||||
"True\n",
|
||||
"True\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"L.set_state_dict({'weight': t_ps[0].T, 'bias': t_ps[1]})\n",
|
||||
"L2.set_state_dict({'weight': t_ps[2].T, 'bias': t_ps[3]})\n",
|
||||
"\n",
|
||||
"ps = []\n",
|
||||
"for n, p in L.named_parameters():\n",
|
||||
" ps.append(p)\n",
|
||||
"\n",
|
||||
"for n, p in L2.state_dict().items():\n",
|
||||
" ps.append(p)\n",
|
||||
" \n",
|
||||
"for p, tp in zip(ps, t_ps):\n",
|
||||
" print(np.allclose(p.numpy(), tp.T))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "committed-jacob",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "extreme-traffic",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "optimum-milwaukee",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "viral-indian",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"True\n",
|
||||
"True\n",
|
||||
"True\n",
|
||||
"True\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# data = np.load('enc_0_ff_out.npz', allow_pickle=True)\n",
|
||||
"# t_norm_ff = data['norm_ff']\n",
|
||||
"# t_ff_out = data['ff_out']\n",
|
||||
"# t_ff_l_x = data['ff_l_x']\n",
|
||||
"# t_ff_l_a_x = data['ff_l_a_x']\n",
|
||||
"# t_ff_l_a_l_x = data['ff_l_a_l_x']\n",
|
||||
"# t_ps = data['ps']\n",
|
||||
"TL = torch.nn.Linear(256, 2048)\n",
|
||||
"TL2 = torch.nn.Linear(2048, 256)\n",
|
||||
"TL.load_state_dict({'weight': torch.tensor(t_ps[0]), 'bias': torch.tensor(t_ps[1])})\n",
|
||||
"TL2.load_state_dict({'weight': torch.tensor(t_ps[2]), 'bias': torch.tensor(t_ps[3])})\n",
|
||||
"\n",
|
||||
"# for n, p in TL.named_parameters():\n",
|
||||
"# print(n, p)\n",
|
||||
"# for n, p in TL2.named_parameters():\n",
|
||||
"# print(n, p)\n",
|
||||
"\n",
|
||||
"ps = []\n",
|
||||
"for n, p in TL.state_dict().items():\n",
|
||||
" ps.append(p.data.numpy())\n",
|
||||
" \n",
|
||||
"for n, p in TL2.state_dict().items():\n",
|
||||
" ps.append(p.data.numpy())\n",
|
||||
" \n",
|
||||
"for p, tp in zip(ps, t_ps):\n",
|
||||
" print(np.allclose(p, tp))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "skilled-vietnamese",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[[[ 0.67277956 0.08313607 -0.62761104 ... -0.17480263 0.42718208\n",
|
||||
" -0.5787626 ]\n",
|
||||
" [ 0.91516656 0.5393416 1.7159258 ... 0.06144593 0.06486575\n",
|
||||
" -0.03350811]\n",
|
||||
" [ 0.438351 0.6227843 0.24096036 ... 1.0912522 -0.90929437\n",
|
||||
" -1.012989 ]\n",
|
||||
" ...\n",
|
||||
" [ 0.68631977 0.14240924 0.10763275 ... -0.11513516 0.48065388\n",
|
||||
" 0.04070369]\n",
|
||||
" [-0.9525228 0.23197874 0.31264272 ... 0.5312439 0.18773697\n",
|
||||
" -0.8450228 ]\n",
|
||||
" [ 0.42024016 -0.04561988 0.54541194 ... -0.41933843 -0.00436018\n",
|
||||
" -0.06663495]]\n",
|
||||
"\n",
|
||||
" [[-0.11638781 -0.33566502 -0.20887226 ... 0.17423287 -0.9195841\n",
|
||||
" -0.8161046 ]\n",
|
||||
" [-0.3469874 0.88269687 -0.11887559 ... -0.15566081 0.16357468\n",
|
||||
" -0.20766167]\n",
|
||||
" [-0.3847657 0.3984318 -0.06963477 ... -0.00360622 1.2360432\n",
|
||||
" -0.26811332]\n",
|
||||
" ...\n",
|
||||
" [ 0.08230796 -0.46158582 0.54582864 ... 0.15747628 -0.44790155\n",
|
||||
" 0.06020184]\n",
|
||||
" [-0.8095085 0.43163058 -0.42837143 ... 0.8627463 0.90656304\n",
|
||||
" 0.15847842]\n",
|
||||
" [-1.485811 -0.18216592 -0.8882585 ... 0.32596245 0.7822631\n",
|
||||
" -0.6460344 ]]]\n",
|
||||
"[[[ 0.67278004 0.08313602 -0.6276114 ... -0.17480245 0.42718196\n",
|
||||
" -0.5787625 ]\n",
|
||||
" [ 0.91516703 0.5393413 1.7159253 ... 0.06144581 0.06486579\n",
|
||||
" -0.03350812]\n",
|
||||
" [ 0.43835106 0.62278455 0.24096027 ... 1.0912521 -0.9092943\n",
|
||||
" -1.0129892 ]\n",
|
||||
" ...\n",
|
||||
" [ 0.6863195 0.14240888 0.10763284 ... -0.11513527 0.48065376\n",
|
||||
" 0.04070365]\n",
|
||||
" [-0.9525231 0.23197863 0.31264275 ... 0.53124386 0.18773702\n",
|
||||
" -0.84502304]\n",
|
||||
" [ 0.42024007 -0.04561983 0.545412 ... -0.41933888 -0.00436005\n",
|
||||
" -0.066635 ]]\n",
|
||||
"\n",
|
||||
" [[-0.11638767 -0.33566508 -0.20887226 ... 0.17423296 -0.9195838\n",
|
||||
" -0.8161046 ]\n",
|
||||
" [-0.34698725 0.88269705 -0.11887549 ... -0.15566081 0.16357464\n",
|
||||
" -0.20766166]\n",
|
||||
" [-0.3847657 0.3984319 -0.06963488 ... -0.00360619 1.2360426\n",
|
||||
" -0.26811326]\n",
|
||||
" ...\n",
|
||||
" [ 0.08230786 -0.4615857 0.5458287 ... 0.15747619 -0.44790167\n",
|
||||
" 0.06020182]\n",
|
||||
" [-0.8095083 0.4316307 -0.42837155 ... 0.862746 0.9065631\n",
|
||||
" 0.15847899]\n",
|
||||
" [-1.485811 -0.18216613 -0.8882584 ... 0.32596254 0.7822631\n",
|
||||
" -0.6460344 ]]]\n",
|
||||
"True\n",
|
||||
"False\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"y = L(px)\n",
|
||||
"print(y.numpy())\n",
|
||||
"\n",
|
||||
"ty = TL(tx)\n",
|
||||
"print(ty.data.numpy())\n",
|
||||
"print(np.allclose(px.numpy(), tx.detach().numpy()))\n",
|
||||
"print(np.allclose(y.numpy(), ty.detach().numpy()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "incorrect-allah",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "prostate-cameroon",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "governmental-surge",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[[ 0.04476918 0.554463 -0.3027508 ... -0.49600336 0.3751858\n",
|
||||
" 0.8254095 ]\n",
|
||||
" [ 0.95594174 -0.29528382 -1.2899452 ... 0.43718258 0.05584608\n",
|
||||
" -0.06974669]]\n",
|
||||
"[[ 0.04476918 0.5544631 -0.3027507 ... -0.49600336 0.37518573\n",
|
||||
" 0.8254096 ]\n",
|
||||
" [ 0.95594174 -0.29528376 -1.2899454 ... 0.4371827 0.05584623\n",
|
||||
" -0.0697467 ]]\n",
|
||||
"True\n",
|
||||
"False\n",
|
||||
"True\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"x = np.random.randn(2, 256)\n",
|
||||
"px = paddle.to_tensor(x, dtype='float32')\n",
|
||||
"tx = torch.tensor(x, dtype=torch.float32)\n",
|
||||
"y = L(px)\n",
|
||||
"print(y.numpy())\n",
|
||||
"ty = TL(tx)\n",
|
||||
"print(ty.data.numpy())\n",
|
||||
"print(np.allclose(px.numpy(), tx.detach().numpy()))\n",
|
||||
"print(np.allclose(y.numpy(), ty.detach().numpy()))\n",
|
||||
"print(np.allclose(y.numpy(), ty.detach().numpy(), atol=1e-5))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "confidential-jacket",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "improved-civilization",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"5e7e7c9fde8350084abf1898cf52651cfc84b17a\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(paddle.version.commit)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "d1e2d3b4",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['__builtins__',\n",
|
||||
" '__cached__',\n",
|
||||
" '__doc__',\n",
|
||||
" '__file__',\n",
|
||||
" '__loader__',\n",
|
||||
" '__name__',\n",
|
||||
" '__package__',\n",
|
||||
" '__spec__',\n",
|
||||
" 'commit',\n",
|
||||
" 'full_version',\n",
|
||||
" 'istaged',\n",
|
||||
" 'major',\n",
|
||||
" 'minor',\n",
|
||||
" 'mkl',\n",
|
||||
" 'patch',\n",
|
||||
" 'rc',\n",
|
||||
" 'show',\n",
|
||||
" 'with_mkl']"
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"dir(paddle.version)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "c880c719",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"2.1.0\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(paddle.version.full_version)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "f26977bf",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"commit: 5e7e7c9fde8350084abf1898cf52651cfc84b17a\n",
|
||||
"None\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(paddle.version.show())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "04ad47f6",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"1.6.0\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(torch.__version__)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "e1e03830",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['__builtins__',\n",
|
||||
" '__cached__',\n",
|
||||
" '__doc__',\n",
|
||||
" '__file__',\n",
|
||||
" '__loader__',\n",
|
||||
" '__name__',\n",
|
||||
" '__package__',\n",
|
||||
" '__spec__',\n",
|
||||
" '__version__',\n",
|
||||
" 'cuda',\n",
|
||||
" 'debug',\n",
|
||||
" 'git_version',\n",
|
||||
" 'hip']"
|
||||
]
|
||||
},
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"dir(torch.version)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"id": "4ad0389b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'b31f58de6fa8bbda5353b3c77d9be4914399724d'"
|
||||
]
|
||||
},
|
||||
"execution_count": 19,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"torch.version.git_version"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"id": "7870ea10",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'10.2'"
|
||||
]
|
||||
},
|
||||
"execution_count": 21,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"torch.version.cuda"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "db8ee5a7",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6321ec2a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@ -1,389 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "emerging-meter",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/workspace/DeepSpeech-2.x/tools/venv/lib/python3.7/site-packages/paddle/fluid/layers/utils.py:26: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
|
||||
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||||
" def convert_to_list(value, n, name, dtype=np.int):\n",
|
||||
"/workspace/DeepSpeech-2.x/tools/venv/lib/python3.7/site-packages/scipy/fftpack/__init__.py:103: DeprecationWarning: The module numpy.dual is deprecated. Instead of using dual, use the functions directly from numpy or scipy.\n",
|
||||
" from numpy.dual import register_func\n",
|
||||
"/workspace/DeepSpeech-2.x/tools/venv/lib/python3.7/site-packages/scipy/special/orthogonal.py:81: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
|
||||
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||||
" from numpy import (exp, inf, pi, sqrt, floor, sin, cos, around, int,\n",
|
||||
"/workspace/DeepSpeech-2.x/tools/venv/lib/python3.7/site-packages/numba/core/types/__init__.py:108: DeprecationWarning: `np.long` is a deprecated alias for `np.compat.long`. To silence this warning, use `np.compat.long` by itself. In the likely event your code does not need to work on Python 2 you can use the builtin `int` for which `np.compat.long` is itself an alias. Doing this will not modify any behaviour and is safe. When replacing `np.long`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
|
||||
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||||
" long_ = _make_signed(np.long)\n",
|
||||
"/workspace/DeepSpeech-2.x/tools/venv/lib/python3.7/site-packages/numba/core/types/__init__.py:109: DeprecationWarning: `np.long` is a deprecated alias for `np.compat.long`. To silence this warning, use `np.compat.long` by itself. In the likely event your code does not need to work on Python 2 you can use the builtin `int` for which `np.compat.long` is itself an alias. Doing this will not modify any behaviour and is safe. When replacing `np.long`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
|
||||
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||||
" ulong = _make_unsigned(np.long)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import math\n",
|
||||
"import random\n",
|
||||
"import tarfile\n",
|
||||
"import logging\n",
|
||||
"import numpy as np\n",
|
||||
"from collections import namedtuple\n",
|
||||
"from functools import partial\n",
|
||||
"\n",
|
||||
"import paddle\n",
|
||||
"from paddle.io import Dataset\n",
|
||||
"from paddle.io import DataLoader\n",
|
||||
"from paddle.io import BatchSampler\n",
|
||||
"from paddle.io import DistributedBatchSampler\n",
|
||||
"from paddle import distributed as dist\n",
|
||||
"\n",
|
||||
"from data_utils.utility import read_manifest\n",
|
||||
"from data_utils.augmentor.augmentation import AugmentationPipeline\n",
|
||||
"from data_utils.featurizer.speech_featurizer import SpeechFeaturizer\n",
|
||||
"from data_utils.speech import SpeechSegment\n",
|
||||
"from data_utils.normalizer import FeatureNormalizer\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"from data_utils.dataset import (\n",
|
||||
" DeepSpeech2Dataset,\n",
|
||||
" DeepSpeech2DistributedBatchSampler,\n",
|
||||
" DeepSpeech2BatchSampler,\n",
|
||||
" SpeechCollator,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"id": "excessive-american",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def create_dataloader(manifest_path,\t\n",
|
||||
" vocab_filepath,\t\n",
|
||||
" mean_std_filepath,\t\n",
|
||||
" augmentation_config='{}',\t\n",
|
||||
" max_duration=float('inf'),\t\n",
|
||||
" min_duration=0.0,\t\n",
|
||||
" stride_ms=10.0,\t\n",
|
||||
" window_ms=20.0,\t\n",
|
||||
" max_freq=None,\t\n",
|
||||
" specgram_type='linear',\t\n",
|
||||
" use_dB_normalization=True,\t\n",
|
||||
" random_seed=0,\t\n",
|
||||
" keep_transcription_text=False,\t\n",
|
||||
" is_training=False,\t\n",
|
||||
" batch_size=1,\t\n",
|
||||
" num_workers=0,\t\n",
|
||||
" sortagrad=False,\t\n",
|
||||
" shuffle_method=None,\t\n",
|
||||
" dist=False):\t\n",
|
||||
"\n",
|
||||
" dataset = DeepSpeech2Dataset(\t\n",
|
||||
" manifest_path,\t\n",
|
||||
" vocab_filepath,\t\n",
|
||||
" mean_std_filepath,\t\n",
|
||||
" augmentation_config=augmentation_config,\t\n",
|
||||
" max_duration=max_duration,\t\n",
|
||||
" min_duration=min_duration,\t\n",
|
||||
" stride_ms=stride_ms,\t\n",
|
||||
" window_ms=window_ms,\t\n",
|
||||
" max_freq=max_freq,\t\n",
|
||||
" specgram_type=specgram_type,\t\n",
|
||||
" use_dB_normalization=use_dB_normalization,\t\n",
|
||||
" random_seed=random_seed,\t\n",
|
||||
" keep_transcription_text=keep_transcription_text)\t\n",
|
||||
"\n",
|
||||
" if dist:\t\n",
|
||||
" batch_sampler = DeepSpeech2DistributedBatchSampler(\t\n",
|
||||
" dataset,\t\n",
|
||||
" batch_size,\t\n",
|
||||
" num_replicas=None,\t\n",
|
||||
" rank=None,\t\n",
|
||||
" shuffle=is_training,\t\n",
|
||||
" drop_last=is_training,\t\n",
|
||||
" sortagrad=is_training,\t\n",
|
||||
" shuffle_method=shuffle_method)\t\n",
|
||||
" else:\t\n",
|
||||
" batch_sampler = DeepSpeech2BatchSampler(\t\n",
|
||||
" dataset,\t\n",
|
||||
" shuffle=is_training,\t\n",
|
||||
" batch_size=batch_size,\t\n",
|
||||
" drop_last=is_training,\t\n",
|
||||
" sortagrad=is_training,\t\n",
|
||||
" shuffle_method=shuffle_method)\t\n",
|
||||
"\n",
|
||||
" def padding_batch(batch, padding_to=-1, flatten=False, is_training=True):\t\n",
|
||||
" \"\"\"\t\n",
|
||||
" Padding audio features with zeros to make them have the same shape (or\t\n",
|
||||
" a user-defined shape) within one bach.\t\n",
|
||||
"\n",
|
||||
" If ``padding_to`` is -1, the maximun shape in the batch will be used\t\n",
|
||||
" as the target shape for padding. Otherwise, `padding_to` will be the\t\n",
|
||||
" target shape (only refers to the second axis).\t\n",
|
||||
"\n",
|
||||
" If `flatten` is True, features will be flatten to 1darray.\t\n",
|
||||
" \"\"\"\t\n",
|
||||
" new_batch = []\t\n",
|
||||
" # get target shape\t\n",
|
||||
" max_length = max([audio.shape[1] for audio, text in batch])\t\n",
|
||||
" if padding_to != -1:\t\n",
|
||||
" if padding_to < max_length:\t\n",
|
||||
" raise ValueError(\"If padding_to is not -1, it should be larger \"\t\n",
|
||||
" \"than any instance's shape in the batch\")\t\n",
|
||||
" max_length = padding_to\t\n",
|
||||
" max_text_length = max([len(text) for audio, text in batch])\t\n",
|
||||
" # padding\t\n",
|
||||
" padded_audios = []\t\n",
|
||||
" audio_lens = []\t\n",
|
||||
" texts, text_lens = [], []\t\n",
|
||||
" for audio, text in batch:\t\n",
|
||||
" padded_audio = np.zeros([audio.shape[0], max_length])\t\n",
|
||||
" padded_audio[:, :audio.shape[1]] = audio\t\n",
|
||||
" if flatten:\t\n",
|
||||
" padded_audio = padded_audio.flatten()\t\n",
|
||||
" padded_audios.append(padded_audio)\t\n",
|
||||
" audio_lens.append(audio.shape[1])\t\n",
|
||||
"\n",
|
||||
" padded_text = np.zeros([max_text_length])\n",
|
||||
" if is_training:\n",
|
||||
" padded_text[:len(text)] = text\t# ids\n",
|
||||
" else:\n",
|
||||
" padded_text[:len(text)] = [ord(t) for t in text] # string\n",
|
||||
" \n",
|
||||
" texts.append(padded_text)\t\n",
|
||||
" text_lens.append(len(text))\t\n",
|
||||
"\n",
|
||||
" padded_audios = np.array(padded_audios).astype('float32')\t\n",
|
||||
" audio_lens = np.array(audio_lens).astype('int64')\t\n",
|
||||
" texts = np.array(texts).astype('int32')\t\n",
|
||||
" text_lens = np.array(text_lens).astype('int64')\t\n",
|
||||
" return padded_audios, texts, audio_lens, text_lens\t\n",
|
||||
"\n",
|
||||
" loader = DataLoader(\t\n",
|
||||
" dataset,\t\n",
|
||||
" batch_sampler=batch_sampler,\t\n",
|
||||
" collate_fn=partial(padding_batch, is_training=is_training),\t\n",
|
||||
" num_workers=num_workers)\t\n",
|
||||
" return loader"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"id": "naval-brave",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'num_samples': 5, 'beam_size': 500, 'num_proc_bsearch': 8, 'num_conv_layers': 2, 'num_rnn_layers': 3, 'rnn_layer_size': 2048, 'alpha': 2.5, 'beta': 0.3, 'cutoff_prob': 1.0, 'cutoff_top_n': 40, 'use_gru': False, 'use_gpu': True, 'share_rnn_weights': True, 'infer_manifest': 'examples/aishell/data/manifest.dev', 'mean_std_path': 'examples/aishell/data/mean_std.npz', 'vocab_path': 'examples/aishell/data/vocab.txt', 'lang_model_path': 'models/lm/common_crawl_00.prune01111.trie.klm', 'model_path': 'examples/aishell/checkpoints/step_final', 'decoding_method': 'ctc_beam_search', 'error_rate_type': 'wer', 'specgram_type': 'linear'}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"import argparse\n",
|
||||
"import functools\n",
|
||||
"from utils.utility import add_arguments, print_arguments\n",
|
||||
"parser = argparse.ArgumentParser(description=__doc__)\n",
|
||||
"add_arg = functools.partial(add_arguments, argparser=parser)\n",
|
||||
"# yapf: disable\n",
|
||||
"add_arg('num_samples', int, 5, \"# of samples to infer.\")\n",
|
||||
"add_arg('beam_size', int, 500, \"Beam search width.\")\n",
|
||||
"add_arg('num_proc_bsearch', int, 8, \"# of CPUs for beam search.\")\n",
|
||||
"add_arg('num_conv_layers', int, 2, \"# of convolution layers.\")\n",
|
||||
"add_arg('num_rnn_layers', int, 3, \"# of recurrent layers.\")\n",
|
||||
"add_arg('rnn_layer_size', int, 2048, \"# of recurrent cells per layer.\")\n",
|
||||
"add_arg('alpha', float, 2.5, \"Coef of LM for beam search.\")\n",
|
||||
"add_arg('beta', float, 0.3, \"Coef of WC for beam search.\")\n",
|
||||
"add_arg('cutoff_prob', float, 1.0, \"Cutoff probability for pruning.\")\n",
|
||||
"add_arg('cutoff_top_n', int, 40, \"Cutoff number for pruning.\")\n",
|
||||
"add_arg('use_gru', bool, False, \"Use GRUs instead of simple RNNs.\")\n",
|
||||
"add_arg('use_gpu', bool, True, \"Use GPU or not.\")\n",
|
||||
"add_arg('share_rnn_weights',bool, True, \"Share input-hidden weights across \"\n",
|
||||
" \"bi-directional RNNs. Not for GRU.\")\n",
|
||||
"add_arg('infer_manifest', str,\n",
|
||||
" 'examples/aishell/data/manifest.dev',\n",
|
||||
" \"Filepath of manifest to infer.\")\n",
|
||||
"add_arg('mean_std_path', str,\n",
|
||||
" 'examples/aishell/data/mean_std.npz',\n",
|
||||
" \"Filepath of normalizer's mean & std.\")\n",
|
||||
"add_arg('vocab_path', str,\n",
|
||||
" 'examples/aishell/data/vocab.txt',\n",
|
||||
" \"Filepath of vocabulary.\")\n",
|
||||
"add_arg('lang_model_path', str,\n",
|
||||
" 'models/lm/common_crawl_00.prune01111.trie.klm',\n",
|
||||
" \"Filepath for language model.\")\n",
|
||||
"add_arg('model_path', str,\n",
|
||||
" 'examples/aishell/checkpoints/step_final',\n",
|
||||
" \"If None, the training starts from scratch, \"\n",
|
||||
" \"otherwise, it resumes from the pre-trained model.\")\n",
|
||||
"add_arg('decoding_method', str,\n",
|
||||
" 'ctc_beam_search',\n",
|
||||
" \"Decoding method. Options: ctc_beam_search, ctc_greedy\",\n",
|
||||
" choices = ['ctc_beam_search', 'ctc_greedy'])\n",
|
||||
"add_arg('error_rate_type', str,\n",
|
||||
" 'wer',\n",
|
||||
" \"Error rate type for evaluation.\",\n",
|
||||
" choices=['wer', 'cer'])\n",
|
||||
"add_arg('specgram_type', str,\n",
|
||||
" 'linear',\n",
|
||||
" \"Audio feature type. Options: linear, mfcc.\",\n",
|
||||
" choices=['linear', 'mfcc'])\n",
|
||||
"# yapf: disable\n",
|
||||
"args = parser.parse_args([])\n",
|
||||
"print(vars(args))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"id": "bearing-physics",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_reader = create_dataloader(\n",
|
||||
" manifest_path=args.infer_manifest,\n",
|
||||
" vocab_filepath=args.vocab_path,\n",
|
||||
" mean_std_filepath=args.mean_std_path,\n",
|
||||
" augmentation_config='{}',\n",
|
||||
" #max_duration=float('inf'),\n",
|
||||
" max_duration=27.0,\n",
|
||||
" min_duration=0.0,\n",
|
||||
" stride_ms=10.0,\n",
|
||||
" window_ms=20.0,\n",
|
||||
" max_freq=None,\n",
|
||||
" specgram_type=args.specgram_type,\n",
|
||||
" use_dB_normalization=True,\n",
|
||||
" random_seed=0,\n",
|
||||
" keep_transcription_text=True,\n",
|
||||
" is_training=False,\n",
|
||||
" batch_size=args.num_samples,\n",
|
||||
" sortagrad=True,\n",
|
||||
" shuffle_method=None,\n",
|
||||
" dist=False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 30,
|
||||
"id": "classified-melissa",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"test Tensor(shape=[5, 6], dtype=int32, place=CUDAPinnedPlace, stop_gradient=True,\n",
|
||||
" [[22823, 26102, 20195, 37324, 0 , 0 ],\n",
|
||||
" [22238, 26469, 23601, 22909, 0 , 0 ],\n",
|
||||
" [20108, 26376, 22235, 26085, 0 , 0 ],\n",
|
||||
" [36824, 35201, 20445, 25345, 32654, 24863],\n",
|
||||
" [29042, 27748, 21463, 23456, 0 , 0 ]])\n",
|
||||
"test raw 大时代里\n",
|
||||
"test raw 煲汤受宠\n",
|
||||
"audio len Tensor(shape=[5], dtype=int64, place=CUDAPinnedPlace, stop_gradient=True,\n",
|
||||
" [163, 167, 180, 186, 186])\n",
|
||||
"test len Tensor(shape=[5], dtype=int64, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [4, 4, 4, 6, 4])\n",
|
||||
"audio Tensor(shape=[5, 161, 186], dtype=float32, place=CUDAPinnedPlace, stop_gradient=True,\n",
|
||||
" [[[ 1.11669052, 0.79015088, 0.93658292, ..., 0. , 0. , 0. ],\n",
|
||||
" [ 0.83549136, 0.72643483, 0.83578080, ..., 0. , 0. , 0. ],\n",
|
||||
" [-0.89155018, -0.18894747, -0.53357804, ..., 0. , 0. , 0. ],\n",
|
||||
" ...,\n",
|
||||
" [ 0.33386710, -0.81240511, 0.12869737, ..., 0. , 0. , 0. ],\n",
|
||||
" [-0.17537928, 0.58380985, 0.70696265, ..., 0. , 0. , 0. ],\n",
|
||||
" [-0.84175998, 1.22041416, 0.07929770, ..., 0. , 0. , 0. ]],\n",
|
||||
"\n",
|
||||
" [[-0.35964420, 0.77392709, 0.71409988, ..., 0. , 0. , 0. ],\n",
|
||||
" [-0.15990183, 0.42962283, 0.06222462, ..., 0. , 0. , 0. ],\n",
|
||||
" [-0.31166190, -0.74864638, -0.52836996, ..., 0. , 0. , 0. ],\n",
|
||||
" ...,\n",
|
||||
" [-0.27546275, 0.32889456, 0.12410031, ..., 0. , 0. , 0. ],\n",
|
||||
" [ 0.16264282, 0.49418071, -0.15960945, ..., 0. , 0. , 0. ],\n",
|
||||
" [ 0.12476666, 0.00516864, 1.16021466, ..., 0. , 0. , 0. ]],\n",
|
||||
"\n",
|
||||
" [[ 0.90202141, 1.48541915, 0.92062062, ..., 0. , 0. , 0. ],\n",
|
||||
" [ 0.82661545, 1.37171340, 0.86746097, ..., 0. , 0. , 0. ],\n",
|
||||
" [-0.62287915, -0.48645937, 0.35041964, ..., 0. , 0. , 0. ],\n",
|
||||
" ...,\n",
|
||||
" [ 0.07376949, 0.07138316, 0.76355994, ..., 0. , 0. , 0. ],\n",
|
||||
" [-0.32306790, 0.43247896, 1.27311838, ..., 0. , 0. , 0. ],\n",
|
||||
" [-0.97667056, 0.60747612, 0.79181534, ..., 0. , 0. , 0. ]],\n",
|
||||
"\n",
|
||||
" [[ 0.72022128, 0.95428467, 0.92766261, ..., 0.29105374, -0.45564806, -0.62151009],\n",
|
||||
" [ 0.42083180, 0.49279949, 0.82724041, ..., -0.17333922, -1.45363355, -0.61673522],\n",
|
||||
" [-0.76116520, -0.84750438, -0.09512503, ..., -1.01497340, -1.42781055, -0.80859023],\n",
|
||||
" ...,\n",
|
||||
" [-0.23009977, 1.06155431, 1.09065628, ..., 0.25581080, 0.53794998, -1.22650719],\n",
|
||||
" [-1.37693381, 0.30778193, 0.17152318, ..., 0.51650339, 0.25580606, 0.83097816],\n",
|
||||
" [-1.62180591, 1.30567718, 1.09928656, ..., -0.77590007, 1.27712476, 0.53189957]],\n",
|
||||
"\n",
|
||||
" [[ 1.03205252, -0.51535392, 0.21077573, ..., 0.76618457, 1.27425683, 1.52250278],\n",
|
||||
" [ 0.82059991, 0.43990925, 0.13090958, ..., 0.86662549, 1.01687658, 1.48495352],\n",
|
||||
" [-0.75489789, -0.01997089, -0.65174174, ..., 0.09061214, -0.55211234, -0.01614586],\n",
|
||||
" ...,\n",
|
||||
" [ 0.50985396, 1.84555030, 0.79185146, ..., 1.13666189, 1.19898069, 1.98158395],\n",
|
||||
" [ 1.98721015, 2.52385354, 1.11714780, ..., 0.19416514, 1.11329341, 0.64460152],\n",
|
||||
" [ 2.69512844, 1.90993905, 0.50245082, ..., -0.50902629, 0.03333465, -1.24584770]]])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"for idx, (audio, audio_len, text, text_len) in enumerate(batch_reader()):\n",
|
||||
" print('test', text)\n",
|
||||
" print(\"test raw\", ''.join( chr(i) for i in text[0][:int(text_len[0])] ))\n",
|
||||
" print(\"test raw\", ''.join( chr(i) for i in text[-1][:int(text_len[-1])] ))\n",
|
||||
" print('audio len', audio_len)\n",
|
||||
" print('test len', text_len)\n",
|
||||
" print('audio', audio)\n",
|
||||
" break"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "unexpected-skating",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "minus-modern",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,290 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "breeding-haven",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/ssd5/zhanghui/DeepSpeech2.x\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'/home/ssd5/zhanghui/DeepSpeech2.x'"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%cd ..\n",
|
||||
"%pwd"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "appropriate-theta",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"LICENSE deepspeech examples\t\t requirements.txt tools\r\n",
|
||||
"README.md docs\t libsndfile-1.0.28\t setup.sh\t utils\r\n",
|
||||
"README_cn.md env.sh\t libsndfile-1.0.28.tar.gz tests\r\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!ls"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "entire-bloom",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/ssd5/zhanghui/DeepSpeech2.x/tools/venv/lib/python3.7/site-packages/paddle/fluid/layers/utils.py:26: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
|
||||
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||||
" def convert_to_list(value, n, name, dtype=np.int):\n",
|
||||
"WARNING:root:override cat of paddle.Tensor if exists or register, remove this when fixed!\n",
|
||||
"WARNING:root:register user masked_fill to paddle.Tensor, remove this when fixed!\n",
|
||||
"WARNING:root:register user masked_fill_ to paddle.Tensor, remove this when fixed!\n",
|
||||
"WARNING:root:register user repeat to paddle.Tensor, remove this when fixed!\n",
|
||||
"WARNING:root:register user glu to paddle.nn.functional, remove this when fixed!\n",
|
||||
"WARNING:root:register user GLU to paddle.nn, remove this when fixed!\n",
|
||||
"WARNING:root:register user ConstantPad2d to paddle.nn, remove this when fixed!\n",
|
||||
"WARNING:root:override ctc_loss of paddle.nn.functional if exists, remove this when fixed!\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from deepspeech.modules import loss"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "governmental-aircraft",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/ssd5/zhanghui/DeepSpeech2.x/tools/venv/lib/python3.7/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
|
||||
" and should_run_async(code)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import paddle"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "proprietary-disaster",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<function deepspeech.modules.repeat(xs: paddle.VarBase, *size: Any) -> paddle.VarBase>"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"paddle.Tensor.repeat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "first-diagram",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<property at 0x7fb515eeeb88>"
|
||||
]
|
||||
},
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"paddle.Tensor.size"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "intelligent-david",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<function paddle.tensor.manipulation.concat(x, axis=0, name=None)>"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"paddle.Tensor.cat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "bronze-tenant",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"a = paddle.to_tensor([12,32, 10, 12, 123,32 ,4])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "balanced-bearing",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"7"
|
||||
]
|
||||
},
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"a.size"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"id": "extreme-republic",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def size(xs: paddle.Tensor, *args: int) -> paddle.Tensor:\n",
|
||||
" nargs = len(args)\n",
|
||||
" assert (nargs <= 1)\n",
|
||||
" s = paddle.shape(xs)\n",
|
||||
" if nargs == 1:\n",
|
||||
" return s[args[0]]\n",
|
||||
" else:\n",
|
||||
" return s\n",
|
||||
"\n",
|
||||
"# logger.warn(\n",
|
||||
"# \"override size of paddle.Tensor if exists or register, remove this when fixed!\"\n",
|
||||
"# )\n",
|
||||
"paddle.Tensor.size = size"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"id": "gross-addiction",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"Tensor(shape=[1], dtype=int32, place=CPUPlace, stop_gradient=True,\n",
|
||||
" [7])"
|
||||
]
|
||||
},
|
||||
"execution_count": 21,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"a.size(0)\n",
|
||||
"a.size()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"id": "adverse-dining",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"Tensor(shape=[1], dtype=int32, place=CPUPlace, stop_gradient=True,\n",
|
||||
" [7])"
|
||||
]
|
||||
},
|
||||
"execution_count": 22,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"a.size()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "popular-potato",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -1,229 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 32,
|
||||
"id": "academic-surname",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import paddle\n",
|
||||
"from paddle import nn"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 33,
|
||||
"id": "fundamental-treasure",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Parameter containing:\n",
|
||||
"Tensor(shape=[256], dtype=float32, place=CUDAPlace(0), stop_gradient=False,\n",
|
||||
" [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])\n",
|
||||
"Parameter containing:\n",
|
||||
"Tensor(shape=[256], dtype=float32, place=CUDAPlace(0), stop_gradient=False,\n",
|
||||
" [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"L = nn.LayerNorm(256, epsilon=1e-12)\n",
|
||||
"for p in L.parameters():\n",
|
||||
" print(p)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 34,
|
||||
"id": "consolidated-elephant",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 46,
|
||||
"id": "moderate-noise",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"float64\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"x = np.random.randn(2, 51, 256)\n",
|
||||
"print(x.dtype)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 47,
|
||||
"id": "cooked-progressive",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"y = L(paddle.to_tensor(x, dtype='float32'))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 48,
|
||||
"id": "optimum-milwaukee",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 49,
|
||||
"id": "viral-indian",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Parameter containing:\n",
|
||||
"tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
|
||||
" 1., 1., 1., 1.], requires_grad=True)\n",
|
||||
"Parameter containing:\n",
|
||||
"tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
|
||||
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
|
||||
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
|
||||
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
|
||||
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
|
||||
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
|
||||
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
|
||||
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
|
||||
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
|
||||
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
|
||||
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
|
||||
" requires_grad=True)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"TL = torch.nn.LayerNorm(256, eps=1e-12)\n",
|
||||
"for p in TL.parameters():\n",
|
||||
" print(p)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 50,
|
||||
"id": "skilled-vietnamese",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ty = TL(torch.tensor(x, dtype=torch.float32))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 51,
|
||||
"id": "incorrect-allah",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"False"
|
||||
]
|
||||
},
|
||||
"execution_count": 51,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"np.allclose(y.numpy(), ty.detach().numpy())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "prostate-cameroon",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 52,
|
||||
"id": "governmental-surge",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"True"
|
||||
]
|
||||
},
|
||||
"execution_count": 52,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"x = np.random.randn(2, 256)\n",
|
||||
"y = L(paddle.to_tensor(x, dtype='float32'))\n",
|
||||
"ty = TL(torch.tensor(x, dtype=torch.float32))\n",
|
||||
"np.allclose(y.numpy(), ty.detach().numpy())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "confidential-jacket",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -1,449 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "primary-organic",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 38,
|
||||
"id": "stopped-semester",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def mask_finished_scores(score: torch.Tensor,\n",
|
||||
" flag: torch.Tensor) -> torch.Tensor:\n",
|
||||
" \"\"\"\n",
|
||||
" If a sequence is finished, we only allow one alive branch. This function\n",
|
||||
" aims to give one branch a zero score and the rest -inf score.\n",
|
||||
" Args:\n",
|
||||
" score (torch.Tensor): A real value array with shape\n",
|
||||
" (batch_size * beam_size, beam_size).\n",
|
||||
" flag (torch.Tensor): A bool array with shape\n",
|
||||
" (batch_size * beam_size, 1).\n",
|
||||
" Returns:\n",
|
||||
" torch.Tensor: (batch_size * beam_size, beam_size).\n",
|
||||
" \"\"\"\n",
|
||||
" beam_size = score.size(-1)\n",
|
||||
" zero_mask = torch.zeros_like(flag, dtype=torch.bool)\n",
|
||||
" if beam_size > 1:\n",
|
||||
" unfinished = torch.cat((zero_mask, flag.repeat([1, beam_size - 1])),\n",
|
||||
" dim=1)\n",
|
||||
" finished = torch.cat((flag, zero_mask.repeat([1, beam_size - 1])),\n",
|
||||
" dim=1)\n",
|
||||
" else:\n",
|
||||
" unfinished = zero_mask\n",
|
||||
" finished = flag\n",
|
||||
" print(unfinished)\n",
|
||||
" print(finished)\n",
|
||||
" score.masked_fill_(unfinished, -float('inf'))\n",
|
||||
" score.masked_fill_(finished, 0)\n",
|
||||
" return score"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 58,
|
||||
"id": "agreed-portuguese",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"tensor([[ True],\n",
|
||||
" [False]])\n",
|
||||
"tensor([[-0.8841, 0.7381, -0.9986],\n",
|
||||
" [ 0.2675, -0.7971, 0.3798]])\n",
|
||||
"tensor([[ True, True],\n",
|
||||
" [False, False]])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"score = torch.randn((2, 3))\n",
|
||||
"flag = torch.ones((2, 1), dtype=torch.bool)\n",
|
||||
"flag[1] = False\n",
|
||||
"print(flag)\n",
|
||||
"print(score)\n",
|
||||
"print(flag.repeat([1, 2]))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 59,
|
||||
"id": "clean-aspect",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"tensor([[False, True, True],\n",
|
||||
" [False, False, False]])\n",
|
||||
"tensor([[ True, False, False],\n",
|
||||
" [False, False, False]])\n",
|
||||
"tensor([[ 0.0000, -inf, -inf],\n",
|
||||
" [ 0.2675, -0.7971, 0.3798]])\n",
|
||||
"tensor([[ 0.0000, -inf, -inf],\n",
|
||||
" [ 0.2675, -0.7971, 0.3798]])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"r = mask_finished_scores(score, flag)\n",
|
||||
"print(r)\n",
|
||||
"print(score)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 55,
|
||||
"id": "thrown-airline",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Tensor(shape=[2, 1], dtype=bool, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[True ],\n",
|
||||
" [False]])\n",
|
||||
"Tensor(shape=[2, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[ 2.05994511, 1.87704289, 0.01988174],\n",
|
||||
" [-0.40165186, 0.77547729, -0.64469045]])\n",
|
||||
"Tensor(shape=[2, 2], dtype=bool, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[True , True ],\n",
|
||||
" [False, False]])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import paddle\n",
|
||||
"\n",
|
||||
"score = paddle.randn((2, 3))\n",
|
||||
"flag = paddle.ones((2, 1), dtype='bool')\n",
|
||||
"flag[1] = False\n",
|
||||
"print(flag)\n",
|
||||
"print(score)\n",
|
||||
"print(flag.tile([1, 2]))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 56,
|
||||
"id": "internal-patent",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Tensor(shape=[2, 3], dtype=bool, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[False, True , True ],\n",
|
||||
" [False, False, False]])\n",
|
||||
"Tensor(shape=[2, 3], dtype=bool, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[True , False, False],\n",
|
||||
" [False, False, False]])\n",
|
||||
"x Tensor(shape=[2, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[ 2.05994511, 1.87704289, 0.01988174],\n",
|
||||
" [-0.40165186, 0.77547729, -0.64469045]])\n",
|
||||
"2 Tensor(shape=[2, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[ 2.05994511, 1.87704289, 0.01988174],\n",
|
||||
" [-0.40165186, 0.77547729, -0.64469045]])\n",
|
||||
"3 Tensor(shape=[2, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[ 2.05994511, -inf. , -inf. ],\n",
|
||||
" [-0.40165186, 0.77547729, -0.64469045]])\n",
|
||||
"x Tensor(shape=[2, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[ 2.05994511, -inf. , -inf. ],\n",
|
||||
" [-0.40165186, 0.77547729, -0.64469045]])\n",
|
||||
"2 Tensor(shape=[2, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[ 2.05994511, -inf. , -inf. ],\n",
|
||||
" [-0.40165186, 0.77547729, -0.64469045]])\n",
|
||||
"3 Tensor(shape=[2, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[ 0. , -inf. , -inf. ],\n",
|
||||
" [-0.40165186, 0.77547729, -0.64469045]])\n",
|
||||
"Tensor(shape=[2, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[ 0. , -inf. , -inf. ],\n",
|
||||
" [-0.40165186, 0.77547729, -0.64469045]])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"paddle.bool = 'bool'\n",
|
||||
"\n",
|
||||
"def masked_fill(xs:paddle.Tensor, mask:paddle.Tensor, value:float):\n",
|
||||
" print(xs)\n",
|
||||
" trues = paddle.ones_like(xs) * value\n",
|
||||
" assert xs.shape == mask.shape\n",
|
||||
" xs = paddle.where(mask, trues, xs)\n",
|
||||
" return xs\n",
|
||||
"\n",
|
||||
"def masked_fill_(xs:paddle.Tensor, mask:paddle.Tensor, value:float):\n",
|
||||
" print('x', xs)\n",
|
||||
" trues = paddle.ones_like(xs) * value\n",
|
||||
" assert xs.shape == mask.shape\n",
|
||||
" ret = paddle.where(mask, trues, xs)\n",
|
||||
" print('2', xs)\n",
|
||||
" paddle.assign(ret, output=xs)\n",
|
||||
" print('3', xs)\n",
|
||||
"\n",
|
||||
"paddle.Tensor.masked_fill = masked_fill\n",
|
||||
"paddle.Tensor.masked_fill_ = masked_fill_\n",
|
||||
"\n",
|
||||
"def mask_finished_scores_pd(score: paddle.Tensor,\n",
|
||||
" flag: paddle.Tensor) -> paddle.Tensor:\n",
|
||||
" \"\"\"\n",
|
||||
" If a sequence is finished, we only allow one alive branch. This function\n",
|
||||
" aims to give one branch a zero score and the rest -inf score.\n",
|
||||
" Args:\n",
|
||||
" score (torch.Tensor): A real value array with shape\n",
|
||||
" (batch_size * beam_size, beam_size).\n",
|
||||
" flag (torch.Tensor): A bool array with shape\n",
|
||||
" (batch_size * beam_size, 1).\n",
|
||||
" Returns:\n",
|
||||
" torch.Tensor: (batch_size * beam_size, beam_size).\n",
|
||||
" \"\"\"\n",
|
||||
" beam_size = score.shape[-1]\n",
|
||||
" zero_mask = paddle.zeros_like(flag, dtype=paddle.bool)\n",
|
||||
" if beam_size > 1:\n",
|
||||
" unfinished = paddle.concat((zero_mask, flag.tile([1, beam_size - 1])),\n",
|
||||
" axis=1)\n",
|
||||
" finished = paddle.concat((flag, zero_mask.tile([1, beam_size - 1])),\n",
|
||||
" axis=1)\n",
|
||||
" else:\n",
|
||||
" unfinished = zero_mask\n",
|
||||
" finished = flag\n",
|
||||
" print(unfinished)\n",
|
||||
" print(finished)\n",
|
||||
" \n",
|
||||
" #score.masked_fill_(unfinished, -float('inf'))\n",
|
||||
" #score.masked_fill_(finished, 0)\n",
|
||||
"# infs = paddle.ones_like(score) * -float('inf')\n",
|
||||
"# score = paddle.where(unfinished, infs, score)\n",
|
||||
"# score = paddle.where(finished, paddle.zeros_like(score), score)\n",
|
||||
"\n",
|
||||
"# score = score.masked_fill(unfinished, -float('inf'))\n",
|
||||
"# score = score.masked_fill(finished, 0)\n",
|
||||
" score.masked_fill_(unfinished, -float('inf'))\n",
|
||||
" score.masked_fill_(finished, 0)\n",
|
||||
" return score\n",
|
||||
"\n",
|
||||
"r = mask_finished_scores_pd(score, flag)\n",
|
||||
"print(r)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 57,
|
||||
"id": "vocal-prime",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<bound method PyCapsule.value of Tensor(shape=[2, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[ 0. , -inf. , -inf. ],\n",
|
||||
" [-0.40165186, 0.77547729, -0.64469045]])>"
|
||||
]
|
||||
},
|
||||
"execution_count": 57,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"score.value"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 71,
|
||||
"id": "bacterial-adolescent",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from typing import Union, Any"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 72,
|
||||
"id": "absent-fiber",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def repeat(xs : paddle.Tensor, *size: Any):\n",
|
||||
" print(size)\n",
|
||||
" return paddle.tile(xs, size)\n",
|
||||
"paddle.Tensor.repeat = repeat"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 73,
|
||||
"id": "material-harbor",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"(1, 2)\n",
|
||||
"Tensor(shape=[2, 2], dtype=bool, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[True , True ],\n",
|
||||
" [False, False]])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"flag = paddle.ones((2, 1), dtype='bool')\n",
|
||||
"flag[1] = False\n",
|
||||
"print(flag.repeat(1, 2))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 84,
|
||||
"id": "acute-brighton",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"(Tensor(shape=[1], dtype=int64, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [1]), 2)\n",
|
||||
"Tensor(shape=[2, 2], dtype=bool, place=CUDAPlace(0), stop_gradient=True,\n",
|
||||
" [[True , True ],\n",
|
||||
" [False, False]])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"flag = paddle.ones((2, 1), dtype='bool')\n",
|
||||
"flag[1] = False\n",
|
||||
"print(flag.repeat(paddle.to_tensor(1), 2))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 85,
|
||||
"id": "european-rugby",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def size(xs, *args: int):\n",
|
||||
" nargs = len(args)\n",
|
||||
" s = paddle.shape(xs)\n",
|
||||
" assert(nargs <= 1)\n",
|
||||
" if nargs == 1:\n",
|
||||
" return s[args[0]]\n",
|
||||
" else:\n",
|
||||
" return s\n",
|
||||
"paddle.Tensor.size = size"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 86,
|
||||
"id": "moral-special",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"Tensor(shape=[2], dtype=int32, place=CPUPlace, stop_gradient=True,\n",
|
||||
" [2, 1])"
|
||||
]
|
||||
},
|
||||
"execution_count": 86,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"flag.size()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 87,
|
||||
"id": "ahead-coach",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"Tensor(shape=[1], dtype=int32, place=CPUPlace, stop_gradient=True,\n",
|
||||
" [1])"
|
||||
]
|
||||
},
|
||||
"execution_count": 87,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"flag.size(1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 88,
|
||||
"id": "incomplete-fitness",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"Tensor(shape=[1], dtype=int32, place=CPUPlace, stop_gradient=True,\n",
|
||||
" [2])"
|
||||
]
|
||||
},
|
||||
"execution_count": 88,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"flag.size(0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "upset-connectivity",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -1,231 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "designing-borough",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/workspace/DeepSpeech-2.x/tools/venv/lib/python3.7/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
|
||||
" and should_run_async(code)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[[ 0.0000000e+00 0.0000000e+00 0.0000000e+00 ... 0.0000000e+00\n",
|
||||
" 0.0000000e+00 0.0000000e+00]\n",
|
||||
" [ 8.4147096e-01 8.0196178e-01 7.6172036e-01 ... 1.2409373e-04\n",
|
||||
" 1.1547816e-04 1.0746076e-04]\n",
|
||||
" [ 9.0929741e-01 9.5814437e-01 9.8704624e-01 ... 2.4818745e-04\n",
|
||||
" 2.3095631e-04 2.1492151e-04]\n",
|
||||
" ...\n",
|
||||
" [ 3.7960774e-01 7.4510968e-01 7.3418564e-01 ... 1.2036801e-02\n",
|
||||
" 1.1201146e-02 1.0423505e-02]\n",
|
||||
" [-5.7338190e-01 -8.9752287e-02 -4.1488394e-02 ... 1.2160885e-02\n",
|
||||
" 1.1316618e-02 1.0530960e-02]\n",
|
||||
" [-9.9920684e-01 -8.5234123e-01 -7.8794664e-01 ... 1.2284970e-02\n",
|
||||
" 1.1432089e-02 1.0638415e-02]]\n",
|
||||
"True\n",
|
||||
"True\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"import math\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"max_len=100\n",
|
||||
"d_model=256\n",
|
||||
"\n",
|
||||
"pe = torch.zeros(max_len, d_model)\n",
|
||||
"position = torch.arange(0, max_len,\n",
|
||||
" dtype=torch.float32).unsqueeze(1)\n",
|
||||
"toruch_position = position\n",
|
||||
"div_term = torch.exp(\n",
|
||||
" torch.arange(0, d_model, 2, dtype=torch.float32) *\n",
|
||||
" -(math.log(10000.0) / d_model))\n",
|
||||
"tourch_div_term = div_term.cpu().detach().numpy()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"torhc_sin = torch.sin(position * div_term)\n",
|
||||
"torhc_cos = torch.cos(position * div_term)\n",
|
||||
"print(torhc_sin.cpu().detach().numpy())\n",
|
||||
"np_sin = np.sin((position * div_term).cpu().detach().numpy())\n",
|
||||
"np_cos = np.cos((position * div_term).cpu().detach().numpy())\n",
|
||||
"print(np.allclose(np_sin, torhc_sin.cpu().detach().numpy()))\n",
|
||||
"print(np.allclose(np_cos, torhc_cos.cpu().detach().numpy()))\n",
|
||||
"pe[:, 0::2] = torhc_sin\n",
|
||||
"pe[:, 1::2] = torhc_cos\n",
|
||||
"tourch_pe = pe.cpu().detach().numpy()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "swiss-referral",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"True\n",
|
||||
"True\n",
|
||||
"False\n",
|
||||
"False\n",
|
||||
"False\n",
|
||||
"False\n",
|
||||
"[[ 1. 1. 1. ... 1. 1.\n",
|
||||
" 1. ]\n",
|
||||
" [ 0.5403023 0.59737533 0.6479059 ... 1. 1.\n",
|
||||
" 1. ]\n",
|
||||
" [-0.41614684 -0.28628543 -0.1604359 ... 0.99999994 1.\n",
|
||||
" 1. ]\n",
|
||||
" ...\n",
|
||||
" [-0.92514753 -0.66694194 -0.67894876 ... 0.9999276 0.99993724\n",
|
||||
" 0.9999457 ]\n",
|
||||
" [-0.81928825 -0.9959641 -0.999139 ... 0.99992603 0.999936\n",
|
||||
" 0.99994457]\n",
|
||||
" [ 0.03982088 -0.52298605 -0.6157435 ... 0.99992454 0.9999347\n",
|
||||
" 0.99994344]]\n",
|
||||
"----\n",
|
||||
"[[ 1. 1. 1. ... 1. 1.\n",
|
||||
" 1. ]\n",
|
||||
" [ 0.54030234 0.59737533 0.6479059 ... 1. 1.\n",
|
||||
" 1. ]\n",
|
||||
" [-0.41614684 -0.28628543 -0.1604359 ... 1. 1.\n",
|
||||
" 1. ]\n",
|
||||
" ...\n",
|
||||
" [-0.92514753 -0.66694194 -0.67894876 ... 0.9999276 0.9999373\n",
|
||||
" 0.9999457 ]\n",
|
||||
" [-0.81928825 -0.9959641 -0.999139 ... 0.99992603 0.999936\n",
|
||||
" 0.99994457]\n",
|
||||
" [ 0.03982088 -0.5229861 -0.6157435 ... 0.99992454 0.9999347\n",
|
||||
" 0.99994344]]\n",
|
||||
")))))))\n",
|
||||
"[[ 0.0000000e+00 0.0000000e+00 0.0000000e+00 ... 0.0000000e+00\n",
|
||||
" 0.0000000e+00 0.0000000e+00]\n",
|
||||
" [ 8.4147096e-01 8.0196178e-01 7.6172036e-01 ... 1.2409373e-04\n",
|
||||
" 1.1547816e-04 1.0746076e-04]\n",
|
||||
" [ 9.0929741e-01 9.5814437e-01 9.8704624e-01 ... 2.4818745e-04\n",
|
||||
" 2.3095631e-04 2.1492151e-04]\n",
|
||||
" ...\n",
|
||||
" [ 3.7960774e-01 7.4510968e-01 7.3418564e-01 ... 1.2036801e-02\n",
|
||||
" 1.1201146e-02 1.0423505e-02]\n",
|
||||
" [-5.7338190e-01 -8.9752287e-02 -4.1488394e-02 ... 1.2160885e-02\n",
|
||||
" 1.1316618e-02 1.0530960e-02]\n",
|
||||
" [-9.9920684e-01 -8.5234123e-01 -7.8794664e-01 ... 1.2284970e-02\n",
|
||||
" 1.1432089e-02 1.0638415e-02]]\n",
|
||||
"----\n",
|
||||
"[[ 0.0000000e+00 0.0000000e+00 0.0000000e+00 ... 0.0000000e+00\n",
|
||||
" 0.0000000e+00 0.0000000e+00]\n",
|
||||
" [ 8.4147096e-01 8.0196178e-01 7.6172036e-01 ... 1.2409373e-04\n",
|
||||
" 1.1547816e-04 1.0746076e-04]\n",
|
||||
" [ 9.0929741e-01 9.5814437e-01 9.8704624e-01 ... 2.4818745e-04\n",
|
||||
" 2.3095631e-04 2.1492151e-04]\n",
|
||||
" ...\n",
|
||||
" [ 3.7960774e-01 7.4510968e-01 7.3418564e-01 ... 1.2036801e-02\n",
|
||||
" 1.1201146e-02 1.0423505e-02]\n",
|
||||
" [-5.7338190e-01 -8.9752287e-02 -4.1488394e-02 ... 1.2160885e-02\n",
|
||||
" 1.1316618e-02 1.0530960e-02]\n",
|
||||
" [-9.9920684e-01 -8.5234123e-01 -7.8794664e-01 ... 1.2284970e-02\n",
|
||||
" 1.1432089e-02 1.0638415e-02]]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import paddle\n",
|
||||
"paddle.set_device('cpu')\n",
|
||||
"ppe = paddle.zeros((max_len, d_model), dtype='float32')\n",
|
||||
"position = paddle.arange(0, max_len,\n",
|
||||
" dtype='float32').unsqueeze(1)\n",
|
||||
"print(np.allclose(position.numpy(), toruch_position))\n",
|
||||
"div_term = paddle.exp(\n",
|
||||
" paddle.arange(0, d_model, 2, dtype='float32') *\n",
|
||||
" -(math.log(10000.0) / d_model))\n",
|
||||
"print(np.allclose(div_term.numpy(), tourch_div_term))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"p_sin = paddle.sin(position * div_term)\n",
|
||||
"p_cos = paddle.cos(position * div_term)\n",
|
||||
"print(np.allclose(np_sin, p_sin.numpy(), rtol=1.e-6, atol=0))\n",
|
||||
"print(np.allclose(np_cos, p_cos.numpy(), rtol=1.e-6, atol=0))\n",
|
||||
"ppe[:, 0::2] = p_sin\n",
|
||||
"ppe[:, 1::2] = p_cos\n",
|
||||
"print(np.allclose(p_sin.numpy(), torhc_sin.cpu().detach().numpy()))\n",
|
||||
"print(np.allclose(p_cos.numpy(), torhc_cos.cpu().detach().numpy()))\n",
|
||||
"print(p_cos.numpy())\n",
|
||||
"print(\"----\")\n",
|
||||
"print(torhc_cos.cpu().detach().numpy())\n",
|
||||
"print(\")))))))\")\n",
|
||||
"print(p_sin.numpy())\n",
|
||||
"print(\"----\")\n",
|
||||
"print(torhc_sin.cpu().detach().numpy())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "integrated-boards",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"False\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(np.allclose(ppe.numpy(), pe.numpy()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "flying-reserve",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "revised-divide",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,58 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Evaluation for DeepSpeech2 model."""
|
||||
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
|
||||
from deepspeech.exps.deepspeech2.model import DeepSpeech2ExportTester as ExportTester
|
||||
from deepspeech.training.cli import default_argument_parser
|
||||
from deepspeech.utils.utility import print_arguments
|
||||
|
||||
|
||||
def main_sp(config, args):
|
||||
exp = ExportTester(config, args)
|
||||
exp.setup()
|
||||
exp.run_test()
|
||||
|
||||
|
||||
def main(config, args):
|
||||
main_sp(config, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = default_argument_parser()
|
||||
# save asr result to
|
||||
parser.add_argument(
|
||||
"--result_file", type=str, help="path of save the asr result")
|
||||
#load jit model from
|
||||
parser.add_argument(
|
||||
"--export_path", type=str, help="path of the jit model to save")
|
||||
parser.add_argument("--model_type")
|
||||
args = parser.parse_args()
|
||||
print_arguments(args, globals())
|
||||
if args.model_type is None:
|
||||
args.model_type = 'offline'
|
||||
print("model_type:{}".format(args.model_type))
|
||||
|
||||
# https://yaml.org/type/float.html
|
||||
config = get_cfg_defaults(args.model_type)
|
||||
if args.config:
|
||||
config.merge_from_file(args.config)
|
||||
if args.opts:
|
||||
config.merge_from_list(args.opts)
|
||||
config.freeze()
|
||||
print(config)
|
||||
if args.dump_config:
|
||||
with open(args.dump_config, 'w') as f:
|
||||
print(config, file=f)
|
||||
|
||||
main(config, args)
|
||||
@ -1,191 +0,0 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Beam search parameters tuning for DeepSpeech2 model."""
|
||||
import functools
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
from paddle.io import DataLoader
|
||||
|
||||
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
|
||||
from deepspeech.io.collator import SpeechCollator
|
||||
from deepspeech.io.dataset import ManifestDataset
|
||||
from deepspeech.models.deepspeech2 import DeepSpeech2Model
|
||||
from deepspeech.training.cli import default_argument_parser
|
||||
from deepspeech.utils import error_rate
|
||||
from deepspeech.utils.utility import add_arguments
|
||||
from deepspeech.utils.utility import print_arguments
|
||||
|
||||
|
||||
def tune(config, args):
|
||||
"""Tune parameters alpha and beta incrementally."""
|
||||
if not args.num_alphas >= 0:
|
||||
raise ValueError("num_alphas must be non-negative!")
|
||||
if not args.num_betas >= 0:
|
||||
raise ValueError("num_betas must be non-negative!")
|
||||
config.defrost()
|
||||
config.data.manfiest = config.data.dev_manifest
|
||||
config.data.augmentation_config = ""
|
||||
config.data.keep_transcription_text = True
|
||||
dev_dataset = ManifestDataset.from_config(config)
|
||||
|
||||
valid_loader = DataLoader(
|
||||
dev_dataset,
|
||||
batch_size=config.data.batch_size,
|
||||
shuffle=False,
|
||||
drop_last=False,
|
||||
collate_fn=SpeechCollator(keep_transcription_text=True))
|
||||
|
||||
model = DeepSpeech2Model.from_pretrained(dev_dataset, config,
|
||||
args.checkpoint_path)
|
||||
model.eval()
|
||||
|
||||
# decoders only accept string encoded in utf-8
|
||||
vocab_list = valid_loader.dataset.vocab_list
|
||||
errors_func = error_rate.char_errors if config.decoding.error_rate_type == 'cer' else error_rate.word_errors
|
||||
|
||||
# create grid for search
|
||||
cand_alphas = np.linspace(args.alpha_from, args.alpha_to, args.num_alphas)
|
||||
cand_betas = np.linspace(args.beta_from, args.beta_to, args.num_betas)
|
||||
params_grid = [(alpha, beta) for alpha in cand_alphas
|
||||
for beta in cand_betas]
|
||||
|
||||
err_sum = [0.0 for i in range(len(params_grid))]
|
||||
err_ave = [0.0 for i in range(len(params_grid))]
|
||||
|
||||
num_ins, len_refs, cur_batch = 0, 0, 0
|
||||
# initialize external scorer
|
||||
model.decoder.init_decode(args.alpha_from, args.beta_from,
|
||||
config.decoding.lang_model_path, vocab_list,
|
||||
config.decoding.decoding_method)
|
||||
## incremental tuning parameters over multiple batches
|
||||
print("start tuning ...")
|
||||
for infer_data in valid_loader():
|
||||
if (args.num_batches >= 0) and (cur_batch >= args.num_batches):
|
||||
break
|
||||
|
||||
def ordid2token(texts, texts_len):
|
||||
""" ord() id to chr() chr """
|
||||
trans = []
|
||||
for text, n in zip(texts, texts_len):
|
||||
n = n.numpy().item()
|
||||
ids = text[:n]
|
||||
trans.append(''.join([chr(i) for i in ids]))
|
||||
return trans
|
||||
|
||||
audio, audio_len, text, text_len = infer_data
|
||||
target_transcripts = ordid2token(text, text_len)
|
||||
num_ins += audio.shape[0]
|
||||
|
||||
# model infer
|
||||
eouts, eouts_len = model.encoder(audio, audio_len)
|
||||
probs = model.decoder.softmax(eouts)
|
||||
|
||||
# grid search
|
||||
for index, (alpha, beta) in enumerate(params_grid):
|
||||
print(f"tuneing: alpha={alpha} beta={beta}")
|
||||
result_transcripts = model.decoder.decode_probs(
|
||||
probs.numpy(), eouts_len, vocab_list,
|
||||
config.decoding.decoding_method,
|
||||
config.decoding.lang_model_path, alpha, beta,
|
||||
config.decoding.beam_size, config.decoding.cutoff_prob,
|
||||
config.decoding.cutoff_top_n, config.decoding.num_proc_bsearch)
|
||||
|
||||
for target, result in zip(target_transcripts, result_transcripts):
|
||||
errors, len_ref = errors_func(target, result)
|
||||
err_sum[index] += errors
|
||||
|
||||
# accumulate the length of references of every batchπ
|
||||
# in the first iteration
|
||||
if args.alpha_from == alpha and args.beta_from == beta:
|
||||
len_refs += len_ref
|
||||
|
||||
err_ave[index] = err_sum[index] / len_refs
|
||||
if index % 2 == 0:
|
||||
sys.stdout.write('.')
|
||||
sys.stdout.flush()
|
||||
print("tuneing: one grid done!")
|
||||
|
||||
# output on-line tuning result at the end of current batch
|
||||
err_ave_min = min(err_ave)
|
||||
min_index = err_ave.index(err_ave_min)
|
||||
print("\nBatch %d [%d/?], current opt (alpha, beta) = (%s, %s), "
|
||||
" min [%s] = %f" %
|
||||
(cur_batch, num_ins, "%.3f" % params_grid[min_index][0],
|
||||
"%.3f" % params_grid[min_index][1],
|
||||
config.decoding.error_rate_type, err_ave_min))
|
||||
cur_batch += 1
|
||||
|
||||
# output WER/CER at every (alpha, beta)
|
||||
print("\nFinal %s:\n" % config.decoding.error_rate_type)
|
||||
for index in range(len(params_grid)):
|
||||
print("(alpha, beta) = (%s, %s), [%s] = %f" %
|
||||
("%.3f" % params_grid[index][0], "%.3f" % params_grid[index][1],
|
||||
config.decoding.error_rate_type, err_ave[index]))
|
||||
|
||||
err_ave_min = min(err_ave)
|
||||
min_index = err_ave.index(err_ave_min)
|
||||
print("\nFinish tuning on %d batches, final opt (alpha, beta) = (%s, %s)" %
|
||||
(cur_batch, "%.3f" % params_grid[min_index][0],
|
||||
"%.3f" % params_grid[min_index][1]))
|
||||
|
||||
print("finish tuning")
|
||||
|
||||
|
||||
def main(config, args):
|
||||
tune(config, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = default_argument_parser()
|
||||
add_arg = functools.partial(add_arguments, argparser=parser)
|
||||
add_arg('num_batches', int, -1, "# of batches tuning on. "
|
||||
"Default -1, on whole dev set.")
|
||||
add_arg('num_alphas', int, 45, "# of alpha candidates for tuning.")
|
||||
add_arg('num_betas', int, 8, "# of beta candidates for tuning.")
|
||||
add_arg('alpha_from', float, 1.0, "Where alpha starts tuning from.")
|
||||
add_arg('alpha_to', float, 3.2, "Where alpha ends tuning with.")
|
||||
add_arg('beta_from', float, 0.1, "Where beta starts tuning from.")
|
||||
add_arg('beta_to', float, 0.45, "Where beta ends tuning with.")
|
||||
|
||||
add_arg('batch_size', int, 256, "# of samples per batch.")
|
||||
add_arg('beam_size', int, 500, "Beam search width.")
|
||||
add_arg('num_proc_bsearch', int, 8, "# of CPUs for beam search.")
|
||||
add_arg('cutoff_prob', float, 1.0, "Cutoff probability for pruning.")
|
||||
add_arg('cutoff_top_n', int, 40, "Cutoff number for pruning.")
|
||||
|
||||
args = parser.parse_args()
|
||||
print_arguments(args, globals())
|
||||
|
||||
# https://yaml.org/type/float.html
|
||||
config = get_cfg_defaults()
|
||||
if args.config:
|
||||
config.merge_from_file(args.config)
|
||||
if args.opts:
|
||||
config.merge_from_list(args.opts)
|
||||
|
||||
config.data.batch_size = args.batch_size
|
||||
config.decoding.beam_size = args.beam_size
|
||||
config.decoding.num_proc_bsearch = args.num_proc_bsearch
|
||||
config.decoding.cutoff_prob = args.cutoff_prob
|
||||
config.decoding.cutoff_top_n = args.cutoff_top_n
|
||||
|
||||
config.freeze()
|
||||
print(config)
|
||||
|
||||
if args.dump_config:
|
||||
with open(args.dump_config, 'w') as f:
|
||||
print(config, file=f)
|
||||
|
||||
main(config, args)
|
||||
@ -0,0 +1,220 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains U2 model."""
|
||||
import paddle
|
||||
from paddle import distributed as dist
|
||||
from paddle.io import DataLoader
|
||||
|
||||
from deepspeech.io.collator import SpeechCollator
|
||||
from deepspeech.io.dataset import ManifestDataset
|
||||
from deepspeech.io.sampler import SortagradBatchSampler
|
||||
from deepspeech.io.sampler import SortagradDistributedBatchSampler
|
||||
from deepspeech.models.u2 import U2Evaluator
|
||||
from deepspeech.models.u2 import U2Model
|
||||
from deepspeech.models.u2 import U2Updater
|
||||
from deepspeech.training.extensions.snapshot import Snapshot
|
||||
from deepspeech.training.extensions.visualizer import VisualDL
|
||||
from deepspeech.training.optimizer import OptimizerFactory
|
||||
from deepspeech.training.scheduler import LRSchedulerFactory
|
||||
from deepspeech.training.timer import Timer
|
||||
from deepspeech.training.trainer import Trainer
|
||||
from deepspeech.training.updaters.trainer import Trainer as NewTrainer
|
||||
from deepspeech.utils import layer_tools
|
||||
from deepspeech.utils.log import Log
|
||||
from deepspeech.utils.utility import UpdateConfig
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
|
||||
class U2Trainer(Trainer):
|
||||
def __init__(self, config, args):
|
||||
super().__init__(config, args)
|
||||
|
||||
def setup_dataloader(self):
|
||||
config = self.config.clone()
|
||||
config.defrost()
|
||||
config.collator.keep_transcription_text = False
|
||||
|
||||
# train/valid dataset, return token ids
|
||||
config.data.manifest = config.data.train_manifest
|
||||
train_dataset = ManifestDataset.from_config(config)
|
||||
|
||||
config.data.manifest = config.data.dev_manifest
|
||||
dev_dataset = ManifestDataset.from_config(config)
|
||||
|
||||
collate_fn_train = SpeechCollator.from_config(config)
|
||||
|
||||
config.collator.augmentation_config = ""
|
||||
collate_fn_dev = SpeechCollator.from_config(config)
|
||||
|
||||
if self.parallel:
|
||||
batch_sampler = SortagradDistributedBatchSampler(
|
||||
train_dataset,
|
||||
batch_size=config.collator.batch_size,
|
||||
num_replicas=None,
|
||||
rank=None,
|
||||
shuffle=True,
|
||||
drop_last=True,
|
||||
sortagrad=config.collator.sortagrad,
|
||||
shuffle_method=config.collator.shuffle_method)
|
||||
else:
|
||||
batch_sampler = SortagradBatchSampler(
|
||||
train_dataset,
|
||||
shuffle=True,
|
||||
batch_size=config.collator.batch_size,
|
||||
drop_last=True,
|
||||
sortagrad=config.collator.sortagrad,
|
||||
shuffle_method=config.collator.shuffle_method)
|
||||
self.train_loader = DataLoader(
|
||||
train_dataset,
|
||||
batch_sampler=batch_sampler,
|
||||
collate_fn=collate_fn_train,
|
||||
num_workers=config.collator.num_workers, )
|
||||
self.valid_loader = DataLoader(
|
||||
dev_dataset,
|
||||
batch_size=config.collator.batch_size,
|
||||
shuffle=False,
|
||||
drop_last=False,
|
||||
collate_fn=collate_fn_dev)
|
||||
|
||||
# test dataset, return raw text
|
||||
config.data.manifest = config.data.test_manifest
|
||||
# filter test examples, will cause less examples, but no mismatch with training
|
||||
# and can use large batch size , save training time, so filter test egs now.
|
||||
config.data.min_input_len = 0.0 # second
|
||||
config.data.max_input_len = float('inf') # second
|
||||
config.data.min_output_len = 0.0 # tokens
|
||||
config.data.max_output_len = float('inf') # tokens
|
||||
config.data.min_output_input_ratio = 0.00
|
||||
config.data.max_output_input_ratio = float('inf')
|
||||
|
||||
test_dataset = ManifestDataset.from_config(config)
|
||||
# return text ord id
|
||||
config.collator.keep_transcription_text = True
|
||||
config.collator.augmentation_config = ""
|
||||
self.test_loader = DataLoader(
|
||||
test_dataset,
|
||||
batch_size=config.decoding.batch_size,
|
||||
shuffle=False,
|
||||
drop_last=False,
|
||||
collate_fn=SpeechCollator.from_config(config))
|
||||
# return text token id
|
||||
config.collator.keep_transcription_text = False
|
||||
self.align_loader = DataLoader(
|
||||
test_dataset,
|
||||
batch_size=config.decoding.batch_size,
|
||||
shuffle=False,
|
||||
drop_last=False,
|
||||
collate_fn=SpeechCollator.from_config(config))
|
||||
logger.info("Setup train/valid/test/align Dataloader!")
|
||||
|
||||
def setup_model(self):
|
||||
config = self.config
|
||||
model_conf = config.model
|
||||
with UpdateConfig(model_conf):
|
||||
model_conf.input_dim = self.train_loader.collate_fn.feature_size
|
||||
model_conf.output_dim = self.train_loader.collate_fn.vocab_size
|
||||
|
||||
model = U2Model.from_config(model_conf)
|
||||
|
||||
if self.parallel:
|
||||
model = paddle.DataParallel(model)
|
||||
|
||||
model.train()
|
||||
logger.info(f"{model}")
|
||||
layer_tools.print_params(model, logger.info)
|
||||
|
||||
train_config = config.training
|
||||
optim_type = train_config.optim
|
||||
optim_conf = train_config.optim_conf
|
||||
scheduler_type = train_config.scheduler
|
||||
scheduler_conf = train_config.scheduler_conf
|
||||
|
||||
scheduler_args = {
|
||||
"learning_rate": optim_conf.lr,
|
||||
"verbose": False,
|
||||
"warmup_steps": scheduler_conf.warmup_steps,
|
||||
"gamma": scheduler_conf.lr_decay,
|
||||
"d_model": model_conf.encoder_conf.output_size,
|
||||
}
|
||||
lr_scheduler = LRSchedulerFactory.from_args(scheduler_type,
|
||||
scheduler_args)
|
||||
|
||||
def optimizer_args(
|
||||
config,
|
||||
parameters,
|
||||
lr_scheduler=None, ):
|
||||
train_config = config.training
|
||||
optim_type = train_config.optim
|
||||
optim_conf = train_config.optim_conf
|
||||
scheduler_type = train_config.scheduler
|
||||
scheduler_conf = train_config.scheduler_conf
|
||||
return {
|
||||
"grad_clip": train_config.global_grad_clip,
|
||||
"weight_decay": optim_conf.weight_decay,
|
||||
"learning_rate": lr_scheduler
|
||||
if lr_scheduler else optim_conf.lr,
|
||||
"parameters": parameters,
|
||||
"epsilon": 1e-9 if optim_type == 'noam' else None,
|
||||
"beta1": 0.9 if optim_type == 'noam' else None,
|
||||
"beat2": 0.98 if optim_type == 'noam' else None,
|
||||
}
|
||||
|
||||
optimzer_args = optimizer_args(config, model.parameters(), lr_scheduler)
|
||||
optimizer = OptimizerFactory.from_args(optim_type, optimzer_args)
|
||||
|
||||
self.model = model
|
||||
self.optimizer = optimizer
|
||||
self.lr_scheduler = lr_scheduler
|
||||
logger.info("Setup model/optimizer/lr_scheduler!")
|
||||
|
||||
def setup_updater(self):
|
||||
output_dir = self.output_dir
|
||||
config = self.config.training
|
||||
|
||||
updater = U2Updater(
|
||||
model=self.model,
|
||||
optimizer=self.optimizer,
|
||||
scheduler=self.lr_scheduler,
|
||||
dataloader=self.train_loader,
|
||||
output_dir=output_dir,
|
||||
accum_grad=config.accum_grad)
|
||||
|
||||
trainer = NewTrainer(updater, (config.n_epoch, 'epoch'), output_dir)
|
||||
|
||||
evaluator = U2Evaluator(self.model, self.valid_loader)
|
||||
|
||||
trainer.extend(evaluator, trigger=(1, "epoch"))
|
||||
|
||||
if dist.get_rank() == 0:
|
||||
trainer.extend(VisualDL(output_dir), trigger=(1, "iteration"))
|
||||
num_snapshots = config.checkpoint.kbest_n
|
||||
trainer.extend(
|
||||
Snapshot(
|
||||
mode='kbest',
|
||||
max_size=num_snapshots,
|
||||
indicator='VALID/LOSS',
|
||||
less_better=True),
|
||||
trigger=(1, 'epoch'))
|
||||
# print(trainer.extensions)
|
||||
# trainer.run()
|
||||
self.trainer = trainer
|
||||
|
||||
def run(self):
|
||||
"""The routine of the experiment after setup. This method is intended
|
||||
to be used by the user.
|
||||
"""
|
||||
self.setup_updater()
|
||||
with Timer("Training Done: {}"):
|
||||
self.trainer.run()
|
||||
@ -0,0 +1,13 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
@ -0,0 +1,83 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Evaluation for U2 model."""
|
||||
import cProfile
|
||||
|
||||
from yacs.config import CfgNode
|
||||
|
||||
from deepspeech.training.cli import default_argument_parser
|
||||
from deepspeech.utils.dynamic_import import dynamic_import
|
||||
from deepspeech.utils.utility import print_arguments
|
||||
|
||||
model_test_alias = {
|
||||
"u2": "deepspeech.exps.u2.model:U2Tester",
|
||||
"u2_kaldi": "deepspeech.exps.u2_kaldi.model:U2Tester",
|
||||
}
|
||||
|
||||
|
||||
def main_sp(config, args):
|
||||
class_obj = dynamic_import(args.model_name, model_test_alias)
|
||||
exp = class_obj(config, args)
|
||||
exp.setup()
|
||||
|
||||
if args.run_mode == 'test':
|
||||
exp.run_test()
|
||||
elif args.run_mode == 'export':
|
||||
exp.run_export()
|
||||
elif args.run_mode == 'align':
|
||||
exp.run_align()
|
||||
|
||||
|
||||
def main(config, args):
|
||||
main_sp(config, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = default_argument_parser()
|
||||
parser.add_argument(
|
||||
'--model-name',
|
||||
type=str,
|
||||
default='u2_kaldi',
|
||||
help='model name, e.g: deepspeech2, u2, u2_kaldi, u2_st')
|
||||
parser.add_argument(
|
||||
'--run-mode',
|
||||
type=str,
|
||||
default='test',
|
||||
help='run mode, e.g. test, align, export')
|
||||
parser.add_argument(
|
||||
'--dict-path', type=str, default=None, help='dict path.')
|
||||
# save asr result to
|
||||
parser.add_argument(
|
||||
"--result-file", type=str, help="path of save the asr result")
|
||||
# save jit model to
|
||||
parser.add_argument(
|
||||
"--export-path", type=str, help="path of the jit model to save")
|
||||
args = parser.parse_args()
|
||||
print_arguments(args, globals())
|
||||
|
||||
config = CfgNode()
|
||||
config.set_new_allowed(True)
|
||||
config.merge_from_file(args.config)
|
||||
if args.opts:
|
||||
config.merge_from_list(args.opts)
|
||||
config.freeze()
|
||||
print(config)
|
||||
if args.dump_config:
|
||||
with open(args.dump_config, 'w') as f:
|
||||
print(config, file=f)
|
||||
|
||||
# Setting for profiling
|
||||
pr = cProfile.Profile()
|
||||
pr.runcall(main, config, args)
|
||||
pr.dump_stats('test.profile')
|
||||
@ -0,0 +1,69 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Trainer for U2 model."""
|
||||
import cProfile
|
||||
import os
|
||||
|
||||
from paddle import distributed as dist
|
||||
from yacs.config import CfgNode
|
||||
|
||||
from deepspeech.training.cli import default_argument_parser
|
||||
from deepspeech.utils.dynamic_import import dynamic_import
|
||||
from deepspeech.utils.utility import print_arguments
|
||||
|
||||
model_train_alias = {
|
||||
"u2": "deepspeech.exps.u2.model:U2Trainer",
|
||||
"u2_kaldi": "deepspeech.exps.u2_kaldi.model:U2Trainer",
|
||||
}
|
||||
|
||||
|
||||
def main_sp(config, args):
|
||||
class_obj = dynamic_import(args.model_name, model_train_alias)
|
||||
exp = class_obj(config, args)
|
||||
exp.setup()
|
||||
exp.run()
|
||||
|
||||
|
||||
def main(config, args):
|
||||
if args.device == "gpu" and args.nprocs > 1:
|
||||
dist.spawn(main_sp, args=(config, args), nprocs=args.nprocs)
|
||||
else:
|
||||
main_sp(config, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = default_argument_parser()
|
||||
parser.add_argument(
|
||||
'--model-name',
|
||||
type=str,
|
||||
default='u2_kaldi',
|
||||
help='model name, e.g: deepspeech2, u2, u2_kaldi, u2_st')
|
||||
args = parser.parse_args()
|
||||
print_arguments(args, globals())
|
||||
|
||||
config = CfgNode()
|
||||
config.set_new_allowed(True)
|
||||
config.merge_from_file(args.config)
|
||||
if args.opts:
|
||||
config.merge_from_list(args.opts)
|
||||
config.freeze()
|
||||
print(config)
|
||||
if args.dump_config:
|
||||
with open(args.dump_config, 'w') as f:
|
||||
print(config, file=f)
|
||||
|
||||
# Setting for profiling
|
||||
pr = cProfile.Profile()
|
||||
pr.runcall(main, config, args)
|
||||
pr.dump_stats(os.path.join(args.output, 'train.profile'))
|
||||
@ -0,0 +1,671 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains U2 model."""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
from paddle import distributed as dist
|
||||
from yacs.config import CfgNode
|
||||
|
||||
from deepspeech.frontend.featurizer import TextFeaturizer
|
||||
from deepspeech.frontend.utility import load_dict
|
||||
from deepspeech.io.dataloader import BatchDataLoader
|
||||
from deepspeech.models.u2 import U2Model
|
||||
from deepspeech.training.optimizer import OptimizerFactory
|
||||
from deepspeech.training.scheduler import LRSchedulerFactory
|
||||
from deepspeech.training.timer import Timer
|
||||
from deepspeech.training.trainer import Trainer
|
||||
from deepspeech.utils import ctc_utils
|
||||
from deepspeech.utils import error_rate
|
||||
from deepspeech.utils import layer_tools
|
||||
from deepspeech.utils import mp_tools
|
||||
from deepspeech.utils import text_grid
|
||||
from deepspeech.utils import utility
|
||||
from deepspeech.utils.log import Log
|
||||
from deepspeech.utils.utility import UpdateConfig
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
|
||||
def get_cfg_defaults():
|
||||
"""Get a yacs CfgNode object with default values for my_project."""
|
||||
# Return a clone so that the defaults will not be altered
|
||||
# This is for the "local variable" use pattern
|
||||
_C = CfgNode()
|
||||
|
||||
_C.model = U2Model.params()
|
||||
|
||||
_C.training = U2Trainer.params()
|
||||
|
||||
_C.decoding = U2Tester.params()
|
||||
|
||||
config = _C.clone()
|
||||
config.set_new_allowed(True)
|
||||
return config
|
||||
|
||||
|
||||
class U2Trainer(Trainer):
|
||||
@classmethod
|
||||
def params(cls, config: Optional[CfgNode]=None) -> CfgNode:
|
||||
# training config
|
||||
default = CfgNode(
|
||||
dict(
|
||||
n_epoch=50, # train epochs
|
||||
log_interval=100, # steps
|
||||
accum_grad=1, # accum grad by # steps
|
||||
checkpoint=dict(
|
||||
kbest_n=50,
|
||||
latest_n=5, ), ))
|
||||
if config is not None:
|
||||
config.merge_from_other_cfg(default)
|
||||
return default
|
||||
|
||||
def __init__(self, config, args):
|
||||
super().__init__(config, args)
|
||||
|
||||
def train_batch(self, batch_index, batch_data, msg):
|
||||
train_conf = self.config.training
|
||||
start = time.time()
|
||||
|
||||
# forward
|
||||
utt, audio, audio_len, text, text_len = batch_data
|
||||
loss, attention_loss, ctc_loss = self.model(audio, audio_len, text,
|
||||
text_len)
|
||||
|
||||
# loss div by `batch_size * accum_grad`
|
||||
loss /= train_conf.accum_grad
|
||||
losses_np = {'loss': float(loss) * train_conf.accum_grad}
|
||||
if attention_loss:
|
||||
losses_np['att_loss'] = float(attention_loss)
|
||||
if ctc_loss:
|
||||
losses_np['ctc_loss'] = float(ctc_loss)
|
||||
|
||||
# loss backward
|
||||
if (batch_index + 1) % train_conf.accum_grad != 0:
|
||||
# Disable gradient synchronizations across DDP processes.
|
||||
# Within this context, gradients will be accumulated on module
|
||||
# variables, which will later be synchronized.
|
||||
context = self.model.no_sync
|
||||
else:
|
||||
# Used for single gpu training and DDP gradient synchronization
|
||||
# processes.
|
||||
context = nullcontext
|
||||
with context():
|
||||
loss.backward()
|
||||
layer_tools.print_grads(self.model, print_func=None)
|
||||
|
||||
# optimizer step
|
||||
if (batch_index + 1) % train_conf.accum_grad == 0:
|
||||
self.optimizer.step()
|
||||
self.optimizer.clear_grad()
|
||||
self.lr_scheduler.step()
|
||||
self.iteration += 1
|
||||
|
||||
iteration_time = time.time() - start
|
||||
|
||||
if (batch_index + 1) % train_conf.log_interval == 0:
|
||||
msg += "train time: {:>.3f}s, ".format(iteration_time)
|
||||
msg += "batch size: {}, ".format(self.config.collator.batch_size)
|
||||
msg += "accum: {}, ".format(train_conf.accum_grad)
|
||||
msg += ', '.join('{}: {:>.6f}'.format(k, v)
|
||||
for k, v in losses_np.items())
|
||||
logger.info(msg)
|
||||
|
||||
if dist.get_rank() == 0 and self.visualizer:
|
||||
losses_np_v = losses_np.copy()
|
||||
losses_np_v.update({"lr": self.lr_scheduler()})
|
||||
self.visualizer.add_scalars("step", losses_np_v,
|
||||
self.iteration - 1)
|
||||
|
||||
@paddle.no_grad()
|
||||
def valid(self):
|
||||
self.model.eval()
|
||||
logger.info(f"Valid Total Examples: {len(self.valid_loader.dataset)}")
|
||||
valid_losses = defaultdict(list)
|
||||
num_seen_utts = 1
|
||||
total_loss = 0.0
|
||||
|
||||
for i, batch in enumerate(self.valid_loader):
|
||||
utt, audio, audio_len, text, text_len = batch
|
||||
loss, attention_loss, ctc_loss = self.model(audio, audio_len, text,
|
||||
text_len)
|
||||
if paddle.isfinite(loss):
|
||||
num_utts = batch[1].shape[0]
|
||||
num_seen_utts += num_utts
|
||||
total_loss += float(loss) * num_utts
|
||||
valid_losses['val_loss'].append(float(loss))
|
||||
if attention_loss:
|
||||
valid_losses['val_att_loss'].append(float(attention_loss))
|
||||
if ctc_loss:
|
||||
valid_losses['val_ctc_loss'].append(float(ctc_loss))
|
||||
|
||||
if (i + 1) % self.config.training.log_interval == 0:
|
||||
valid_dump = {k: np.mean(v) for k, v in valid_losses.items()}
|
||||
valid_dump['val_history_loss'] = total_loss / num_seen_utts
|
||||
|
||||
# logging
|
||||
msg = f"Valid: Rank: {dist.get_rank()}, "
|
||||
msg += "epoch: {}, ".format(self.epoch)
|
||||
msg += "step: {}, ".format(self.iteration)
|
||||
msg += "batch: {}/{}, ".format(i + 1, len(self.valid_loader))
|
||||
msg += ', '.join('{}: {:>.6f}'.format(k, v)
|
||||
for k, v in valid_dump.items())
|
||||
logger.info(msg)
|
||||
|
||||
logger.info('Rank {} Val info val_loss {}'.format(
|
||||
dist.get_rank(), total_loss / num_seen_utts))
|
||||
return total_loss, num_seen_utts
|
||||
|
||||
def train(self):
|
||||
"""The training process control by step."""
|
||||
# !!!IMPORTANT!!!
|
||||
# Try to export the model by script, if fails, we should refine
|
||||
# the code to satisfy the script export requirements
|
||||
# script_model = paddle.jit.to_static(self.model)
|
||||
# script_model_path = str(self.checkpoint_dir / 'init')
|
||||
# paddle.jit.save(script_model, script_model_path)
|
||||
|
||||
from_scratch = self.resume_or_scratch()
|
||||
if from_scratch:
|
||||
# save init model, i.e. 0 epoch
|
||||
self.save(tag='init')
|
||||
self.lr_scheduler.step(self.iteration)
|
||||
|
||||
logger.info(f"Train Total Examples: {len(self.train_loader.dataset)}")
|
||||
while self.epoch < self.config.training.n_epoch:
|
||||
with Timer("Epoch-Train Time Cost: {}"):
|
||||
self.model.train()
|
||||
try:
|
||||
data_start_time = time.time()
|
||||
for batch_index, batch in enumerate(self.train_loader):
|
||||
dataload_time = time.time() - data_start_time
|
||||
msg = "Train: Rank: {}, ".format(dist.get_rank())
|
||||
msg += "epoch: {}, ".format(self.epoch)
|
||||
msg += "step: {}, ".format(self.iteration)
|
||||
msg += "batch : {}/{}, ".format(batch_index + 1,
|
||||
len(self.train_loader))
|
||||
msg += "lr: {:>.8f}, ".format(self.lr_scheduler())
|
||||
msg += "data time: {:>.3f}s, ".format(dataload_time)
|
||||
self.train_batch(batch_index, batch, msg)
|
||||
self.after_train_batch()
|
||||
data_start_time = time.time()
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
raise e
|
||||
|
||||
with Timer("Eval Time Cost: {}"):
|
||||
total_loss, num_seen_utts = self.valid()
|
||||
if dist.get_world_size() > 1:
|
||||
num_seen_utts = paddle.to_tensor(num_seen_utts)
|
||||
# the default operator in all_reduce function is sum.
|
||||
dist.all_reduce(num_seen_utts)
|
||||
total_loss = paddle.to_tensor(total_loss)
|
||||
dist.all_reduce(total_loss)
|
||||
cv_loss = total_loss / num_seen_utts
|
||||
cv_loss = float(cv_loss)
|
||||
else:
|
||||
cv_loss = total_loss / num_seen_utts
|
||||
|
||||
logger.info(
|
||||
'Epoch {} Val info val_loss {}'.format(self.epoch, cv_loss))
|
||||
if self.visualizer:
|
||||
self.visualizer.add_scalars(
|
||||
'epoch', {'cv_loss': cv_loss,
|
||||
'lr': self.lr_scheduler()}, self.epoch)
|
||||
self.save(tag=self.epoch, infos={'val_loss': cv_loss})
|
||||
self.new_epoch()
|
||||
|
||||
def setup_dataloader(self):
|
||||
config = self.config.clone()
|
||||
# train/valid dataset, return token ids
|
||||
self.train_loader = BatchDataLoader(
|
||||
json_file=config.data.train_manifest,
|
||||
train_mode=True,
|
||||
sortagrad=False,
|
||||
batch_size=config.collator.batch_size,
|
||||
maxlen_in=float('inf'),
|
||||
maxlen_out=float('inf'),
|
||||
minibatches=0,
|
||||
mini_batch_size=self.args.nprocs,
|
||||
batch_count='auto',
|
||||
batch_bins=0,
|
||||
batch_frames_in=0,
|
||||
batch_frames_out=0,
|
||||
batch_frames_inout=0,
|
||||
preprocess_conf=config.collator.augmentation_config,
|
||||
n_iter_processes=config.collator.num_workers,
|
||||
subsampling_factor=1,
|
||||
num_encs=1)
|
||||
|
||||
self.valid_loader = BatchDataLoader(
|
||||
json_file=config.data.dev_manifest,
|
||||
train_mode=False,
|
||||
sortagrad=False,
|
||||
batch_size=config.collator.batch_size,
|
||||
maxlen_in=float('inf'),
|
||||
maxlen_out=float('inf'),
|
||||
minibatches=0,
|
||||
mini_batch_size=self.args.nprocs,
|
||||
batch_count='auto',
|
||||
batch_bins=0,
|
||||
batch_frames_in=0,
|
||||
batch_frames_out=0,
|
||||
batch_frames_inout=0,
|
||||
preprocess_conf=None,
|
||||
n_iter_processes=1,
|
||||
subsampling_factor=1,
|
||||
num_encs=1)
|
||||
|
||||
# test dataset, return raw text
|
||||
self.test_loader = BatchDataLoader(
|
||||
json_file=config.data.test_manifest,
|
||||
train_mode=False,
|
||||
sortagrad=False,
|
||||
batch_size=config.decoding.batch_size,
|
||||
maxlen_in=float('inf'),
|
||||
maxlen_out=float('inf'),
|
||||
minibatches=0,
|
||||
mini_batch_size=1,
|
||||
batch_count='auto',
|
||||
batch_bins=0,
|
||||
batch_frames_in=0,
|
||||
batch_frames_out=0,
|
||||
batch_frames_inout=0,
|
||||
preprocess_conf=None,
|
||||
n_iter_processes=1,
|
||||
subsampling_factor=1,
|
||||
num_encs=1)
|
||||
|
||||
self.align_loader = BatchDataLoader(
|
||||
json_file=config.data.test_manifest,
|
||||
train_mode=False,
|
||||
sortagrad=False,
|
||||
batch_size=config.decoding.batch_size,
|
||||
maxlen_in=float('inf'),
|
||||
maxlen_out=float('inf'),
|
||||
minibatches=0,
|
||||
mini_batch_size=1,
|
||||
batch_count='auto',
|
||||
batch_bins=0,
|
||||
batch_frames_in=0,
|
||||
batch_frames_out=0,
|
||||
batch_frames_inout=0,
|
||||
preprocess_conf=None,
|
||||
n_iter_processes=1,
|
||||
subsampling_factor=1,
|
||||
num_encs=1)
|
||||
logger.info("Setup train/valid/test/align Dataloader!")
|
||||
|
||||
def setup_model(self):
|
||||
config = self.config
|
||||
|
||||
# model
|
||||
model_conf = config.model
|
||||
with UpdateConfig(model_conf):
|
||||
model_conf.input_dim = self.train_loader.feat_dim
|
||||
model_conf.output_dim = self.train_loader.vocab_size
|
||||
|
||||
model = U2Model.from_config(model_conf)
|
||||
if self.parallel:
|
||||
model = paddle.DataParallel(model)
|
||||
logger.info(f"{model}")
|
||||
layer_tools.print_params(model, logger.info)
|
||||
|
||||
# lr
|
||||
scheduler_conf = config.scheduler_conf
|
||||
scheduler_args = {
|
||||
"learning_rate": scheduler_conf.lr,
|
||||
"warmup_steps": scheduler_conf.warmup_steps,
|
||||
"gamma": scheduler_conf.lr_decay,
|
||||
"d_model": model_conf.encoder_conf.output_size,
|
||||
"verbose": False,
|
||||
}
|
||||
lr_scheduler = LRSchedulerFactory.from_args(config.scheduler,
|
||||
scheduler_args)
|
||||
|
||||
# opt
|
||||
def optimizer_args(
|
||||
config,
|
||||
parameters,
|
||||
lr_scheduler=None, ):
|
||||
optim_conf = config.optim_conf
|
||||
return {
|
||||
"grad_clip": optim_conf.global_grad_clip,
|
||||
"weight_decay": optim_conf.weight_decay,
|
||||
"learning_rate": lr_scheduler,
|
||||
"parameters": parameters,
|
||||
}
|
||||
|
||||
optimzer_args = optimizer_args(config, model.parameters(), lr_scheduler)
|
||||
optimizer = OptimizerFactory.from_args(config.optim, optimzer_args)
|
||||
|
||||
self.model = model
|
||||
self.lr_scheduler = lr_scheduler
|
||||
self.optimizer = optimizer
|
||||
logger.info("Setup model/optimizer/lr_scheduler!")
|
||||
|
||||
|
||||
class U2Tester(U2Trainer):
|
||||
@classmethod
|
||||
def params(cls, config: Optional[CfgNode]=None) -> CfgNode:
|
||||
# decoding config
|
||||
default = CfgNode(
|
||||
dict(
|
||||
alpha=2.5, # Coef of LM for beam search.
|
||||
beta=0.3, # Coef of WC for beam search.
|
||||
cutoff_prob=1.0, # Cutoff probability for pruning.
|
||||
cutoff_top_n=40, # Cutoff number for pruning.
|
||||
lang_model_path='models/lm/common_crawl_00.prune01111.trie.klm', # Filepath for language model.
|
||||
decoding_method='attention', # Decoding method. Options: 'attention', 'ctc_greedy_search',
|
||||
# 'ctc_prefix_beam_search', 'attention_rescoring'
|
||||
error_rate_type='wer', # Error rate type for evaluation. Options `wer`, 'cer'
|
||||
num_proc_bsearch=8, # # of CPUs for beam search.
|
||||
beam_size=10, # Beam search width.
|
||||
batch_size=16, # decoding batch size
|
||||
ctc_weight=0.0, # ctc weight for attention rescoring decode mode.
|
||||
decoding_chunk_size=-1, # decoding chunk size. Defaults to -1.
|
||||
# <0: for decoding, use full chunk.
|
||||
# >0: for decoding, use fixed chunk size as set.
|
||||
# 0: used for training, it's prohibited here.
|
||||
num_decoding_left_chunks=-1, # number of left chunks for decoding. Defaults to -1.
|
||||
simulate_streaming=False, # simulate streaming inference. Defaults to False.
|
||||
))
|
||||
|
||||
if config is not None:
|
||||
config.merge_from_other_cfg(default)
|
||||
return default
|
||||
|
||||
def __init__(self, config, args):
|
||||
super().__init__(config, args)
|
||||
|
||||
def id2token(self, texts, texts_len, text_feature):
|
||||
""" ord() id to chr() chr """
|
||||
trans = []
|
||||
for text, n in zip(texts, texts_len):
|
||||
n = n.numpy().item()
|
||||
ids = text[:n]
|
||||
trans.append(text_feature.defeaturize(ids.numpy().tolist()))
|
||||
return trans
|
||||
|
||||
def compute_metrics(self,
|
||||
utts,
|
||||
audio,
|
||||
audio_len,
|
||||
texts,
|
||||
texts_len,
|
||||
fout=None):
|
||||
cfg = self.config.decoding
|
||||
errors_sum, len_refs, num_ins = 0.0, 0, 0
|
||||
errors_func = error_rate.char_errors if cfg.error_rate_type == 'cer' else error_rate.word_errors
|
||||
error_rate_func = error_rate.cer if cfg.error_rate_type == 'cer' else error_rate.wer
|
||||
|
||||
start_time = time.time()
|
||||
text_feature = TextFeaturizer(
|
||||
unit_type=self.config.collator.unit_type,
|
||||
vocab_filepath=self.config.collator.vocab_filepath,
|
||||
spm_model_prefix=self.config.collator.spm_model_prefix)
|
||||
target_transcripts = self.id2token(texts, texts_len, text_feature)
|
||||
result_transcripts = self.model.decode(
|
||||
audio,
|
||||
audio_len,
|
||||
text_feature=text_feature,
|
||||
decoding_method=cfg.decoding_method,
|
||||
lang_model_path=cfg.lang_model_path,
|
||||
beam_alpha=cfg.alpha,
|
||||
beam_beta=cfg.beta,
|
||||
beam_size=cfg.beam_size,
|
||||
cutoff_prob=cfg.cutoff_prob,
|
||||
cutoff_top_n=cfg.cutoff_top_n,
|
||||
num_processes=cfg.num_proc_bsearch,
|
||||
ctc_weight=cfg.ctc_weight,
|
||||
decoding_chunk_size=cfg.decoding_chunk_size,
|
||||
num_decoding_left_chunks=cfg.num_decoding_left_chunks,
|
||||
simulate_streaming=cfg.simulate_streaming)
|
||||
decode_time = time.time() - start_time
|
||||
|
||||
for utt, target, result in zip(utts, target_transcripts,
|
||||
result_transcripts):
|
||||
errors, len_ref = errors_func(target, result)
|
||||
errors_sum += errors
|
||||
len_refs += len_ref
|
||||
num_ins += 1
|
||||
if fout:
|
||||
fout.write(utt + " " + result + "\n")
|
||||
logger.info("\nTarget Transcription: %s\nOutput Transcription: %s" %
|
||||
(target, result))
|
||||
logger.info("One example error rate [%s] = %f" %
|
||||
(cfg.error_rate_type, error_rate_func(target, result)))
|
||||
|
||||
return dict(
|
||||
errors_sum=errors_sum,
|
||||
len_refs=len_refs,
|
||||
num_ins=num_ins, # num examples
|
||||
error_rate=errors_sum / len_refs,
|
||||
error_rate_type=cfg.error_rate_type,
|
||||
num_frames=audio_len.sum().numpy().item(),
|
||||
decode_time=decode_time)
|
||||
|
||||
@mp_tools.rank_zero_only
|
||||
@paddle.no_grad()
|
||||
def test(self):
|
||||
assert self.args.result_file
|
||||
self.model.eval()
|
||||
logger.info(f"Test Total Examples: {len(self.test_loader.dataset)}")
|
||||
|
||||
stride_ms = self.config.collator.stride_ms
|
||||
error_rate_type = None
|
||||
errors_sum, len_refs, num_ins = 0.0, 0, 0
|
||||
num_frames = 0.0
|
||||
num_time = 0.0
|
||||
with open(self.args.result_file, 'w') as fout:
|
||||
for i, batch in enumerate(self.test_loader):
|
||||
metrics = self.compute_metrics(*batch, fout=fout)
|
||||
num_frames += metrics['num_frames']
|
||||
num_time += metrics["decode_time"]
|
||||
errors_sum += metrics['errors_sum']
|
||||
len_refs += metrics['len_refs']
|
||||
num_ins += metrics['num_ins']
|
||||
error_rate_type = metrics['error_rate_type']
|
||||
rtf = num_time / (num_frames * stride_ms)
|
||||
logger.info(
|
||||
"RTF: %f, Error rate [%s] (%d/?) = %f" %
|
||||
(rtf, error_rate_type, num_ins, errors_sum / len_refs))
|
||||
|
||||
rtf = num_time / (num_frames * stride_ms)
|
||||
msg = "Test: "
|
||||
msg += "epoch: {}, ".format(self.epoch)
|
||||
msg += "step: {}, ".format(self.iteration)
|
||||
msg += "RTF: {}, ".format(rtf)
|
||||
msg += "Final error rate [%s] (%d/%d) = %f" % (
|
||||
error_rate_type, num_ins, num_ins, errors_sum / len_refs)
|
||||
logger.info(msg)
|
||||
|
||||
# test meta results
|
||||
err_meta_path = os.path.splitext(self.args.result_file)[0] + '.err'
|
||||
err_type_str = "{}".format(error_rate_type)
|
||||
with open(err_meta_path, 'w') as f:
|
||||
data = json.dumps({
|
||||
"epoch":
|
||||
self.epoch,
|
||||
"step":
|
||||
self.iteration,
|
||||
"rtf":
|
||||
rtf,
|
||||
error_rate_type:
|
||||
errors_sum / len_refs,
|
||||
"dataset_hour": (num_frames * stride_ms) / 1000.0 / 3600.0,
|
||||
"process_hour":
|
||||
num_time / 1000.0 / 3600.0,
|
||||
"num_examples":
|
||||
num_ins,
|
||||
"err_sum":
|
||||
errors_sum,
|
||||
"ref_len":
|
||||
len_refs,
|
||||
"decode_method":
|
||||
self.config.decoding.decoding_method,
|
||||
})
|
||||
f.write(data + '\n')
|
||||
|
||||
def run_test(self):
|
||||
self.resume_or_scratch()
|
||||
try:
|
||||
self.test()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(-1)
|
||||
|
||||
@paddle.no_grad()
|
||||
def align(self):
|
||||
if self.config.decoding.batch_size > 1:
|
||||
logger.fatal('alignment mode must be running with batch_size == 1')
|
||||
sys.exit(1)
|
||||
|
||||
# xxx.align
|
||||
assert self.args.result_file and self.args.result_file.endswith(
|
||||
'.align')
|
||||
|
||||
self.model.eval()
|
||||
logger.info(f"Align Total Examples: {len(self.align_loader.dataset)}")
|
||||
|
||||
stride_ms = self.config.collater.stride_ms
|
||||
token_dict = self.args.char_list
|
||||
|
||||
with open(self.args.result_file, 'w') as fout:
|
||||
# one example in batch
|
||||
for i, batch in enumerate(self.align_loader):
|
||||
key, feat, feats_length, target, target_length = batch
|
||||
|
||||
# 1. Encoder
|
||||
encoder_out, encoder_mask = self.model._forward_encoder(
|
||||
feat, feats_length) # (B, maxlen, encoder_dim)
|
||||
maxlen = encoder_out.size(1)
|
||||
ctc_probs = self.model.ctc.log_softmax(
|
||||
encoder_out) # (1, maxlen, vocab_size)
|
||||
|
||||
# 2. alignment
|
||||
ctc_probs = ctc_probs.squeeze(0)
|
||||
target = target.squeeze(0)
|
||||
alignment = ctc_utils.forced_align(ctc_probs, target)
|
||||
logger.info("align ids", key[0], alignment)
|
||||
fout.write('{} {}\n'.format(key[0], alignment))
|
||||
|
||||
# 3. gen praat
|
||||
# segment alignment
|
||||
align_segs = text_grid.segment_alignment(alignment)
|
||||
logger.info("align tokens", key[0], align_segs)
|
||||
# IntervalTier, List["start end token\n"]
|
||||
subsample = utility.get_subsample(self.config)
|
||||
tierformat = text_grid.align_to_tierformat(
|
||||
align_segs, subsample, token_dict)
|
||||
# write tier
|
||||
align_output_path = os.path.join(
|
||||
os.path.dirname(self.args.result_file), "align")
|
||||
tier_path = os.path.join(align_output_path, key[0] + ".tier")
|
||||
with open(tier_path, 'w') as f:
|
||||
f.writelines(tierformat)
|
||||
# write textgrid
|
||||
textgrid_path = os.path.join(align_output_path,
|
||||
key[0] + ".TextGrid")
|
||||
second_per_frame = 1. / (1000. /
|
||||
stride_ms) # 25ms window, 10ms stride
|
||||
second_per_example = (
|
||||
len(alignment) + 1) * subsample * second_per_frame
|
||||
text_grid.generate_textgrid(
|
||||
maxtime=second_per_example,
|
||||
intervals=tierformat,
|
||||
output=textgrid_path)
|
||||
|
||||
def run_align(self):
|
||||
self.resume_or_scratch()
|
||||
try:
|
||||
self.align()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(-1)
|
||||
|
||||
def load_inferspec(self):
|
||||
"""infer model and input spec.
|
||||
|
||||
Returns:
|
||||
nn.Layer: inference model
|
||||
List[paddle.static.InputSpec]: input spec.
|
||||
"""
|
||||
from deepspeech.models.u2 import U2InferModel
|
||||
infer_model = U2InferModel.from_pretrained(self.test_loader,
|
||||
self.config.model.clone(),
|
||||
self.args.checkpoint_path)
|
||||
feat_dim = self.test_loader.feat_dim
|
||||
input_spec = [
|
||||
paddle.static.InputSpec(shape=[1, None, feat_dim],
|
||||
dtype='float32'), # audio, [B,T,D]
|
||||
paddle.static.InputSpec(shape=[1],
|
||||
dtype='int64'), # audio_length, [B]
|
||||
]
|
||||
return infer_model, input_spec
|
||||
|
||||
def export(self):
|
||||
infer_model, input_spec = self.load_inferspec()
|
||||
assert isinstance(input_spec, list), type(input_spec)
|
||||
infer_model.eval()
|
||||
static_model = paddle.jit.to_static(infer_model, input_spec=input_spec)
|
||||
logger.info(f"Export code: {static_model.forward.code}")
|
||||
paddle.jit.save(static_model, self.args.export_path)
|
||||
|
||||
def run_export(self):
|
||||
try:
|
||||
self.export()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(-1)
|
||||
|
||||
def setup_dict(self):
|
||||
# load dictionary for debug log
|
||||
self.args.char_list = load_dict(self.args.dict_path,
|
||||
"maskctc" in self.args.model_name)
|
||||
|
||||
def setup(self):
|
||||
"""Setup the experiment.
|
||||
"""
|
||||
paddle.set_device(self.args.device)
|
||||
|
||||
self.setup_output_dir()
|
||||
self.setup_checkpointer()
|
||||
|
||||
self.setup_dataloader()
|
||||
self.setup_model()
|
||||
|
||||
self.setup_dict()
|
||||
|
||||
self.iteration = 0
|
||||
self.epoch = 0
|
||||
|
||||
def setup_output_dir(self):
|
||||
"""Create a directory used for output.
|
||||
"""
|
||||
# output dir
|
||||
if self.args.output:
|
||||
output_dir = Path(self.args.output).expanduser()
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
output_dir = Path(
|
||||
self.args.checkpoint_path).expanduser().parent.parent
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.output_dir = output_dir
|
||||
@ -0,0 +1,13 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
@ -0,0 +1,51 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Export for U2 model."""
|
||||
from deepspeech.exps.u2_st.config import get_cfg_defaults
|
||||
from deepspeech.exps.u2_st.model import U2STTester as Tester
|
||||
from deepspeech.training.cli import default_argument_parser
|
||||
from deepspeech.utils.utility import print_arguments
|
||||
|
||||
|
||||
def main_sp(config, args):
|
||||
exp = Tester(config, args)
|
||||
exp.setup()
|
||||
exp.run_export()
|
||||
|
||||
|
||||
def main(config, args):
|
||||
main_sp(config, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = default_argument_parser()
|
||||
# save jit model to
|
||||
parser.add_argument(
|
||||
"--export_path", type=str, help="path of the jit model to save")
|
||||
args = parser.parse_args()
|
||||
print_arguments(args, globals())
|
||||
|
||||
# https://yaml.org/type/float.html
|
||||
config = get_cfg_defaults()
|
||||
if args.config:
|
||||
config.merge_from_file(args.config)
|
||||
if args.opts:
|
||||
config.merge_from_list(args.opts)
|
||||
config.freeze()
|
||||
print(config)
|
||||
if args.dump_config:
|
||||
with open(args.dump_config, 'w') as f:
|
||||
print(config, file=f)
|
||||
|
||||
main(config, args)
|
||||
@ -0,0 +1,58 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Evaluation for U2 model."""
|
||||
import cProfile
|
||||
|
||||
from deepspeech.exps.u2_st.config import get_cfg_defaults
|
||||
from deepspeech.exps.u2_st.model import U2STTester as Tester
|
||||
from deepspeech.training.cli import default_argument_parser
|
||||
from deepspeech.utils.utility import print_arguments
|
||||
|
||||
# TODO(hui zhang): dynamic load
|
||||
|
||||
|
||||
def main_sp(config, args):
|
||||
exp = Tester(config, args)
|
||||
exp.setup()
|
||||
exp.run_test()
|
||||
|
||||
|
||||
def main(config, args):
|
||||
main_sp(config, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = default_argument_parser()
|
||||
# save asr result to
|
||||
parser.add_argument(
|
||||
"--result_file", type=str, help="path of save the asr result")
|
||||
args = parser.parse_args()
|
||||
print_arguments(args, globals())
|
||||
|
||||
# https://yaml.org/type/float.html
|
||||
config = get_cfg_defaults()
|
||||
if args.config:
|
||||
config.merge_from_file(args.config)
|
||||
if args.opts:
|
||||
config.merge_from_list(args.opts)
|
||||
config.freeze()
|
||||
print(config)
|
||||
if args.dump_config:
|
||||
with open(args.dump_config, 'w') as f:
|
||||
print(config, file=f)
|
||||
|
||||
# Setting for profiling
|
||||
pr = cProfile.Profile()
|
||||
pr.runcall(main, config, args)
|
||||
pr.dump_stats('test.profile')
|
||||
@ -0,0 +1,59 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Trainer for U2 model."""
|
||||
import cProfile
|
||||
import os
|
||||
|
||||
from paddle import distributed as dist
|
||||
|
||||
from deepspeech.exps.u2_st.config import get_cfg_defaults
|
||||
from deepspeech.exps.u2_st.model import U2STTrainer as Trainer
|
||||
from deepspeech.training.cli import default_argument_parser
|
||||
from deepspeech.utils.utility import print_arguments
|
||||
|
||||
|
||||
def main_sp(config, args):
|
||||
exp = Trainer(config, args)
|
||||
exp.setup()
|
||||
exp.run()
|
||||
|
||||
|
||||
def main(config, args):
|
||||
if args.device == "gpu" and args.nprocs > 1:
|
||||
dist.spawn(main_sp, args=(config, args), nprocs=args.nprocs)
|
||||
else:
|
||||
main_sp(config, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = default_argument_parser()
|
||||
args = parser.parse_args()
|
||||
print_arguments(args, globals())
|
||||
|
||||
# https://yaml.org/type/float.html
|
||||
config = get_cfg_defaults()
|
||||
if args.config:
|
||||
config.merge_from_file(args.config)
|
||||
if args.opts:
|
||||
config.merge_from_list(args.opts)
|
||||
config.freeze()
|
||||
print(config)
|
||||
if args.dump_config:
|
||||
with open(args.dump_config, 'w') as f:
|
||||
print(config, file=f)
|
||||
|
||||
# Setting for profiling
|
||||
pr = cProfile.Profile()
|
||||
pr.runcall(main, config, args)
|
||||
pr.dump_stats(os.path.join(args.output, 'train.profile'))
|
||||
@ -0,0 +1,41 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from yacs.config import CfgNode
|
||||
|
||||
from deepspeech.exps.u2_st.model import U2STTester
|
||||
from deepspeech.exps.u2_st.model import U2STTrainer
|
||||
from deepspeech.io.collator_st import SpeechCollator
|
||||
from deepspeech.io.dataset import ManifestDataset
|
||||
from deepspeech.models.u2_st import U2STModel
|
||||
|
||||
_C = CfgNode()
|
||||
|
||||
_C.data = ManifestDataset.params()
|
||||
|
||||
_C.collator = SpeechCollator.params()
|
||||
|
||||
_C.model = U2STModel.params()
|
||||
|
||||
_C.training = U2STTrainer.params()
|
||||
|
||||
_C.decoding = U2STTester.params()
|
||||
|
||||
|
||||
def get_cfg_defaults():
|
||||
"""Get a yacs CfgNode object with default values for my_project."""
|
||||
# Return a clone so that the defaults will not be altered
|
||||
# This is for the "local variable" use pattern
|
||||
config = _C.clone()
|
||||
config.set_new_allowed(True)
|
||||
return config
|
||||
@ -0,0 +1,695 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains U2 model."""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
from paddle import distributed as dist
|
||||
from paddle.io import DataLoader
|
||||
from yacs.config import CfgNode
|
||||
|
||||
from deepspeech.io.collator_st import KaldiPrePorocessedCollator
|
||||
from deepspeech.io.collator_st import SpeechCollator
|
||||
from deepspeech.io.collator_st import TripletKaldiPrePorocessedCollator
|
||||
from deepspeech.io.collator_st import TripletSpeechCollator
|
||||
from deepspeech.io.dataset import ManifestDataset
|
||||
from deepspeech.io.dataset import TripletManifestDataset
|
||||
from deepspeech.io.sampler import SortagradBatchSampler
|
||||
from deepspeech.io.sampler import SortagradDistributedBatchSampler
|
||||
from deepspeech.models.u2_st import U2STModel
|
||||
from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog
|
||||
from deepspeech.training.scheduler import WarmupLR
|
||||
from deepspeech.training.timer import Timer
|
||||
from deepspeech.training.trainer import Trainer
|
||||
from deepspeech.utils import bleu_score
|
||||
from deepspeech.utils import ctc_utils
|
||||
from deepspeech.utils import layer_tools
|
||||
from deepspeech.utils import mp_tools
|
||||
from deepspeech.utils import text_grid
|
||||
from deepspeech.utils import utility
|
||||
from deepspeech.utils.log import Log
|
||||
from deepspeech.utils.utility import UpdateConfig
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
|
||||
class U2STTrainer(Trainer):
|
||||
@classmethod
|
||||
def params(cls, config: Optional[CfgNode]=None) -> CfgNode:
|
||||
# training config
|
||||
default = CfgNode(
|
||||
dict(
|
||||
n_epoch=50, # train epochs
|
||||
log_interval=100, # steps
|
||||
accum_grad=1, # accum grad by # steps
|
||||
global_grad_clip=5.0, # the global norm clip
|
||||
))
|
||||
default.optim = 'adam'
|
||||
default.optim_conf = CfgNode(
|
||||
dict(
|
||||
lr=5e-4, # learning rate
|
||||
weight_decay=1e-6, # the coeff of weight decay
|
||||
))
|
||||
default.scheduler = 'warmuplr'
|
||||
default.scheduler_conf = CfgNode(
|
||||
dict(
|
||||
warmup_steps=25000,
|
||||
lr_decay=1.0, # learning rate decay
|
||||
))
|
||||
|
||||
if config is not None:
|
||||
config.merge_from_other_cfg(default)
|
||||
return default
|
||||
|
||||
def __init__(self, config, args):
|
||||
super().__init__(config, args)
|
||||
|
||||
def train_batch(self, batch_index, batch_data, msg):
|
||||
train_conf = self.config.training
|
||||
start = time.time()
|
||||
# forward
|
||||
utt, audio, audio_len, text, text_len = batch_data
|
||||
if isinstance(text, list) and isinstance(text_len, list):
|
||||
# joint training with ASR. Two decoding texts [translation, transcription]
|
||||
text, text_transcript = text
|
||||
text_len, text_transcript_len = text_len
|
||||
loss, st_loss, attention_loss, ctc_loss = self.model(
|
||||
audio, audio_len, text, text_len, text_transcript,
|
||||
text_transcript_len)
|
||||
else:
|
||||
loss, st_loss, attention_loss, ctc_loss = self.model(
|
||||
audio, audio_len, text, text_len)
|
||||
|
||||
# loss div by `batch_size * accum_grad`
|
||||
loss /= train_conf.accum_grad
|
||||
losses_np = {'loss': float(loss) * train_conf.accum_grad}
|
||||
if attention_loss:
|
||||
losses_np['att_loss'] = float(attention_loss)
|
||||
if ctc_loss:
|
||||
losses_np['ctc_loss'] = float(ctc_loss)
|
||||
|
||||
# loss backward
|
||||
if (batch_index + 1) % train_conf.accum_grad != 0:
|
||||
# Disable gradient synchronizations across DDP processes.
|
||||
# Within this context, gradients will be accumulated on module
|
||||
# variables, which will later be synchronized.
|
||||
context = self.model.no_sync
|
||||
else:
|
||||
# Used for single gpu training and DDP gradient synchronization
|
||||
# processes.
|
||||
context = nullcontext
|
||||
with context():
|
||||
loss.backward()
|
||||
layer_tools.print_grads(self.model, print_func=None)
|
||||
|
||||
# optimizer step
|
||||
if (batch_index + 1) % train_conf.accum_grad == 0:
|
||||
self.optimizer.step()
|
||||
self.optimizer.clear_grad()
|
||||
self.lr_scheduler.step()
|
||||
self.iteration += 1
|
||||
|
||||
iteration_time = time.time() - start
|
||||
|
||||
if (batch_index + 1) % train_conf.log_interval == 0:
|
||||
msg += "train time: {:>.3f}s, ".format(iteration_time)
|
||||
msg += "batch size: {}, ".format(self.config.collator.batch_size)
|
||||
msg += "accum: {}, ".format(train_conf.accum_grad)
|
||||
msg += ', '.join('{}: {:>.6f}'.format(k, v)
|
||||
for k, v in losses_np.items())
|
||||
logger.info(msg)
|
||||
|
||||
if dist.get_rank() == 0 and self.visualizer:
|
||||
losses_np_v = losses_np.copy()
|
||||
losses_np_v.update({"lr": self.lr_scheduler()})
|
||||
self.visualizer.add_scalars("step", losses_np_v,
|
||||
self.iteration - 1)
|
||||
|
||||
@paddle.no_grad()
|
||||
def valid(self):
|
||||
self.model.eval()
|
||||
logger.info(f"Valid Total Examples: {len(self.valid_loader.dataset)}")
|
||||
valid_losses = defaultdict(list)
|
||||
num_seen_utts = 1
|
||||
total_loss = 0.0
|
||||
for i, batch in enumerate(self.valid_loader):
|
||||
utt, audio, audio_len, text, text_len = batch
|
||||
if isinstance(text, list) and isinstance(text_len, list):
|
||||
text, text_transcript = text
|
||||
text_len, text_transcript_len = text_len
|
||||
loss, st_loss, attention_loss, ctc_loss = self.model(
|
||||
audio, audio_len, text, text_len, text_transcript,
|
||||
text_transcript_len)
|
||||
else:
|
||||
loss, st_loss, attention_loss, ctc_loss = self.model(
|
||||
audio, audio_len, text, text_len)
|
||||
if paddle.isfinite(loss):
|
||||
num_utts = batch[1].shape[0]
|
||||
num_seen_utts += num_utts
|
||||
total_loss += float(st_loss) * num_utts
|
||||
valid_losses['val_loss'].append(float(st_loss))
|
||||
if attention_loss:
|
||||
valid_losses['val_att_loss'].append(float(attention_loss))
|
||||
if ctc_loss:
|
||||
valid_losses['val_ctc_loss'].append(float(ctc_loss))
|
||||
|
||||
if (i + 1) % self.config.training.log_interval == 0:
|
||||
valid_dump = {k: np.mean(v) for k, v in valid_losses.items()}
|
||||
valid_dump['val_history_st_loss'] = total_loss / num_seen_utts
|
||||
|
||||
# logging
|
||||
msg = f"Valid: Rank: {dist.get_rank()}, "
|
||||
msg += "epoch: {}, ".format(self.epoch)
|
||||
msg += "step: {}, ".format(self.iteration)
|
||||
msg += "batch: {}/{}, ".format(i + 1, len(self.valid_loader))
|
||||
msg += ', '.join('{}: {:>.6f}'.format(k, v)
|
||||
for k, v in valid_dump.items())
|
||||
logger.info(msg)
|
||||
|
||||
logger.info('Rank {} Val info st_val_loss {}'.format(
|
||||
dist.get_rank(), total_loss / num_seen_utts))
|
||||
return total_loss, num_seen_utts
|
||||
|
||||
def train(self):
|
||||
"""The training process control by step."""
|
||||
# !!!IMPORTANT!!!
|
||||
# Try to export the model by script, if fails, we should refine
|
||||
# the code to satisfy the script export requirements
|
||||
# script_model = paddle.jit.to_static(self.model)
|
||||
# script_model_path = str(self.checkpoint_dir / 'init')
|
||||
# paddle.jit.save(script_model, script_model_path)
|
||||
|
||||
from_scratch = self.resume_or_scratch()
|
||||
if from_scratch:
|
||||
# save init model, i.e. 0 epoch
|
||||
self.save(tag='init')
|
||||
|
||||
self.lr_scheduler.step(self.iteration)
|
||||
if self.parallel:
|
||||
self.train_loader.batch_sampler.set_epoch(self.epoch)
|
||||
|
||||
logger.info(f"Train Total Examples: {len(self.train_loader.dataset)}")
|
||||
while self.epoch < self.config.training.n_epoch:
|
||||
with Timer("Epoch-Train Time Cost: {}"):
|
||||
self.model.train()
|
||||
try:
|
||||
data_start_time = time.time()
|
||||
for batch_index, batch in enumerate(self.train_loader):
|
||||
dataload_time = time.time() - data_start_time
|
||||
msg = "Train: Rank: {}, ".format(dist.get_rank())
|
||||
msg += "epoch: {}, ".format(self.epoch)
|
||||
msg += "step: {}, ".format(self.iteration)
|
||||
msg += "batch : {}/{}, ".format(batch_index + 1,
|
||||
len(self.train_loader))
|
||||
msg += "lr: {:>.8f}, ".format(self.lr_scheduler())
|
||||
msg += "data time: {:>.3f}s, ".format(dataload_time)
|
||||
self.train_batch(batch_index, batch, msg)
|
||||
self.after_train_batch()
|
||||
data_start_time = time.time()
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
raise e
|
||||
|
||||
with Timer("Eval Time Cost: {}"):
|
||||
total_loss, num_seen_utts = self.valid()
|
||||
if dist.get_world_size() > 1:
|
||||
num_seen_utts = paddle.to_tensor(num_seen_utts)
|
||||
# the default operator in all_reduce function is sum.
|
||||
dist.all_reduce(num_seen_utts)
|
||||
total_loss = paddle.to_tensor(total_loss)
|
||||
dist.all_reduce(total_loss)
|
||||
cv_loss = total_loss / num_seen_utts
|
||||
cv_loss = float(cv_loss)
|
||||
else:
|
||||
cv_loss = total_loss / num_seen_utts
|
||||
|
||||
logger.info(
|
||||
'Epoch {} Val info val_loss {}'.format(self.epoch, cv_loss))
|
||||
if self.visualizer:
|
||||
self.visualizer.add_scalars(
|
||||
'epoch', {'cv_loss': cv_loss,
|
||||
'lr': self.lr_scheduler()}, self.epoch)
|
||||
self.save(tag=self.epoch, infos={'val_loss': cv_loss})
|
||||
self.new_epoch()
|
||||
|
||||
def setup_dataloader(self):
|
||||
config = self.config.clone()
|
||||
config.defrost()
|
||||
config.collator.keep_transcription_text = False
|
||||
|
||||
# train/valid dataset, return token ids
|
||||
Dataset = TripletManifestDataset if config.model.model_conf.asr_weight > 0. else ManifestDataset
|
||||
config.data.manifest = config.data.train_manifest
|
||||
train_dataset = Dataset.from_config(config)
|
||||
|
||||
config.data.manifest = config.data.dev_manifest
|
||||
dev_dataset = Dataset.from_config(config)
|
||||
|
||||
if config.collator.raw_wav:
|
||||
if config.model.model_conf.asr_weight > 0.:
|
||||
Collator = TripletSpeechCollator
|
||||
TestCollator = SpeechCollator
|
||||
else:
|
||||
TestCollator = Collator = SpeechCollator
|
||||
# Not yet implement the mtl loader for raw_wav.
|
||||
else:
|
||||
if config.model.model_conf.asr_weight > 0.:
|
||||
Collator = TripletKaldiPrePorocessedCollator
|
||||
TestCollator = KaldiPrePorocessedCollator
|
||||
else:
|
||||
TestCollator = Collator = KaldiPrePorocessedCollator
|
||||
|
||||
collate_fn_train = Collator.from_config(config)
|
||||
|
||||
config.collator.augmentation_config = ""
|
||||
collate_fn_dev = Collator.from_config(config)
|
||||
|
||||
if self.parallel:
|
||||
batch_sampler = SortagradDistributedBatchSampler(
|
||||
train_dataset,
|
||||
batch_size=config.collator.batch_size,
|
||||
num_replicas=None,
|
||||
rank=None,
|
||||
shuffle=True,
|
||||
drop_last=True,
|
||||
sortagrad=config.collator.sortagrad,
|
||||
shuffle_method=config.collator.shuffle_method)
|
||||
else:
|
||||
batch_sampler = SortagradBatchSampler(
|
||||
train_dataset,
|
||||
shuffle=True,
|
||||
batch_size=config.collator.batch_size,
|
||||
drop_last=True,
|
||||
sortagrad=config.collator.sortagrad,
|
||||
shuffle_method=config.collator.shuffle_method)
|
||||
self.train_loader = DataLoader(
|
||||
train_dataset,
|
||||
batch_sampler=batch_sampler,
|
||||
collate_fn=collate_fn_train,
|
||||
num_workers=config.collator.num_workers, )
|
||||
self.valid_loader = DataLoader(
|
||||
dev_dataset,
|
||||
batch_size=config.collator.batch_size,
|
||||
shuffle=False,
|
||||
drop_last=False,
|
||||
collate_fn=collate_fn_dev)
|
||||
|
||||
# test dataset, return raw text
|
||||
config.data.manifest = config.data.test_manifest
|
||||
# filter test examples, will cause less examples, but no mismatch with training
|
||||
# and can use large batch size , save training time, so filter test egs now.
|
||||
# config.data.min_input_len = 0.0 # second
|
||||
# config.data.max_input_len = float('inf') # second
|
||||
# config.data.min_output_len = 0.0 # tokens
|
||||
# config.data.max_output_len = float('inf') # tokens
|
||||
# config.data.min_output_input_ratio = 0.00
|
||||
# config.data.max_output_input_ratio = float('inf')
|
||||
test_dataset = ManifestDataset.from_config(config)
|
||||
# return text ord id
|
||||
config.collator.keep_transcription_text = True
|
||||
config.collator.augmentation_config = ""
|
||||
self.test_loader = DataLoader(
|
||||
test_dataset,
|
||||
batch_size=config.decoding.batch_size,
|
||||
shuffle=False,
|
||||
drop_last=False,
|
||||
collate_fn=TestCollator.from_config(config))
|
||||
# return text token id
|
||||
config.collator.keep_transcription_text = False
|
||||
self.align_loader = DataLoader(
|
||||
test_dataset,
|
||||
batch_size=config.decoding.batch_size,
|
||||
shuffle=False,
|
||||
drop_last=False,
|
||||
collate_fn=TestCollator.from_config(config))
|
||||
logger.info("Setup train/valid/test/align Dataloader!")
|
||||
|
||||
def setup_model(self):
|
||||
config = self.config
|
||||
model_conf = config.model
|
||||
with UpdateConfig(model_conf):
|
||||
model_conf.input_dim = self.train_loader.collate_fn.feature_size
|
||||
model_conf.output_dim = self.train_loader.collate_fn.vocab_size
|
||||
|
||||
model = U2STModel.from_config(model_conf)
|
||||
|
||||
if self.parallel:
|
||||
model = paddle.DataParallel(model)
|
||||
|
||||
logger.info(f"{model}")
|
||||
layer_tools.print_params(model, logger.info)
|
||||
|
||||
train_config = config.training
|
||||
optim_type = train_config.optim
|
||||
optim_conf = train_config.optim_conf
|
||||
scheduler_type = train_config.scheduler
|
||||
scheduler_conf = train_config.scheduler_conf
|
||||
|
||||
if scheduler_type == 'expdecaylr':
|
||||
lr_scheduler = paddle.optimizer.lr.ExponentialDecay(
|
||||
learning_rate=optim_conf.lr,
|
||||
gamma=scheduler_conf.lr_decay,
|
||||
verbose=False)
|
||||
elif scheduler_type == 'warmuplr':
|
||||
lr_scheduler = WarmupLR(
|
||||
learning_rate=optim_conf.lr,
|
||||
warmup_steps=scheduler_conf.warmup_steps,
|
||||
verbose=False)
|
||||
elif scheduler_type == 'noam':
|
||||
lr_scheduler = paddle.optimizer.lr.NoamDecay(
|
||||
learning_rate=optim_conf.lr,
|
||||
d_model=model_conf.encoder_conf.output_size,
|
||||
warmup_steps=scheduler_conf.warmup_steps,
|
||||
verbose=False)
|
||||
else:
|
||||
raise ValueError(f"Not support scheduler: {scheduler_type}")
|
||||
|
||||
grad_clip = ClipGradByGlobalNormWithLog(train_config.global_grad_clip)
|
||||
weight_decay = paddle.regularizer.L2Decay(optim_conf.weight_decay)
|
||||
if optim_type == 'adam':
|
||||
optimizer = paddle.optimizer.Adam(
|
||||
learning_rate=lr_scheduler,
|
||||
parameters=model.parameters(),
|
||||
weight_decay=weight_decay,
|
||||
grad_clip=grad_clip)
|
||||
else:
|
||||
raise ValueError(f"Not support optim: {optim_type}")
|
||||
|
||||
self.model = model
|
||||
self.optimizer = optimizer
|
||||
self.lr_scheduler = lr_scheduler
|
||||
logger.info("Setup model/optimizer/lr_scheduler!")
|
||||
|
||||
|
||||
class U2STTester(U2STTrainer):
|
||||
@classmethod
|
||||
def params(cls, config: Optional[CfgNode]=None) -> CfgNode:
|
||||
# decoding config
|
||||
default = CfgNode(
|
||||
dict(
|
||||
alpha=2.5, # Coef of LM for beam search.
|
||||
beta=0.3, # Coef of WC for beam search.
|
||||
cutoff_prob=1.0, # Cutoff probability for pruning.
|
||||
cutoff_top_n=40, # Cutoff number for pruning.
|
||||
lang_model_path='models/lm/common_crawl_00.prune01111.trie.klm', # Filepath for language model.
|
||||
decoding_method='attention', # Decoding method. Options: 'attention', 'ctc_greedy_search',
|
||||
# 'ctc_prefix_beam_search', 'attention_rescoring'
|
||||
error_rate_type='bleu', # Error rate type for evaluation. Options `bleu`, 'char_bleu'
|
||||
num_proc_bsearch=8, # # of CPUs for beam search.
|
||||
beam_size=10, # Beam search width.
|
||||
batch_size=16, # decoding batch size
|
||||
ctc_weight=0.0, # ctc weight for attention rescoring decode mode.
|
||||
decoding_chunk_size=-1, # decoding chunk size. Defaults to -1.
|
||||
# <0: for decoding, use full chunk.
|
||||
# >0: for decoding, use fixed chunk size as set.
|
||||
# 0: used for training, it's prohibited here.
|
||||
num_decoding_left_chunks=-1, # number of left chunks for decoding. Defaults to -1.
|
||||
simulate_streaming=False, # simulate streaming inference. Defaults to False.
|
||||
))
|
||||
|
||||
if config is not None:
|
||||
config.merge_from_other_cfg(default)
|
||||
return default
|
||||
|
||||
def __init__(self, config, args):
|
||||
super().__init__(config, args)
|
||||
|
||||
def ordid2token(self, texts, texts_len):
|
||||
""" ord() id to chr() chr """
|
||||
trans = []
|
||||
for text, n in zip(texts, texts_len):
|
||||
n = n.numpy().item()
|
||||
ids = text[:n]
|
||||
trans.append(''.join([chr(i) for i in ids]))
|
||||
return trans
|
||||
|
||||
def compute_translation_metrics(self,
|
||||
utts,
|
||||
audio,
|
||||
audio_len,
|
||||
texts,
|
||||
texts_len,
|
||||
bleu_func,
|
||||
fout=None):
|
||||
cfg = self.config.decoding
|
||||
len_refs, num_ins = 0, 0
|
||||
|
||||
start_time = time.time()
|
||||
text_feature = self.test_loader.collate_fn.text_feature
|
||||
|
||||
refs = [
|
||||
"".join(chr(t) for t in text[:text_len])
|
||||
for text, text_len in zip(texts, texts_len)
|
||||
]
|
||||
# from IPython import embed
|
||||
# import os
|
||||
# embed()
|
||||
# os._exit(0)
|
||||
hyps = self.model.decode(
|
||||
audio,
|
||||
audio_len,
|
||||
text_feature=text_feature,
|
||||
decoding_method=cfg.decoding_method,
|
||||
lang_model_path=cfg.lang_model_path,
|
||||
beam_alpha=cfg.alpha,
|
||||
beam_beta=cfg.beta,
|
||||
beam_size=cfg.beam_size,
|
||||
cutoff_prob=cfg.cutoff_prob,
|
||||
cutoff_top_n=cfg.cutoff_top_n,
|
||||
num_processes=cfg.num_proc_bsearch,
|
||||
ctc_weight=cfg.ctc_weight,
|
||||
decoding_chunk_size=cfg.decoding_chunk_size,
|
||||
num_decoding_left_chunks=cfg.num_decoding_left_chunks,
|
||||
simulate_streaming=cfg.simulate_streaming)
|
||||
decode_time = time.time() - start_time
|
||||
|
||||
for utt, target, result in zip(utts, refs, hyps):
|
||||
len_refs += len(target.split())
|
||||
num_ins += 1
|
||||
if fout:
|
||||
fout.write(utt + " " + result + "\n")
|
||||
logger.info("\nReference: %s\nHypothesis: %s" % (target, result))
|
||||
logger.info("One example BLEU = %s" %
|
||||
(bleu_func([result], [[target]]).prec_str))
|
||||
|
||||
return dict(
|
||||
hyps=hyps,
|
||||
refs=refs,
|
||||
bleu=bleu_func(hyps, [refs]).score,
|
||||
len_refs=len_refs,
|
||||
num_ins=num_ins, # num examples
|
||||
num_frames=audio_len.sum().numpy().item(),
|
||||
decode_time=decode_time)
|
||||
|
||||
@mp_tools.rank_zero_only
|
||||
@paddle.no_grad()
|
||||
def test(self):
|
||||
assert self.args.result_file
|
||||
self.model.eval()
|
||||
logger.info(f"Test Total Examples: {len(self.test_loader.dataset)}")
|
||||
|
||||
cfg = self.config.decoding
|
||||
bleu_func = bleu_score.char_bleu if cfg.error_rate_type == 'char-bleu' else bleu_score.bleu
|
||||
|
||||
stride_ms = self.test_loader.collate_fn.stride_ms
|
||||
hyps, refs = [], []
|
||||
len_refs, num_ins = 0, 0
|
||||
num_frames = 0.0
|
||||
num_time = 0.0
|
||||
with open(self.args.result_file, 'w') as fout:
|
||||
for i, batch in enumerate(self.test_loader):
|
||||
metrics = self.compute_translation_metrics(
|
||||
*batch, bleu_func=bleu_func, fout=fout)
|
||||
hyps += metrics['hyps']
|
||||
refs += metrics['refs']
|
||||
bleu = metrics['bleu']
|
||||
num_frames += metrics['num_frames']
|
||||
num_time += metrics["decode_time"]
|
||||
len_refs += metrics['len_refs']
|
||||
num_ins += metrics['num_ins']
|
||||
rtf = num_time / (num_frames * stride_ms)
|
||||
logger.info("RTF: %f, BELU (%d) = %f" % (rtf, num_ins, bleu))
|
||||
|
||||
rtf = num_time / (num_frames * stride_ms)
|
||||
msg = "Test: "
|
||||
msg += "epoch: {}, ".format(self.epoch)
|
||||
msg += "step: {}, ".format(self.iteration)
|
||||
msg += "RTF: {}, ".format(rtf)
|
||||
msg += "Test set [%s]: %s" % (len(hyps), str(bleu_func(hyps, [refs])))
|
||||
logger.info(msg)
|
||||
bleu_meta_path = os.path.splitext(self.args.result_file)[0] + '.bleu'
|
||||
err_type_str = "BLEU"
|
||||
with open(bleu_meta_path, 'w') as f:
|
||||
data = json.dumps({
|
||||
"epoch":
|
||||
self.epoch,
|
||||
"step":
|
||||
self.iteration,
|
||||
"rtf":
|
||||
rtf,
|
||||
err_type_str:
|
||||
bleu_func(hyps, [refs]).score,
|
||||
"dataset_hour": (num_frames * stride_ms) / 1000.0 / 3600.0,
|
||||
"process_hour":
|
||||
num_time / 1000.0 / 3600.0,
|
||||
"num_examples":
|
||||
num_ins,
|
||||
"decode_method":
|
||||
self.config.decoding.decoding_method,
|
||||
})
|
||||
f.write(data + '\n')
|
||||
|
||||
def run_test(self):
|
||||
self.resume_or_scratch()
|
||||
try:
|
||||
self.test()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(-1)
|
||||
|
||||
@paddle.no_grad()
|
||||
def align(self):
|
||||
if self.config.decoding.batch_size > 1:
|
||||
logger.fatal('alignment mode must be running with batch_size == 1')
|
||||
sys.exit(1)
|
||||
|
||||
# xxx.align
|
||||
assert self.args.result_file and self.args.result_file.endswith(
|
||||
'.align')
|
||||
|
||||
self.model.eval()
|
||||
logger.info(f"Align Total Examples: {len(self.align_loader.dataset)}")
|
||||
|
||||
stride_ms = self.align_loader.collate_fn.stride_ms
|
||||
token_dict = self.align_loader.collate_fn.vocab_list
|
||||
with open(self.args.result_file, 'w') as fout:
|
||||
# one example in batch
|
||||
for i, batch in enumerate(self.align_loader):
|
||||
key, feat, feats_length, target, target_length = batch
|
||||
|
||||
# 1. Encoder
|
||||
encoder_out, encoder_mask = self.model._forward_encoder(
|
||||
feat, feats_length) # (B, maxlen, encoder_dim)
|
||||
maxlen = encoder_out.size(1)
|
||||
ctc_probs = self.model.ctc.log_softmax(
|
||||
encoder_out) # (1, maxlen, vocab_size)
|
||||
|
||||
# 2. alignment
|
||||
ctc_probs = ctc_probs.squeeze(0)
|
||||
target = target.squeeze(0)
|
||||
alignment = ctc_utils.forced_align(ctc_probs, target)
|
||||
logger.info("align ids", key[0], alignment)
|
||||
fout.write('{} {}\n'.format(key[0], alignment))
|
||||
|
||||
# 3. gen praat
|
||||
# segment alignment
|
||||
align_segs = text_grid.segment_alignment(alignment)
|
||||
logger.info("align tokens", key[0], align_segs)
|
||||
# IntervalTier, List["start end token\n"]
|
||||
subsample = utility.get_subsample(self.config)
|
||||
tierformat = text_grid.align_to_tierformat(
|
||||
align_segs, subsample, token_dict)
|
||||
# write tier
|
||||
align_output_path = os.path.join(
|
||||
os.path.dirname(self.args.result_file), "align")
|
||||
tier_path = os.path.join(align_output_path, key[0] + ".tier")
|
||||
with open(tier_path, 'w') as f:
|
||||
f.writelines(tierformat)
|
||||
# write textgrid
|
||||
textgrid_path = os.path.join(align_output_path,
|
||||
key[0] + ".TextGrid")
|
||||
second_per_frame = 1. / (1000. /
|
||||
stride_ms) # 25ms window, 10ms stride
|
||||
second_per_example = (
|
||||
len(alignment) + 1) * subsample * second_per_frame
|
||||
text_grid.generate_textgrid(
|
||||
maxtime=second_per_example,
|
||||
intervals=tierformat,
|
||||
output=textgrid_path)
|
||||
|
||||
def run_align(self):
|
||||
self.resume_or_scratch()
|
||||
try:
|
||||
self.align()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(-1)
|
||||
|
||||
def load_inferspec(self):
|
||||
"""infer model and input spec.
|
||||
|
||||
Returns:
|
||||
nn.Layer: inference model
|
||||
List[paddle.static.InputSpec]: input spec.
|
||||
"""
|
||||
from deepspeech.models.u2 import U2InferModel
|
||||
infer_model = U2InferModel.from_pretrained(self.test_loader,
|
||||
self.config.model.clone(),
|
||||
self.args.checkpoint_path)
|
||||
feat_dim = self.test_loader.collate_fn.feature_size
|
||||
input_spec = [
|
||||
paddle.static.InputSpec(shape=[1, None, feat_dim],
|
||||
dtype='float32'), # audio, [B,T,D]
|
||||
paddle.static.InputSpec(shape=[1],
|
||||
dtype='int64'), # audio_length, [B]
|
||||
]
|
||||
return infer_model, input_spec
|
||||
|
||||
def export(self):
|
||||
infer_model, input_spec = self.load_inferspec()
|
||||
assert isinstance(input_spec, list), type(input_spec)
|
||||
infer_model.eval()
|
||||
static_model = paddle.jit.to_static(infer_model, input_spec=input_spec)
|
||||
logger.info(f"Export code: {static_model.forward.code}")
|
||||
paddle.jit.save(static_model, self.args.export_path)
|
||||
|
||||
def run_export(self):
|
||||
try:
|
||||
self.export()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(-1)
|
||||
|
||||
def setup(self):
|
||||
"""Setup the experiment.
|
||||
"""
|
||||
paddle.set_device(self.args.device)
|
||||
|
||||
self.setup_output_dir()
|
||||
self.setup_checkpointer()
|
||||
|
||||
self.setup_dataloader()
|
||||
self.setup_model()
|
||||
|
||||
self.iteration = 0
|
||||
self.epoch = 0
|
||||
|
||||
def setup_output_dir(self):
|
||||
"""Create a directory used for output.
|
||||
"""
|
||||
# output dir
|
||||
if self.args.output:
|
||||
output_dir = Path(self.args.output).expanduser()
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
output_dir = Path(
|
||||
self.args.checkpoint_path).expanduser().parent.parent
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.output_dir = output_dir
|
||||
@ -0,0 +1,469 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import itertools
|
||||
|
||||
import numpy as np
|
||||
|
||||
from deepspeech.utils.log import Log
|
||||
|
||||
__all__ = ["make_batchset"]
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
|
||||
def batchfy_by_seq(
|
||||
sorted_data,
|
||||
batch_size,
|
||||
max_length_in,
|
||||
max_length_out,
|
||||
min_batch_size=1,
|
||||
shortest_first=False,
|
||||
ikey="input",
|
||||
iaxis=0,
|
||||
okey="output",
|
||||
oaxis=0, ):
|
||||
"""Make batch set from json dictionary
|
||||
|
||||
:param List[(str, Dict[str, Any])] sorted_data: dictionary loaded from data.json
|
||||
:param int batch_size: batch size
|
||||
:param int max_length_in: maximum length of input to decide adaptive batch size
|
||||
:param int max_length_out: maximum length of output to decide adaptive batch size
|
||||
:param int min_batch_size: mininum batch size (for multi-gpu)
|
||||
:param bool shortest_first: Sort from batch with shortest samples
|
||||
to longest if true, otherwise reverse
|
||||
:param str ikey: key to access input
|
||||
(for ASR ikey="input", for TTS, MT ikey="output".)
|
||||
:param int iaxis: dimension to access input
|
||||
(for ASR, TTS iaxis=0, for MT iaxis="1".)
|
||||
:param str okey: key to access output
|
||||
(for ASR, MT okey="output". for TTS okey="input".)
|
||||
:param int oaxis: dimension to access output
|
||||
(for ASR, TTS, MT oaxis=0, reserved for future research, -1 means all axis.)
|
||||
:return: List[List[Tuple[str, dict]]] list of batches
|
||||
"""
|
||||
if batch_size <= 0:
|
||||
raise ValueError(f"Invalid batch_size={batch_size}")
|
||||
|
||||
# check #utts is more than min_batch_size
|
||||
if len(sorted_data) < min_batch_size:
|
||||
raise ValueError(
|
||||
f"#utts({len(sorted_data)}) is less than min_batch_size({min_batch_size})."
|
||||
)
|
||||
|
||||
# make list of minibatches
|
||||
minibatches = []
|
||||
start = 0
|
||||
while True:
|
||||
_, info = sorted_data[start]
|
||||
ilen = int(info[ikey][iaxis]["shape"][0])
|
||||
olen = (int(info[okey][oaxis]["shape"][0]) if oaxis >= 0 else
|
||||
max(map(lambda x: int(x["shape"][0]), info[okey])))
|
||||
factor = max(int(ilen / max_length_in), int(olen / max_length_out))
|
||||
# change batchsize depending on the input and output length
|
||||
# if ilen = 1000 and max_length_in = 800
|
||||
# then b = batchsize / 2
|
||||
# and max(min_batches, .) avoids batchsize = 0
|
||||
bs = max(min_batch_size, int(batch_size / (1 + factor)))
|
||||
end = min(len(sorted_data), start + bs)
|
||||
minibatch = sorted_data[start:end]
|
||||
if shortest_first:
|
||||
minibatch.reverse()
|
||||
|
||||
# check each batch is more than minimum batchsize
|
||||
if len(minibatch) < min_batch_size:
|
||||
mod = min_batch_size - len(minibatch) % min_batch_size
|
||||
additional_minibatch = [
|
||||
sorted_data[i] for i in np.random.randint(0, start, mod)
|
||||
]
|
||||
if shortest_first:
|
||||
additional_minibatch.reverse()
|
||||
minibatch.extend(additional_minibatch)
|
||||
minibatches.append(minibatch)
|
||||
|
||||
if end == len(sorted_data):
|
||||
break
|
||||
start = end
|
||||
|
||||
# batch: List[List[Tuple[str, dict]]]
|
||||
return minibatches
|
||||
|
||||
|
||||
def batchfy_by_bin(
|
||||
sorted_data,
|
||||
batch_bins,
|
||||
num_batches=0,
|
||||
min_batch_size=1,
|
||||
shortest_first=False,
|
||||
ikey="input",
|
||||
okey="output", ):
|
||||
"""Make variably sized batch set, which maximizes
|
||||
|
||||
the number of bins up to `batch_bins`.
|
||||
|
||||
:param List[(str, Dict[str, Any])] sorted_data: dictionary loaded from data.json
|
||||
:param int batch_bins: Maximum frames of a batch
|
||||
:param int num_batches: # number of batches to use (for debug)
|
||||
:param int min_batch_size: minimum batch size (for multi-gpu)
|
||||
:param int test: Return only every `test` batches
|
||||
:param bool shortest_first: Sort from batch with shortest samples
|
||||
to longest if true, otherwise reverse
|
||||
|
||||
:param str ikey: key to access input (for ASR ikey="input", for TTS ikey="output".)
|
||||
:param str okey: key to access output (for ASR okey="output". for TTS okey="input".)
|
||||
|
||||
:return: List[Tuple[str, Dict[str, List[Dict[str, Any]]]] list of batches
|
||||
"""
|
||||
if batch_bins <= 0:
|
||||
raise ValueError(f"invalid batch_bins={batch_bins}")
|
||||
length = len(sorted_data)
|
||||
idim = int(sorted_data[0][1][ikey][0]["shape"][1])
|
||||
odim = int(sorted_data[0][1][okey][0]["shape"][1])
|
||||
logger.info("# utts: " + str(len(sorted_data)))
|
||||
minibatches = []
|
||||
start = 0
|
||||
n = 0
|
||||
while True:
|
||||
# Dynamic batch size depending on size of samples
|
||||
b = 0
|
||||
next_size = 0
|
||||
max_olen = 0
|
||||
while next_size < batch_bins and (start + b) < length:
|
||||
ilen = int(sorted_data[start + b][1][ikey][0]["shape"][0]) * idim
|
||||
olen = int(sorted_data[start + b][1][okey][0]["shape"][0]) * odim
|
||||
if olen > max_olen:
|
||||
max_olen = olen
|
||||
next_size = (max_olen + ilen) * (b + 1)
|
||||
if next_size <= batch_bins:
|
||||
b += 1
|
||||
elif next_size == 0:
|
||||
raise ValueError(
|
||||
f"Can't fit one sample in batch_bins ({batch_bins}): "
|
||||
f"Please increase the value")
|
||||
end = min(length, start + max(min_batch_size, b))
|
||||
batch = sorted_data[start:end]
|
||||
if shortest_first:
|
||||
batch.reverse()
|
||||
minibatches.append(batch)
|
||||
# Check for min_batch_size and fixes the batches if needed
|
||||
i = -1
|
||||
while len(minibatches[i]) < min_batch_size:
|
||||
missing = min_batch_size - len(minibatches[i])
|
||||
if -i == len(minibatches):
|
||||
minibatches[i + 1].extend(minibatches[i])
|
||||
minibatches = minibatches[1:]
|
||||
break
|
||||
else:
|
||||
minibatches[i].extend(minibatches[i - 1][:missing])
|
||||
minibatches[i - 1] = minibatches[i - 1][missing:]
|
||||
i -= 1
|
||||
if end == length:
|
||||
break
|
||||
start = end
|
||||
n += 1
|
||||
if num_batches > 0:
|
||||
minibatches = minibatches[:num_batches]
|
||||
lengths = [len(x) for x in minibatches]
|
||||
logger.info(
|
||||
str(len(minibatches)) + " batches containing from " + str(min(lengths))
|
||||
+ " to " + str(max(lengths)) + " samples " + "(avg " + str(
|
||||
int(np.mean(lengths))) + " samples).")
|
||||
return minibatches
|
||||
|
||||
|
||||
def batchfy_by_frame(
|
||||
sorted_data,
|
||||
max_frames_in,
|
||||
max_frames_out,
|
||||
max_frames_inout,
|
||||
num_batches=0,
|
||||
min_batch_size=1,
|
||||
shortest_first=False,
|
||||
ikey="input",
|
||||
okey="output", ):
|
||||
"""Make variable batch set, which maximizes the number of frames to max_batch_frame.
|
||||
|
||||
:param List[(str, Dict[str, Any])] sorteddata: dictionary loaded from data.json
|
||||
:param int max_frames_in: Maximum input frames of a batch
|
||||
:param int max_frames_out: Maximum output frames of a batch
|
||||
:param int max_frames_inout: Maximum input+output frames of a batch
|
||||
:param int num_batches: # number of batches to use (for debug)
|
||||
:param int min_batch_size: minimum batch size (for multi-gpu)
|
||||
:param int test: Return only every `test` batches
|
||||
:param bool shortest_first: Sort from batch with shortest samples
|
||||
to longest if true, otherwise reverse
|
||||
|
||||
:param str ikey: key to access input (for ASR ikey="input", for TTS ikey="output".)
|
||||
:param str okey: key to access output (for ASR okey="output". for TTS okey="input".)
|
||||
|
||||
:return: List[Tuple[str, Dict[str, List[Dict[str, Any]]]] list of batches
|
||||
"""
|
||||
if max_frames_in <= 0 and max_frames_out <= 0 and max_frames_inout <= 0:
|
||||
raise ValueError(
|
||||
"At least, one of `--batch-frames-in`, `--batch-frames-out` or "
|
||||
"`--batch-frames-inout` should be > 0")
|
||||
length = len(sorted_data)
|
||||
minibatches = []
|
||||
start = 0
|
||||
end = 0
|
||||
while end != length:
|
||||
# Dynamic batch size depending on size of samples
|
||||
b = 0
|
||||
max_olen = 0
|
||||
max_ilen = 0
|
||||
while (start + b) < length:
|
||||
ilen = int(sorted_data[start + b][1][ikey][0]["shape"][0])
|
||||
if ilen > max_frames_in and max_frames_in != 0:
|
||||
raise ValueError(
|
||||
f"Can't fit one sample in --batch-frames-in ({max_frames_in}): "
|
||||
f"Please increase the value")
|
||||
olen = int(sorted_data[start + b][1][okey][0]["shape"][0])
|
||||
if olen > max_frames_out and max_frames_out != 0:
|
||||
raise ValueError(
|
||||
f"Can't fit one sample in --batch-frames-out ({max_frames_out}): "
|
||||
f"Please increase the value")
|
||||
if ilen + olen > max_frames_inout and max_frames_inout != 0:
|
||||
raise ValueError(
|
||||
f"Can't fit one sample in --batch-frames-out ({max_frames_inout}): "
|
||||
f"Please increase the value")
|
||||
max_olen = max(max_olen, olen)
|
||||
max_ilen = max(max_ilen, ilen)
|
||||
in_ok = max_ilen * (b + 1) <= max_frames_in or max_frames_in == 0
|
||||
out_ok = max_olen * (b + 1) <= max_frames_out or max_frames_out == 0
|
||||
inout_ok = (max_ilen + max_olen) * (
|
||||
b + 1) <= max_frames_inout or max_frames_inout == 0
|
||||
if in_ok and out_ok and inout_ok:
|
||||
# add more seq in the minibatch
|
||||
b += 1
|
||||
else:
|
||||
# no more seq in the minibatch
|
||||
break
|
||||
end = min(length, start + b)
|
||||
batch = sorted_data[start:end]
|
||||
if shortest_first:
|
||||
batch.reverse()
|
||||
minibatches.append(batch)
|
||||
# Check for min_batch_size and fixes the batches if needed
|
||||
i = -1
|
||||
while len(minibatches[i]) < min_batch_size:
|
||||
missing = min_batch_size - len(minibatches[i])
|
||||
if -i == len(minibatches):
|
||||
minibatches[i + 1].extend(minibatches[i])
|
||||
minibatches = minibatches[1:]
|
||||
break
|
||||
else:
|
||||
minibatches[i].extend(minibatches[i - 1][:missing])
|
||||
minibatches[i - 1] = minibatches[i - 1][missing:]
|
||||
i -= 1
|
||||
start = end
|
||||
if num_batches > 0:
|
||||
minibatches = minibatches[:num_batches]
|
||||
lengths = [len(x) for x in minibatches]
|
||||
logger.info(
|
||||
str(len(minibatches)) + " batches containing from " + str(min(lengths))
|
||||
+ " to " + str(max(lengths)) + " samples" + "(avg " + str(
|
||||
int(np.mean(lengths))) + " samples).")
|
||||
|
||||
return minibatches
|
||||
|
||||
|
||||
def batchfy_shuffle(data, batch_size, min_batch_size, num_batches,
|
||||
shortest_first):
|
||||
import random
|
||||
|
||||
logger.info("use shuffled batch.")
|
||||
sorted_data = random.sample(data.items(), len(data.items()))
|
||||
logger.info("# utts: " + str(len(sorted_data)))
|
||||
# make list of minibatches
|
||||
minibatches = []
|
||||
start = 0
|
||||
while True:
|
||||
end = min(len(sorted_data), start + batch_size)
|
||||
# check each batch is more than minimum batchsize
|
||||
minibatch = sorted_data[start:end]
|
||||
if shortest_first:
|
||||
minibatch.reverse()
|
||||
if len(minibatch) < min_batch_size:
|
||||
mod = min_batch_size - len(minibatch) % min_batch_size
|
||||
additional_minibatch = [
|
||||
sorted_data[i] for i in np.random.randint(0, start, mod)
|
||||
]
|
||||
if shortest_first:
|
||||
additional_minibatch.reverse()
|
||||
minibatch.extend(additional_minibatch)
|
||||
minibatches.append(minibatch)
|
||||
if end == len(sorted_data):
|
||||
break
|
||||
start = end
|
||||
|
||||
# for debugging
|
||||
if num_batches > 0:
|
||||
minibatches = minibatches[:num_batches]
|
||||
logger.info("# minibatches: " + str(len(minibatches)))
|
||||
return minibatches
|
||||
|
||||
|
||||
BATCH_COUNT_CHOICES = ["auto", "seq", "bin", "frame"]
|
||||
BATCH_SORT_KEY_CHOICES = ["input", "output", "shuffle"]
|
||||
|
||||
|
||||
def make_batchset(
|
||||
data,
|
||||
batch_size=0,
|
||||
max_length_in=float("inf"),
|
||||
max_length_out=float("inf"),
|
||||
num_batches=0,
|
||||
min_batch_size=1,
|
||||
shortest_first=False,
|
||||
batch_sort_key="input",
|
||||
count="auto",
|
||||
batch_bins=0,
|
||||
batch_frames_in=0,
|
||||
batch_frames_out=0,
|
||||
batch_frames_inout=0,
|
||||
iaxis=0,
|
||||
oaxis=0, ):
|
||||
"""Make batch set from json dictionary
|
||||
|
||||
if utts have "category" value,
|
||||
|
||||
>>> data = [{'category': 'A', 'input': ..., 'utt':'utt1'},
|
||||
... {'category': 'B', 'input': ..., 'utt':'utt2'},
|
||||
... {'category': 'B', 'input': ..., 'utt':'utt3'},
|
||||
... {'category': 'A', 'input': ..., 'utt':'utt4'}]
|
||||
>>> make_batchset(data, batchsize=2, ...)
|
||||
[[('utt1', ...), ('utt4', ...)], [('utt2', ...), ('utt3': ...)]]
|
||||
|
||||
Note that if any utts doesn't have "category",
|
||||
perform as same as batchfy_by_{count}
|
||||
|
||||
:param List[Dict[str, Any]] data: dictionary loaded from data.json
|
||||
:param int batch_size: maximum number of sequences in a minibatch.
|
||||
:param int batch_bins: maximum number of bins (frames x dim) in a minibatch.
|
||||
:param int batch_frames_in: maximum number of input frames in a minibatch.
|
||||
:param int batch_frames_out: maximum number of output frames in a minibatch.
|
||||
:param int batch_frames_out: maximum number of input+output frames in a minibatch.
|
||||
:param str count: strategy to count maximum size of batch.
|
||||
For choices, see espnet.asr.batchfy.BATCH_COUNT_CHOICES
|
||||
|
||||
:param int max_length_in: maximum length of input to decide adaptive batch size
|
||||
:param int max_length_out: maximum length of output to decide adaptive batch size
|
||||
:param int num_batches: # number of batches to use (for debug)
|
||||
:param int min_batch_size: minimum batch size (for multi-gpu)
|
||||
:param bool shortest_first: Sort from batch with shortest samples
|
||||
to longest if true, otherwise reverse
|
||||
:param str batch_sort_key: how to sort data before creating minibatches
|
||||
["input", "output", "shuffle"]
|
||||
:param bool swap_io: if True, use "input" as output and "output"
|
||||
as input in `data` dict
|
||||
:param bool mt: if True, use 0-axis of "output" as output and 1-axis of "output"
|
||||
as input in `data` dict
|
||||
:param int iaxis: dimension to access input
|
||||
(for ASR, TTS iaxis=0, for MT iaxis="1".)
|
||||
:param int oaxis: dimension to access output (for ASR, TTS, MT oaxis=0,
|
||||
reserved for future research, -1 means all axis.)
|
||||
:return: List[List[Tuple[str, dict]]] list of batches
|
||||
"""
|
||||
# check args
|
||||
if count not in BATCH_COUNT_CHOICES:
|
||||
raise ValueError(
|
||||
f"arg 'count' ({count}) should be one of {BATCH_COUNT_CHOICES}")
|
||||
if batch_sort_key not in BATCH_SORT_KEY_CHOICES:
|
||||
raise ValueError(f"arg 'batch_sort_key' ({batch_sort_key}) should be "
|
||||
f"one of {BATCH_SORT_KEY_CHOICES}")
|
||||
|
||||
ikey = "input"
|
||||
okey = "output"
|
||||
batch_sort_axis = 0 # index of list
|
||||
if count == "auto":
|
||||
if batch_size != 0:
|
||||
count = "seq"
|
||||
elif batch_bins != 0:
|
||||
count = "bin"
|
||||
elif batch_frames_in != 0 or batch_frames_out != 0 or batch_frames_inout != 0:
|
||||
count = "frame"
|
||||
else:
|
||||
raise ValueError(
|
||||
f"cannot detect `count` manually set one of {BATCH_COUNT_CHOICES}"
|
||||
)
|
||||
logger.info(f"count is auto detected as {count}")
|
||||
|
||||
if count != "seq" and batch_sort_key == "shuffle":
|
||||
raise ValueError(
|
||||
"batch_sort_key=shuffle is only available if batch_count=seq")
|
||||
|
||||
category2data = {} # Dict[str, dict]
|
||||
for v in data:
|
||||
k = v['utt']
|
||||
category2data.setdefault(v.get("category"), {})[k] = v
|
||||
|
||||
batches_list = [] # List[List[List[Tuple[str, dict]]]]
|
||||
for d in category2data.values():
|
||||
if batch_sort_key == "shuffle":
|
||||
batches = batchfy_shuffle(d, batch_size, min_batch_size,
|
||||
num_batches, shortest_first)
|
||||
batches_list.append(batches)
|
||||
continue
|
||||
|
||||
# sort it by input lengths (long to short)
|
||||
sorted_data = sorted(
|
||||
d.items(),
|
||||
key=lambda data: int(data[1][batch_sort_key][batch_sort_axis]["shape"][0]),
|
||||
reverse=not shortest_first, )
|
||||
logger.info("# utts: " + str(len(sorted_data)))
|
||||
|
||||
if count == "seq":
|
||||
batches = batchfy_by_seq(
|
||||
sorted_data,
|
||||
batch_size=batch_size,
|
||||
max_length_in=max_length_in,
|
||||
max_length_out=max_length_out,
|
||||
min_batch_size=min_batch_size,
|
||||
shortest_first=shortest_first,
|
||||
ikey=ikey,
|
||||
iaxis=iaxis,
|
||||
okey=okey,
|
||||
oaxis=oaxis, )
|
||||
if count == "bin":
|
||||
batches = batchfy_by_bin(
|
||||
sorted_data,
|
||||
batch_bins=batch_bins,
|
||||
min_batch_size=min_batch_size,
|
||||
shortest_first=shortest_first,
|
||||
ikey=ikey,
|
||||
okey=okey, )
|
||||
if count == "frame":
|
||||
batches = batchfy_by_frame(
|
||||
sorted_data,
|
||||
max_frames_in=batch_frames_in,
|
||||
max_frames_out=batch_frames_out,
|
||||
max_frames_inout=batch_frames_inout,
|
||||
min_batch_size=min_batch_size,
|
||||
shortest_first=shortest_first,
|
||||
ikey=ikey,
|
||||
okey=okey, )
|
||||
batches_list.append(batches)
|
||||
|
||||
if len(batches_list) == 1:
|
||||
batches = batches_list[0]
|
||||
else:
|
||||
# Concat list. This way is faster than "sum(batch_list, [])"
|
||||
batches = list(itertools.chain(*batches_list))
|
||||
|
||||
# for debugging
|
||||
if num_batches > 0:
|
||||
batches = batches[:num_batches]
|
||||
logger.info("# minibatches: " + str(len(batches)))
|
||||
|
||||
# batch: List[List[Tuple[str, dict]]]
|
||||
return batches
|
||||
@ -0,0 +1,631 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import io
|
||||
from collections import namedtuple
|
||||
from typing import Optional
|
||||
|
||||
import kaldiio
|
||||
import numpy as np
|
||||
from yacs.config import CfgNode
|
||||
|
||||
from deepspeech.frontend.augmentor.augmentation import AugmentationPipeline
|
||||
from deepspeech.frontend.featurizer.speech_featurizer import SpeechFeaturizer
|
||||
from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer
|
||||
from deepspeech.frontend.normalizer import FeatureNormalizer
|
||||
from deepspeech.frontend.speech import SpeechSegment
|
||||
from deepspeech.frontend.utility import IGNORE_ID
|
||||
from deepspeech.io.utility import pad_sequence
|
||||
from deepspeech.utils.log import Log
|
||||
|
||||
__all__ = ["SpeechCollator", "KaldiPrePorocessedCollator"]
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
# namedtupe need global for pickle.
|
||||
TarLocalData = namedtuple('TarLocalData', ['tar2info', 'tar2object'])
|
||||
|
||||
|
||||
class SpeechCollator():
|
||||
@classmethod
|
||||
def params(cls, config: Optional[CfgNode]=None) -> CfgNode:
|
||||
default = CfgNode(
|
||||
dict(
|
||||
augmentation_config="",
|
||||
random_seed=0,
|
||||
mean_std_filepath="",
|
||||
unit_type="char",
|
||||
vocab_filepath="",
|
||||
spm_model_prefix="",
|
||||
specgram_type='linear', # 'linear', 'mfcc', 'fbank'
|
||||
feat_dim=0, # 'mfcc', 'fbank'
|
||||
delta_delta=False, # 'mfcc', 'fbank'
|
||||
stride_ms=10.0, # ms
|
||||
window_ms=20.0, # ms
|
||||
n_fft=None, # fft points
|
||||
max_freq=None, # None for samplerate/2
|
||||
target_sample_rate=16000, # target sample rate
|
||||
use_dB_normalization=True,
|
||||
target_dB=-20,
|
||||
dither=1.0, # feature dither
|
||||
keep_transcription_text=False))
|
||||
|
||||
if config is not None:
|
||||
config.merge_from_other_cfg(default)
|
||||
return default
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config):
|
||||
"""Build a SpeechCollator object from a config.
|
||||
|
||||
Args:
|
||||
config (yacs.config.CfgNode): configs object.
|
||||
|
||||
Returns:
|
||||
SpeechCollator: collator object.
|
||||
"""
|
||||
assert 'augmentation_config' in config.collator
|
||||
assert 'keep_transcription_text' in config.collator
|
||||
assert 'mean_std_filepath' in config.collator
|
||||
assert 'vocab_filepath' in config.collator
|
||||
assert 'specgram_type' in config.collator
|
||||
assert 'n_fft' in config.collator
|
||||
assert config.collator
|
||||
|
||||
if isinstance(config.collator.augmentation_config, (str, bytes)):
|
||||
if config.collator.augmentation_config:
|
||||
aug_file = io.open(
|
||||
config.collator.augmentation_config,
|
||||
mode='r',
|
||||
encoding='utf8')
|
||||
else:
|
||||
aug_file = io.StringIO(initial_value='{}', newline='')
|
||||
else:
|
||||
aug_file = config.collator.augmentation_config
|
||||
assert isinstance(aug_file, io.StringIO)
|
||||
|
||||
speech_collator = cls(
|
||||
aug_file=aug_file,
|
||||
random_seed=0,
|
||||
mean_std_filepath=config.collator.mean_std_filepath,
|
||||
unit_type=config.collator.unit_type,
|
||||
vocab_filepath=config.collator.vocab_filepath,
|
||||
spm_model_prefix=config.collator.spm_model_prefix,
|
||||
specgram_type=config.collator.specgram_type,
|
||||
feat_dim=config.collator.feat_dim,
|
||||
delta_delta=config.collator.delta_delta,
|
||||
stride_ms=config.collator.stride_ms,
|
||||
window_ms=config.collator.window_ms,
|
||||
n_fft=config.collator.n_fft,
|
||||
max_freq=config.collator.max_freq,
|
||||
target_sample_rate=config.collator.target_sample_rate,
|
||||
use_dB_normalization=config.collator.use_dB_normalization,
|
||||
target_dB=config.collator.target_dB,
|
||||
dither=config.collator.dither,
|
||||
keep_transcription_text=config.collator.keep_transcription_text)
|
||||
return speech_collator
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
aug_file,
|
||||
mean_std_filepath,
|
||||
vocab_filepath,
|
||||
spm_model_prefix,
|
||||
random_seed=0,
|
||||
unit_type="char",
|
||||
specgram_type='linear', # 'linear', 'mfcc', 'fbank'
|
||||
feat_dim=0, # 'mfcc', 'fbank'
|
||||
delta_delta=False, # 'mfcc', 'fbank'
|
||||
stride_ms=10.0, # ms
|
||||
window_ms=20.0, # ms
|
||||
n_fft=None, # fft points
|
||||
max_freq=None, # None for samplerate/2
|
||||
target_sample_rate=16000, # target sample rate
|
||||
use_dB_normalization=True,
|
||||
target_dB=-20,
|
||||
dither=1.0,
|
||||
keep_transcription_text=True):
|
||||
"""SpeechCollator Collator
|
||||
|
||||
Args:
|
||||
unit_type(str): token unit type, e.g. char, word, spm
|
||||
vocab_filepath (str): vocab file path.
|
||||
mean_std_filepath (str): mean and std file path, which suffix is *.npy
|
||||
spm_model_prefix (str): spm model prefix, need if `unit_type` is spm.
|
||||
augmentation_config (str, optional): augmentation json str. Defaults to '{}'.
|
||||
stride_ms (float, optional): stride size in ms. Defaults to 10.0.
|
||||
window_ms (float, optional): window size in ms. Defaults to 20.0.
|
||||
n_fft (int, optional): fft points for rfft. Defaults to None.
|
||||
max_freq (int, optional): max cut freq. Defaults to None.
|
||||
target_sample_rate (int, optional): target sample rate which used for training. Defaults to 16000.
|
||||
specgram_type (str, optional): 'linear', 'mfcc' or 'fbank'. Defaults to 'linear'.
|
||||
feat_dim (int, optional): audio feature dim, using by 'mfcc' or 'fbank'. Defaults to None.
|
||||
delta_delta (bool, optional): audio feature with delta-delta, using by 'fbank' or 'mfcc'. Defaults to False.
|
||||
use_dB_normalization (bool, optional): do dB normalization. Defaults to True.
|
||||
target_dB (int, optional): target dB. Defaults to -20.
|
||||
random_seed (int, optional): for random generator. Defaults to 0.
|
||||
keep_transcription_text (bool, optional): True, when not in training mode, will not do tokenizer; Defaults to False.
|
||||
if ``keep_transcription_text`` is False, text is token ids else is raw string.
|
||||
|
||||
Do augmentations
|
||||
Padding audio features with zeros to make them have the same shape (or
|
||||
a user-defined shape) within one batch.
|
||||
"""
|
||||
self._keep_transcription_text = keep_transcription_text
|
||||
|
||||
self._local_data = TarLocalData(tar2info={}, tar2object={})
|
||||
self._augmentation_pipeline = AugmentationPipeline(
|
||||
augmentation_config=aug_file.read(), random_seed=random_seed)
|
||||
|
||||
self._normalizer = FeatureNormalizer(
|
||||
mean_std_filepath) if mean_std_filepath else None
|
||||
|
||||
self._stride_ms = stride_ms
|
||||
self._target_sample_rate = target_sample_rate
|
||||
|
||||
self._speech_featurizer = SpeechFeaturizer(
|
||||
unit_type=unit_type,
|
||||
vocab_filepath=vocab_filepath,
|
||||
spm_model_prefix=spm_model_prefix,
|
||||
specgram_type=specgram_type,
|
||||
feat_dim=feat_dim,
|
||||
delta_delta=delta_delta,
|
||||
stride_ms=stride_ms,
|
||||
window_ms=window_ms,
|
||||
n_fft=n_fft,
|
||||
max_freq=max_freq,
|
||||
target_sample_rate=target_sample_rate,
|
||||
use_dB_normalization=use_dB_normalization,
|
||||
target_dB=target_dB,
|
||||
dither=dither)
|
||||
|
||||
def _parse_tar(self, file):
|
||||
"""Parse a tar file to get a tarfile object
|
||||
and a map containing tarinfoes
|
||||
"""
|
||||
result = {}
|
||||
f = tarfile.open(file)
|
||||
for tarinfo in f.getmembers():
|
||||
result[tarinfo.name] = tarinfo
|
||||
return f, result
|
||||
|
||||
def _subfile_from_tar(self, file):
|
||||
"""Get subfile object from tar.
|
||||
|
||||
It will return a subfile object from tar file
|
||||
and cached tar file info for next reading request.
|
||||
"""
|
||||
tarpath, filename = file.split(':', 1)[1].split('#', 1)
|
||||
if 'tar2info' not in self._local_data.__dict__:
|
||||
self._local_data.tar2info = {}
|
||||
if 'tar2object' not in self._local_data.__dict__:
|
||||
self._local_data.tar2object = {}
|
||||
if tarpath not in self._local_data.tar2info:
|
||||
object, infoes = self._parse_tar(tarpath)
|
||||
self._local_data.tar2info[tarpath] = infoes
|
||||
self._local_data.tar2object[tarpath] = object
|
||||
return self._local_data.tar2object[tarpath].extractfile(
|
||||
self._local_data.tar2info[tarpath][filename])
|
||||
|
||||
@property
|
||||
def manifest(self):
|
||||
return self._manifest
|
||||
|
||||
@property
|
||||
def vocab_size(self):
|
||||
return self._speech_featurizer.vocab_size
|
||||
|
||||
@property
|
||||
def vocab_list(self):
|
||||
return self._speech_featurizer.vocab_list
|
||||
|
||||
@property
|
||||
def vocab_dict(self):
|
||||
return self._speech_featurizer.vocab_dict
|
||||
|
||||
@property
|
||||
def text_feature(self):
|
||||
return self._speech_featurizer.text_feature
|
||||
|
||||
@property
|
||||
def feature_size(self):
|
||||
return self._speech_featurizer.feature_size
|
||||
|
||||
@property
|
||||
def stride_ms(self):
|
||||
return self._speech_featurizer.stride_ms
|
||||
|
||||
def process_utterance(self, audio_file, translation):
|
||||
"""Load, augment, featurize and normalize for speech data.
|
||||
|
||||
:param audio_file: Filepath or file object of audio file.
|
||||
:type audio_file: str | file
|
||||
:param translation: translation text.
|
||||
:type translation: str
|
||||
:return: Tuple of audio feature tensor and data of translation part,
|
||||
where translation part could be token ids or text.
|
||||
:rtype: tuple of (2darray, list)
|
||||
"""
|
||||
if isinstance(audio_file, str) and audio_file.startswith('tar:'):
|
||||
speech_segment = SpeechSegment.from_file(
|
||||
self._subfile_from_tar(audio_file), translation)
|
||||
else:
|
||||
speech_segment = SpeechSegment.from_file(audio_file, translation)
|
||||
|
||||
# audio augment
|
||||
self._augmentation_pipeline.transform_audio(speech_segment)
|
||||
|
||||
specgram, translation_part = self._speech_featurizer.featurize(
|
||||
speech_segment, self._keep_transcription_text)
|
||||
if self._normalizer:
|
||||
specgram = self._normalizer.apply(specgram)
|
||||
|
||||
# specgram augment
|
||||
specgram = self._augmentation_pipeline.transform_feature(specgram)
|
||||
return specgram, translation_part
|
||||
|
||||
def __call__(self, batch):
|
||||
"""batch examples
|
||||
|
||||
Args:
|
||||
batch ([List]): batch is (audio, text)
|
||||
audio (np.ndarray) shape (T, D)
|
||||
text (List[int] or str): shape (U,)
|
||||
|
||||
Returns:
|
||||
tuple(audio, text, audio_lens, text_lens): batched data.
|
||||
audio : (B, Tmax, D)
|
||||
audio_lens: (B)
|
||||
text : (B, Umax)
|
||||
text_lens: (B)
|
||||
"""
|
||||
audios = []
|
||||
audio_lens = []
|
||||
texts = []
|
||||
text_lens = []
|
||||
utts = []
|
||||
for utt, audio, text in batch:
|
||||
audio, text = self.process_utterance(audio, text)
|
||||
#utt
|
||||
utts.append(utt)
|
||||
# audio
|
||||
audios.append(audio) # [T, D]
|
||||
audio_lens.append(audio.shape[0])
|
||||
# text
|
||||
# for training, text is token ids
|
||||
# else text is string, convert to unicode ord
|
||||
tokens = []
|
||||
if self._keep_transcription_text:
|
||||
assert isinstance(text, str), (type(text), text)
|
||||
tokens = [ord(t) for t in text]
|
||||
else:
|
||||
tokens = text # token ids
|
||||
tokens = tokens if isinstance(tokens, np.ndarray) else np.array(
|
||||
tokens, dtype=np.int64)
|
||||
texts.append(tokens)
|
||||
text_lens.append(tokens.shape[0])
|
||||
|
||||
padded_audios = pad_sequence(
|
||||
audios, padding_value=0.0).astype(np.float32) #[B, T, D]
|
||||
audio_lens = np.array(audio_lens).astype(np.int64)
|
||||
padded_texts = pad_sequence(
|
||||
texts, padding_value=IGNORE_ID).astype(np.int64)
|
||||
text_lens = np.array(text_lens).astype(np.int64)
|
||||
return utts, padded_audios, audio_lens, padded_texts, text_lens
|
||||
|
||||
|
||||
class TripletSpeechCollator(SpeechCollator):
|
||||
def process_utterance(self, audio_file, translation, transcript):
|
||||
"""Load, augment, featurize and normalize for speech data.
|
||||
|
||||
:param audio_file: Filepath or file object of audio file.
|
||||
:type audio_file: str | file
|
||||
:param translation: translation text.
|
||||
:type translation: str
|
||||
:return: Tuple of audio feature tensor and data of translation part,
|
||||
where translation part could be token ids or text.
|
||||
:rtype: tuple of (2darray, list)
|
||||
"""
|
||||
if isinstance(audio_file, str) and audio_file.startswith('tar:'):
|
||||
speech_segment = SpeechSegment.from_file(
|
||||
self._subfile_from_tar(audio_file), translation)
|
||||
else:
|
||||
speech_segment = SpeechSegment.from_file(audio_file, translation)
|
||||
|
||||
# audio augment
|
||||
self._augmentation_pipeline.transform_audio(speech_segment)
|
||||
|
||||
specgram, translation_part = self._speech_featurizer.featurize(
|
||||
speech_segment, self._keep_transcription_text)
|
||||
transcript_part = self._speech_featurizer._text_featurizer.featurize(
|
||||
transcript)
|
||||
if self._normalizer:
|
||||
specgram = self._normalizer.apply(specgram)
|
||||
|
||||
# specgram augment
|
||||
specgram = self._augmentation_pipeline.transform_feature(specgram)
|
||||
return specgram, translation_part, transcript_part
|
||||
|
||||
def __call__(self, batch):
|
||||
"""batch examples
|
||||
|
||||
Args:
|
||||
batch ([List]): batch is (audio, text)
|
||||
audio (np.ndarray) shape (T, D)
|
||||
text (List[int] or str): shape (U,)
|
||||
|
||||
Returns:
|
||||
tuple(audio, text, audio_lens, text_lens): batched data.
|
||||
audio : (B, Tmax, D)
|
||||
audio_lens: (B)
|
||||
text : (B, Umax)
|
||||
text_lens: (B)
|
||||
"""
|
||||
audios = []
|
||||
audio_lens = []
|
||||
translation_text = []
|
||||
translation_text_lens = []
|
||||
transcription_text = []
|
||||
transcription_text_lens = []
|
||||
|
||||
utts = []
|
||||
for utt, audio, translation, transcription in batch:
|
||||
audio, translation, transcription = self.process_utterance(
|
||||
audio, translation, transcription)
|
||||
#utt
|
||||
utts.append(utt)
|
||||
# audio
|
||||
audios.append(audio) # [T, D]
|
||||
audio_lens.append(audio.shape[0])
|
||||
# text
|
||||
# for training, text is token ids
|
||||
# else text is string, convert to unicode ord
|
||||
tokens = [[], []]
|
||||
for idx, text in enumerate([translation, transcription]):
|
||||
if self._keep_transcription_text:
|
||||
assert isinstance(text, str), (type(text), text)
|
||||
tokens[idx] = [ord(t) for t in text]
|
||||
else:
|
||||
tokens[idx] = text # token ids
|
||||
tokens[idx] = tokens[idx] if isinstance(
|
||||
tokens[idx], np.ndarray) else np.array(
|
||||
tokens[idx], dtype=np.int64)
|
||||
translation_text.append(tokens[0])
|
||||
translation_text_lens.append(tokens[0].shape[0])
|
||||
transcription_text.append(tokens[1])
|
||||
transcription_text_lens.append(tokens[1].shape[0])
|
||||
|
||||
padded_audios = pad_sequence(
|
||||
audios, padding_value=0.0).astype(np.float32) #[B, T, D]
|
||||
audio_lens = np.array(audio_lens).astype(np.int64)
|
||||
padded_translation = pad_sequence(
|
||||
translation_text, padding_value=IGNORE_ID).astype(np.int64)
|
||||
translation_lens = np.array(translation_text_lens).astype(np.int64)
|
||||
padded_transcription = pad_sequence(
|
||||
transcription_text, padding_value=IGNORE_ID).astype(np.int64)
|
||||
transcription_lens = np.array(transcription_text_lens).astype(np.int64)
|
||||
return utts, padded_audios, audio_lens, (
|
||||
padded_translation, padded_transcription), (translation_lens,
|
||||
transcription_lens)
|
||||
|
||||
|
||||
class KaldiPrePorocessedCollator(SpeechCollator):
|
||||
@classmethod
|
||||
def params(cls, config: Optional[CfgNode]=None) -> CfgNode:
|
||||
default = CfgNode(
|
||||
dict(
|
||||
augmentation_config="",
|
||||
random_seed=0,
|
||||
unit_type="char",
|
||||
vocab_filepath="",
|
||||
spm_model_prefix="",
|
||||
feat_dim=0,
|
||||
stride_ms=10.0,
|
||||
keep_transcription_text=False))
|
||||
|
||||
if config is not None:
|
||||
config.merge_from_other_cfg(default)
|
||||
return default
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config):
|
||||
"""Build a SpeechCollator object from a config.
|
||||
|
||||
Args:
|
||||
config (yacs.config.CfgNode): configs object.
|
||||
|
||||
Returns:
|
||||
SpeechCollator: collator object.
|
||||
"""
|
||||
assert 'augmentation_config' in config.collator
|
||||
assert 'keep_transcription_text' in config.collator
|
||||
assert 'vocab_filepath' in config.collator
|
||||
assert config.collator
|
||||
|
||||
if isinstance(config.collator.augmentation_config, (str, bytes)):
|
||||
if config.collator.augmentation_config:
|
||||
aug_file = io.open(
|
||||
config.collator.augmentation_config,
|
||||
mode='r',
|
||||
encoding='utf8')
|
||||
else:
|
||||
aug_file = io.StringIO(initial_value='{}', newline='')
|
||||
else:
|
||||
aug_file = config.collator.augmentation_config
|
||||
assert isinstance(aug_file, io.StringIO)
|
||||
|
||||
speech_collator = cls(
|
||||
aug_file=aug_file,
|
||||
random_seed=0,
|
||||
unit_type=config.collator.unit_type,
|
||||
vocab_filepath=config.collator.vocab_filepath,
|
||||
spm_model_prefix=config.collator.spm_model_prefix,
|
||||
feat_dim=config.collator.feat_dim,
|
||||
stride_ms=config.collator.stride_ms,
|
||||
keep_transcription_text=config.collator.keep_transcription_text)
|
||||
return speech_collator
|
||||
|
||||
def __init__(self,
|
||||
aug_file,
|
||||
vocab_filepath,
|
||||
spm_model_prefix,
|
||||
random_seed=0,
|
||||
unit_type="char",
|
||||
feat_dim=0,
|
||||
stride_ms=10.0,
|
||||
keep_transcription_text=True):
|
||||
"""SpeechCollator Collator
|
||||
|
||||
Args:
|
||||
unit_type(str): token unit type, e.g. char, word, spm
|
||||
vocab_filepath (str): vocab file path.
|
||||
spm_model_prefix (str): spm model prefix, need if `unit_type` is spm.
|
||||
augmentation_config (str, optional): augmentation json str. Defaults to '{}'.
|
||||
random_seed (int, optional): for random generator. Defaults to 0.
|
||||
keep_transcription_text (bool, optional): True, when not in training mode, will not do tokenizer; Defaults to False.
|
||||
if ``keep_transcription_text`` is False, text is token ids else is raw string.
|
||||
|
||||
Do augmentations
|
||||
Padding audio features with zeros to make them have the same shape (or
|
||||
a user-defined shape) within one batch.
|
||||
"""
|
||||
self._keep_transcription_text = keep_transcription_text
|
||||
self._feat_dim = feat_dim
|
||||
self._stride_ms = stride_ms
|
||||
|
||||
self._local_data = TarLocalData(tar2info={}, tar2object={})
|
||||
self._augmentation_pipeline = AugmentationPipeline(
|
||||
augmentation_config=aug_file.read(), random_seed=random_seed)
|
||||
|
||||
self._text_featurizer = TextFeaturizer(unit_type, vocab_filepath,
|
||||
spm_model_prefix)
|
||||
|
||||
def process_utterance(self, audio_file, translation):
|
||||
"""Load, augment, featurize and normalize for speech data.
|
||||
|
||||
:param audio_file: Filepath or file object of kaldi processed feature.
|
||||
:type audio_file: str | file
|
||||
:param translation: Translation text.
|
||||
:type translation: str
|
||||
:return: Tuple of audio feature tensor and data of translation part,
|
||||
where translation part could be token ids or text.
|
||||
:rtype: tuple of (2darray, list)
|
||||
"""
|
||||
specgram = kaldiio.load_mat(audio_file)
|
||||
assert specgram.shape[
|
||||
1] == self._feat_dim, 'expect feat dim {}, but got {}'.format(
|
||||
self._feat_dim, specgram.shape[1])
|
||||
|
||||
# specgram augment
|
||||
specgram = self._augmentation_pipeline.transform_feature(specgram)
|
||||
|
||||
if self._keep_transcription_text:
|
||||
return specgram, translation
|
||||
else:
|
||||
text_ids = self._text_featurizer.featurize(translation)
|
||||
return specgram, text_ids
|
||||
|
||||
|
||||
class TripletKaldiPrePorocessedCollator(KaldiPrePorocessedCollator):
|
||||
def process_utterance(self, audio_file, translation, transcript):
|
||||
"""Load, augment, featurize and normalize for speech data.
|
||||
|
||||
:param audio_file: Filepath or file object of kali processed feature.
|
||||
:type audio_file: str | file
|
||||
:param translation: Translation text.
|
||||
:type translation: str
|
||||
:param transcript: Transcription text.
|
||||
:type transcript: str
|
||||
:return: Tuple of audio feature tensor and data of translation and transcription parts,
|
||||
where translation and transcription parts could be token ids or text.
|
||||
:rtype: tuple of (2darray, (list, list))
|
||||
"""
|
||||
specgram = kaldiio.load_mat(audio_file)
|
||||
assert specgram.shape[
|
||||
1] == self._feat_dim, 'expect feat dim {}, but got {}'.format(
|
||||
self._feat_dim, specgram.shape[1])
|
||||
|
||||
# specgram augment
|
||||
specgram = self._augmentation_pipeline.transform_feature(specgram)
|
||||
|
||||
if self._keep_transcription_text:
|
||||
return specgram, translation, transcript
|
||||
else:
|
||||
translation_text_ids = self._text_featurizer.featurize(translation)
|
||||
transcript_text_ids = self._text_featurizer.featurize(transcript)
|
||||
return specgram, translation_text_ids, transcript_text_ids
|
||||
|
||||
def __call__(self, batch):
|
||||
"""batch examples
|
||||
|
||||
Args:
|
||||
batch ([List]): batch is (audio, text)
|
||||
audio (np.ndarray) shape (T, D)
|
||||
translation (List[int] or str): shape (U,)
|
||||
transcription (List[int] or str): shape (V,)
|
||||
|
||||
Returns:
|
||||
tuple(audio, text, audio_lens, text_lens): batched data.
|
||||
audio : (B, Tmax, D)
|
||||
audio_lens: (B)
|
||||
translation_text : (B, Umax)
|
||||
translation_text_lens: (B)
|
||||
transcription_text : (B, Vmax)
|
||||
transcription_text_lens: (B)
|
||||
"""
|
||||
audios = []
|
||||
audio_lens = []
|
||||
translation_text = []
|
||||
translation_text_lens = []
|
||||
transcription_text = []
|
||||
transcription_text_lens = []
|
||||
|
||||
utts = []
|
||||
for utt, audio, translation, transcription in batch:
|
||||
audio, translation, transcription = self.process_utterance(
|
||||
audio, translation, transcription)
|
||||
#utt
|
||||
utts.append(utt)
|
||||
# audio
|
||||
audios.append(audio) # [T, D]
|
||||
audio_lens.append(audio.shape[0])
|
||||
# text
|
||||
# for training, text is token ids
|
||||
# else text is string, convert to unicode ord
|
||||
tokens = [[], []]
|
||||
for idx, text in enumerate([translation, transcription]):
|
||||
if self._keep_transcription_text:
|
||||
assert isinstance(text, str), (type(text), text)
|
||||
tokens[idx] = [ord(t) for t in text]
|
||||
else:
|
||||
tokens[idx] = text # token ids
|
||||
tokens[idx] = tokens[idx] if isinstance(
|
||||
tokens[idx], np.ndarray) else np.array(
|
||||
tokens[idx], dtype=np.int64)
|
||||
translation_text.append(tokens[0])
|
||||
translation_text_lens.append(tokens[0].shape[0])
|
||||
transcription_text.append(tokens[1])
|
||||
transcription_text_lens.append(tokens[1].shape[0])
|
||||
|
||||
padded_audios = pad_sequence(
|
||||
audios, padding_value=0.0).astype(np.float32) #[B, T, D]
|
||||
audio_lens = np.array(audio_lens).astype(np.int64)
|
||||
padded_translation = pad_sequence(
|
||||
translation_text, padding_value=IGNORE_ID).astype(np.int64)
|
||||
translation_lens = np.array(translation_text_lens).astype(np.int64)
|
||||
padded_transcription = pad_sequence(
|
||||
transcription_text, padding_value=IGNORE_ID).astype(np.int64)
|
||||
transcription_lens = np.array(transcription_text_lens).astype(np.int64)
|
||||
return utts, padded_audios, audio_lens, (
|
||||
padded_translation, padded_transcription), (translation_lens,
|
||||
transcription_lens)
|
||||
@ -0,0 +1,81 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import numpy as np
|
||||
|
||||
from deepspeech.io.utility import pad_list
|
||||
from deepspeech.utils.log import Log
|
||||
|
||||
__all__ = ["CustomConverter"]
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
|
||||
class CustomConverter():
|
||||
"""Custom batch converter.
|
||||
|
||||
Args:
|
||||
subsampling_factor (int): The subsampling factor.
|
||||
dtype (np.dtype): Data type to convert.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, subsampling_factor=1, dtype=np.float32):
|
||||
"""Construct a CustomConverter object."""
|
||||
self.subsampling_factor = subsampling_factor
|
||||
self.ignore_id = -1
|
||||
self.dtype = dtype
|
||||
|
||||
def __call__(self, batch):
|
||||
"""Transform a batch and send it to a device.
|
||||
|
||||
Args:
|
||||
batch (list): The batch to transform.
|
||||
|
||||
Returns:
|
||||
tuple(np.ndarray, nn.ndarray, nn.ndarray)
|
||||
|
||||
"""
|
||||
# batch should be located in list
|
||||
assert len(batch) == 1
|
||||
(xs, ys), utts = batch[0]
|
||||
assert xs[0] is not None, "please check Reader and Augmentation impl."
|
||||
|
||||
# perform subsampling
|
||||
if self.subsampling_factor > 1:
|
||||
xs = [x[::self.subsampling_factor, :] for x in xs]
|
||||
|
||||
# get batch of lengths of input sequences
|
||||
ilens = np.array([x.shape[0] for x in xs])
|
||||
|
||||
# perform padding and convert to tensor
|
||||
# currently only support real number
|
||||
if xs[0].dtype.kind == "c":
|
||||
xs_pad_real = pad_list([x.real for x in xs], 0).astype(self.dtype)
|
||||
xs_pad_imag = pad_list([x.imag for x in xs], 0).astype(self.dtype)
|
||||
# Note(kamo):
|
||||
# {'real': ..., 'imag': ...} will be changed to ComplexTensor in E2E.
|
||||
# Don't create ComplexTensor and give it E2E here
|
||||
# because torch.nn.DataParellel can't handle it.
|
||||
xs_pad = {"real": xs_pad_real, "imag": xs_pad_imag}
|
||||
else:
|
||||
xs_pad = pad_list(xs, 0).astype(self.dtype)
|
||||
|
||||
# NOTE: this is for multi-output (e.g., speech translation)
|
||||
ys_pad = pad_list(
|
||||
[np.array(y[0][:]) if isinstance(y, tuple) else y for y in ys],
|
||||
self.ignore_id)
|
||||
|
||||
olens = np.array(
|
||||
[y[0].shape[0] if isinstance(y, tuple) else y.shape[0] for y in ys])
|
||||
return utts, xs_pad, ilens, ys_pad, olens
|
||||
@ -0,0 +1,170 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Text
|
||||
|
||||
import numpy as np
|
||||
from paddle.io import DataLoader
|
||||
|
||||
from deepspeech.frontend.utility import read_manifest
|
||||
from deepspeech.io.batchfy import make_batchset
|
||||
from deepspeech.io.converter import CustomConverter
|
||||
from deepspeech.io.dataset import TransformDataset
|
||||
from deepspeech.io.reader import LoadInputsAndTargets
|
||||
from deepspeech.utils.log import Log
|
||||
|
||||
__all__ = ["BatchDataLoader"]
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
|
||||
def feat_dim_and_vocab_size(data_json: List[Dict[Text, Any]],
|
||||
mode: Text="asr",
|
||||
iaxis=0,
|
||||
oaxis=0):
|
||||
if mode == 'asr':
|
||||
feat_dim = data_json[0]['input'][oaxis]['shape'][1]
|
||||
vocab_size = data_json[0]['output'][oaxis]['shape'][1]
|
||||
else:
|
||||
raise ValueError(f"{mode} mode not support!")
|
||||
return feat_dim, vocab_size
|
||||
|
||||
|
||||
def batch_collate(x):
|
||||
"""de-minibatch, since user compose batch.
|
||||
|
||||
Args:
|
||||
x (List[Tuple]): [(utts, xs, ilens, ys, olens)]
|
||||
|
||||
Returns:
|
||||
Tuple: (utts, xs, ilens, ys, olens)
|
||||
"""
|
||||
return x[0]
|
||||
|
||||
|
||||
class BatchDataLoader():
|
||||
def __init__(self,
|
||||
json_file: str,
|
||||
train_mode: bool,
|
||||
sortagrad: bool=False,
|
||||
batch_size: int=0,
|
||||
maxlen_in: float=float('inf'),
|
||||
maxlen_out: float=float('inf'),
|
||||
minibatches: int=0,
|
||||
mini_batch_size: int=1,
|
||||
batch_count: str='auto',
|
||||
batch_bins: int=0,
|
||||
batch_frames_in: int=0,
|
||||
batch_frames_out: int=0,
|
||||
batch_frames_inout: int=0,
|
||||
preprocess_conf=None,
|
||||
n_iter_processes: int=1,
|
||||
subsampling_factor: int=1,
|
||||
num_encs: int=1):
|
||||
self.json_file = json_file
|
||||
self.train_mode = train_mode
|
||||
self.use_sortagrad = sortagrad == -1 or sortagrad > 0
|
||||
self.batch_size = batch_size
|
||||
self.maxlen_in = maxlen_in
|
||||
self.maxlen_out = maxlen_out
|
||||
self.batch_count = batch_count
|
||||
self.batch_bins = batch_bins
|
||||
self.batch_frames_in = batch_frames_in
|
||||
self.batch_frames_out = batch_frames_out
|
||||
self.batch_frames_inout = batch_frames_inout
|
||||
self.subsampling_factor = subsampling_factor
|
||||
self.num_encs = num_encs
|
||||
self.preprocess_conf = preprocess_conf
|
||||
self.n_iter_processes = n_iter_processes
|
||||
|
||||
# read json data
|
||||
self.data_json = read_manifest(json_file)
|
||||
self.feat_dim, self.vocab_size = feat_dim_and_vocab_size(
|
||||
self.data_json, mode='asr')
|
||||
|
||||
# make minibatch list (variable length)
|
||||
self.minibaches = make_batchset(
|
||||
self.data_json,
|
||||
batch_size,
|
||||
maxlen_in,
|
||||
maxlen_out,
|
||||
minibatches, # for debug
|
||||
min_batch_size=mini_batch_size,
|
||||
shortest_first=self.use_sortagrad,
|
||||
count=batch_count,
|
||||
batch_bins=batch_bins,
|
||||
batch_frames_in=batch_frames_in,
|
||||
batch_frames_out=batch_frames_out,
|
||||
batch_frames_inout=batch_frames_inout,
|
||||
iaxis=0,
|
||||
oaxis=0, )
|
||||
|
||||
# data reader
|
||||
self.reader = LoadInputsAndTargets(
|
||||
mode="asr",
|
||||
load_output=True,
|
||||
preprocess_conf=preprocess_conf,
|
||||
preprocess_args={"train":
|
||||
train_mode}, # Switch the mode of preprocessing
|
||||
)
|
||||
|
||||
# Setup a converter
|
||||
if num_encs == 1:
|
||||
self.converter = CustomConverter(
|
||||
subsampling_factor=subsampling_factor, dtype=np.float32)
|
||||
else:
|
||||
assert NotImplementedError("not impl CustomConverterMulEnc.")
|
||||
|
||||
# hack to make batchsize argument as 1
|
||||
# actual bathsize is included in a list
|
||||
# default collate function converts numpy array to pytorch tensor
|
||||
# we used an empty collate function instead which returns list
|
||||
self.dataset = TransformDataset(self.minibaches, self.converter,
|
||||
self.reader)
|
||||
|
||||
self.dataloader = DataLoader(
|
||||
dataset=self.dataset,
|
||||
batch_size=1,
|
||||
shuffle=not self.use_sortagrad if self.train_mode else False,
|
||||
collate_fn=batch_collate,
|
||||
num_workers=self.n_iter_processes, )
|
||||
|
||||
def __repr__(self):
|
||||
echo = f"<{self.__class__.__module__}.{self.__class__.__name__} object at {hex(id(self))}> "
|
||||
echo += f"train_mode: {self.train_mode}, "
|
||||
echo += f"sortagrad: {self.use_sortagrad}, "
|
||||
echo += f"batch_size: {self.batch_size}, "
|
||||
echo += f"maxlen_in: {self.maxlen_in}, "
|
||||
echo += f"maxlen_out: {self.maxlen_out}, "
|
||||
echo += f"batch_count: {self.batch_count}, "
|
||||
echo += f"batch_bins: {self.batch_bins}, "
|
||||
echo += f"batch_frames_in: {self.batch_frames_in}, "
|
||||
echo += f"batch_frames_out: {self.batch_frames_out}, "
|
||||
echo += f"batch_frames_inout: {self.batch_frames_inout}, "
|
||||
echo += f"subsampling_factor: {self.subsampling_factor}, "
|
||||
echo += f"num_encs: {self.num_encs}, "
|
||||
echo += f"num_workers: {self.n_iter_processes}, "
|
||||
echo += f"file: {self.json_file}"
|
||||
return echo
|
||||
|
||||
def __len__(self):
|
||||
return len(self.dataloader)
|
||||
|
||||
def __iter__(self):
|
||||
return self.dataloader.__iter__()
|
||||
|
||||
def __call__(self):
|
||||
return self.__iter__()
|
||||
@ -0,0 +1,410 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from collections import OrderedDict
|
||||
|
||||
import kaldiio
|
||||
import numpy as np
|
||||
import soundfile
|
||||
|
||||
from deepspeech.frontend.augmentor.augmentation import AugmentationPipeline
|
||||
from deepspeech.utils.log import Log
|
||||
|
||||
__all__ = ["LoadInputsAndTargets"]
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
|
||||
class LoadInputsAndTargets():
|
||||
"""Create a mini-batch from a list of dicts
|
||||
|
||||
>>> batch = [('utt1',
|
||||
... dict(input=[dict(feat='some.ark:123',
|
||||
... filetype='mat',
|
||||
... name='input1',
|
||||
... shape=[100, 80])],
|
||||
... output=[dict(tokenid='1 2 3 4',
|
||||
... name='target1',
|
||||
... shape=[4, 31])]]))
|
||||
>>> l = LoadInputsAndTargets()
|
||||
>>> feat, target = l(batch)
|
||||
|
||||
:param: str mode: Specify the task mode, "asr" or "tts"
|
||||
:param: str preprocess_conf: The path of a json file for pre-processing
|
||||
:param: bool load_input: If False, not to load the input data
|
||||
:param: bool load_output: If False, not to load the output data
|
||||
:param: bool sort_in_input_length: Sort the mini-batch in descending order
|
||||
of the input length
|
||||
:param: bool use_speaker_embedding: Used for tts mode only
|
||||
:param: bool use_second_target: Used for tts mode only
|
||||
:param: dict preprocess_args: Set some optional arguments for preprocessing
|
||||
:param: Optional[dict] preprocess_args: Used for tts mode only
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mode="asr",
|
||||
preprocess_conf=None,
|
||||
load_input=True,
|
||||
load_output=True,
|
||||
sort_in_input_length=True,
|
||||
preprocess_args=None,
|
||||
keep_all_data_on_mem=False, ):
|
||||
self._loaders = {}
|
||||
|
||||
if mode not in ["asr"]:
|
||||
raise ValueError("Only asr are allowed: mode={}".format(mode))
|
||||
|
||||
if preprocess_conf is not None:
|
||||
with open(preprocess_conf, 'r') as fin:
|
||||
self.preprocessing = AugmentationPipeline(fin.read())
|
||||
logger.warning(
|
||||
"[Experimental feature] Some preprocessing will be done "
|
||||
"for the mini-batch creation using {}".format(
|
||||
self.preprocessing))
|
||||
else:
|
||||
# If conf doesn't exist, this function don't touch anything.
|
||||
self.preprocessing = None
|
||||
|
||||
self.mode = mode
|
||||
self.load_output = load_output
|
||||
self.load_input = load_input
|
||||
self.sort_in_input_length = sort_in_input_length
|
||||
if preprocess_args is None:
|
||||
self.preprocess_args = {}
|
||||
else:
|
||||
assert isinstance(preprocess_args, dict), type(preprocess_args)
|
||||
self.preprocess_args = dict(preprocess_args)
|
||||
|
||||
self.keep_all_data_on_mem = keep_all_data_on_mem
|
||||
|
||||
def __call__(self, batch, return_uttid=False):
|
||||
"""Function to load inputs and targets from list of dicts
|
||||
|
||||
:param List[Tuple[str, dict]] batch: list of dict which is subset of
|
||||
loaded data.json
|
||||
:param bool return_uttid: return utterance ID information for visualization
|
||||
:return: list of input token id sequences [(L_1), (L_2), ..., (L_B)]
|
||||
:return: list of input feature sequences
|
||||
[(T_1, D), (T_2, D), ..., (T_B, D)]
|
||||
:rtype: list of float ndarray
|
||||
:return: list of target token id sequences [(L_1), (L_2), ..., (L_B)]
|
||||
:rtype: list of int ndarray
|
||||
|
||||
"""
|
||||
x_feats_dict = OrderedDict() # OrderedDict[str, List[np.ndarray]]
|
||||
y_feats_dict = OrderedDict() # OrderedDict[str, List[np.ndarray]]
|
||||
uttid_list = [] # List[str]
|
||||
|
||||
for uttid, info in batch:
|
||||
uttid_list.append(uttid)
|
||||
|
||||
if self.load_input:
|
||||
# Note(kamo): This for-loop is for multiple inputs
|
||||
for idx, inp in enumerate(info["input"]):
|
||||
# {"input":
|
||||
# [{"feat": "some/path.h5:F01_050C0101_PED_REAL",
|
||||
# "filetype": "hdf5",
|
||||
# "name": "input1", ...}], ...}
|
||||
x = self._get_from_loader(
|
||||
filepath=inp["feat"],
|
||||
filetype=inp.get("filetype", "mat"))
|
||||
x_feats_dict.setdefault(inp["name"], []).append(x)
|
||||
|
||||
if self.load_output:
|
||||
for idx, inp in enumerate(info["output"]):
|
||||
if "tokenid" in inp:
|
||||
# ======= Legacy format for output =======
|
||||
# {"output": [{"tokenid": "1 2 3 4"}])
|
||||
x = np.fromiter(
|
||||
map(int, inp["tokenid"].split()), dtype=np.int64)
|
||||
else:
|
||||
# ======= New format =======
|
||||
# {"input":
|
||||
# [{"feat": "some/path.h5:F01_050C0101_PED_REAL",
|
||||
# "filetype": "hdf5",
|
||||
# "name": "target1", ...}], ...}
|
||||
x = self._get_from_loader(
|
||||
filepath=inp["feat"],
|
||||
filetype=inp.get("filetype", "mat"))
|
||||
|
||||
y_feats_dict.setdefault(inp["name"], []).append(x)
|
||||
|
||||
if self.mode == "asr":
|
||||
return_batch, uttid_list = self._create_batch_asr(
|
||||
x_feats_dict, y_feats_dict, uttid_list)
|
||||
else:
|
||||
raise NotImplementedError(self.mode)
|
||||
|
||||
if self.preprocessing is not None:
|
||||
# Apply pre-processing all input features
|
||||
for x_name in return_batch.keys():
|
||||
if x_name.startswith("input"):
|
||||
return_batch[x_name] = self.preprocessing(
|
||||
return_batch[x_name], uttid_list,
|
||||
**self.preprocess_args)
|
||||
|
||||
if return_uttid:
|
||||
return tuple(return_batch.values()), uttid_list
|
||||
|
||||
# Doesn't return the names now.
|
||||
return tuple(return_batch.values())
|
||||
|
||||
def _create_batch_asr(self, x_feats_dict, y_feats_dict, uttid_list):
|
||||
"""Create a OrderedDict for the mini-batch
|
||||
|
||||
:param OrderedDict x_feats_dict:
|
||||
e.g. {"input1": [ndarray, ndarray, ...],
|
||||
"input2": [ndarray, ndarray, ...]}
|
||||
:param OrderedDict y_feats_dict:
|
||||
e.g. {"target1": [ndarray, ndarray, ...],
|
||||
"target2": [ndarray, ndarray, ...]}
|
||||
:param: List[str] uttid_list:
|
||||
Give uttid_list to sort in the same order as the mini-batch
|
||||
:return: batch, uttid_list
|
||||
:rtype: Tuple[OrderedDict, List[str]]
|
||||
"""
|
||||
# handle single-input and multi-input (paralell) asr mode
|
||||
xs = list(x_feats_dict.values())
|
||||
|
||||
if self.load_output:
|
||||
ys = list(y_feats_dict.values())
|
||||
assert len(xs[0]) == len(ys[0]), (len(xs[0]), len(ys[0]))
|
||||
|
||||
# get index of non-zero length samples
|
||||
nonzero_idx = list(
|
||||
filter(lambda i: len(ys[0][i]) > 0, range(len(ys[0]))))
|
||||
for n in range(1, len(y_feats_dict)):
|
||||
nonzero_idx = filter(lambda i: len(ys[n][i]) > 0, nonzero_idx)
|
||||
else:
|
||||
# Note(kamo): Be careful not to make nonzero_idx to a generator
|
||||
nonzero_idx = list(range(len(xs[0])))
|
||||
|
||||
if self.sort_in_input_length:
|
||||
# sort in input lengths based on the first input
|
||||
nonzero_sorted_idx = sorted(
|
||||
nonzero_idx, key=lambda i: -len(xs[0][i]))
|
||||
else:
|
||||
nonzero_sorted_idx = nonzero_idx
|
||||
|
||||
if len(nonzero_sorted_idx) != len(xs[0]):
|
||||
logger.warning(
|
||||
"Target sequences include empty tokenid (batch {} -> {}).".
|
||||
format(len(xs[0]), len(nonzero_sorted_idx)))
|
||||
|
||||
# remove zero-length samples
|
||||
xs = [[x[i] for i in nonzero_sorted_idx] for x in xs]
|
||||
uttid_list = [uttid_list[i] for i in nonzero_sorted_idx]
|
||||
|
||||
x_names = list(x_feats_dict.keys())
|
||||
if self.load_output:
|
||||
ys = [[y[i] for i in nonzero_sorted_idx] for y in ys]
|
||||
y_names = list(y_feats_dict.keys())
|
||||
|
||||
# Keeping x_name and y_name, e.g. input1, for future extension
|
||||
return_batch = OrderedDict([
|
||||
* [(x_name, x) for x_name, x in zip(x_names, xs)],
|
||||
* [(y_name, y) for y_name, y in zip(y_names, ys)],
|
||||
])
|
||||
else:
|
||||
return_batch = OrderedDict(
|
||||
[(x_name, x) for x_name, x in zip(x_names, xs)])
|
||||
return return_batch, uttid_list
|
||||
|
||||
def _get_from_loader(self, filepath, filetype):
|
||||
"""Return ndarray
|
||||
|
||||
In order to make the fds to be opened only at the first referring,
|
||||
the loader are stored in self._loaders
|
||||
|
||||
>>> ndarray = loader.get_from_loader(
|
||||
... 'some/path.h5:F01_050C0101_PED_REAL', filetype='hdf5')
|
||||
|
||||
:param: str filepath:
|
||||
:param: str filetype:
|
||||
:return:
|
||||
:rtype: np.ndarray
|
||||
"""
|
||||
if filetype == "hdf5":
|
||||
# e.g.
|
||||
# {"input": [{"feat": "some/path.h5:F01_050C0101_PED_REAL",
|
||||
# "filetype": "hdf5",
|
||||
# -> filepath = "some/path.h5", key = "F01_050C0101_PED_REAL"
|
||||
filepath, key = filepath.split(":", 1)
|
||||
|
||||
loader = self._loaders.get(filepath)
|
||||
if loader is None:
|
||||
# To avoid disk access, create loader only for the first time
|
||||
loader = h5py.File(filepath, "r")
|
||||
self._loaders[filepath] = loader
|
||||
return loader[key][()]
|
||||
elif filetype == "sound.hdf5":
|
||||
# e.g.
|
||||
# {"input": [{"feat": "some/path.h5:F01_050C0101_PED_REAL",
|
||||
# "filetype": "sound.hdf5",
|
||||
# -> filepath = "some/path.h5", key = "F01_050C0101_PED_REAL"
|
||||
filepath, key = filepath.split(":", 1)
|
||||
|
||||
loader = self._loaders.get(filepath)
|
||||
if loader is None:
|
||||
# To avoid disk access, create loader only for the first time
|
||||
loader = SoundHDF5File(filepath, "r", dtype="int16")
|
||||
self._loaders[filepath] = loader
|
||||
array, rate = loader[key]
|
||||
return array
|
||||
elif filetype == "sound":
|
||||
# e.g.
|
||||
# {"input": [{"feat": "some/path.wav",
|
||||
# "filetype": "sound"},
|
||||
# Assume PCM16
|
||||
if not self.keep_all_data_on_mem:
|
||||
array, _ = soundfile.read(filepath, dtype="int16")
|
||||
return array
|
||||
if filepath not in self._loaders:
|
||||
array, _ = soundfile.read(filepath, dtype="int16")
|
||||
self._loaders[filepath] = array
|
||||
return self._loaders[filepath]
|
||||
elif filetype == "npz":
|
||||
# e.g.
|
||||
# {"input": [{"feat": "some/path.npz:F01_050C0101_PED_REAL",
|
||||
# "filetype": "npz",
|
||||
filepath, key = filepath.split(":", 1)
|
||||
|
||||
loader = self._loaders.get(filepath)
|
||||
if loader is None:
|
||||
# To avoid disk access, create loader only for the first time
|
||||
loader = np.load(filepath)
|
||||
self._loaders[filepath] = loader
|
||||
return loader[key]
|
||||
elif filetype == "npy":
|
||||
# e.g.
|
||||
# {"input": [{"feat": "some/path.npy",
|
||||
# "filetype": "npy"},
|
||||
if not self.keep_all_data_on_mem:
|
||||
return np.load(filepath)
|
||||
if filepath not in self._loaders:
|
||||
self._loaders[filepath] = np.load(filepath)
|
||||
return self._loaders[filepath]
|
||||
elif filetype in ["mat", "vec"]:
|
||||
# e.g.
|
||||
# {"input": [{"feat": "some/path.ark:123",
|
||||
# "filetype": "mat"}]},
|
||||
# In this case, "123" indicates the starting points of the matrix
|
||||
# load_mat can load both matrix and vector
|
||||
if not self.keep_all_data_on_mem:
|
||||
return kaldiio.load_mat(filepath)
|
||||
if filepath not in self._loaders:
|
||||
self._loaders[filepath] = kaldiio.load_mat(filepath)
|
||||
return self._loaders[filepath]
|
||||
elif filetype == "scp":
|
||||
# e.g.
|
||||
# {"input": [{"feat": "some/path.scp:F01_050C0101_PED_REAL",
|
||||
# "filetype": "scp",
|
||||
filepath, key = filepath.split(":", 1)
|
||||
loader = self._loaders.get(filepath)
|
||||
if loader is None:
|
||||
# To avoid disk access, create loader only for the first time
|
||||
loader = kaldiio.load_scp(filepath)
|
||||
self._loaders[filepath] = loader
|
||||
return loader[key]
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"Not supported: loader_type={}".format(filetype))
|
||||
|
||||
|
||||
class SoundHDF5File():
|
||||
"""Collecting sound files to a HDF5 file
|
||||
|
||||
>>> f = SoundHDF5File('a.flac.h5', mode='a')
|
||||
>>> array = np.random.randint(0, 100, 100, dtype=np.int16)
|
||||
>>> f['id'] = (array, 16000)
|
||||
>>> array, rate = f['id']
|
||||
|
||||
|
||||
:param: str filepath:
|
||||
:param: str mode:
|
||||
:param: str format: The type used when saving wav. flac, nist, htk, etc.
|
||||
:param: str dtype:
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
filepath,
|
||||
mode="r+",
|
||||
format=None,
|
||||
dtype="int16",
|
||||
**kwargs):
|
||||
self.filepath = filepath
|
||||
self.mode = mode
|
||||
self.dtype = dtype
|
||||
|
||||
self.file = h5py.File(filepath, mode, **kwargs)
|
||||
if format is None:
|
||||
# filepath = a.flac.h5 -> format = flac
|
||||
second_ext = os.path.splitext(os.path.splitext(filepath)[0])[1]
|
||||
format = second_ext[1:]
|
||||
if format.upper() not in soundfile.available_formats():
|
||||
# If not found, flac is selected
|
||||
format = "flac"
|
||||
|
||||
# This format affects only saving
|
||||
self.format = format
|
||||
|
||||
def __repr__(self):
|
||||
return '<SoundHDF5 file "{}" (mode {}, format {}, type {})>'.format(
|
||||
self.filepath, self.mode, self.format, self.dtype)
|
||||
|
||||
def create_dataset(self, name, shape=None, data=None, **kwds):
|
||||
f = io.BytesIO()
|
||||
array, rate = data
|
||||
soundfile.write(f, array, rate, format=self.format)
|
||||
self.file.create_dataset(
|
||||
name, shape=shape, data=np.void(f.getvalue()), **kwds)
|
||||
|
||||
def __setitem__(self, name, data):
|
||||
self.create_dataset(name, data=data)
|
||||
|
||||
def __getitem__(self, key):
|
||||
data = self.file[key][()]
|
||||
f = io.BytesIO(data.tobytes())
|
||||
array, rate = soundfile.read(f, dtype=self.dtype)
|
||||
return array, rate
|
||||
|
||||
def keys(self):
|
||||
return self.file.keys()
|
||||
|
||||
def values(self):
|
||||
for k in self.file:
|
||||
yield self[k]
|
||||
|
||||
def items(self):
|
||||
for k in self.file:
|
||||
yield k, self[k]
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.file)
|
||||
|
||||
def __contains__(self, item):
|
||||
return item in self.file
|
||||
|
||||
def __len__(self, item):
|
||||
return len(self.file)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.file.close()
|
||||
|
||||
def close(self):
|
||||
self.file.close()
|
||||
@ -0,0 +1,17 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from .deepspeech2 import DeepSpeech2InferModel
|
||||
from .deepspeech2 import DeepSpeech2Model
|
||||
|
||||
__all__ = ['DeepSpeech2Model', 'DeepSpeech2InferModel']
|
||||
@ -0,0 +1,163 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from paddle import nn
|
||||
from paddle.nn import functional as F
|
||||
|
||||
from deepspeech.modules.activation import brelu
|
||||
from deepspeech.modules.mask import make_non_pad_mask
|
||||
from deepspeech.utils.log import Log
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
__all__ = ['ConvStack', "conv_output_size"]
|
||||
|
||||
|
||||
def conv_output_size(I, F, P, S):
|
||||
# https://stanford.edu/~shervine/teaching/cs-230/cheatsheet-convolutional-neural-networks#hyperparameters
|
||||
# Output size after Conv:
|
||||
# By noting I the length of the input volume size,
|
||||
# F the length of the filter,
|
||||
# P the amount of zero padding,
|
||||
# S the stride,
|
||||
# then the output size O of the feature map along that dimension is given by:
|
||||
# O = (I - F + Pstart + Pend) // S + 1
|
||||
# When Pstart == Pend == P, we can replace Pstart + Pend by 2P.
|
||||
# When Pstart == Pend == 0
|
||||
# O = (I - F - S) // S
|
||||
# https://iq.opengenus.org/output-size-of-convolution/
|
||||
# Output height = (Input height + padding height top + padding height bottom - kernel height) / (stride height) + 1
|
||||
# Output width = (Output width + padding width right + padding width left - kernel width) / (stride width) + 1
|
||||
return (I - F + 2 * P - S) // S
|
||||
|
||||
|
||||
class ConvBn(nn.Layer):
|
||||
"""Convolution layer with batch normalization.
|
||||
|
||||
:param kernel_size: The x dimension of a filter kernel. Or input a tuple for
|
||||
two image dimension.
|
||||
:type kernel_size: int|tuple|list
|
||||
:param num_channels_in: Number of input channels.
|
||||
:type num_channels_in: int
|
||||
:param num_channels_out: Number of output channels.
|
||||
:type num_channels_out: int
|
||||
:param stride: The x dimension of the stride. Or input a tuple for two
|
||||
image dimension.
|
||||
:type stride: int|tuple|list
|
||||
:param padding: The x dimension of the padding. Or input a tuple for two
|
||||
image dimension.
|
||||
:type padding: int|tuple|list
|
||||
:param act: Activation type, relu|brelu
|
||||
:type act: string
|
||||
:return: Batch norm layer after convolution layer.
|
||||
:rtype: Variable
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, num_channels_in, num_channels_out, kernel_size, stride,
|
||||
padding, act):
|
||||
|
||||
super().__init__()
|
||||
assert len(kernel_size) == 2
|
||||
assert len(stride) == 2
|
||||
assert len(padding) == 2
|
||||
self.kernel_size = kernel_size
|
||||
self.stride = stride
|
||||
self.padding = padding
|
||||
|
||||
self.conv = nn.Conv2D(
|
||||
num_channels_in,
|
||||
num_channels_out,
|
||||
kernel_size=kernel_size,
|
||||
stride=stride,
|
||||
padding=padding,
|
||||
weight_attr=None,
|
||||
bias_attr=False,
|
||||
data_format='NCHW')
|
||||
|
||||
self.bn = nn.BatchNorm2D(
|
||||
num_channels_out,
|
||||
weight_attr=None,
|
||||
bias_attr=None,
|
||||
data_format='NCHW')
|
||||
self.act = F.relu if act == 'relu' else brelu
|
||||
|
||||
def forward(self, x, x_len):
|
||||
"""
|
||||
x(Tensor): audio, shape [B, C, D, T]
|
||||
"""
|
||||
x = self.conv(x)
|
||||
x = self.bn(x)
|
||||
x = self.act(x)
|
||||
|
||||
x_len = (x_len - self.kernel_size[1] + 2 * self.padding[1]
|
||||
) // self.stride[1] + 1
|
||||
|
||||
# reset padding part to 0
|
||||
masks = make_non_pad_mask(x_len) #[B, T]
|
||||
masks = masks.unsqueeze(1).unsqueeze(1) # [B, 1, 1, T]
|
||||
# https://github.com/PaddlePaddle/Paddle/pull/29265
|
||||
# rhs will type promote to lhs
|
||||
x = x * masks
|
||||
return x, x_len
|
||||
|
||||
|
||||
class ConvStack(nn.Layer):
|
||||
"""Convolution group with stacked convolution layers.
|
||||
|
||||
:param feat_size: audio feature dim.
|
||||
:type feat_size: int
|
||||
:param num_stacks: Number of stacked convolution layers.
|
||||
:type num_stacks: int
|
||||
"""
|
||||
|
||||
def __init__(self, feat_size, num_stacks):
|
||||
super().__init__()
|
||||
self.feat_size = feat_size # D
|
||||
self.num_stacks = num_stacks
|
||||
|
||||
self.conv_in = ConvBn(
|
||||
num_channels_in=1,
|
||||
num_channels_out=32,
|
||||
kernel_size=(41, 11), #[D, T]
|
||||
stride=(2, 3),
|
||||
padding=(20, 5),
|
||||
act='brelu')
|
||||
|
||||
out_channel = 32
|
||||
convs = [
|
||||
ConvBn(
|
||||
num_channels_in=32,
|
||||
num_channels_out=out_channel,
|
||||
kernel_size=(21, 11),
|
||||
stride=(2, 1),
|
||||
padding=(10, 5),
|
||||
act='brelu') for i in range(num_stacks - 1)
|
||||
]
|
||||
self.conv_stack = nn.LayerList(convs)
|
||||
|
||||
# conv output feat_dim
|
||||
output_height = (feat_size - 1) // 2 + 1
|
||||
for i in range(self.num_stacks - 1):
|
||||
output_height = (output_height - 1) // 2 + 1
|
||||
self.output_height = out_channel * output_height
|
||||
|
||||
def forward(self, x, x_len):
|
||||
"""
|
||||
x: shape [B, C, D, T]
|
||||
x_len : shape [B]
|
||||
"""
|
||||
x, x_len = self.conv_in(x, x_len)
|
||||
for i, conv in enumerate(self.conv_stack):
|
||||
x, x_len = conv(x, x_len)
|
||||
return x, x_len
|
||||
@ -0,0 +1,315 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import math
|
||||
|
||||
import paddle
|
||||
from paddle import nn
|
||||
from paddle.nn import functional as F
|
||||
from paddle.nn import initializer as I
|
||||
|
||||
from deepspeech.modules.activation import brelu
|
||||
from deepspeech.modules.mask import make_non_pad_mask
|
||||
from deepspeech.utils.log import Log
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
__all__ = ['RNNStack']
|
||||
|
||||
|
||||
class RNNCell(nn.RNNCellBase):
|
||||
r"""
|
||||
Elman RNN (SimpleRNN) cell. Given the inputs and previous states, it
|
||||
computes the outputs and updates states.
|
||||
The formula used is as follows:
|
||||
.. math::
|
||||
h_{t} & = act(x_{t} + b_{ih} + W_{hh}h_{t-1} + b_{hh})
|
||||
y_{t} & = h_{t}
|
||||
|
||||
where :math:`act` is for :attr:`activation`.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
hidden_size: int,
|
||||
activation="tanh",
|
||||
weight_ih_attr=None,
|
||||
weight_hh_attr=None,
|
||||
bias_ih_attr=None,
|
||||
bias_hh_attr=None,
|
||||
name=None):
|
||||
super().__init__()
|
||||
std = 1.0 / math.sqrt(hidden_size)
|
||||
self.weight_hh = self.create_parameter(
|
||||
(hidden_size, hidden_size),
|
||||
weight_hh_attr,
|
||||
default_initializer=I.Uniform(-std, std))
|
||||
self.bias_ih = None
|
||||
self.bias_hh = self.create_parameter(
|
||||
(hidden_size, ),
|
||||
bias_hh_attr,
|
||||
is_bias=True,
|
||||
default_initializer=I.Uniform(-std, std))
|
||||
|
||||
self.hidden_size = hidden_size
|
||||
if activation not in ["tanh", "relu", "brelu"]:
|
||||
raise ValueError(
|
||||
"activation for SimpleRNNCell should be tanh or relu, "
|
||||
"but get {}".format(activation))
|
||||
self.activation = activation
|
||||
self._activation_fn = paddle.tanh \
|
||||
if activation == "tanh" \
|
||||
else F.relu
|
||||
if activation == 'brelu':
|
||||
self._activation_fn = brelu
|
||||
|
||||
def forward(self, inputs, states=None):
|
||||
if states is None:
|
||||
states = self.get_initial_states(inputs, self.state_shape)
|
||||
pre_h = states
|
||||
i2h = inputs
|
||||
if self.bias_ih is not None:
|
||||
i2h += self.bias_ih
|
||||
h2h = paddle.matmul(pre_h, self.weight_hh, transpose_y=True)
|
||||
if self.bias_hh is not None:
|
||||
h2h += self.bias_hh
|
||||
h = self._activation_fn(i2h + h2h)
|
||||
return h, h
|
||||
|
||||
@property
|
||||
def state_shape(self):
|
||||
return (self.hidden_size, )
|
||||
|
||||
|
||||
class GRUCell(nn.RNNCellBase):
|
||||
r"""
|
||||
Gated Recurrent Unit (GRU) RNN cell. Given the inputs and previous states,
|
||||
it computes the outputs and updates states.
|
||||
The formula for GRU used is as follows:
|
||||
.. math::
|
||||
r_{t} & = \sigma(W_{ir}x_{t} + b_{ir} + W_{hr}h_{t-1} + b_{hr})
|
||||
z_{t} & = \sigma(W_{iz}x_{t} + b_{iz} + W_{hz}h_{t-1} + b_{hz})
|
||||
\widetilde{h}_{t} & = \tanh(W_{ic}x_{t} + b_{ic} + r_{t} * (W_{hc}h_{t-1} + b_{hc}))
|
||||
h_{t} & = z_{t} * h_{t-1} + (1 - z_{t}) * \widetilde{h}_{t}
|
||||
y_{t} & = h_{t}
|
||||
|
||||
where :math:`\sigma` is the sigmoid fucntion, and * is the elemetwise
|
||||
multiplication operator.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
input_size: int,
|
||||
hidden_size: int,
|
||||
weight_ih_attr=None,
|
||||
weight_hh_attr=None,
|
||||
bias_ih_attr=None,
|
||||
bias_hh_attr=None,
|
||||
name=None):
|
||||
super().__init__()
|
||||
std = 1.0 / math.sqrt(hidden_size)
|
||||
self.weight_hh = self.create_parameter(
|
||||
(3 * hidden_size, hidden_size),
|
||||
weight_hh_attr,
|
||||
default_initializer=I.Uniform(-std, std))
|
||||
self.bias_ih = None
|
||||
self.bias_hh = self.create_parameter(
|
||||
(3 * hidden_size, ),
|
||||
bias_hh_attr,
|
||||
is_bias=True,
|
||||
default_initializer=I.Uniform(-std, std))
|
||||
|
||||
self.hidden_size = hidden_size
|
||||
self.input_size = input_size
|
||||
self._gate_activation = F.sigmoid
|
||||
self._activation = paddle.tanh
|
||||
|
||||
def forward(self, inputs, states=None):
|
||||
if states is None:
|
||||
states = self.get_initial_states(inputs, self.state_shape)
|
||||
|
||||
pre_hidden = states
|
||||
x_gates = inputs
|
||||
if self.bias_ih is not None:
|
||||
x_gates = x_gates + self.bias_ih
|
||||
h_gates = paddle.matmul(pre_hidden, self.weight_hh, transpose_y=True)
|
||||
if self.bias_hh is not None:
|
||||
h_gates = h_gates + self.bias_hh
|
||||
|
||||
x_r, x_z, x_c = paddle.split(x_gates, num_or_sections=3, axis=1)
|
||||
h_r, h_z, h_c = paddle.split(h_gates, num_or_sections=3, axis=1)
|
||||
|
||||
r = self._gate_activation(x_r + h_r)
|
||||
z = self._gate_activation(x_z + h_z)
|
||||
c = self._activation(x_c + r * h_c) # apply reset gate after mm
|
||||
h = (pre_hidden - c) * z + c
|
||||
# https://www.paddlepaddle.org.cn/documentation/docs/zh/api/paddle/fluid/layers/dynamic_gru_cn.html#dynamic-gru
|
||||
|
||||
return h, h
|
||||
|
||||
@property
|
||||
def state_shape(self):
|
||||
r"""
|
||||
The `state_shape` of GRUCell is a shape `[hidden_size]` (-1 for batch
|
||||
size would be automatically inserted into shape). The shape corresponds
|
||||
to the shape of :math:`h_{t-1}`.
|
||||
"""
|
||||
return (self.hidden_size, )
|
||||
|
||||
|
||||
class BiRNNWithBN(nn.Layer):
|
||||
"""Bidirectonal simple rnn layer with sequence-wise batch normalization.
|
||||
The batch normalization is only performed on input-state weights.
|
||||
|
||||
:param size: Dimension of RNN cells.
|
||||
:type size: int
|
||||
:param share_weights: Whether to share input-hidden weights between
|
||||
forward and backward directional RNNs.
|
||||
:type share_weights: bool
|
||||
:return: Bidirectional simple rnn layer.
|
||||
:rtype: Variable
|
||||
"""
|
||||
|
||||
def __init__(self, i_size: int, h_size: int, share_weights: bool):
|
||||
super().__init__()
|
||||
self.share_weights = share_weights
|
||||
if self.share_weights:
|
||||
#input-hidden weights shared between bi-directional rnn.
|
||||
self.fw_fc = nn.Linear(i_size, h_size, bias_attr=False)
|
||||
# batch norm is only performed on input-state projection
|
||||
self.fw_bn = nn.BatchNorm1D(
|
||||
h_size, bias_attr=None, data_format='NLC')
|
||||
self.bw_fc = self.fw_fc
|
||||
self.bw_bn = self.fw_bn
|
||||
else:
|
||||
self.fw_fc = nn.Linear(i_size, h_size, bias_attr=False)
|
||||
self.fw_bn = nn.BatchNorm1D(
|
||||
h_size, bias_attr=None, data_format='NLC')
|
||||
self.bw_fc = nn.Linear(i_size, h_size, bias_attr=False)
|
||||
self.bw_bn = nn.BatchNorm1D(
|
||||
h_size, bias_attr=None, data_format='NLC')
|
||||
|
||||
self.fw_cell = RNNCell(hidden_size=h_size, activation='brelu')
|
||||
self.bw_cell = RNNCell(hidden_size=h_size, activation='brelu')
|
||||
self.fw_rnn = nn.RNN(
|
||||
self.fw_cell, is_reverse=False, time_major=False) #[B, T, D]
|
||||
self.bw_rnn = nn.RNN(
|
||||
self.bw_cell, is_reverse=True, time_major=False) #[B, T, D]
|
||||
|
||||
def forward(self, x: paddle.Tensor, x_len: paddle.Tensor):
|
||||
# x, shape [B, T, D]
|
||||
fw_x = self.fw_bn(self.fw_fc(x))
|
||||
bw_x = self.bw_bn(self.bw_fc(x))
|
||||
fw_x, _ = self.fw_rnn(inputs=fw_x, sequence_length=x_len)
|
||||
bw_x, _ = self.bw_rnn(inputs=bw_x, sequence_length=x_len)
|
||||
x = paddle.concat([fw_x, bw_x], axis=-1)
|
||||
return x, x_len
|
||||
|
||||
|
||||
class BiGRUWithBN(nn.Layer):
|
||||
"""Bidirectonal gru layer with sequence-wise batch normalization.
|
||||
The batch normalization is only performed on input-state weights.
|
||||
|
||||
:param name: Name of the layer.
|
||||
:type name: string
|
||||
:param input: Input layer.
|
||||
:type input: Variable
|
||||
:param size: Dimension of GRU cells.
|
||||
:type size: int
|
||||
:param act: Activation type.
|
||||
:type act: string
|
||||
:return: Bidirectional GRU layer.
|
||||
:rtype: Variable
|
||||
"""
|
||||
|
||||
def __init__(self, i_size: int, h_size: int):
|
||||
super().__init__()
|
||||
hidden_size = h_size * 3
|
||||
|
||||
self.fw_fc = nn.Linear(i_size, hidden_size, bias_attr=False)
|
||||
self.fw_bn = nn.BatchNorm1D(
|
||||
hidden_size, bias_attr=None, data_format='NLC')
|
||||
self.bw_fc = nn.Linear(i_size, hidden_size, bias_attr=False)
|
||||
self.bw_bn = nn.BatchNorm1D(
|
||||
hidden_size, bias_attr=None, data_format='NLC')
|
||||
|
||||
self.fw_cell = GRUCell(input_size=hidden_size, hidden_size=h_size)
|
||||
self.bw_cell = GRUCell(input_size=hidden_size, hidden_size=h_size)
|
||||
self.fw_rnn = nn.RNN(
|
||||
self.fw_cell, is_reverse=False, time_major=False) #[B, T, D]
|
||||
self.bw_rnn = nn.RNN(
|
||||
self.bw_cell, is_reverse=True, time_major=False) #[B, T, D]
|
||||
|
||||
def forward(self, x, x_len):
|
||||
# x, shape [B, T, D]
|
||||
fw_x = self.fw_bn(self.fw_fc(x))
|
||||
bw_x = self.bw_bn(self.bw_fc(x))
|
||||
fw_x, _ = self.fw_rnn(inputs=fw_x, sequence_length=x_len)
|
||||
bw_x, _ = self.bw_rnn(inputs=bw_x, sequence_length=x_len)
|
||||
x = paddle.concat([fw_x, bw_x], axis=-1)
|
||||
return x, x_len
|
||||
|
||||
|
||||
class RNNStack(nn.Layer):
|
||||
"""RNN group with stacked bidirectional simple RNN or GRU layers.
|
||||
|
||||
:param input: Input layer.
|
||||
:type input: Variable
|
||||
:param size: Dimension of RNN cells in each layer.
|
||||
:type size: int
|
||||
:param num_stacks: Number of stacked rnn layers.
|
||||
:type num_stacks: int
|
||||
:param use_gru: Use gru if set True. Use simple rnn if set False.
|
||||
:type use_gru: bool
|
||||
:param share_rnn_weights: Whether to share input-hidden weights between
|
||||
forward and backward directional RNNs.
|
||||
It is only available when use_gru=False.
|
||||
:type share_weights: bool
|
||||
:return: Output layer of the RNN group.
|
||||
:rtype: Variable
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
i_size: int,
|
||||
h_size: int,
|
||||
num_stacks: int,
|
||||
use_gru: bool,
|
||||
share_rnn_weights: bool):
|
||||
super().__init__()
|
||||
rnn_stacks = []
|
||||
for i in range(num_stacks):
|
||||
if use_gru:
|
||||
#default:GRU using tanh
|
||||
rnn_stacks.append(BiGRUWithBN(i_size=i_size, h_size=h_size))
|
||||
else:
|
||||
rnn_stacks.append(
|
||||
BiRNNWithBN(
|
||||
i_size=i_size,
|
||||
h_size=h_size,
|
||||
share_weights=share_rnn_weights))
|
||||
i_size = h_size * 2
|
||||
|
||||
self.rnn_stacks = nn.LayerList(rnn_stacks)
|
||||
|
||||
def forward(self, x: paddle.Tensor, x_len: paddle.Tensor):
|
||||
"""
|
||||
x: shape [B, T, D]
|
||||
x_len: shpae [B]
|
||||
"""
|
||||
for i, rnn in enumerate(self.rnn_stacks):
|
||||
x, x_len = rnn(x, x_len)
|
||||
masks = make_non_pad_mask(x_len) #[B, T]
|
||||
masks = masks.unsqueeze(-1) # [B, T, 1]
|
||||
# https://github.com/PaddlePaddle/Paddle/pull/29265
|
||||
# rhs will type promote to lhs
|
||||
x = x * masks
|
||||
|
||||
return x, x_len
|
||||
@ -0,0 +1,17 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from .deepspeech2 import DeepSpeech2InferModelOnline
|
||||
from .deepspeech2 import DeepSpeech2ModelOnline
|
||||
|
||||
__all__ = ['DeepSpeech2ModelOnline', 'DeepSpeech2InferModelOnline']
|
||||
@ -0,0 +1,33 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import paddle
|
||||
|
||||
from deepspeech.modules.subsampling import Conv2dSubsampling4
|
||||
|
||||
|
||||
class Conv2dSubsampling4Online(Conv2dSubsampling4):
|
||||
def __init__(self, idim: int, odim: int, dropout_rate: float):
|
||||
super().__init__(idim, odim, dropout_rate, None)
|
||||
self.output_dim = ((idim - 1) // 2 - 1) // 2 * odim
|
||||
self.receptive_field_length = 2 * (
|
||||
3 - 1) + 3 # stride_1 * (kernel_size_2 - 1) + kerel_size_1
|
||||
|
||||
def forward(self, x: paddle.Tensor,
|
||||
x_len: paddle.Tensor) -> [paddle.Tensor, paddle.Tensor]:
|
||||
x = x.unsqueeze(1) # (b, c=1, t, f)
|
||||
x = self.conv(x)
|
||||
#b, c, t, f = paddle.shape(x) #not work under jit
|
||||
x = x.transpose([0, 2, 1, 3]).reshape([0, 0, -1])
|
||||
x_len = ((x_len - 1) // 2 - 1) // 2
|
||||
return x, x_len
|
||||
@ -0,0 +1,438 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Deepspeech2 ASR Online Model"""
|
||||
from typing import Optional
|
||||
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
from paddle import nn
|
||||
from yacs.config import CfgNode
|
||||
|
||||
from deepspeech.models.ds2_online.conv import Conv2dSubsampling4Online
|
||||
from deepspeech.modules.ctc import CTCDecoder
|
||||
from deepspeech.utils import layer_tools
|
||||
from deepspeech.utils.checkpoint import Checkpoint
|
||||
from deepspeech.utils.log import Log
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
__all__ = ['DeepSpeech2ModelOnline', 'DeepSpeech2InferModelOnline']
|
||||
|
||||
|
||||
class CRNNEncoder(nn.Layer):
|
||||
def __init__(self,
|
||||
feat_size,
|
||||
dict_size,
|
||||
num_conv_layers=2,
|
||||
num_rnn_layers=4,
|
||||
rnn_size=1024,
|
||||
rnn_direction='forward',
|
||||
num_fc_layers=2,
|
||||
fc_layers_size_list=[512, 256],
|
||||
use_gru=False):
|
||||
super().__init__()
|
||||
self.rnn_size = rnn_size
|
||||
self.feat_size = feat_size # 161 for linear
|
||||
self.dict_size = dict_size
|
||||
self.num_rnn_layers = num_rnn_layers
|
||||
self.num_fc_layers = num_fc_layers
|
||||
self.rnn_direction = rnn_direction
|
||||
self.fc_layers_size_list = fc_layers_size_list
|
||||
self.use_gru = use_gru
|
||||
self.conv = Conv2dSubsampling4Online(feat_size, 32, dropout_rate=0.0)
|
||||
|
||||
self.output_dim = self.conv.output_dim
|
||||
|
||||
i_size = self.conv.output_dim
|
||||
self.rnn = nn.LayerList()
|
||||
self.layernorm_list = nn.LayerList()
|
||||
self.fc_layers_list = nn.LayerList()
|
||||
if rnn_direction == 'bidirect' or rnn_direction == 'bidirectional':
|
||||
layernorm_size = 2 * rnn_size
|
||||
elif rnn_direction == 'forward':
|
||||
layernorm_size = rnn_size
|
||||
else:
|
||||
raise Exception("Wrong rnn direction")
|
||||
for i in range(0, num_rnn_layers):
|
||||
if i == 0:
|
||||
rnn_input_size = i_size
|
||||
else:
|
||||
rnn_input_size = layernorm_size
|
||||
if use_gru is True:
|
||||
self.rnn.append(
|
||||
nn.GRU(
|
||||
input_size=rnn_input_size,
|
||||
hidden_size=rnn_size,
|
||||
num_layers=1,
|
||||
direction=rnn_direction))
|
||||
else:
|
||||
self.rnn.append(
|
||||
nn.LSTM(
|
||||
input_size=rnn_input_size,
|
||||
hidden_size=rnn_size,
|
||||
num_layers=1,
|
||||
direction=rnn_direction))
|
||||
self.layernorm_list.append(nn.LayerNorm(layernorm_size))
|
||||
self.output_dim = layernorm_size
|
||||
|
||||
fc_input_size = layernorm_size
|
||||
for i in range(self.num_fc_layers):
|
||||
self.fc_layers_list.append(
|
||||
nn.Linear(fc_input_size, fc_layers_size_list[i]))
|
||||
fc_input_size = fc_layers_size_list[i]
|
||||
self.output_dim = fc_layers_size_list[i]
|
||||
|
||||
@property
|
||||
def output_size(self):
|
||||
return self.output_dim
|
||||
|
||||
def forward(self, x, x_lens, init_state_h_box=None, init_state_c_box=None):
|
||||
"""Compute Encoder outputs
|
||||
|
||||
Args:
|
||||
x (Tensor): [B, T, D]
|
||||
x_lens (Tensor): [B]
|
||||
init_state_h_box(Tensor): init_states h for RNN layers: [num_rnn_layers * num_directions, batch_size, hidden_size]
|
||||
init_state_c_box(Tensor): init_states c for RNN layers: [num_rnn_layers * num_directions, batch_size, hidden_size]
|
||||
Return:
|
||||
x (Tensor): encoder outputs, [B, T, D]
|
||||
x_lens (Tensor): encoder length, [B]
|
||||
final_state_h_box(Tensor): final_states h for RNN layers: [num_rnn_layers * num_directions, batch_size, hidden_size]
|
||||
final_state_c_box(Tensor): final_states c for RNN layers: [num_rnn_layers * num_directions, batch_size, hidden_size]
|
||||
"""
|
||||
if init_state_h_box is not None:
|
||||
init_state_list = None
|
||||
|
||||
if self.use_gru is True:
|
||||
init_state_h_list = paddle.split(
|
||||
init_state_h_box, self.num_rnn_layers, axis=0)
|
||||
init_state_list = init_state_h_list
|
||||
else:
|
||||
init_state_h_list = paddle.split(
|
||||
init_state_h_box, self.num_rnn_layers, axis=0)
|
||||
init_state_c_list = paddle.split(
|
||||
init_state_c_box, self.num_rnn_layers, axis=0)
|
||||
init_state_list = [(init_state_h_list[i], init_state_c_list[i])
|
||||
for i in range(self.num_rnn_layers)]
|
||||
else:
|
||||
init_state_list = [None] * self.num_rnn_layers
|
||||
|
||||
x, x_lens = self.conv(x, x_lens)
|
||||
final_chunk_state_list = []
|
||||
for i in range(0, self.num_rnn_layers):
|
||||
x, final_state = self.rnn[i](x, init_state_list[i],
|
||||
x_lens) #[B, T, D]
|
||||
final_chunk_state_list.append(final_state)
|
||||
x = self.layernorm_list[i](x)
|
||||
|
||||
for i in range(self.num_fc_layers):
|
||||
x = self.fc_layers_list[i](x)
|
||||
x = F.relu(x)
|
||||
|
||||
if self.use_gru is True:
|
||||
final_chunk_state_h_box = paddle.concat(
|
||||
final_chunk_state_list, axis=0)
|
||||
final_chunk_state_c_box = init_state_c_box
|
||||
else:
|
||||
final_chunk_state_h_list = [
|
||||
final_chunk_state_list[i][0] for i in range(self.num_rnn_layers)
|
||||
]
|
||||
final_chunk_state_c_list = [
|
||||
final_chunk_state_list[i][1] for i in range(self.num_rnn_layers)
|
||||
]
|
||||
final_chunk_state_h_box = paddle.concat(
|
||||
final_chunk_state_h_list, axis=0)
|
||||
final_chunk_state_c_box = paddle.concat(
|
||||
final_chunk_state_c_list, axis=0)
|
||||
|
||||
return x, x_lens, final_chunk_state_h_box, final_chunk_state_c_box
|
||||
|
||||
def forward_chunk_by_chunk(self, x, x_lens, decoder_chunk_size=8):
|
||||
"""Compute Encoder outputs
|
||||
|
||||
Args:
|
||||
x (Tensor): [B, T, D]
|
||||
x_lens (Tensor): [B]
|
||||
decoder_chunk_size: The chunk size of decoder
|
||||
Returns:
|
||||
eouts_list (List of Tensor): The list of encoder outputs in chunk_size: [B, chunk_size, D] * num_chunks
|
||||
eouts_lens_list (List of Tensor): The list of encoder length in chunk_size: [B] * num_chunks
|
||||
final_state_h_box(Tensor): final_states h for RNN layers: [num_rnn_layers * num_directions, batch_size, hidden_size]
|
||||
final_state_c_box(Tensor): final_states c for RNN layers: [num_rnn_layers * num_directions, batch_size, hidden_size]
|
||||
"""
|
||||
subsampling_rate = self.conv.subsampling_rate
|
||||
receptive_field_length = self.conv.receptive_field_length
|
||||
chunk_size = (decoder_chunk_size - 1
|
||||
) * subsampling_rate + receptive_field_length
|
||||
chunk_stride = subsampling_rate * decoder_chunk_size
|
||||
max_len = x.shape[1]
|
||||
assert (chunk_size <= max_len)
|
||||
|
||||
eouts_chunk_list = []
|
||||
eouts_chunk_lens_list = []
|
||||
if (max_len - chunk_size) % chunk_stride != 0:
|
||||
padding_len = chunk_stride - (max_len - chunk_size) % chunk_stride
|
||||
else:
|
||||
padding_len = 0
|
||||
padding = paddle.zeros((x.shape[0], padding_len, x.shape[2]))
|
||||
padded_x = paddle.concat([x, padding], axis=1)
|
||||
num_chunk = (max_len + padding_len - chunk_size) / chunk_stride + 1
|
||||
num_chunk = int(num_chunk)
|
||||
chunk_state_h_box = None
|
||||
chunk_state_c_box = None
|
||||
final_state_h_box = None
|
||||
final_state_c_box = None
|
||||
for i in range(0, num_chunk):
|
||||
start = i * chunk_stride
|
||||
end = start + chunk_size
|
||||
x_chunk = padded_x[:, start:end, :]
|
||||
|
||||
x_len_left = paddle.where(x_lens - i * chunk_stride < 0,
|
||||
paddle.zeros_like(x_lens),
|
||||
x_lens - i * chunk_stride)
|
||||
x_chunk_len_tmp = paddle.ones_like(x_lens) * chunk_size
|
||||
x_chunk_lens = paddle.where(x_len_left < x_chunk_len_tmp,
|
||||
x_len_left, x_chunk_len_tmp)
|
||||
|
||||
eouts_chunk, eouts_chunk_lens, chunk_state_h_box, chunk_state_c_box = self.forward(
|
||||
x_chunk, x_chunk_lens, chunk_state_h_box, chunk_state_c_box)
|
||||
|
||||
eouts_chunk_list.append(eouts_chunk)
|
||||
eouts_chunk_lens_list.append(eouts_chunk_lens)
|
||||
final_state_h_box = chunk_state_h_box
|
||||
final_state_c_box = chunk_state_c_box
|
||||
return eouts_chunk_list, eouts_chunk_lens_list, final_state_h_box, final_state_c_box
|
||||
|
||||
|
||||
class DeepSpeech2ModelOnline(nn.Layer):
|
||||
"""The DeepSpeech2 network structure for online.
|
||||
|
||||
:param audio: Audio spectrogram data layer.
|
||||
:type audio: Variable
|
||||
:param text: Transcription text data layer.
|
||||
:type text: Variable
|
||||
:param audio_len: Valid sequence length data layer.
|
||||
:type audio_len: Variable
|
||||
:param feat_size: feature size for audio.
|
||||
:type feat_size: int
|
||||
:param dict_size: Dictionary size for tokenized transcription.
|
||||
:type dict_size: int
|
||||
:param num_conv_layers: Number of stacking convolution layers.
|
||||
:type num_conv_layers: int
|
||||
:param num_rnn_layers: Number of stacking RNN layers.
|
||||
:type num_rnn_layers: int
|
||||
:param rnn_size: RNN layer size (dimension of RNN cells).
|
||||
:type rnn_size: int
|
||||
:param num_fc_layers: Number of stacking FC layers.
|
||||
:type num_fc_layers: int
|
||||
:param fc_layers_size_list: The list of FC layer sizes.
|
||||
:type fc_layers_size_list: [int,]
|
||||
:param use_gru: Use gru if set True. Use simple rnn if set False.
|
||||
:type use_gru: bool
|
||||
:return: A tuple of an output unnormalized log probability layer (
|
||||
before softmax) and a ctc cost layer.
|
||||
:rtype: tuple of LayerOutput
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def params(cls, config: Optional[CfgNode]=None) -> CfgNode:
|
||||
default = CfgNode(
|
||||
dict(
|
||||
num_conv_layers=2, #Number of stacking convolution layers.
|
||||
num_rnn_layers=4, #Number of stacking RNN layers.
|
||||
rnn_layer_size=1024, #RNN layer size (number of RNN cells).
|
||||
num_fc_layers=2,
|
||||
fc_layers_size_list=[512, 256],
|
||||
use_gru=True, #Use gru if set True. Use simple rnn if set False.
|
||||
blank_id=0, # index of blank in vocob.txt
|
||||
))
|
||||
if config is not None:
|
||||
config.merge_from_other_cfg(default)
|
||||
return default
|
||||
|
||||
def __init__(self,
|
||||
feat_size,
|
||||
dict_size,
|
||||
num_conv_layers=2,
|
||||
num_rnn_layers=4,
|
||||
rnn_size=1024,
|
||||
rnn_direction='forward',
|
||||
num_fc_layers=2,
|
||||
fc_layers_size_list=[512, 256],
|
||||
use_gru=False,
|
||||
blank_id=0):
|
||||
super().__init__()
|
||||
self.encoder = CRNNEncoder(
|
||||
feat_size=feat_size,
|
||||
dict_size=dict_size,
|
||||
num_conv_layers=num_conv_layers,
|
||||
num_rnn_layers=num_rnn_layers,
|
||||
rnn_direction=rnn_direction,
|
||||
num_fc_layers=num_fc_layers,
|
||||
fc_layers_size_list=fc_layers_size_list,
|
||||
rnn_size=rnn_size,
|
||||
use_gru=use_gru)
|
||||
|
||||
self.decoder = CTCDecoder(
|
||||
odim=dict_size, # <blank> is in vocab
|
||||
enc_n_units=self.encoder.output_size,
|
||||
blank_id=blank_id,
|
||||
dropout_rate=0.0,
|
||||
reduction=True, # sum
|
||||
batch_average=True, # sum / batch_size
|
||||
grad_norm_type='instance')
|
||||
|
||||
def forward(self, audio, audio_len, text, text_len):
|
||||
"""Compute Model loss
|
||||
|
||||
Args:
|
||||
audio (Tenosr): [B, T, D]
|
||||
audio_len (Tensor): [B]
|
||||
text (Tensor): [B, U]
|
||||
text_len (Tensor): [B]
|
||||
|
||||
Returns:
|
||||
loss (Tenosr): [1]
|
||||
"""
|
||||
eouts, eouts_len, final_state_h_box, final_state_c_box = self.encoder(
|
||||
audio, audio_len, None, None)
|
||||
loss = self.decoder(eouts, eouts_len, text, text_len)
|
||||
return loss
|
||||
|
||||
@paddle.no_grad()
|
||||
def decode(self, audio, audio_len, vocab_list, decoding_method,
|
||||
lang_model_path, beam_alpha, beam_beta, beam_size, cutoff_prob,
|
||||
cutoff_top_n, num_processes):
|
||||
# init once
|
||||
# decoders only accept string encoded in utf-8
|
||||
self.decoder.init_decode(
|
||||
beam_alpha=beam_alpha,
|
||||
beam_beta=beam_beta,
|
||||
lang_model_path=lang_model_path,
|
||||
vocab_list=vocab_list,
|
||||
decoding_method=decoding_method)
|
||||
|
||||
eouts, eouts_len, final_state_h_box, final_state_c_box = self.encoder(
|
||||
audio, audio_len, None, None)
|
||||
probs = self.decoder.softmax(eouts)
|
||||
return self.decoder.decode_probs(
|
||||
probs.numpy(), eouts_len, vocab_list, decoding_method,
|
||||
lang_model_path, beam_alpha, beam_beta, beam_size, cutoff_prob,
|
||||
cutoff_top_n, num_processes)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, dataloader, config, checkpoint_path):
|
||||
"""Build a DeepSpeech2Model model from a pretrained model.
|
||||
Parameters
|
||||
----------
|
||||
dataloader: paddle.io.DataLoader
|
||||
|
||||
config: yacs.config.CfgNode
|
||||
model configs
|
||||
|
||||
checkpoint_path: Path or str
|
||||
the path of pretrained model checkpoint, without extension name
|
||||
|
||||
Returns
|
||||
-------
|
||||
DeepSpeech2ModelOnline
|
||||
The model built from pretrained result.
|
||||
"""
|
||||
model = cls(feat_size=dataloader.collate_fn.feature_size,
|
||||
dict_size=dataloader.collate_fn.vocab_size,
|
||||
num_conv_layers=config.model.num_conv_layers,
|
||||
num_rnn_layers=config.model.num_rnn_layers,
|
||||
rnn_size=config.model.rnn_layer_size,
|
||||
rnn_direction=config.model.rnn_direction,
|
||||
num_fc_layers=config.model.num_fc_layers,
|
||||
fc_layers_size_list=config.model.fc_layers_size_list,
|
||||
use_gru=config.model.use_gru,
|
||||
blank_id=config.model.blank_id)
|
||||
infos = Checkpoint().load_parameters(
|
||||
model, checkpoint_path=checkpoint_path)
|
||||
logger.info(f"checkpoint info: {infos}")
|
||||
layer_tools.summary(model)
|
||||
return model
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config):
|
||||
"""Build a DeepSpeec2ModelOnline from config
|
||||
Parameters
|
||||
|
||||
config: yacs.config.CfgNode
|
||||
config.model
|
||||
Returns
|
||||
-------
|
||||
DeepSpeech2ModelOnline
|
||||
The model built from config.
|
||||
"""
|
||||
model = cls(feat_size=config.feat_size,
|
||||
dict_size=config.dict_size,
|
||||
num_conv_layers=config.num_conv_layers,
|
||||
num_rnn_layers=config.num_rnn_layers,
|
||||
rnn_size=config.rnn_layer_size,
|
||||
rnn_direction=config.rnn_direction,
|
||||
num_fc_layers=config.num_fc_layers,
|
||||
fc_layers_size_list=config.fc_layers_size_list,
|
||||
use_gru=config.use_gru,
|
||||
blank_id=config.blank_id)
|
||||
return model
|
||||
|
||||
|
||||
class DeepSpeech2InferModelOnline(DeepSpeech2ModelOnline):
|
||||
def __init__(self,
|
||||
feat_size,
|
||||
dict_size,
|
||||
num_conv_layers=2,
|
||||
num_rnn_layers=4,
|
||||
rnn_size=1024,
|
||||
rnn_direction='forward',
|
||||
num_fc_layers=2,
|
||||
fc_layers_size_list=[512, 256],
|
||||
use_gru=False,
|
||||
blank_id=0):
|
||||
super().__init__(
|
||||
feat_size=feat_size,
|
||||
dict_size=dict_size,
|
||||
num_conv_layers=num_conv_layers,
|
||||
num_rnn_layers=num_rnn_layers,
|
||||
rnn_size=rnn_size,
|
||||
rnn_direction=rnn_direction,
|
||||
num_fc_layers=num_fc_layers,
|
||||
fc_layers_size_list=fc_layers_size_list,
|
||||
use_gru=use_gru,
|
||||
blank_id=blank_id)
|
||||
|
||||
def forward(self, audio_chunk, audio_chunk_lens, chunk_state_h_box,
|
||||
chunk_state_c_box):
|
||||
eouts_chunk, eouts_chunk_lens, final_state_h_box, final_state_c_box = self.encoder(
|
||||
audio_chunk, audio_chunk_lens, chunk_state_h_box, chunk_state_c_box)
|
||||
probs_chunk = self.decoder.softmax(eouts_chunk)
|
||||
return probs_chunk, eouts_chunk_lens, final_state_h_box, final_state_c_box
|
||||
|
||||
def export(self):
|
||||
static_model = paddle.jit.to_static(
|
||||
self,
|
||||
input_spec=[
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None,
|
||||
self.encoder.feat_size], #[B, chunk_size, feat_dim]
|
||||
dtype='float32'),
|
||||
paddle.static.InputSpec(shape=[None],
|
||||
dtype='int64'), # audio_length, [B]
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None, None], dtype='float32'),
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None, None], dtype='float32')
|
||||
])
|
||||
return static_model
|
||||
@ -0,0 +1,19 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from .u2 import U2InferModel
|
||||
from .u2 import U2Model
|
||||
from .updater import U2Evaluator
|
||||
from .updater import U2Updater
|
||||
|
||||
__all__ = ["U2Model", "U2InferModel", "U2Evaluator", "U2Updater"]
|
||||
@ -0,0 +1,149 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from contextlib import nullcontext
|
||||
|
||||
import paddle
|
||||
from paddle import distributed as dist
|
||||
|
||||
from deepspeech.training.extensions.evaluator import StandardEvaluator
|
||||
from deepspeech.training.reporter import report
|
||||
from deepspeech.training.timer import Timer
|
||||
from deepspeech.training.updaters.standard_updater import StandardUpdater
|
||||
from deepspeech.utils import layer_tools
|
||||
from deepspeech.utils.log import Log
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
|
||||
class U2Evaluator(StandardEvaluator):
|
||||
def __init__(self, model, dataloader):
|
||||
super().__init__(model, dataloader)
|
||||
self.msg = ""
|
||||
self.num_seen_utts = 0
|
||||
self.total_loss = 0.0
|
||||
|
||||
def evaluate_core(self, batch):
|
||||
self.msg = "Valid: Rank: {}, ".format(dist.get_rank())
|
||||
losses_dict = {}
|
||||
|
||||
loss, attention_loss, ctc_loss = self.model(*batch[1:])
|
||||
if paddle.isfinite(loss):
|
||||
num_utts = batch[1].shape[0]
|
||||
self.num_seen_utts += num_utts
|
||||
self.total_loss += float(loss) * num_utts
|
||||
|
||||
losses_dict['loss'] = float(loss)
|
||||
if attention_loss:
|
||||
losses_dict['att_loss'] = float(attention_loss)
|
||||
if ctc_loss:
|
||||
losses_dict['ctc_loss'] = float(ctc_loss)
|
||||
|
||||
for k, v in losses_dict.items():
|
||||
report("eval/" + k, v)
|
||||
|
||||
self.msg += ', '.join('{}: {:>.6f}'.format(k, v)
|
||||
for k, v in losses_dict.items())
|
||||
logger.info(self.msg)
|
||||
return self.total_loss, self.num_seen_utts
|
||||
|
||||
|
||||
class U2Updater(StandardUpdater):
|
||||
def __init__(self,
|
||||
model,
|
||||
optimizer,
|
||||
scheduler,
|
||||
dataloader,
|
||||
init_state=None,
|
||||
accum_grad=1,
|
||||
**kwargs):
|
||||
super().__init__(
|
||||
model, optimizer, scheduler, dataloader, init_state=init_state)
|
||||
self.accum_grad = accum_grad
|
||||
self.forward_count = 0
|
||||
self.msg = ""
|
||||
|
||||
def update_core(self, batch):
|
||||
"""One Step
|
||||
|
||||
Args:
|
||||
batch (List[Object]): utts, xs, xlens, ys, ylens
|
||||
"""
|
||||
losses_dict = {}
|
||||
self.msg = "Rank: {}, ".format(dist.get_rank())
|
||||
|
||||
# forward
|
||||
batch_size = batch[1].shape[0]
|
||||
loss, attention_loss, ctc_loss = self.model(*batch[1:])
|
||||
# loss div by `batch_size * accum_grad`
|
||||
loss /= self.accum_grad
|
||||
|
||||
# loss backward
|
||||
if (self.forward_count + 1) != self.accum_grad:
|
||||
# Disable gradient synchronizations across DDP processes.
|
||||
# Within this context, gradients will be accumulated on module
|
||||
# variables, which will later be synchronized.
|
||||
context = self.model.no_sync
|
||||
else:
|
||||
# Used for single gpu training and DDP gradient synchronization
|
||||
# processes.
|
||||
context = nullcontext
|
||||
|
||||
with context():
|
||||
loss.backward()
|
||||
layer_tools.print_grads(self.model, print_func=None)
|
||||
|
||||
# loss info
|
||||
losses_dict['loss'] = float(loss) * self.accum_grad
|
||||
if attention_loss:
|
||||
losses_dict['att_loss'] = float(attention_loss)
|
||||
if ctc_loss:
|
||||
losses_dict['ctc_loss'] = float(ctc_loss)
|
||||
# report loss
|
||||
for k, v in losses_dict.items():
|
||||
report("train/" + k, v)
|
||||
# loss msg
|
||||
self.msg += "batch size: {}, ".format(batch_size)
|
||||
self.msg += "accum: {}, ".format(self.accum_grad)
|
||||
self.msg += ', '.join('{}: {:>.6f}'.format(k, v)
|
||||
for k, v in losses_dict.items())
|
||||
|
||||
# Truncate the graph
|
||||
loss.detach()
|
||||
|
||||
# update parameters
|
||||
self.forward_count += 1
|
||||
if self.forward_count != self.accum_grad:
|
||||
return
|
||||
self.forward_count = 0
|
||||
|
||||
self.optimizer.step()
|
||||
self.optimizer.clear_grad()
|
||||
self.scheduler.step()
|
||||
|
||||
def update(self):
|
||||
# model is default in train mode
|
||||
|
||||
# training for a step is implemented here
|
||||
with Timer("data time cost:{}"):
|
||||
batch = self.read_batch()
|
||||
with Timer("step time cost:{}"):
|
||||
self.update_core(batch)
|
||||
|
||||
# #iterations with accum_grad > 1
|
||||
# Ref.: https://github.com/espnet/espnet/issues/777
|
||||
if self.forward_count == 0:
|
||||
self.state.iteration += 1
|
||||
if self.updates_per_epoch is not None:
|
||||
if self.state.iteration % self.updates_per_epoch == 0:
|
||||
self.state.epoch += 1
|
||||
@ -0,0 +1,728 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""U2 ASR Model
|
||||
Unified Streaming and Non-streaming Two-pass End-to-end Model for Speech Recognition
|
||||
(https://arxiv.org/pdf/2012.05481.pdf)
|
||||
"""
|
||||
import time
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Tuple
|
||||
|
||||
import paddle
|
||||
from paddle import jit
|
||||
from paddle import nn
|
||||
from yacs.config import CfgNode
|
||||
|
||||
from deepspeech.frontend.utility import IGNORE_ID
|
||||
from deepspeech.frontend.utility import load_cmvn
|
||||
from deepspeech.modules.cmvn import GlobalCMVN
|
||||
from deepspeech.modules.ctc import CTCDecoder
|
||||
from deepspeech.modules.decoder import TransformerDecoder
|
||||
from deepspeech.modules.encoder import ConformerEncoder
|
||||
from deepspeech.modules.encoder import TransformerEncoder
|
||||
from deepspeech.modules.loss import LabelSmoothingLoss
|
||||
from deepspeech.modules.mask import mask_finished_preds
|
||||
from deepspeech.modules.mask import mask_finished_scores
|
||||
from deepspeech.modules.mask import subsequent_mask
|
||||
from deepspeech.utils import checkpoint
|
||||
from deepspeech.utils import layer_tools
|
||||
from deepspeech.utils.log import Log
|
||||
from deepspeech.utils.tensor_utils import add_sos_eos
|
||||
from deepspeech.utils.tensor_utils import th_accuracy
|
||||
from deepspeech.utils.utility import UpdateConfig
|
||||
|
||||
__all__ = ["U2STModel", "U2STInferModel"]
|
||||
|
||||
logger = Log(__name__).getlog()
|
||||
|
||||
|
||||
class U2STBaseModel(nn.Layer):
|
||||
"""CTC-Attention hybrid Encoder-Decoder model"""
|
||||
|
||||
@classmethod
|
||||
def params(cls, config: Optional[CfgNode]=None) -> CfgNode:
|
||||
# network architecture
|
||||
default = CfgNode()
|
||||
# allow add new item when merge_with_file
|
||||
default.cmvn_file = ""
|
||||
default.cmvn_file_type = "json"
|
||||
default.input_dim = 0
|
||||
default.output_dim = 0
|
||||
# encoder related
|
||||
default.encoder = 'transformer'
|
||||
default.encoder_conf = CfgNode(
|
||||
dict(
|
||||
output_size=256, # dimension of attention
|
||||
attention_heads=4,
|
||||
linear_units=2048, # the number of units of position-wise feed forward
|
||||
num_blocks=12, # the number of encoder blocks
|
||||
dropout_rate=0.1,
|
||||
positional_dropout_rate=0.1,
|
||||
attention_dropout_rate=0.0,
|
||||
input_layer='conv2d', # encoder input type, you can chose conv2d, conv2d6 and conv2d8
|
||||
normalize_before=True,
|
||||
# use_cnn_module=True,
|
||||
# cnn_module_kernel=15,
|
||||
# activation_type='swish',
|
||||
# pos_enc_layer_type='rel_pos',
|
||||
# selfattention_layer_type='rel_selfattn',
|
||||
))
|
||||
# decoder related
|
||||
default.decoder = 'transformer'
|
||||
default.decoder_conf = CfgNode(
|
||||
dict(
|
||||
attention_heads=4,
|
||||
linear_units=2048,
|
||||
num_blocks=6,
|
||||
dropout_rate=0.1,
|
||||
positional_dropout_rate=0.1,
|
||||
self_attention_dropout_rate=0.0,
|
||||
src_attention_dropout_rate=0.0, ))
|
||||
# hybrid CTC/attention
|
||||
default.model_conf = CfgNode(
|
||||
dict(
|
||||
asr_weight=0.0,
|
||||
ctc_weight=0.0,
|
||||
lsm_weight=0.1, # label smoothing option
|
||||
length_normalized_loss=False, ))
|
||||
|
||||
if config is not None:
|
||||
config.merge_from_other_cfg(default)
|
||||
return default
|
||||
|
||||
def __init__(self,
|
||||
vocab_size: int,
|
||||
encoder: TransformerEncoder,
|
||||
st_decoder: TransformerDecoder,
|
||||
decoder: TransformerDecoder=None,
|
||||
ctc: CTCDecoder=None,
|
||||
ctc_weight: float=0.0,
|
||||
asr_weight: float=0.0,
|
||||
ignore_id: int=IGNORE_ID,
|
||||
lsm_weight: float=0.0,
|
||||
length_normalized_loss: bool=False):
|
||||
assert 0.0 <= ctc_weight <= 1.0, ctc_weight
|
||||
|
||||
super().__init__()
|
||||
# note that eos is the same as sos (equivalent ID)
|
||||
self.sos = vocab_size - 1
|
||||
self.eos = vocab_size - 1
|
||||
self.vocab_size = vocab_size
|
||||
self.ignore_id = ignore_id
|
||||
self.ctc_weight = ctc_weight
|
||||
self.asr_weight = asr_weight
|
||||
|
||||
self.encoder = encoder
|
||||
self.st_decoder = st_decoder
|
||||
self.decoder = decoder
|
||||
self.ctc = ctc
|
||||
self.criterion_att = LabelSmoothingLoss(
|
||||
size=vocab_size,
|
||||
padding_idx=ignore_id,
|
||||
smoothing=lsm_weight,
|
||||
normalize_length=length_normalized_loss, )
|
||||
|
||||
def forward(
|
||||
self,
|
||||
speech: paddle.Tensor,
|
||||
speech_lengths: paddle.Tensor,
|
||||
text: paddle.Tensor,
|
||||
text_lengths: paddle.Tensor,
|
||||
asr_text: paddle.Tensor=None,
|
||||
asr_text_lengths: paddle.Tensor=None,
|
||||
) -> Tuple[Optional[paddle.Tensor], Optional[paddle.Tensor], Optional[
|
||||
paddle.Tensor]]:
|
||||
"""Frontend + Encoder + Decoder + Calc loss
|
||||
Args:
|
||||
speech: (Batch, Length, ...)
|
||||
speech_lengths: (Batch, )
|
||||
text: (Batch, Length)
|
||||
text_lengths: (Batch,)
|
||||
Returns:
|
||||
total_loss, attention_loss, ctc_loss
|
||||
"""
|
||||
assert text_lengths.dim() == 1, text_lengths.shape
|
||||
# Check that batch_size is unified
|
||||
assert (speech.shape[0] == speech_lengths.shape[0] == text.shape[0] ==
|
||||
text_lengths.shape[0]), (speech.shape, speech_lengths.shape,
|
||||
text.shape, text_lengths.shape)
|
||||
# 1. Encoder
|
||||
start = time.time()
|
||||
encoder_out, encoder_mask = self.encoder(speech, speech_lengths)
|
||||
encoder_time = time.time() - start
|
||||
#logger.debug(f"encoder time: {encoder_time}")
|
||||
encoder_out_lens = encoder_mask.squeeze(1).sum(1) #[B, 1, T] -> [B]
|
||||
|
||||
# 2a. ST-decoder branch
|
||||
start = time.time()
|
||||
loss_st, acc_st = self._calc_st_loss(encoder_out, encoder_mask, text,
|
||||
text_lengths)
|
||||
decoder_time = time.time() - start
|
||||
|
||||
loss_asr_att = None
|
||||
loss_asr_ctc = None
|
||||
# 2b. ASR Attention-decoder branch
|
||||
if self.asr_weight > 0.:
|
||||
if self.ctc_weight != 1.0:
|
||||
start = time.time()
|
||||
loss_asr_att, acc_att = self._calc_att_loss(
|
||||
encoder_out, encoder_mask, asr_text, asr_text_lengths)
|
||||
decoder_time = time.time() - start
|
||||
|
||||
# 2c. CTC branch
|
||||
if self.ctc_weight != 0.0:
|
||||
start = time.time()
|
||||
loss_asr_ctc = self.ctc(encoder_out, encoder_out_lens, asr_text,
|
||||
asr_text_lengths)
|
||||
ctc_time = time.time() - start
|
||||
|
||||
if loss_asr_ctc is None:
|
||||
loss_asr = loss_asr_att
|
||||
elif loss_asr_att is None:
|
||||
loss_asr = loss_asr_ctc
|
||||
else:
|
||||
loss_asr = self.ctc_weight * loss_asr_ctc + (1 - self.ctc_weight
|
||||
) * loss_asr_att
|
||||
loss = self.asr_weight * loss_asr + (1 - self.asr_weight) * loss_st
|
||||
else:
|
||||
loss = loss_st
|
||||
return loss, loss_st, loss_asr_att, loss_asr_ctc
|
||||
|
||||
def _calc_st_loss(
|
||||
self,
|
||||
encoder_out: paddle.Tensor,
|
||||
encoder_mask: paddle.Tensor,
|
||||
ys_pad: paddle.Tensor,
|
||||
ys_pad_lens: paddle.Tensor, ) -> Tuple[paddle.Tensor, float]:
|
||||
"""Calc attention loss.
|
||||
|
||||
Args:
|
||||
encoder_out (paddle.Tensor): [B, Tmax, D]
|
||||
encoder_mask (paddle.Tensor): [B, 1, Tmax]
|
||||
ys_pad (paddle.Tensor): [B, Umax]
|
||||
ys_pad_lens (paddle.Tensor): [B]
|
||||
|
||||
Returns:
|
||||
Tuple[paddle.Tensor, float]: attention_loss, accuracy rate
|
||||
"""
|
||||
ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos,
|
||||
self.ignore_id)
|
||||
ys_in_lens = ys_pad_lens + 1
|
||||
|
||||
# 1. Forward decoder
|
||||
decoder_out, _ = self.st_decoder(encoder_out, encoder_mask, ys_in_pad,
|
||||
ys_in_lens)
|
||||
|
||||
# 2. Compute attention loss
|
||||
loss_att = self.criterion_att(decoder_out, ys_out_pad)
|
||||
acc_att = th_accuracy(
|
||||
decoder_out.view(-1, self.vocab_size),
|
||||
ys_out_pad,
|
||||
ignore_label=self.ignore_id, )
|
||||
return loss_att, acc_att
|
||||
|
||||
def _calc_att_loss(
|
||||
self,
|
||||
encoder_out: paddle.Tensor,
|
||||
encoder_mask: paddle.Tensor,
|
||||
ys_pad: paddle.Tensor,
|
||||
ys_pad_lens: paddle.Tensor, ) -> Tuple[paddle.Tensor, float]:
|
||||
"""Calc attention loss.
|
||||
|
||||
Args:
|
||||
encoder_out (paddle.Tensor): [B, Tmax, D]
|
||||
encoder_mask (paddle.Tensor): [B, 1, Tmax]
|
||||
ys_pad (paddle.Tensor): [B, Umax]
|
||||
ys_pad_lens (paddle.Tensor): [B]
|
||||
|
||||
Returns:
|
||||
Tuple[paddle.Tensor, float]: attention_loss, accuracy rate
|
||||
"""
|
||||
ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos,
|
||||
self.ignore_id)
|
||||
ys_in_lens = ys_pad_lens + 1
|
||||
|
||||
# 1. Forward decoder
|
||||
decoder_out, _ = self.decoder(encoder_out, encoder_mask, ys_in_pad,
|
||||
ys_in_lens)
|
||||
|
||||
# 2. Compute attention loss
|
||||
loss_att = self.criterion_att(decoder_out, ys_out_pad)
|
||||
acc_att = th_accuracy(
|
||||
decoder_out.view(-1, self.vocab_size),
|
||||
ys_out_pad,
|
||||
ignore_label=self.ignore_id, )
|
||||
return loss_att, acc_att
|
||||
|
||||
def _forward_encoder(
|
||||
self,
|
||||
speech: paddle.Tensor,
|
||||
speech_lengths: paddle.Tensor,
|
||||
decoding_chunk_size: int=-1,
|
||||
num_decoding_left_chunks: int=-1,
|
||||
simulate_streaming: bool=False,
|
||||
) -> Tuple[paddle.Tensor, paddle.Tensor]:
|
||||
"""Encoder pass.
|
||||
|
||||
Args:
|
||||
speech (paddle.Tensor): [B, Tmax, D]
|
||||
speech_lengths (paddle.Tensor): [B]
|
||||
decoding_chunk_size (int, optional): chuck size. Defaults to -1.
|
||||
num_decoding_left_chunks (int, optional): nums chunks. Defaults to -1.
|
||||
simulate_streaming (bool, optional): streaming or not. Defaults to False.
|
||||
|
||||
Returns:
|
||||
Tuple[paddle.Tensor, paddle.Tensor]:
|
||||
encoder hiddens (B, Tmax, D),
|
||||
encoder hiddens mask (B, 1, Tmax).
|
||||
"""
|
||||
# Let's assume B = batch_size
|
||||
# 1. Encoder
|
||||
if simulate_streaming and decoding_chunk_size > 0:
|
||||
encoder_out, encoder_mask = self.encoder.forward_chunk_by_chunk(
|
||||
speech,
|
||||
decoding_chunk_size=decoding_chunk_size,
|
||||
num_decoding_left_chunks=num_decoding_left_chunks
|
||||
) # (B, maxlen, encoder_dim)
|
||||
else:
|
||||
encoder_out, encoder_mask = self.encoder(
|
||||
speech,
|
||||
speech_lengths,
|
||||
decoding_chunk_size=decoding_chunk_size,
|
||||
num_decoding_left_chunks=num_decoding_left_chunks
|
||||
) # (B, maxlen, encoder_dim)
|
||||
return encoder_out, encoder_mask
|
||||
|
||||
def translate(
|
||||
self,
|
||||
speech: paddle.Tensor,
|
||||
speech_lengths: paddle.Tensor,
|
||||
beam_size: int=10,
|
||||
decoding_chunk_size: int=-1,
|
||||
num_decoding_left_chunks: int=-1,
|
||||
simulate_streaming: bool=False, ) -> paddle.Tensor:
|
||||
""" Apply beam search on attention decoder
|
||||
Args:
|
||||
speech (paddle.Tensor): (batch, max_len, feat_dim)
|
||||
speech_length (paddle.Tensor): (batch, )
|
||||
beam_size (int): beam size for beam search
|
||||
decoding_chunk_size (int): decoding chunk for dynamic chunk
|
||||
trained model.
|
||||
<0: for decoding, use full chunk.
|
||||
>0: for decoding, use fixed chunk size as set.
|
||||
0: used for training, it's prohibited here
|
||||
simulate_streaming (bool): whether do encoder forward in a
|
||||
streaming fashion
|
||||
Returns:
|
||||
paddle.Tensor: decoding result, (batch, max_result_len)
|
||||
"""
|
||||
assert speech.shape[0] == speech_lengths.shape[0]
|
||||
assert decoding_chunk_size != 0
|
||||
device = speech.place
|
||||
batch_size = speech.shape[0]
|
||||
|
||||
# Let's assume B = batch_size and N = beam_size
|
||||
# 1. Encoder
|
||||
encoder_out, encoder_mask = self._forward_encoder(
|
||||
speech, speech_lengths, decoding_chunk_size,
|
||||
num_decoding_left_chunks,
|
||||
simulate_streaming) # (B, maxlen, encoder_dim)
|
||||
maxlen = encoder_out.size(1)
|
||||
encoder_dim = encoder_out.size(2)
|
||||
running_size = batch_size * beam_size
|
||||
encoder_out = encoder_out.unsqueeze(1).repeat(1, beam_size, 1, 1).view(
|
||||
running_size, maxlen, encoder_dim) # (B*N, maxlen, encoder_dim)
|
||||
encoder_mask = encoder_mask.unsqueeze(1).repeat(
|
||||
1, beam_size, 1, 1).view(running_size, 1,
|
||||
maxlen) # (B*N, 1, max_len)
|
||||
|
||||
hyps = paddle.ones(
|
||||
[running_size, 1], dtype=paddle.long).fill_(self.sos) # (B*N, 1)
|
||||
# log scale score
|
||||
scores = paddle.to_tensor(
|
||||
[0.0] + [-float('inf')] * (beam_size - 1), dtype=paddle.float)
|
||||
scores = scores.to(device).repeat(batch_size).unsqueeze(1).to(
|
||||
device) # (B*N, 1)
|
||||
end_flag = paddle.zeros_like(scores, dtype=paddle.bool) # (B*N, 1)
|
||||
cache: Optional[List[paddle.Tensor]] = None
|
||||
# 2. Decoder forward step by step
|
||||
for i in range(1, maxlen + 1):
|
||||
# Stop if all batch and all beam produce eos
|
||||
if end_flag.sum() == running_size:
|
||||
break
|
||||
|
||||
# 2.1 Forward decoder step
|
||||
hyps_mask = subsequent_mask(i).unsqueeze(0).repeat(
|
||||
running_size, 1, 1).to(device) # (B*N, i, i)
|
||||
# logp: (B*N, vocab)
|
||||
logp, cache = self.st_decoder.forward_one_step(
|
||||
encoder_out, encoder_mask, hyps, hyps_mask, cache)
|
||||
|
||||
# 2.2 First beam prune: select topk best prob at current time
|
||||
top_k_logp, top_k_index = logp.topk(beam_size) # (B*N, N)
|
||||
top_k_logp = mask_finished_scores(top_k_logp, end_flag)
|
||||
top_k_index = mask_finished_preds(top_k_index, end_flag, self.eos)
|
||||
|
||||
# 2.3 Seconde beam prune: select topk score with history
|
||||
scores = scores + top_k_logp # (B*N, N), broadcast add
|
||||
scores = scores.view(batch_size, beam_size * beam_size) # (B, N*N)
|
||||
scores, offset_k_index = scores.topk(k=beam_size) # (B, N)
|
||||
scores = scores.view(-1, 1) # (B*N, 1)
|
||||
|
||||
# 2.4. Compute base index in top_k_index,
|
||||
# regard top_k_index as (B*N*N),regard offset_k_index as (B*N),
|
||||
# then find offset_k_index in top_k_index
|
||||
base_k_index = paddle.arange(batch_size).view(-1, 1).repeat(
|
||||
1, beam_size) # (B, N)
|
||||
base_k_index = base_k_index * beam_size * beam_size
|
||||
best_k_index = base_k_index.view(-1) + offset_k_index.view(
|
||||
-1) # (B*N)
|
||||
|
||||
# 2.5 Update best hyps
|
||||
best_k_pred = paddle.index_select(
|
||||
top_k_index.view(-1), index=best_k_index, axis=0) # (B*N)
|
||||
best_hyps_index = best_k_index // beam_size
|
||||
last_best_k_hyps = paddle.index_select(
|
||||
hyps, index=best_hyps_index, axis=0) # (B*N, i)
|
||||
hyps = paddle.cat(
|
||||
(last_best_k_hyps, best_k_pred.view(-1, 1)),
|
||||
dim=1) # (B*N, i+1)
|
||||
|
||||
# 2.6 Update end flag
|
||||
end_flag = paddle.eq(hyps[:, -1], self.eos).view(-1, 1)
|
||||
|
||||
# 3. Select best of best
|
||||
scores = scores.view(batch_size, beam_size)
|
||||
# TODO: length normalization
|
||||
best_index = paddle.argmax(scores, axis=-1).long() # (B)
|
||||
best_hyps_index = best_index + paddle.arange(
|
||||
batch_size, dtype=paddle.long) * beam_size
|
||||
best_hyps = paddle.index_select(hyps, index=best_hyps_index, axis=0)
|
||||
best_hyps = best_hyps[:, 1:]
|
||||
return best_hyps
|
||||
|
||||
# @jit.to_static
|
||||
def subsampling_rate(self) -> int:
|
||||
""" Export interface for c++ call, return subsampling_rate of the
|
||||
model
|
||||
"""
|
||||
return self.encoder.embed.subsampling_rate
|
||||
|
||||
# @jit.to_static
|
||||
def right_context(self) -> int:
|
||||
""" Export interface for c++ call, return right_context of the model
|
||||
"""
|
||||
return self.encoder.embed.right_context
|
||||
|
||||
# @jit.to_static
|
||||
def sos_symbol(self) -> int:
|
||||
""" Export interface for c++ call, return sos symbol id of the model
|
||||
"""
|
||||
return self.sos
|
||||
|
||||
# @jit.to_static
|
||||
def eos_symbol(self) -> int:
|
||||
""" Export interface for c++ call, return eos symbol id of the model
|
||||
"""
|
||||
return self.eos
|
||||
|
||||
@jit.to_static
|
||||
def forward_encoder_chunk(
|
||||
self,
|
||||
xs: paddle.Tensor,
|
||||
offset: int,
|
||||
required_cache_size: int,
|
||||
subsampling_cache: Optional[paddle.Tensor]=None,
|
||||
elayers_output_cache: Optional[List[paddle.Tensor]]=None,
|
||||
conformer_cnn_cache: Optional[List[paddle.Tensor]]=None,
|
||||
) -> Tuple[paddle.Tensor, paddle.Tensor, List[paddle.Tensor], List[
|
||||
paddle.Tensor]]:
|
||||
""" Export interface for c++ call, give input chunk xs, and return
|
||||
output from time 0 to current chunk.
|
||||
Args:
|
||||
xs (paddle.Tensor): chunk input
|
||||
subsampling_cache (Optional[paddle.Tensor]): subsampling cache
|
||||
elayers_output_cache (Optional[List[paddle.Tensor]]):
|
||||
transformer/conformer encoder layers output cache
|
||||
conformer_cnn_cache (Optional[List[paddle.Tensor]]): conformer
|
||||
cnn cache
|
||||
Returns:
|
||||
paddle.Tensor: output, it ranges from time 0 to current chunk.
|
||||
paddle.Tensor: subsampling cache
|
||||
List[paddle.Tensor]: attention cache
|
||||
List[paddle.Tensor]: conformer cnn cache
|
||||
"""
|
||||
return self.encoder.forward_chunk(
|
||||
xs, offset, required_cache_size, subsampling_cache,
|
||||
elayers_output_cache, conformer_cnn_cache)
|
||||
|
||||
# @jit.to_static
|
||||
def ctc_activation(self, xs: paddle.Tensor) -> paddle.Tensor:
|
||||
""" Export interface for c++ call, apply linear transform and log
|
||||
softmax before ctc
|
||||
Args:
|
||||
xs (paddle.Tensor): encoder output
|
||||
Returns:
|
||||
paddle.Tensor: activation before ctc
|
||||
"""
|
||||
return self.ctc.log_softmax(xs)
|
||||
|
||||
@jit.to_static
|
||||
def forward_attention_decoder(
|
||||
self,
|
||||
hyps: paddle.Tensor,
|
||||
hyps_lens: paddle.Tensor,
|
||||
encoder_out: paddle.Tensor, ) -> paddle.Tensor:
|
||||
""" Export interface for c++ call, forward decoder with multiple
|
||||
hypothesis from ctc prefix beam search and one encoder output
|
||||
Args:
|
||||
hyps (paddle.Tensor): hyps from ctc prefix beam search, already
|
||||
pad sos at the begining, (B, T)
|
||||
hyps_lens (paddle.Tensor): length of each hyp in hyps, (B)
|
||||
encoder_out (paddle.Tensor): corresponding encoder output, (B=1, T, D)
|
||||
Returns:
|
||||
paddle.Tensor: decoder output, (B, L)
|
||||
"""
|
||||
assert encoder_out.size(0) == 1
|
||||
num_hyps = hyps.size(0)
|
||||
assert hyps_lens.size(0) == num_hyps
|
||||
encoder_out = encoder_out.repeat(num_hyps, 1, 1)
|
||||
# (B, 1, T)
|
||||
encoder_mask = paddle.ones(
|
||||
[num_hyps, 1, encoder_out.size(1)], dtype=paddle.bool)
|
||||
# (num_hyps, max_hyps_len, vocab_size)
|
||||
decoder_out, _ = self.decoder(encoder_out, encoder_mask, hyps,
|
||||
hyps_lens)
|
||||
decoder_out = paddle.nn.functional.log_softmax(decoder_out, dim=-1)
|
||||
return decoder_out
|
||||
|
||||
@paddle.no_grad()
|
||||
def decode(self,
|
||||
feats: paddle.Tensor,
|
||||
feats_lengths: paddle.Tensor,
|
||||
text_feature: Dict[str, int],
|
||||
decoding_method: str,
|
||||
lang_model_path: str,
|
||||
beam_alpha: float,
|
||||
beam_beta: float,
|
||||
beam_size: int,
|
||||
cutoff_prob: float,
|
||||
cutoff_top_n: int,
|
||||
num_processes: int,
|
||||
ctc_weight: float=0.0,
|
||||
decoding_chunk_size: int=-1,
|
||||
num_decoding_left_chunks: int=-1,
|
||||
simulate_streaming: bool=False):
|
||||
"""u2 decoding.
|
||||
|
||||
Args:
|
||||
feats (Tenosr): audio features, (B, T, D)
|
||||
feats_lengths (Tenosr): (B)
|
||||
text_feature (TextFeaturizer): text feature object.
|
||||
decoding_method (str): decoding mode, e.g.
|
||||
'fullsentence',
|
||||
'simultaneous'
|
||||
lang_model_path (str): lm path.
|
||||
beam_alpha (float): lm weight.
|
||||
beam_beta (float): length penalty.
|
||||
beam_size (int): beam size for search
|
||||
cutoff_prob (float): for prune.
|
||||
cutoff_top_n (int): for prune.
|
||||
num_processes (int):
|
||||
ctc_weight (float, optional): ctc weight for attention rescoring decode mode. Defaults to 0.0.
|
||||
decoding_chunk_size (int, optional): decoding chunk size. Defaults to -1.
|
||||
<0: for decoding, use full chunk.
|
||||
>0: for decoding, use fixed chunk size as set.
|
||||
0: used for training, it's prohibited here.
|
||||
num_decoding_left_chunks (int, optional):
|
||||
number of left chunks for decoding. Defaults to -1.
|
||||
simulate_streaming (bool, optional): simulate streaming inference. Defaults to False.
|
||||
|
||||
Raises:
|
||||
ValueError: when not support decoding_method.
|
||||
|
||||
Returns:
|
||||
List[List[int]]: transcripts.
|
||||
"""
|
||||
batch_size = feats.size(0)
|
||||
|
||||
if decoding_method == 'fullsentence':
|
||||
hyps = self.translate(
|
||||
feats,
|
||||
feats_lengths,
|
||||
beam_size=beam_size,
|
||||
decoding_chunk_size=decoding_chunk_size,
|
||||
num_decoding_left_chunks=num_decoding_left_chunks,
|
||||
simulate_streaming=simulate_streaming)
|
||||
hyps = [hyp.tolist() for hyp in hyps]
|
||||
else:
|
||||
raise ValueError(f"Not support decoding method: {decoding_method}")
|
||||
|
||||
res = [text_feature.defeaturize(hyp) for hyp in hyps]
|
||||
return res
|
||||
|
||||
|
||||
class U2STModel(U2STBaseModel):
|
||||
def __init__(self, configs: dict):
|
||||
vocab_size, encoder, decoder = U2STModel._init_from_config(configs)
|
||||
|
||||
if isinstance(decoder, Tuple):
|
||||
st_decoder, asr_decoder, ctc = decoder
|
||||
super().__init__(
|
||||
vocab_size=vocab_size,
|
||||
encoder=encoder,
|
||||
st_decoder=st_decoder,
|
||||
decoder=asr_decoder,
|
||||
ctc=ctc,
|
||||
**configs['model_conf'])
|
||||
else:
|
||||
super().__init__(
|
||||
vocab_size=vocab_size,
|
||||
encoder=encoder,
|
||||
st_decoder=decoder,
|
||||
**configs['model_conf'])
|
||||
|
||||
@classmethod
|
||||
def _init_from_config(cls, configs: dict):
|
||||
"""init sub module for model.
|
||||
|
||||
Args:
|
||||
configs (dict): config dict.
|
||||
|
||||
Raises:
|
||||
ValueError: raise when using not support encoder type.
|
||||
|
||||
Returns:
|
||||
int, nn.Layer, nn.Layer, nn.Layer: vocab size, encoder, decoder, ctc
|
||||
"""
|
||||
if configs['cmvn_file'] is not None:
|
||||
mean, istd = load_cmvn(configs['cmvn_file'],
|
||||
configs['cmvn_file_type'])
|
||||
global_cmvn = GlobalCMVN(
|
||||
paddle.to_tensor(mean, dtype=paddle.float),
|
||||
paddle.to_tensor(istd, dtype=paddle.float))
|
||||
else:
|
||||
global_cmvn = None
|
||||
|
||||
input_dim = configs['input_dim']
|
||||
vocab_size = configs['output_dim']
|
||||
assert input_dim != 0, input_dim
|
||||
assert vocab_size != 0, vocab_size
|
||||
|
||||
encoder_type = configs.get('encoder', 'transformer')
|
||||
logger.info(f"U2 Encoder type: {encoder_type}")
|
||||
if encoder_type == 'transformer':
|
||||
encoder = TransformerEncoder(
|
||||
input_dim, global_cmvn=global_cmvn, **configs['encoder_conf'])
|
||||
elif encoder_type == 'conformer':
|
||||
encoder = ConformerEncoder(
|
||||
input_dim, global_cmvn=global_cmvn, **configs['encoder_conf'])
|
||||
else:
|
||||
raise ValueError(f"not support encoder type:{encoder_type}")
|
||||
|
||||
st_decoder = TransformerDecoder(vocab_size,
|
||||
encoder.output_size(),
|
||||
**configs['decoder_conf'])
|
||||
|
||||
asr_weight = configs['model_conf']['asr_weight']
|
||||
logger.info(f"ASR Joint Training Weight: {asr_weight}")
|
||||
|
||||
if asr_weight > 0.:
|
||||
decoder = TransformerDecoder(vocab_size,
|
||||
encoder.output_size(),
|
||||
**configs['decoder_conf'])
|
||||
# ctc decoder and ctc loss
|
||||
model_conf = configs['model_conf']
|
||||
ctc = CTCDecoder(
|
||||
odim=vocab_size,
|
||||
enc_n_units=encoder.output_size(),
|
||||
blank_id=0,
|
||||
dropout_rate=model_conf['ctc_dropout_rate'],
|
||||
reduction=True, # sum
|
||||
batch_average=True, # sum / batch_size
|
||||
grad_norm_type=model_conf['ctc_grad_norm_type'])
|
||||
|
||||
return vocab_size, encoder, (st_decoder, decoder, ctc)
|
||||
else:
|
||||
return vocab_size, encoder, st_decoder
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, configs: dict):
|
||||
"""init model.
|
||||
|
||||
Args:
|
||||
configs (dict): config dict.
|
||||
|
||||
Raises:
|
||||
ValueError: raise when using not support encoder type.
|
||||
|
||||
Returns:
|
||||
nn.Layer: U2STModel
|
||||
"""
|
||||
model = cls(configs)
|
||||
return model
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, dataloader, config, checkpoint_path):
|
||||
"""Build a DeepSpeech2Model model from a pretrained model.
|
||||
|
||||
Args:
|
||||
dataloader (paddle.io.DataLoader): not used.
|
||||
config (yacs.config.CfgNode): model configs
|
||||
checkpoint_path (Path or str): the path of pretrained model checkpoint, without extension name
|
||||
|
||||
Returns:
|
||||
DeepSpeech2Model: The model built from pretrained result.
|
||||
"""
|
||||
with UpdateConfig(config):
|
||||
config.input_dim = dataloader.collate_fn.feature_size
|
||||
config.output_dim = dataloader.collate_fn.vocab_size
|
||||
|
||||
model = cls.from_config(config)
|
||||
|
||||
if checkpoint_path:
|
||||
infos = checkpoint.load_parameters(
|
||||
model, checkpoint_path=checkpoint_path)
|
||||
logger.info(f"checkpoint info: {infos}")
|
||||
layer_tools.summary(model)
|
||||
return model
|
||||
|
||||
|
||||
class U2STInferModel(U2STModel):
|
||||
def __init__(self, configs: dict):
|
||||
super().__init__(configs)
|
||||
|
||||
def forward(self,
|
||||
feats,
|
||||
feats_lengths,
|
||||
decoding_chunk_size=-1,
|
||||
num_decoding_left_chunks=-1,
|
||||
simulate_streaming=False):
|
||||
"""export model function
|
||||
|
||||
Args:
|
||||
feats (Tensor): [B, T, D]
|
||||
feats_lengths (Tensor): [B]
|
||||
|
||||
Returns:
|
||||
List[List[int]]: best path result
|
||||
"""
|
||||
return self.translate(
|
||||
feats,
|
||||
feats_lengths,
|
||||
decoding_chunk_size=decoding_chunk_size,
|
||||
num_decoding_left_chunks=num_decoding_left_chunks,
|
||||
simulate_streaming=simulate_streaming)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue