msb-public
/
PaddleSpeech
mirror of https://github.com/PaddlePaddle/PaddleSpeech
1
0
Code Issues Projects Releases Wiki Activity

reorder imports

Browse Source
pull/578/head
Hui Zhang 5 years ago
parent 708b7fd39b
commit fa6e333422
94 changed files with 402 additions and 373 deletions
Show Stats Download Patch File Download Diff File

117
.notebook/python_test.ipynb
Unescape View File

@ -637,7 +637,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 59, "execution_count": 59,
"id": "featured-cookbook", "id": "norwegian-cleveland",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -660,7 +660,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 35, "execution_count": 35,
"id": "concrete-rubber", "id": "endless-kidney",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -705,7 +705,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": 3,
"id": "surgical-shore", "id": "later-louisiana",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -728,7 +728,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": 4,
"id": "medieval-daily", "id": "funded-nudist",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -748,7 +748,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 5,
"id": "derived-meeting", "id": "contrary-affiliation",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -758,7 +758,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": 6,
"id": "hawaiian-engagement", "id": "friendly-interpretation",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -776,7 +776,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 7, "execution_count": 7,
"id": "moderate-greece", "id": "bottom-wilderness",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -809,7 +809,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 17, "execution_count": 17,
"id": "duplicate-colon", "id": "acquired-jacksonville",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -839,7 +839,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 18, "execution_count": 18,
"id": "actual-junior", "id": "entertaining-capture",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -871,7 +871,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 19, "execution_count": 19,
"id": "looking-belle", "id": "amber-grade",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -903,7 +903,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 31, "execution_count": 31,
"id": "turned-friend", "id": "hidden-playback",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -935,7 +935,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 32, "execution_count": 32,
"id": "wrapped-foundation", "id": "twelve-university",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -966,7 +966,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 20, "execution_count": 20,
"id": "applied-abuse", "id": "minor-endorsement",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -977,7 +977,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 46, "execution_count": 46,
"id": "experimental-invasion", "id": "upper-majority",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -997,7 +997,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 30, "execution_count": 30,
"id": "established-lunch", "id": "supreme-coverage",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -1007,7 +1007,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 31, "execution_count": 31,
"id": "residential-logic", "id": "tough-domain",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -1017,7 +1017,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 47, "execution_count": 47,
"id": "maritime-consequence", "id": "indian-empire",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1038,7 +1038,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": 1,
"id": "african-identification", "id": "horizontal-paragraph",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -1049,7 +1049,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 56, "execution_count": 56,
"id": "upset-regard", "id": "homeless-zoning",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -1059,7 +1059,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 58, "execution_count": 58,
"id": "wrapped-netherlands", "id": "floating-atmosphere",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -1069,7 +1069,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 60, "execution_count": 60,
"id": "engaging-statement", "id": "stupid-reducing",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -1079,7 +1079,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 66, "execution_count": 66,
"id": "joined-handy", "id": "practical-airline",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -1089,7 +1089,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 95, "execution_count": 95,
"id": "absolute-kenya", "id": "apart-comfort",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1110,7 +1110,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 81, "execution_count": 81,
"id": "integral-subsection", "id": "underlying-brand",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1131,7 +1131,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 11,
"id": "conservative-sport", "id": "german-things",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -1161,7 +1161,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 12, "execution_count": 12,
"id": "connected-brass", "id": "third-regression",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -1172,7 +1172,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 14,
"id": "worse-couple", "id": "thick-korea",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1201,7 +1201,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 13, "execution_count": 13,
"id": "unlimited-effects", "id": "institutional-hands",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1230,7 +1230,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 15, "execution_count": 15,
"id": "stable-artist", "id": "brave-native",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1260,7 +1260,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 29, "execution_count": 29,
"id": "romance-laser", "id": "turkish-ticket",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1290,7 +1290,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 30, "execution_count": 30,
"id": "pharmaceutical-colombia", "id": "executed-excerpt",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1308,7 +1308,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 22, "execution_count": 22,
"id": "forty-harvest", "id": "continental-boring",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1343,7 +1343,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 23, "execution_count": 23,
"id": "thirty-software", "id": "linear-assembly",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1376,7 +1376,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 33, "execution_count": 33,
"id": "portuguese-addiction", "id": "applied-louis",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -1386,7 +1386,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 34, "execution_count": 34,
"id": "everyday-vessel", "id": "historic-struggle",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1428,17 +1428,60 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 35, "execution_count": 35,
"id": "latter-expression", "id": "monthly-roads",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"getpass?" "getpass?"
] ]
}, },
{
"cell_type": "code",
"execution_count": 38,
"id": "boxed-peoples",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Worker:"
]
}
],
"source": [
"import multiprocessing\n",
"import cProfile\n",
"import time\n",
"\n",
"def worker(num):\n",
" time.sleep(3)\n",
" print('Worker:', num)\n",
"\n",
"def profile_worker(num):\n",
" cProfile.runctx('worker(num)', globals(), locals(), 'profile-%d.out' %num)\n",
"\n",
"\n",
"\n",
"for i in range(5):\n",
" p = multiprocessing.Process(target=profile_worker, args=(i,))\n",
" p.start()"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "fresh-tender",
"metadata": {},
"outputs": [],
"source": [
"!ls"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "small-professor", "id": "religious-peripheral",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [] "source": []

8
.pre-commit-config.yaml
Unescape View File

@ -52,7 +52,7 @@
language: system language: system
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|py)$ files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|py)$
#exclude: (?=decoders/swig).*(\.cpp|\.h)$ #exclude: (?=decoders/swig).*(\.cpp|\.h)$
#- repo: https://github.com/asottile/reorder_python_imports - repo: https://github.com/asottile/reorder_python_imports
# rev: v2.4.0 rev: v2.4.0
# hooks: hooks:
# - id: reorder-python-imports - id: reorder-python-imports

8
deepspeech/__init__.py
Unescape View File

@ -11,18 +11,18 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from typing import Union from typing import Any
from typing import List from typing import List
from typing import Tuple from typing import Tuple
from typing import Any from typing import Union
import paddle import paddle
from paddle import nn from paddle import nn
from paddle.nn import functional as F
#TODO(Hui Zhang): remove fluid import
from paddle.fluid import core from paddle.fluid import core
from paddle.nn import functional as F
from deepspeech.utils.log import Log from deepspeech.utils.log import Log
#TODO(Hui Zhang): remove fluid import
logger = Log(__name__).getlog() logger = Log(__name__).getlog()
########### hcak logging ############# ########### hcak logging #############

6
deepspeech/decoders/decoders_deprecated.py
Unescape View File

@ -12,11 +12,11 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains various CTC decoders.""" """Contains various CTC decoders."""
import multiprocessing
from itertools import groupby from itertools import groupby
import numpy as np
from math import log from math import log
import multiprocessing
import numpy as np
def ctc_greedy_decoder(probs_seq, vocabulary): def ctc_greedy_decoder(probs_seq, vocabulary):

2
deepspeech/decoders/scorer_deprecated.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""External Scorer for Beam Search Decoder.""" """External Scorer for Beam Search Decoder."""
import os import os
import kenlm import kenlm
import numpy as np import numpy as np

12
deepspeech/decoders/swig/setup.py
Unescape View File

@ -12,14 +12,16 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Script to build and install decoder package.""" """Script to build and install decoder package."""
import argparse
from setuptools import setup, Extension, distutils
import glob import glob
import platform import multiprocessing.pool
import os import os
import platform
import sys import sys
import multiprocessing.pool
import argparse from setuptools import distutils
from setuptools import Extension
from setuptools import setup
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument( parser.add_argument(

1
deepspeech/decoders/swig_wrapper.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Wrapper for various CTC decoders in SWIG.""" """Wrapper for various CTC decoders in SWIG."""
import swig_decoders import swig_decoders

2
deepspeech/decoders/tests/test_decoders.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Test decoders.""" """Test decoders."""
import unittest import unittest
from deepspeech.decoders import decoders_deprecated as decoder from deepspeech.decoders import decoders_deprecated as decoder

5
deepspeech/exps/deepspeech2/bin/deploy/client.py
Unescape View File

@ -12,9 +12,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Client-end for the ASR demo.""" """Client-end for the ASR demo."""
import keyboard
import sys
import argparse import argparse
import sys
import keyboard
import pyaudio import pyaudio
from deepspeech.utils.socket_server import socket_send from deepspeech.utils.socket_server import socket_send

3
deepspeech/exps/deepspeech2/bin/deploy/record.py
Unescape View File

@ -13,9 +13,10 @@
# limitations under the License. # limitations under the License.
"""Record wav from Microphone""" """Record wav from Microphone"""
# http://people.csail.mit.edu/hubert/pyaudio/ # http://people.csail.mit.edu/hubert/pyaudio/
import pyaudio
import wave import wave
import pyaudio
CHUNK = 1024 CHUNK = 1024
FORMAT = pyaudio.paInt16 FORMAT = pyaudio.paInt16
CHANNELS = 1 CHANNELS = 1

24
deepspeech/exps/deepspeech2/bin/deploy/runtime.py
Unescape View File

@ -13,23 +13,21 @@
# limitations under the License. # limitations under the License.
"""Server-end for the ASR demo.""" """Server-end for the ASR demo."""
import functools import functools
import paddle
import numpy as np
from deepspeech.utils.socket_server import warm_up_test import numpy as np
from deepspeech.utils.socket_server import AsrTCPServer import paddle
from deepspeech.utils.socket_server import AsrRequestHandler from paddle.inference import Config
from paddle.inference import create_predictor
from deepspeech.training.cli import default_argument_parser
from deepspeech.exps.deepspeech2.config import get_cfg_defaults from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.utils.utility import add_arguments, print_arguments
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.io.dataset import ManifestDataset from deepspeech.io.dataset import ManifestDataset
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from paddle.inference import Config from deepspeech.training.cli import default_argument_parser
from paddle.inference import create_predictor from deepspeech.utils.socket_server import AsrRequestHandler
from deepspeech.utils.socket_server import AsrTCPServer
from deepspeech.utils.socket_server import warm_up_test
from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments
def init_predictor(args): def init_predictor(args):

19
deepspeech/exps/deepspeech2/bin/deploy/server.py
Unescape View File

@ -13,20 +13,19 @@
# limitations under the License. # limitations under the License.
"""Server-end for the ASR demo.""" """Server-end for the ASR demo."""
import functools import functools
import paddle
import numpy as np
from deepspeech.utils.socket_server import warm_up_test import numpy as np
from deepspeech.utils.socket_server import AsrTCPServer import paddle
from deepspeech.utils.socket_server import AsrRequestHandler
from deepspeech.training.cli import default_argument_parser
from deepspeech.exps.deepspeech2.config import get_cfg_defaults from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.utils.utility import add_arguments, print_arguments
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.io.dataset import ManifestDataset from deepspeech.io.dataset import ManifestDataset
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.socket_server import AsrRequestHandler
from deepspeech.utils.socket_server import AsrTCPServer
from deepspeech.utils.socket_server import warm_up_test
from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments
def start_server(config, args): def start_server(config, args):

5
deepspeech/exps/deepspeech2/bin/export.py
Unescape View File

@ -12,11 +12,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Export for DeepSpeech2 model.""" """Export for DeepSpeech2 model."""
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
from deepspeech.exps.deepspeech2.config import get_cfg_defaults from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
def main_sp(config, args): def main_sp(config, args):

4
deepspeech/exps/deepspeech2/bin/infer.py
Unescape View File

@ -12,12 +12,12 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Inferer for DeepSpeech2 model.""" """Inferer for DeepSpeech2 model."""
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester
from deepspeech.training.cli import default_argument_parser from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments from deepspeech.utils.utility import print_arguments
# TODO(hui zhang): dynamic load # TODO(hui zhang): dynamic load
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester
def main_sp(config, args): def main_sp(config, args):

5
deepspeech/exps/deepspeech2/bin/test.py
Unescape View File

@ -12,11 +12,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Evaluation for DeepSpeech2 model.""" """Evaluation for DeepSpeech2 model."""
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
from deepspeech.exps.deepspeech2.config import get_cfg_defaults from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
def main_sp(config, args): def main_sp(config, args):

5
deepspeech/exps/deepspeech2/bin/train.py
Unescape View File

@ -14,11 +14,10 @@
"""Trainer for DeepSpeech2 model.""" """Trainer for DeepSpeech2 model."""
from paddle import distributed as dist from paddle import distributed as dist
from deepspeech.utils.utility import print_arguments
from deepspeech.training.cli import default_argument_parser
from deepspeech.exps.deepspeech2.config import get_cfg_defaults from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.exps.deepspeech2.model import DeepSpeech2Trainer as Trainer from deepspeech.exps.deepspeech2.model import DeepSpeech2Trainer as Trainer
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
def main_sp(config, args): def main_sp(config, args):

16
deepspeech/exps/deepspeech2/bin/tune.py
Unescape View File

@ -12,22 +12,20 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Beam search parameters tuning for DeepSpeech2 model.""" """Beam search parameters tuning for DeepSpeech2 model."""
import sys
import numpy as np
import functools import functools
import sys
import numpy as np
from paddle.io import DataLoader from paddle.io import DataLoader
from deepspeech.utils import error_rate from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.utils.utility import add_arguments, print_arguments
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.io.collator import SpeechCollator from deepspeech.io.collator import SpeechCollator
from deepspeech.io.dataset import ManifestDataset from deepspeech.io.dataset import ManifestDataset
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.training.cli import default_argument_parser from deepspeech.training.cli import default_argument_parser
from deepspeech.exps.deepspeech2.config import get_cfg_defaults from deepspeech.utils import error_rate
from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments
def tune(config, args): def tune(config, args):

2
deepspeech/exps/deepspeech2/config.py
Unescape View File

@ -11,8 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from yacs.config import CfgNode as CN from yacs.config import CfgNode as CN
from deepspeech.models.deepspeech2 import DeepSpeech2Model from deepspeech.models.deepspeech2 import DeepSpeech2Model
_C = CN() _C = CN()

24
deepspeech/exps/deepspeech2/model.py
Unescape View File

@ -12,31 +12,27 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains DeepSpeech2 model.""" """Contains DeepSpeech2 model."""
import time import time
from deepspeech.utils.log import Log
import numpy as np
from collections import defaultdict from collections import defaultdict
from pathlib import Path from pathlib import Path
import numpy as np
import paddle import paddle
from paddle import distributed as dist from paddle import distributed as dist
from paddle.io import DataLoader from paddle.io import DataLoader
from deepspeech.training.trainer import Trainer
from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog
from deepspeech.utils import mp_tools
from deepspeech.utils import layer_tools
from deepspeech.utils import error_rate
from deepspeech.io.collator import SpeechCollator from deepspeech.io.collator import SpeechCollator
from deepspeech.io.sampler import SortagradDistributedBatchSampler
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.io.dataset import ManifestDataset from deepspeech.io.dataset import ManifestDataset
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.models.deepspeech2 import DeepSpeech2Model from deepspeech.io.sampler import SortagradDistributedBatchSampler
from deepspeech.models.deepspeech2 import DeepSpeech2InferModel from deepspeech.models.deepspeech2 import DeepSpeech2InferModel
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog
from deepspeech.training.trainer import Trainer
from deepspeech.utils import error_rate
from deepspeech.utils import layer_tools
from deepspeech.utils import mp_tools
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

6
deepspeech/exps/u2/bin/export.py
Unescape View File

@ -12,12 +12,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Export for U2 model.""" """Export for U2 model."""
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
from deepspeech.exps.u2.config import get_cfg_defaults from deepspeech.exps.u2.config import get_cfg_defaults
from deepspeech.exps.u2.model import U2Tester as Tester from deepspeech.exps.u2.model import U2Tester as Tester
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
def main_sp(config, args): def main_sp(config, args):

6
deepspeech/exps/u2/bin/test.py
Unescape View File

@ -12,15 +12,15 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Evaluation for U2 model.""" """Evaluation for U2 model."""
import os
import cProfile import cProfile
import os
from deepspeech.exps.u2.config import get_cfg_defaults
from deepspeech.exps.u2.model import U2Tester as Tester
from deepspeech.training.cli import default_argument_parser from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments from deepspeech.utils.utility import print_arguments
# TODO(hui zhang): dynamic load # TODO(hui zhang): dynamic load
from deepspeech.exps.u2.config import get_cfg_defaults
from deepspeech.exps.u2.model import U2Tester as Tester
def main_sp(config, args): def main_sp(config, args):

9
deepspeech/exps/u2/bin/train.py
Unescape View File

@ -12,16 +12,15 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Trainer for U2 model.""" """Trainer for U2 model."""
import os
import cProfile import cProfile
from paddle import distributed as dist import os
from deepspeech.utils.utility import print_arguments from paddle import distributed as dist
from deepspeech.training.cli import default_argument_parser
from deepspeech.exps.u2.config import get_cfg_defaults from deepspeech.exps.u2.config import get_cfg_defaults
from deepspeech.exps.u2.model import U2Trainer as Trainer from deepspeech.exps.u2.model import U2Trainer as Trainer
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
def main_sp(config, args): def main_sp(config, args):

5
deepspeech/exps/u2/config.py
Unescape View File

@ -11,13 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from yacs.config import CfgNode from yacs.config import CfgNode
from deepspeech.exps.u2.model import U2Tester
from deepspeech.exps.u2.model import U2Trainer
from deepspeech.io.dataset import ManifestDataset from deepspeech.io.dataset import ManifestDataset
from deepspeech.models.u2 import U2Model from deepspeech.models.u2 import U2Model
from deepspeech.exps.u2.model import U2Trainer
from deepspeech.exps.u2.model import U2Tester
_C = CfgNode() _C = CfgNode()

26
deepspeech/exps/u2/model.py
Unescape View File

@ -12,34 +12,30 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains U2 model.""" """Contains U2 model."""
import sys import sys
import time import time
from deepspeech.utils.log import Log
import numpy as np
from collections import defaultdict from collections import defaultdict
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
from yacs.config import CfgNode
import numpy as np
import paddle import paddle
from paddle import distributed as dist from paddle import distributed as dist
from paddle.io import DataLoader from paddle.io import DataLoader
from yacs.config import CfgNode
from deepspeech.training.trainer import Trainer
from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog
from deepspeech.training.scheduler import WarmupLR
from deepspeech.utils import mp_tools
from deepspeech.utils import layer_tools
from deepspeech.utils import error_rate
from deepspeech.io.collator import SpeechCollator from deepspeech.io.collator import SpeechCollator
from deepspeech.io.sampler import SortagradDistributedBatchSampler
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.io.dataset import ManifestDataset from deepspeech.io.dataset import ManifestDataset
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.io.sampler import SortagradDistributedBatchSampler
from deepspeech.models.u2 import U2Model from deepspeech.models.u2 import U2Model
from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog
from deepspeech.training.scheduler import WarmupLR
from deepspeech.training.trainer import Trainer
from deepspeech.utils import error_rate
from deepspeech.utils import layer_tools
from deepspeech.utils import mp_tools
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

12
deepspeech/frontend/audio.py
Unescape View File

@ -12,16 +12,16 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the audio segment class.""" """Contains the audio segment class."""
import copy
import numpy as np
import io import io
import struct import random
import re import re
import soundfile import struct
import numpy as np
import resampy import resampy
import soundfile
from scipy import signal from scipy import signal
import random
import copy
class AudioSegment(object): class AudioSegment(object):

14
deepspeech/frontend/augmentor/augmentation.py
Unescape View File

@ -12,21 +12,19 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the data augmentation pipeline.""" """Contains the data augmentation pipeline."""
import json import json
import numpy as np import numpy as np
# audio augment
from deepspeech.frontend.augmentor.volume_perturb import VolumePerturbAugmentor
from deepspeech.frontend.augmentor.shift_perturb import ShiftPerturbAugmentor
from deepspeech.frontend.augmentor.speed_perturb import SpeedPerturbAugmentor
from deepspeech.frontend.augmentor.noise_perturb import NoisePerturbAugmentor
from deepspeech.frontend.augmentor.impulse_response import ImpulseResponseAugmentor from deepspeech.frontend.augmentor.impulse_response import ImpulseResponseAugmentor
from deepspeech.frontend.augmentor.resample import ResampleAugmentor from deepspeech.frontend.augmentor.noise_perturb import NoisePerturbAugmentor
from deepspeech.frontend.augmentor.online_bayesian_normalization import \ from deepspeech.frontend.augmentor.online_bayesian_normalization import \
OnlineBayesianNormalizationAugmentor OnlineBayesianNormalizationAugmentor
# feature augment from deepspeech.frontend.augmentor.resample import ResampleAugmentor
from deepspeech.frontend.augmentor.shift_perturb import ShiftPerturbAugmentor
from deepspeech.frontend.augmentor.spec_augment import SpecAugmentor from deepspeech.frontend.augmentor.spec_augment import SpecAugmentor
from deepspeech.frontend.augmentor.speed_perturb import SpeedPerturbAugmentor
from deepspeech.frontend.augmentor.volume_perturb import VolumePerturbAugmentor
class AugmentationPipeline(): class AugmentationPipeline():

4
deepspeech/frontend/augmentor/base.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the abstract base class for augmentation models.""" """Contains the abstract base class for augmentation models."""
from abc import ABCMeta
from abc import ABCMeta, abstractmethod from abc import abstractmethod
class AugmentorBase(): class AugmentorBase():

3
deepspeech/frontend/augmentor/impulse_response.py
Unescape View File

@ -12,10 +12,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the impulse response augmentation model.""" """Contains the impulse response augmentation model."""
from deepspeech.frontend.audio import AudioSegment
from deepspeech.frontend.augmentor.base import AugmentorBase from deepspeech.frontend.augmentor.base import AugmentorBase
from deepspeech.frontend.utility import read_manifest from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.audio import AudioSegment
class ImpulseResponseAugmentor(AugmentorBase): class ImpulseResponseAugmentor(AugmentorBase):

3
deepspeech/frontend/augmentor/noise_perturb.py
Unescape View File

@ -12,10 +12,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the noise perturb augmentation model.""" """Contains the noise perturb augmentation model."""
from deepspeech.frontend.audio import AudioSegment
from deepspeech.frontend.augmentor.base import AugmentorBase from deepspeech.frontend.augmentor.base import AugmentorBase
from deepspeech.frontend.utility import read_manifest from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.audio import AudioSegment
class NoisePerturbAugmentor(AugmentorBase): class NoisePerturbAugmentor(AugmentorBase):

1
deepspeech/frontend/augmentor/online_bayesian_normalization.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contain the online bayesian normalization augmentation model.""" """Contain the online bayesian normalization augmentation model."""
from deepspeech.frontend.augmentor.base import AugmentorBase from deepspeech.frontend.augmentor.base import AugmentorBase

1
deepspeech/frontend/augmentor/resample.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contain the resample augmentation model.""" """Contain the resample augmentation model."""
from deepspeech.frontend.augmentor.base import AugmentorBase from deepspeech.frontend.augmentor.base import AugmentorBase

1
deepspeech/frontend/augmentor/shift_perturb.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the volume perturb augmentation model.""" """Contains the volume perturb augmentation model."""
from deepspeech.frontend.augmentor.base import AugmentorBase from deepspeech.frontend.augmentor.base import AugmentorBase

2
deepspeech/frontend/augmentor/spec_augment.py
Unescape View File

@ -12,10 +12,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the volume perturb augmentation model.""" """Contains the volume perturb augmentation model."""
from deepspeech.utils.log import Log
import numpy as np import numpy as np
from deepspeech.frontend.augmentor.base import AugmentorBase from deepspeech.frontend.augmentor.base import AugmentorBase
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

2
deepspeech/frontend/augmentor/speed_perturb.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contain the speech perturbation augmentation model.""" """Contain the speech perturbation augmentation model."""
import numpy as np import numpy as np
from deepspeech.frontend.augmentor.base import AugmentorBase from deepspeech.frontend.augmentor.base import AugmentorBase

1
deepspeech/frontend/augmentor/volume_perturb.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the volume perturb augmentation model.""" """Contains the volume perturb augmentation model."""
from deepspeech.frontend.augmentor.base import AugmentorBase from deepspeech.frontend.augmentor.base import AugmentorBase

5
deepspeech/frontend/featurizer/audio_featurizer.py
Unescape View File

@ -12,11 +12,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the audio featurizer class.""" """Contains the audio featurizer class."""
import numpy as np import numpy as np
from python_speech_features import mfcc
from python_speech_features import logfbank
from python_speech_features import delta from python_speech_features import delta
from python_speech_features import logfbank
from python_speech_features import mfcc
class AudioFeaturizer(object): class AudioFeaturizer(object):

1
deepspeech/frontend/featurizer/speech_featurizer.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the speech featurizer class.""" """Contains the speech featurizer class."""
from deepspeech.frontend.featurizer.audio_featurizer import AudioFeaturizer from deepspeech.frontend.featurizer.audio_featurizer import AudioFeaturizer
from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer

3
deepspeech/frontend/featurizer/text_featurizer.py
Unescape View File

@ -12,11 +12,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the text featurizer class.""" """Contains the text featurizer class."""
import sentencepiece as spm import sentencepiece as spm
from deepspeech.frontend.utility import UNK
from deepspeech.frontend.utility import EOS from deepspeech.frontend.utility import EOS
from deepspeech.frontend.utility import UNK
class TextFeaturizer(object): class TextFeaturizer(object):

7
deepspeech/frontend/normalizer.py
Unescape View File

@ -12,12 +12,13 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains feature normalizers.""" """Contains feature normalizers."""
import random
import numpy as np import numpy as np
import random
from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.utility import load_cmvn
from deepspeech.frontend.audio import AudioSegment from deepspeech.frontend.audio import AudioSegment
from deepspeech.frontend.utility import load_cmvn
from deepspeech.frontend.utility import read_manifest
class FeatureNormalizer(object): class FeatureNormalizer(object):

2
deepspeech/frontend/speech.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains the speech segment class.""" """Contains the speech segment class."""
import numpy as np import numpy as np
from deepspeech.frontend.audio import AudioSegment from deepspeech.frontend.audio import AudioSegment

8
deepspeech/frontend/utility.py
Unescape View File

@ -12,10 +12,12 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains data helper functions.""" """Contains data helper functions."""
import numpy as np
import math
import json
import codecs import codecs
import json
import math
import numpy as np
from deepspeech.utils.log import Log from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

5
deepspeech/io/__init__.py
Unescape View File

@ -11,14 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import numpy as np import numpy as np
from paddle.io import DataLoader from paddle.io import DataLoader
from deepspeech.io.collator import SpeechCollator from deepspeech.io.collator import SpeechCollator
from deepspeech.io.sampler import SortagradDistributedBatchSampler
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.io.dataset import ManifestDataset from deepspeech.io.dataset import ManifestDataset
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.io.sampler import SortagradDistributedBatchSampler
def create_dataloader(manifest_path, def create_dataloader(manifest_path,

5
deepspeech/io/collator.py
Unescape View File

@ -11,12 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from deepspeech.utils.log import Log
import numpy as np import numpy as np
from deepspeech.io.utility import pad_sequence
from deepspeech.frontend.utility import IGNORE_ID from deepspeech.frontend.utility import IGNORE_ID
from deepspeech.io.utility import pad_sequence
from deepspeech.utils.log import Log
__all__ = ["SpeechCollator"] __all__ = ["SpeechCollator"]

13
deepspeech/io/dataset.py
Unescape View File

@ -11,23 +11,22 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import time
import io import io
import random import random
import tarfile import tarfile
from deepspeech.utils.log import Log import time
from typing import Optional
from yacs.config import CfgNode
from collections import namedtuple from collections import namedtuple
from typing import Optional
from paddle.io import Dataset from paddle.io import Dataset
from yacs.config import CfgNode
from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.augmentor.augmentation import AugmentationPipeline from deepspeech.frontend.augmentor.augmentation import AugmentationPipeline
from deepspeech.frontend.featurizer.speech_featurizer import SpeechFeaturizer from deepspeech.frontend.featurizer.speech_featurizer import SpeechFeaturizer
from deepspeech.frontend.speech import SpeechSegment
from deepspeech.frontend.normalizer import FeatureNormalizer from deepspeech.frontend.normalizer import FeatureNormalizer
from deepspeech.frontend.speech import SpeechSegment
from deepspeech.frontend.utility import read_manifest
from deepspeech.utils.log import Log
__all__ = [ __all__ = [
"ManifestDataset", "ManifestDataset",

8
deepspeech/io/sampler.py
Unescape View File

@ -11,14 +11,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import math import math
from deepspeech.utils.log import Log
import numpy as np
import numpy as np
from paddle import distributed as dist
from paddle.io import BatchSampler from paddle.io import BatchSampler
from paddle.io import DistributedBatchSampler from paddle.io import DistributedBatchSampler
from paddle import distributed as dist
from deepspeech.utils.log import Log
__all__ = [ __all__ = [
"SortagradDistributedBatchSampler", "SortagradDistributedBatchSampler",

5
deepspeech/io/utility.py
Unescape View File

@ -11,10 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from typing import List
from deepspeech.utils.log import Log
import numpy as np import numpy as np
from typing import List
from deepspeech.utils.log import Log
__all__ = ["pad_sequence"] __all__ = ["pad_sequence"]

7
deepspeech/models/deepspeech2.py
Unescape View File

@ -12,19 +12,18 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Deepspeech2 ASR Model""" """Deepspeech2 ASR Model"""
from deepspeech.utils.log import Log
from typing import Optional from typing import Optional
from yacs.config import CfgNode
import paddle import paddle
from paddle import nn from paddle import nn
from yacs.config import CfgNode
from deepspeech.modules.conv import ConvStack from deepspeech.modules.conv import ConvStack
from deepspeech.modules.rnn import RNNStack
from deepspeech.modules.ctc import CTCDecoder from deepspeech.modules.ctc import CTCDecoder
from deepspeech.modules.rnn import RNNStack
from deepspeech.utils import checkpoint from deepspeech.utils import checkpoint
from deepspeech.utils import layer_tools from deepspeech.utils import layer_tools
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

37
deepspeech/models/u2.py
Unescape View File

@ -15,40 +15,39 @@
Unified Streaming and Non-streaming Two-pass End-to-end Model for Speech Recognition Unified Streaming and Non-streaming Two-pass End-to-end Model for Speech Recognition
(https://arxiv.org/pdf/2012.05481.pdf) (https://arxiv.org/pdf/2012.05481.pdf)
""" """
import time
import sys import sys
import time
from collections import defaultdict from collections import defaultdict
from deepspeech.utils.log import Log from typing import Dict
from yacs.config import CfgNode from typing import List
from typing import List, Optional, Tuple, Dict from typing import Optional
from typing import Tuple
import paddle import paddle
from paddle import jit from paddle import jit
from paddle import nn from paddle import nn
from yacs.config import CfgNode
from deepspeech.modules.mask import make_pad_mask from deepspeech.frontend.utility import IGNORE_ID
from deepspeech.modules.mask import mask_finished_preds from deepspeech.frontend.utility import load_cmvn
from deepspeech.modules.mask import mask_finished_scores
from deepspeech.modules.mask import subsequent_mask
from deepspeech.modules.cmvn import GlobalCMVN from deepspeech.modules.cmvn import GlobalCMVN
from deepspeech.modules.encoder import ConformerEncoder
from deepspeech.modules.encoder import TransformerEncoder
from deepspeech.modules.ctc import CTCDecoder from deepspeech.modules.ctc import CTCDecoder
from deepspeech.modules.decoder import TransformerDecoder from deepspeech.modules.decoder import TransformerDecoder
from deepspeech.modules.encoder import ConformerEncoder
from deepspeech.modules.encoder import TransformerEncoder
from deepspeech.modules.loss import LabelSmoothingLoss from deepspeech.modules.loss import LabelSmoothingLoss
from deepspeech.modules.mask import make_pad_mask
from deepspeech.frontend.utility import load_cmvn from deepspeech.modules.mask import mask_finished_preds
from deepspeech.frontend.utility import IGNORE_ID from deepspeech.modules.mask import mask_finished_scores
from deepspeech.modules.mask import subsequent_mask
from deepspeech.utils import checkpoint from deepspeech.utils import checkpoint
from deepspeech.utils import layer_tools from deepspeech.utils import layer_tools
from deepspeech.utils.utility import log_add from deepspeech.utils.ctc_utils import remove_duplicates_and_blank
from deepspeech.utils.log import Log
from deepspeech.utils.tensor_utils import add_sos_eos from deepspeech.utils.tensor_utils import add_sos_eos
from deepspeech.utils.tensor_utils import th_accuracy
from deepspeech.utils.tensor_utils import pad_sequence from deepspeech.utils.tensor_utils import pad_sequence
from deepspeech.utils.ctc_utils import remove_duplicates_and_blank from deepspeech.utils.tensor_utils import th_accuracy
from deepspeech.utils.utility import log_add
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

4
deepspeech/modules/activation.py
Unescape View File

@ -11,13 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from deepspeech.utils.log import Log
from collections import OrderedDict from collections import OrderedDict
import paddle import paddle
from paddle import nn from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()
__all__ = ["get_activation", "brelu", "LinearGLUBlock", "ConvGLUBlock"] __all__ = ["get_activation", "brelu", "LinearGLUBlock", "ConvGLUBlock"]

6
deepspeech/modules/attention.py
Unescape View File

@ -13,13 +13,15 @@
# limitations under the License. # limitations under the License.
"""Multi-Head Attention layer definition.""" """Multi-Head Attention layer definition."""
import math import math
from deepspeech.utils.log import Log from typing import Optional
from typing import Optional, Tuple from typing import Tuple
import paddle import paddle
from paddle import nn from paddle import nn
from paddle.nn import initializer as I from paddle.nn import initializer as I
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()
__all__ = ["MultiHeadedAttention", "RelPositionMultiHeadedAttention"] __all__ = ["MultiHeadedAttention", "RelPositionMultiHeadedAttention"]

5
deepspeech/modules/cmvn.py
Unescape View File

@ -11,12 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from deepspeech.utils.log import Log
import paddle import paddle
from paddle import nn from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()
__all__ = ['GlobalCMVN'] __all__ = ['GlobalCMVN']

9
deepspeech/modules/conformer_convolution.py
Unescape View File

@ -12,13 +12,14 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""ConvolutionModule definition.""" """ConvolutionModule definition."""
from typing import Optional, Tuple from typing import Optional
from typeguard import check_argument_types from typing import Tuple
from deepspeech.utils.log import Log
import paddle import paddle
from paddle import nn from paddle import nn
from typeguard import check_argument_types
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

6
deepspeech/modules/conv.py
Unescape View File

@ -11,14 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from deepspeech.utils.log import Log
from paddle import nn from paddle import nn
from paddle.nn import functional as F from paddle.nn import functional as F
from deepspeech.modules.mask import sequence_mask
from deepspeech.modules.activation import brelu from deepspeech.modules.activation import brelu
from deepspeech.modules.mask import sequence_mask
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

13
deepspeech/modules/ctc.py
Unescape View File

@ -11,20 +11,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from deepspeech.utils.log import Log
from typeguard import check_argument_types
import paddle import paddle
from paddle import nn from paddle import nn
from paddle.nn import functional as F from paddle.nn import functional as F
from typeguard import check_argument_types
from deepspeech.decoders.swig_wrapper import ctc_beam_search_decoder_batch
from deepspeech.decoders.swig_wrapper import ctc_greedy_decoder
from deepspeech.decoders.swig_wrapper import Scorer
from deepspeech.modules.loss import CTCLoss from deepspeech.modules.loss import CTCLoss
from deepspeech.utils import ctc_utils from deepspeech.utils import ctc_utils
from deepspeech.utils.log import Log
from deepspeech.decoders.swig_wrapper import Scorer
from deepspeech.decoders.swig_wrapper import ctc_greedy_decoder
from deepspeech.decoders.swig_wrapper import ctc_beam_search_decoder_batch
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

12
deepspeech/modules/decoder.py
Unescape View File

@ -12,19 +12,21 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Decoder definition.""" """Decoder definition."""
from typing import Tuple, List, Optional from typing import List
from typeguard import check_argument_types from typing import Optional
from deepspeech.utils.log import Log from typing import Tuple
import paddle import paddle
from paddle import nn from paddle import nn
from typeguard import check_argument_types
from deepspeech.modules.attention import MultiHeadedAttention from deepspeech.modules.attention import MultiHeadedAttention
from deepspeech.modules.decoder_layer import DecoderLayer from deepspeech.modules.decoder_layer import DecoderLayer
from deepspeech.modules.embedding import PositionalEncoding from deepspeech.modules.embedding import PositionalEncoding
from deepspeech.modules.positionwise_feed_forward import PositionwiseFeedForward
from deepspeech.modules.mask import subsequent_mask
from deepspeech.modules.mask import make_non_pad_mask from deepspeech.modules.mask import make_non_pad_mask
from deepspeech.modules.mask import subsequent_mask
from deepspeech.modules.positionwise_feed_forward import PositionwiseFeedForward
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

6
deepspeech/modules/decoder_layer.py
Unescape View File

@ -12,12 +12,14 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Decoder self-attention layer definition.""" """Decoder self-attention layer definition."""
from typing import Optional, Tuple from typing import Optional
from deepspeech.utils.log import Log from typing import Tuple
import paddle import paddle
from paddle import nn from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()
__all__ = ["DecoderLayer"] __all__ = ["DecoderLayer"]

4
deepspeech/modules/embedding.py
Unescape View File

@ -12,14 +12,14 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Positonal Encoding Module.""" """Positonal Encoding Module."""
import math import math
from deepspeech.utils.log import Log
from typing import Tuple from typing import Tuple
import paddle import paddle
from paddle import nn from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()
__all__ = ["PositionalEncoding", "RelPositionalEncoding"] __all__ = ["PositionalEncoding", "RelPositionalEncoding"]

16
deepspeech/modules/encoder.py
Unescape View File

@ -12,28 +12,30 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Encoder definition.""" """Encoder definition."""
from deepspeech.utils.log import Log from typing import List
from typing import Tuple, List, Optional from typing import Optional
from typeguard import check_argument_types from typing import Tuple
import paddle import paddle
from paddle import nn from paddle import nn
from typeguard import check_argument_types
from deepspeech.modules.activation import get_activation
from deepspeech.modules.attention import MultiHeadedAttention from deepspeech.modules.attention import MultiHeadedAttention
from deepspeech.modules.attention import RelPositionMultiHeadedAttention from deepspeech.modules.attention import RelPositionMultiHeadedAttention
from deepspeech.modules.conformer_convolution import ConvolutionModule from deepspeech.modules.conformer_convolution import ConvolutionModule
from deepspeech.modules.embedding import PositionalEncoding from deepspeech.modules.embedding import PositionalEncoding
from deepspeech.modules.embedding import RelPositionalEncoding from deepspeech.modules.embedding import RelPositionalEncoding
from deepspeech.modules.encoder_layer import TransformerEncoderLayer
from deepspeech.modules.encoder_layer import ConformerEncoderLayer from deepspeech.modules.encoder_layer import ConformerEncoderLayer
from deepspeech.modules.encoder_layer import TransformerEncoderLayer
from deepspeech.modules.mask import add_optional_chunk_mask
from deepspeech.modules.mask import make_non_pad_mask
from deepspeech.modules.positionwise_feed_forward import PositionwiseFeedForward from deepspeech.modules.positionwise_feed_forward import PositionwiseFeedForward
from deepspeech.modules.subsampling import Conv2dSubsampling4 from deepspeech.modules.subsampling import Conv2dSubsampling4
from deepspeech.modules.subsampling import Conv2dSubsampling6 from deepspeech.modules.subsampling import Conv2dSubsampling6
from deepspeech.modules.subsampling import Conv2dSubsampling8 from deepspeech.modules.subsampling import Conv2dSubsampling8
from deepspeech.modules.subsampling import LinearNoSubsampling from deepspeech.modules.subsampling import LinearNoSubsampling
from deepspeech.modules.mask import make_non_pad_mask from deepspeech.utils.log import Log
from deepspeech.modules.mask import add_optional_chunk_mask
from deepspeech.modules.activation import get_activation
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

6
deepspeech/modules/encoder_layer.py
Unescape View File

@ -12,12 +12,14 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Encoder self-attention layer definition.""" """Encoder self-attention layer definition."""
from typing import Optional, Tuple from typing import Optional
from deepspeech.utils.log import Log from typing import Tuple
import paddle import paddle
from paddle import nn from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()
__all__ = ["TransformerEncoderLayer", "ConformerEncoderLayer"] __all__ = ["TransformerEncoderLayer", "ConformerEncoderLayer"]

5
deepspeech/modules/loss.py
Unescape View File

@ -11,13 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from deepspeech.utils.log import Log
import paddle import paddle
from paddle import nn from paddle import nn
from paddle.nn import functional as F from paddle.nn import functional as F
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()
__all__ = ['CTCLoss', "LabelSmoothingLoss"] __all__ = ['CTCLoss', "LabelSmoothingLoss"]

3
deepspeech/modules/mask.py
Unescape View File

@ -11,11 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import paddle
from deepspeech.utils.log import Log from deepspeech.utils.log import Log
import paddle
logger = Log(__name__).getlog() logger = Log(__name__).getlog()
__all__ = [ __all__ = [

4
deepspeech/modules/positionwise_feed_forward.py
Unescape View File

@ -12,11 +12,11 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Positionwise feed forward layer definition.""" """Positionwise feed forward layer definition."""
from deepspeech.utils.log import Log
import paddle import paddle
from paddle import nn from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()
__all__ = ["PositionwiseFeedForward"] __all__ = ["PositionwiseFeedForward"]

5
deepspeech/modules/rnn.py
Unescape View File

@ -11,17 +11,16 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import math import math
from deepspeech.utils.log import Log
import paddle import paddle
from paddle import nn from paddle import nn
from paddle.nn import functional as F from paddle.nn import functional as F
from paddle.nn import initializer as I from paddle.nn import initializer as I
from deepspeech.modules.mask import sequence_mask
from deepspeech.modules.activation import brelu from deepspeech.modules.activation import brelu
from deepspeech.modules.mask import sequence_mask
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

3
deepspeech/modules/subsampling.py
Unescape View File

@ -12,14 +12,13 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Subsampling layer definition.""" """Subsampling layer definition."""
from typing import Tuple from typing import Tuple
from deepspeech.utils.log import Log
import paddle import paddle
from paddle import nn from paddle import nn
from deepspeech.modules.embedding import PositionalEncoding from deepspeech.modules.embedding import PositionalEncoding
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

1
deepspeech/training/cli.py
Unescape View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import argparse import argparse

9
deepspeech/training/gradclip.py
Unescape View File

@ -11,13 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from deepspeech.utils.log import Log
import paddle import paddle
from paddle.fluid.dygraph import base as imperative_base
from paddle.fluid import layers
from paddle.fluid import core from paddle.fluid import core
from paddle.fluid import layers
from paddle.fluid.dygraph import base as imperative_base
from deepspeech.utils.log import Log
__all__ = ["ClipGradByGlobalNormWithLog"] __all__ = ["ClipGradByGlobalNormWithLog"]

6
deepspeech/training/scheduler.py
Unescape View File

@ -11,12 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from deepspeech.utils.log import Log
from typing import Union from typing import Union
from typeguard import check_argument_types
from paddle.optimizer.lr import LRScheduler from paddle.optimizer.lr import LRScheduler
from typeguard import check_argument_types
from deepspeech.utils.log import Log
__all__ = ["WarmupLR"] __all__ = ["WarmupLR"]

3
deepspeech/training/trainer.py
Unescape View File

@ -11,9 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import time import time
from deepspeech.utils.log import Log
from pathlib import Path from pathlib import Path
import paddle import paddle
@ -22,6 +20,7 @@ from tensorboardX import SummaryWriter
from deepspeech.utils import checkpoint from deepspeech.utils import checkpoint
from deepspeech.utils import mp_tools from deepspeech.utils import mp_tools
from deepspeech.utils.log import Log
__all__ = ["Trainer"] __all__ = ["Trainer"]

5
deepspeech/utils/checkpoint.py
Unescape View File

@ -11,11 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import json
import os import os
from deepspeech.utils.log import Log
import re import re
import json
from typing import Union from typing import Union
import paddle import paddle
@ -23,6 +21,7 @@ from paddle import distributed as dist
from paddle.optimizer import Optimizer from paddle.optimizer import Optimizer
from deepspeech.utils import mp_tools from deepspeech.utils import mp_tools
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

6
deepspeech/utils/ctc_utils.py
Unescape View File

@ -11,13 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from deepspeech.utils.log import Log
import numpy as np
from typing import List from typing import List
import numpy as np
import paddle import paddle
from deepspeech.utils.log import Log
logger = Log(__name__).getlog() logger = Log(__name__).getlog()
__all__ = ["forced_align", "remove_duplicates_and_blank", "insert_blank"] __all__ = ["forced_align", "remove_duplicates_and_blank", "insert_blank"]

2
deepspeech/utils/layer_tools.py
Unescape View File

@ -11,9 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import numpy as np import numpy as np
from paddle import nn from paddle import nn
__all__ = [ __all__ = [

5
deepspeech/utils/log.py
Unescape View File

@ -11,12 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import getpass
import logging import logging
import os import os
import sys
import getpass
import socket import socket
import sys
import time import time

2
deepspeech/utils/mp_tools.py
Unescape View File

@ -11,9 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from functools import wraps
from paddle import distributed as dist from paddle import distributed as dist
from functools import wraps
__all__ = ["rank_zero_only"] __all__ = ["rank_zero_only"]

6
deepspeech/utils/socket_server.py
Unescape View File

@ -11,15 +11,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os import os
import random import random
import time
from time import gmtime, strftime
import socket import socket
import socketserver import socketserver
import struct import struct
import time
import wave import wave
from time import gmtime
from time import strftime
from deepspeech.frontend.utility import read_manifest from deepspeech.frontend.utility import read_manifest

6
deepspeech/utils/tensor_utils.py
Unescape View File

@ -12,11 +12,13 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Unility functions for Transformer.""" """Unility functions for Transformer."""
from deepspeech.utils.log import Log from typing import List
from typing import Tuple, List from typing import Tuple
import paddle import paddle
from deepspeech.utils.log import Log
__all__ = ["pad_sequence", "add_sos_eos", "th_accuracy"] __all__ = ["pad_sequence", "add_sos_eos", "th_accuracy"]
logger = Log(__name__).getlog() logger = Log(__name__).getlog()

5
deepspeech/utils/utility.py
Unescape View File

@ -12,10 +12,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Contains common utility functions.""" """Contains common utility functions."""
import os
import math
import distutils.util import distutils.util
import math
import os
from typing import List from typing import List
__all__ = ['print_arguments', 'add_arguments', "log_add"] __all__ = ['print_arguments', 'add_arguments', "log_add"]

12
examples/dataset/aishell/aishell.py
Unescape View File

@ -18,13 +18,15 @@ Manifest file is a json-format file with each line containing the
meta data (i.e. audio filepath, transcript and audio duration) meta data (i.e. audio filepath, transcript and audio duration)
of each audio file in the data set. of each audio file in the data set.
""" """
import argparse
import os
import codecs import codecs
import soundfile
import json import json
import argparse import os
from utils.utility import download, unpack
import soundfile
from utils.utility import download
from utils.utility import unpack
DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech') DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')

10
examples/dataset/chime3_background/chime3_background.py
Unescape View File

@ -18,14 +18,14 @@ Manifest file is a json-format file with each line containing the
meta data (i.e. audio filepath, transcript and audio duration) meta data (i.e. audio filepath, transcript and audio duration)
of each audio file in the data set. of each audio file in the data set.
""" """
import argparse
import io
import json
import os import os
import wget
import zipfile import zipfile
import argparse
import soundfile import soundfile
import json import wget
import io
from paddle.v2.dataset.common import md5file from paddle.v2.dataset.common import md5file
# DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech') # DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')

14
examples/dataset/librispeech/librispeech.py
Unescape View File

@ -18,15 +18,17 @@ Manifest file is a json-format file with each line containing the
meta data (i.e. audio filepath, transcript and audio duration) meta data (i.e. audio filepath, transcript and audio duration)
of each audio file in the data set. of each audio file in the data set.
""" """
import distutils.util
import os
import argparse import argparse
import soundfile
import json
import codecs import codecs
import distutils.util
import io import io
from utils.utility import download, unpack import json
import os
import soundfile
from utils.utility import download
from utils.utility import unpack
URL_ROOT = "http://www.openslr.org/resources/12" URL_ROOT = "http://www.openslr.org/resources/12"
#URL_ROOT = "https://openslr.magicdatatech.com/resources/12" #URL_ROOT = "https://openslr.magicdatatech.com/resources/12"

12
examples/dataset/mini_librispeech/mini_librispeech.py
Unescape View File

@ -18,14 +18,16 @@ Manifest file is a json-format file with each line containing the
meta data (i.e. audio filepath, transcript and audio duration) meta data (i.e. audio filepath, transcript and audio duration)
of each audio file in the data set. of each audio file in the data set.
""" """
import os
import argparse import argparse
import soundfile
import json
import codecs import codecs
import io import io
from utils.utility import download, unpack import json
import os
import soundfile
from utils.utility import download
from utils.utility import unpack
URL_ROOT = "http://www.openslr.org/resources/31" URL_ROOT = "http://www.openslr.org/resources/31"
URL_TRAIN_CLEAN = URL_ROOT + "/train-clean-5.tar.gz" URL_TRAIN_CLEAN = URL_ROOT + "/train-clean-5.tar.gz"

11
examples/dataset/musan/musan.py
Unescape View File

@ -22,12 +22,15 @@ from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
import os import argparse
import codecs import codecs
import soundfile
import json import json
import argparse import os
from utils.utility import download, unpack
import soundfile
from utils.utility import download
from utils.utility import unpack
DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech') DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')

11
examples/dataset/rir_noise/rir_noise.py
Unescape View File

@ -22,12 +22,15 @@ from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
import os import argparse
import codecs import codecs
import soundfile
import json import json
import argparse import os
from utils.utility import download, unzip
import soundfile
from utils.utility import download
from utils.utility import unzip
DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech') DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')

13
examples/dataset/voxforge/voxforge.py
Unescape View File

@ -18,16 +18,19 @@ Manifest file is a json-format file with each line containing the
meta data (i.e. audio filepath, transcript and audio duration) meta data (i.e. audio filepath, transcript and audio duration)
of each audio file in the data set. of each audio file in the data set.
""" """
import argparse
import os
import codecs import codecs
import datetime import datetime
import soundfile
import json import json
import argparse import os
import shutil import shutil
import subprocess import subprocess
from utils.utility import download_multi, unpack, getfile_insensitive
import soundfile
from utils.utility import download_multi
from utils.utility import getfile_insensitive
from utils.utility import unpack
DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech') DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')

5
tests/deepspeech2_model_test.py
Unescape View File

@ -11,10 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import unittest
import paddle
import numpy as np import numpy as np
import unittest import paddle
from deepspeech.models.deepspeech2 import DeepSpeech2Model from deepspeech.models.deepspeech2 import DeepSpeech2Model

2
tests/error_rate_test.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Test error rate.""" """Test error rate."""
import unittest import unittest
from deepspeech.utils import error_rate from deepspeech.utils import error_rate

7
tests/mask_test.py
Unescape View File

@ -11,13 +11,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import unittest
import paddle
import numpy as np import numpy as np
import unittest import paddle
from deepspeech.modules.mask import sequence_mask
from deepspeech.modules.mask import make_non_pad_mask from deepspeech.modules.mask import make_non_pad_mask
from deepspeech.modules.mask import make_pad_mask from deepspeech.modules.mask import make_pad_mask
from deepspeech.modules.mask import sequence_mask
class TestU2Model(unittest.TestCase): class TestU2Model(unittest.TestCase):

5
tests/u2_model_test.py
Unescape View File

@ -11,11 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import paddle
import unittest import unittest
import numpy as np import numpy as np
import paddle
from yacs.config import CfgNode as CN from yacs.config import CfgNode as CN
from deepspeech.models.u2 import U2Model from deepspeech.models.u2 import U2Model

9
utils/build_vocab.py
Unescape View File

@ -14,20 +14,19 @@
"""Build vocabulary from manifest files. """Build vocabulary from manifest files.
Each item in vocabulary file is a character. Each item in vocabulary file is a character.
""" """
import argparse import argparse
import functools import functools
from collections import Counter
import os import os
import tempfile import tempfile
from collections import Counter
from deepspeech.frontend.utility import read_manifest from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer
from deepspeech.frontend.utility import UNK
from deepspeech.frontend.utility import BLANK from deepspeech.frontend.utility import BLANK
from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.utility import SOS from deepspeech.frontend.utility import SOS
from deepspeech.frontend.utility import UNK
from deepspeech.utils.utility import add_arguments from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments from deepspeech.utils.utility import print_arguments
from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser) add_arg = functools.partial(add_arguments, argparser=parser)

7
utils/compute_mean_std.py
Unescape View File

@ -12,13 +12,14 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Compute mean and std for feature normalizer, and save to file.""" """Compute mean and std for feature normalizer, and save to file."""
import argparse import argparse
import functools import functools
from deepspeech.frontend.normalizer import FeatureNormalizer
from deepspeech.frontend.augmentor.augmentation import AugmentationPipeline from deepspeech.frontend.augmentor.augmentation import AugmentationPipeline
from deepspeech.frontend.featurizer.audio_featurizer import AudioFeaturizer from deepspeech.frontend.featurizer.audio_featurizer import AudioFeaturizer
from deepspeech.utils.utility import add_arguments, print_arguments from deepspeech.frontend.normalizer import FeatureNormalizer
from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser) add_arg = functools.partial(add_arguments, argparser=parser)

4
utils/format_data.py
Unescape View File

@ -16,11 +16,11 @@ import argparse
import functools import functools
import json import json
from deepspeech.frontend.utility import read_manifest from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer
from deepspeech.frontend.utility import load_cmvn from deepspeech.frontend.utility import load_cmvn
from deepspeech.frontend.utility import read_manifest
from deepspeech.utils.utility import add_arguments from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments from deepspeech.utils.utility import print_arguments
from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser) add_arg = functools.partial(add_arguments, argparser=parser)

8
utils/spm_decode
Unescape View File

@ -4,8 +4,10 @@
# #
# This source code is licensed under the license found in the # This source code is licensed under the license found in the
# https://github.com/pytorch/fairseq/blob/master/LICENSE # https://github.com/pytorch/fairseq/blob/master/LICENSE
from __future__ import absolute_import
from __future__ import absolute_import, division, print_function, unicode_literals from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import argparse import argparse
import sys import sys
@ -46,4 +48,4 @@ def main():
if __name__ == "__main__": if __name__ == "__main__":
main() main()

8
utils/spm_encode
Unescape View File

@ -4,8 +4,10 @@
# #
# This source code is licensed under the license found in # This source code is licensed under the license found in
# https://github.com/pytorch/fairseq/blob/master/LICENSE # https://github.com/pytorch/fairseq/blob/master/LICENSE
from __future__ import absolute_import
from __future__ import absolute_import, division, print_function, unicode_literals from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import argparse import argparse
import contextlib import contextlib
@ -96,4 +98,4 @@ def main():
if __name__ == "__main__": if __name__ == "__main__":
main() main()

2
utils/spm_train
Unescape View File

@ -10,4 +10,4 @@ import sentencepiece as spm
if __name__ == "__main__": if __name__ == "__main__":
spm.SentencePieceTrainer.Train(" ".join(sys.argv[1:])) spm.SentencePieceTrainer.Train(" ".join(sys.argv[1:]))

2
utils/utility.py
Unescape View File

@ -11,10 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os import os
import tarfile import tarfile
import zipfile import zipfile
from paddle.dataset.common import md5file from paddle.dataset.common import md5file

Write Preview
Loading…
Cancel
Save