msb-public
/
PaddleSpeech
mirror of https://github.com/PaddlePaddle/PaddleSpeech
1
0
Code Issues Projects Releases Wiki Activity

reorder imports

Browse Source
pull/578/head
Hui Zhang 4 years ago
parent 708b7fd39b
commit fa6e333422
94 changed files with 402 additions and 373 deletions
Show Stats Download Patch File Download Diff File

117
.notebook/python_test.ipynb
Unescape View File

@ -637,7 +637,7 @@
{
"cell_type": "code",
"execution_count": 59,
"id": "featured-cookbook",
"id": "norwegian-cleveland",
"metadata": {},
"outputs": [
{
@ -660,7 +660,7 @@
{
"cell_type": "code",
"execution_count": 35,
"id": "concrete-rubber",
"id": "endless-kidney",
"metadata": {},
"outputs": [
{
@ -705,7 +705,7 @@
{
"cell_type": "code",
"execution_count": 3,
"id": "surgical-shore",
"id": "later-louisiana",
"metadata": {},
"outputs": [
{
@ -728,7 +728,7 @@
{
"cell_type": "code",
"execution_count": 4,
"id": "medieval-daily",
"id": "funded-nudist",
"metadata": {},
"outputs": [
{
@ -748,7 +748,7 @@
{
"cell_type": "code",
"execution_count": 5,
"id": "derived-meeting",
"id": "contrary-affiliation",
"metadata": {},
"outputs": [],
"source": [
@ -758,7 +758,7 @@
{
"cell_type": "code",
"execution_count": 6,
"id": "hawaiian-engagement",
"id": "friendly-interpretation",
"metadata": {},
"outputs": [
{
@ -776,7 +776,7 @@
{
"cell_type": "code",
"execution_count": 7,
"id": "moderate-greece",
"id": "bottom-wilderness",
"metadata": {},
"outputs": [
{
@ -809,7 +809,7 @@
{
"cell_type": "code",
"execution_count": 17,
"id": "duplicate-colon",
"id": "acquired-jacksonville",
"metadata": {},
"outputs": [
{
@ -839,7 +839,7 @@
{
"cell_type": "code",
"execution_count": 18,
"id": "actual-junior",
"id": "entertaining-capture",
"metadata": {},
"outputs": [
{
@ -871,7 +871,7 @@
{
"cell_type": "code",
"execution_count": 19,
"id": "looking-belle",
"id": "amber-grade",
"metadata": {},
"outputs": [
{
@ -903,7 +903,7 @@
{
"cell_type": "code",
"execution_count": 31,
"id": "turned-friend",
"id": "hidden-playback",
"metadata": {},
"outputs": [
{
@ -935,7 +935,7 @@
{
"cell_type": "code",
"execution_count": 32,
"id": "wrapped-foundation",
"id": "twelve-university",
"metadata": {},
"outputs": [
{
@ -966,7 +966,7 @@
{
"cell_type": "code",
"execution_count": 20,
"id": "applied-abuse",
"id": "minor-endorsement",
"metadata": {},
"outputs": [],
"source": [
@ -977,7 +977,7 @@
{
"cell_type": "code",
"execution_count": 46,
"id": "experimental-invasion",
"id": "upper-majority",
"metadata": {},
"outputs": [
{
@ -997,7 +997,7 @@
{
"cell_type": "code",
"execution_count": 30,
"id": "established-lunch",
"id": "supreme-coverage",
"metadata": {},
"outputs": [],
"source": [
@ -1007,7 +1007,7 @@
{
"cell_type": "code",
"execution_count": 31,
"id": "residential-logic",
"id": "tough-domain",
"metadata": {},
"outputs": [],
"source": [
@ -1017,7 +1017,7 @@
{
"cell_type": "code",
"execution_count": 47,
"id": "maritime-consequence",
"id": "indian-empire",
"metadata": {},
"outputs": [
{
@ -1038,7 +1038,7 @@
{
"cell_type": "code",
"execution_count": 1,
"id": "african-identification",
"id": "horizontal-paragraph",
"metadata": {},
"outputs": [],
"source": [
@ -1049,7 +1049,7 @@
{
"cell_type": "code",
"execution_count": 56,
"id": "upset-regard",
"id": "homeless-zoning",
"metadata": {},
"outputs": [],
"source": [
@ -1059,7 +1059,7 @@
{
"cell_type": "code",
"execution_count": 58,
"id": "wrapped-netherlands",
"id": "floating-atmosphere",
"metadata": {},
"outputs": [],
"source": [
@ -1069,7 +1069,7 @@
{
"cell_type": "code",
"execution_count": 60,
"id": "engaging-statement",
"id": "stupid-reducing",
"metadata": {},
"outputs": [],
"source": [
@ -1079,7 +1079,7 @@
{
"cell_type": "code",
"execution_count": 66,
"id": "joined-handy",
"id": "practical-airline",
"metadata": {},
"outputs": [],
"source": [
@ -1089,7 +1089,7 @@
{
"cell_type": "code",
"execution_count": 95,
"id": "absolute-kenya",
"id": "apart-comfort",
"metadata": {},
"outputs": [
{
@ -1110,7 +1110,7 @@
{
"cell_type": "code",
"execution_count": 81,
"id": "integral-subsection",
"id": "underlying-brand",
"metadata": {},
"outputs": [
{
@ -1131,7 +1131,7 @@
{
"cell_type": "code",
"execution_count": 11,
"id": "conservative-sport",
"id": "german-things",
"metadata": {},
"outputs": [],
"source": [
@ -1161,7 +1161,7 @@
{
"cell_type": "code",
"execution_count": 12,
"id": "connected-brass",
"id": "third-regression",
"metadata": {},
"outputs": [],
"source": [
@ -1172,7 +1172,7 @@
{
"cell_type": "code",
"execution_count": 14,
"id": "worse-couple",
"id": "thick-korea",
"metadata": {},
"outputs": [
{
@ -1201,7 +1201,7 @@
{
"cell_type": "code",
"execution_count": 13,
"id": "unlimited-effects",
"id": "institutional-hands",
"metadata": {},
"outputs": [
{
@ -1230,7 +1230,7 @@
{
"cell_type": "code",
"execution_count": 15,
"id": "stable-artist",
"id": "brave-native",
"metadata": {},
"outputs": [
{
@ -1260,7 +1260,7 @@
{
"cell_type": "code",
"execution_count": 29,
"id": "romance-laser",
"id": "turkish-ticket",
"metadata": {},
"outputs": [
{
@ -1290,7 +1290,7 @@
{
"cell_type": "code",
"execution_count": 30,
"id": "pharmaceutical-colombia",
"id": "executed-excerpt",
"metadata": {},
"outputs": [
{
@ -1308,7 +1308,7 @@
{
"cell_type": "code",
"execution_count": 22,
"id": "forty-harvest",
"id": "continental-boring",
"metadata": {},
"outputs": [
{
@ -1343,7 +1343,7 @@
{
"cell_type": "code",
"execution_count": 23,
"id": "thirty-software",
"id": "linear-assembly",
"metadata": {},
"outputs": [
{
@ -1376,7 +1376,7 @@
{
"cell_type": "code",
"execution_count": 33,
"id": "portuguese-addiction",
"id": "applied-louis",
"metadata": {},
"outputs": [],
"source": [
@ -1386,7 +1386,7 @@
{
"cell_type": "code",
"execution_count": 34,
"id": "everyday-vessel",
"id": "historic-struggle",
"metadata": {},
"outputs": [
{
@ -1428,17 +1428,60 @@
{
"cell_type": "code",
"execution_count": 35,
"id": "latter-expression",
"id": "monthly-roads",
"metadata": {},
"outputs": [],
"source": [
"getpass?"
]
},
{
"cell_type": "code",
"execution_count": 38,
"id": "boxed-peoples",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Worker:"
]
}
],
"source": [
"import multiprocessing\n",
"import cProfile\n",
"import time\n",
"\n",
"def worker(num):\n",
" time.sleep(3)\n",
" print('Worker:', num)\n",
"\n",
"def profile_worker(num):\n",
" cProfile.runctx('worker(num)', globals(), locals(), 'profile-%d.out' %num)\n",
"\n",
"\n",
"\n",
"for i in range(5):\n",
" p = multiprocessing.Process(target=profile_worker, args=(i,))\n",
" p.start()"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "fresh-tender",
"metadata": {},
"outputs": [],
"source": [
"!ls"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "small-professor",
"id": "religious-peripheral",
"metadata": {},
"outputs": [],
"source": []

8
.pre-commit-config.yaml
Unescape View File

@ -52,7 +52,7 @@
language: system
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|py)$
#exclude: (?=decoders/swig).*(\.cpp|\.h)$
#- repo: https://github.com/asottile/reorder_python_imports
# rev: v2.4.0
# hooks:
# - id: reorder-python-imports
- repo: https://github.com/asottile/reorder_python_imports
rev: v2.4.0
hooks:
- id: reorder-python-imports

8
deepspeech/__init__.py
Unescape View File

@ -11,18 +11,18 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Union
from typing import Any
from typing import List
from typing import Tuple
from typing import Any
from typing import Union
import paddle
from paddle import nn
from paddle.nn import functional as F
#TODO(Hui Zhang): remove fluid import
from paddle.fluid import core
from paddle.nn import functional as F
from deepspeech.utils.log import Log
#TODO(Hui Zhang): remove fluid import
logger = Log(__name__).getlog()
########### hcak logging #############

6
deepspeech/decoders/decoders_deprecated.py
Unescape View File

@ -12,11 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains various CTC decoders."""
import multiprocessing
from itertools import groupby
import numpy as np
from math import log
import multiprocessing
import numpy as np
def ctc_greedy_decoder(probs_seq, vocabulary):

2
deepspeech/decoders/scorer_deprecated.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""External Scorer for Beam Search Decoder."""
import os
import kenlm
import numpy as np

12
deepspeech/decoders/swig/setup.py
Unescape View File

@ -12,14 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Script to build and install decoder package."""
from setuptools import setup, Extension, distutils
import argparse
import glob
import platform
import multiprocessing.pool
import os
import platform
import sys
import multiprocessing.pool
import argparse
from setuptools import distutils
from setuptools import Extension
from setuptools import setup
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(

1
deepspeech/decoders/swig_wrapper.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Wrapper for various CTC decoders in SWIG."""
import swig_decoders

2
deepspeech/decoders/tests/test_decoders.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test decoders."""
import unittest
from deepspeech.decoders import decoders_deprecated as decoder

5
deepspeech/exps/deepspeech2/bin/deploy/client.py
Unescape View File

@ -12,9 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Client-end for the ASR demo."""
import keyboard
import sys
import argparse
import sys
import keyboard
import pyaudio
from deepspeech.utils.socket_server import socket_send

3
deepspeech/exps/deepspeech2/bin/deploy/record.py
Unescape View File

@ -13,9 +13,10 @@
# limitations under the License.
"""Record wav from Microphone"""
# http://people.csail.mit.edu/hubert/pyaudio/
import pyaudio
import wave
import pyaudio
CHUNK = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 1

24
deepspeech/exps/deepspeech2/bin/deploy/runtime.py
Unescape View File

@ -13,23 +13,21 @@
# limitations under the License.
"""Server-end for the ASR demo."""
import functools
import paddle
import numpy as np
from deepspeech.utils.socket_server import warm_up_test
from deepspeech.utils.socket_server import AsrTCPServer
from deepspeech.utils.socket_server import AsrRequestHandler
import numpy as np
import paddle
from paddle.inference import Config
from paddle.inference import create_predictor
from deepspeech.training.cli import default_argument_parser
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.utils.utility import add_arguments, print_arguments
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.io.dataset import ManifestDataset
from paddle.inference import Config
from paddle.inference import create_predictor
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.socket_server import AsrRequestHandler
from deepspeech.utils.socket_server import AsrTCPServer
from deepspeech.utils.socket_server import warm_up_test
from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments
def init_predictor(args):

19
deepspeech/exps/deepspeech2/bin/deploy/server.py
Unescape View File

@ -13,20 +13,19 @@
# limitations under the License.
"""Server-end for the ASR demo."""
import functools
import paddle
import numpy as np
from deepspeech.utils.socket_server import warm_up_test
from deepspeech.utils.socket_server import AsrTCPServer
from deepspeech.utils.socket_server import AsrRequestHandler
import numpy as np
import paddle
from deepspeech.training.cli import default_argument_parser
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.utils.utility import add_arguments, print_arguments
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.io.dataset import ManifestDataset
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.socket_server import AsrRequestHandler
from deepspeech.utils.socket_server import AsrTCPServer
from deepspeech.utils.socket_server import warm_up_test
from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments
def start_server(config, args):

5
deepspeech/exps/deepspeech2/bin/export.py
Unescape View File

@ -12,11 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Export for DeepSpeech2 model."""
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
def main_sp(config, args):

4
deepspeech/exps/deepspeech2/bin/infer.py
Unescape View File

@ -12,12 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inferer for DeepSpeech2 model."""
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
# TODO(hui zhang): dynamic load
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester
def main_sp(config, args):

5
deepspeech/exps/deepspeech2/bin/test.py
Unescape View File

@ -12,11 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Evaluation for DeepSpeech2 model."""
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
def main_sp(config, args):

5
deepspeech/exps/deepspeech2/bin/train.py
Unescape View File

@ -14,11 +14,10 @@
"""Trainer for DeepSpeech2 model."""
from paddle import distributed as dist
from deepspeech.utils.utility import print_arguments
from deepspeech.training.cli import default_argument_parser
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.exps.deepspeech2.model import DeepSpeech2Trainer as Trainer
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
def main_sp(config, args):

16
deepspeech/exps/deepspeech2/bin/tune.py
Unescape View File

@ -12,22 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Beam search parameters tuning for DeepSpeech2 model."""
import sys
import numpy as np
import functools
import sys
import numpy as np
from paddle.io import DataLoader
from deepspeech.utils import error_rate
from deepspeech.utils.utility import add_arguments, print_arguments
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.io.collator import SpeechCollator
from deepspeech.io.dataset import ManifestDataset
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.training.cli import default_argument_parser
from deepspeech.exps.deepspeech2.config import get_cfg_defaults
from deepspeech.utils import error_rate
from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments
def tune(config, args):

2
deepspeech/exps/deepspeech2/config.py
Unescape View File

@ -11,8 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from yacs.config import CfgNode as CN
from deepspeech.models.deepspeech2 import DeepSpeech2Model
_C = CN()

24
deepspeech/exps/deepspeech2/model.py
Unescape View File

@ -12,31 +12,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains DeepSpeech2 model."""
import time
from deepspeech.utils.log import Log
import numpy as np
from collections import defaultdict
from pathlib import Path
import numpy as np
import paddle
from paddle import distributed as dist
from paddle.io import DataLoader
from deepspeech.training.trainer import Trainer
from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog
from deepspeech.utils import mp_tools
from deepspeech.utils import layer_tools
from deepspeech.utils import error_rate
from deepspeech.io.collator import SpeechCollator
from deepspeech.io.sampler import SortagradDistributedBatchSampler
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.io.dataset import ManifestDataset
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.io.sampler import SortagradDistributedBatchSampler
from deepspeech.models.deepspeech2 import DeepSpeech2InferModel
from deepspeech.models.deepspeech2 import DeepSpeech2Model
from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog
from deepspeech.training.trainer import Trainer
from deepspeech.utils import error_rate
from deepspeech.utils import layer_tools
from deepspeech.utils import mp_tools
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

6
deepspeech/exps/u2/bin/export.py
Unescape View File

@ -12,12 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Export for U2 model."""
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
from deepspeech.exps.u2.config import get_cfg_defaults
from deepspeech.exps.u2.model import U2Tester as Tester
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
def main_sp(config, args):

6
deepspeech/exps/u2/bin/test.py
Unescape View File

@ -12,15 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Evaluation for U2 model."""
import os
import cProfile
import os
from deepspeech.exps.u2.config import get_cfg_defaults
from deepspeech.exps.u2.model import U2Tester as Tester
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
# TODO(hui zhang): dynamic load
from deepspeech.exps.u2.config import get_cfg_defaults
from deepspeech.exps.u2.model import U2Tester as Tester
def main_sp(config, args):

9
deepspeech/exps/u2/bin/train.py
Unescape View File

@ -12,16 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Trainer for U2 model."""
import os
import cProfile
from paddle import distributed as dist
import os
from deepspeech.utils.utility import print_arguments
from deepspeech.training.cli import default_argument_parser
from paddle import distributed as dist
from deepspeech.exps.u2.config import get_cfg_defaults
from deepspeech.exps.u2.model import U2Trainer as Trainer
from deepspeech.training.cli import default_argument_parser
from deepspeech.utils.utility import print_arguments
def main_sp(config, args):

5
deepspeech/exps/u2/config.py
Unescape View File

@ -11,13 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from yacs.config import CfgNode
from deepspeech.exps.u2.model import U2Tester
from deepspeech.exps.u2.model import U2Trainer
from deepspeech.io.dataset import ManifestDataset
from deepspeech.models.u2 import U2Model
from deepspeech.exps.u2.model import U2Trainer
from deepspeech.exps.u2.model import U2Tester
_C = CfgNode()

26
deepspeech/exps/u2/model.py
Unescape View File

@ -12,34 +12,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains U2 model."""
import sys
import time
from deepspeech.utils.log import Log
import numpy as np
from collections import defaultdict
from pathlib import Path
from typing import Optional
from yacs.config import CfgNode
import numpy as np
import paddle
from paddle import distributed as dist
from paddle.io import DataLoader
from deepspeech.training.trainer import Trainer
from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog
from deepspeech.training.scheduler import WarmupLR
from deepspeech.utils import mp_tools
from deepspeech.utils import layer_tools
from deepspeech.utils import error_rate
from yacs.config import CfgNode
from deepspeech.io.collator import SpeechCollator
from deepspeech.io.sampler import SortagradDistributedBatchSampler
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.io.dataset import ManifestDataset
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.io.sampler import SortagradDistributedBatchSampler
from deepspeech.models.u2 import U2Model
from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog
from deepspeech.training.scheduler import WarmupLR
from deepspeech.training.trainer import Trainer
from deepspeech.utils import error_rate
from deepspeech.utils import layer_tools
from deepspeech.utils import mp_tools
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

12
deepspeech/frontend/audio.py
Unescape View File

@ -12,16 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the audio segment class."""
import numpy as np
import copy
import io
import struct
import random
import re
import soundfile
import struct
import numpy as np
import resampy
import soundfile
from scipy import signal
import random
import copy
class AudioSegment(object):

14
deepspeech/frontend/augmentor/augmentation.py
Unescape View File

@ -12,21 +12,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the data augmentation pipeline."""
import json
import numpy as np
# audio augment
from deepspeech.frontend.augmentor.volume_perturb import VolumePerturbAugmentor
from deepspeech.frontend.augmentor.shift_perturb import ShiftPerturbAugmentor
from deepspeech.frontend.augmentor.speed_perturb import SpeedPerturbAugmentor
from deepspeech.frontend.augmentor.noise_perturb import NoisePerturbAugmentor
from deepspeech.frontend.augmentor.impulse_response import ImpulseResponseAugmentor
from deepspeech.frontend.augmentor.resample import ResampleAugmentor
from deepspeech.frontend.augmentor.noise_perturb import NoisePerturbAugmentor
from deepspeech.frontend.augmentor.online_bayesian_normalization import \
OnlineBayesianNormalizationAugmentor
# feature augment
from deepspeech.frontend.augmentor.resample import ResampleAugmentor
from deepspeech.frontend.augmentor.shift_perturb import ShiftPerturbAugmentor
from deepspeech.frontend.augmentor.spec_augment import SpecAugmentor
from deepspeech.frontend.augmentor.speed_perturb import SpeedPerturbAugmentor
from deepspeech.frontend.augmentor.volume_perturb import VolumePerturbAugmentor
class AugmentationPipeline():

4
deepspeech/frontend/augmentor/base.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the abstract base class for augmentation models."""
from abc import ABCMeta, abstractmethod
from abc import ABCMeta
from abc import abstractmethod
class AugmentorBase():

3
deepspeech/frontend/augmentor/impulse_response.py
Unescape View File

@ -12,10 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the impulse response augmentation model."""
from deepspeech.frontend.audio import AudioSegment
from deepspeech.frontend.augmentor.base import AugmentorBase
from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.audio import AudioSegment
class ImpulseResponseAugmentor(AugmentorBase):

3
deepspeech/frontend/augmentor/noise_perturb.py
Unescape View File

@ -12,10 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the noise perturb augmentation model."""
from deepspeech.frontend.audio import AudioSegment
from deepspeech.frontend.augmentor.base import AugmentorBase
from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.audio import AudioSegment
class NoisePerturbAugmentor(AugmentorBase):

1
deepspeech/frontend/augmentor/online_bayesian_normalization.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contain the online bayesian normalization augmentation model."""
from deepspeech.frontend.augmentor.base import AugmentorBase

1
deepspeech/frontend/augmentor/resample.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contain the resample augmentation model."""
from deepspeech.frontend.augmentor.base import AugmentorBase

1
deepspeech/frontend/augmentor/shift_perturb.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the volume perturb augmentation model."""
from deepspeech.frontend.augmentor.base import AugmentorBase

2
deepspeech/frontend/augmentor/spec_augment.py
Unescape View File

@ -12,10 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the volume perturb augmentation model."""
from deepspeech.utils.log import Log
import numpy as np
from deepspeech.frontend.augmentor.base import AugmentorBase
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

2
deepspeech/frontend/augmentor/speed_perturb.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contain the speech perturbation augmentation model."""
import numpy as np
from deepspeech.frontend.augmentor.base import AugmentorBase

1
deepspeech/frontend/augmentor/volume_perturb.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the volume perturb augmentation model."""
from deepspeech.frontend.augmentor.base import AugmentorBase

5
deepspeech/frontend/featurizer/audio_featurizer.py
Unescape View File

@ -12,11 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the audio featurizer class."""
import numpy as np
from python_speech_features import mfcc
from python_speech_features import logfbank
from python_speech_features import delta
from python_speech_features import logfbank
from python_speech_features import mfcc
class AudioFeaturizer(object):

1
deepspeech/frontend/featurizer/speech_featurizer.py
Unescape View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the speech featurizer class."""
from deepspeech.frontend.featurizer.audio_featurizer import AudioFeaturizer
from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer

3
deepspeech/frontend/featurizer/text_featurizer.py
Unescape View File

@ -12,11 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the text featurizer class."""
import sentencepiece as spm
from deepspeech.frontend.utility import UNK
from deepspeech.frontend.utility import EOS
from deepspeech.frontend.utility import UNK
class TextFeaturizer(object):

7
deepspeech/frontend/normalizer.py
Unescape View File

@ -12,12 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains feature normalizers."""
import random
import numpy as np
import random
from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.utility import load_cmvn
from deepspeech.frontend.audio import AudioSegment
from deepspeech.frontend.utility import load_cmvn
from deepspeech.frontend.utility import read_manifest
class FeatureNormalizer(object):

2
deepspeech/frontend/speech.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the speech segment class."""
import numpy as np
from deepspeech.frontend.audio import AudioSegment

8
deepspeech/frontend/utility.py
Unescape View File

@ -12,10 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains data helper functions."""
import numpy as np
import math
import json
import codecs
import json
import math
import numpy as np
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

5
deepspeech/io/__init__.py
Unescape View File

@ -11,14 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
from paddle.io import DataLoader
from deepspeech.io.collator import SpeechCollator
from deepspeech.io.sampler import SortagradDistributedBatchSampler
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.io.dataset import ManifestDataset
from deepspeech.io.sampler import SortagradBatchSampler
from deepspeech.io.sampler import SortagradDistributedBatchSampler
def create_dataloader(manifest_path,

5
deepspeech/io/collator.py
Unescape View File

@ -11,12 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deepspeech.utils.log import Log
import numpy as np
from deepspeech.io.utility import pad_sequence
from deepspeech.frontend.utility import IGNORE_ID
from deepspeech.io.utility import pad_sequence
from deepspeech.utils.log import Log
__all__ = ["SpeechCollator"]

13
deepspeech/io/dataset.py
Unescape View File

@ -11,23 +11,22 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
import io
import random
import tarfile
from deepspeech.utils.log import Log
from typing import Optional
from yacs.config import CfgNode
import time
from collections import namedtuple
from typing import Optional
from paddle.io import Dataset
from yacs.config import CfgNode
from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.augmentor.augmentation import AugmentationPipeline
from deepspeech.frontend.featurizer.speech_featurizer import SpeechFeaturizer
from deepspeech.frontend.speech import SpeechSegment
from deepspeech.frontend.normalizer import FeatureNormalizer
from deepspeech.frontend.speech import SpeechSegment
from deepspeech.frontend.utility import read_manifest
from deepspeech.utils.log import Log
__all__ = [
"ManifestDataset",

8
deepspeech/io/sampler.py
Unescape View File

@ -11,14 +11,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math
from deepspeech.utils.log import Log
import numpy as np
import numpy as np
from paddle import distributed as dist
from paddle.io import BatchSampler
from paddle.io import DistributedBatchSampler
from paddle import distributed as dist
from deepspeech.utils.log import Log
__all__ = [
"SortagradDistributedBatchSampler",

5
deepspeech/io/utility.py
Unescape View File

@ -11,10 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import List
from deepspeech.utils.log import Log
import numpy as np
from typing import List
from deepspeech.utils.log import Log
__all__ = ["pad_sequence"]

7
deepspeech/models/deepspeech2.py
Unescape View File

@ -12,19 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Deepspeech2 ASR Model"""
from deepspeech.utils.log import Log
from typing import Optional
from yacs.config import CfgNode
import paddle
from paddle import nn
from yacs.config import CfgNode
from deepspeech.modules.conv import ConvStack
from deepspeech.modules.rnn import RNNStack
from deepspeech.modules.ctc import CTCDecoder
from deepspeech.modules.rnn import RNNStack
from deepspeech.utils import checkpoint
from deepspeech.utils import layer_tools
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

37
deepspeech/models/u2.py
Unescape View File

@ -15,40 +15,39 @@
Unified Streaming and Non-streaming Two-pass End-to-end Model for Speech Recognition
(https://arxiv.org/pdf/2012.05481.pdf)
"""
import time
import sys
import time
from collections import defaultdict
from deepspeech.utils.log import Log
from yacs.config import CfgNode
from typing import List, Optional, Tuple, Dict
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
import paddle
from paddle import jit
from paddle import nn
from yacs.config import CfgNode
from deepspeech.modules.mask import make_pad_mask
from deepspeech.modules.mask import mask_finished_preds
from deepspeech.modules.mask import mask_finished_scores
from deepspeech.modules.mask import subsequent_mask
from deepspeech.frontend.utility import IGNORE_ID
from deepspeech.frontend.utility import load_cmvn
from deepspeech.modules.cmvn import GlobalCMVN
from deepspeech.modules.encoder import ConformerEncoder
from deepspeech.modules.encoder import TransformerEncoder
from deepspeech.modules.ctc import CTCDecoder
from deepspeech.modules.decoder import TransformerDecoder
from deepspeech.modules.encoder import ConformerEncoder
from deepspeech.modules.encoder import TransformerEncoder
from deepspeech.modules.loss import LabelSmoothingLoss
from deepspeech.frontend.utility import load_cmvn
from deepspeech.frontend.utility import IGNORE_ID
from deepspeech.modules.mask import make_pad_mask
from deepspeech.modules.mask import mask_finished_preds
from deepspeech.modules.mask import mask_finished_scores
from deepspeech.modules.mask import subsequent_mask
from deepspeech.utils import checkpoint
from deepspeech.utils import layer_tools
from deepspeech.utils.utility import log_add
from deepspeech.utils.ctc_utils import remove_duplicates_and_blank
from deepspeech.utils.log import Log
from deepspeech.utils.tensor_utils import add_sos_eos
from deepspeech.utils.tensor_utils import th_accuracy
from deepspeech.utils.tensor_utils import pad_sequence
from deepspeech.utils.ctc_utils import remove_duplicates_and_blank
from deepspeech.utils.tensor_utils import th_accuracy
from deepspeech.utils.utility import log_add
logger = Log(__name__).getlog()

4
deepspeech/modules/activation.py
Unescape View File

@ -11,13 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deepspeech.utils.log import Log
from collections import OrderedDict
import paddle
from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()
__all__ = ["get_activation", "brelu", "LinearGLUBlock", "ConvGLUBlock"]

6
deepspeech/modules/attention.py
Unescape View File

@ -13,13 +13,15 @@
# limitations under the License.
"""Multi-Head Attention layer definition."""
import math
from deepspeech.utils.log import Log
from typing import Optional, Tuple
from typing import Optional
from typing import Tuple
import paddle
from paddle import nn
from paddle.nn import initializer as I
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()
__all__ = ["MultiHeadedAttention", "RelPositionMultiHeadedAttention"]

5
deepspeech/modules/cmvn.py
Unescape View File

@ -11,12 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deepspeech.utils.log import Log
import paddle
from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()
__all__ = ['GlobalCMVN']

9
deepspeech/modules/conformer_convolution.py
Unescape View File

@ -12,13 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""ConvolutionModule definition."""
from typing import Optional, Tuple
from typeguard import check_argument_types
from deepspeech.utils.log import Log
from typing import Optional
from typing import Tuple
import paddle
from paddle import nn
from typeguard import check_argument_types
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

6
deepspeech/modules/conv.py
Unescape View File

@ -11,14 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deepspeech.utils.log import Log
from paddle import nn
from paddle.nn import functional as F
from deepspeech.modules.mask import sequence_mask
from deepspeech.modules.activation import brelu
from deepspeech.modules.mask import sequence_mask
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

13
deepspeech/modules/ctc.py
Unescape View File

@ -11,20 +11,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deepspeech.utils.log import Log
from typeguard import check_argument_types
import paddle
from paddle import nn
from paddle.nn import functional as F
from typeguard import check_argument_types
from deepspeech.decoders.swig_wrapper import ctc_beam_search_decoder_batch
from deepspeech.decoders.swig_wrapper import ctc_greedy_decoder
from deepspeech.decoders.swig_wrapper import Scorer
from deepspeech.modules.loss import CTCLoss
from deepspeech.utils import ctc_utils
from deepspeech.decoders.swig_wrapper import Scorer
from deepspeech.decoders.swig_wrapper import ctc_greedy_decoder
from deepspeech.decoders.swig_wrapper import ctc_beam_search_decoder_batch
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

12
deepspeech/modules/decoder.py
Unescape View File

@ -12,19 +12,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Decoder definition."""
from typing import Tuple, List, Optional
from typeguard import check_argument_types
from deepspeech.utils.log import Log
from typing import List
from typing import Optional
from typing import Tuple
import paddle
from paddle import nn
from typeguard import check_argument_types
from deepspeech.modules.attention import MultiHeadedAttention
from deepspeech.modules.decoder_layer import DecoderLayer
from deepspeech.modules.embedding import PositionalEncoding
from deepspeech.modules.positionwise_feed_forward import PositionwiseFeedForward
from deepspeech.modules.mask import subsequent_mask
from deepspeech.modules.mask import make_non_pad_mask
from deepspeech.modules.mask import subsequent_mask
from deepspeech.modules.positionwise_feed_forward import PositionwiseFeedForward
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

6
deepspeech/modules/decoder_layer.py
Unescape View File

@ -12,12 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Decoder self-attention layer definition."""
from typing import Optional, Tuple
from deepspeech.utils.log import Log
from typing import Optional
from typing import Tuple
import paddle
from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()
__all__ = ["DecoderLayer"]

4
deepspeech/modules/embedding.py
Unescape View File

@ -12,14 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Positonal Encoding Module."""
import math
from deepspeech.utils.log import Log
from typing import Tuple
import paddle
from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()
__all__ = ["PositionalEncoding", "RelPositionalEncoding"]

16
deepspeech/modules/encoder.py
Unescape View File

@ -12,28 +12,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Encoder definition."""
from deepspeech.utils.log import Log
from typing import Tuple, List, Optional
from typeguard import check_argument_types
from typing import List
from typing import Optional
from typing import Tuple
import paddle
from paddle import nn
from typeguard import check_argument_types
from deepspeech.modules.activation import get_activation
from deepspeech.modules.attention import MultiHeadedAttention
from deepspeech.modules.attention import RelPositionMultiHeadedAttention
from deepspeech.modules.conformer_convolution import ConvolutionModule
from deepspeech.modules.embedding import PositionalEncoding
from deepspeech.modules.embedding import RelPositionalEncoding
from deepspeech.modules.encoder_layer import TransformerEncoderLayer
from deepspeech.modules.encoder_layer import ConformerEncoderLayer
from deepspeech.modules.encoder_layer import TransformerEncoderLayer
from deepspeech.modules.mask import add_optional_chunk_mask
from deepspeech.modules.mask import make_non_pad_mask
from deepspeech.modules.positionwise_feed_forward import PositionwiseFeedForward
from deepspeech.modules.subsampling import Conv2dSubsampling4
from deepspeech.modules.subsampling import Conv2dSubsampling6
from deepspeech.modules.subsampling import Conv2dSubsampling8
from deepspeech.modules.subsampling import LinearNoSubsampling
from deepspeech.modules.mask import make_non_pad_mask
from deepspeech.modules.mask import add_optional_chunk_mask
from deepspeech.modules.activation import get_activation
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

6
deepspeech/modules/encoder_layer.py
Unescape View File

@ -12,12 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Encoder self-attention layer definition."""
from typing import Optional, Tuple
from deepspeech.utils.log import Log
from typing import Optional
from typing import Tuple
import paddle
from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()
__all__ = ["TransformerEncoderLayer", "ConformerEncoderLayer"]

5
deepspeech/modules/loss.py
Unescape View File

@ -11,13 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deepspeech.utils.log import Log
import paddle
from paddle import nn
from paddle.nn import functional as F
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()
__all__ = ['CTCLoss', "LabelSmoothingLoss"]

3
deepspeech/modules/mask.py
Unescape View File

@ -11,11 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle
from deepspeech.utils.log import Log
import paddle
logger = Log(__name__).getlog()
__all__ = [

4
deepspeech/modules/positionwise_feed_forward.py
Unescape View File

@ -12,11 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Positionwise feed forward layer definition."""
from deepspeech.utils.log import Log
import paddle
from paddle import nn
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()
__all__ = ["PositionwiseFeedForward"]

5
deepspeech/modules/rnn.py
Unescape View File

@ -11,17 +11,16 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math
from deepspeech.utils.log import Log
import paddle
from paddle import nn
from paddle.nn import functional as F
from paddle.nn import initializer as I
from deepspeech.modules.mask import sequence_mask
from deepspeech.modules.activation import brelu
from deepspeech.modules.mask import sequence_mask
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

3
deepspeech/modules/subsampling.py
Unescape View File

@ -12,14 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Subsampling layer definition."""
from typing import Tuple
from deepspeech.utils.log import Log
import paddle
from paddle import nn
from deepspeech.modules.embedding import PositionalEncoding
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

1
deepspeech/training/cli.py
Unescape View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse

9
deepspeech/training/gradclip.py
Unescape View File

@ -11,13 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deepspeech.utils.log import Log
import paddle
from paddle.fluid.dygraph import base as imperative_base
from paddle.fluid import layers
from paddle.fluid import core
from paddle.fluid import layers
from paddle.fluid.dygraph import base as imperative_base
from deepspeech.utils.log import Log
__all__ = ["ClipGradByGlobalNormWithLog"]

6
deepspeech/training/scheduler.py
Unescape View File

@ -11,12 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deepspeech.utils.log import Log
from typing import Union
from typeguard import check_argument_types
from paddle.optimizer.lr import LRScheduler
from typeguard import check_argument_types
from deepspeech.utils.log import Log
__all__ = ["WarmupLR"]

3
deepspeech/training/trainer.py
Unescape View File

@ -11,9 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from deepspeech.utils.log import Log
from pathlib import Path
import paddle
@ -22,6 +20,7 @@ from tensorboardX import SummaryWriter
from deepspeech.utils import checkpoint
from deepspeech.utils import mp_tools
from deepspeech.utils.log import Log
__all__ = ["Trainer"]

5
deepspeech/utils/checkpoint.py
Unescape View File

@ -11,11 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
from deepspeech.utils.log import Log
import re
import json
from typing import Union
import paddle
@ -23,6 +21,7 @@ from paddle import distributed as dist
from paddle.optimizer import Optimizer
from deepspeech.utils import mp_tools
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()

6
deepspeech/utils/ctc_utils.py
Unescape View File

@ -11,13 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deepspeech.utils.log import Log
import numpy as np
from typing import List
import numpy as np
import paddle
from deepspeech.utils.log import Log
logger = Log(__name__).getlog()
__all__ = ["forced_align", "remove_duplicates_and_blank", "insert_blank"]

2
deepspeech/utils/layer_tools.py
Unescape View File

@ -11,9 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
from paddle import nn
__all__ = [

5
deepspeech/utils/log.py
Unescape View File

@ -11,12 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import getpass
import logging
import os
import sys
import getpass
import socket
import sys
import time

2
deepspeech/utils/mp_tools.py
Unescape View File

@ -11,9 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from functools import wraps
from paddle import distributed as dist
from functools import wraps
__all__ = ["rank_zero_only"]

6
deepspeech/utils/socket_server.py
Unescape View File

@ -11,15 +11,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import random
import time
from time import gmtime, strftime
import socket
import socketserver
import struct
import time
import wave
from time import gmtime
from time import strftime
from deepspeech.frontend.utility import read_manifest

6
deepspeech/utils/tensor_utils.py
Unescape View File

@ -12,11 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unility functions for Transformer."""
from deepspeech.utils.log import Log
from typing import Tuple, List
from typing import List
from typing import Tuple
import paddle
from deepspeech.utils.log import Log
__all__ = ["pad_sequence", "add_sos_eos", "th_accuracy"]
logger = Log(__name__).getlog()

5
deepspeech/utils/utility.py
Unescape View File

@ -12,10 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains common utility functions."""
import os
import math
import distutils.util
import math
import os
from typing import List
__all__ = ['print_arguments', 'add_arguments', "log_add"]

12
examples/dataset/aishell/aishell.py
Unescape View File

@ -18,13 +18,15 @@ Manifest file is a json-format file with each line containing the
meta data (i.e. audio filepath, transcript and audio duration)
of each audio file in the data set.
"""
import os
import argparse
import codecs
import soundfile
import json
import argparse
from utils.utility import download, unpack
import os
import soundfile
from utils.utility import download
from utils.utility import unpack
DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')

10
examples/dataset/chime3_background/chime3_background.py
Unescape View File

@ -18,14 +18,14 @@ Manifest file is a json-format file with each line containing the
meta data (i.e. audio filepath, transcript and audio duration)
of each audio file in the data set.
"""
import argparse
import io
import json
import os
import wget
import zipfile
import argparse
import soundfile
import json
import io
import wget
from paddle.v2.dataset.common import md5file
# DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')

14
examples/dataset/librispeech/librispeech.py
Unescape View File

@ -18,15 +18,17 @@ Manifest file is a json-format file with each line containing the
meta data (i.e. audio filepath, transcript and audio duration)
of each audio file in the data set.
"""
import distutils.util
import os
import argparse
import soundfile
import json
import codecs
import distutils.util
import io
from utils.utility import download, unpack
import json
import os
import soundfile
from utils.utility import download
from utils.utility import unpack
URL_ROOT = "http://www.openslr.org/resources/12"
#URL_ROOT = "https://openslr.magicdatatech.com/resources/12"

12
examples/dataset/mini_librispeech/mini_librispeech.py
Unescape View File

@ -18,14 +18,16 @@ Manifest file is a json-format file with each line containing the
meta data (i.e. audio filepath, transcript and audio duration)
of each audio file in the data set.
"""
import os
import argparse
import soundfile
import json
import codecs
import io
from utils.utility import download, unpack
import json
import os
import soundfile
from utils.utility import download
from utils.utility import unpack
URL_ROOT = "http://www.openslr.org/resources/31"
URL_TRAIN_CLEAN = URL_ROOT + "/train-clean-5.tar.gz"

11
examples/dataset/musan/musan.py
Unescape View File

@ -22,12 +22,15 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import argparse
import codecs
import soundfile
import json
import argparse
from utils.utility import download, unpack
import os
import soundfile
from utils.utility import download
from utils.utility import unpack
DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')

11
examples/dataset/rir_noise/rir_noise.py
Unescape View File

@ -22,12 +22,15 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import argparse
import codecs
import soundfile
import json
import argparse
from utils.utility import download, unzip
import os
import soundfile
from utils.utility import download
from utils.utility import unzip
DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')

13
examples/dataset/voxforge/voxforge.py
Unescape View File

@ -18,16 +18,19 @@ Manifest file is a json-format file with each line containing the
meta data (i.e. audio filepath, transcript and audio duration)
of each audio file in the data set.
"""
import os
import argparse
import codecs
import datetime
import soundfile
import json
import argparse
import os
import shutil
import subprocess
from utils.utility import download_multi, unpack, getfile_insensitive
import soundfile
from utils.utility import download_multi
from utils.utility import getfile_insensitive
from utils.utility import unpack
DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')

5
tests/deepspeech2_model_test.py
Unescape View File

@ -11,10 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import paddle
import numpy as np
import unittest
import paddle
from deepspeech.models.deepspeech2 import DeepSpeech2Model

2
tests/error_rate_test.py
Unescape View File

@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test error rate."""
import unittest
from deepspeech.utils import error_rate

7
tests/mask_test.py
Unescape View File

@ -11,13 +11,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import paddle
import numpy as np
import unittest
from deepspeech.modules.mask import sequence_mask
import paddle
from deepspeech.modules.mask import make_non_pad_mask
from deepspeech.modules.mask import make_pad_mask
from deepspeech.modules.mask import sequence_mask
class TestU2Model(unittest.TestCase):

5
tests/u2_model_test.py
Unescape View File

@ -11,11 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle
import unittest
import numpy as np
import paddle
from yacs.config import CfgNode as CN
from deepspeech.models.u2 import U2Model

9
utils/build_vocab.py
Unescape View File

@ -14,20 +14,19 @@
"""Build vocabulary from manifest files.
Each item in vocabulary file is a character.
"""
import argparse
import functools
from collections import Counter
import os
import tempfile
from collections import Counter
from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.utility import UNK
from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer
from deepspeech.frontend.utility import BLANK
from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.utility import SOS
from deepspeech.frontend.utility import UNK
from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments
from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer
parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)

7
utils/compute_mean_std.py
Unescape View File

@ -12,13 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Compute mean and std for feature normalizer, and save to file."""
import argparse
import functools
from deepspeech.frontend.normalizer import FeatureNormalizer
from deepspeech.frontend.augmentor.augmentation import AugmentationPipeline
from deepspeech.frontend.featurizer.audio_featurizer import AudioFeaturizer
from deepspeech.utils.utility import add_arguments, print_arguments
from deepspeech.frontend.normalizer import FeatureNormalizer
from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments
parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)

4
utils/format_data.py
Unescape View File

@ -16,11 +16,11 @@ import argparse
import functools
import json
from deepspeech.frontend.utility import read_manifest
from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer
from deepspeech.frontend.utility import load_cmvn
from deepspeech.frontend.utility import read_manifest
from deepspeech.utils.utility import add_arguments
from deepspeech.utils.utility import print_arguments
from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer
parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)

6
utils/spm_decode
Unescape View File

@ -4,8 +4,10 @@
#
# This source code is licensed under the license found in the
# https://github.com/pytorch/fairseq/blob/master/LICENSE
from __future__ import absolute_import, division, print_function, unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import argparse
import sys

6
utils/spm_encode
Unescape View File

@ -4,8 +4,10 @@
#
# This source code is licensed under the license found in
# https://github.com/pytorch/fairseq/blob/master/LICENSE
from __future__ import absolute_import, division, print_function, unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import argparse
import contextlib

2
utils/utility.py
Unescape View File

@ -11,10 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import tarfile
import zipfile
from paddle.dataset.common import md5file

Write Preview
Loading…
Cancel
Save