diff --git a/.notebook/python_test.ipynb b/.notebook/python_test.ipynb index f8ed532c4..99bbe0caa 100644 --- a/.notebook/python_test.ipynb +++ b/.notebook/python_test.ipynb @@ -637,7 +637,7 @@ { "cell_type": "code", "execution_count": 59, - "id": "featured-cookbook", + "id": "norwegian-cleveland", "metadata": {}, "outputs": [ { @@ -660,7 +660,7 @@ { "cell_type": "code", "execution_count": 35, - "id": "concrete-rubber", + "id": "endless-kidney", "metadata": {}, "outputs": [ { @@ -705,7 +705,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "surgical-shore", + "id": "later-louisiana", "metadata": {}, "outputs": [ { @@ -728,7 +728,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "medieval-daily", + "id": "funded-nudist", "metadata": {}, "outputs": [ { @@ -748,7 +748,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "derived-meeting", + "id": "contrary-affiliation", "metadata": {}, "outputs": [], "source": [ @@ -758,7 +758,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "hawaiian-engagement", + "id": "friendly-interpretation", "metadata": {}, "outputs": [ { @@ -776,7 +776,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "moderate-greece", + "id": "bottom-wilderness", "metadata": {}, "outputs": [ { @@ -809,7 +809,7 @@ { "cell_type": "code", "execution_count": 17, - "id": "duplicate-colon", + "id": "acquired-jacksonville", "metadata": {}, "outputs": [ { @@ -839,7 +839,7 @@ { "cell_type": "code", "execution_count": 18, - "id": "actual-junior", + "id": "entertaining-capture", "metadata": {}, "outputs": [ { @@ -871,7 +871,7 @@ { "cell_type": "code", "execution_count": 19, - "id": "looking-belle", + "id": "amber-grade", "metadata": {}, "outputs": [ { @@ -903,7 +903,7 @@ { "cell_type": "code", "execution_count": 31, - "id": "turned-friend", + "id": "hidden-playback", "metadata": {}, "outputs": [ { @@ -935,7 +935,7 @@ { "cell_type": "code", "execution_count": 32, - "id": "wrapped-foundation", + "id": "twelve-university", "metadata": {}, "outputs": [ { @@ -966,7 +966,7 @@ { "cell_type": "code", "execution_count": 20, - "id": "applied-abuse", + "id": "minor-endorsement", "metadata": {}, "outputs": [], "source": [ @@ -977,7 +977,7 @@ { "cell_type": "code", "execution_count": 46, - "id": "experimental-invasion", + "id": "upper-majority", "metadata": {}, "outputs": [ { @@ -997,7 +997,7 @@ { "cell_type": "code", "execution_count": 30, - "id": "established-lunch", + "id": "supreme-coverage", "metadata": {}, "outputs": [], "source": [ @@ -1007,7 +1007,7 @@ { "cell_type": "code", "execution_count": 31, - "id": "residential-logic", + "id": "tough-domain", "metadata": {}, "outputs": [], "source": [ @@ -1017,7 +1017,7 @@ { "cell_type": "code", "execution_count": 47, - "id": "maritime-consequence", + "id": "indian-empire", "metadata": {}, "outputs": [ { @@ -1038,7 +1038,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "african-identification", + "id": "horizontal-paragraph", "metadata": {}, "outputs": [], "source": [ @@ -1049,7 +1049,7 @@ { "cell_type": "code", "execution_count": 56, - "id": "upset-regard", + "id": "homeless-zoning", "metadata": {}, "outputs": [], "source": [ @@ -1059,7 +1059,7 @@ { "cell_type": "code", "execution_count": 58, - "id": "wrapped-netherlands", + "id": "floating-atmosphere", "metadata": {}, "outputs": [], "source": [ @@ -1069,7 +1069,7 @@ { "cell_type": "code", "execution_count": 60, - "id": "engaging-statement", + "id": "stupid-reducing", "metadata": {}, "outputs": [], "source": [ @@ -1079,7 +1079,7 @@ { "cell_type": "code", "execution_count": 66, - "id": "joined-handy", + "id": "practical-airline", "metadata": {}, "outputs": [], "source": [ @@ -1089,7 +1089,7 @@ { "cell_type": "code", "execution_count": 95, - "id": "absolute-kenya", + "id": "apart-comfort", "metadata": {}, "outputs": [ { @@ -1110,7 +1110,7 @@ { "cell_type": "code", "execution_count": 81, - "id": "integral-subsection", + "id": "underlying-brand", "metadata": {}, "outputs": [ { @@ -1131,7 +1131,7 @@ { "cell_type": "code", "execution_count": 11, - "id": "conservative-sport", + "id": "german-things", "metadata": {}, "outputs": [], "source": [ @@ -1161,7 +1161,7 @@ { "cell_type": "code", "execution_count": 12, - "id": "connected-brass", + "id": "third-regression", "metadata": {}, "outputs": [], "source": [ @@ -1172,7 +1172,7 @@ { "cell_type": "code", "execution_count": 14, - "id": "worse-couple", + "id": "thick-korea", "metadata": {}, "outputs": [ { @@ -1201,7 +1201,7 @@ { "cell_type": "code", "execution_count": 13, - "id": "unlimited-effects", + "id": "institutional-hands", "metadata": {}, "outputs": [ { @@ -1230,7 +1230,7 @@ { "cell_type": "code", "execution_count": 15, - "id": "stable-artist", + "id": "brave-native", "metadata": {}, "outputs": [ { @@ -1260,7 +1260,7 @@ { "cell_type": "code", "execution_count": 29, - "id": "romance-laser", + "id": "turkish-ticket", "metadata": {}, "outputs": [ { @@ -1290,7 +1290,7 @@ { "cell_type": "code", "execution_count": 30, - "id": "pharmaceutical-colombia", + "id": "executed-excerpt", "metadata": {}, "outputs": [ { @@ -1308,7 +1308,7 @@ { "cell_type": "code", "execution_count": 22, - "id": "forty-harvest", + "id": "continental-boring", "metadata": {}, "outputs": [ { @@ -1343,7 +1343,7 @@ { "cell_type": "code", "execution_count": 23, - "id": "thirty-software", + "id": "linear-assembly", "metadata": {}, "outputs": [ { @@ -1376,7 +1376,7 @@ { "cell_type": "code", "execution_count": 33, - "id": "portuguese-addiction", + "id": "applied-louis", "metadata": {}, "outputs": [], "source": [ @@ -1386,7 +1386,7 @@ { "cell_type": "code", "execution_count": 34, - "id": "everyday-vessel", + "id": "historic-struggle", "metadata": {}, "outputs": [ { @@ -1428,17 +1428,60 @@ { "cell_type": "code", "execution_count": 35, - "id": "latter-expression", + "id": "monthly-roads", "metadata": {}, "outputs": [], "source": [ "getpass?" ] }, + { + "cell_type": "code", + "execution_count": 38, + "id": "boxed-peoples", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Worker:" + ] + } + ], + "source": [ + "import multiprocessing\n", + "import cProfile\n", + "import time\n", + "\n", + "def worker(num):\n", + " time.sleep(3)\n", + " print('Worker:', num)\n", + "\n", + "def profile_worker(num):\n", + " cProfile.runctx('worker(num)', globals(), locals(), 'profile-%d.out' %num)\n", + "\n", + "\n", + "\n", + "for i in range(5):\n", + " p = multiprocessing.Process(target=profile_worker, args=(i,))\n", + " p.start()" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "fresh-tender", + "metadata": {}, + "outputs": [], + "source": [ + "!ls" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "small-professor", + "id": "religious-peripheral", "metadata": {}, "outputs": [], "source": [] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c18efbc15..02c084bb8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,7 +52,7 @@ language: system files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|py)$ #exclude: (?=decoders/swig).*(\.cpp|\.h)$ -#- repo: https://github.com/asottile/reorder_python_imports -# rev: v2.4.0 -# hooks: -# - id: reorder-python-imports +- repo: https://github.com/asottile/reorder_python_imports + rev: v2.4.0 + hooks: + - id: reorder-python-imports diff --git a/deepspeech/__init__.py b/deepspeech/__init__.py index 8be0e93f4..39523d897 100644 --- a/deepspeech/__init__.py +++ b/deepspeech/__init__.py @@ -11,18 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Union +from typing import Any from typing import List from typing import Tuple -from typing import Any +from typing import Union import paddle from paddle import nn -from paddle.nn import functional as F -#TODO(Hui Zhang): remove fluid import from paddle.fluid import core +from paddle.nn import functional as F from deepspeech.utils.log import Log +#TODO(Hui Zhang): remove fluid import logger = Log(__name__).getlog() ########### hcak logging ############# diff --git a/deepspeech/decoders/decoders_deprecated.py b/deepspeech/decoders/decoders_deprecated.py index af05e61b9..fef08807e 100644 --- a/deepspeech/decoders/decoders_deprecated.py +++ b/deepspeech/decoders/decoders_deprecated.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains various CTC decoders.""" - +import multiprocessing from itertools import groupby -import numpy as np from math import log -import multiprocessing + +import numpy as np def ctc_greedy_decoder(probs_seq, vocabulary): diff --git a/deepspeech/decoders/scorer_deprecated.py b/deepspeech/decoders/scorer_deprecated.py index ad4d745fe..d81fb2e3c 100644 --- a/deepspeech/decoders/scorer_deprecated.py +++ b/deepspeech/decoders/scorer_deprecated.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """External Scorer for Beam Search Decoder.""" - import os + import kenlm import numpy as np diff --git a/deepspeech/decoders/swig/setup.py b/deepspeech/decoders/swig/setup.py index b02dc7671..86af475af 100644 --- a/deepspeech/decoders/swig/setup.py +++ b/deepspeech/decoders/swig/setup.py @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. """Script to build and install decoder package.""" - -from setuptools import setup, Extension, distutils +import argparse import glob -import platform +import multiprocessing.pool import os +import platform import sys -import multiprocessing.pool -import argparse + +from setuptools import distutils +from setuptools import Extension +from setuptools import setup parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/deepspeech/decoders/swig_wrapper.py b/deepspeech/decoders/swig_wrapper.py index 3051f4e82..3ffdb9c74 100644 --- a/deepspeech/decoders/swig_wrapper.py +++ b/deepspeech/decoders/swig_wrapper.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """Wrapper for various CTC decoders in SWIG.""" - import swig_decoders diff --git a/deepspeech/decoders/tests/test_decoders.py b/deepspeech/decoders/tests/test_decoders.py index e225d2bc1..6e6265ef4 100644 --- a/deepspeech/decoders/tests/test_decoders.py +++ b/deepspeech/decoders/tests/test_decoders.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """Test decoders.""" - import unittest + from deepspeech.decoders import decoders_deprecated as decoder diff --git a/deepspeech/exps/deepspeech2/bin/deploy/client.py b/deepspeech/exps/deepspeech2/bin/deploy/client.py index e27e561bd..0a83f8a58 100644 --- a/deepspeech/exps/deepspeech2/bin/deploy/client.py +++ b/deepspeech/exps/deepspeech2/bin/deploy/client.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. """Client-end for the ASR demo.""" -import keyboard -import sys import argparse +import sys + +import keyboard import pyaudio from deepspeech.utils.socket_server import socket_send diff --git a/deepspeech/exps/deepspeech2/bin/deploy/record.py b/deepspeech/exps/deepspeech2/bin/deploy/record.py index 717747593..94ad0f1cb 100644 --- a/deepspeech/exps/deepspeech2/bin/deploy/record.py +++ b/deepspeech/exps/deepspeech2/bin/deploy/record.py @@ -13,9 +13,10 @@ # limitations under the License. """Record wav from Microphone""" # http://people.csail.mit.edu/hubert/pyaudio/ -import pyaudio import wave +import pyaudio + CHUNK = 1024 FORMAT = pyaudio.paInt16 CHANNELS = 1 diff --git a/deepspeech/exps/deepspeech2/bin/deploy/runtime.py b/deepspeech/exps/deepspeech2/bin/deploy/runtime.py index 8212fd864..f3125e04d 100644 --- a/deepspeech/exps/deepspeech2/bin/deploy/runtime.py +++ b/deepspeech/exps/deepspeech2/bin/deploy/runtime.py @@ -13,23 +13,21 @@ # limitations under the License. """Server-end for the ASR demo.""" import functools -import paddle -import numpy as np -from deepspeech.utils.socket_server import warm_up_test -from deepspeech.utils.socket_server import AsrTCPServer -from deepspeech.utils.socket_server import AsrRequestHandler +import numpy as np +import paddle +from paddle.inference import Config +from paddle.inference import create_predictor -from deepspeech.training.cli import default_argument_parser from deepspeech.exps.deepspeech2.config import get_cfg_defaults - -from deepspeech.utils.utility import add_arguments, print_arguments - -from deepspeech.models.deepspeech2 import DeepSpeech2Model from deepspeech.io.dataset import ManifestDataset - -from paddle.inference import Config -from paddle.inference import create_predictor +from deepspeech.models.deepspeech2 import DeepSpeech2Model +from deepspeech.training.cli import default_argument_parser +from deepspeech.utils.socket_server import AsrRequestHandler +from deepspeech.utils.socket_server import AsrTCPServer +from deepspeech.utils.socket_server import warm_up_test +from deepspeech.utils.utility import add_arguments +from deepspeech.utils.utility import print_arguments def init_predictor(args): diff --git a/deepspeech/exps/deepspeech2/bin/deploy/server.py b/deepspeech/exps/deepspeech2/bin/deploy/server.py index 89023c85e..b2ff37e06 100644 --- a/deepspeech/exps/deepspeech2/bin/deploy/server.py +++ b/deepspeech/exps/deepspeech2/bin/deploy/server.py @@ -13,20 +13,19 @@ # limitations under the License. """Server-end for the ASR demo.""" import functools -import paddle -import numpy as np -from deepspeech.utils.socket_server import warm_up_test -from deepspeech.utils.socket_server import AsrTCPServer -from deepspeech.utils.socket_server import AsrRequestHandler +import numpy as np +import paddle -from deepspeech.training.cli import default_argument_parser from deepspeech.exps.deepspeech2.config import get_cfg_defaults - -from deepspeech.utils.utility import add_arguments, print_arguments - -from deepspeech.models.deepspeech2 import DeepSpeech2Model from deepspeech.io.dataset import ManifestDataset +from deepspeech.models.deepspeech2 import DeepSpeech2Model +from deepspeech.training.cli import default_argument_parser +from deepspeech.utils.socket_server import AsrRequestHandler +from deepspeech.utils.socket_server import AsrTCPServer +from deepspeech.utils.socket_server import warm_up_test +from deepspeech.utils.utility import add_arguments +from deepspeech.utils.utility import print_arguments def start_server(config, args): diff --git a/deepspeech/exps/deepspeech2/bin/export.py b/deepspeech/exps/deepspeech2/bin/export.py index 7eb543308..a1607d583 100644 --- a/deepspeech/exps/deepspeech2/bin/export.py +++ b/deepspeech/exps/deepspeech2/bin/export.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. """Export for DeepSpeech2 model.""" -from deepspeech.training.cli import default_argument_parser -from deepspeech.utils.utility import print_arguments - from deepspeech.exps.deepspeech2.config import get_cfg_defaults from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester +from deepspeech.training.cli import default_argument_parser +from deepspeech.utils.utility import print_arguments def main_sp(config, args): diff --git a/deepspeech/exps/deepspeech2/bin/infer.py b/deepspeech/exps/deepspeech2/bin/infer.py index 9dda38513..76faf9d62 100644 --- a/deepspeech/exps/deepspeech2/bin/infer.py +++ b/deepspeech/exps/deepspeech2/bin/infer.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. """Inferer for DeepSpeech2 model.""" +from deepspeech.exps.deepspeech2.config import get_cfg_defaults +from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester from deepspeech.training.cli import default_argument_parser from deepspeech.utils.utility import print_arguments # TODO(hui zhang): dynamic load -from deepspeech.exps.deepspeech2.config import get_cfg_defaults -from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester def main_sp(config, args): diff --git a/deepspeech/exps/deepspeech2/bin/test.py b/deepspeech/exps/deepspeech2/bin/test.py index 0f5ec4000..f4edf08a8 100644 --- a/deepspeech/exps/deepspeech2/bin/test.py +++ b/deepspeech/exps/deepspeech2/bin/test.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. """Evaluation for DeepSpeech2 model.""" -from deepspeech.training.cli import default_argument_parser -from deepspeech.utils.utility import print_arguments - from deepspeech.exps.deepspeech2.config import get_cfg_defaults from deepspeech.exps.deepspeech2.model import DeepSpeech2Tester as Tester +from deepspeech.training.cli import default_argument_parser +from deepspeech.utils.utility import print_arguments def main_sp(config, args): diff --git a/deepspeech/exps/deepspeech2/bin/train.py b/deepspeech/exps/deepspeech2/bin/train.py index ef43d793a..5e5c1e2a4 100644 --- a/deepspeech/exps/deepspeech2/bin/train.py +++ b/deepspeech/exps/deepspeech2/bin/train.py @@ -14,11 +14,10 @@ """Trainer for DeepSpeech2 model.""" from paddle import distributed as dist -from deepspeech.utils.utility import print_arguments -from deepspeech.training.cli import default_argument_parser - from deepspeech.exps.deepspeech2.config import get_cfg_defaults from deepspeech.exps.deepspeech2.model import DeepSpeech2Trainer as Trainer +from deepspeech.training.cli import default_argument_parser +from deepspeech.utils.utility import print_arguments def main_sp(config, args): diff --git a/deepspeech/exps/deepspeech2/bin/tune.py b/deepspeech/exps/deepspeech2/bin/tune.py index 5efb433fa..02e329a11 100644 --- a/deepspeech/exps/deepspeech2/bin/tune.py +++ b/deepspeech/exps/deepspeech2/bin/tune.py @@ -12,22 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. """Beam search parameters tuning for DeepSpeech2 model.""" - -import sys -import numpy as np import functools +import sys +import numpy as np from paddle.io import DataLoader -from deepspeech.utils import error_rate -from deepspeech.utils.utility import add_arguments, print_arguments - -from deepspeech.models.deepspeech2 import DeepSpeech2Model +from deepspeech.exps.deepspeech2.config import get_cfg_defaults from deepspeech.io.collator import SpeechCollator from deepspeech.io.dataset import ManifestDataset - +from deepspeech.models.deepspeech2 import DeepSpeech2Model from deepspeech.training.cli import default_argument_parser -from deepspeech.exps.deepspeech2.config import get_cfg_defaults +from deepspeech.utils import error_rate +from deepspeech.utils.utility import add_arguments +from deepspeech.utils.utility import print_arguments def tune(config, args): diff --git a/deepspeech/exps/deepspeech2/config.py b/deepspeech/exps/deepspeech2/config.py index e1ea385b2..85f7fe1e1 100644 --- a/deepspeech/exps/deepspeech2/config.py +++ b/deepspeech/exps/deepspeech2/config.py @@ -11,8 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from yacs.config import CfgNode as CN + from deepspeech.models.deepspeech2 import DeepSpeech2Model _C = CN() diff --git a/deepspeech/exps/deepspeech2/model.py b/deepspeech/exps/deepspeech2/model.py index 033509177..c72e3d0a6 100644 --- a/deepspeech/exps/deepspeech2/model.py +++ b/deepspeech/exps/deepspeech2/model.py @@ -12,31 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains DeepSpeech2 model.""" - import time -from deepspeech.utils.log import Log -import numpy as np from collections import defaultdict from pathlib import Path +import numpy as np import paddle from paddle import distributed as dist from paddle.io import DataLoader -from deepspeech.training.trainer import Trainer -from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog - -from deepspeech.utils import mp_tools -from deepspeech.utils import layer_tools -from deepspeech.utils import error_rate - from deepspeech.io.collator import SpeechCollator -from deepspeech.io.sampler import SortagradDistributedBatchSampler -from deepspeech.io.sampler import SortagradBatchSampler from deepspeech.io.dataset import ManifestDataset - -from deepspeech.models.deepspeech2 import DeepSpeech2Model +from deepspeech.io.sampler import SortagradBatchSampler +from deepspeech.io.sampler import SortagradDistributedBatchSampler from deepspeech.models.deepspeech2 import DeepSpeech2InferModel +from deepspeech.models.deepspeech2 import DeepSpeech2Model +from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog +from deepspeech.training.trainer import Trainer +from deepspeech.utils import error_rate +from deepspeech.utils import layer_tools +from deepspeech.utils import mp_tools +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/exps/u2/bin/export.py b/deepspeech/exps/u2/bin/export.py index 1fa0799ee..292c78389 100644 --- a/deepspeech/exps/u2/bin/export.py +++ b/deepspeech/exps/u2/bin/export.py @@ -12,12 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. """Export for U2 model.""" - -from deepspeech.training.cli import default_argument_parser -from deepspeech.utils.utility import print_arguments - from deepspeech.exps.u2.config import get_cfg_defaults from deepspeech.exps.u2.model import U2Tester as Tester +from deepspeech.training.cli import default_argument_parser +from deepspeech.utils.utility import print_arguments def main_sp(config, args): diff --git a/deepspeech/exps/u2/bin/test.py b/deepspeech/exps/u2/bin/test.py index 7d8e719cc..a80106f74 100644 --- a/deepspeech/exps/u2/bin/test.py +++ b/deepspeech/exps/u2/bin/test.py @@ -12,15 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. """Evaluation for U2 model.""" -import os import cProfile +import os +from deepspeech.exps.u2.config import get_cfg_defaults +from deepspeech.exps.u2.model import U2Tester as Tester from deepspeech.training.cli import default_argument_parser from deepspeech.utils.utility import print_arguments # TODO(hui zhang): dynamic load -from deepspeech.exps.u2.config import get_cfg_defaults -from deepspeech.exps.u2.model import U2Tester as Tester def main_sp(config, args): diff --git a/deepspeech/exps/u2/bin/train.py b/deepspeech/exps/u2/bin/train.py index 02845e9cc..f23716781 100644 --- a/deepspeech/exps/u2/bin/train.py +++ b/deepspeech/exps/u2/bin/train.py @@ -12,16 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. """Trainer for U2 model.""" - -import os import cProfile -from paddle import distributed as dist +import os -from deepspeech.utils.utility import print_arguments -from deepspeech.training.cli import default_argument_parser +from paddle import distributed as dist from deepspeech.exps.u2.config import get_cfg_defaults from deepspeech.exps.u2.model import U2Trainer as Trainer +from deepspeech.training.cli import default_argument_parser +from deepspeech.utils.utility import print_arguments def main_sp(config, args): diff --git a/deepspeech/exps/u2/config.py b/deepspeech/exps/u2/config.py index 34228cb93..460c24304 100644 --- a/deepspeech/exps/u2/config.py +++ b/deepspeech/exps/u2/config.py @@ -11,13 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from yacs.config import CfgNode +from deepspeech.exps.u2.model import U2Tester +from deepspeech.exps.u2.model import U2Trainer from deepspeech.io.dataset import ManifestDataset from deepspeech.models.u2 import U2Model -from deepspeech.exps.u2.model import U2Trainer -from deepspeech.exps.u2.model import U2Tester _C = CfgNode() diff --git a/deepspeech/exps/u2/model.py b/deepspeech/exps/u2/model.py index ae66cc07a..7c623a03f 100644 --- a/deepspeech/exps/u2/model.py +++ b/deepspeech/exps/u2/model.py @@ -12,34 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains U2 model.""" - import sys import time -from deepspeech.utils.log import Log -import numpy as np from collections import defaultdict from pathlib import Path from typing import Optional -from yacs.config import CfgNode +import numpy as np import paddle from paddle import distributed as dist from paddle.io import DataLoader - -from deepspeech.training.trainer import Trainer -from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog -from deepspeech.training.scheduler import WarmupLR - -from deepspeech.utils import mp_tools -from deepspeech.utils import layer_tools -from deepspeech.utils import error_rate +from yacs.config import CfgNode from deepspeech.io.collator import SpeechCollator -from deepspeech.io.sampler import SortagradDistributedBatchSampler -from deepspeech.io.sampler import SortagradBatchSampler from deepspeech.io.dataset import ManifestDataset - +from deepspeech.io.sampler import SortagradBatchSampler +from deepspeech.io.sampler import SortagradDistributedBatchSampler from deepspeech.models.u2 import U2Model +from deepspeech.training.gradclip import ClipGradByGlobalNormWithLog +from deepspeech.training.scheduler import WarmupLR +from deepspeech.training.trainer import Trainer +from deepspeech.utils import error_rate +from deepspeech.utils import layer_tools +from deepspeech.utils import mp_tools +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/frontend/audio.py b/deepspeech/frontend/audio.py index 18c75a177..a45f91d49 100644 --- a/deepspeech/frontend/audio.py +++ b/deepspeech/frontend/audio.py @@ -12,16 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the audio segment class.""" - -import numpy as np +import copy import io -import struct +import random import re -import soundfile +import struct + +import numpy as np import resampy +import soundfile from scipy import signal -import random -import copy class AudioSegment(object): diff --git a/deepspeech/frontend/augmentor/augmentation.py b/deepspeech/frontend/augmentor/augmentation.py index 9204ae3f5..cc0564daf 100644 --- a/deepspeech/frontend/augmentor/augmentation.py +++ b/deepspeech/frontend/augmentor/augmentation.py @@ -12,21 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the data augmentation pipeline.""" - import json + import numpy as np -# audio augment -from deepspeech.frontend.augmentor.volume_perturb import VolumePerturbAugmentor -from deepspeech.frontend.augmentor.shift_perturb import ShiftPerturbAugmentor -from deepspeech.frontend.augmentor.speed_perturb import SpeedPerturbAugmentor -from deepspeech.frontend.augmentor.noise_perturb import NoisePerturbAugmentor from deepspeech.frontend.augmentor.impulse_response import ImpulseResponseAugmentor -from deepspeech.frontend.augmentor.resample import ResampleAugmentor +from deepspeech.frontend.augmentor.noise_perturb import NoisePerturbAugmentor from deepspeech.frontend.augmentor.online_bayesian_normalization import \ OnlineBayesianNormalizationAugmentor -# feature augment +from deepspeech.frontend.augmentor.resample import ResampleAugmentor +from deepspeech.frontend.augmentor.shift_perturb import ShiftPerturbAugmentor from deepspeech.frontend.augmentor.spec_augment import SpecAugmentor +from deepspeech.frontend.augmentor.speed_perturb import SpeedPerturbAugmentor +from deepspeech.frontend.augmentor.volume_perturb import VolumePerturbAugmentor class AugmentationPipeline(): diff --git a/deepspeech/frontend/augmentor/base.py b/deepspeech/frontend/augmentor/base.py index fcc49d3f7..e6f5c1e9f 100644 --- a/deepspeech/frontend/augmentor/base.py +++ b/deepspeech/frontend/augmentor/base.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the abstract base class for augmentation models.""" - -from abc import ABCMeta, abstractmethod +from abc import ABCMeta +from abc import abstractmethod class AugmentorBase(): diff --git a/deepspeech/frontend/augmentor/impulse_response.py b/deepspeech/frontend/augmentor/impulse_response.py index fb359445f..fbd617b42 100644 --- a/deepspeech/frontend/augmentor/impulse_response.py +++ b/deepspeech/frontend/augmentor/impulse_response.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the impulse response augmentation model.""" - +from deepspeech.frontend.audio import AudioSegment from deepspeech.frontend.augmentor.base import AugmentorBase from deepspeech.frontend.utility import read_manifest -from deepspeech.frontend.audio import AudioSegment class ImpulseResponseAugmentor(AugmentorBase): diff --git a/deepspeech/frontend/augmentor/noise_perturb.py b/deepspeech/frontend/augmentor/noise_perturb.py index e24da4c3e..b3c07f5c1 100644 --- a/deepspeech/frontend/augmentor/noise_perturb.py +++ b/deepspeech/frontend/augmentor/noise_perturb.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the noise perturb augmentation model.""" - +from deepspeech.frontend.audio import AudioSegment from deepspeech.frontend.augmentor.base import AugmentorBase from deepspeech.frontend.utility import read_manifest -from deepspeech.frontend.audio import AudioSegment class NoisePerturbAugmentor(AugmentorBase): diff --git a/deepspeech/frontend/augmentor/online_bayesian_normalization.py b/deepspeech/frontend/augmentor/online_bayesian_normalization.py index 14c260dfd..5af3b9b03 100644 --- a/deepspeech/frontend/augmentor/online_bayesian_normalization.py +++ b/deepspeech/frontend/augmentor/online_bayesian_normalization.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contain the online bayesian normalization augmentation model.""" - from deepspeech.frontend.augmentor.base import AugmentorBase diff --git a/deepspeech/frontend/augmentor/resample.py b/deepspeech/frontend/augmentor/resample.py index 8ef574cbb..9afce635d 100644 --- a/deepspeech/frontend/augmentor/resample.py +++ b/deepspeech/frontend/augmentor/resample.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contain the resample augmentation model.""" - from deepspeech.frontend.augmentor.base import AugmentorBase diff --git a/deepspeech/frontend/augmentor/shift_perturb.py b/deepspeech/frontend/augmentor/shift_perturb.py index 2edbf594d..9cc3fe2d0 100644 --- a/deepspeech/frontend/augmentor/shift_perturb.py +++ b/deepspeech/frontend/augmentor/shift_perturb.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the volume perturb augmentation model.""" - from deepspeech.frontend.augmentor.base import AugmentorBase diff --git a/deepspeech/frontend/augmentor/spec_augment.py b/deepspeech/frontend/augmentor/spec_augment.py index 59240be5a..518bbe096 100644 --- a/deepspeech/frontend/augmentor/spec_augment.py +++ b/deepspeech/frontend/augmentor/spec_augment.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the volume perturb augmentation model.""" -from deepspeech.utils.log import Log import numpy as np from deepspeech.frontend.augmentor.base import AugmentorBase +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/frontend/augmentor/speed_perturb.py b/deepspeech/frontend/augmentor/speed_perturb.py index f3fbdd629..d0977c131 100644 --- a/deepspeech/frontend/augmentor/speed_perturb.py +++ b/deepspeech/frontend/augmentor/speed_perturb.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contain the speech perturbation augmentation model.""" - import numpy as np + from deepspeech.frontend.augmentor.base import AugmentorBase diff --git a/deepspeech/frontend/augmentor/volume_perturb.py b/deepspeech/frontend/augmentor/volume_perturb.py index dc64d0e9e..0d76e7a05 100644 --- a/deepspeech/frontend/augmentor/volume_perturb.py +++ b/deepspeech/frontend/augmentor/volume_perturb.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the volume perturb augmentation model.""" - from deepspeech.frontend.augmentor.base import AugmentorBase diff --git a/deepspeech/frontend/featurizer/audio_featurizer.py b/deepspeech/frontend/featurizer/audio_featurizer.py index 72e70c0d1..eddfdc8c5 100644 --- a/deepspeech/frontend/featurizer/audio_featurizer.py +++ b/deepspeech/frontend/featurizer/audio_featurizer.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the audio featurizer class.""" - import numpy as np -from python_speech_features import mfcc -from python_speech_features import logfbank from python_speech_features import delta +from python_speech_features import logfbank +from python_speech_features import mfcc class AudioFeaturizer(object): diff --git a/deepspeech/frontend/featurizer/speech_featurizer.py b/deepspeech/frontend/featurizer/speech_featurizer.py index 920bec538..83bade7e8 100644 --- a/deepspeech/frontend/featurizer/speech_featurizer.py +++ b/deepspeech/frontend/featurizer/speech_featurizer.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the speech featurizer class.""" - from deepspeech.frontend.featurizer.audio_featurizer import AudioFeaturizer from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer diff --git a/deepspeech/frontend/featurizer/text_featurizer.py b/deepspeech/frontend/featurizer/text_featurizer.py index a704e37ef..1ba6ac7f9 100644 --- a/deepspeech/frontend/featurizer/text_featurizer.py +++ b/deepspeech/frontend/featurizer/text_featurizer.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the text featurizer class.""" - import sentencepiece as spm -from deepspeech.frontend.utility import UNK from deepspeech.frontend.utility import EOS +from deepspeech.frontend.utility import UNK class TextFeaturizer(object): diff --git a/deepspeech/frontend/normalizer.py b/deepspeech/frontend/normalizer.py index 7be421fb1..e9524bf38 100644 --- a/deepspeech/frontend/normalizer.py +++ b/deepspeech/frontend/normalizer.py @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains feature normalizers.""" +import random import numpy as np -import random -from deepspeech.frontend.utility import read_manifest -from deepspeech.frontend.utility import load_cmvn + from deepspeech.frontend.audio import AudioSegment +from deepspeech.frontend.utility import load_cmvn +from deepspeech.frontend.utility import read_manifest class FeatureNormalizer(object): diff --git a/deepspeech/frontend/speech.py b/deepspeech/frontend/speech.py index d43733856..e58795c0e 100644 --- a/deepspeech/frontend/speech.py +++ b/deepspeech/frontend/speech.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains the speech segment class.""" - import numpy as np + from deepspeech.frontend.audio import AudioSegment diff --git a/deepspeech/frontend/utility.py b/deepspeech/frontend/utility.py index 657a71a9f..4b17c841a 100644 --- a/deepspeech/frontend/utility.py +++ b/deepspeech/frontend/utility.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains data helper functions.""" -import numpy as np -import math -import json import codecs +import json +import math + +import numpy as np + from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/io/__init__.py b/deepspeech/io/__init__.py index 9fe0e0ebb..e180f18ee 100644 --- a/deepspeech/io/__init__.py +++ b/deepspeech/io/__init__.py @@ -11,14 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import numpy as np from paddle.io import DataLoader from deepspeech.io.collator import SpeechCollator -from deepspeech.io.sampler import SortagradDistributedBatchSampler -from deepspeech.io.sampler import SortagradBatchSampler from deepspeech.io.dataset import ManifestDataset +from deepspeech.io.sampler import SortagradBatchSampler +from deepspeech.io.sampler import SortagradDistributedBatchSampler def create_dataloader(manifest_path, diff --git a/deepspeech/io/collator.py b/deepspeech/io/collator.py index 310333d04..0b7dbd13e 100644 --- a/deepspeech/io/collator.py +++ b/deepspeech/io/collator.py @@ -11,12 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from deepspeech.utils.log import Log import numpy as np -from deepspeech.io.utility import pad_sequence from deepspeech.frontend.utility import IGNORE_ID +from deepspeech.io.utility import pad_sequence +from deepspeech.utils.log import Log __all__ = ["SpeechCollator"] diff --git a/deepspeech/io/dataset.py b/deepspeech/io/dataset.py index a33724947..e1ee7a01b 100644 --- a/deepspeech/io/dataset.py +++ b/deepspeech/io/dataset.py @@ -11,23 +11,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import time import io import random import tarfile -from deepspeech.utils.log import Log -from typing import Optional -from yacs.config import CfgNode +import time from collections import namedtuple +from typing import Optional from paddle.io import Dataset +from yacs.config import CfgNode -from deepspeech.frontend.utility import read_manifest from deepspeech.frontend.augmentor.augmentation import AugmentationPipeline from deepspeech.frontend.featurizer.speech_featurizer import SpeechFeaturizer -from deepspeech.frontend.speech import SpeechSegment from deepspeech.frontend.normalizer import FeatureNormalizer +from deepspeech.frontend.speech import SpeechSegment +from deepspeech.frontend.utility import read_manifest +from deepspeech.utils.log import Log __all__ = [ "ManifestDataset", diff --git a/deepspeech/io/sampler.py b/deepspeech/io/sampler.py index 910169af6..763a3781e 100644 --- a/deepspeech/io/sampler.py +++ b/deepspeech/io/sampler.py @@ -11,14 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import math -from deepspeech.utils.log import Log -import numpy as np +import numpy as np +from paddle import distributed as dist from paddle.io import BatchSampler from paddle.io import DistributedBatchSampler -from paddle import distributed as dist + +from deepspeech.utils.log import Log __all__ = [ "SortagradDistributedBatchSampler", diff --git a/deepspeech/io/utility.py b/deepspeech/io/utility.py index de0e52349..0cd37428b 100644 --- a/deepspeech/io/utility.py +++ b/deepspeech/io/utility.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import List -from deepspeech.utils.log import Log import numpy as np -from typing import List + +from deepspeech.utils.log import Log __all__ = ["pad_sequence"] diff --git a/deepspeech/models/deepspeech2.py b/deepspeech/models/deepspeech2.py index a3ad9fed4..75b771b89 100644 --- a/deepspeech/models/deepspeech2.py +++ b/deepspeech/models/deepspeech2.py @@ -12,19 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. """Deepspeech2 ASR Model""" -from deepspeech.utils.log import Log from typing import Optional -from yacs.config import CfgNode import paddle from paddle import nn +from yacs.config import CfgNode from deepspeech.modules.conv import ConvStack -from deepspeech.modules.rnn import RNNStack from deepspeech.modules.ctc import CTCDecoder - +from deepspeech.modules.rnn import RNNStack from deepspeech.utils import checkpoint from deepspeech.utils import layer_tools +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/models/u2.py b/deepspeech/models/u2.py index 04a57c1ee..16e5eaad6 100644 --- a/deepspeech/models/u2.py +++ b/deepspeech/models/u2.py @@ -15,40 +15,39 @@ Unified Streaming and Non-streaming Two-pass End-to-end Model for Speech Recognition (https://arxiv.org/pdf/2012.05481.pdf) """ - -import time import sys +import time from collections import defaultdict -from deepspeech.utils.log import Log -from yacs.config import CfgNode -from typing import List, Optional, Tuple, Dict +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple import paddle from paddle import jit from paddle import nn +from yacs.config import CfgNode -from deepspeech.modules.mask import make_pad_mask -from deepspeech.modules.mask import mask_finished_preds -from deepspeech.modules.mask import mask_finished_scores -from deepspeech.modules.mask import subsequent_mask - +from deepspeech.frontend.utility import IGNORE_ID +from deepspeech.frontend.utility import load_cmvn from deepspeech.modules.cmvn import GlobalCMVN -from deepspeech.modules.encoder import ConformerEncoder -from deepspeech.modules.encoder import TransformerEncoder from deepspeech.modules.ctc import CTCDecoder from deepspeech.modules.decoder import TransformerDecoder +from deepspeech.modules.encoder import ConformerEncoder +from deepspeech.modules.encoder import TransformerEncoder from deepspeech.modules.loss import LabelSmoothingLoss - -from deepspeech.frontend.utility import load_cmvn -from deepspeech.frontend.utility import IGNORE_ID - +from deepspeech.modules.mask import make_pad_mask +from deepspeech.modules.mask import mask_finished_preds +from deepspeech.modules.mask import mask_finished_scores +from deepspeech.modules.mask import subsequent_mask from deepspeech.utils import checkpoint from deepspeech.utils import layer_tools -from deepspeech.utils.utility import log_add +from deepspeech.utils.ctc_utils import remove_duplicates_and_blank +from deepspeech.utils.log import Log from deepspeech.utils.tensor_utils import add_sos_eos -from deepspeech.utils.tensor_utils import th_accuracy from deepspeech.utils.tensor_utils import pad_sequence -from deepspeech.utils.ctc_utils import remove_duplicates_and_blank +from deepspeech.utils.tensor_utils import th_accuracy +from deepspeech.utils.utility import log_add logger = Log(__name__).getlog() diff --git a/deepspeech/modules/activation.py b/deepspeech/modules/activation.py index eb0297dd3..0fe66b739 100644 --- a/deepspeech/modules/activation.py +++ b/deepspeech/modules/activation.py @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from deepspeech.utils.log import Log from collections import OrderedDict import paddle from paddle import nn +from deepspeech.utils.log import Log + logger = Log(__name__).getlog() __all__ = ["get_activation", "brelu", "LinearGLUBlock", "ConvGLUBlock"] diff --git a/deepspeech/modules/attention.py b/deepspeech/modules/attention.py index 19c338177..4401a4a55 100644 --- a/deepspeech/modules/attention.py +++ b/deepspeech/modules/attention.py @@ -13,13 +13,15 @@ # limitations under the License. """Multi-Head Attention layer definition.""" import math -from deepspeech.utils.log import Log -from typing import Optional, Tuple +from typing import Optional +from typing import Tuple import paddle from paddle import nn from paddle.nn import initializer as I +from deepspeech.utils.log import Log + logger = Log(__name__).getlog() __all__ = ["MultiHeadedAttention", "RelPositionMultiHeadedAttention"] diff --git a/deepspeech/modules/cmvn.py b/deepspeech/modules/cmvn.py index 0d65242bc..b98af052e 100644 --- a/deepspeech/modules/cmvn.py +++ b/deepspeech/modules/cmvn.py @@ -11,12 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from deepspeech.utils.log import Log - import paddle from paddle import nn +from deepspeech.utils.log import Log + logger = Log(__name__).getlog() __all__ = ['GlobalCMVN'] diff --git a/deepspeech/modules/conformer_convolution.py b/deepspeech/modules/conformer_convolution.py index 2e6d40bb1..b2f194c4a 100644 --- a/deepspeech/modules/conformer_convolution.py +++ b/deepspeech/modules/conformer_convolution.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. """ConvolutionModule definition.""" -from typing import Optional, Tuple -from typeguard import check_argument_types - -from deepspeech.utils.log import Log +from typing import Optional +from typing import Tuple import paddle from paddle import nn +from typeguard import check_argument_types + +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/modules/conv.py b/deepspeech/modules/conv.py index c86b36f47..f0f0d7463 100644 --- a/deepspeech/modules/conv.py +++ b/deepspeech/modules/conv.py @@ -11,14 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from deepspeech.utils.log import Log - from paddle import nn from paddle.nn import functional as F -from deepspeech.modules.mask import sequence_mask from deepspeech.modules.activation import brelu +from deepspeech.modules.mask import sequence_mask +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/modules/ctc.py b/deepspeech/modules/ctc.py index fc0fc8f38..31e489a3d 100644 --- a/deepspeech/modules/ctc.py +++ b/deepspeech/modules/ctc.py @@ -11,20 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from deepspeech.utils.log import Log -from typeguard import check_argument_types - import paddle from paddle import nn from paddle.nn import functional as F +from typeguard import check_argument_types +from deepspeech.decoders.swig_wrapper import ctc_beam_search_decoder_batch +from deepspeech.decoders.swig_wrapper import ctc_greedy_decoder +from deepspeech.decoders.swig_wrapper import Scorer from deepspeech.modules.loss import CTCLoss from deepspeech.utils import ctc_utils - -from deepspeech.decoders.swig_wrapper import Scorer -from deepspeech.decoders.swig_wrapper import ctc_greedy_decoder -from deepspeech.decoders.swig_wrapper import ctc_beam_search_decoder_batch +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/modules/decoder.py b/deepspeech/modules/decoder.py index 6d6e0944a..696a6315b 100644 --- a/deepspeech/modules/decoder.py +++ b/deepspeech/modules/decoder.py @@ -12,19 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. """Decoder definition.""" -from typing import Tuple, List, Optional -from typeguard import check_argument_types -from deepspeech.utils.log import Log +from typing import List +from typing import Optional +from typing import Tuple import paddle from paddle import nn +from typeguard import check_argument_types from deepspeech.modules.attention import MultiHeadedAttention from deepspeech.modules.decoder_layer import DecoderLayer from deepspeech.modules.embedding import PositionalEncoding -from deepspeech.modules.positionwise_feed_forward import PositionwiseFeedForward -from deepspeech.modules.mask import subsequent_mask from deepspeech.modules.mask import make_non_pad_mask +from deepspeech.modules.mask import subsequent_mask +from deepspeech.modules.positionwise_feed_forward import PositionwiseFeedForward +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/modules/decoder_layer.py b/deepspeech/modules/decoder_layer.py index 84490290e..c6fac5412 100644 --- a/deepspeech/modules/decoder_layer.py +++ b/deepspeech/modules/decoder_layer.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. """Decoder self-attention layer definition.""" -from typing import Optional, Tuple -from deepspeech.utils.log import Log +from typing import Optional +from typing import Tuple import paddle from paddle import nn +from deepspeech.utils.log import Log + logger = Log(__name__).getlog() __all__ = ["DecoderLayer"] diff --git a/deepspeech/modules/embedding.py b/deepspeech/modules/embedding.py index 8d7a842e1..98b4e1291 100644 --- a/deepspeech/modules/embedding.py +++ b/deepspeech/modules/embedding.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. """Positonal Encoding Module.""" - import math -from deepspeech.utils.log import Log from typing import Tuple import paddle from paddle import nn +from deepspeech.utils.log import Log + logger = Log(__name__).getlog() __all__ = ["PositionalEncoding", "RelPositionalEncoding"] diff --git a/deepspeech/modules/encoder.py b/deepspeech/modules/encoder.py index 70b01eb4a..41fcbac26 100644 --- a/deepspeech/modules/encoder.py +++ b/deepspeech/modules/encoder.py @@ -12,28 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. """Encoder definition.""" -from deepspeech.utils.log import Log -from typing import Tuple, List, Optional -from typeguard import check_argument_types +from typing import List +from typing import Optional +from typing import Tuple import paddle from paddle import nn +from typeguard import check_argument_types +from deepspeech.modules.activation import get_activation from deepspeech.modules.attention import MultiHeadedAttention from deepspeech.modules.attention import RelPositionMultiHeadedAttention from deepspeech.modules.conformer_convolution import ConvolutionModule from deepspeech.modules.embedding import PositionalEncoding from deepspeech.modules.embedding import RelPositionalEncoding -from deepspeech.modules.encoder_layer import TransformerEncoderLayer from deepspeech.modules.encoder_layer import ConformerEncoderLayer +from deepspeech.modules.encoder_layer import TransformerEncoderLayer +from deepspeech.modules.mask import add_optional_chunk_mask +from deepspeech.modules.mask import make_non_pad_mask from deepspeech.modules.positionwise_feed_forward import PositionwiseFeedForward from deepspeech.modules.subsampling import Conv2dSubsampling4 from deepspeech.modules.subsampling import Conv2dSubsampling6 from deepspeech.modules.subsampling import Conv2dSubsampling8 from deepspeech.modules.subsampling import LinearNoSubsampling -from deepspeech.modules.mask import make_non_pad_mask -from deepspeech.modules.mask import add_optional_chunk_mask -from deepspeech.modules.activation import get_activation +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/modules/encoder_layer.py b/deepspeech/modules/encoder_layer.py index 5dec45652..1db556ca5 100644 --- a/deepspeech/modules/encoder_layer.py +++ b/deepspeech/modules/encoder_layer.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. """Encoder self-attention layer definition.""" -from typing import Optional, Tuple -from deepspeech.utils.log import Log +from typing import Optional +from typing import Tuple import paddle from paddle import nn +from deepspeech.utils.log import Log + logger = Log(__name__).getlog() __all__ = ["TransformerEncoderLayer", "ConformerEncoderLayer"] diff --git a/deepspeech/modules/loss.py b/deepspeech/modules/loss.py index d78f689d0..870dd7cd9 100644 --- a/deepspeech/modules/loss.py +++ b/deepspeech/modules/loss.py @@ -11,13 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from deepspeech.utils.log import Log - import paddle from paddle import nn from paddle.nn import functional as F +from deepspeech.utils.log import Log + logger = Log(__name__).getlog() __all__ = ['CTCLoss', "LabelSmoothingLoss"] diff --git a/deepspeech/modules/mask.py b/deepspeech/modules/mask.py index 2877b063e..fa6e0d552 100644 --- a/deepspeech/modules/mask.py +++ b/deepspeech/modules/mask.py @@ -11,11 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import paddle from deepspeech.utils.log import Log -import paddle - logger = Log(__name__).getlog() __all__ = [ diff --git a/deepspeech/modules/positionwise_feed_forward.py b/deepspeech/modules/positionwise_feed_forward.py index 3242dd6f1..9732e1aa4 100644 --- a/deepspeech/modules/positionwise_feed_forward.py +++ b/deepspeech/modules/positionwise_feed_forward.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. """Positionwise feed forward layer definition.""" -from deepspeech.utils.log import Log - import paddle from paddle import nn +from deepspeech.utils.log import Log + logger = Log(__name__).getlog() __all__ = ["PositionwiseFeedForward"] diff --git a/deepspeech/modules/rnn.py b/deepspeech/modules/rnn.py index 6d26fb4f8..cef731e35 100644 --- a/deepspeech/modules/rnn.py +++ b/deepspeech/modules/rnn.py @@ -11,17 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import math -from deepspeech.utils.log import Log import paddle from paddle import nn from paddle.nn import functional as F from paddle.nn import initializer as I -from deepspeech.modules.mask import sequence_mask from deepspeech.modules.activation import brelu +from deepspeech.modules.mask import sequence_mask +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/modules/subsampling.py b/deepspeech/modules/subsampling.py index 8ffb20cfb..512d3c290 100644 --- a/deepspeech/modules/subsampling.py +++ b/deepspeech/modules/subsampling.py @@ -12,14 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. """Subsampling layer definition.""" - from typing import Tuple -from deepspeech.utils.log import Log import paddle from paddle import nn from deepspeech.modules.embedding import PositionalEncoding +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/training/cli.py b/deepspeech/training/cli.py index d06672825..b83d989d6 100644 --- a/deepspeech/training/cli.py +++ b/deepspeech/training/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import argparse diff --git a/deepspeech/training/gradclip.py b/deepspeech/training/gradclip.py index 8f62383d4..348308fdb 100644 --- a/deepspeech/training/gradclip.py +++ b/deepspeech/training/gradclip.py @@ -11,13 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from deepspeech.utils.log import Log - import paddle -from paddle.fluid.dygraph import base as imperative_base -from paddle.fluid import layers from paddle.fluid import core +from paddle.fluid import layers +from paddle.fluid.dygraph import base as imperative_base + +from deepspeech.utils.log import Log __all__ = ["ClipGradByGlobalNormWithLog"] diff --git a/deepspeech/training/scheduler.py b/deepspeech/training/scheduler.py index b2400a73a..07195c298 100644 --- a/deepspeech/training/scheduler.py +++ b/deepspeech/training/scheduler.py @@ -11,12 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from deepspeech.utils.log import Log from typing import Union -from typeguard import check_argument_types from paddle.optimizer.lr import LRScheduler +from typeguard import check_argument_types + +from deepspeech.utils.log import Log __all__ = ["WarmupLR"] diff --git a/deepspeech/training/trainer.py b/deepspeech/training/trainer.py index 779fe032a..ff5910d16 100644 --- a/deepspeech/training/trainer.py +++ b/deepspeech/training/trainer.py @@ -11,9 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import time -from deepspeech.utils.log import Log from pathlib import Path import paddle @@ -22,6 +20,7 @@ from tensorboardX import SummaryWriter from deepspeech.utils import checkpoint from deepspeech.utils import mp_tools +from deepspeech.utils.log import Log __all__ = ["Trainer"] diff --git a/deepspeech/utils/checkpoint.py b/deepspeech/utils/checkpoint.py index 1174d8559..882b65241 100644 --- a/deepspeech/utils/checkpoint.py +++ b/deepspeech/utils/checkpoint.py @@ -11,11 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import json import os -from deepspeech.utils.log import Log import re -import json from typing import Union import paddle @@ -23,6 +21,7 @@ from paddle import distributed as dist from paddle.optimizer import Optimizer from deepspeech.utils import mp_tools +from deepspeech.utils.log import Log logger = Log(__name__).getlog() diff --git a/deepspeech/utils/ctc_utils.py b/deepspeech/utils/ctc_utils.py index fe6464502..73669fea6 100644 --- a/deepspeech/utils/ctc_utils.py +++ b/deepspeech/utils/ctc_utils.py @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from deepspeech.utils.log import Log -import numpy as np from typing import List +import numpy as np import paddle +from deepspeech.utils.log import Log + logger = Log(__name__).getlog() __all__ = ["forced_align", "remove_duplicates_and_blank", "insert_blank"] diff --git a/deepspeech/utils/layer_tools.py b/deepspeech/utils/layer_tools.py index 0138ff1a9..0ff4f6f54 100644 --- a/deepspeech/utils/layer_tools.py +++ b/deepspeech/utils/layer_tools.py @@ -11,9 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import numpy as np - from paddle import nn __all__ = [ diff --git a/deepspeech/utils/log.py b/deepspeech/utils/log.py index d753a1181..4f3ade870 100644 --- a/deepspeech/utils/log.py +++ b/deepspeech/utils/log.py @@ -11,12 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import getpass import logging import os -import sys -import getpass import socket +import sys import time diff --git a/deepspeech/utils/mp_tools.py b/deepspeech/utils/mp_tools.py index 871735a4f..d3e25aab6 100644 --- a/deepspeech/utils/mp_tools.py +++ b/deepspeech/utils/mp_tools.py @@ -11,9 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from functools import wraps from paddle import distributed as dist -from functools import wraps __all__ = ["rank_zero_only"] diff --git a/deepspeech/utils/socket_server.py b/deepspeech/utils/socket_server.py index 8a4f7dbc5..adcbf3bb2 100644 --- a/deepspeech/utils/socket_server.py +++ b/deepspeech/utils/socket_server.py @@ -11,15 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import os import random -import time -from time import gmtime, strftime import socket import socketserver import struct +import time import wave +from time import gmtime +from time import strftime from deepspeech.frontend.utility import read_manifest diff --git a/deepspeech/utils/tensor_utils.py b/deepspeech/utils/tensor_utils.py index 0a78125f8..6c423167c 100644 --- a/deepspeech/utils/tensor_utils.py +++ b/deepspeech/utils/tensor_utils.py @@ -12,11 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. """Unility functions for Transformer.""" -from deepspeech.utils.log import Log -from typing import Tuple, List +from typing import List +from typing import Tuple import paddle +from deepspeech.utils.log import Log + __all__ = ["pad_sequence", "add_sos_eos", "th_accuracy"] logger = Log(__name__).getlog() diff --git a/deepspeech/utils/utility.py b/deepspeech/utils/utility.py index 3407e2e8f..64570026b 100644 --- a/deepspeech/utils/utility.py +++ b/deepspeech/utils/utility.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains common utility functions.""" - -import os -import math import distutils.util +import math +import os from typing import List __all__ = ['print_arguments', 'add_arguments', "log_add"] diff --git a/examples/dataset/aishell/aishell.py b/examples/dataset/aishell/aishell.py index 2037546dd..a0cabe352 100644 --- a/examples/dataset/aishell/aishell.py +++ b/examples/dataset/aishell/aishell.py @@ -18,13 +18,15 @@ Manifest file is a json-format file with each line containing the meta data (i.e. audio filepath, transcript and audio duration) of each audio file in the data set. """ - -import os +import argparse import codecs -import soundfile import json -import argparse -from utils.utility import download, unpack +import os + +import soundfile + +from utils.utility import download +from utils.utility import unpack DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech') diff --git a/examples/dataset/chime3_background/chime3_background.py b/examples/dataset/chime3_background/chime3_background.py index 04c18c83e..1f5439aab 100644 --- a/examples/dataset/chime3_background/chime3_background.py +++ b/examples/dataset/chime3_background/chime3_background.py @@ -18,14 +18,14 @@ Manifest file is a json-format file with each line containing the meta data (i.e. audio filepath, transcript and audio duration) of each audio file in the data set. """ - +import argparse +import io +import json import os -import wget import zipfile -import argparse + import soundfile -import json -import io +import wget from paddle.v2.dataset.common import md5file # DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech') diff --git a/examples/dataset/librispeech/librispeech.py b/examples/dataset/librispeech/librispeech.py index 849b6c73e..5374c6680 100644 --- a/examples/dataset/librispeech/librispeech.py +++ b/examples/dataset/librispeech/librispeech.py @@ -18,15 +18,17 @@ Manifest file is a json-format file with each line containing the meta data (i.e. audio filepath, transcript and audio duration) of each audio file in the data set. """ - -import distutils.util -import os import argparse -import soundfile -import json import codecs +import distutils.util import io -from utils.utility import download, unpack +import json +import os + +import soundfile + +from utils.utility import download +from utils.utility import unpack URL_ROOT = "http://www.openslr.org/resources/12" #URL_ROOT = "https://openslr.magicdatatech.com/resources/12" diff --git a/examples/dataset/mini_librispeech/mini_librispeech.py b/examples/dataset/mini_librispeech/mini_librispeech.py index 9b441c14a..3529b13eb 100644 --- a/examples/dataset/mini_librispeech/mini_librispeech.py +++ b/examples/dataset/mini_librispeech/mini_librispeech.py @@ -18,14 +18,16 @@ Manifest file is a json-format file with each line containing the meta data (i.e. audio filepath, transcript and audio duration) of each audio file in the data set. """ - -import os import argparse -import soundfile -import json import codecs import io -from utils.utility import download, unpack +import json +import os + +import soundfile + +from utils.utility import download +from utils.utility import unpack URL_ROOT = "http://www.openslr.org/resources/31" URL_TRAIN_CLEAN = URL_ROOT + "/train-clean-5.tar.gz" diff --git a/examples/dataset/musan/musan.py b/examples/dataset/musan/musan.py index 84322051f..2ac701bed 100644 --- a/examples/dataset/musan/musan.py +++ b/examples/dataset/musan/musan.py @@ -22,12 +22,15 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os +import argparse import codecs -import soundfile import json -import argparse -from utils.utility import download, unpack +import os + +import soundfile + +from utils.utility import download +from utils.utility import unpack DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech') diff --git a/examples/dataset/rir_noise/rir_noise.py b/examples/dataset/rir_noise/rir_noise.py index ca91f57d5..e7b122890 100644 --- a/examples/dataset/rir_noise/rir_noise.py +++ b/examples/dataset/rir_noise/rir_noise.py @@ -22,12 +22,15 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os +import argparse import codecs -import soundfile import json -import argparse -from utils.utility import download, unzip +import os + +import soundfile + +from utils.utility import download +from utils.utility import unzip DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech') diff --git a/examples/dataset/voxforge/voxforge.py b/examples/dataset/voxforge/voxforge.py index 5c84d1eeb..36282bd60 100644 --- a/examples/dataset/voxforge/voxforge.py +++ b/examples/dataset/voxforge/voxforge.py @@ -18,16 +18,19 @@ Manifest file is a json-format file with each line containing the meta data (i.e. audio filepath, transcript and audio duration) of each audio file in the data set. """ - -import os +import argparse import codecs import datetime -import soundfile import json -import argparse +import os import shutil import subprocess -from utils.utility import download_multi, unpack, getfile_insensitive + +import soundfile + +from utils.utility import download_multi +from utils.utility import getfile_insensitive +from utils.utility import unpack DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech') diff --git a/tests/deepspeech2_model_test.py b/tests/deepspeech2_model_test.py index 98c82cd25..1776736f5 100644 --- a/tests/deepspeech2_model_test.py +++ b/tests/deepspeech2_model_test.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import unittest -import paddle import numpy as np -import unittest +import paddle + from deepspeech.models.deepspeech2 import DeepSpeech2Model diff --git a/tests/error_rate_test.py b/tests/error_rate_test.py index 646d5739f..aef25a2ce 100644 --- a/tests/error_rate_test.py +++ b/tests/error_rate_test.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """Test error rate.""" - import unittest + from deepspeech.utils import error_rate diff --git a/tests/mask_test.py b/tests/mask_test.py index afdd53ba8..c4a843e32 100644 --- a/tests/mask_test.py +++ b/tests/mask_test.py @@ -11,13 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import unittest -import paddle import numpy as np -import unittest -from deepspeech.modules.mask import sequence_mask +import paddle + from deepspeech.modules.mask import make_non_pad_mask from deepspeech.modules.mask import make_pad_mask +from deepspeech.modules.mask import sequence_mask class TestU2Model(unittest.TestCase): diff --git a/tests/u2_model_test.py b/tests/u2_model_test.py index a10cdc93b..58600e6ba 100644 --- a/tests/u2_model_test.py +++ b/tests/u2_model_test.py @@ -11,11 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import paddle - import unittest + import numpy as np +import paddle from yacs.config import CfgNode as CN from deepspeech.models.u2 import U2Model diff --git a/utils/build_vocab.py b/utils/build_vocab.py index 805762597..609ce8e7e 100644 --- a/utils/build_vocab.py +++ b/utils/build_vocab.py @@ -14,20 +14,19 @@ """Build vocabulary from manifest files. Each item in vocabulary file is a character. """ - import argparse import functools -from collections import Counter import os import tempfile +from collections import Counter -from deepspeech.frontend.utility import read_manifest -from deepspeech.frontend.utility import UNK +from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer from deepspeech.frontend.utility import BLANK +from deepspeech.frontend.utility import read_manifest from deepspeech.frontend.utility import SOS +from deepspeech.frontend.utility import UNK from deepspeech.utils.utility import add_arguments from deepspeech.utils.utility import print_arguments -from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer parser = argparse.ArgumentParser(description=__doc__) add_arg = functools.partial(add_arguments, argparser=parser) diff --git a/utils/compute_mean_std.py b/utils/compute_mean_std.py index eebbbb923..948e18a68 100644 --- a/utils/compute_mean_std.py +++ b/utils/compute_mean_std.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. """Compute mean and std for feature normalizer, and save to file.""" - import argparse import functools -from deepspeech.frontend.normalizer import FeatureNormalizer + from deepspeech.frontend.augmentor.augmentation import AugmentationPipeline from deepspeech.frontend.featurizer.audio_featurizer import AudioFeaturizer -from deepspeech.utils.utility import add_arguments, print_arguments +from deepspeech.frontend.normalizer import FeatureNormalizer +from deepspeech.utils.utility import add_arguments +from deepspeech.utils.utility import print_arguments parser = argparse.ArgumentParser(description=__doc__) add_arg = functools.partial(add_arguments, argparser=parser) diff --git a/utils/format_data.py b/utils/format_data.py index 356a9c7d1..235cc8cbe 100644 --- a/utils/format_data.py +++ b/utils/format_data.py @@ -16,11 +16,11 @@ import argparse import functools import json -from deepspeech.frontend.utility import read_manifest +from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer from deepspeech.frontend.utility import load_cmvn +from deepspeech.frontend.utility import read_manifest from deepspeech.utils.utility import add_arguments from deepspeech.utils.utility import print_arguments -from deepspeech.frontend.featurizer.text_featurizer import TextFeaturizer parser = argparse.ArgumentParser(description=__doc__) add_arg = functools.partial(add_arguments, argparser=parser) diff --git a/utils/spm_decode b/utils/spm_decode index a94aa2a8d..8497ad811 100755 --- a/utils/spm_decode +++ b/utils/spm_decode @@ -4,8 +4,10 @@ # # This source code is licensed under the license found in the # https://github.com/pytorch/fairseq/blob/master/LICENSE - -from __future__ import absolute_import, division, print_function, unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import argparse import sys @@ -46,4 +48,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/utils/spm_encode b/utils/spm_encode index 081e40e96..99357045a 100755 --- a/utils/spm_encode +++ b/utils/spm_encode @@ -4,8 +4,10 @@ # # This source code is licensed under the license found in # https://github.com/pytorch/fairseq/blob/master/LICENSE - -from __future__ import absolute_import, division, print_function, unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import argparse import contextlib @@ -96,4 +98,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/utils/spm_train b/utils/spm_train index 44330a55f..0b247aee0 100755 --- a/utils/spm_train +++ b/utils/spm_train @@ -10,4 +10,4 @@ import sentencepiece as spm if __name__ == "__main__": - spm.SentencePieceTrainer.Train(" ".join(sys.argv[1:])) \ No newline at end of file + spm.SentencePieceTrainer.Train(" ".join(sys.argv[1:])) diff --git a/utils/utility.py b/utils/utility.py index aaedfa956..0333bc559 100644 --- a/utils/utility.py +++ b/utils/utility.py @@ -11,10 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import os import tarfile import zipfile + from paddle.dataset.common import md5file