From b11608c232da750ad5eecb19e7f2f20e14864886 Mon Sep 17 00:00:00 2001 From: huangyuxin Date: Wed, 21 Jul 2021 08:44:12 +0000 Subject: [PATCH 01/15] added autoLog, but gpu_util is always 0.0% --- deepspeech/exps/deepspeech2/model.py | 34 +++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/deepspeech/exps/deepspeech2/model.py b/deepspeech/exps/deepspeech2/model.py index 209e8b023..3810694a1 100644 --- a/deepspeech/exps/deepspeech2/model.py +++ b/deepspeech/exps/deepspeech2/model.py @@ -37,6 +37,11 @@ from deepspeech.utils import mp_tools from deepspeech.utils.log import Log logger = Log(__name__).getlog() +import auto_log +import os +from paddle import inference + + class DeepSpeech2Trainer(Trainer): @classmethod @@ -223,6 +228,27 @@ class DeepSpeech2Tester(DeepSpeech2Trainer): def __init__(self, config, args): super().__init__(config, args) + # added by hyx + pid = os.getpid() + infer_config = inference.Config() + infer_config.enable_use_gpu(10000, 2) + logger = None + autolog = auto_log.AutoLogger( + model_name="tiny_s0", + model_precision="fp32", + batch_size=config.decoding.batch_size, + data_shape="dynamic", + save_path="./output/auto_log.lpg", + inference_config=infer_config, + pids=pid, + process_name=None, + gpu_ids=2, + time_keys=[ + 'preprocess_time', 'inference_time', 'postprocess_time' + ], + warmup=0) + self.autolog = autolog + logger = autolog.logger def ordid2token(self, texts, texts_len): """ ord() id to chr() chr """ @@ -248,6 +274,8 @@ class DeepSpeech2Tester(DeepSpeech2Trainer): vocab_list = self.test_loader.collate_fn.vocab_list target_transcripts = self.ordid2token(texts, texts_len) + self.autolog.times.start() + self.autolog.times.stamp() result_transcripts = self.model.decode( audio, audio_len, @@ -260,7 +288,9 @@ class DeepSpeech2Tester(DeepSpeech2Trainer): cutoff_prob=cfg.cutoff_prob, cutoff_top_n=cfg.cutoff_top_n, num_processes=cfg.num_proc_bsearch) - + self.autolog.times.stamp() + self.autolog.times.stamp() + self.autolog.times.end() for utt, target, result in zip(utts, target_transcripts, result_transcripts): errors, len_ref = errors_func(target, result) @@ -291,6 +321,7 @@ class DeepSpeech2Tester(DeepSpeech2Trainer): errors_sum, len_refs, num_ins = 0.0, 0, 0 with open(self.args.result_file, 'w') as fout: for i, batch in enumerate(self.test_loader): + logger.info("batch: {}".format(i)) utts, audio, audio_len, texts, texts_len = batch metrics = self.compute_metrics(utts, audio, audio_len, texts, texts_len, fout) @@ -308,6 +339,7 @@ class DeepSpeech2Tester(DeepSpeech2Trainer): msg += "Final error rate [%s] (%d/%d) = %f" % ( error_rate_type, num_ins, num_ins, errors_sum / len_refs) logger.info(msg) + self.autolog.report() def run_test(self): self.resume_or_scratch() From 85a931a359c11cfbaeaa0e12d6ad0ac17877be1f Mon Sep 17 00:00:00 2001 From: huangyuxin Date: Thu, 22 Jul 2021 02:56:52 +0000 Subject: [PATCH 02/15] added autolog --- deepspeech/exps/deepspeech2/model.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/deepspeech/exps/deepspeech2/model.py b/deepspeech/exps/deepspeech2/model.py index 3810694a1..54ad449cd 100644 --- a/deepspeech/exps/deepspeech2/model.py +++ b/deepspeech/exps/deepspeech2/model.py @@ -35,12 +35,12 @@ from deepspeech.utils import error_rate from deepspeech.utils import layer_tools from deepspeech.utils import mp_tools from deepspeech.utils.log import Log -logger = Log(__name__).getlog() import auto_log import os from paddle import inference +logger = Log(__name__).getlog() class DeepSpeech2Trainer(Trainer): @@ -228,13 +228,12 @@ class DeepSpeech2Tester(DeepSpeech2Trainer): def __init__(self, config, args): super().__init__(config, args) - # added by hyx pid = os.getpid() + gpu_id = int(os.environ['CUDA_VISIBLE_DEVICES'].split(',')[0]) infer_config = inference.Config() - infer_config.enable_use_gpu(10000, 2) - logger = None + infer_config.enable_use_gpu(100, gpu_id) autolog = auto_log.AutoLogger( - model_name="tiny_s0", + model_name="deepspeech2", model_precision="fp32", batch_size=config.decoding.batch_size, data_shape="dynamic", @@ -242,13 +241,14 @@ class DeepSpeech2Tester(DeepSpeech2Trainer): inference_config=infer_config, pids=pid, process_name=None, - gpu_ids=2, + gpu_ids=gpu_id, time_keys=[ 'preprocess_time', 'inference_time', 'postprocess_time' ], warmup=0) self.autolog = autolog logger = autolog.logger + logger.info("gpu_id:{}".format(gpu_id)) def ordid2token(self, texts, texts_len): """ ord() id to chr() chr """ @@ -291,6 +291,7 @@ class DeepSpeech2Tester(DeepSpeech2Trainer): self.autolog.times.stamp() self.autolog.times.stamp() self.autolog.times.end() + for utt, target, result in zip(utts, target_transcripts, result_transcripts): errors, len_ref = errors_func(target, result) From 4ffec0e7d2ab26316405396648755f136b3ff70a Mon Sep 17 00:00:00 2001 From: huangyuxin Date: Thu, 22 Jul 2021 02:59:45 +0000 Subject: [PATCH 03/15] add autolog --- examples/aishell/s0/run.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/aishell/s0/run.sh b/examples/aishell/s0/run.sh index c9708dcc9..6f4c23bd4 100755 --- a/examples/aishell/s0/run.sh +++ b/examples/aishell/s0/run.sh @@ -2,9 +2,9 @@ set -e source path.sh -gpus=0,1,2,3 -stage=0 -stop_stage=100 +gpus=2 +stage=3 +stop_stage=3 conf_path=conf/deepspeech2.yaml avg_num=1 @@ -31,10 +31,10 @@ fi if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then # test ckpt avg_n - CUDA_VISIBLE_DEVICES=0 ./local/test.sh ${conf_path} exp/${ckpt}/checkpoints/${avg_ckpt} || exit -1 + CUDA_VISIBLE_DEVICES=${gpus} ./local/test.sh ${conf_path} exp/${ckpt}/checkpoints/${avg_ckpt} || exit -1 fi if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then # export ckpt avg_n - CUDA_VISIBLE_DEVICES=0 ./local/export.sh ${conf_path} exp/${ckpt}/checkpoints/${avg_ckpt} exp/${ckpt}/checkpoints/${avg_ckpt}.jit + CUDA_VISIBLE_DEVICES=${gpus} ./local/export.sh ${conf_path} exp/${ckpt}/checkpoints/${avg_ckpt} exp/${ckpt}/checkpoints/${avg_ckpt}.jit fi From a1d497db044139b83a7064c23568c03da9808e46 Mon Sep 17 00:00:00 2001 From: huangyuxin Date: Thu, 22 Jul 2021 03:12:07 +0000 Subject: [PATCH 04/15] add autoLog --- examples/aishell/s0/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/aishell/s0/run.sh b/examples/aishell/s0/run.sh index 6f4c23bd4..0ddf5fb5d 100755 --- a/examples/aishell/s0/run.sh +++ b/examples/aishell/s0/run.sh @@ -3,8 +3,8 @@ set -e source path.sh gpus=2 -stage=3 -stop_stage=3 +stage=0 +stop_stage=4 conf_path=conf/deepspeech2.yaml avg_num=1 From 521184bd32ca1fe5712ea1b911136776d44d5518 Mon Sep 17 00:00:00 2001 From: Jackwaterveg Date: Thu, 22 Jul 2021 12:19:58 +0000 Subject: [PATCH 05/15] added autoLog --- examples/tiny/s0/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tiny/s0/run.sh b/examples/tiny/s0/run.sh index d7e153e8d..9ecea1b36 100755 --- a/examples/tiny/s0/run.sh +++ b/examples/tiny/s0/run.sh @@ -2,7 +2,7 @@ set -e source path.sh -gpus=0 +gpus=2 stage=0 stop_stage=100 conf_path=conf/deepspeech2.yaml From 054dcd2b6d7bb654b3103add4717dafb90b915ff Mon Sep 17 00:00:00 2001 From: Jackwaterveg Date: Thu, 22 Jul 2021 12:58:43 +0000 Subject: [PATCH 06/15] change autoLog --- deepspeech/exps/deepspeech2/model.py | 28 +++--------------------- deepspeech/utils/log.py | 32 ++++++++++++++++++++++++++++ examples/aishell/s0/run.sh | 8 +++---- 3 files changed, 39 insertions(+), 29 deletions(-) diff --git a/deepspeech/exps/deepspeech2/model.py b/deepspeech/exps/deepspeech2/model.py index 54ad449cd..8a611291f 100644 --- a/deepspeech/exps/deepspeech2/model.py +++ b/deepspeech/exps/deepspeech2/model.py @@ -35,10 +35,8 @@ from deepspeech.utils import error_rate from deepspeech.utils import layer_tools from deepspeech.utils import mp_tools from deepspeech.utils.log import Log +from deepspeech.utils.log import Autolog -import auto_log -import os -from paddle import inference logger = Log(__name__).getlog() @@ -228,28 +226,8 @@ class DeepSpeech2Tester(DeepSpeech2Trainer): def __init__(self, config, args): super().__init__(config, args) - pid = os.getpid() - gpu_id = int(os.environ['CUDA_VISIBLE_DEVICES'].split(',')[0]) - infer_config = inference.Config() - infer_config.enable_use_gpu(100, gpu_id) - autolog = auto_log.AutoLogger( - model_name="deepspeech2", - model_precision="fp32", - batch_size=config.decoding.batch_size, - data_shape="dynamic", - save_path="./output/auto_log.lpg", - inference_config=infer_config, - pids=pid, - process_name=None, - gpu_ids=gpu_id, - time_keys=[ - 'preprocess_time', 'inference_time', 'postprocess_time' - ], - warmup=0) - self.autolog = autolog - logger = autolog.logger - logger.info("gpu_id:{}".format(gpu_id)) - + self.autolog = Autolog(batch_size = config.decoding.batch_size, model_name = "deepspeech2", model_precision = "fp32").getlog() + def ordid2token(self, texts, texts_len): """ ord() id to chr() chr """ trans = [] diff --git a/deepspeech/utils/log.py b/deepspeech/utils/log.py index 499b1872f..d5522d8d1 100644 --- a/deepspeech/utils/log.py +++ b/deepspeech/utils/log.py @@ -17,6 +17,12 @@ import os import socket import sys +import auto_log +import os +from paddle import inference + + + FORMAT_STR = '[%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s' DATE_FMT_STR = '%Y/%m/%d %H:%M:%S' @@ -146,3 +152,29 @@ class Log(): def getlog(self): return self.logger + +class Autolog: + + def __init__(self, batch_size, model_name = "DeepSpeech", model_precision = "fp32"): + pid = os.getpid() + gpu_id = int(os.environ['CUDA_VISIBLE_DEVICES'].split(',')[0]) + infer_config = inference.Config() + infer_config.enable_use_gpu(100, gpu_id) + autolog = auto_log.AutoLogger( + model_name = model_name, + model_precision = model_precision, + batch_size = batch_size, + data_shape="dynamic", + save_path="./output/auto_log.lpg", + inference_config = infer_config, + pids = pid, + process_name = None, + gpu_ids = gpu_id, + time_keys=[ + 'preprocess_time', 'inference_time', 'postprocess_time' + ], + warmup=0) + self.autolog = autolog + + def getlog(self): + return self.autolog diff --git a/examples/aishell/s0/run.sh b/examples/aishell/s0/run.sh index 0ddf5fb5d..c9708dcc9 100755 --- a/examples/aishell/s0/run.sh +++ b/examples/aishell/s0/run.sh @@ -2,9 +2,9 @@ set -e source path.sh -gpus=2 +gpus=0,1,2,3 stage=0 -stop_stage=4 +stop_stage=100 conf_path=conf/deepspeech2.yaml avg_num=1 @@ -31,10 +31,10 @@ fi if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then # test ckpt avg_n - CUDA_VISIBLE_DEVICES=${gpus} ./local/test.sh ${conf_path} exp/${ckpt}/checkpoints/${avg_ckpt} || exit -1 + CUDA_VISIBLE_DEVICES=0 ./local/test.sh ${conf_path} exp/${ckpt}/checkpoints/${avg_ckpt} || exit -1 fi if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then # export ckpt avg_n - CUDA_VISIBLE_DEVICES=${gpus} ./local/export.sh ${conf_path} exp/${ckpt}/checkpoints/${avg_ckpt} exp/${ckpt}/checkpoints/${avg_ckpt}.jit + CUDA_VISIBLE_DEVICES=0 ./local/export.sh ${conf_path} exp/${ckpt}/checkpoints/${avg_ckpt} exp/${ckpt}/checkpoints/${avg_ckpt}.jit fi From 39cb7b6d7a167f89708d0a99c49e968162a0c1cb Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 22 Jul 2021 21:09:53 +0800 Subject: [PATCH 07/15] Update install.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 增加了AutoLog的安装 --- doc/src/install.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/src/install.md b/doc/src/install.md index 01049a2fc..7998f90ad 100644 --- a/doc/src/install.md +++ b/doc/src/install.md @@ -42,6 +42,13 @@ bash setup.sh ```bash source tools/venv/bin/activate ``` +## install AutoLog +```bash +git clone https://github.com/LDOUBLEV/AutoLog +pip3 install -r requirements.txt +python3 setup.py bdist_wheel +pip3 install ./dist/AutoLog-1.0.0-py3-none-any.whl +```bash ## Running in Docker Container (optional) From 3158bfa3c2a49b3e578eb544f542aa13c9b8a99e Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 22 Jul 2021 21:10:26 +0800 Subject: [PATCH 08/15] Update install.md --- doc/src/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/install.md b/doc/src/install.md index 7998f90ad..f2c4ff289 100644 --- a/doc/src/install.md +++ b/doc/src/install.md @@ -48,7 +48,7 @@ git clone https://github.com/LDOUBLEV/AutoLog pip3 install -r requirements.txt python3 setup.py bdist_wheel pip3 install ./dist/AutoLog-1.0.0-py3-none-any.whl -```bash +``` ## Running in Docker Container (optional) From 5fff669767020d46a3080041aa52b2bbc4de8430 Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 22 Jul 2021 21:10:47 +0800 Subject: [PATCH 09/15] Update install.md --- doc/src/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/install.md b/doc/src/install.md index f2c4ff289..5a4c879d4 100644 --- a/doc/src/install.md +++ b/doc/src/install.md @@ -42,7 +42,7 @@ bash setup.sh ```bash source tools/venv/bin/activate ``` -## install AutoLog +## Install AutoLog ```bash git clone https://github.com/LDOUBLEV/AutoLog pip3 install -r requirements.txt From 8dd72ccc1c4733d46ae58da88f680afab9fd7e55 Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 22 Jul 2021 21:13:14 +0800 Subject: [PATCH 10/15] Update install.md --- doc/src/install.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/src/install.md b/doc/src/install.md index 5a4c879d4..ca744b303 100644 --- a/doc/src/install.md +++ b/doc/src/install.md @@ -43,6 +43,7 @@ bash setup.sh source tools/venv/bin/activate ``` ## Install AutoLog +Installing AutoLog via `git`: ```bash git clone https://github.com/LDOUBLEV/AutoLog pip3 install -r requirements.txt From 72b9e5c46c5a3413f3741fc910cb03947018af20 Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 22 Jul 2021 21:14:14 +0800 Subject: [PATCH 11/15] Update install.md --- doc/src/install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/install.md b/doc/src/install.md index ca744b303..35373abc7 100644 --- a/doc/src/install.md +++ b/doc/src/install.md @@ -42,8 +42,8 @@ bash setup.sh ```bash source tools/venv/bin/activate ``` -## Install AutoLog -Installing AutoLog via `git`: + +- Installing AutoLog via `git`: ```bash git clone https://github.com/LDOUBLEV/AutoLog pip3 install -r requirements.txt From eba280d5dfac19649915316c56164b43428a98c6 Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 22 Jul 2021 21:15:06 +0800 Subject: [PATCH 12/15] Update install.md --- doc/src/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/install.md b/doc/src/install.md index 35373abc7..fd42224db 100644 --- a/doc/src/install.md +++ b/doc/src/install.md @@ -43,7 +43,7 @@ bash setup.sh source tools/venv/bin/activate ``` -- Installing AutoLog via `git`: +- Besides, installing AutoLog via `git`: ```bash git clone https://github.com/LDOUBLEV/AutoLog pip3 install -r requirements.txt From d965f6b4c19b14f9be9b86cde054b8421835906b Mon Sep 17 00:00:00 2001 From: Jackwaterveg Date: Thu, 22 Jul 2021 13:19:05 +0000 Subject: [PATCH 13/15] autoLog --- examples/tiny/s0/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tiny/s0/run.sh b/examples/tiny/s0/run.sh index 9ecea1b36..d7e153e8d 100755 --- a/examples/tiny/s0/run.sh +++ b/examples/tiny/s0/run.sh @@ -2,7 +2,7 @@ set -e source path.sh -gpus=2 +gpus=0 stage=0 stop_stage=100 conf_path=conf/deepspeech2.yaml From d17e93ddd369823d289b6f64983b48197c32060c Mon Sep 17 00:00:00 2001 From: Jackwaterveg Date: Thu, 22 Jul 2021 13:21:16 +0000 Subject: [PATCH 14/15] autoLog --- deepspeech/exps/deepspeech2/model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deepspeech/exps/deepspeech2/model.py b/deepspeech/exps/deepspeech2/model.py index 8a611291f..5e659092a 100644 --- a/deepspeech/exps/deepspeech2/model.py +++ b/deepspeech/exps/deepspeech2/model.py @@ -300,7 +300,6 @@ class DeepSpeech2Tester(DeepSpeech2Trainer): errors_sum, len_refs, num_ins = 0.0, 0, 0 with open(self.args.result_file, 'w') as fout: for i, batch in enumerate(self.test_loader): - logger.info("batch: {}".format(i)) utts, audio, audio_len, texts, texts_len = batch metrics = self.compute_metrics(utts, audio, audio_len, texts, texts_len, fout) From dee578bb4495923ec627cdb07259fed82fc972de Mon Sep 17 00:00:00 2001 From: Jackwaterveg Date: Fri, 23 Jul 2021 09:36:17 +0000 Subject: [PATCH 15/15] added the way of installing auto_log into setup.sh --- doc/src/install.md | 8 -------- setup.sh | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/doc/src/install.md b/doc/src/install.md index fd42224db..01049a2fc 100644 --- a/doc/src/install.md +++ b/doc/src/install.md @@ -43,14 +43,6 @@ bash setup.sh source tools/venv/bin/activate ``` -- Besides, installing AutoLog via `git`: -```bash -git clone https://github.com/LDOUBLEV/AutoLog -pip3 install -r requirements.txt -python3 setup.py bdist_wheel -pip3 install ./dist/AutoLog-1.0.0-py3-none-any.whl -``` - ## Running in Docker Container (optional) Docker is an open source tool to build, ship, and run distributed applications in an isolated environment. A Docker image for this project has been provided in [hub.docker.com](https://hub.docker.com) with all the dependencies installed. This Docker image requires the support of NVIDIA GPU, so please make sure its availiability and the [nvidia-docker](https://github.com/NVIDIA/nvidia-docker) has been installed. diff --git a/setup.sh b/setup.sh index 11daa102a..0c602e618 100644 --- a/setup.sh +++ b/setup.sh @@ -43,6 +43,23 @@ if [ $? != 0 ]; then rm libsndfile-1.0.28.tar.gz fi +#install auto-log +python3 -c "import auto_log" +if [ $? != 0 ]; then + info_msg "Install auto_log into default system path" + git clone https://github.com/LDOUBLEV/AutoLog + if [ $? != 0 ]; then + error_msg "Download auto_log failed !!!" + exit 1 + fi + cd AutoLog + pip3 install -r requirements.txt + python3 setup.py bdist_wheel + pip3 install ./dist/[Aa]uto*.whl + cd .. + rm -rf AutoLog +fi + # install decoders python3 -c "import pkg_resources; pkg_resources.require(\"swig_decoders==1.1\")" if [ $? != 0 ]; then @@ -66,4 +83,5 @@ if [ $? != 0 ]; then fi popd + info_msg "Install all dependencies successfully."