From 63d78c88f513c5c9f2385c5c660d3fb570b51ea1 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Tue, 30 Mar 2021 09:40:49 +0000 Subject: [PATCH] refactor egs --- examples/aishell/README.md | 8 +-- examples/aishell/s0/README.md | 7 +++ .../aishell/{ => s0}/conf/augmentation.config | 0 .../aishell/{ => s0}/conf/deepspeech2.yaml | 0 examples/aishell/{ => s0}/local/client.sh | 0 examples/aishell/{ => s0}/local/data.sh | 0 .../aishell/{ => s0}/local/download_lm_ch.sh | 0 .../aishell/{ => s0}/local/download_model.sh | 0 examples/aishell/{ => s0}/local/export.sh | 0 examples/aishell/{ => s0}/local/infer.sh | 0 examples/aishell/{ => s0}/local/server.sh | 0 examples/aishell/{ => s0}/local/test.sh | 0 examples/aishell/{ => s0}/local/train.sh | 0 examples/aishell/{ => s0}/local/tune.sh | 0 examples/aishell/{ => s0}/path.sh | 2 +- examples/aishell/{ => s0}/run.sh | 0 examples/librispeech/README.md | 8 +-- examples/librispeech/s0/README.md | 7 +++ .../{ => s0}/conf/augmentation.config | 0 .../{ => s0}/conf/deepspeech2.yaml | 0 examples/librispeech/{ => s0}/local/data.sh | 0 .../{ => s0}/local/download_lm_en.sh | 0 .../{ => s0}/local/download_model.sh | 0 examples/librispeech/{ => s0}/local/export.sh | 0 examples/librispeech/{ => s0}/local/infer.sh | 0 examples/librispeech/{ => s0}/local/test.sh | 0 examples/librispeech/{ => s0}/local/train.sh | 0 examples/librispeech/{ => s0}/local/tune.sh | 0 examples/librispeech/{ => s0}/path.sh | 2 +- examples/librispeech/{ => s0}/run.sh | 0 examples/tiny/README.md | 52 +------------------ examples/tiny/s0/README.md | 51 ++++++++++++++++++ .../tiny/{ => s0}/conf/augmentation.config | 0 examples/tiny/{ => s0}/conf/deepspeech2.yaml | 0 examples/tiny/{ => s0}/local/data.sh | 0 .../tiny/{ => s0}/local/download_lm_en.sh | 0 .../tiny/{ => s0}/local/download_model.sh | 0 examples/tiny/{ => s0}/local/export.sh | 0 examples/tiny/{ => s0}/local/infer.sh | 0 examples/tiny/{ => s0}/local/test.sh | 0 examples/tiny/{ => s0}/local/train.sh | 0 examples/tiny/{ => s0}/local/tune.sh | 0 examples/tiny/{ => s0}/path.sh | 2 +- examples/tiny/{ => s0}/run.sh | 0 44 files changed, 71 insertions(+), 68 deletions(-) create mode 100644 examples/aishell/s0/README.md rename examples/aishell/{ => s0}/conf/augmentation.config (100%) rename examples/aishell/{ => s0}/conf/deepspeech2.yaml (100%) rename examples/aishell/{ => s0}/local/client.sh (100%) rename examples/aishell/{ => s0}/local/data.sh (100%) rename examples/aishell/{ => s0}/local/download_lm_ch.sh (100%) rename examples/aishell/{ => s0}/local/download_model.sh (100%) rename examples/aishell/{ => s0}/local/export.sh (100%) rename examples/aishell/{ => s0}/local/infer.sh (100%) rename examples/aishell/{ => s0}/local/server.sh (100%) rename examples/aishell/{ => s0}/local/test.sh (100%) rename examples/aishell/{ => s0}/local/train.sh (100%) rename examples/aishell/{ => s0}/local/tune.sh (100%) rename examples/aishell/{ => s0}/path.sh (90%) rename examples/aishell/{ => s0}/run.sh (100%) create mode 100644 examples/librispeech/s0/README.md rename examples/librispeech/{ => s0}/conf/augmentation.config (100%) rename examples/librispeech/{ => s0}/conf/deepspeech2.yaml (100%) rename examples/librispeech/{ => s0}/local/data.sh (100%) rename examples/librispeech/{ => s0}/local/download_lm_en.sh (100%) rename examples/librispeech/{ => s0}/local/download_model.sh (100%) rename examples/librispeech/{ => s0}/local/export.sh (100%) rename examples/librispeech/{ => s0}/local/infer.sh (100%) rename examples/librispeech/{ => s0}/local/test.sh (100%) rename examples/librispeech/{ => s0}/local/train.sh (100%) rename examples/librispeech/{ => s0}/local/tune.sh (100%) rename examples/librispeech/{ => s0}/path.sh (90%) rename examples/librispeech/{ => s0}/run.sh (100%) create mode 100644 examples/tiny/s0/README.md rename examples/tiny/{ => s0}/conf/augmentation.config (100%) rename examples/tiny/{ => s0}/conf/deepspeech2.yaml (100%) rename examples/tiny/{ => s0}/local/data.sh (100%) rename examples/tiny/{ => s0}/local/download_lm_en.sh (100%) rename examples/tiny/{ => s0}/local/download_model.sh (100%) rename examples/tiny/{ => s0}/local/export.sh (100%) rename examples/tiny/{ => s0}/local/infer.sh (100%) rename examples/tiny/{ => s0}/local/test.sh (100%) rename examples/tiny/{ => s0}/local/train.sh (100%) rename examples/tiny/{ => s0}/local/tune.sh (100%) rename examples/tiny/{ => s0}/path.sh (90%) rename examples/tiny/{ => s0}/run.sh (100%) diff --git a/examples/aishell/README.md b/examples/aishell/README.md index 6d67d19a9..e109e1ae4 100644 --- a/examples/aishell/README.md +++ b/examples/aishell/README.md @@ -1,7 +1 @@ -# Aishell-1 - -## CTC -| Model | Config | Test set | CER | -| --- | --- | --- | --- | -| DeepSpeech2 | conf/deepspeech2.yaml | test | 0.078977 | -| DeepSpeech2 | release 1.8.5 | test | 0.080447 | +* s0 for deepspeech2 diff --git a/examples/aishell/s0/README.md b/examples/aishell/s0/README.md new file mode 100644 index 000000000..6d67d19a9 --- /dev/null +++ b/examples/aishell/s0/README.md @@ -0,0 +1,7 @@ +# Aishell-1 + +## CTC +| Model | Config | Test set | CER | +| --- | --- | --- | --- | +| DeepSpeech2 | conf/deepspeech2.yaml | test | 0.078977 | +| DeepSpeech2 | release 1.8.5 | test | 0.080447 | diff --git a/examples/aishell/conf/augmentation.config b/examples/aishell/s0/conf/augmentation.config similarity index 100% rename from examples/aishell/conf/augmentation.config rename to examples/aishell/s0/conf/augmentation.config diff --git a/examples/aishell/conf/deepspeech2.yaml b/examples/aishell/s0/conf/deepspeech2.yaml similarity index 100% rename from examples/aishell/conf/deepspeech2.yaml rename to examples/aishell/s0/conf/deepspeech2.yaml diff --git a/examples/aishell/local/client.sh b/examples/aishell/s0/local/client.sh similarity index 100% rename from examples/aishell/local/client.sh rename to examples/aishell/s0/local/client.sh diff --git a/examples/aishell/local/data.sh b/examples/aishell/s0/local/data.sh similarity index 100% rename from examples/aishell/local/data.sh rename to examples/aishell/s0/local/data.sh diff --git a/examples/aishell/local/download_lm_ch.sh b/examples/aishell/s0/local/download_lm_ch.sh similarity index 100% rename from examples/aishell/local/download_lm_ch.sh rename to examples/aishell/s0/local/download_lm_ch.sh diff --git a/examples/aishell/local/download_model.sh b/examples/aishell/s0/local/download_model.sh similarity index 100% rename from examples/aishell/local/download_model.sh rename to examples/aishell/s0/local/download_model.sh diff --git a/examples/aishell/local/export.sh b/examples/aishell/s0/local/export.sh similarity index 100% rename from examples/aishell/local/export.sh rename to examples/aishell/s0/local/export.sh diff --git a/examples/aishell/local/infer.sh b/examples/aishell/s0/local/infer.sh similarity index 100% rename from examples/aishell/local/infer.sh rename to examples/aishell/s0/local/infer.sh diff --git a/examples/aishell/local/server.sh b/examples/aishell/s0/local/server.sh similarity index 100% rename from examples/aishell/local/server.sh rename to examples/aishell/s0/local/server.sh diff --git a/examples/aishell/local/test.sh b/examples/aishell/s0/local/test.sh similarity index 100% rename from examples/aishell/local/test.sh rename to examples/aishell/s0/local/test.sh diff --git a/examples/aishell/local/train.sh b/examples/aishell/s0/local/train.sh similarity index 100% rename from examples/aishell/local/train.sh rename to examples/aishell/s0/local/train.sh diff --git a/examples/aishell/local/tune.sh b/examples/aishell/s0/local/tune.sh similarity index 100% rename from examples/aishell/local/tune.sh rename to examples/aishell/s0/local/tune.sh diff --git a/examples/aishell/path.sh b/examples/aishell/s0/path.sh similarity index 90% rename from examples/aishell/path.sh rename to examples/aishell/s0/path.sh index debdbba46..f67893266 100644 --- a/examples/aishell/path.sh +++ b/examples/aishell/s0/path.sh @@ -1,4 +1,4 @@ -export MAIN_ROOT=${PWD}/../../ +export MAIN_ROOT=${PWD}/../../../ export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} export LC_ALL=C diff --git a/examples/aishell/run.sh b/examples/aishell/s0/run.sh similarity index 100% rename from examples/aishell/run.sh rename to examples/aishell/s0/run.sh diff --git a/examples/librispeech/README.md b/examples/librispeech/README.md index 1e694df1c..e109e1ae4 100644 --- a/examples/librispeech/README.md +++ b/examples/librispeech/README.md @@ -1,7 +1 @@ -# LibriSpeech - -## CTC -| Model | Config | Test set | WER | -| --- | --- | --- | --- | -| DeepSpeech2 | conf/deepspeech2.yaml | test-clean | 0.073973 | -| DeepSpeech2 | release 1.8.5 | test-clean | 0.074939 | +* s0 for deepspeech2 diff --git a/examples/librispeech/s0/README.md b/examples/librispeech/s0/README.md new file mode 100644 index 000000000..1e694df1c --- /dev/null +++ b/examples/librispeech/s0/README.md @@ -0,0 +1,7 @@ +# LibriSpeech + +## CTC +| Model | Config | Test set | WER | +| --- | --- | --- | --- | +| DeepSpeech2 | conf/deepspeech2.yaml | test-clean | 0.073973 | +| DeepSpeech2 | release 1.8.5 | test-clean | 0.074939 | diff --git a/examples/librispeech/conf/augmentation.config b/examples/librispeech/s0/conf/augmentation.config similarity index 100% rename from examples/librispeech/conf/augmentation.config rename to examples/librispeech/s0/conf/augmentation.config diff --git a/examples/librispeech/conf/deepspeech2.yaml b/examples/librispeech/s0/conf/deepspeech2.yaml similarity index 100% rename from examples/librispeech/conf/deepspeech2.yaml rename to examples/librispeech/s0/conf/deepspeech2.yaml diff --git a/examples/librispeech/local/data.sh b/examples/librispeech/s0/local/data.sh similarity index 100% rename from examples/librispeech/local/data.sh rename to examples/librispeech/s0/local/data.sh diff --git a/examples/librispeech/local/download_lm_en.sh b/examples/librispeech/s0/local/download_lm_en.sh similarity index 100% rename from examples/librispeech/local/download_lm_en.sh rename to examples/librispeech/s0/local/download_lm_en.sh diff --git a/examples/librispeech/local/download_model.sh b/examples/librispeech/s0/local/download_model.sh similarity index 100% rename from examples/librispeech/local/download_model.sh rename to examples/librispeech/s0/local/download_model.sh diff --git a/examples/librispeech/local/export.sh b/examples/librispeech/s0/local/export.sh similarity index 100% rename from examples/librispeech/local/export.sh rename to examples/librispeech/s0/local/export.sh diff --git a/examples/librispeech/local/infer.sh b/examples/librispeech/s0/local/infer.sh similarity index 100% rename from examples/librispeech/local/infer.sh rename to examples/librispeech/s0/local/infer.sh diff --git a/examples/librispeech/local/test.sh b/examples/librispeech/s0/local/test.sh similarity index 100% rename from examples/librispeech/local/test.sh rename to examples/librispeech/s0/local/test.sh diff --git a/examples/librispeech/local/train.sh b/examples/librispeech/s0/local/train.sh similarity index 100% rename from examples/librispeech/local/train.sh rename to examples/librispeech/s0/local/train.sh diff --git a/examples/librispeech/local/tune.sh b/examples/librispeech/s0/local/tune.sh similarity index 100% rename from examples/librispeech/local/tune.sh rename to examples/librispeech/s0/local/tune.sh diff --git a/examples/librispeech/path.sh b/examples/librispeech/s0/path.sh similarity index 90% rename from examples/librispeech/path.sh rename to examples/librispeech/s0/path.sh index a179631b3..44b178383 100644 --- a/examples/librispeech/path.sh +++ b/examples/librispeech/s0/path.sh @@ -1,4 +1,4 @@ -export MAIN_ROOT=${PWD}/../../ +export MAIN_ROOT=${PWD}/../../../ export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} export LC_ALL=C diff --git a/examples/librispeech/run.sh b/examples/librispeech/s0/run.sh similarity index 100% rename from examples/librispeech/run.sh rename to examples/librispeech/s0/run.sh diff --git a/examples/tiny/README.md b/examples/tiny/README.md index 0f96864c0..e109e1ae4 100644 --- a/examples/tiny/README.md +++ b/examples/tiny/README.md @@ -1,51 +1 @@ -# Tiny Example - -1. `source path.sh` -3. set `CUDA_VISIBLE_DEVICES` as you need. -2. demo scrpt is `bash run.sh`. You can run commond separately as needed. - -## Steps -- Prepare the data - - ```bash - bash local/data.sh - ``` - - `data.sh` will download dataset, generate manifests, collect normalizer's statistics and build vocabulary. Once the data preparation is done, you will find the data (only part of LibriSpeech) downloaded in `${MAIN_ROOT}/dataset/librispeech` and the corresponding manifest files generated in `${PWD}/data` as well as a mean stddev file and a vocabulary file. It has to be run for the very first time you run this dataset and is reusable for all further experiments. - -- Train your own ASR model - - ```bash - bash local/train.sh - ``` - - `train.sh` will start a training job, with training logs printed to stdout and model checkpoint of every pass/epoch saved to `${PWD}/checkpoints`. These checkpoints could be used for training resuming, inference, evaluation and deployment. - -- Case inference with an existing model - - ```bash - bash local/infer.sh - ``` - - `infer.sh` will show us some speech-to-text decoding results for several (default: 10) samples with the trained model. The performance might not be good now as the current model is only trained with a toy subset of LibriSpeech. To see the results with a better model, you can download a well-trained (trained for several days, with the complete LibriSpeech) model and do the inference. - -- Evaluate an existing model - - ```bash - bash local/test.sh - ``` - - `test.sh` will evaluate the model with Word Error Rate (or Character Error Rate) measurement. Similarly, you can also download a well-trained model and test its performance: - - -- Export jit model - - ```bash - bash local/export.sh ckpt_path saved_jit_model_path - ``` - -- Tune hyper paerameter - - ```bash - bash local/tune.sh - ``` +* s0 for deepspeech2 diff --git a/examples/tiny/s0/README.md b/examples/tiny/s0/README.md new file mode 100644 index 000000000..0f96864c0 --- /dev/null +++ b/examples/tiny/s0/README.md @@ -0,0 +1,51 @@ +# Tiny Example + +1. `source path.sh` +3. set `CUDA_VISIBLE_DEVICES` as you need. +2. demo scrpt is `bash run.sh`. You can run commond separately as needed. + +## Steps +- Prepare the data + + ```bash + bash local/data.sh + ``` + + `data.sh` will download dataset, generate manifests, collect normalizer's statistics and build vocabulary. Once the data preparation is done, you will find the data (only part of LibriSpeech) downloaded in `${MAIN_ROOT}/dataset/librispeech` and the corresponding manifest files generated in `${PWD}/data` as well as a mean stddev file and a vocabulary file. It has to be run for the very first time you run this dataset and is reusable for all further experiments. + +- Train your own ASR model + + ```bash + bash local/train.sh + ``` + + `train.sh` will start a training job, with training logs printed to stdout and model checkpoint of every pass/epoch saved to `${PWD}/checkpoints`. These checkpoints could be used for training resuming, inference, evaluation and deployment. + +- Case inference with an existing model + + ```bash + bash local/infer.sh + ``` + + `infer.sh` will show us some speech-to-text decoding results for several (default: 10) samples with the trained model. The performance might not be good now as the current model is only trained with a toy subset of LibriSpeech. To see the results with a better model, you can download a well-trained (trained for several days, with the complete LibriSpeech) model and do the inference. + +- Evaluate an existing model + + ```bash + bash local/test.sh + ``` + + `test.sh` will evaluate the model with Word Error Rate (or Character Error Rate) measurement. Similarly, you can also download a well-trained model and test its performance: + + +- Export jit model + + ```bash + bash local/export.sh ckpt_path saved_jit_model_path + ``` + +- Tune hyper paerameter + + ```bash + bash local/tune.sh + ``` diff --git a/examples/tiny/conf/augmentation.config b/examples/tiny/s0/conf/augmentation.config similarity index 100% rename from examples/tiny/conf/augmentation.config rename to examples/tiny/s0/conf/augmentation.config diff --git a/examples/tiny/conf/deepspeech2.yaml b/examples/tiny/s0/conf/deepspeech2.yaml similarity index 100% rename from examples/tiny/conf/deepspeech2.yaml rename to examples/tiny/s0/conf/deepspeech2.yaml diff --git a/examples/tiny/local/data.sh b/examples/tiny/s0/local/data.sh similarity index 100% rename from examples/tiny/local/data.sh rename to examples/tiny/s0/local/data.sh diff --git a/examples/tiny/local/download_lm_en.sh b/examples/tiny/s0/local/download_lm_en.sh similarity index 100% rename from examples/tiny/local/download_lm_en.sh rename to examples/tiny/s0/local/download_lm_en.sh diff --git a/examples/tiny/local/download_model.sh b/examples/tiny/s0/local/download_model.sh similarity index 100% rename from examples/tiny/local/download_model.sh rename to examples/tiny/s0/local/download_model.sh diff --git a/examples/tiny/local/export.sh b/examples/tiny/s0/local/export.sh similarity index 100% rename from examples/tiny/local/export.sh rename to examples/tiny/s0/local/export.sh diff --git a/examples/tiny/local/infer.sh b/examples/tiny/s0/local/infer.sh similarity index 100% rename from examples/tiny/local/infer.sh rename to examples/tiny/s0/local/infer.sh diff --git a/examples/tiny/local/test.sh b/examples/tiny/s0/local/test.sh similarity index 100% rename from examples/tiny/local/test.sh rename to examples/tiny/s0/local/test.sh diff --git a/examples/tiny/local/train.sh b/examples/tiny/s0/local/train.sh similarity index 100% rename from examples/tiny/local/train.sh rename to examples/tiny/s0/local/train.sh diff --git a/examples/tiny/local/tune.sh b/examples/tiny/s0/local/tune.sh similarity index 100% rename from examples/tiny/local/tune.sh rename to examples/tiny/s0/local/tune.sh diff --git a/examples/tiny/path.sh b/examples/tiny/s0/path.sh similarity index 90% rename from examples/tiny/path.sh rename to examples/tiny/s0/path.sh index a179631b3..44b178383 100644 --- a/examples/tiny/path.sh +++ b/examples/tiny/s0/path.sh @@ -1,4 +1,4 @@ -export MAIN_ROOT=${PWD}/../../ +export MAIN_ROOT=${PWD}/../../../ export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} export LC_ALL=C diff --git a/examples/tiny/run.sh b/examples/tiny/s0/run.sh similarity index 100% rename from examples/tiny/run.sh rename to examples/tiny/s0/run.sh