From 5dba8c4f510baa044fa35d13290fc955e728c607 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Wed, 22 Sep 2021 09:32:00 +0000
Subject: [PATCH 01/13] fix espnet kaldi libri s2 config

---
 examples/librispeech/s2/conf/transformer.yaml | 6 +++---
 examples/librispeech/s2/local/test.sh         | 2 +-
 examples/librispeech/s2/run.sh                | 6 +++---
 utils/avg_model.py                            | 4 ++--
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/examples/librispeech/s2/conf/transformer.yaml b/examples/librispeech/s2/conf/transformer.yaml
index edf5b81dc..b86224ff4 100644
--- a/examples/librispeech/s2/conf/transformer.yaml
+++ b/examples/librispeech/s2/conf/transformer.yaml
@@ -12,7 +12,7 @@ collator:
   stride_ms: 10.0
   window_ms: 25.0
   sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs 
-  batch_size: 32 
+  batch_size: 30 
   maxlen_in: 512  # if input length  > maxlen-in, batchsize is automatically reduced
   maxlen_out: 150  # if output length > maxlen-out, batchsize is automatically reduced
   minibatches: 0 # for debug
@@ -59,7 +59,7 @@ model:
     model_conf:
         ctc_weight: 0.3
         ctc_dropoutrate: 0.0
-        ctc_grad_norm_type: instance
+        ctc_grad_norm_type: batch
         lsm_weight: 0.1     # label smoothing option
         length_normalized_loss: false
 
@@ -83,7 +83,7 @@ scheduler_conf:
   lr_decay: 1.0
 
 decoding:
-  batch_size: 64
+  batch_size: 1
   error_rate_type: wer
   decoding_method: attention  # 'attention', 'ctc_greedy_search', 'ctc_prefix_beam_search', 'attention_rescoring'
   lang_model_path: data/lm/common_crawl_00.prune01111.trie.klm
diff --git a/examples/librispeech/s2/local/test.sh b/examples/librispeech/s2/local/test.sh
index efd06f35e..893d67b5b 100755
--- a/examples/librispeech/s2/local/test.sh
+++ b/examples/librispeech/s2/local/test.sh
@@ -36,7 +36,7 @@ for type in attention ctc_greedy_search; do
         # stream decoding only support batchsize=1
         batch_size=1
     else
-        batch_size=64
+        batch_size=1
     fi
     python3 -u ${BIN_DIR}/test.py \
     --model-name u2_kaldi \
diff --git a/examples/librispeech/s2/run.sh b/examples/librispeech/s2/run.sh
index 46c8ea5d8..8dd937369 100755
--- a/examples/librispeech/s2/run.sh
+++ b/examples/librispeech/s2/run.sh
@@ -6,7 +6,7 @@ stage=0
 stop_stage=100
 conf_path=conf/transformer.yaml
 dict_path=data/train_960_unigram5000_units.txt
-avg_num=5
+avg_num=10
 source ${MAIN_ROOT}/utils/parse_options.sh || exit 1;
 
 avg_ckpt=avg_${avg_num}
@@ -20,12 +20,12 @@ fi
 
 if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
     # train model, all `ckpt` under `exp` dir
-    CUDA_VISIBLE_DEVICES=0,1,2,3 ./local/train.sh ${conf_path}  ${ckpt}
+    CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 ./local/train.sh ${conf_path}  ${ckpt}
 fi
 
 if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
     # avg n best model
-    avg.sh best exp/${ckpt}/checkpoints ${avg_num}
+    avg.sh latest exp/${ckpt}/checkpoints ${avg_num}
 fi
 
 if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
diff --git a/utils/avg_model.py b/utils/avg_model.py
index 8ec792f52..3a0739c95 100755
--- a/utils/avg_model.py
+++ b/utils/avg_model.py
@@ -80,8 +80,8 @@ def main(args):
         data = json.dumps({
             "avg_ckpt": args.dst_model,
             "ckpt": path_list,
-            "epoch": selected_epochs.tolist(),
-            "val_loss": beat_val_scores.tolist(),
+            "epoch": selected_epochs,
+            "val_loss": beat_val_scores,
         })
         f.write(data + "\n")
 

From fffa6477dbd4e5909639a6f8dcb5b3acbd7242e7 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Wed, 22 Sep 2021 10:04:35 +0000
Subject: [PATCH 02/13] nprocs 0 for cpu, other for gpu

---
 deepspeech/exps/deepspeech2/bin/train.py          |  2 +-
 deepspeech/exps/deepspeech2/model.py              |  4 ++--
 deepspeech/exps/u2/bin/train.py                   |  2 +-
 deepspeech/exps/u2/model.py                       |  2 +-
 deepspeech/exps/u2_kaldi/bin/train.py             |  2 +-
 deepspeech/exps/u2_kaldi/model.py                 |  2 +-
 deepspeech/exps/u2_st/bin/train.py                |  2 +-
 deepspeech/exps/u2_st/model.py                    |  2 +-
 deepspeech/training/cli.py                        |  8 +-------
 deepspeech/training/trainer.py                    |  6 +++---
 examples/aishell/s0/local/export.sh               |  6 ------
 examples/aishell/s0/local/test.sh                 |  7 +------
 examples/aishell/s0/local/test_export.sh          |  7 +------
 examples/aishell/s0/local/train.sh                |  6 ------
 examples/aishell/s1/local/align.sh                |  7 +------
 examples/aishell/s1/local/export.sh               |  6 ------
 examples/aishell/s1/local/test.sh                 | 11 ++---------
 examples/aishell/s1/local/train.sh                |  6 ------
 examples/callcenter/s1/local/align.sh             |  8 +-------
 examples/callcenter/s1/local/export.sh            |  6 ------
 examples/callcenter/s1/local/test.sh              | 10 ++--------
 examples/callcenter/s1/local/train.sh             |  5 -----
 examples/librispeech/s0/local/export.sh           |  6 ------
 examples/librispeech/s0/local/test.sh             |  7 +------
 examples/librispeech/s0/local/train.sh            |  7 -------
 examples/librispeech/s1/local/align.sh            |  7 +------
 examples/librispeech/s1/local/export.sh           |  6 ------
 examples/librispeech/s1/local/test.sh             | 11 ++---------
 examples/librispeech/s1/local/train.sh            | 10 +++-------
 examples/librispeech/s2/local/align.sh            |  7 +------
 examples/librispeech/s2/local/export.sh           |  6 ------
 examples/librispeech/s2/local/train.sh            |  7 -------
 examples/ted_en_zh/t0/local/test.sh               |  7 +------
 examples/ted_en_zh/t0/local/train.sh              |  7 -------
 examples/timit/s1/local/align.sh                  |  7 +------
 examples/timit/s1/local/export.sh                 |  6 ------
 examples/timit/s1/local/test.sh                   | 11 ++---------
 examples/timit/s1/local/train.sh                  |  7 -------
 examples/tiny/s0/local/export.sh                  |  6 ------
 examples/tiny/s0/local/test.sh                    |  7 +------
 examples/tiny/s0/local/train.sh                   |  7 -------
 examples/tiny/s1/local/align.sh                   |  7 +------
 examples/tiny/s1/local/export.sh                  |  6 ------
 examples/tiny/s1/local/test.sh                    | 10 ++--------
 examples/tiny/s1/local/train.sh                   |  6 ------
 examples/v18_to_v2x/deepspeech2x/model.py         |  2 +-
 examples/v18_to_v2x/exp_aishell/local/test.sh     |  7 +------
 examples/v18_to_v2x/exp_baidu_en8k/local/test.sh  |  7 +------
 examples/v18_to_v2x/exp_librispeech/local/test.sh |  7 +------
 tests/chains/ds2_params_lite_train_infer.txt      |  4 ++--
 50 files changed, 43 insertions(+), 269 deletions(-)

diff --git a/deepspeech/exps/deepspeech2/bin/train.py b/deepspeech/exps/deepspeech2/bin/train.py
index 69ff043a0..6740f288f 100644
--- a/deepspeech/exps/deepspeech2/bin/train.py
+++ b/deepspeech/exps/deepspeech2/bin/train.py
@@ -27,7 +27,7 @@ def main_sp(config, args):
 
 
 def main(config, args):
-    if args.device == "gpu" and args.nprocs > 1:
+    if args.nprocs > 0:
         dist.spawn(main_sp, args=(config, args), nprocs=args.nprocs)
     else:
         main_sp(config, args)
diff --git a/deepspeech/exps/deepspeech2/model.py b/deepspeech/exps/deepspeech2/model.py
index 7bf029300..8af2b02ab 100644
--- a/deepspeech/exps/deepspeech2/model.py
+++ b/deepspeech/exps/deepspeech2/model.py
@@ -403,7 +403,7 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device(self.args.device)
+        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()
@@ -635,7 +635,7 @@ class DeepSpeech2ExportTester(DeepSpeech2Tester):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device(self.args.device)
+        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
 
diff --git a/deepspeech/exps/u2/bin/train.py b/deepspeech/exps/u2/bin/train.py
index b664401a2..17fb08a6c 100644
--- a/deepspeech/exps/u2/bin/train.py
+++ b/deepspeech/exps/u2/bin/train.py
@@ -32,7 +32,7 @@ def main_sp(config, args):
 
 
 def main(config, args):
-    if args.device == "gpu" and args.nprocs > 1:
+    if args.nprocs > 0:
         dist.spawn(main_sp, args=(config, args), nprocs=args.nprocs)
     else:
         main_sp(config, args)
diff --git a/deepspeech/exps/u2/model.py b/deepspeech/exps/u2/model.py
index 6bf019005..bc46a104b 100644
--- a/deepspeech/exps/u2/model.py
+++ b/deepspeech/exps/u2/model.py
@@ -653,7 +653,7 @@ class U2Tester(U2Trainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device(self.args.device)
+        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()
diff --git a/deepspeech/exps/u2_kaldi/bin/train.py b/deepspeech/exps/u2_kaldi/bin/train.py
index 1dcd154d3..d909727f3 100644
--- a/deepspeech/exps/u2_kaldi/bin/train.py
+++ b/deepspeech/exps/u2_kaldi/bin/train.py
@@ -36,7 +36,7 @@ def main_sp(config, args):
 
 
 def main(config, args):
-    if args.device == "gpu" and args.nprocs > 1:
+    if args.nprocs > 0:
         dist.spawn(main_sp, args=(config, args), nprocs=args.nprocs)
     else:
         main_sp(config, args)
diff --git a/deepspeech/exps/u2_kaldi/model.py b/deepspeech/exps/u2_kaldi/model.py
index bc7cd4fd3..be89c3d69 100644
--- a/deepspeech/exps/u2_kaldi/model.py
+++ b/deepspeech/exps/u2_kaldi/model.py
@@ -637,7 +637,7 @@ class U2Tester(U2Trainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device(self.args.device)
+        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()
diff --git a/deepspeech/exps/u2_st/bin/train.py b/deepspeech/exps/u2_st/bin/train.py
index 86a0f0000..1e6a746b8 100644
--- a/deepspeech/exps/u2_st/bin/train.py
+++ b/deepspeech/exps/u2_st/bin/train.py
@@ -30,7 +30,7 @@ def main_sp(config, args):
 
 
 def main(config, args):
-    if args.device == "gpu" and args.nprocs > 1:
+    if args.nprocs > 0:
         dist.spawn(main_sp, args=(config, args), nprocs=args.nprocs)
     else:
         main_sp(config, args)
diff --git a/deepspeech/exps/u2_st/model.py b/deepspeech/exps/u2_st/model.py
index 4f95bc42b..55dadee82 100644
--- a/deepspeech/exps/u2_st/model.py
+++ b/deepspeech/exps/u2_st/model.py
@@ -661,7 +661,7 @@ class U2STTester(U2STTrainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device(self.args.device)
+        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()
diff --git a/deepspeech/training/cli.py b/deepspeech/training/cli.py
index 07c213dbc..aa263a066 100644
--- a/deepspeech/training/cli.py
+++ b/deepspeech/training/cli.py
@@ -30,7 +30,7 @@ def default_argument_parser():
 
     The ``--checkpoint_path`` specifies the checkpoint to load from.
 
-    The ``--device`` and ``--nprocs`` specifies how to run the training.
+    The ``--nprocs`` specifies how to run the training.
 
 
     See Also
@@ -51,12 +51,6 @@ def default_argument_parser():
         default=None,
         help="seed to use for paddle, np and random. None or 0 for random, else set seed."
     )
-    train_group.add_argument(
-        "--device",
-        type=str,
-        default='gpu',
-        choices=["cpu", "gpu"],
-        help="device cpu and gpu are supported.")
     train_group.add_argument(
         "--nprocs",
         type=int,
diff --git a/deepspeech/training/trainer.py b/deepspeech/training/trainer.py
index 8b1adcd03..b180f4890 100644
--- a/deepspeech/training/trainer.py
+++ b/deepspeech/training/trainer.py
@@ -86,7 +86,7 @@ class Trainer():
     >>>     config.merge_from_list(args.opts)
     >>> config.freeze()
     >>>
-    >>> if args.nprocs > 1 and args.device == "gpu":
+    >>> if args.nprocs > 0:
     >>>     dist.spawn(main_sp, args=(config, args), nprocs=args.nprocs)
     >>> else:
     >>>     main_sp(config, args)
@@ -119,7 +119,7 @@ class Trainer():
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device(self.args.device)
+        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
         if self.parallel:
             self.init_parallel()
 
@@ -139,7 +139,7 @@ class Trainer():
         """A flag indicating whether the experiment should run with
         multiprocessing.
         """
-        return self.args.device == "gpu" and self.args.nprocs > 1
+        return elf.args.nprocs > 0
 
     def init_parallel(self):
         """Init environment for multiprocess training.
diff --git a/examples/aishell/s0/local/export.sh b/examples/aishell/s0/local/export.sh
index 2e09e5f5e..a5e62c28d 100755
--- a/examples/aishell/s0/local/export.sh
+++ b/examples/aishell/s0/local/export.sh
@@ -13,13 +13,7 @@ ckpt_path_prefix=$2
 jit_model_export_path=$3
 model_type=$4
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 python3 -u ${BIN_DIR}/export.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --checkpoint_path ${ckpt_path_prefix} \
diff --git a/examples/aishell/s0/local/test.sh b/examples/aishell/s0/local/test.sh
index 9fd0bc8d5..2ae0740b3 100755
--- a/examples/aishell/s0/local/test.sh
+++ b/examples/aishell/s0/local/test.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 model_type=$3
@@ -23,8 +19,7 @@ if [ $? -ne 0 ]; then
 fi
 
 python3 -u ${BIN_DIR}/test.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${ckpt_prefix}.rsl \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/aishell/s0/local/test_export.sh b/examples/aishell/s0/local/test_export.sh
index b6d580979..a9a6b122d 100755
--- a/examples/aishell/s0/local/test_export.sh
+++ b/examples/aishell/s0/local/test_export.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 jit_model_export_path=$2
 model_type=$3
@@ -23,8 +19,7 @@ if [ $? -ne 0 ]; then
 fi
 
 python3 -u ${BIN_DIR}/test_export.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${jit_model_export_path}.rsl \
 --export_path ${jit_model_export_path} \
diff --git a/examples/aishell/s0/local/train.sh b/examples/aishell/s0/local/train.sh
index 668ad0ead..edbf33830 100755
--- a/examples/aishell/s0/local/train.sh
+++ b/examples/aishell/s0/local/train.sh
@@ -12,11 +12,6 @@ config_path=$1
 ckpt_name=$2
 model_type=$3
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 mkdir -p exp
 
 # seed may break model convergence
@@ -26,7 +21,6 @@ if [ ${seed} != 0 ]; then
 fi
 
 python3 -u ${BIN_DIR}/train.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
diff --git a/examples/aishell/s1/local/align.sh b/examples/aishell/s1/local/align.sh
index ad6c84bc8..279461aaf 100755
--- a/examples/aishell/s1/local/align.sh
+++ b/examples/aishell/s1/local/align.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 
@@ -22,8 +18,7 @@ mkdir -p ${output_dir}
 # align dump in `result_file`
 # .tier, .TextGrid dump in `dir of result_file`
 python3 -u ${BIN_DIR}/alignment.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${output_dir}/${type}.align \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/aishell/s1/local/export.sh b/examples/aishell/s1/local/export.sh
index f99a15bad..b562218e7 100755
--- a/examples/aishell/s1/local/export.sh
+++ b/examples/aishell/s1/local/export.sh
@@ -12,13 +12,7 @@ config_path=$1
 ckpt_path_prefix=$2
 jit_model_export_path=$3
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 python3 -u ${BIN_DIR}/export.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --checkpoint_path ${ckpt_path_prefix} \
diff --git a/examples/aishell/s1/local/test.sh b/examples/aishell/s1/local/test.sh
index f7e99ad7f..c87412c9b 100755
--- a/examples/aishell/s1/local/test.sh
+++ b/examples/aishell/s1/local/test.sh
@@ -8,11 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 config_path=$1
 ckpt_prefix=$2
 
@@ -39,8 +34,7 @@ for type in attention ctc_greedy_search; do
     output_dir=${ckpt_prefix}
     mkdir -p ${output_dir}
     python3 -u ${BIN_DIR}/test.py \
-    --device ${device} \
-    --nproc 1 \
+    --nproc ${ngpu} \
     --config ${config_path} \
     --result_file ${output_dir}/${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
@@ -58,8 +52,7 @@ for type in ctc_prefix_beam_search attention_rescoring; do
     output_dir=${ckpt_prefix}
     mkdir -p ${output_dir}
     python3 -u ${BIN_DIR}/test.py \
-    --device ${device} \
-    --nproc 1 \
+    --nproc ${ngpu} \
     --config ${config_path} \
     --result_file ${output_dir}/${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/aishell/s1/local/train.sh b/examples/aishell/s1/local/train.sh
index 5097d4d03..71af3a006 100755
--- a/examples/aishell/s1/local/train.sh
+++ b/examples/aishell/s1/local/train.sh
@@ -12,11 +12,6 @@ source ${MAIN_ROOT}/utils/parse_options.sh || exit 1;
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 if [ ${seed} != 0  ]; then
     export FLAGS_cudnn_deterministic=True
     echo "using seed $seed & FLAGS_cudnn_deterministic=True ..."
@@ -34,7 +29,6 @@ mkdir -p exp
 
 python3 -u ${BIN_DIR}/train.py \
 --seed ${seed} \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
diff --git a/examples/callcenter/s1/local/align.sh b/examples/callcenter/s1/local/align.sh
index f2c878c20..b679e2ea7 100755
--- a/examples/callcenter/s1/local/align.sh
+++ b/examples/callcenter/s1/local/align.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 
@@ -20,7 +16,6 @@ ckpt_name=$(basename ${ckpt_prefxi})
 mkdir -p exp
 
 
-
 batch_size=1
 output_dir=${ckpt_prefix}
 mkdir -p ${output_dir}
@@ -28,8 +23,7 @@ mkdir -p ${output_dir}
 # align dump in `result_file`
 # .tier, .TextGrid dump in `dir of result_file`
 python3 -u ${BIN_DIR}/alignment.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${output_dir}/${type}.align \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/callcenter/s1/local/export.sh b/examples/callcenter/s1/local/export.sh
index d171899cd..d5f912e90 100755
--- a/examples/callcenter/s1/local/export.sh
+++ b/examples/callcenter/s1/local/export.sh
@@ -12,13 +12,7 @@ config_path=$1
 ckpt_path_prefix=$2
 jit_model_export_path=$3
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 python3 -u ${BIN_DIR}/export.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --checkpoint_path ${ckpt_path_prefix} \
diff --git a/examples/callcenter/s1/local/test.sh b/examples/callcenter/s1/local/test.sh
index 7a5b1cdb1..dca3137dd 100755
--- a/examples/callcenter/s1/local/test.sh
+++ b/examples/callcenter/s1/local/test.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 
@@ -32,8 +28,7 @@ for type in attention ctc_greedy_search; do
     output_dir=${ckpt_prefix}
     mkdir -p ${output_dir}
     python3 -u ${BIN_DIR}/test.py \
-    --device ${device} \
-    --nproc 1 \
+    --nproc ${ngpu} \
     --config ${config_path} \
     --result_file ${output_dir}/${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
@@ -51,8 +46,7 @@ for type in ctc_prefix_beam_search attention_rescoring; do
     output_dir=${ckpt_prefix}
     mkdir -p ${output_dir}
     python3 -u ${BIN_DIR}/test.py \
-    --device ${device} \
-    --nproc 1 \
+    --nproc ${ngpu} \
     --config ${config_path} \
     --result_file ${output_dir}/${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/callcenter/s1/local/train.sh b/examples/callcenter/s1/local/train.sh
index d5dc15b03..eb8f86626 100755
--- a/examples/callcenter/s1/local/train.sh
+++ b/examples/callcenter/s1/local/train.sh
@@ -11,10 +11,6 @@ echo "using $ngpu gpus..."
 config_path=$1
 ckpt_name=$2
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 echo "using ${device}..."
 
 mkdir -p exp
@@ -26,7 +22,6 @@ if [ ${seed} != 0 ]; then
 fi
 
 python3 -u ${BIN_DIR}/train.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
diff --git a/examples/librispeech/s0/local/export.sh b/examples/librispeech/s0/local/export.sh
index 2e09e5f5e..a5e62c28d 100755
--- a/examples/librispeech/s0/local/export.sh
+++ b/examples/librispeech/s0/local/export.sh
@@ -13,13 +13,7 @@ ckpt_path_prefix=$2
 jit_model_export_path=$3
 model_type=$4
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 python3 -u ${BIN_DIR}/export.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --checkpoint_path ${ckpt_path_prefix} \
diff --git a/examples/librispeech/s0/local/test.sh b/examples/librispeech/s0/local/test.sh
index b5b68c599..4d00f30b8 100755
--- a/examples/librispeech/s0/local/test.sh
+++ b/examples/librispeech/s0/local/test.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 model_type=$3
@@ -23,8 +19,7 @@ if [ $? -ne 0 ]; then
 fi
 
 python3 -u ${BIN_DIR}/test.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${ckpt_prefix}.rsl \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/librispeech/s0/local/train.sh b/examples/librispeech/s0/local/train.sh
index 6aee372a4..519df7fe9 100755
--- a/examples/librispeech/s0/local/train.sh
+++ b/examples/librispeech/s0/local/train.sh
@@ -12,12 +12,6 @@ config_path=$1
 ckpt_name=$2
 model_type=$3
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-echo "using ${device}..."
-
 mkdir -p exp
 
 # seed may break model convergence
@@ -27,7 +21,6 @@ if [ ${seed} != 0 ]; then
 fi
 
 python3 -u ${BIN_DIR}/train.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
diff --git a/examples/librispeech/s1/local/align.sh b/examples/librispeech/s1/local/align.sh
index ad6c84bc8..279461aaf 100755
--- a/examples/librispeech/s1/local/align.sh
+++ b/examples/librispeech/s1/local/align.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 
@@ -22,8 +18,7 @@ mkdir -p ${output_dir}
 # align dump in `result_file`
 # .tier, .TextGrid dump in `dir of result_file`
 python3 -u ${BIN_DIR}/alignment.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${output_dir}/${type}.align \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/librispeech/s1/local/export.sh b/examples/librispeech/s1/local/export.sh
index f99a15bad..b562218e7 100755
--- a/examples/librispeech/s1/local/export.sh
+++ b/examples/librispeech/s1/local/export.sh
@@ -12,13 +12,7 @@ config_path=$1
 ckpt_path_prefix=$2
 jit_model_export_path=$3
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 python3 -u ${BIN_DIR}/export.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --checkpoint_path ${ckpt_path_prefix} \
diff --git a/examples/librispeech/s1/local/test.sh b/examples/librispeech/s1/local/test.sh
index 3bd3f0bba..f7ec34ab5 100755
--- a/examples/librispeech/s1/local/test.sh
+++ b/examples/librispeech/s1/local/test.sh
@@ -8,11 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 config_path=$1
 ckpt_prefix=$2
 
@@ -38,8 +33,7 @@ for type in attention ctc_greedy_search; do
         batch_size=64
     fi
     python3 -u ${BIN_DIR}/test.py \
-    --device ${device} \
-    --nproc 1 \
+    --nproc ${ngpu} \
     --config ${config_path} \
     --result_file ${ckpt_prefix}.${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
@@ -55,8 +49,7 @@ for type in ctc_prefix_beam_search attention_rescoring; do
     echo "decoding ${type}"
     batch_size=1
     python3 -u ${BIN_DIR}/test.py \
-    --device ${device} \
-    --nproc 1 \
+    --nproc ${ngpu} \
     --config ${config_path} \
     --result_file ${ckpt_prefix}.${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/librispeech/s1/local/train.sh b/examples/librispeech/s1/local/train.sh
index f905b766e..8f92c6469 100755
--- a/examples/librispeech/s1/local/train.sh
+++ b/examples/librispeech/s1/local/train.sh
@@ -11,12 +11,6 @@ echo "using $ngpu gpus..."
 config_path=$1
 ckpt_name=$2
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-echo "using ${device}..."
-
 mkdir -p exp
 
 # seed may break model convergence
@@ -25,8 +19,10 @@ if [ ${seed} != 0 ]; then
     export FLAGS_cudnn_deterministic=True
 fi
 
+# export FLAGS_cudnn_exhaustive_search=true
+# export FLAGS_conv_workspace_size_limit=4000
+
 python3 -u ${BIN_DIR}/train.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
diff --git a/examples/librispeech/s2/local/align.sh b/examples/librispeech/s2/local/align.sh
index b3d8fa5f5..b45f4a0f5 100755
--- a/examples/librispeech/s2/local/align.sh
+++ b/examples/librispeech/s2/local/align.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 dict_path=$2
 ckpt_prefix=$3
@@ -26,8 +22,7 @@ python3 -u ${BIN_DIR}/test.py \
 --model-name 'u2_kaldi' \
 --run-mode 'align' \
 --dict-path ${dict_path} \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result-file ${output_dir}/${type}.align \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/librispeech/s2/local/export.sh b/examples/librispeech/s2/local/export.sh
index efa70a2b9..9c66dc62a 100755
--- a/examples/librispeech/s2/local/export.sh
+++ b/examples/librispeech/s2/local/export.sh
@@ -12,15 +12,9 @@ config_path=$1
 ckpt_path_prefix=$2
 jit_model_export_path=$3
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 python3 -u ${BIN_DIR}/test.py \
 --model-name 'u2_kaldi' \
 --run-mode 'export' \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --checkpoint_path ${ckpt_path_prefix} \
diff --git a/examples/librispeech/s2/local/train.sh b/examples/librispeech/s2/local/train.sh
index 66754201f..33b46c20f 100755
--- a/examples/librispeech/s2/local/train.sh
+++ b/examples/librispeech/s2/local/train.sh
@@ -11,12 +11,6 @@ echo "using $ngpu gpus..."
 config_path=$1
 ckpt_name=$2
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-echo "using ${device}..."
-
 mkdir -p exp
 
 # seed may break model convergence
@@ -27,7 +21,6 @@ fi
 
 python3 -u ${BIN_DIR}/train.py \
 --model-name u2_kaldi \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
diff --git a/examples/ted_en_zh/t0/local/test.sh b/examples/ted_en_zh/t0/local/test.sh
index 642328e88..344750855 100755
--- a/examples/ted_en_zh/t0/local/test.sh
+++ b/examples/ted_en_zh/t0/local/test.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 
@@ -19,8 +15,7 @@ for type in fullsentence; do
     echo "decoding ${type}"
     batch_size=32
     python3 -u ${BIN_DIR}/test.py \
-    --device ${device} \
-    --nproc 1 \
+    --nproc ${ngpu} \
     --config ${config_path} \
     --result_file ${ckpt_prefix}.${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/ted_en_zh/t0/local/train.sh b/examples/ted_en_zh/t0/local/train.sh
index f905b766e..e5fd19ddb 100755
--- a/examples/ted_en_zh/t0/local/train.sh
+++ b/examples/ted_en_zh/t0/local/train.sh
@@ -11,12 +11,6 @@ echo "using $ngpu gpus..."
 config_path=$1
 ckpt_name=$2
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-echo "using ${device}..."
-
 mkdir -p exp
 
 # seed may break model convergence
@@ -26,7 +20,6 @@ if [ ${seed} != 0 ]; then
 fi
 
 python3 -u ${BIN_DIR}/train.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
diff --git a/examples/timit/s1/local/align.sh b/examples/timit/s1/local/align.sh
index ad6c84bc8..279461aaf 100755
--- a/examples/timit/s1/local/align.sh
+++ b/examples/timit/s1/local/align.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 
@@ -22,8 +18,7 @@ mkdir -p ${output_dir}
 # align dump in `result_file`
 # .tier, .TextGrid dump in `dir of result_file`
 python3 -u ${BIN_DIR}/alignment.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${output_dir}/${type}.align \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/timit/s1/local/export.sh b/examples/timit/s1/local/export.sh
index f99a15bad..b562218e7 100755
--- a/examples/timit/s1/local/export.sh
+++ b/examples/timit/s1/local/export.sh
@@ -12,13 +12,7 @@ config_path=$1
 ckpt_path_prefix=$2
 jit_model_export_path=$3
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 python3 -u ${BIN_DIR}/export.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --checkpoint_path ${ckpt_path_prefix} \
diff --git a/examples/timit/s1/local/test.sh b/examples/timit/s1/local/test.sh
index a137924e2..868c8fda8 100755
--- a/examples/timit/s1/local/test.sh
+++ b/examples/timit/s1/local/test.sh
@@ -8,11 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 config_path=$1
 ckpt_prefix=$2
 
@@ -37,8 +32,7 @@ for type in attention ctc_greedy_search; do
         batch_size=64
     fi
     python3 -u ${BIN_DIR}/test.py \
-    --device ${device} \
-    --nproc 1 \
+    --nproc ${ngpu} \
     --config ${config_path} \
     --result_file ${ckpt_prefix}.${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
@@ -54,8 +48,7 @@ for type in ctc_prefix_beam_search attention_rescoring; do
     echo "decoding ${type}"
     batch_size=1
     python3 -u ${BIN_DIR}/test.py \
-    --device ${device} \
-    --nproc 1 \
+    --nproc ${ngpu}  \
     --config ${config_path} \
     --result_file ${ckpt_prefix}.${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/timit/s1/local/train.sh b/examples/timit/s1/local/train.sh
index 180d8b5a7..89a64327c 100755
--- a/examples/timit/s1/local/train.sh
+++ b/examples/timit/s1/local/train.sh
@@ -11,12 +11,6 @@ echo "using $ngpu gpus..."
 config_path=$1
 ckpt_name=$2
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-echo "using ${device}..."
-
 mkdir -p exp
 
 # seed may break model convergence
@@ -26,7 +20,6 @@ if [ ${seed} != 0  ]; then
 fi
 
 python3 -u ${BIN_DIR}/train.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
diff --git a/examples/tiny/s0/local/export.sh b/examples/tiny/s0/local/export.sh
index 2e09e5f5e..a5e62c28d 100755
--- a/examples/tiny/s0/local/export.sh
+++ b/examples/tiny/s0/local/export.sh
@@ -13,13 +13,7 @@ ckpt_path_prefix=$2
 jit_model_export_path=$3
 model_type=$4
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 python3 -u ${BIN_DIR}/export.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --checkpoint_path ${ckpt_path_prefix} \
diff --git a/examples/tiny/s0/local/test.sh b/examples/tiny/s0/local/test.sh
index b5b68c599..4d00f30b8 100755
--- a/examples/tiny/s0/local/test.sh
+++ b/examples/tiny/s0/local/test.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 model_type=$3
@@ -23,8 +19,7 @@ if [ $? -ne 0 ]; then
 fi
 
 python3 -u ${BIN_DIR}/test.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${ckpt_prefix}.rsl \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/tiny/s0/local/train.sh b/examples/tiny/s0/local/train.sh
index 9a76c7ade..5b87780ae 100755
--- a/examples/tiny/s0/local/train.sh
+++ b/examples/tiny/s0/local/train.sh
@@ -10,17 +10,11 @@ source ${MAIN_ROOT}/utils/parse_options.sh || exit 1;
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 if [ ${seed} != 0  ]; then
     export FLAGS_cudnn_deterministic=True
     echo "using seed $seed & FLAGS_cudnn_deterministic=True ..."
 fi
 
-
 if [ $# != 3 ];then
     echo "usage: CUDA_VISIBLE_DEVICES=0 ${0} config_path ckpt_name model_type"
     exit -1
@@ -33,7 +27,6 @@ model_type=$3
 mkdir -p exp
 
 python3 -u ${BIN_DIR}/train.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
diff --git a/examples/tiny/s1/local/align.sh b/examples/tiny/s1/local/align.sh
index ad6c84bc8..279461aaf 100755
--- a/examples/tiny/s1/local/align.sh
+++ b/examples/tiny/s1/local/align.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 
@@ -22,8 +18,7 @@ mkdir -p ${output_dir}
 # align dump in `result_file`
 # .tier, .TextGrid dump in `dir of result_file`
 python3 -u ${BIN_DIR}/alignment.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${output_dir}/${type}.align \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/tiny/s1/local/export.sh b/examples/tiny/s1/local/export.sh
index f99a15bad..b562218e7 100755
--- a/examples/tiny/s1/local/export.sh
+++ b/examples/tiny/s1/local/export.sh
@@ -12,13 +12,7 @@ config_path=$1
 ckpt_path_prefix=$2
 jit_model_export_path=$3
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 python3 -u ${BIN_DIR}/export.py \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --checkpoint_path ${ckpt_path_prefix} \
diff --git a/examples/tiny/s1/local/test.sh b/examples/tiny/s1/local/test.sh
index 4d3ed081a..19872bb36 100755
--- a/examples/tiny/s1/local/test.sh
+++ b/examples/tiny/s1/local/test.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 
@@ -35,8 +31,7 @@ for type in attention ctc_greedy_search; do
         batch_size=64
     fi
     python3 -u ${BIN_DIR}/test.py \
-    --device ${device} \
-    --nproc 1 \
+    --nproc ${ngpu} \
     --config ${config_path} \
     --result_file ${ckpt_prefix}.${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
@@ -52,8 +47,7 @@ for type in ctc_prefix_beam_search attention_rescoring; do
     echo "decoding ${type}"
     batch_size=1
     python3 -u ${BIN_DIR}/test.py \
-    --device ${device} \
-    --nproc 1 \
+    --nproc ${ngpu} \
     --config ${config_path} \
     --result_file ${ckpt_prefix}.${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/tiny/s1/local/train.sh b/examples/tiny/s1/local/train.sh
index 5097d4d03..71af3a006 100755
--- a/examples/tiny/s1/local/train.sh
+++ b/examples/tiny/s1/local/train.sh
@@ -12,11 +12,6 @@ source ${MAIN_ROOT}/utils/parse_options.sh || exit 1;
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 if [ ${seed} != 0  ]; then
     export FLAGS_cudnn_deterministic=True
     echo "using seed $seed & FLAGS_cudnn_deterministic=True ..."
@@ -34,7 +29,6 @@ mkdir -p exp
 
 python3 -u ${BIN_DIR}/train.py \
 --seed ${seed} \
---device ${device} \
 --nproc ${ngpu} \
 --config ${config_path} \
 --output exp/${ckpt_name} \
diff --git a/examples/v18_to_v2x/deepspeech2x/model.py b/examples/v18_to_v2x/deepspeech2x/model.py
index 1fe1e2d68..cbbc502d2 100644
--- a/examples/v18_to_v2x/deepspeech2x/model.py
+++ b/examples/v18_to_v2x/deepspeech2x/model.py
@@ -401,7 +401,7 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device(self.args.device)
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()
diff --git a/examples/v18_to_v2x/exp_aishell/local/test.sh b/examples/v18_to_v2x/exp_aishell/local/test.sh
index 9fd0bc8d5..2ae0740b3 100755
--- a/examples/v18_to_v2x/exp_aishell/local/test.sh
+++ b/examples/v18_to_v2x/exp_aishell/local/test.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 model_type=$3
@@ -23,8 +19,7 @@ if [ $? -ne 0 ]; then
 fi
 
 python3 -u ${BIN_DIR}/test.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${ckpt_prefix}.rsl \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/v18_to_v2x/exp_baidu_en8k/local/test.sh b/examples/v18_to_v2x/exp_baidu_en8k/local/test.sh
index b5b68c599..4d00f30b8 100755
--- a/examples/v18_to_v2x/exp_baidu_en8k/local/test.sh
+++ b/examples/v18_to_v2x/exp_baidu_en8k/local/test.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 model_type=$3
@@ -23,8 +19,7 @@ if [ $? -ne 0 ]; then
 fi
 
 python3 -u ${BIN_DIR}/test.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${ckpt_prefix}.rsl \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/examples/v18_to_v2x/exp_librispeech/local/test.sh b/examples/v18_to_v2x/exp_librispeech/local/test.sh
index b5b68c599..4d00f30b8 100755
--- a/examples/v18_to_v2x/exp_librispeech/local/test.sh
+++ b/examples/v18_to_v2x/exp_librispeech/local/test.sh
@@ -8,10 +8,6 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
 config_path=$1
 ckpt_prefix=$2
 model_type=$3
@@ -23,8 +19,7 @@ if [ $? -ne 0 ]; then
 fi
 
 python3 -u ${BIN_DIR}/test.py \
---device ${device} \
---nproc 1 \
+--nproc ${ngpu} \
 --config ${config_path} \
 --result_file ${ckpt_prefix}.rsl \
 --checkpoint_path ${ckpt_prefix} \
diff --git a/tests/chains/ds2_params_lite_train_infer.txt b/tests/chains/ds2_params_lite_train_infer.txt
index 82a9da9a9..47f7db95b 100644
--- a/tests/chains/ds2_params_lite_train_infer.txt
+++ b/tests/chains/ds2_params_lite_train_infer.txt
@@ -13,7 +13,7 @@ null:null
 null:null
 ##
 trainer:norm_train
-norm_train: ../../../deepspeech/exps/deepspeech2/bin/train.py --nproc 1 --config conf/deepspeech2.yaml --model_type offline --device gpu
+norm_train: ../../../deepspeech/exps/deepspeech2/bin/train.py --nproc 1 --config conf/deepspeech2.yaml --model_type offline
 pact_train:null
 fpgm_train:null
 distill_train:null
@@ -21,7 +21,7 @@ null:null
 null:null
 ##
 ===========================eval_params===========================
-eval: ../../../deepspeech/exps/deepspeech2/bin/test.py --nproc 1 --config conf/deepspeech2.yaml --result_file tests/9.rsl  --model_type offline --device gpu
+eval: ../../../deepspeech/exps/deepspeech2/bin/test.py --nproc 1 --config conf/deepspeech2.yaml --result_file tests/9.rsl  --model_type offline
 null:null
 ##
 ===========================infer_params===========================

From da3a94b0e0f7892951a78b50881cdceb68460d89 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Wed, 22 Sep 2021 12:17:38 +0000
Subject: [PATCH 03/13] fix set_device; more utils; args.opts support multi
 same name

---
 deepspeech/exps/deepspeech2/model.py  |   4 +-
 deepspeech/exps/u2/model.py           |   2 +-
 deepspeech/exps/u2_kaldi/model.py     |   2 +-
 deepspeech/exps/u2_st/model.py        |   2 +-
 deepspeech/training/cli.py            |  23 +-
 deepspeech/training/trainer.py        |   4 +-
 examples/librispeech/s1/local/test.sh |   6 +-
 examples/librispeech/s2/cmd.sh        |  89 +++++++
 examples/librispeech/s2/local/test.sh | 102 ++++----
 examples/librispeech/s2/run.sh        |   4 +-
 utils/README.md                       |   3 +
 utils/build_kenlm_model_from_arpa.sh  |  44 ++++
 utils/duration_from_maniefst.sh       |   0
 utils/log.sh                          |   0
 utils/parallel/run.pl                 | 356 ++++++++++++++++++++++++++
 utils/parse_options.sh                |   0
 utils/pd_env_collect.sh               |   0
 utils/profile.sh                      |   0
 utils/run.pl                          |   1 +
 utils/score_sclite.sh                 | 125 +++++++++
 utils/spk2utt_to_utt2spk.pl           |  25 ++
 utils/split_data.sh                   |  79 ++++++
 utils/split_json.sh                   |  31 +++
 utils/split_scp.pl                    | 212 +++++++++++++++
 utils/train_arpa_with_kenlm.sh        |  67 +++++
 utils/utility.sh                      |   0
 utils/utt2spk_to_spk2utt.pl           |  38 +++
 27 files changed, 1158 insertions(+), 61 deletions(-)
 create mode 100644 examples/librispeech/s2/cmd.sh
 create mode 100644 utils/README.md
 create mode 100755 utils/build_kenlm_model_from_arpa.sh
 mode change 100644 => 100755 utils/duration_from_maniefst.sh
 mode change 100644 => 100755 utils/log.sh
 create mode 100755 utils/parallel/run.pl
 mode change 100644 => 100755 utils/parse_options.sh
 mode change 100644 => 100755 utils/pd_env_collect.sh
 mode change 100644 => 100755 utils/profile.sh
 create mode 100755 utils/run.pl
 create mode 100755 utils/score_sclite.sh
 create mode 100755 utils/spk2utt_to_utt2spk.pl
 create mode 100755 utils/split_data.sh
 create mode 100755 utils/split_json.sh
 create mode 100644 utils/split_scp.pl 
 create mode 100755 utils/train_arpa_with_kenlm.sh
 mode change 100644 => 100755 utils/utility.sh
 create mode 100755 utils/utt2spk_to_spk2utt.pl

diff --git a/deepspeech/exps/deepspeech2/model.py b/deepspeech/exps/deepspeech2/model.py
index 8af2b02ab..120539814 100644
--- a/deepspeech/exps/deepspeech2/model.py
+++ b/deepspeech/exps/deepspeech2/model.py
@@ -403,7 +403,7 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()
@@ -635,7 +635,7 @@ class DeepSpeech2ExportTester(DeepSpeech2Tester):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
 
diff --git a/deepspeech/exps/u2/model.py b/deepspeech/exps/u2/model.py
index bc46a104b..4dd054896 100644
--- a/deepspeech/exps/u2/model.py
+++ b/deepspeech/exps/u2/model.py
@@ -653,7 +653,7 @@ class U2Tester(U2Trainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()
diff --git a/deepspeech/exps/u2_kaldi/model.py b/deepspeech/exps/u2_kaldi/model.py
index be89c3d69..e8482aa9c 100644
--- a/deepspeech/exps/u2_kaldi/model.py
+++ b/deepspeech/exps/u2_kaldi/model.py
@@ -637,7 +637,7 @@ class U2Tester(U2Trainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()
diff --git a/deepspeech/exps/u2_st/model.py b/deepspeech/exps/u2_st/model.py
index 55dadee82..c98f5e69e 100644
--- a/deepspeech/exps/u2_st/model.py
+++ b/deepspeech/exps/u2_st/model.py
@@ -661,7 +661,7 @@ class U2STTester(U2STTrainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()
diff --git a/deepspeech/training/cli.py b/deepspeech/training/cli.py
index aa263a066..e079293c7 100644
--- a/deepspeech/training/cli.py
+++ b/deepspeech/training/cli.py
@@ -14,6 +14,20 @@
 import argparse
 
 
+class ExtendAction(argparse.Action):
+    """
+    [Since Python 3.8, the "extend" is available directly in stdlib]
+    (https://docs.python.org/3.8/library/argparse.html#action).
+    If you only have to support 3.8+ then defining it yourself is no longer required. 
+    Usage of stdlib "extend" action is exactly the same way as this answer originally described:
+    """
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = getattr(namespace, self.dest) or []
+        items.extend(values)
+        setattr(namespace, self.dest, items)
+
+
 def default_argument_parser():
     r"""A simple yet genral argument parser for experiments with parakeet.
 
@@ -42,6 +56,7 @@ def default_argument_parser():
         the parser
     """
     parser = argparse.ArgumentParser()
+    parser.register('action', 'extend', ExtendAction)
 
     train_group = parser.add_argument_group(
         title='Train Options', description=None)
@@ -64,10 +79,10 @@ def default_argument_parser():
         "--checkpoint_path", type=str, help="path to load checkpoint")
     train_group.add_argument(
         "--opts",
-        type=str,
-        default=[],
-        nargs='+',
-        help="overwrite --config file, passing in LIST[KEY VALUE] pairs")
+        action='extend',
+        nargs=2,
+        metavar=('key', 'val'),
+        help="overwrite --config field, passing (KEY VALUE) pairs")
     train_group.add_argument(
         "--dump-config", metavar="FILE", help="dump config to `this` file.")
 
diff --git a/deepspeech/training/trainer.py b/deepspeech/training/trainer.py
index b180f4890..79b1562e4 100644
--- a/deepspeech/training/trainer.py
+++ b/deepspeech/training/trainer.py
@@ -119,7 +119,7 @@ class Trainer():
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
         if self.parallel:
             self.init_parallel()
 
@@ -139,7 +139,7 @@ class Trainer():
         """A flag indicating whether the experiment should run with
         multiprocessing.
         """
-        return elf.args.nprocs > 0
+        return self.args.nprocs > 0
 
     def init_parallel(self):
         """Init environment for multiprocess training.
diff --git a/examples/librispeech/s1/local/test.sh b/examples/librispeech/s1/local/test.sh
index f7ec34ab5..62b92e1e3 100755
--- a/examples/librispeech/s1/local/test.sh
+++ b/examples/librispeech/s1/local/test.sh
@@ -37,7 +37,8 @@ for type in attention ctc_greedy_search; do
     --config ${config_path} \
     --result_file ${ckpt_prefix}.${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
-    --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
+    --opts decoding.decoding_method ${type} \
+    --opts decoding.batch_size ${batch_size}
 
     if [ $? -ne 0 ]; then
         echo "Failed in evaluation!"
@@ -53,7 +54,8 @@ for type in ctc_prefix_beam_search attention_rescoring; do
     --config ${config_path} \
     --result_file ${ckpt_prefix}.${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
-    --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
+    --opts decoding.decoding_method ${type} \
+    --opts decoding.batch_size ${batch_size}
 
     if [ $? -ne 0 ]; then
         echo "Failed in evaluation!"
diff --git a/examples/librispeech/s2/cmd.sh b/examples/librispeech/s2/cmd.sh
new file mode 100644
index 000000000..7b70ef5e0
--- /dev/null
+++ b/examples/librispeech/s2/cmd.sh
@@ -0,0 +1,89 @@
+# ====== About run.pl, queue.pl, slurm.pl, and ssh.pl ======
+# Usage: <cmd>.pl [options] JOB=1:<nj> <log> <command...>
+# e.g.
+#   run.pl --mem 4G JOB=1:10 echo.JOB.log echo JOB
+#
+# Options:
+#   --time <time>: Limit the maximum time to execute.
+#   --mem <mem>: Limit the maximum memory usage.
+#   -–max-jobs-run <njob>: Limit the number parallel jobs. This is ignored for non-array jobs.
+#   --num-threads <ngpu>: Specify the number of CPU core.
+#   --gpu <ngpu>: Specify the number of GPU devices.
+#   --config: Change the configuration file from default.
+#
+# "JOB=1:10" is used for "array jobs" and it can control the number of parallel jobs.
+# The left string of "=", i.e. "JOB", is replaced by <N>(Nth job) in the command and the log file name,
+# e.g. "echo JOB" is changed to "echo 3" for the 3rd job and "echo 8" for 8th job respectively.
+# Note that the number must start with a positive number, so you can't use "JOB=0:10" for example.
+#
+# run.pl, queue.pl, slurm.pl, and ssh.pl have unified interface, not depending on its backend.
+# These options are mapping to specific options for each backend and
+# it is configured by "conf/queue.conf" and "conf/slurm.conf" by default.
+# If jobs failed, your configuration might be wrong for your environment.
+#
+#
+# The official documentation for run.pl, queue.pl, slurm.pl, and ssh.pl:
+#   "Parallelization in Kaldi": http://kaldi-asr.org/doc/queue.html
+# =========================================================~
+
+
+# Select the backend used by run.sh from "local", "sge", "slurm", or "ssh"
+cmd_backend='local'
+
+# Local machine, without any Job scheduling system
+if [ "${cmd_backend}" = local ]; then
+
+    # The other usage
+    export train_cmd="run.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="run.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="run.pl"
+
+# "qsub" (SGE, Torque, PBS, etc.)
+elif [ "${cmd_backend}" = sge ]; then
+    # The default setting is written in conf/queue.conf.
+    # You must change "-q g.q" for the "queue" for your environment.
+    # To know the "queue" names, type "qhost -q"
+    # Note that to use "--gpu *", you have to setup "complex_value" for the system scheduler.
+
+    export train_cmd="queue.pl"
+    export cuda_cmd="queue.pl"
+    export decode_cmd="queue.pl"
+
+# "sbatch" (Slurm)
+elif [ "${cmd_backend}" = slurm ]; then
+    # The default setting is written in conf/slurm.conf.
+    # You must change "-p cpu" and "-p gpu" for the "partion" for your environment.
+    # To know the "partion" names, type "sinfo".
+    # You can use "--gpu * " by default for slurm and it is interpreted as "--gres gpu:*"
+    # The devices are allocated exclusively using "${CUDA_VISIBLE_DEVICES}".
+
+    export train_cmd="slurm.pl"
+    export cuda_cmd="slurm.pl"
+    export decode_cmd="slurm.pl"
+
+elif [ "${cmd_backend}" = ssh ]; then
+    # You have to create ".queue/machines" to specify the host to execute jobs.
+    # e.g. .queue/machines
+    #   host1
+    #   host2
+    #   host3
+    # Assuming you can login them without any password, i.e. You have to set ssh keys.
+
+    export train_cmd="ssh.pl"
+    export cuda_cmd="ssh.pl"
+    export decode_cmd="ssh.pl"
+
+# This is an example of specifying several unique options in the JHU CLSP cluster setup.
+# Users can modify/add their own command options according to their cluster environments.
+elif [ "${cmd_backend}" = jhu ]; then
+
+    export train_cmd="queue.pl --mem 2G"
+    export cuda_cmd="queue-freegpu.pl --mem 2G --gpu 1 --config conf/gpu.conf"
+    export decode_cmd="queue.pl --mem 4G"
+
+else
+    echo "$0: Error: Unknown cmd_backend=${cmd_backend}" 1>&2
+    return 1
+fi
diff --git a/examples/librispeech/s2/local/test.sh b/examples/librispeech/s2/local/test.sh
index 893d67b5b..09644bb2c 100755
--- a/examples/librispeech/s2/local/test.sh
+++ b/examples/librispeech/s2/local/test.sh
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+expdir=exp
+datadir=data
+
+lmtag=
+
+recog_set="test-clean test-other dev-clean dev-other"
+recog_set="test-clean"
+
+# bpemode (unigram or bpe)
+nbpe=5000
+bpemode=unigram
+bpeprefix="data/bpe_${bpemode}_${nbpe}"
+bpemodel=${bpeprefix}.model
+
 if [ $# != 3 ];then
     echo "usage: ${0} config_path dict_path ckpt_path_prefix"
     exit -1
@@ -8,13 +22,8 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 config_path=$1
-dict_path=$2
+dict=$2
 ckpt_prefix=$3
 
 chunk_mode=false
@@ -30,50 +39,49 @@ echo "chunk mode ${chunk_mode}"
 #    exit 1
 #fi
 
-for type in attention ctc_greedy_search; do
-    echo "decoding ${type}"
-    if [ ${chunk_mode} == true ];then
-        # stream decoding only support batchsize=1
-        batch_size=1
-    else
+pids=() # initialize pids
+
+for dmethd in attention ctc_greedy_search ctc_prefix_beam_search attention_rescoring; do
+(
+    for rtask in ${recog_set}; do
+    (
+        decode_dir=decode_${rtask}_${dmethd}_$(basename ${config_path%.*})_${lmtag}
+        feat_recog_dir=${datadir}
+        mkdir -p ${expdir}/${decode_dir}
+        mkdir -p ${feat_recog_dir}
+
+        # split data
+        split_json.sh ${feat_recog_dir}/manifest.${rtask} ${nj}
+
+        #### use CPU for decoding
+        ngpu=0
+
+        # set batchsize 0 to disable batch decoding
         batch_size=1
-    fi
-    python3 -u ${BIN_DIR}/test.py \
-    --model-name u2_kaldi \
-    --run-mode test \
-    --dict-path ${dict_path} \
-    --device ${device} \
-    --nproc 1 \
-    --config ${config_path} \
-    --result-file ${ckpt_prefix}.${type}.rsl \
-    --checkpoint_path ${ckpt_prefix} \
-    --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
-
-    if [ $? -ne 0 ]; then
-        echo "Failed in evaluation!"
-        exit 1
-    fi
-done
+        ${decode_cmd} JOB=1:${nj} ${expdir}/${decode_dir}/log/decode.JOB.log \
+            python3 -u ${BIN_DIR}/test.py \
+            --model-name u2_kaldi \
+            --run-mode test \
+            --nproc ${ngpu} \
+            --dict-path ${dict} \
+            --config ${config_path} \
+            --checkpoint_path ${ckpt_prefix} \
+            --result-file ${expdir}/${decode_dir}/data.JOB.json \
+            --opts decoding.decoding_method ${dmethd} \
+            --opts decoding.batch_size ${batch_size} \
+            --opts data.test_manifest ${feat_recog_dir}/split${nj}/JOB/manifest.${rtask}
+
+        score_sclite.sh --bpe ${nbpe} --bpemodel ${bpemodel}.model --wer true ${expdir}/${decode_dir} ${dict}
 
-for type in ctc_prefix_beam_search attention_rescoring; do
-    echo "decoding ${type}"
-    batch_size=1
-    python3 -u ${BIN_DIR}/test.py \
-    --model-name u2_kaldi \
-    --run-mode test \
-    --dict-path ${dict_path} \
-    --device ${device} \
-    --nproc 1 \
-    --config ${config_path} \
-    --result-file ${ckpt_prefix}.${type}.rsl \
-    --checkpoint_path ${ckpt_prefix} \
-    --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
-
-    if [ $? -ne 0 ]; then
-        echo "Failed in evaluation!"
-        exit 1
-    fi
+    ) &
+    pids+=($!) # store background pids
+    done
+) &
+pids+=($!) # store background pids
 done
 
+i=0; for pid in "${pids[@]}"; do wait ${pid} || ((++i)); done
+[ ${i} -gt 0 ] && echo "$0: ${i} background jobs are failed." && false
+echo "Finished"
 
 exit 0
diff --git a/examples/librispeech/s2/run.sh b/examples/librispeech/s2/run.sh
index 8dd937369..46b6ac1b4 100755
--- a/examples/librispeech/s2/run.sh
+++ b/examples/librispeech/s2/run.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
-source path.sh
+
+. ./path.sh || exit 1;
+. ./cmd.sh || exit 1;
 
 stage=0
 stop_stage=100
diff --git a/utils/README.md b/utils/README.md
new file mode 100644
index 000000000..d48faf699
--- /dev/null
+++ b/utils/README.md
@@ -0,0 +1,3 @@
+# Utils
+
+* [kaldi utils](https://github.com/kaldi-asr/kaldi/blob/cbed4ff688/egs/wsj/s5/utils)
\ No newline at end of file
diff --git a/utils/build_kenlm_model_from_arpa.sh b/utils/build_kenlm_model_from_arpa.sh
new file mode 100755
index 000000000..9e1113b89
--- /dev/null
+++ b/utils/build_kenlm_model_from_arpa.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# 2020 author Jiayu DU
+# Apache 2.0
+
+# This script reads in an Arpa format language model, and converts it into the
+# KenLM format language model.
+
+[ -f path.sh ] && . ./path.sh;
+
+# begin configuration section
+kenlm_opts="" # e.g. "-q 8 -b 8" for 8bits quantization
+model_type="trie" # "trie" or "probing". trie is smaller, probing is faster.
+# end configuration section
+
+. utils/parse_options.sh
+
+if [ $# != 2 ]; then
+  echo "Usage: "
+  echo "  $0 [options] <arpa-lm-path> <kenlm-path>"
+  echo "e.g.:"
+  echo "  $0 data/local/lm/4gram.arpa data/lang_test/G.trie"
+  echo "Options:"
+  echo "  --model-type can be either \"trie\" or \"probing\""
+  echo "  --kenlm-opts directly pass through to kenlm"
+  echo "    e.g. for 8bits quantization, feed \"-q 8 -b 8\""
+  exit 1;
+fi
+
+export LC_ALL=C
+
+arpa_lm=$1
+kenlm=$2
+
+if ! which build_binary >& /dev/null ; then
+  echo "$0: cannot find KenLM's build_binary tool,"
+  echo "check kenlm installation (tools/extras/install_kenlm_query_only.sh)."
+  exit 1
+fi
+
+mkdir -p $(dirname $kenlm)
+build_binary  $kenlm_opts  $model_type  $arpa_lm  $kenlm
+
+echo "$0: Successfully built arpa into kenlm format: $kenlm"
+exit 0
\ No newline at end of file
diff --git a/utils/duration_from_maniefst.sh b/utils/duration_from_maniefst.sh
old mode 100644
new mode 100755
diff --git a/utils/log.sh b/utils/log.sh
old mode 100644
new mode 100755
diff --git a/utils/parallel/run.pl b/utils/parallel/run.pl
new file mode 100755
index 000000000..c89eb3da8
--- /dev/null
+++ b/utils/parallel/run.pl
@@ -0,0 +1,356 @@
+#!/usr/bin/env perl
+use warnings; #sed replacement for -w perl parameter
+# In general, doing
+#  run.pl some.log a b c is like running the command a b c in
+# the bash shell, and putting the standard error and output into some.log.
+# To run parallel jobs (backgrounded on the host machine), you can do (e.g.)
+#  run.pl JOB=1:4 some.JOB.log a b c JOB is like running the command a b c JOB
+# and putting it in some.JOB.log, for each one. [Note: JOB can be any identifier].
+# If any of the jobs fails, this script will fail.
+
+# A typical example is:
+#  run.pl some.log my-prog "--opt=foo bar" foo \|  other-prog baz
+# and run.pl will run something like:
+# ( my-prog '--opt=foo bar' foo |  other-prog baz ) >& some.log
+#
+# Basically it takes the command-line arguments, quotes them
+# as necessary to preserve spaces, and evaluates them with bash.
+# In addition it puts the command line at the top of the log, and
+# the start and end times of the command at the beginning and end.
+# The reason why this is useful is so that we can create a different
+# version of this program that uses a queueing system instead.
+
+#use Data::Dumper;
+
+@ARGV < 2 && die "usage: run.pl log-file command-line arguments...";
+
+#print STDERR "COMMAND-LINE: " .  Dumper(\@ARGV) . "\n";
+$job_pick = 'all';
+$max_jobs_run = -1;
+$jobstart = 1;
+$jobend = 1;
+$ignored_opts = ""; # These will be ignored.
+
+# First parse an option like JOB=1:4, and any
+# options that would normally be given to
+# queue.pl, which we will just discard.
+
+for (my $x = 1; $x <= 2; $x++) { # This for-loop is to
+  # allow the JOB=1:n option to be interleaved with the
+  # options to qsub.
+  while (@ARGV >= 2 && $ARGV[0] =~ m:^-:) {
+    # parse any options that would normally go to qsub, but which will be ignored here.
+    my $switch = shift @ARGV;
+    if ($switch eq "-V") {
+      $ignored_opts .= "-V ";
+    } elsif ($switch eq "--max-jobs-run" || $switch eq "-tc") {
+      # we do support the option --max-jobs-run n, and its GridEngine form -tc n.
+      # if the command appears multiple times uses the smallest option.
+      if ( $max_jobs_run <= 0 ) {
+          $max_jobs_run =  shift @ARGV;
+      } else {
+        my $new_constraint = shift @ARGV;
+        if ( ($new_constraint < $max_jobs_run) ) {
+          $max_jobs_run = $new_constraint;
+        }
+      }
+      
+      if (! ($max_jobs_run > 0)) {
+        die "run.pl: invalid option --max-jobs-run $max_jobs_run";
+      }
+    } else {
+      my $argument = shift @ARGV;
+      if ($argument =~ m/^--/) {
+        print STDERR "run.pl: WARNING: suspicious argument '$argument' to $switch; starts with '-'\n";
+      }
+      if ($switch eq "-sync" && $argument =~ m/^[yY]/) {
+        $ignored_opts .= "-sync "; # Note: in the
+        # corresponding code in queue.pl it says instead, just "$sync = 1;".
+      } elsif ($switch eq "-pe") { # e.g. -pe smp 5
+        my $argument2 = shift @ARGV;
+        $ignored_opts .= "$switch $argument $argument2 ";
+      } elsif ($switch eq "--gpu") {
+        $using_gpu = $argument;
+      } elsif ($switch eq "--pick") {
+        if($argument =~ m/^(all|failed|incomplete)$/) {
+          $job_pick = $argument;
+        } else {
+          print STDERR "run.pl: ERROR: --pick argument must be one of 'all', 'failed' or 'incomplete'"
+        }
+      } else {
+        # Ignore option.
+        $ignored_opts .= "$switch $argument ";
+      }
+    }
+  }
+  if ($ARGV[0] =~ m/^([\w_][\w\d_]*)+=(\d+):(\d+)$/) { # e.g. JOB=1:20
+    $jobname = $1;
+    $jobstart = $2;
+    $jobend = $3;
+    if ($jobstart > $jobend) {
+      die "run.pl: invalid job range $ARGV[0]";
+    }
+    if ($jobstart <= 0) {
+      die "run.pl: invalid job range $ARGV[0], start must be strictly positive (this is required for GridEngine compatibility).";
+    }
+    shift;
+  } elsif ($ARGV[0] =~ m/^([\w_][\w\d_]*)+=(\d+)$/) { # e.g. JOB=1.
+    $jobname = $1;
+    $jobstart = $2;
+    $jobend = $2;
+    shift;
+  } elsif ($ARGV[0] =~ m/.+\=.*\:.*$/) {
+    print STDERR "run.pl: Warning: suspicious first argument to run.pl: $ARGV[0]\n";
+  }
+}
+
+# Users found this message confusing so we are removing it.
+# if ($ignored_opts ne "") {
+#   print STDERR "run.pl: Warning: ignoring options \"$ignored_opts\"\n";
+# }
+
+if ($max_jobs_run == -1) { # If --max-jobs-run option not set,
+                           # then work out the number of processors if possible,
+                           # and set it based on that.
+  $max_jobs_run = 0;
+  if ($using_gpu) {
+    if (open(P, "nvidia-smi -L |")) {
+      $max_jobs_run++ while (<P>);
+      close(P);
+    }
+    if ($max_jobs_run == 0) {
+      $max_jobs_run = 1;
+      print STDERR "run.pl: Warning: failed to detect number of GPUs from nvidia-smi, using ${max_jobs_run}\n";
+    }
+  } elsif (open(P, "</proc/cpuinfo")) {  # Linux
+    while (<P>) { if (m/^processor/) { $max_jobs_run++; } }
+    if ($max_jobs_run == 0) {
+      print STDERR "run.pl: Warning: failed to detect any processors from /proc/cpuinfo\n";
+      $max_jobs_run = 10;  # reasonable default.
+    }
+    close(P);
+  } elsif (open(P, "sysctl -a |")) {  # BSD/Darwin
+    while (<P>) {
+      if (m/hw\.ncpu\s*[:=]\s*(\d+)/) { # hw.ncpu = 4, or hw.ncpu: 4
+        $max_jobs_run = $1;
+        last;
+      }
+    }
+    close(P);
+    if ($max_jobs_run == 0) {
+      print STDERR "run.pl: Warning: failed to detect any processors from sysctl -a\n";
+      $max_jobs_run = 10;  # reasonable default.
+    }
+  } else {
+    # allow at most 32 jobs at once, on non-UNIX systems; change this code
+    # if you need to change this default.
+    $max_jobs_run = 32;
+  }
+  # The just-computed value of $max_jobs_run is just the number of processors
+  # (or our best guess); and if it happens that the number of jobs we need to
+  # run is just slightly above $max_jobs_run, it will make sense to increase
+  # $max_jobs_run to equal the number of jobs, so we don't have a small number
+  # of leftover jobs.
+  $num_jobs = $jobend - $jobstart + 1;
+  if (!$using_gpu &&
+      $num_jobs > $max_jobs_run && $num_jobs < 1.4 * $max_jobs_run) {
+    $max_jobs_run = $num_jobs;
+  }
+}
+
+sub pick_or_exit {
+  # pick_or_exit ( $logfile ) 
+  # Invoked before each job is started helps to run jobs selectively.
+  #
+  # Given the name of the output logfile decides whether the job must be 
+  # executed (by returning from the subroutine) or not (by terminating the
+  # process calling exit)
+  # 
+  # PRE: $job_pick is a global variable set by command line switch --pick
+  #      and indicates which class of jobs must be executed.
+  #
+  # 1) If a failed job is not executed the process exit code will indicate 
+  #    failure, just as if the task was just executed  and failed.
+  #
+  # 2) If a task is incomplete it will be executed. Incomplete may be either
+  #    a job whose log file does not contain the accounting notes in the end,
+  #    or a job whose log file does not exist.
+  #
+  # 3) If the $job_pick is set to 'all' (default behavior) a task will be
+  #    executed regardless of the result of previous attempts.
+  #
+  # This logic could have been implemented in the main execution loop
+  # but a subroutine to preserve the current level of readability of
+  # that part of the code.
+  #
+  # Alexandre Felipe, (o.alexandre.felipe@gmail.com) 14th of August of 2020
+  #
+  if($job_pick eq 'all'){
+    return; # no need to bother with the previous log
+  }
+  open my $fh, "<", $_[0] or return; # job not executed yet
+  my $log_line;
+  my $cur_line;
+  while ($cur_line = <$fh>) {
+    if( $cur_line =~ m/# Ended \(code .*/ ) {
+      $log_line = $cur_line;
+    }
+  }
+  close $fh;
+  if (! defined($log_line)){
+    return; # incomplete
+  }
+  if ( $log_line =~ m/# Ended \(code 0\).*/ ) {
+    exit(0); # complete
+  } elsif ( $log_line =~ m/# Ended \(code \d+(; signal \d+)?\).*/ ){
+    if ($job_pick !~ m/^(failed|all)$/) {
+      exit(1); # failed but not going to run
+    } else {
+      return; # failed
+    }
+  } elsif ( $log_line =~ m/.*\S.*/ ) {
+    return; # incomplete jobs are always run
+  }
+}
+
+
+$logfile = shift @ARGV;
+
+if (defined $jobname && $logfile !~ m/$jobname/ &&
+    $jobend > $jobstart) {
+  print STDERR "run.pl: you are trying to run a parallel job but "
+    . "you are putting the output into just one log file ($logfile)\n";
+  exit(1);
+}
+
+$cmd = "";
+
+foreach $x (@ARGV) {
+    if ($x =~ m/^\S+$/) { $cmd .=  $x . " "; }
+    elsif ($x =~ m:\":) { $cmd .= "'$x' "; }
+    else { $cmd .= "\"$x\" "; }
+}
+
+#$Data::Dumper::Indent=0;
+$ret = 0;
+$numfail = 0;
+%active_pids=();
+
+use POSIX ":sys_wait_h";
+for ($jobid = $jobstart; $jobid <= $jobend; $jobid++) {
+  if (scalar(keys %active_pids) >= $max_jobs_run) {
+
+    # Lets wait for a change in any child's status
+    # Then we have to work out which child finished
+    $r = waitpid(-1, 0);
+    $code = $?;
+    if ($r < 0 ) { die "run.pl: Error waiting for child process"; } # should never happen.
+    if ( defined $active_pids{$r} ) {
+        $jid=$active_pids{$r};
+        $fail[$jid]=$code;
+        if ($code !=0) { $numfail++;}
+        delete $active_pids{$r};
+        # print STDERR "Finished: $r/$jid " .  Dumper(\%active_pids) . "\n";
+    } else {
+        die "run.pl: Cannot find the PID of the child process that just finished.";
+    }
+
+    # In theory we could do a non-blocking waitpid over all jobs running just
+    # to find out if only one or more jobs finished during the previous waitpid()
+    # However, we just omit this and will reap the next one in the next pass
+    # through the for(;;) cycle
+  }
+  $childpid = fork();
+  if (!defined $childpid) { die "run.pl: Error forking in run.pl (writing to $logfile)"; }
+  if ($childpid == 0) { # We're in the child... this branch
+    # executes the job and returns (possibly with an error status).
+    if (defined $jobname) {
+      $cmd =~ s/$jobname/$jobid/g;
+      $logfile =~ s/$jobname/$jobid/g;
+    }
+    # exit if the job does not need to be executed
+    pick_or_exit( $logfile );
+
+    system("mkdir -p `dirname $logfile` 2>/dev/null");
+    open(F, ">$logfile") || die "run.pl: Error opening log file $logfile";
+    print F "# " . $cmd . "\n";
+    print F "# Started at " . `date`;
+    $starttime = `date +'%s'`;
+    print F "#\n";
+    close(F);
+
+    # Pipe into bash.. make sure we're not using any other shell.
+    open(B, "|bash") || die "run.pl: Error opening shell command";
+    print B "( " . $cmd . ") 2>>$logfile >> $logfile";
+    close(B);                   # If there was an error, exit status is in $?
+    $ret = $?;
+
+    $lowbits = $ret & 127;
+    $highbits = $ret >> 8;
+    if ($lowbits != 0) { $return_str = "code $highbits; signal $lowbits" }
+    else { $return_str = "code $highbits"; }
+
+    $endtime = `date +'%s'`;
+    open(F, ">>$logfile") || die "run.pl: Error opening log file $logfile (again)";
+    $enddate = `date`;
+    chop $enddate;
+    print F "# Accounting: time=" . ($endtime - $starttime) . " threads=1\n";
+    print F "# Ended ($return_str) at " . $enddate . ", elapsed time " . ($endtime-$starttime) . " seconds\n";
+    close(F);
+    exit($ret == 0 ? 0 : 1);
+  } else {
+    $pid[$jobid] = $childpid;
+    $active_pids{$childpid} = $jobid;
+    # print STDERR "Queued: " .  Dumper(\%active_pids) . "\n";
+  }
+}
+
+# Now we have submitted all the jobs, lets wait until all the jobs finish
+foreach $child (keys %active_pids) {
+    $jobid=$active_pids{$child};
+    $r = waitpid($pid[$jobid], 0);
+    $code = $?;
+    if ($r == -1) { die "run.pl: Error waiting for child process"; } # should never happen.
+    if ($r != 0) { $fail[$jobid]=$code; $numfail++ if $code!=0; } # Completed successfully
+}
+
+# Some sanity checks:
+# The $fail array should not contain undefined codes
+# The number of non-zeros in that array  should be equal to $numfail
+# We cannot do foreach() here, as the JOB ids do not start at zero
+$failed_jids=0;
+for ($jobid = $jobstart; $jobid <= $jobend; $jobid++) {
+  $job_return = $fail[$jobid];
+  if (not defined $job_return ) {
+    # print Dumper(\@fail);
+
+    die "run.pl: Sanity check failed: we have indication that some jobs are running " .
+      "even after we waited for all jobs to finish" ;
+  }
+  if ($job_return != 0 ){ $failed_jids++;}
+}
+if ($failed_jids != $numfail) {
+  die "run.pl: Sanity check failed: cannot find out how many jobs failed ($failed_jids x $numfail)."
+}
+if ($numfail > 0) { $ret = 1; }
+
+if ($ret != 0) {
+  $njobs = $jobend - $jobstart + 1;
+  if ($njobs == 1) {
+    if (defined $jobname) {
+      $logfile =~ s/$jobname/$jobstart/; # only one numbered job, so replace name with
+                                         # that job.
+    }
+    print STDERR "run.pl: job failed, log is in $logfile\n";
+    if ($logfile =~ m/JOB/) {
+      print STDERR "run.pl: probably you forgot to put JOB=1:\$nj in your script.";
+    }
+  }
+  else {
+    $logfile =~ s/$jobname/*/g;
+    print STDERR "run.pl: $numfail / $njobs failed, log is in $logfile\n";
+  }
+}
+
+
+exit ($ret);
\ No newline at end of file
diff --git a/utils/parse_options.sh b/utils/parse_options.sh
old mode 100644
new mode 100755
diff --git a/utils/pd_env_collect.sh b/utils/pd_env_collect.sh
old mode 100644
new mode 100755
diff --git a/utils/profile.sh b/utils/profile.sh
old mode 100644
new mode 100755
diff --git a/utils/run.pl b/utils/run.pl
new file mode 100755
index 000000000..8b84c7f08
--- /dev/null
+++ b/utils/run.pl
@@ -0,0 +1 @@
+parallel/run.pl
\ No newline at end of file
diff --git a/utils/score_sclite.sh b/utils/score_sclite.sh
new file mode 100755
index 000000000..3aa4cd072
--- /dev/null
+++ b/utils/score_sclite.sh
@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+
+# Copyright 2017 Johns Hopkins University (Shinji Watanabe)
+#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
+
+[ -f ./path.sh ] && . ./path.sh
+
+nlsyms=""
+wer=false
+bpe=""
+bpemodel=""
+remove_blank=true
+filter=""
+num_spkrs=1
+help_message="Usage: $0 <data-dir> <dict>"
+
+. utils/parse_options.sh
+
+if [ $# != 2 ]; then
+    echo "${help_message}"
+    exit 1;
+fi
+
+dir=$1
+dic=$2
+
+concatjson.py ${dir}/data.*.json > ${dir}/data.json
+
+if [ $num_spkrs -eq 1 ]; then
+  json2trn.py ${dir}/data.json ${dic} --num-spkrs ${num_spkrs} --refs ${dir}/ref.trn --hyps ${dir}/hyp.trn
+
+  if ${remove_blank}; then
+      sed -i.bak2 -r 's/<blank> //g' ${dir}/hyp.trn
+  fi
+  if [ -n "${nlsyms}" ]; then
+      cp ${dir}/ref.trn ${dir}/ref.trn.org
+      cp ${dir}/hyp.trn ${dir}/hyp.trn.org
+      filt.py -v ${nlsyms} ${dir}/ref.trn.org > ${dir}/ref.trn
+      filt.py -v ${nlsyms} ${dir}/hyp.trn.org > ${dir}/hyp.trn
+  fi
+  if [ -n "${filter}" ]; then
+      sed -i.bak3 -f ${filter} ${dir}/hyp.trn
+      sed -i.bak3 -f ${filter} ${dir}/ref.trn
+  fi
+
+  sclite -r ${dir}/ref.trn trn -h ${dir}/hyp.trn trn -i rm -o all stdout > ${dir}/result.txt
+
+  echo "write a CER (or TER) result in ${dir}/result.txt"
+  grep -e Avg -e SPKR -m 2 ${dir}/result.txt
+
+  if ${wer}; then
+      if [ -n "$bpe" ]; then
+  	    spm_decode --model=${bpemodel} --input_format=piece < ${dir}/ref.trn | sed -e "s/▁/ /g" > ${dir}/ref.wrd.trn
+  	    spm_decode --model=${bpemodel} --input_format=piece < ${dir}/hyp.trn | sed -e "s/▁/ /g" > ${dir}/hyp.wrd.trn
+      else
+  	    sed -e "s/ //g" -e "s/(/ (/" -e "s/<space>/ /g" ${dir}/ref.trn > ${dir}/ref.wrd.trn
+  	    sed -e "s/ //g" -e "s/(/ (/" -e "s/<space>/ /g" ${dir}/hyp.trn > ${dir}/hyp.wrd.trn
+      fi
+      sclite -r ${dir}/ref.wrd.trn trn -h ${dir}/hyp.wrd.trn trn -i rm -o all stdout > ${dir}/result.wrd.txt
+
+      echo "write a WER result in ${dir}/result.wrd.txt"
+      grep -e Avg -e SPKR -m 2 ${dir}/result.wrd.txt
+  fi
+elif [ ${num_spkrs} -lt 4 ]; then
+  ref_trns=""
+  hyp_trns=""
+  for i in $(seq ${num_spkrs}); do
+      ref_trns=${ref_trns}"${dir}/ref${i}.trn "
+      hyp_trns=${hyp_trns}"${dir}/hyp${i}.trn "
+  done
+  json2trn.py ${dir}/data.json ${dic} --num-spkrs ${num_spkrs} --refs ${ref_trns} --hyps ${hyp_trns}
+
+  for n in $(seq ${num_spkrs}); do
+      if ${remove_blank}; then
+          sed -i.bak2 -r 's/<blank> //g' ${dir}/hyp${n}.trn
+      fi
+      if [ -n "${nlsyms}" ]; then
+          cp ${dir}/ref${n}.trn ${dir}/ref${n}.trn.org
+          cp ${dir}/hyp${n}.trn ${dir}/hyp${n}.trn.org
+          filt.py -v ${nlsyms} ${dir}/ref${n}.trn.org > ${dir}/ref${n}.trn
+          filt.py -v ${nlsyms} ${dir}/hyp${n}.trn.org > ${dir}/hyp${n}.trn
+      fi
+      if [ -n "${filter}" ]; then
+          sed -i.bak3 -f ${filter} ${dir}/hyp${n}.trn
+          sed -i.bak3 -f ${filter} ${dir}/ref${n}.trn
+      fi
+  done
+
+  results_str=""
+  for (( i=0; i<$((num_spkrs * num_spkrs)); i++ )); do
+      ind_r=$((i / num_spkrs + 1))
+      ind_h=$((i % num_spkrs + 1))
+      results_str=${results_str}"${dir}/result_r${ind_r}h${ind_h}.txt "
+      sclite -r ${dir}/ref${ind_r}.trn trn -h ${dir}/hyp${ind_h}.trn trn -i rm -o all stdout > ${dir}/result_r${ind_r}h${ind_h}.txt
+  done
+
+  echo "write CER (or TER) results in ${dir}/result_r*h*.txt"
+  eval_perm_free_error.py --num-spkrs ${num_spkrs} \
+      ${results_str} > ${dir}/min_perm_result.json
+  sed -n '2,4p' ${dir}/min_perm_result.json
+
+  if ${wer}; then
+      for n in $(seq ${num_spkrs}); do
+          if [ -n "$bpe" ]; then
+              spm_decode --model=${bpemodel} --input_format=piece < ${dir}/ref${n}.trn | sed -e "s/▁/ /g" > ${dir}/ref${n}.wrd.trn
+              spm_decode --model=${bpemodel} --input_format=piece < ${dir}/hyp${n}.trn | sed -e "s/▁/ /g" > ${dir}/hyp${n}.wrd.trn
+          else
+              sed -e "s/ //g" -e "s/(/ (/" -e "s/<space>/ /g" ${dir}/ref${n}.trn > ${dir}/ref${n}.wrd.trn
+              sed -e "s/ //g" -e "s/(/ (/" -e "s/<space>/ /g" ${dir}/hyp${n}.trn > ${dir}/hyp${n}.wrd.trn
+          fi
+      done
+      results_str=""
+      for (( i=0; i<$((num_spkrs * num_spkrs)); i++ )); do
+          ind_r=$((i / num_spkrs + 1))
+          ind_h=$((i % num_spkrs + 1))
+          results_str=${results_str}"${dir}/result_r${ind_r}h${ind_h}.wrd.txt "
+          sclite -r ${dir}/ref${ind_r}.wrd.trn trn -h ${dir}/hyp${ind_h}.wrd.trn trn -i rm -o all stdout > ${dir}/result_r${ind_r}h${ind_h}.wrd.txt
+      done
+
+      echo "write WER results in ${dir}/result_r*h*.wrd.txt"
+      eval_perm_free_error.py --num-spkrs ${num_spkrs} \
+          ${results_str} > ${dir}/min_perm_result.wrd.json
+      sed -n '2,4p' ${dir}/min_perm_result.wrd.json
+  fi
+fi
diff --git a/utils/spk2utt_to_utt2spk.pl b/utils/spk2utt_to_utt2spk.pl
new file mode 100755
index 000000000..9f7fb335c
--- /dev/null
+++ b/utils/spk2utt_to_utt2spk.pl
@@ -0,0 +1,25 @@
+#!/usr/bin/env perl
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+while(<>){ 
+    @A = split(" ", $_);
+    @A > 1 || die "Invalid line in spk2utt file: $_";
+    $s = shift @A;
+    foreach $u ( @A ) {
+        print "$u $s\n";
+    }
+}
diff --git a/utils/split_data.sh b/utils/split_data.sh
new file mode 100755
index 000000000..8e6c5d5bf
--- /dev/null
+++ b/utils/split_data.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+
+if [ $# != 2 ]; then
+  echo "Usage: split_data.sh data-dir num-to-split"
+  exit 1
+fi
+
+data=$1
+numsplit=$2
+
+if [ $numsplit -le 0 ]; then
+  echo "Invalid num-split argument $numsplit";
+  exit 1;
+fi
+
+n=0;
+feats=""
+wavs=""
+utt2spks=""
+texts=""
+
+nu=`cat $data/utt2spk | wc -l`
+nf=`cat $data/feats.scp | wc -l`
+nt=`cat $data/text | wc -l`
+if [ $nu -ne $nf ]; then
+  echo "split_data.sh: warning, #lines is (utt2spk,feats.scp) is ($nu,$nf);"
+  echo "this script may produce incorrectly split data."
+  echo "use utils/fix_data_dir.sh to fix this."
+fi
+if [ $nt -ne 0 -a $nu -ne $nt ]; then
+  echo "split_data.sh: warning, #lines is (utt2spk,text) is ($nu,$nt);"
+  echo "this script may produce incorrectly split data."
+  echo "use utils/fix_data_dir.sh to fix this."
+fi
+
+# utilsscripts/get_split.pl returns "0 1 2 3" or "00 01 .. 18 19" or whatever.
+# for n in `get_splits.pl $numsplit`; do
+for n in `seq 1 $numsplit`; do  # Changed this to usual number sequence -Arnab
+  mkdir -p $data/split$numsplit/$n
+  feats="$feats $data/split$numsplit/$n/feats.scp"
+  wavs="$wavs $data/split$numsplit/$n/wav.scp"
+  texts="$texts $data/split$numsplit/$n/text"
+  utt2spks="$utt2spks $data/split$numsplit/$n/utt2spk"
+done
+
+split_scp.pl --utt2spk=$data/utt2spk $data/utt2spk $utt2spks
+split_scp.pl --utt2spk=$data/utt2spk $data/feats.scp $feats
+[ -f $data/wav.scp ] && \
+  split_scp.pl --utt2spk=$data/utt2spk $data/wav.scp $wavs
+[ -f $data/text ] && \
+  split_scp.pl --utt2spk=$data/utt2spk $data/text $texts
+
+# for n in `get_splits.pl $numsplit`; do
+for n in `seq 1 $numsplit`; do  # Changed this to usual number sequence -Arnab
+  utt2spk_to_spk2utt.pl $data/split$numsplit/$n/utt2spk \
+    > $data/split$numsplit/$n/spk2utt
+  # for completeness, also split the spk2gender file
+  [ -f $data/spk2gender ] && \
+    filter_scp.pl $data/split$numsplit/$n/spk2utt $data/spk2gender \
+    > $data/split$numsplit/$n/spk2gender 
+done
+
+exit 0
\ No newline at end of file
diff --git a/utils/split_json.sh b/utils/split_json.sh
new file mode 100755
index 000000000..6c7d38b91
--- /dev/null
+++ b/utils/split_json.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -o errexit
+
+if [ $# != 2 ]; then
+  echo "Usage: split_data.sh manifest num-to-split"
+  exit 1
+fi
+
+data=data
+
+jsonfile=$1
+numsplit=$2
+
+if [ $numsplit -le 0 ]; then
+  echo "Invalid num-split argument $numsplit";
+  exit 1;
+fi
+
+n=0;
+jsons=""
+
+# utilsscripts/get_split.pl returns "0 1 2 3" or "00 01 .. 18 19" or whatever.
+# for n in `get_splits.pl $numsplit`; do
+for n in `seq 1 $numsplit`; do  # Changed this to usual number sequence -Arnab
+  mkdir -p $data/split$numsplit/$n
+  jsons="$jsons $data/split$numsplit/$n/${jsonfile}"
+done
+
+split_scp.pl $data/${jsonfile} $jsons
+
+exit 0
\ No newline at end of file
diff --git a/utils/split_scp.pl  b/utils/split_scp.pl 
new file mode 100644
index 000000000..fc28e0b65
--- /dev/null
+++ b/utils/split_scp.pl 	
@@ -0,0 +1,212 @@
+#!/usr/bin/env perl
+use warnings; #sed replacement for -w perl parameter
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+# This program splits up any kind of .scp or archive-type file.
+# If there is no utt2spk option it will work on any text  file and
+# will split it up with an approximately equal number of lines in
+# each but.
+# With the --utt2spk option it will work on anything that has the 
+# utterance-id as the first entry on each line; the utt2spk file is
+# of the form "utterance speaker" (on each line).
+# It splits it into equal size chunks as far as it can.  If you use
+# the utt2spk option it will make sure these chunks coincide with
+# speaker boundaries.  In this case, if there are more chunks
+# than speakers (and in some other circumstances), some of the 
+# resulting  chunks will be empty and it
+# will print a warning.
+# You will normally call this like:
+# split_scp.pl scp scp.1 scp.2 scp.3 ...
+# or
+# split_scp.pl --utt2spk=utt2spk scp scp.1 scp.2 scp.3 ...
+# Note that you can use this script to split the utt2spk file itself,
+# e.g. split_scp.pl --utt2spk=utt2spk utt2spk utt2spk.1 utt2spk.2 ...
+
+# You can also call the scripts like:
+# split_scp.pl -j 3 0 scp scp.0
+# [note: with this option, it assumes zero-based indexing of the split parts,
+# i.e. the second number must be 0 <= n < num-jobs.]
+
+$num_jobs = 0;
+$job_id = 0;
+$utt2spk_file = "";
+
+for ($x = 1; $x <= 2; $x++) {
+    if ($ARGV[0] eq "-j") {
+        shift @ARGV;
+        $num_jobs = shift @ARGV;
+        $job_id = shift @ARGV;
+        if ($num_jobs <= 0 || $job_id < 0 || $job_id >= $num_jobs) {
+            die "Invalid num-jobs and job-id: $num_jobs and $job_id";
+        }
+    }
+    if ($ARGV[0] =~ "--utt2spk=(.+)") {
+        $utt2spk_file=$1;
+        shift;
+    }
+}
+
+if(($num_jobs == 0 && @ARGV < 2) || ($num_jobs > 0 && (@ARGV < 1 || @ARGV > 2))) {
+    die "Usage: split_scp.pl [--utt2spk=<utt2spk_file>] in.scp out1.scp out2.scp ... \n" .
+        " or: split_scp.pl -j num-jobs job-id [--utt2spk=<utt2spk_file>] in.scp [out.scp]\n" .
+        " ... where 0 <= job-id < num-jobs.";
+}
+   
+$inscp = shift @ARGV;
+if ($num_jobs == 0) { # without -j option
+    @OUTPUTS = @ARGV;
+} else {
+    for ($j = 0; $j < $num_jobs; $j++) {
+        if ($j == $job_id) { 
+            if (@ARGV > 0) { push @OUTPUTS, $ARGV[0]; }
+            else { push @OUTPUTS, "-"; }
+        } else {
+            push @OUTPUTS, "/dev/null";
+        }
+    }
+} 
+
+if ($utt2spk_file ne "") {  # We have the --utt2spk option...
+    open(U, "<$utt2spk_file") || die "Failed to open utt2spk file $utt2spk_file";
+    while(<U>) {
+        @A = split;
+        @A == 2 || die "Bad line $_ in utt2spk file $utt2spk_file";
+        ($u,$s) = @A;
+        $utt2spk{$u} = $s;
+    }
+    open(I, "<$inscp") || die "Opening input scp file $inscp";
+    @spkrs = ();
+    while(<I>) {
+        @A = split;
+        if(@A == 0) { die "Empty or space-only line in scp file $inscp"; }
+        $u = $A[0];
+        $s = $utt2spk{$u};
+        if(!defined $s) { die "No such utterance $u in utt2spk file $utt2spk_file"; }
+        if(!defined $spk_count{$s}) { 
+            push @spkrs, $s; 
+            $spk_count{$s} = 0;
+            $spk_data{$s} = "";
+        }
+        $spk_count{$s}++;
+        $spk_data{$s} = $spk_data{$s} . $_;
+    }
+    # Now split as equally as possible ..
+    # First allocate spks to files by allocating an approximately
+    # equal number of speakers.
+    $numspks = @spkrs;  # number of speakers.
+    $numscps = @OUTPUTS; # number of output files.
+    for($scpidx = 0; $scpidx < $numscps; $scpidx++) {
+        $scparray[$scpidx] = []; # [] is array reference.
+    }
+    for ($spkidx = 0; $spkidx < $numspks; $spkidx++) {
+        $scpidx = int(($spkidx*$numscps) / $numspks);
+        $spk = $spkrs[$spkidx];
+        push @{$scparray[$scpidx]}, $spk;
+        $scpcount[$scpidx] += $spk_count{$spk};
+    }
+
+    # Now will try to reassign beginning + ending speakers
+    # to different scp's and see if it gets more balanced.
+    # Suppose objf we're minimizing is sum_i (num utts in scp[i] - average)^2.
+    # We can show that if considering changing just 2 scp's, we minimize
+    # this by minimizing the squared difference in sizes.  This is
+    # equivalent to minimizing the absolute difference in sizes.  This
+    # shows this method is bound to converge.
+
+    $changed = 1;
+    while($changed) {
+        $changed = 0;
+        for($scpidx = 0; $scpidx < $numscps; $scpidx++) {
+            # First try to reassign ending spk of this scp.
+            if($scpidx < $numscps-1) {
+                $sz = @{$scparray[$scpidx]};
+                if($sz > 0) {
+                    $spk = $scparray[$scpidx]->[$sz-1];
+                    $count = $spk_count{$spk};
+                    $nutt1 = $scpcount[$scpidx];
+                    $nutt2 = $scpcount[$scpidx+1];
+                    if( abs( ($nutt2+$count) - ($nutt1-$count))
+                        < abs($nutt2 - $nutt1))  { # Would decrease
+                        # size-diff by reassigning spk...
+                        $scpcount[$scpidx+1] += $count;
+                        $scpcount[$scpidx] -= $count;
+                        pop @{$scparray[$scpidx]};
+                        unshift @{$scparray[$scpidx+1]}, $spk;
+                        $changed = 1;
+                    }
+                }
+            }
+            if($scpidx > 0 && @{$scparray[$scpidx]} > 0) {
+                $spk = $scparray[$scpidx]->[0];
+                $count = $spk_count{$spk};
+                $nutt1 = $scpcount[$scpidx-1];
+                $nutt2 = $scpcount[$scpidx];
+                if( abs( ($nutt2-$count) - ($nutt1+$count))
+                    < abs($nutt2 - $nutt1))  { # Would decrease
+                    # size-diff by reassigning spk...
+                    $scpcount[$scpidx-1] += $count;
+                    $scpcount[$scpidx] -= $count;
+                    shift @{$scparray[$scpidx]};
+                    push @{$scparray[$scpidx-1]}, $spk;
+                    $changed = 1;
+                }
+            }
+        }
+    }
+    # Now print out the files...
+    for($scpidx = 0; $scpidx < $numscps; $scpidx++) {
+        $scpfn = $OUTPUTS[$scpidx];
+        open(F, ">$scpfn") || die "Could not open scp file $scpfn for writing.";
+        $count = 0;
+        if(@{$scparray[$scpidx]} == 0) {
+            print STDERR "Warning: split_scp.pl producing empty .scp file $scpfn (too many splits and too few speakers?)\n";
+        } else {
+            foreach $spk ( @{$scparray[$scpidx]} ) {
+                print F $spk_data{$spk};
+                $count += $spk_count{$spk};
+            }
+            if($count != $scpcount[$scpidx]) { die "Count mismatch [code error]"; }
+        }
+        close(F);
+    }
+} else { 
+   # This block is the "normal" case where there is no --utt2spk 
+   # option and we just break into equal size chunks.
+
+    open(I, "<$inscp") || die "Opening input scp file $inscp";
+
+    $numscps = @OUTPUTS;  # size of array.
+    @F = ();
+    while(<I>) {
+        push @F, $_;
+    }
+    $numlines = @F;
+    if($numlines == 0) {
+        print STDERR "split_scp.pl: warning: empty input scp file $inscp";
+    }
+    $linesperscp = int( ($numlines+($numscps-1)) / $numscps); # the +$(numscps-1) forces rounding up.
+# [just doing int() rounds down].
+    for($scpidx = 0; $scpidx < @OUTPUTS; $scpidx++) {
+        $scpfile = $OUTPUTS[$scpidx];
+        open(O, ">$scpfile") || die "Opening output scp file $scpfile";
+        for($n = $linesperscp * $scpidx; $n < $numlines && $n < $linesperscp*($scpidx+1); $n++) {
+            print O $F[$n];
+        }
+        close(O) || die "Closing scp file $scpfile";
+    }
+}
\ No newline at end of file
diff --git a/utils/train_arpa_with_kenlm.sh b/utils/train_arpa_with_kenlm.sh
new file mode 100755
index 000000000..8af646ceb
--- /dev/null
+++ b/utils/train_arpa_with_kenlm.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# 2020 Author Jiayu DU
+# Apache 2.0
+
+# This script uses kenlm to estimate an arpa model from plain text,
+# it is a resort when you hit memory limit dealing with large corpus
+# kenlm estimates arpa using on-disk structure,
+# as long as you have big enough hard disk, memory shouldn't be a problem.
+# by default, kenlm use up to 50% of your local memory,
+# you can control this through -S option
+
+[ -f path.sh ] && . ./path.sh;
+
+kenlm_opts="" # e.g. "-o 4 -S 50% --prune 0 5 7 7"
+
+if [ $# != 4 ]; then
+  echo "$0 <text> <kaldi_symbol_table> <working_dir> <arpa_name>"
+  echo "e.g. $0 train.txt words.txt wdir 4gram"
+  exit 1
+fi
+
+text=$1
+symbol_table=$2
+dir=$3
+arpa_name=$4
+
+if ! which lmplz >& /dev/null ; then
+  echo "$0: cannot find training tool *lmplz*."
+  echo "tools/extras/install_kenlm_query_only.sh installs kenlm at tools/kenlm"
+  echo "it only supports runtime mode, to actually train an arpa using KenLM,"
+  echo "you need a complete KenLM installation(depends on EIGEN and BOOST),"
+  echo "follow KenLM's building instructions at (https://github.com/kpu/kenlm)"
+  exit 1
+fi
+
+# the text should be properly pre-processed, e.g:
+#   cleand, normalized and possibly word-segmented
+
+# get rid off irrelavent symbols
+grep -v '<eps>' $symbol_table \
+  | grep -v '#0' \
+  | grep -v '<unk>' | grep -v '<UNK>' \
+  | grep -v '<s>' | grep -v '</s>' \
+  | awk '{print $1}' \
+  > $dir/ngram.vocab
+
+# To make sure that kenlm & kaldi have strictly the same vocabulary:
+# 1. feed vocabulary into kenlm via --limit_vocab_file
+# 2. cat vocabulary to training text, so each word at least appear once
+# 
+# TL;DR reason:
+# Unlike SRILM's -limit-vocab, kenlm's --limit_vocab_file option 
+# spcifies a *valid* set of vocabulary, whereas *valid but unseen* 
+# words are discarded in final arpa.
+# So the trick is, 
+# we explicitly add kaldi's vocab(one word per line) to training text, 
+# making each word appear at least once.
+# kenlm never prunes unigram, 
+# so this always generates consistent kenlm vocabuary as kaldi has.
+# The effect of this is like add-one smoothing to unigram counts,
+# shouldn't have significant impacts in practice.
+cat $dir/ngram.vocab $text \
+  | lmplz $kenlm_opts --limit_vocab_file $dir/ngram.vocab \
+  > $dir/${arpa_name}.arpa
+
+echo "$0: Done training arpa to: $dir/${arpa_name}.arpa"
\ No newline at end of file
diff --git a/utils/utility.sh b/utils/utility.sh
old mode 100644
new mode 100755
diff --git a/utils/utt2spk_to_spk2utt.pl b/utils/utt2spk_to_spk2utt.pl
new file mode 100755
index 000000000..127363579
--- /dev/null
+++ b/utils/utt2spk_to_spk2utt.pl
@@ -0,0 +1,38 @@
+#!/usr/bin/env perl
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+# converts an utt2spk file to a spk2utt file.
+# Takes input from the stdin or from a file argument;
+# output goes to the standard out.
+
+if ( @ARGV > 1 ) {
+    die "Usage: utt2spk_to_spk2utt.pl [ utt2spk ] > spk2utt";
+}
+
+while(<>){ 
+    @A = split(" ", $_);
+    @A == 2 || die "Invalid line in utt2spk file: $_";
+    ($u,$s) = @A;
+    if(!$seen_spk{$s}) {
+        $seen_spk{$s} = 1;
+        push @spklist, $s;
+    }
+    push (@{$spk_hash{$s}}, "$u");
+}
+foreach $s (@spklist) {
+    $l = join(' ',@{$spk_hash{$s}});
+    print "$s $l\n";
+}
\ No newline at end of file

From 3db194eb896643cdeb753da1dd710ba518578c00 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Wed, 22 Sep 2021 12:26:00 +0000
Subject: [PATCH 04/13] dump decode result as jsonlines

---
 deepspeech/exps/deepspeech2/model.py | 10 ++++++----
 deepspeech/exps/u2/model.py          | 10 ++++++----
 deepspeech/exps/u2_kaldi/model.py    | 10 ++++++----
 deepspeech/exps/u2_st/model.py       |  9 ++++++---
 4 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/deepspeech/exps/deepspeech2/model.py b/deepspeech/exps/deepspeech2/model.py
index 120539814..646f6f236 100644
--- a/deepspeech/exps/deepspeech2/model.py
+++ b/deepspeech/exps/deepspeech2/model.py
@@ -18,6 +18,7 @@ from collections import defaultdict
 from contextlib import nullcontext
 from pathlib import Path
 from typing import Optional
+import jsonlines
 
 import numpy as np
 import paddle
@@ -305,9 +306,10 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
             len_refs += len_ref
             num_ins += 1
             if fout:
-                fout.write(utt + " " + result + "\n")
-            logger.info("\nTarget Transcription: %s\nOutput Transcription: %s" %
-                        (target, result))
+                fout.write({"utt": utt, "ref", target, "hyp": result})
+            logger.info(f"Utt: {utt}")
+            logger.info(f"Ref: {target}")
+            logger.info(f"Hyp: {result}")
             logger.info("Current error rate [%s] = %f" %
                         (cfg.error_rate_type, error_rate_func(target, result)))
 
@@ -350,7 +352,7 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
         cfg = self.config
         error_rate_type = None
         errors_sum, len_refs, num_ins = 0.0, 0, 0
-        with open(self.args.result_file, 'w') as fout:
+        with jsonlines.open(self.args.result_file, 'w') as fout:
             for i, batch in enumerate(self.test_loader):
                 utts, audio, audio_len, texts, texts_len = batch
                 metrics = self.compute_metrics(utts, audio, audio_len, texts,
diff --git a/deepspeech/exps/u2/model.py b/deepspeech/exps/u2/model.py
index 4dd054896..f1970334d 100644
--- a/deepspeech/exps/u2/model.py
+++ b/deepspeech/exps/u2/model.py
@@ -21,6 +21,7 @@ from collections import OrderedDict
 from contextlib import nullcontext
 from pathlib import Path
 from typing import Optional
+import jsonlines
 
 import numpy as np
 import paddle
@@ -466,9 +467,10 @@ class U2Tester(U2Trainer):
             len_refs += len_ref
             num_ins += 1
             if fout:
-                fout.write(utt + " " + result + "\n")
-            logger.info("\nTarget Transcription: %s\nOutput Transcription: %s" %
-                        (target, result))
+                fout.write({"utt": utt, "ref", target, "hyp": result})
+            logger.info(f"Utt: {utt}")
+            logger.info(f"Ref: {target}")
+            logger.info(f"Hyp: {result}")
             logger.info("One example error rate [%s] = %f" %
                         (cfg.error_rate_type, error_rate_func(target, result)))
 
@@ -493,7 +495,7 @@ class U2Tester(U2Trainer):
         errors_sum, len_refs, num_ins = 0.0, 0, 0
         num_frames = 0.0
         num_time = 0.0
-        with open(self.args.result_file, 'w') as fout:
+        with jsonlines.open(self.args.result_file, 'w') as fout:
             for i, batch in enumerate(self.test_loader):
                 metrics = self.compute_metrics(*batch, fout=fout)
                 num_frames += metrics['num_frames']
diff --git a/deepspeech/exps/u2_kaldi/model.py b/deepspeech/exps/u2_kaldi/model.py
index e8482aa9c..00d780817 100644
--- a/deepspeech/exps/u2_kaldi/model.py
+++ b/deepspeech/exps/u2_kaldi/model.py
@@ -20,6 +20,7 @@ from collections import defaultdict
 from contextlib import nullcontext
 from pathlib import Path
 from typing import Optional
+import jsonlines
 
 import numpy as np
 import paddle
@@ -445,9 +446,10 @@ class U2Tester(U2Trainer):
             len_refs += len_ref
             num_ins += 1
             if fout:
-                fout.write(utt + " " + result + "\n")
-            logger.info("\nTarget Transcription: %s\nOutput Transcription: %s" %
-                        (target, result))
+                fout.write({"utt": utt, "ref", target, "hyp": result})
+            logger.info(f"Utt: {utt}")
+            logger.info(f"Ref: {target}")
+            logger.info(f"Hyp: {result}")
             logger.info("One example error rate [%s] = %f" %
                         (cfg.error_rate_type, error_rate_func(target, result)))
 
@@ -472,7 +474,7 @@ class U2Tester(U2Trainer):
         errors_sum, len_refs, num_ins = 0.0, 0, 0
         num_frames = 0.0
         num_time = 0.0
-        with open(self.args.result_file, 'w') as fout:
+        with jsonlines.open(self.args.result_file, 'w') as fout:
             for i, batch in enumerate(self.test_loader):
                 metrics = self.compute_metrics(*batch, fout=fout)
                 num_frames += metrics['num_frames']
diff --git a/deepspeech/exps/u2_st/model.py b/deepspeech/exps/u2_st/model.py
index c98f5e69e..86bb649bf 100644
--- a/deepspeech/exps/u2_st/model.py
+++ b/deepspeech/exps/u2_st/model.py
@@ -20,6 +20,7 @@ from collections import defaultdict
 from contextlib import nullcontext
 from pathlib import Path
 from typing import Optional
+import jsonlines
 
 import numpy as np
 import paddle
@@ -479,8 +480,10 @@ class U2STTester(U2STTrainer):
             len_refs += len(target.split())
             num_ins += 1
             if fout:
-                fout.write(utt + " " + result + "\n")
-            logger.info("\nReference: %s\nHypothesis: %s" % (target, result))
+                fout.write({"utt": utt, "ref", target, "hyp": result})
+            logger.info(f"Utt: {utt}")
+            logger.info(f"Ref: {target}")
+            logger.info(f"Hyp: {result}")
             logger.info("One example BLEU = %s" %
                         (bleu_func([result], [[target]]).prec_str))
 
@@ -508,7 +511,7 @@ class U2STTester(U2STTrainer):
         len_refs, num_ins = 0, 0
         num_frames = 0.0
         num_time = 0.0
-        with open(self.args.result_file, 'w') as fout:
+        with jsonlines.open(self.args.result_file, 'w') as fout:
             for i, batch in enumerate(self.test_loader):
                 metrics = self.compute_translation_metrics(
                     *batch, bleu_func=bleu_func, fout=fout)

From 16275235330719ce50d17b6ccc67e96543be38d1 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Wed, 22 Sep 2021 12:46:24 +0000
Subject: [PATCH 05/13] add score_sclite

---
 examples/librispeech/s2/path.sh |  2 +-
 tools/Makefile                  | 36 ++++++++++++++++++
 utils/filter.py                 | 66 +++++++++++++++++++++++++++++++++
 utils/score_sclite.sh           |  3 +-
 4 files changed, 105 insertions(+), 2 deletions(-)
 create mode 100644 utils/filter.py

diff --git a/examples/librispeech/s2/path.sh b/examples/librispeech/s2/path.sh
index c90e27821..9f6891cd3 100644
--- a/examples/librispeech/s2/path.sh
+++ b/examples/librispeech/s2/path.sh
@@ -1,6 +1,6 @@
 export MAIN_ROOT=`realpath ${PWD}/../../../`
 
-export PATH=${MAIN_ROOT}:${PWD}/utils:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/tools/sckt/bin/sclite:${PWD}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
diff --git a/tools/Makefile b/tools/Makefile
index 62cf990fa..9fb1dc896 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -39,3 +39,39 @@ mfa.done:
 	test -d montreal-forced-aligner || wget https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner/releases/download/v1.0.1/montreal-forced-aligner_linux.tar.gz
 	tar xvf montreal-forced-aligner_linux.tar.gz
 	touch mfa.done
+
+
+# Keep the existing target 'sclite' to avoid breaking the users who might have
+# scripted it in.
+.PHONY: sclite sctk_cleaned sctk_made
+
+sclite sctk_made: sctk/.compiled
+
+sctk/.compiled: sctk
+	rm -f sctk/.compiled
+	$(SCTK_MKENV) $(MAKE) -C sctk config
+	$(SCTK_MKENV) $(MAKE) -C sctk all doc
+	$(MAKE) -C sctk install
+	touch sctk/.compiled
+
+# The GitHub archive unpacks into SCTK-{40-character-long-hash}/
+sctk: sctk-$(SCTK_GITHASH).tar.gz
+	tar zxvf sctk-$(SCTK_GITHASH).tar.gz
+	rm -rf sctk-$(SCTK_GITHASH) sctk
+	mv SCTK-$(SCTK_GITHASH)* sctk-$(SCTK_GITHASH)
+	ln -s sctk-$(SCTK_GITHASH) sctk
+	touch sctk-$(SCTK_GITHASH).tar.gz
+
+sctk-$(SCTK_GITHASH).tar.gz:
+	if [ -d '$(DOWNLOAD_DIR)' ]; then \
+	  cp -p '$(DOWNLOAD_DIR)/sctk-$(SCTK_GITHASH).tar.gz' .; \
+	else \
+	  $(WGET) -nv -T 10 -t 3 -O sctk-$(SCTK_GITHASH).tar.gz \
+	    https://github.com/usnistgov/SCTK/archive/$(SCTK_GITHASH).tar.gz; \
+	fi
+
+sctk_cleaned:
+	-for d in sctk/ sctk-*/; do \
+	   [ ! -f $$d/.compiled ] || $(MAKE) -C $$d clean; \
+	   rm -f $$d/.compiled; \
+	done
\ No newline at end of file
diff --git a/utils/filter.py b/utils/filter.py
new file mode 100644
index 000000000..d31eab4d7
--- /dev/null
+++ b/utils/filter.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+# Apache 2.0
+
+import argparse
+import codecs
+import sys
+
+is_python2 = sys.version_info[0] == 2
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(
+        description="filter words in a text file",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--exclude",
+        "-v",
+        dest="exclude",
+        action="store_true",
+        help="exclude filter words",
+    )
+    parser.add_argument("filt", type=str, help="filter list")
+    parser.add_argument("infile", type=str, help="input file")
+    return parser
+
+
+def main(args):
+    args = get_parser().parse_args(args)
+    filter_file(args.infile, args.filt, args.exclude)
+
+
+def filter_file(infile, filt, exclude):
+    vocab = set()
+    with codecs.open(filt, "r", encoding="utf-8") as vocabfile:
+        for line in vocabfile:
+            vocab.add(line.strip())
+
+    sys.stdout = codecs.getwriter("utf-8")(
+        sys.stdout if is_python2 else sys.stdout.buffer
+    )
+    with codecs.open(infile, "r", encoding="utf-8") as textfile:
+        for line in textfile:
+            if exclude:
+                print(
+                    " ".join(
+                        map(
+                            lambda word: word if word not in vocab else "",
+                            line.strip().split(),
+                        )
+                    )
+                )
+            else:
+                print(
+                    " ".join(
+                        map(
+                            lambda word: word if word in vocab else "<UNK>",
+                            line.strip().split(),
+                        )
+                    )
+                )
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])
diff --git a/utils/score_sclite.sh b/utils/score_sclite.sh
index 3aa4cd072..7ded76eba 100755
--- a/utils/score_sclite.sh
+++ b/utils/score_sclite.sh
@@ -5,6 +5,7 @@
 
 [ -f ./path.sh ] && . ./path.sh
 
+# non language symbol
 nlsyms=""
 wer=false
 bpe=""
@@ -24,7 +25,7 @@ fi
 dir=$1
 dic=$2
 
-concatjson.py ${dir}/data.*.json > ${dir}/data.json
+cat ${dir}/data.*.json > ${dir}/data.json
 
 if [ $num_spkrs -eq 1 ]; then
   json2trn.py ${dir}/data.json ${dic} --num-spkrs ${num_spkrs} --refs ${dir}/ref.trn --hyps ${dir}/hyp.trn

From 4094a753d9786b8388daf4f67e6ea9b4e73d8957 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Wed, 22 Sep 2021 12:49:22 +0000
Subject: [PATCH 06/13] fix decode json file

---
 deepspeech/exps/deepspeech2/model.py |  4 +--
 deepspeech/exps/u2/model.py          |  4 +--
 deepspeech/exps/u2_kaldi/model.py    |  4 +--
 deepspeech/exps/u2_st/model.py       |  4 +--
 utils/README.md                      |  3 ++-
 utils/dump_manifest.py               |  0
 utils/filter.py                      | 37 +++++++++-------------------
 7 files changed, 22 insertions(+), 34 deletions(-)
 mode change 100644 => 100755 utils/dump_manifest.py
 mode change 100644 => 100755 utils/filter.py

diff --git a/deepspeech/exps/deepspeech2/model.py b/deepspeech/exps/deepspeech2/model.py
index 646f6f236..79a676345 100644
--- a/deepspeech/exps/deepspeech2/model.py
+++ b/deepspeech/exps/deepspeech2/model.py
@@ -18,8 +18,8 @@ from collections import defaultdict
 from contextlib import nullcontext
 from pathlib import Path
 from typing import Optional
-import jsonlines
 
+import jsonlines
 import numpy as np
 import paddle
 from paddle import distributed as dist
@@ -306,7 +306,7 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
             len_refs += len_ref
             num_ins += 1
             if fout:
-                fout.write({"utt": utt, "ref", target, "hyp": result})
+                fout.write({"utt": utt, "ref": target, "hyp": result})
             logger.info(f"Utt: {utt}")
             logger.info(f"Ref: {target}")
             logger.info(f"Hyp: {result}")
diff --git a/deepspeech/exps/u2/model.py b/deepspeech/exps/u2/model.py
index f1970334d..5cb0962a7 100644
--- a/deepspeech/exps/u2/model.py
+++ b/deepspeech/exps/u2/model.py
@@ -21,8 +21,8 @@ from collections import OrderedDict
 from contextlib import nullcontext
 from pathlib import Path
 from typing import Optional
-import jsonlines
 
+import jsonlines
 import numpy as np
 import paddle
 from paddle import distributed as dist
@@ -467,7 +467,7 @@ class U2Tester(U2Trainer):
             len_refs += len_ref
             num_ins += 1
             if fout:
-                fout.write({"utt": utt, "ref", target, "hyp": result})
+                fout.write({"utt": utt, "ref": target, "hyp": result})
             logger.info(f"Utt: {utt}")
             logger.info(f"Ref: {target}")
             logger.info(f"Hyp: {result}")
diff --git a/deepspeech/exps/u2_kaldi/model.py b/deepspeech/exps/u2_kaldi/model.py
index 00d780817..d38afe25c 100644
--- a/deepspeech/exps/u2_kaldi/model.py
+++ b/deepspeech/exps/u2_kaldi/model.py
@@ -20,8 +20,8 @@ from collections import defaultdict
 from contextlib import nullcontext
 from pathlib import Path
 from typing import Optional
-import jsonlines
 
+import jsonlines
 import numpy as np
 import paddle
 from paddle import distributed as dist
@@ -446,7 +446,7 @@ class U2Tester(U2Trainer):
             len_refs += len_ref
             num_ins += 1
             if fout:
-                fout.write({"utt": utt, "ref", target, "hyp": result})
+                fout.write({"utt": utt, "ref": target, "hyp": result})
             logger.info(f"Utt: {utt}")
             logger.info(f"Ref: {target}")
             logger.info(f"Hyp: {result}")
diff --git a/deepspeech/exps/u2_st/model.py b/deepspeech/exps/u2_st/model.py
index 86bb649bf..e4e70292c 100644
--- a/deepspeech/exps/u2_st/model.py
+++ b/deepspeech/exps/u2_st/model.py
@@ -20,8 +20,8 @@ from collections import defaultdict
 from contextlib import nullcontext
 from pathlib import Path
 from typing import Optional
-import jsonlines
 
+import jsonlines
 import numpy as np
 import paddle
 from paddle import distributed as dist
@@ -480,7 +480,7 @@ class U2STTester(U2STTrainer):
             len_refs += len(target.split())
             num_ins += 1
             if fout:
-                fout.write({"utt": utt, "ref", target, "hyp": result})
+                fout.write({"utt": utt, "ref": target, "hyp": result})
             logger.info(f"Utt: {utt}")
             logger.info(f"Ref: {target}")
             logger.info(f"Hyp: {result}")
diff --git a/utils/README.md b/utils/README.md
index d48faf699..163be850f 100644
--- a/utils/README.md
+++ b/utils/README.md
@@ -1,3 +1,4 @@
 # Utils
 
-* [kaldi utils](https://github.com/kaldi-asr/kaldi/blob/cbed4ff688/egs/wsj/s5/utils)
\ No newline at end of file
+* [kaldi utils](https://github.com/kaldi-asr/kaldi/blob/cbed4ff688/egs/wsj/s5/utils)
+* [espnet utils)(https://github.com/espnet/espnet/tree/master/utils)
diff --git a/utils/dump_manifest.py b/utils/dump_manifest.py
old mode 100644
new mode 100755
diff --git a/utils/filter.py b/utils/filter.py
old mode 100644
new mode 100755
index d31eab4d7..e32e135fc
--- a/utils/filter.py
+++ b/utils/filter.py
@@ -1,7 +1,5 @@
 #!/usr/bin/env python3
-
 # Apache 2.0
-
 import argparse
 import codecs
 import sys
@@ -12,15 +10,13 @@ is_python2 = sys.version_info[0] == 2
 def get_parser():
     parser = argparse.ArgumentParser(
         description="filter words in a text file",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-    )
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter, )
     parser.add_argument(
         "--exclude",
         "-v",
         dest="exclude",
         action="store_true",
-        help="exclude filter words",
-    )
+        help="exclude filter words", )
     parser.add_argument("filt", type=str, help="filter list")
     parser.add_argument("infile", type=str, help="input file")
     return parser
@@ -37,29 +33,20 @@ def filter_file(infile, filt, exclude):
         for line in vocabfile:
             vocab.add(line.strip())
 
-    sys.stdout = codecs.getwriter("utf-8")(
-        sys.stdout if is_python2 else sys.stdout.buffer
-    )
+    sys.stdout = codecs.getwriter("utf-8")(sys.stdout
+                                           if is_python2 else sys.stdout.buffer)
     with codecs.open(infile, "r", encoding="utf-8") as textfile:
         for line in textfile:
             if exclude:
-                print(
-                    " ".join(
-                        map(
-                            lambda word: word if word not in vocab else "",
-                            line.strip().split(),
-                        )
-                    )
-                )
+                print(" ".join(
+                    map(
+                        lambda word: word if word not in vocab else "",
+                        line.strip().split(), )))
             else:
-                print(
-                    " ".join(
-                        map(
-                            lambda word: word if word in vocab else "<UNK>",
-                            line.strip().split(),
-                        )
-                    )
-                )
+                print(" ".join(
+                    map(
+                        lambda word: word if word in vocab else "<UNK>",
+                        line.strip().split(), )))
 
 
 if __name__ == "__main__":

From c7f12dc8b7daf6fb434c40d4d1fd0a7b33699da4 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 23 Sep 2021 02:54:38 +0000
Subject: [PATCH 07/13] refactor avg_model; fix set_value not support
 start==end

---
 deepspeech/utils/tensor_utils.py | 14 +++++++--
 utils/avg_model.py               | 49 ++++++++++++++++----------------
 2 files changed, 37 insertions(+), 26 deletions(-)

diff --git a/deepspeech/utils/tensor_utils.py b/deepspeech/utils/tensor_utils.py
index bb7f58ded..61798816b 100644
--- a/deepspeech/utils/tensor_utils.py
+++ b/deepspeech/utils/tensor_utils.py
@@ -94,9 +94,19 @@ def pad_sequence(sequences: List[paddle.Tensor],
         length = tensor.shape[0]
         # use index notation to prevent duplicate references to the tensor
         if batch_first:
-            out_tensor[i, :length, ...] = tensor
+            # TODO (Hui Zhang): set_value op not supprot `end==start`
+            # out_tensor[i, :length, ...] = tensor
+            if length != 0:
+                out_tensor[i, :length, ...] = tensor
+            else:
+                out_tensor[i, length, ...] = tensor
         else:
-            out_tensor[:length, i, ...] = tensor
+            # TODO (Hui Zhang): set_value op not supprot `end==start`
+            # out_tensor[:length, i, ...] = tensor
+            if length != 0:
+                out_tensor[:length, i, ...] = tensor
+            else:
+                out_tensor[length, i, ...] = tensor
 
     return out_tensor
 
diff --git a/utils/avg_model.py b/utils/avg_model.py
index 3a0739c95..1fc00cb65 100755
--- a/utils/avg_model.py
+++ b/utils/avg_model.py
@@ -27,33 +27,33 @@ def main(args):
     val_scores = []
     beat_val_scores = []
     selected_epochs = []
-    if args.val_best:
-        jsons = glob.glob(f'{args.ckpt_dir}/[!train]*.json')
-        for y in jsons:
-            with open(y, 'r') as f:
-                dic_json = json.load(f)
-            loss = dic_json['val_loss']
-            epoch = dic_json['epoch']
-            if epoch >= args.min_epoch and epoch <= args.max_epoch:
-                val_scores.append((epoch, loss))
 
-        val_scores = np.array(val_scores)
+    jsons = glob.glob(f'{args.ckpt_dir}/[!train]*.json')
+    jsons = sorted(jsons, key=os.path.getmtime, reverse=True)
+    for y in jsons:
+        with open(y, 'r') as f:
+            dic_json = json.load(f)
+        loss = dic_json['val_loss']
+        epoch = dic_json['epoch']
+        if epoch >= args.min_epoch and epoch <= args.max_epoch:
+            val_scores.append((epoch, loss))
+    val_scores = np.array(val_scores)
+
+    if args.val_best:
         sort_idx = np.argsort(val_scores[:, 1])
         sorted_val_scores = val_scores[sort_idx]
-        path_list = [
-            args.ckpt_dir + '/{}.pdparams'.format(int(epoch))
-            for epoch in sorted_val_scores[:args.num, 0]
-        ]
-
-        beat_val_scores = sorted_val_scores[:args.num, 1]
-        selected_epochs = sorted_val_scores[:args.num, 0].astype(np.int64)
-        print("best val scores = " + str(beat_val_scores))
-        print("selected epochs = " + str(selected_epochs))
     else:
-        path_list = glob.glob(f'{args.ckpt_dir}/[!avg][!final]*.pdparams')
-        path_list = sorted(path_list, key=os.path.getmtime)
-        path_list = path_list[-args.num:]
+        sorted_val_scores = val_scores
+
+    beat_val_scores = sorted_val_scores[:args.num, 1]
+    selected_epochs = sorted_val_scores[:args.num, 0].astype(np.int64)
+    print("selected val scores = " + str(beat_val_scores))
+    print("selected epochs = " + str(selected_epochs))
 
+    path_list = [
+        args.ckpt_dir + '/{}.pdparams'.format(int(epoch))
+        for epoch in sorted_val_scores[:args.num, 0]
+    ]
     print(path_list)
 
     avg = None
@@ -78,10 +78,11 @@ def main(args):
     meta_path = os.path.splitext(args.dst_model)[0] + '.avg.json'
     with open(meta_path, 'w') as f:
         data = json.dumps({
+            "mode": 'val_best' if args.val_best else 'latest',
             "avg_ckpt": args.dst_model,
             "ckpt": path_list,
-            "epoch": selected_epochs,
-            "val_loss": beat_val_scores,
+            "epoch": selected_epochs.tolist(),
+            "val_loss": beat_val_scores.tolist(),
         })
         f.write(data + "\n")
 

From 0866c877517353aa006fc2ead6dc2c35917ffb9a Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 23 Sep 2021 03:00:41 +0000
Subject: [PATCH 08/13] using jsonlines to read manifest and dump decode result

---
 deepspeech/frontend/utility.py | 37 +++++++++++++++-------------------
 1 file changed, 16 insertions(+), 21 deletions(-)

diff --git a/deepspeech/frontend/utility.py b/deepspeech/frontend/utility.py
index c6781cd4e..926abf18e 100644
--- a/deepspeech/frontend/utility.py
+++ b/deepspeech/frontend/utility.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Contains data helper functions."""
-import codecs
 import json
 import math
 from typing import List
@@ -92,26 +91,22 @@ def read_manifest(
     """
 
     manifest = []
-    for json_line in codecs.open(manifest_path, 'r', 'utf-8'):
-        try:
-            json_data = json.loads(json_line)
-        except Exception as e:
-            raise IOError("Error reading manifest: %s" % str(e))
-
-        feat_len = json_data["feat_shape"][
-            0] if 'feat_shape' in json_data else 1.0
-        token_len = json_data["token_shape"][
-            0] if 'token_shape' in json_data else 1.0
-        conditions = [
-            feat_len >= min_input_len,
-            feat_len <= max_input_len,
-            token_len >= min_output_len,
-            token_len <= max_output_len,
-            token_len / feat_len >= min_output_input_ratio,
-            token_len / feat_len <= max_output_input_ratio,
-        ]
-        if all(conditions):
-            manifest.append(json_data)
+    with jsonlines.open(manifest_path, 'r') as reader:
+        for json_data in reader:
+            feat_len = json_data["feat_shape"][
+                0] if 'feat_shape' in json_data else 1.0
+            token_len = json_data["token_shape"][
+                0] if 'token_shape' in json_data else 1.0
+            conditions = [
+                feat_len >= min_input_len,
+                feat_len <= max_input_len,
+                token_len >= min_output_len,
+                token_len <= max_output_len,
+                token_len / feat_len >= min_output_input_ratio,
+                token_len / feat_len <= max_output_input_ratio,
+            ]
+            if all(conditions):
+                manifest.append(json_data)
     return manifest
 
 

From 375bdc5d4a34c832a833312f42d25a763b7f7693 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 23 Sep 2021 03:04:32 +0000
Subject: [PATCH 09/13] refactor ds 1.x exp

---
 examples/{v18_to_v2x => 1xt2x}/.gitignore             |  0
 examples/1xt2x/README.md                              | 11 +++++++++++
 .../exp_aishell => 1xt2x/aishell}/.gitignore          |  0
 .../aishell}/conf/augmentation.json                   |  0
 .../aishell}/conf/deepspeech2.yaml                    |  0
 .../exp_aishell => 1xt2x/aishell}/local/data.sh       |  0
 .../aishell}/local/download_lm_ch.sh                  |  0
 .../aishell}/local/download_model.sh                  |  0
 .../exp_aishell => 1xt2x/aishell}/local/test.sh       |  0
 .../{v18_to_v2x/exp_aishell => 1xt2x/aishell}/path.sh |  0
 .../{v18_to_v2x/exp_aishell => 1xt2x/aishell}/run.sh  |  0
 .../exp_baidu_en8k => 1xt2x/baidu_en8k}/.gitignore    |  0
 .../baidu_en8k}/conf/augmentation.json                |  0
 .../baidu_en8k}/conf/deepspeech2.yaml                 |  0
 .../exp_baidu_en8k => 1xt2x/baidu_en8k}/local/data.sh |  0
 .../baidu_en8k}/local/download_lm_en.sh               |  0
 .../baidu_en8k}/local/download_model.sh               |  0
 .../exp_baidu_en8k => 1xt2x/baidu_en8k}/local/test.sh |  0
 .../exp_baidu_en8k => 1xt2x/baidu_en8k}/path.sh       |  0
 .../exp_baidu_en8k => 1xt2x/baidu_en8k}/run.sh        |  0
 .../{v18_to_v2x => 1xt2x}/deepspeech2x/__init__.py    |  0
 .../{v18_to_v2x => 1xt2x}/deepspeech2x/bin/test.py    |  0
 examples/{v18_to_v2x => 1xt2x}/deepspeech2x/model.py  |  0
 .../deepspeech2x/models/__init__.py                   |  0
 .../deepspeech2x/models/ds2/__init__.py               |  0
 .../deepspeech2x/models/ds2/deepspeech2.py            |  0
 .../deepspeech2x/models/ds2/rnn.py                    |  0
 .../exp_librispeech => 1xt2x/librispeech}/.gitignore  |  0
 .../librispeech}/conf/augmentation.json               |  0
 .../librispeech}/conf/deepspeech2.yaml                |  0
 .../librispeech}/local/data.sh                        |  0
 .../librispeech}/local/download_lm_en.sh              |  0
 .../librispeech}/local/download_model.sh              |  0
 .../librispeech}/local/test.sh                        |  0
 .../exp_librispeech => 1xt2x/librispeech}/path.sh     |  0
 .../exp_librispeech => 1xt2x/librispeech}/run.sh      |  0
 36 files changed, 11 insertions(+)
 rename examples/{v18_to_v2x => 1xt2x}/.gitignore (100%)
 create mode 100644 examples/1xt2x/README.md
 rename examples/{v18_to_v2x/exp_aishell => 1xt2x/aishell}/.gitignore (100%)
 rename examples/{v18_to_v2x/exp_aishell => 1xt2x/aishell}/conf/augmentation.json (100%)
 rename examples/{v18_to_v2x/exp_aishell => 1xt2x/aishell}/conf/deepspeech2.yaml (100%)
 rename examples/{v18_to_v2x/exp_aishell => 1xt2x/aishell}/local/data.sh (100%)
 rename examples/{v18_to_v2x/exp_aishell => 1xt2x/aishell}/local/download_lm_ch.sh (100%)
 rename examples/{v18_to_v2x/exp_aishell => 1xt2x/aishell}/local/download_model.sh (100%)
 rename examples/{v18_to_v2x/exp_aishell => 1xt2x/aishell}/local/test.sh (100%)
 rename examples/{v18_to_v2x/exp_aishell => 1xt2x/aishell}/path.sh (100%)
 rename examples/{v18_to_v2x/exp_aishell => 1xt2x/aishell}/run.sh (100%)
 rename examples/{v18_to_v2x/exp_baidu_en8k => 1xt2x/baidu_en8k}/.gitignore (100%)
 rename examples/{v18_to_v2x/exp_baidu_en8k => 1xt2x/baidu_en8k}/conf/augmentation.json (100%)
 rename examples/{v18_to_v2x/exp_baidu_en8k => 1xt2x/baidu_en8k}/conf/deepspeech2.yaml (100%)
 rename examples/{v18_to_v2x/exp_baidu_en8k => 1xt2x/baidu_en8k}/local/data.sh (100%)
 rename examples/{v18_to_v2x/exp_baidu_en8k => 1xt2x/baidu_en8k}/local/download_lm_en.sh (100%)
 rename examples/{v18_to_v2x/exp_baidu_en8k => 1xt2x/baidu_en8k}/local/download_model.sh (100%)
 rename examples/{v18_to_v2x/exp_baidu_en8k => 1xt2x/baidu_en8k}/local/test.sh (100%)
 rename examples/{v18_to_v2x/exp_baidu_en8k => 1xt2x/baidu_en8k}/path.sh (100%)
 rename examples/{v18_to_v2x/exp_baidu_en8k => 1xt2x/baidu_en8k}/run.sh (100%)
 rename examples/{v18_to_v2x => 1xt2x}/deepspeech2x/__init__.py (100%)
 rename examples/{v18_to_v2x => 1xt2x}/deepspeech2x/bin/test.py (100%)
 rename examples/{v18_to_v2x => 1xt2x}/deepspeech2x/model.py (100%)
 rename examples/{v18_to_v2x => 1xt2x}/deepspeech2x/models/__init__.py (100%)
 rename examples/{v18_to_v2x => 1xt2x}/deepspeech2x/models/ds2/__init__.py (100%)
 rename examples/{v18_to_v2x => 1xt2x}/deepspeech2x/models/ds2/deepspeech2.py (100%)
 rename examples/{v18_to_v2x => 1xt2x}/deepspeech2x/models/ds2/rnn.py (100%)
 rename examples/{v18_to_v2x/exp_librispeech => 1xt2x/librispeech}/.gitignore (100%)
 rename examples/{v18_to_v2x/exp_librispeech => 1xt2x/librispeech}/conf/augmentation.json (100%)
 rename examples/{v18_to_v2x/exp_librispeech => 1xt2x/librispeech}/conf/deepspeech2.yaml (100%)
 rename examples/{v18_to_v2x/exp_librispeech => 1xt2x/librispeech}/local/data.sh (100%)
 rename examples/{v18_to_v2x/exp_librispeech => 1xt2x/librispeech}/local/download_lm_en.sh (100%)
 rename examples/{v18_to_v2x/exp_librispeech => 1xt2x/librispeech}/local/download_model.sh (100%)
 rename examples/{v18_to_v2x/exp_librispeech => 1xt2x/librispeech}/local/test.sh (100%)
 rename examples/{v18_to_v2x/exp_librispeech => 1xt2x/librispeech}/path.sh (100%)
 rename examples/{v18_to_v2x/exp_librispeech => 1xt2x/librispeech}/run.sh (100%)

diff --git a/examples/v18_to_v2x/.gitignore b/examples/1xt2x/.gitignore
similarity index 100%
rename from examples/v18_to_v2x/.gitignore
rename to examples/1xt2x/.gitignore
diff --git a/examples/1xt2x/README.md b/examples/1xt2x/README.md
new file mode 100644
index 000000000..1f5fe8e3b
--- /dev/null
+++ b/examples/1xt2x/README.md
@@ -0,0 +1,11 @@
+# 1xt2x
+
+Convert Deepspeech 1.8 released model to 2.x.
+
+## Model
+* Deepspeech2x
+
+## Exp
+* baidu_en8k
+* aishell
+* librispeech
diff --git a/examples/v18_to_v2x/exp_aishell/.gitignore b/examples/1xt2x/aishell/.gitignore
similarity index 100%
rename from examples/v18_to_v2x/exp_aishell/.gitignore
rename to examples/1xt2x/aishell/.gitignore
diff --git a/examples/v18_to_v2x/exp_aishell/conf/augmentation.json b/examples/1xt2x/aishell/conf/augmentation.json
similarity index 100%
rename from examples/v18_to_v2x/exp_aishell/conf/augmentation.json
rename to examples/1xt2x/aishell/conf/augmentation.json
diff --git a/examples/v18_to_v2x/exp_aishell/conf/deepspeech2.yaml b/examples/1xt2x/aishell/conf/deepspeech2.yaml
similarity index 100%
rename from examples/v18_to_v2x/exp_aishell/conf/deepspeech2.yaml
rename to examples/1xt2x/aishell/conf/deepspeech2.yaml
diff --git a/examples/v18_to_v2x/exp_aishell/local/data.sh b/examples/1xt2x/aishell/local/data.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_aishell/local/data.sh
rename to examples/1xt2x/aishell/local/data.sh
diff --git a/examples/v18_to_v2x/exp_aishell/local/download_lm_ch.sh b/examples/1xt2x/aishell/local/download_lm_ch.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_aishell/local/download_lm_ch.sh
rename to examples/1xt2x/aishell/local/download_lm_ch.sh
diff --git a/examples/v18_to_v2x/exp_aishell/local/download_model.sh b/examples/1xt2x/aishell/local/download_model.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_aishell/local/download_model.sh
rename to examples/1xt2x/aishell/local/download_model.sh
diff --git a/examples/v18_to_v2x/exp_aishell/local/test.sh b/examples/1xt2x/aishell/local/test.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_aishell/local/test.sh
rename to examples/1xt2x/aishell/local/test.sh
diff --git a/examples/v18_to_v2x/exp_aishell/path.sh b/examples/1xt2x/aishell/path.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_aishell/path.sh
rename to examples/1xt2x/aishell/path.sh
diff --git a/examples/v18_to_v2x/exp_aishell/run.sh b/examples/1xt2x/aishell/run.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_aishell/run.sh
rename to examples/1xt2x/aishell/run.sh
diff --git a/examples/v18_to_v2x/exp_baidu_en8k/.gitignore b/examples/1xt2x/baidu_en8k/.gitignore
similarity index 100%
rename from examples/v18_to_v2x/exp_baidu_en8k/.gitignore
rename to examples/1xt2x/baidu_en8k/.gitignore
diff --git a/examples/v18_to_v2x/exp_baidu_en8k/conf/augmentation.json b/examples/1xt2x/baidu_en8k/conf/augmentation.json
similarity index 100%
rename from examples/v18_to_v2x/exp_baidu_en8k/conf/augmentation.json
rename to examples/1xt2x/baidu_en8k/conf/augmentation.json
diff --git a/examples/v18_to_v2x/exp_baidu_en8k/conf/deepspeech2.yaml b/examples/1xt2x/baidu_en8k/conf/deepspeech2.yaml
similarity index 100%
rename from examples/v18_to_v2x/exp_baidu_en8k/conf/deepspeech2.yaml
rename to examples/1xt2x/baidu_en8k/conf/deepspeech2.yaml
diff --git a/examples/v18_to_v2x/exp_baidu_en8k/local/data.sh b/examples/1xt2x/baidu_en8k/local/data.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_baidu_en8k/local/data.sh
rename to examples/1xt2x/baidu_en8k/local/data.sh
diff --git a/examples/v18_to_v2x/exp_baidu_en8k/local/download_lm_en.sh b/examples/1xt2x/baidu_en8k/local/download_lm_en.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_baidu_en8k/local/download_lm_en.sh
rename to examples/1xt2x/baidu_en8k/local/download_lm_en.sh
diff --git a/examples/v18_to_v2x/exp_baidu_en8k/local/download_model.sh b/examples/1xt2x/baidu_en8k/local/download_model.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_baidu_en8k/local/download_model.sh
rename to examples/1xt2x/baidu_en8k/local/download_model.sh
diff --git a/examples/v18_to_v2x/exp_baidu_en8k/local/test.sh b/examples/1xt2x/baidu_en8k/local/test.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_baidu_en8k/local/test.sh
rename to examples/1xt2x/baidu_en8k/local/test.sh
diff --git a/examples/v18_to_v2x/exp_baidu_en8k/path.sh b/examples/1xt2x/baidu_en8k/path.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_baidu_en8k/path.sh
rename to examples/1xt2x/baidu_en8k/path.sh
diff --git a/examples/v18_to_v2x/exp_baidu_en8k/run.sh b/examples/1xt2x/baidu_en8k/run.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_baidu_en8k/run.sh
rename to examples/1xt2x/baidu_en8k/run.sh
diff --git a/examples/v18_to_v2x/deepspeech2x/__init__.py b/examples/1xt2x/deepspeech2x/__init__.py
similarity index 100%
rename from examples/v18_to_v2x/deepspeech2x/__init__.py
rename to examples/1xt2x/deepspeech2x/__init__.py
diff --git a/examples/v18_to_v2x/deepspeech2x/bin/test.py b/examples/1xt2x/deepspeech2x/bin/test.py
similarity index 100%
rename from examples/v18_to_v2x/deepspeech2x/bin/test.py
rename to examples/1xt2x/deepspeech2x/bin/test.py
diff --git a/examples/v18_to_v2x/deepspeech2x/model.py b/examples/1xt2x/deepspeech2x/model.py
similarity index 100%
rename from examples/v18_to_v2x/deepspeech2x/model.py
rename to examples/1xt2x/deepspeech2x/model.py
diff --git a/examples/v18_to_v2x/deepspeech2x/models/__init__.py b/examples/1xt2x/deepspeech2x/models/__init__.py
similarity index 100%
rename from examples/v18_to_v2x/deepspeech2x/models/__init__.py
rename to examples/1xt2x/deepspeech2x/models/__init__.py
diff --git a/examples/v18_to_v2x/deepspeech2x/models/ds2/__init__.py b/examples/1xt2x/deepspeech2x/models/ds2/__init__.py
similarity index 100%
rename from examples/v18_to_v2x/deepspeech2x/models/ds2/__init__.py
rename to examples/1xt2x/deepspeech2x/models/ds2/__init__.py
diff --git a/examples/v18_to_v2x/deepspeech2x/models/ds2/deepspeech2.py b/examples/1xt2x/deepspeech2x/models/ds2/deepspeech2.py
similarity index 100%
rename from examples/v18_to_v2x/deepspeech2x/models/ds2/deepspeech2.py
rename to examples/1xt2x/deepspeech2x/models/ds2/deepspeech2.py
diff --git a/examples/v18_to_v2x/deepspeech2x/models/ds2/rnn.py b/examples/1xt2x/deepspeech2x/models/ds2/rnn.py
similarity index 100%
rename from examples/v18_to_v2x/deepspeech2x/models/ds2/rnn.py
rename to examples/1xt2x/deepspeech2x/models/ds2/rnn.py
diff --git a/examples/v18_to_v2x/exp_librispeech/.gitignore b/examples/1xt2x/librispeech/.gitignore
similarity index 100%
rename from examples/v18_to_v2x/exp_librispeech/.gitignore
rename to examples/1xt2x/librispeech/.gitignore
diff --git a/examples/v18_to_v2x/exp_librispeech/conf/augmentation.json b/examples/1xt2x/librispeech/conf/augmentation.json
similarity index 100%
rename from examples/v18_to_v2x/exp_librispeech/conf/augmentation.json
rename to examples/1xt2x/librispeech/conf/augmentation.json
diff --git a/examples/v18_to_v2x/exp_librispeech/conf/deepspeech2.yaml b/examples/1xt2x/librispeech/conf/deepspeech2.yaml
similarity index 100%
rename from examples/v18_to_v2x/exp_librispeech/conf/deepspeech2.yaml
rename to examples/1xt2x/librispeech/conf/deepspeech2.yaml
diff --git a/examples/v18_to_v2x/exp_librispeech/local/data.sh b/examples/1xt2x/librispeech/local/data.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_librispeech/local/data.sh
rename to examples/1xt2x/librispeech/local/data.sh
diff --git a/examples/v18_to_v2x/exp_librispeech/local/download_lm_en.sh b/examples/1xt2x/librispeech/local/download_lm_en.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_librispeech/local/download_lm_en.sh
rename to examples/1xt2x/librispeech/local/download_lm_en.sh
diff --git a/examples/v18_to_v2x/exp_librispeech/local/download_model.sh b/examples/1xt2x/librispeech/local/download_model.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_librispeech/local/download_model.sh
rename to examples/1xt2x/librispeech/local/download_model.sh
diff --git a/examples/v18_to_v2x/exp_librispeech/local/test.sh b/examples/1xt2x/librispeech/local/test.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_librispeech/local/test.sh
rename to examples/1xt2x/librispeech/local/test.sh
diff --git a/examples/v18_to_v2x/exp_librispeech/path.sh b/examples/1xt2x/librispeech/path.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_librispeech/path.sh
rename to examples/1xt2x/librispeech/path.sh
diff --git a/examples/v18_to_v2x/exp_librispeech/run.sh b/examples/1xt2x/librispeech/run.sh
similarity index 100%
rename from examples/v18_to_v2x/exp_librispeech/run.sh
rename to examples/1xt2x/librispeech/run.sh

From e1b9e49d9eec4c3ad812dedafe66e9dedd8d45fb Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 23 Sep 2021 03:16:49 +0000
Subject: [PATCH 10/13] fix sctk install

---
 .gitignore     |  2 ++
 tools/Makefile | 27 +++++++++++++++++++++++----
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index e4134a082..cd2360e15 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,5 +18,7 @@ tools/sox-14.4.2
 tools/soxbindings
 tools/montreal-forced-aligner/
 tools/Montreal-Forced-Aligner/
+tools/sctk
+tools/sctk-20159b5/
 
 *output/
diff --git a/tools/Makefile b/tools/Makefile
index 9fb1dc896..c925054b8 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -1,8 +1,16 @@
 SHELL:= /bin/bash
 PYTHON:= python3.7
+
+CXX ?= g++
+CC ?= gcc        # used for sph2pipe
+# CXX = clang++  # Uncomment these lines...
+# CC = clang     # ...to build with Clang.
+
+WGET ?= wget
+
 .PHONY: all clean
 
-all: virtualenv kenlm.done sox.done soxbindings.done mfa.done
+all: virtualenv kenlm.done sox.done soxbindings.done mfa.done sclite.done
 
 virtualenv:
 	test -d venv || virtualenv -p $(PYTHON) venv
@@ -41,11 +49,22 @@ mfa.done:
 	touch mfa.done
 
 
+#== SCTK ===============================================================================
+# SCTK official repo does not have version tags. Here's the mapping:
+# # 2.4.9 = 659bc36; 2.4.10 = d914e1b; 2.4.11 = 20159b5.
+SCTK_GITHASH = 20159b5
+
+SCTK_CXFLAGS = -w -march=native
+SCTK_MKENV = CFLAGS="$(CFLAGS) $(SCTK_CXFLAGS)" \
+			              CXXFLAGS="$(CXXFLAGS) -std=c++11 $(SCTK_CXFLAGS)" \
+
+
 # Keep the existing target 'sclite' to avoid breaking the users who might have
 # scripted it in.
-.PHONY: sclite sctk_cleaned sctk_made
+.PHONY: sclite.done sctk_cleaned sctk_made
 
-sclite sctk_made: sctk/.compiled
+sclite.done sctk_made: sctk/.compiled
+	touch sclite.done
 
 sctk/.compiled: sctk
 	rm -f sctk/.compiled
@@ -74,4 +93,4 @@ sctk_cleaned:
 	-for d in sctk/ sctk-*/; do \
 	   [ ! -f $$d/.compiled ] || $(MAKE) -C $$d clean; \
 	   rm -f $$d/.compiled; \
-	done
\ No newline at end of file
+	done

From f91f658e3b5bd4d9654215c15e6e5a4cafc08d08 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 23 Sep 2021 03:18:12 +0000
Subject: [PATCH 11/13] rm dupliate packages

---
 requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index ebf879b51..925e0a317 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,6 @@
 coverage
 gpustat
 jsonlines
-jsonlines
 kaldiio
 loguru
 Pillow

From 99bc4ea9fa1e8d8fb8fe88863bdc884a56be98dc Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 23 Sep 2021 03:21:24 +0000
Subject: [PATCH 12/13] librispeech s1 support sclite and multi process decode

---
 examples/librispeech/s1/cmd.sh        | 89 +++++++++++++++++++++++++
 examples/librispeech/s1/local/test.sh | 96 ++++++++++++++++-----------
 examples/librispeech/s1/path.sh       |  2 +-
 examples/librispeech/s1/run.sh        |  7 +-
 4 files changed, 153 insertions(+), 41 deletions(-)
 create mode 100644 examples/librispeech/s1/cmd.sh

diff --git a/examples/librispeech/s1/cmd.sh b/examples/librispeech/s1/cmd.sh
new file mode 100644
index 000000000..7b70ef5e0
--- /dev/null
+++ b/examples/librispeech/s1/cmd.sh
@@ -0,0 +1,89 @@
+# ====== About run.pl, queue.pl, slurm.pl, and ssh.pl ======
+# Usage: <cmd>.pl [options] JOB=1:<nj> <log> <command...>
+# e.g.
+#   run.pl --mem 4G JOB=1:10 echo.JOB.log echo JOB
+#
+# Options:
+#   --time <time>: Limit the maximum time to execute.
+#   --mem <mem>: Limit the maximum memory usage.
+#   -–max-jobs-run <njob>: Limit the number parallel jobs. This is ignored for non-array jobs.
+#   --num-threads <ngpu>: Specify the number of CPU core.
+#   --gpu <ngpu>: Specify the number of GPU devices.
+#   --config: Change the configuration file from default.
+#
+# "JOB=1:10" is used for "array jobs" and it can control the number of parallel jobs.
+# The left string of "=", i.e. "JOB", is replaced by <N>(Nth job) in the command and the log file name,
+# e.g. "echo JOB" is changed to "echo 3" for the 3rd job and "echo 8" for 8th job respectively.
+# Note that the number must start with a positive number, so you can't use "JOB=0:10" for example.
+#
+# run.pl, queue.pl, slurm.pl, and ssh.pl have unified interface, not depending on its backend.
+# These options are mapping to specific options for each backend and
+# it is configured by "conf/queue.conf" and "conf/slurm.conf" by default.
+# If jobs failed, your configuration might be wrong for your environment.
+#
+#
+# The official documentation for run.pl, queue.pl, slurm.pl, and ssh.pl:
+#   "Parallelization in Kaldi": http://kaldi-asr.org/doc/queue.html
+# =========================================================~
+
+
+# Select the backend used by run.sh from "local", "sge", "slurm", or "ssh"
+cmd_backend='local'
+
+# Local machine, without any Job scheduling system
+if [ "${cmd_backend}" = local ]; then
+
+    # The other usage
+    export train_cmd="run.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="run.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="run.pl"
+
+# "qsub" (SGE, Torque, PBS, etc.)
+elif [ "${cmd_backend}" = sge ]; then
+    # The default setting is written in conf/queue.conf.
+    # You must change "-q g.q" for the "queue" for your environment.
+    # To know the "queue" names, type "qhost -q"
+    # Note that to use "--gpu *", you have to setup "complex_value" for the system scheduler.
+
+    export train_cmd="queue.pl"
+    export cuda_cmd="queue.pl"
+    export decode_cmd="queue.pl"
+
+# "sbatch" (Slurm)
+elif [ "${cmd_backend}" = slurm ]; then
+    # The default setting is written in conf/slurm.conf.
+    # You must change "-p cpu" and "-p gpu" for the "partion" for your environment.
+    # To know the "partion" names, type "sinfo".
+    # You can use "--gpu * " by default for slurm and it is interpreted as "--gres gpu:*"
+    # The devices are allocated exclusively using "${CUDA_VISIBLE_DEVICES}".
+
+    export train_cmd="slurm.pl"
+    export cuda_cmd="slurm.pl"
+    export decode_cmd="slurm.pl"
+
+elif [ "${cmd_backend}" = ssh ]; then
+    # You have to create ".queue/machines" to specify the host to execute jobs.
+    # e.g. .queue/machines
+    #   host1
+    #   host2
+    #   host3
+    # Assuming you can login them without any password, i.e. You have to set ssh keys.
+
+    export train_cmd="ssh.pl"
+    export cuda_cmd="ssh.pl"
+    export decode_cmd="ssh.pl"
+
+# This is an example of specifying several unique options in the JHU CLSP cluster setup.
+# Users can modify/add their own command options according to their cluster environments.
+elif [ "${cmd_backend}" = jhu ]; then
+
+    export train_cmd="queue.pl --mem 2G"
+    export cuda_cmd="queue-freegpu.pl --mem 2G --gpu 1 --config conf/gpu.conf"
+    export decode_cmd="queue.pl --mem 4G"
+
+else
+    echo "$0: Error: Unknown cmd_backend=${cmd_backend}" 1>&2
+    return 1
+fi
diff --git a/examples/librispeech/s1/local/test.sh b/examples/librispeech/s1/local/test.sh
index 62b92e1e3..09644bb2c 100755
--- a/examples/librispeech/s1/local/test.sh
+++ b/examples/librispeech/s1/local/test.sh
@@ -1,7 +1,21 @@
 #!/bin/bash
 
-if [ $# != 2 ];then
-    echo "usage: ${0} config_path ckpt_path_prefix"
+expdir=exp
+datadir=data
+
+lmtag=
+
+recog_set="test-clean test-other dev-clean dev-other"
+recog_set="test-clean"
+
+# bpemode (unigram or bpe)
+nbpe=5000
+bpemode=unigram
+bpeprefix="data/bpe_${bpemode}_${nbpe}"
+bpemodel=${bpeprefix}.model
+
+if [ $# != 3 ];then
+    echo "usage: ${0} config_path dict_path ckpt_path_prefix"
     exit -1
 fi
 
@@ -9,7 +23,8 @@ ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
 config_path=$1
-ckpt_prefix=$2
+dict=$2
+ckpt_prefix=$3
 
 chunk_mode=false
 if [[ ${config_path} =~ ^.*chunk_.*yaml$ ]];then
@@ -24,44 +39,49 @@ echo "chunk mode ${chunk_mode}"
 #    exit 1
 #fi
 
-for type in attention ctc_greedy_search; do
-    echo "decoding ${type}"
-    if [ ${chunk_mode} == true ];then
-        # stream decoding only support batchsize=1
+pids=() # initialize pids
+
+for dmethd in attention ctc_greedy_search ctc_prefix_beam_search attention_rescoring; do
+(
+    for rtask in ${recog_set}; do
+    (
+        decode_dir=decode_${rtask}_${dmethd}_$(basename ${config_path%.*})_${lmtag}
+        feat_recog_dir=${datadir}
+        mkdir -p ${expdir}/${decode_dir}
+        mkdir -p ${feat_recog_dir}
+
+        # split data
+        split_json.sh ${feat_recog_dir}/manifest.${rtask} ${nj}
+
+        #### use CPU for decoding
+        ngpu=0
+
+        # set batchsize 0 to disable batch decoding
         batch_size=1
-    else
-        batch_size=64
-    fi
-    python3 -u ${BIN_DIR}/test.py \
-    --nproc ${ngpu} \
-    --config ${config_path} \
-    --result_file ${ckpt_prefix}.${type}.rsl \
-    --checkpoint_path ${ckpt_prefix} \
-    --opts decoding.decoding_method ${type} \
-    --opts decoding.batch_size ${batch_size}
-
-    if [ $? -ne 0 ]; then
-        echo "Failed in evaluation!"
-        exit 1
-    fi
-done
+        ${decode_cmd} JOB=1:${nj} ${expdir}/${decode_dir}/log/decode.JOB.log \
+            python3 -u ${BIN_DIR}/test.py \
+            --model-name u2_kaldi \
+            --run-mode test \
+            --nproc ${ngpu} \
+            --dict-path ${dict} \
+            --config ${config_path} \
+            --checkpoint_path ${ckpt_prefix} \
+            --result-file ${expdir}/${decode_dir}/data.JOB.json \
+            --opts decoding.decoding_method ${dmethd} \
+            --opts decoding.batch_size ${batch_size} \
+            --opts data.test_manifest ${feat_recog_dir}/split${nj}/JOB/manifest.${rtask}
+
+        score_sclite.sh --bpe ${nbpe} --bpemodel ${bpemodel}.model --wer true ${expdir}/${decode_dir} ${dict}
 
-for type in ctc_prefix_beam_search attention_rescoring; do
-    echo "decoding ${type}"
-    batch_size=1
-    python3 -u ${BIN_DIR}/test.py \
-    --nproc ${ngpu} \
-    --config ${config_path} \
-    --result_file ${ckpt_prefix}.${type}.rsl \
-    --checkpoint_path ${ckpt_prefix} \
-    --opts decoding.decoding_method ${type} \
-    --opts decoding.batch_size ${batch_size}
-
-    if [ $? -ne 0 ]; then
-        echo "Failed in evaluation!"
-        exit 1
-    fi
+    ) &
+    pids+=($!) # store background pids
+    done
+) &
+pids+=($!) # store background pids
 done
 
+i=0; for pid in "${pids[@]}"; do wait ${pid} || ((++i)); done
+[ ${i} -gt 0 ] && echo "$0: ${i} background jobs are failed." && false
+echo "Finished"
 
 exit 0
diff --git a/examples/librispeech/s1/path.sh b/examples/librispeech/s1/path.sh
index 457f7e548..09f5ba3f1 100644
--- a/examples/librispeech/s1/path.sh
+++ b/examples/librispeech/s1/path.sh
@@ -1,6 +1,6 @@
 export MAIN_ROOT=`realpath ${PWD}/../../../`
 
-export PATH=${MAIN_ROOT}:${PWD}/utils:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/tools/sckt/bin/sclite:${PWD}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
diff --git a/examples/librispeech/s1/run.sh b/examples/librispeech/s1/run.sh
index aecd3f617..8749039b7 100755
--- a/examples/librispeech/s1/run.sh
+++ b/examples/librispeech/s1/run.sh
@@ -1,12 +1,15 @@
 #!/bin/bash
 set -e
-source path.sh
+
+. ./path.sh || exit 1;
+. ./cmd.sh || exit 1;
 
 stage=0
 stop_stage=100
 conf_path=conf/transformer.yaml
 avg_num=5
-source ${MAIN_ROOT}/utils/parse_options.sh || exit 1;
+
+. ${MAIN_ROOT}/utils/parse_options.sh || exit 1;
 
 avg_ckpt=avg_${avg_num}
 ckpt=$(basename ${conf_path} | awk -F'.' '{print $1}')

From bf0f5d5d66e11495f31257f48623be0959f84d9d Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 23 Sep 2021 03:40:53 +0000
Subject: [PATCH 13/13] librispeech s1 support multi process decode and sclite

---
 deepspeech/frontend/utility.py        |   1 +
 examples/librispeech/s1/local/test.sh |   8 +-
 examples/librispeech/s1/path.sh       |   2 +-
 examples/librispeech/s2/local/test.sh |   3 +
 examples/librispeech/s2/path.sh       |   2 +-
 utils/run.pl                          |   0
 utils/split_json.sh                   |   4 +-
 utils/split_scp.pl                    |   0
 utils/split_scp.pl                    | 212 --------------------------
 9 files changed, 12 insertions(+), 220 deletions(-)
 mode change 100755 => 120000 utils/run.pl
 create mode 100755 utils/split_scp.pl
 delete mode 100644 utils/split_scp.pl 

diff --git a/deepspeech/frontend/utility.py b/deepspeech/frontend/utility.py
index 926abf18e..f7e2cb214 100644
--- a/deepspeech/frontend/utility.py
+++ b/deepspeech/frontend/utility.py
@@ -18,6 +18,7 @@ from typing import List
 from typing import Optional
 from typing import Text
 
+import jsonlines
 import numpy as np
 
 from deepspeech.utils.log import Log
diff --git a/examples/librispeech/s1/local/test.sh b/examples/librispeech/s1/local/test.sh
index 09644bb2c..7f48d3d51 100755
--- a/examples/librispeech/s1/local/test.sh
+++ b/examples/librispeech/s1/local/test.sh
@@ -1,7 +1,10 @@
 #!/bin/bash
 
+set -e
+
 expdir=exp
 datadir=data
+nj=32
 
 lmtag=
 
@@ -60,13 +63,10 @@ for dmethd in attention ctc_greedy_search ctc_prefix_beam_search attention_resco
         batch_size=1
         ${decode_cmd} JOB=1:${nj} ${expdir}/${decode_dir}/log/decode.JOB.log \
             python3 -u ${BIN_DIR}/test.py \
-            --model-name u2_kaldi \
-            --run-mode test \
             --nproc ${ngpu} \
-            --dict-path ${dict} \
             --config ${config_path} \
+            --result_file ${expdir}/${decode_dir}/data.JOB.json \
             --checkpoint_path ${ckpt_prefix} \
-            --result-file ${expdir}/${decode_dir}/data.JOB.json \
             --opts decoding.decoding_method ${dmethd} \
             --opts decoding.batch_size ${batch_size} \
             --opts data.test_manifest ${feat_recog_dir}/split${nj}/JOB/manifest.${rtask}
diff --git a/examples/librispeech/s1/path.sh b/examples/librispeech/s1/path.sh
index 09f5ba3f1..439f71ae2 100644
--- a/examples/librispeech/s1/path.sh
+++ b/examples/librispeech/s1/path.sh
@@ -1,6 +1,6 @@
 export MAIN_ROOT=`realpath ${PWD}/../../../`
 
-export PATH=${MAIN_ROOT}:${MAIN_ROOT}/tools/sckt/bin/sclite:${PWD}/utils:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/tools/sctk/bin:${PWD}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
diff --git a/examples/librispeech/s2/local/test.sh b/examples/librispeech/s2/local/test.sh
index 09644bb2c..5eeb2d612 100755
--- a/examples/librispeech/s2/local/test.sh
+++ b/examples/librispeech/s2/local/test.sh
@@ -1,7 +1,10 @@
 #!/bin/bash
 
+set -e
+
 expdir=exp
 datadir=data
+nj=32
 
 lmtag=
 
diff --git a/examples/librispeech/s2/path.sh b/examples/librispeech/s2/path.sh
index 9f6891cd3..05a037af8 100644
--- a/examples/librispeech/s2/path.sh
+++ b/examples/librispeech/s2/path.sh
@@ -1,6 +1,6 @@
 export MAIN_ROOT=`realpath ${PWD}/../../../`
 
-export PATH=${MAIN_ROOT}:${MAIN_ROOT}/tools/sckt/bin/sclite:${PWD}/utils:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/tools/sctk/bin:${PWD}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
diff --git a/utils/run.pl b/utils/run.pl
deleted file mode 100755
index 8b84c7f08..000000000
--- a/utils/run.pl
+++ /dev/null
@@ -1 +0,0 @@
-parallel/run.pl
\ No newline at end of file
diff --git a/utils/run.pl b/utils/run.pl
new file mode 120000
index 000000000..8b84c7f08
--- /dev/null
+++ b/utils/run.pl
@@ -0,0 +1 @@
+parallel/run.pl
\ No newline at end of file
diff --git a/utils/split_json.sh b/utils/split_json.sh
index 6c7d38b91..48f64f617 100755
--- a/utils/split_json.sh
+++ b/utils/split_json.sh
@@ -2,7 +2,7 @@
 set -o errexit
 
 if [ $# != 2 ]; then
-  echo "Usage: split_data.sh manifest num-to-split"
+  echo "Usage: split_json.sh manifest num-to-split"
   exit 1
 fi
 
@@ -28,4 +28,4 @@ done
 
 split_scp.pl $data/${jsonfile} $jsons
 
-exit 0
\ No newline at end of file
+exit 0
diff --git a/utils/split_scp.pl b/utils/split_scp.pl
new file mode 100755
index 000000000..e69de29bb
diff --git a/utils/split_scp.pl  b/utils/split_scp.pl 
deleted file mode 100644
index fc28e0b65..000000000
--- a/utils/split_scp.pl 	
+++ /dev/null
@@ -1,212 +0,0 @@
-#!/usr/bin/env perl
-use warnings; #sed replacement for -w perl parameter
-# Copyright 2010-2011 Microsoft Corporation
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
-# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
-# MERCHANTABLITY OR NON-INFRINGEMENT.
-# See the Apache 2 License for the specific language governing permissions and
-# limitations under the License.
-
-
-
-# This program splits up any kind of .scp or archive-type file.
-# If there is no utt2spk option it will work on any text  file and
-# will split it up with an approximately equal number of lines in
-# each but.
-# With the --utt2spk option it will work on anything that has the 
-# utterance-id as the first entry on each line; the utt2spk file is
-# of the form "utterance speaker" (on each line).
-# It splits it into equal size chunks as far as it can.  If you use
-# the utt2spk option it will make sure these chunks coincide with
-# speaker boundaries.  In this case, if there are more chunks
-# than speakers (and in some other circumstances), some of the 
-# resulting  chunks will be empty and it
-# will print a warning.
-# You will normally call this like:
-# split_scp.pl scp scp.1 scp.2 scp.3 ...
-# or
-# split_scp.pl --utt2spk=utt2spk scp scp.1 scp.2 scp.3 ...
-# Note that you can use this script to split the utt2spk file itself,
-# e.g. split_scp.pl --utt2spk=utt2spk utt2spk utt2spk.1 utt2spk.2 ...
-
-# You can also call the scripts like:
-# split_scp.pl -j 3 0 scp scp.0
-# [note: with this option, it assumes zero-based indexing of the split parts,
-# i.e. the second number must be 0 <= n < num-jobs.]
-
-$num_jobs = 0;
-$job_id = 0;
-$utt2spk_file = "";
-
-for ($x = 1; $x <= 2; $x++) {
-    if ($ARGV[0] eq "-j") {
-        shift @ARGV;
-        $num_jobs = shift @ARGV;
-        $job_id = shift @ARGV;
-        if ($num_jobs <= 0 || $job_id < 0 || $job_id >= $num_jobs) {
-            die "Invalid num-jobs and job-id: $num_jobs and $job_id";
-        }
-    }
-    if ($ARGV[0] =~ "--utt2spk=(.+)") {
-        $utt2spk_file=$1;
-        shift;
-    }
-}
-
-if(($num_jobs == 0 && @ARGV < 2) || ($num_jobs > 0 && (@ARGV < 1 || @ARGV > 2))) {
-    die "Usage: split_scp.pl [--utt2spk=<utt2spk_file>] in.scp out1.scp out2.scp ... \n" .
-        " or: split_scp.pl -j num-jobs job-id [--utt2spk=<utt2spk_file>] in.scp [out.scp]\n" .
-        " ... where 0 <= job-id < num-jobs.";
-}
-   
-$inscp = shift @ARGV;
-if ($num_jobs == 0) { # without -j option
-    @OUTPUTS = @ARGV;
-} else {
-    for ($j = 0; $j < $num_jobs; $j++) {
-        if ($j == $job_id) { 
-            if (@ARGV > 0) { push @OUTPUTS, $ARGV[0]; }
-            else { push @OUTPUTS, "-"; }
-        } else {
-            push @OUTPUTS, "/dev/null";
-        }
-    }
-} 
-
-if ($utt2spk_file ne "") {  # We have the --utt2spk option...
-    open(U, "<$utt2spk_file") || die "Failed to open utt2spk file $utt2spk_file";
-    while(<U>) {
-        @A = split;
-        @A == 2 || die "Bad line $_ in utt2spk file $utt2spk_file";
-        ($u,$s) = @A;
-        $utt2spk{$u} = $s;
-    }
-    open(I, "<$inscp") || die "Opening input scp file $inscp";
-    @spkrs = ();
-    while(<I>) {
-        @A = split;
-        if(@A == 0) { die "Empty or space-only line in scp file $inscp"; }
-        $u = $A[0];
-        $s = $utt2spk{$u};
-        if(!defined $s) { die "No such utterance $u in utt2spk file $utt2spk_file"; }
-        if(!defined $spk_count{$s}) { 
-            push @spkrs, $s; 
-            $spk_count{$s} = 0;
-            $spk_data{$s} = "";
-        }
-        $spk_count{$s}++;
-        $spk_data{$s} = $spk_data{$s} . $_;
-    }
-    # Now split as equally as possible ..
-    # First allocate spks to files by allocating an approximately
-    # equal number of speakers.
-    $numspks = @spkrs;  # number of speakers.
-    $numscps = @OUTPUTS; # number of output files.
-    for($scpidx = 0; $scpidx < $numscps; $scpidx++) {
-        $scparray[$scpidx] = []; # [] is array reference.
-    }
-    for ($spkidx = 0; $spkidx < $numspks; $spkidx++) {
-        $scpidx = int(($spkidx*$numscps) / $numspks);
-        $spk = $spkrs[$spkidx];
-        push @{$scparray[$scpidx]}, $spk;
-        $scpcount[$scpidx] += $spk_count{$spk};
-    }
-
-    # Now will try to reassign beginning + ending speakers
-    # to different scp's and see if it gets more balanced.
-    # Suppose objf we're minimizing is sum_i (num utts in scp[i] - average)^2.
-    # We can show that if considering changing just 2 scp's, we minimize
-    # this by minimizing the squared difference in sizes.  This is
-    # equivalent to minimizing the absolute difference in sizes.  This
-    # shows this method is bound to converge.
-
-    $changed = 1;
-    while($changed) {
-        $changed = 0;
-        for($scpidx = 0; $scpidx < $numscps; $scpidx++) {
-            # First try to reassign ending spk of this scp.
-            if($scpidx < $numscps-1) {
-                $sz = @{$scparray[$scpidx]};
-                if($sz > 0) {
-                    $spk = $scparray[$scpidx]->[$sz-1];
-                    $count = $spk_count{$spk};
-                    $nutt1 = $scpcount[$scpidx];
-                    $nutt2 = $scpcount[$scpidx+1];
-                    if( abs( ($nutt2+$count) - ($nutt1-$count))
-                        < abs($nutt2 - $nutt1))  { # Would decrease
-                        # size-diff by reassigning spk...
-                        $scpcount[$scpidx+1] += $count;
-                        $scpcount[$scpidx] -= $count;
-                        pop @{$scparray[$scpidx]};
-                        unshift @{$scparray[$scpidx+1]}, $spk;
-                        $changed = 1;
-                    }
-                }
-            }
-            if($scpidx > 0 && @{$scparray[$scpidx]} > 0) {
-                $spk = $scparray[$scpidx]->[0];
-                $count = $spk_count{$spk};
-                $nutt1 = $scpcount[$scpidx-1];
-                $nutt2 = $scpcount[$scpidx];
-                if( abs( ($nutt2-$count) - ($nutt1+$count))
-                    < abs($nutt2 - $nutt1))  { # Would decrease
-                    # size-diff by reassigning spk...
-                    $scpcount[$scpidx-1] += $count;
-                    $scpcount[$scpidx] -= $count;
-                    shift @{$scparray[$scpidx]};
-                    push @{$scparray[$scpidx-1]}, $spk;
-                    $changed = 1;
-                }
-            }
-        }
-    }
-    # Now print out the files...
-    for($scpidx = 0; $scpidx < $numscps; $scpidx++) {
-        $scpfn = $OUTPUTS[$scpidx];
-        open(F, ">$scpfn") || die "Could not open scp file $scpfn for writing.";
-        $count = 0;
-        if(@{$scparray[$scpidx]} == 0) {
-            print STDERR "Warning: split_scp.pl producing empty .scp file $scpfn (too many splits and too few speakers?)\n";
-        } else {
-            foreach $spk ( @{$scparray[$scpidx]} ) {
-                print F $spk_data{$spk};
-                $count += $spk_count{$spk};
-            }
-            if($count != $scpcount[$scpidx]) { die "Count mismatch [code error]"; }
-        }
-        close(F);
-    }
-} else { 
-   # This block is the "normal" case where there is no --utt2spk 
-   # option and we just break into equal size chunks.
-
-    open(I, "<$inscp") || die "Opening input scp file $inscp";
-
-    $numscps = @OUTPUTS;  # size of array.
-    @F = ();
-    while(<I>) {
-        push @F, $_;
-    }
-    $numlines = @F;
-    if($numlines == 0) {
-        print STDERR "split_scp.pl: warning: empty input scp file $inscp";
-    }
-    $linesperscp = int( ($numlines+($numscps-1)) / $numscps); # the +$(numscps-1) forces rounding up.
-# [just doing int() rounds down].
-    for($scpidx = 0; $scpidx < @OUTPUTS; $scpidx++) {
-        $scpfile = $OUTPUTS[$scpidx];
-        open(O, ">$scpfile") || die "Opening output scp file $scpfile";
-        for($n = $linesperscp * $scpidx; $n < $numlines && $n < $linesperscp*($scpidx+1); $n++) {
-            print O $F[$n];
-        }
-        close(O) || die "Closing scp file $scpfile";
-    }
-}
\ No newline at end of file