refactor tiny script, add transformer and stream conf

examples/librispeech/README.md

@@ -1 +1,3 @@
-* s0 is for deepspeech
+# ASR
+* s0 is for deepspeech2
+* s1 is for U2

examples/tiny/s1/.gitignore

@@ -0,0 +1,3 @@
+data
+exp
+log

examples/tiny/s1/conf/augmentation.json

@@ -1,4 +1,12 @@
 [
+  {
+    "type": "shift",
+    "params": {
+      "min_shift_ms": -5,
+      "max_shift_ms": 5
+    },
+    "prob": 1.0
+  },
   {
     "type": "speed",
     "params": {
@@ -8,14 +16,6 @@
     },
     "prob": 0.0
   },
-  {
-    "type": "shift",
-    "params": {
-      "min_shift_ms": -5,
-      "max_shift_ms": 5
-    },
-    "prob": 1.0
-  },
   {
     "type": "specaug",
     "params": {

examples/tiny/s1/conf/chunk_confermer.yaml

@@ -1,4 +1,43 @@
+# https://yaml.org/type/float.html
+data:
+  train_manifest: data/manifest.tiny
+  dev_manifest: data/manifest.tiny
+  test_manifest: data/manifest.tiny
+  vocab_filepath: data/vocab.txt 
+  unit_type: 'spm'
+  spm_model_prefix: 'data/bpe_unigram_200'
+  mean_std_filepath: ""
+  augmentation_config: conf/augmentation.json
+  batch_size: 4
+  min_input_len: 0.5
+  max_input_len: 20.0
+  min_output_len: 0.0
+  max_output_len: 400.0
+  min_output_input_ratio: 0.05
+  max_output_input_ratio: 10.0
+  raw_wav: True  # use raw_wav or kaldi feature
+  specgram_type: fbank #linear, mfcc, fbank
+  feat_dim: 80
+  delta_delta: False
+  dither: 1.0
+  target_sample_rate: 16000
+  max_freq: None
+  n_fft: None
+  stride_ms: 10.0
+  window_ms: 25.0
+  use_dB_normalization: True
+  target_dB: -20
+  random_seed: 0
+  keep_transcription_text: False
+  sortagrad: True 
+  shuffle_method: batch_shuffle
+  num_workers: 2
+
+
 # network architecture
+model:
+    cmvn_file: "data/mean_std.json"
+    cmvn_file_type: "json"
     # encoder related
     encoder: conformer
     encoder_conf:
@@ -10,14 +49,14 @@ encoder_conf:
         positional_dropout_rate: 0.1
         attention_dropout_rate: 0.0
         input_layer: conv2d # encoder input type, you can chose conv2d, conv2d6 and conv2d8
-    normalize_before: true
+        normalize_before: True
-    cnn_module_kernel: 15
         use_cnn_module: True
+        cnn_module_kernel: 15
         activation_type: 'swish'
         pos_enc_layer_type: 'rel_pos'
         selfattention_layer_type: 'rel_selfattn'
-    causal: true
+        causal: True
-    use_dynamic_chunk: true
+        use_dynamic_chunk: True
         cnn_module_norm: 'layer_norm' # using nn.LayerNorm makes model converge faster
         use_dynamic_left_chunk: false
 
@@ -39,52 +78,38 @@ model_conf:
         length_normalized_loss: false
 
 
-# use raw_wav or kaldi feature
+training:
-raw_wav: true
+  n_epoch: 20
-
-# feature extraction
-collate_conf:
-    # waveform level config
-    wav_distortion_conf:
-        wav_dither: 1.0
-        wav_distortion_rate: 0.0
-        distortion_methods: []
-    speed_perturb: true
-    feature_extraction_conf:
-        feature_type: 'fbank'
-        mel_bins: 80
-        frame_shift: 10
-        frame_length: 25
-        using_pitch: false
-    # spec level config
-    # spec_swap: false
-    feature_dither: 0.0 # add dither [-feature_dither,feature_dither] on fbank feature
-    spec_aug: true
-    spec_aug_conf:
-        warp_for_time: False
-        num_t_mask: 2
-        num_f_mask: 2
-        max_t: 50
-        max_f: 10
-        max_w: 80
-
-# dataset related
-dataset_conf:
-    max_length: 40960
-    min_length: 0
-    batch_type: 'static' # static or dynamic
-    # the size of batch_size should be set according to your gpu memory size, here we used 2080ti gpu whose memory size is 11GB
-    batch_size: 16
-    sort: true
-
-grad_clip: 5
   accum_grad: 1
-max_epoch: 180
+  global_grad_clip: 5.0
-log_interval: 100
-
   optim: adam
   optim_conf:
     lr: 0.001
+    weight_decay: 1e-06
   scheduler: warmuplr     # pytorch v1.1.0+ required
   scheduler_conf:
     warmup_steps: 25000
+    lr_decay: 1.0
+  log_interval: 1
+
+
+decoding:
+  batch_size: 64
+  error_rate_type: wer
+  decoding_method: attention  # 'attention', 'ctc_greedy_search', 'ctc_prefix_beam_search', 'attention_rescoring'
+  lang_model_path: data/lm/common_crawl_00.prune01111.trie.klm
+  alpha: 2.5
+  beta: 0.3
+  beam_size: 10
+  cutoff_prob: 1.0
+  cutoff_top_n: 0
+  num_proc_bsearch: 8
+  ctc_weight: 0.0 # ctc weight for attention rescoring decode mode.
+  decoding_chunk_size: -1 # decoding chunk size. Defaults to -1.
+      # <0: for decoding, use full chunk.
+      # >0: for decoding, use fixed chunk size as set.
+      # 0: used for training, it's prohibited here. 
+  num_decoding_left_chunks: -1  # number of left chunks for decoding. Defaults to -1.
+  simulate_streaming: False  # simulate streaming inference. Defaults to False.
+
+

examples/tiny/s1/conf/chunk_transformer.yaml

@@ -1,4 +1,43 @@
+# https://yaml.org/type/float.html
+data:
+  train_manifest: data/manifest.tiny
+  dev_manifest: data/manifest.tiny
+  test_manifest: data/manifest.tiny
+  vocab_filepath: data/vocab.txt 
+  unit_type: 'spm'
+  spm_model_prefix: 'data/bpe_unigram_200'
+  mean_std_filepath: ""
+  augmentation_config: conf/augmentation.json
+  batch_size: 4
+  min_input_len: 0.5  # second
+  max_input_len: 20.0 # second
+  min_output_len: 0.0 # tokens
+  max_output_len: 400.0 # tokens
+  min_output_input_ratio: 0.05
+  max_output_input_ratio: 10.0
+  raw_wav: True  # use raw_wav or kaldi feature
+  specgram_type: fbank #linear, mfcc, fbank
+  feat_dim: 80
+  delta_delta: False
+  dither: 1.0
+  target_sample_rate: 16000
+  max_freq: None
+  n_fft: None
+  stride_ms: 10.0
+  window_ms: 25.0
+  use_dB_normalization: True
+  target_dB: -20
+  random_seed: 0
+  keep_transcription_text: False
+  sortagrad: True 
+  shuffle_method: batch_shuffle
+  num_workers: 2
+
+
 # network architecture
+model:
+    cmvn_file: "data/mean_std.json"
+    cmvn_file_type: "json"
     # encoder related
     encoder: transformer
     encoder_conf:
@@ -9,7 +48,7 @@ encoder_conf:
         dropout_rate: 0.1
         positional_dropout_rate: 0.1
         attention_dropout_rate: 0.0
-    input_layer: conv2d # encoder architecture type
+        input_layer: conv2d # encoder input type, you can chose conv2d, conv2d6 and conv2d8
         normalize_before: true
         use_dynamic_chunk: true
         use_dynamic_left_chunk: false
@@ -31,53 +70,39 @@ model_conf:
         lsm_weight: 0.1     # label smoothing option
         length_normalized_loss: false
 
-# use raw_wav or kaldi feature
-raw_wav: true
-
-# feature extraction
-collate_conf:
-    # waveform level config
-    wav_distortion_conf:
-        wav_dither: 0.0
-        wav_distortion_rate: 0.0
-        distortion_methods: []
-    speed_perturb: false
-    feature_extraction_conf:
-        feature_type: 'fbank'
-        mel_bins: 80
-        frame_shift: 10
-        frame_length: 25
-        using_pitch: false
-    # spec level config
-    # spec_swap: false
-    feature_dither: 0.0 # add dither [-feature_dither,feature_dither] on fbank feature
-    spec_aug: true
-    spec_aug_conf:
-        warp_for_time: False
-        num_t_mask: 2
-        num_f_mask: 2
-        max_t: 50
-        max_f: 10
-        max_w: 80
-
-
-# dataset related
-dataset_conf:
-    max_length: 40960
-    min_length: 0
-    batch_type: 'static' # static or dynamic
-    # the size of batch_size should be set according to your gpu memory size, here we used 2080ti gpu whose memory size is 11GB
-    batch_size: 16
-    sort: true
 
-grad_clip: 5
+training:
+  n_epoch: 20
   accum_grad: 1
-max_epoch: 180
+  global_grad_clip: 5.0
-log_interval: 100
-
   optim: adam
   optim_conf:
     lr: 0.002
+    weight_decay: 1e-06
   scheduler: warmuplr     # pytorch v1.1.0+ required
   scheduler_conf:
     warmup_steps: 25000
+    lr_decay: 1.0
+  log_interval: 1
+
+
+decoding:
+  batch_size: 64
+  error_rate_type: wer
+  decoding_method: attention  # 'attention', 'ctc_greedy_search', 'ctc_prefix_beam_search', 'attention_rescoring'
+  lang_model_path: data/lm/common_crawl_00.prune01111.trie.klm
+  alpha: 2.5
+  beta: 0.3
+  beam_size: 10
+  cutoff_prob: 1.0
+  cutoff_top_n: 0
+  num_proc_bsearch: 8
+  ctc_weight: 0.0 # ctc weight for attention rescoring decode mode.
+  decoding_chunk_size: -1 # decoding chunk size. Defaults to -1.
+      # <0: for decoding, use full chunk.
+      # >0: for decoding, use fixed chunk size as set.
+      # 0: used for training, it's prohibited here. 
+  num_decoding_left_chunks: -1  # number of left chunks for decoding. Defaults to -1.
+  simulate_streaming: False  # simulate streaming inference. Defaults to False.
+
+

examples/tiny/s1/conf/conformer.yaml

@@ -31,46 +31,7 @@ data:
   keep_transcription_text: False
   sortagrad: True 
   shuffle_method: batch_shuffle
-  num_workers: 0
+  num_workers: 2
-
-
-#   # feature extraction
-# collate_conf:
-#     # waveform level config
-#     wav_distortion_conf:
-#         wav_dither: 0.1
-#         wav_distortion_rate: 0.0
-#         distortion_methods: []
-#     speed_perturb: true
-#     feature_extraction_conf:
-#         feature_type: 'fbank'
-#         mel_bins: 80
-#         frame_shift: 10
-#         frame_length: 25
-#         using_pitch: false
-#     # spec level config
-#     # spec_swap: false
-#     feature_dither: 0.0 # add dither [-feature_dither,feature_dither] on fbank feature
-#     spec_aug: true
-#     spec_aug_conf:
-#         warp_for_time: False
-#         num_t_mask: 2
-#         num_f_mask: 2
-#         max_t: 50
-#         max_f: 10
-#         max_w: 80
-
-
-# # dataset related
-# dataset_conf:
-#     max_length: 40960
-#     min_length: 0
-#     batch_type: 'static' # static or dynamic
-#     # the size of batch_size should be set according to your gpu memory size, here we used 2080ti gpu whose memory size is 11GB
-#     batch_size: 16
-#     sort: true
-
-
 
 
 # network architecture
@@ -129,7 +90,7 @@ training:
 
 
 decoding:
-  batch_size: 16
+  batch_size: 64
   error_rate_type: wer
   decoding_method: attention  # 'attention', 'ctc_greedy_search', 'ctc_prefix_beam_search', 'attention_rescoring'
   lang_model_path: data/lm/common_crawl_00.prune01111.trie.klm

examples/tiny/s1/conf/transformer.yaml

@@ -1,4 +1,43 @@
+# https://yaml.org/type/float.html
+data:
+  train_manifest: data/manifest.tiny
+  dev_manifest: data/manifest.tiny
+  test_manifest: data/manifest.tiny
+  vocab_filepath: data/vocab.txt 
+  unit_type: 'spm'
+  spm_model_prefix: 'data/bpe_unigram_200'
+  mean_std_filepath: ""
+  augmentation_config: conf/augmentation.json
+  batch_size: 4
+  min_input_len: 0.5  # second
+  max_input_len: 20.0 # second
+  min_output_len: 0.0 # tokens
+  max_output_len: 400.0 # tokens
+  min_output_input_ratio: 0.05
+  max_output_input_ratio: 10.0
+  raw_wav: True  # use raw_wav or kaldi feature
+  specgram_type: fbank #linear, mfcc, fbank
+  feat_dim: 80
+  delta_delta: False
+  dither: 1.0
+  target_sample_rate: 16000
+  max_freq: None
+  n_fft: None
+  stride_ms: 10.0
+  window_ms: 25.0
+  use_dB_normalization: True
+  target_dB: -20
+  random_seed: 0
+  keep_transcription_text: False
+  sortagrad: True 
+  shuffle_method: batch_shuffle
+  num_workers: 2
+
+
 # network architecture
+model:
+    cmvn_file: "data/mean_std.json"
+    cmvn_file_type: "json"
     # encoder related
     encoder: transformer
     encoder_conf:
@@ -9,7 +48,7 @@ encoder_conf:
         dropout_rate: 0.1
         positional_dropout_rate: 0.1
         attention_dropout_rate: 0.0
-    input_layer: conv2d # encoder architecture type
+        input_layer: conv2d # encoder input type, you can chose conv2d, conv2d6 and conv2d8
         normalize_before: true
 
     # decoder related
@@ -29,52 +68,39 @@ model_conf:
         lsm_weight: 0.1     # label smoothing option
         length_normalized_loss: false
 
-# use raw_wav or kaldi feature
-raw_wav: true
-
-# feature extraction
-collate_conf:
-    # waveform level config
-    wav_distortion_conf:
-        wav_dither: 0.1
-        wav_distortion_rate: 0.0
-        distortion_methods: []
-    speed_perturb: true
-    feature_extraction_conf:
-        feature_type: 'fbank'
-        mel_bins: 80
-        frame_shift: 10
-        frame_length: 25
-        using_pitch: false
-    # spec level config
-    feature_dither: 0.0 # add dither [-feature_dither,feature_dither] on fbank feature
-    spec_aug: true
-    spec_aug_conf:
-        warp_for_time: False
-        num_t_mask: 2
-        num_f_mask: 2
-        max_t: 50
-        max_f: 10
-        max_w: 80
-
-
-# dataset related
-dataset_conf:
-    max_length: 40960
-    min_length: 0
-    batch_type: 'static' # static or dynamic
-    # the size of batch_size should be set according to your gpu memory size, here we used 2080ti gpu whose memory size is 11GB
-    batch_size: 26
-    sort: true
 
-grad_clip: 5
+training:
+  n_epoch: 20
   accum_grad: 1
-max_epoch: 240
+  global_grad_clip: 5.0
-log_interval: 100
-
   optim: adam
   optim_conf:
     lr: 0.002
+    weight_decay: 1e-06
   scheduler: warmuplr     # pytorch v1.1.0+ required
   scheduler_conf:
     warmup_steps: 25000
+    lr_decay: 1.0
+  log_interval: 1
+
+
+decoding:
+  batch_size: 64
+  error_rate_type: wer
+  decoding_method: attention  # 'attention', 'ctc_greedy_search', 'ctc_prefix_beam_search', 'attention_rescoring'
+  lang_model_path: data/lm/common_crawl_00.prune01111.trie.klm
+  alpha: 2.5
+  beta: 0.3
+  beam_size: 10
+  cutoff_prob: 1.0
+  cutoff_top_n: 0
+  num_proc_bsearch: 8
+  ctc_weight: 0.0 # ctc weight for attention rescoring decode mode.
+  decoding_chunk_size: -1 # decoding chunk size. Defaults to -1.
+      # <0: for decoding, use full chunk.
+      # >0: for decoding, use fixed chunk size as set.
+      # 0: used for training, it's prohibited here. 
+  num_decoding_left_chunks: -1  # number of left chunks for decoding. Defaults to -1.
+  simulate_streaming: False  # simulate streaming inference. Defaults to False.
+
+

examples/tiny/s1/local/avg.sh

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+if [ $# != 2 ];then
+    echo "usage: ${0} ckpt_dir avg_num"
+    exit -1
+fi
+
+ckpt_dir=${1}
+average_num=${2}
+decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams
+
+python3 -u ${MAIN_ROOT}/utils/avg_model.py \
+--dst_model ${decode_checkpoint} \
+--ckpt_dir ${ckpt_dir}  \
+--num ${average_num} \
+--val_best
+            
+if [ $? -ne 0 ]; then
+    echo "Failed in avg ckpt!"
+    exit 1
+fi
+
+exit 0

examples/tiny/s1/local/export.sh

@@ -1,18 +1,22 @@
 #! /usr/bin/env bash
 
-if [ $# != 2 ];then
+if [ $# != 3 ];then
-    echo "usage: export ckpt_path jit_model_path"
+    echo "usage: $0 config_path ckpt_prefix jit_model_path"
     exit -1
 fi
 
+config_path=$1
+ckpt_path_prefix=$2
+jit_model_export_path=$3
+
 python3 -u ${BIN_DIR}/export.py \
---config conf/conformer.yaml \
+--config ${config_path} \
---checkpoint_path ${1} \
+--checkpoint_path ${ckpt_path_prefix} \
---export_path ${2} 
+--export_path ${jit_model_export_path} 
 
 
 if [ $? -ne 0 ]; then
-    echo "Failed in evaluation!"
+    echo "Failed in export!"
     exit 1
 fi
 

examples/tiny/s1/local/test.sh

@@ -1,5 +1,20 @@
 #! /usr/bin/env bash
 
+if [ $# != 2 ];then
+    echo "usage: ${0} config_path ckpt_path_prefix"
+    exit -1
+fi
+
+ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
+echo "using $ngpu gpus..."
+
+device=gpu
+if [ ngpu != 0 ];then
+    device=cpu
+fi
+config_path=$1
+ckpt_prefix=$2
+
 # download language model
 #bash local/download_lm_en.sh
 #if [ $? -ne 0 ]; then
@@ -7,11 +22,11 @@
 #fi
 
 python3 -u ${BIN_DIR}/test.py \
---device 'gpu' \
+--device ${device} \
 --nproc 1 \
---config conf/conformer.yaml \
+--config ${config_path} \
---result_file data/asr.result \
+--result_file ${ckpt_prefix}.rsl \
---output ckpt
+--checkpoint_path ${ckpt_prefix}
 
 if [ $? -ne 0 ]; then
     echo "Failed in evaluation!"

examples/tiny/s1/local/train.sh

@@ -1,18 +1,31 @@
 #! /usr/bin/env bash
 
+if [ $# != 2 ];then
+    echo "usage: CUDA_VISIBLE_DEVICES=0 ${0} config_path ckpt_name"
+    exit -1
+fi
+
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
+config_path=$1
+ckpt_name=$2
+device=gpu
+if [ ngpu != 0 ];then
+    device=cpu
+fi
+
+mkdir -p exp
+
 python3 -u ${BIN_DIR}/train.py \
---device 'gpu' \
+--device ${device} \
 --nproc ${ngpu} \
---config conf/conformer.yaml \
+--config ${config_path} \
---output ckpt-${1}
+--output exp/${ckpt_name}
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"
     exit 1
 fi
 
-
 exit 0

examples/tiny/s1/run.sh

@@ -2,15 +2,19 @@
 set -e
 
 source path.sh
+source ${MAIN_ROOT}/utils/parse_options.sh
 
 # prepare data
 bash ./local/data.sh
 
-# train model
+# train model, all `ckpt` under `exp` dir
-bash ./local/train.sh
+CUDA_VISIBLE_DEVICES=0 ./local/train.sh conf/conformer.yaml test
 
-# test model
+# test ckpt 1
-bash ./local/test.sh
+CUDA_VISIBLE_DEVICES=0 ./local/test.sh conf/conformer.yaml exp/test/checkpoints/1
 
-# infer model
+# avg 1 best model
-bash ./local/infer.sh
+./local/avg.sh exp/test/checkpoints 1
+
+# export ckpt 1
+./local/export.sh conf/conformer.yaml exp/test/checkpoints/1 exp/test/checkpoints/1.jit.model