Merge pull request #703 from PaddlePaddle/shell

fix sha bone; more detial of chunk exp;

examples/aishell/s0/local/client.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 source path.sh
 

examples/aishell/s0/local/data.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 stage=-1
 stop_stage=100

examples/aishell/s0/local/download_lm_ch.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 . ${MAIN_ROOT}/utils/utility.sh
 

examples/aishell/s0/local/export.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 3 ];then
     echo "usage: $0 config_path ckpt_prefix jit_model_path"
@@ -13,7 +13,7 @@ ckpt_path_prefix=$2
 jit_model_export_path=$3
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 

examples/aishell/s0/local/server.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 # TODO: replace the model with a mandarin model
 
 if [[ $# != 1 ]];then

examples/aishell/s0/local/test.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: ${0} config_path ckpt_path_prefix"
@@ -9,7 +9,7 @@ ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 config_path=$1

examples/aishell/s0/local/train.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: CUDA_VISIBLE_DEVICES=0 ${0} config_path ckpt_name"
@@ -12,7 +12,7 @@ config_path=$1
 ckpt_name=$2
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 

examples/aishell/s0/local/tune.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 # grid-search for hyper-parameters in language model
 python3 -u ${BIN_DIR}/tune.py \

examples/aishell/s1/README.md

@@ -12,12 +12,12 @@
 
 ## Chunk Conformer
 
-| Model | Params | Config | Augmentation| Test set | Decode method | Chunk | Loss | WER |  
+| Model | Params | Config | Augmentation| Test set | Decode method | Chunk Size & Left Chunks | Loss | WER |  
 | --- | --- | --- | --- | --- | --- | --- | --- | --- |  
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | attention | 16 | - | 0.061939 |  
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | ctc_greedy_search | 16 | - | 0.070806 |  
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | ctc_prefix_beam_search | 16 | - | 0.070739 |  
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | attention_rescoring | 16 |  - | 0.059400 |  
+| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | attention | 16, -1 | - | 0.061939 |  
+| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | ctc_greedy_search | 16, -1 | - | 0.070806 |  
+| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | ctc_prefix_beam_search | 16, -1 | - | 0.070739 |  
+| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | attention_rescoring | 16, -1 |  - | 0.059400 |  
 
 
 ## Transformer

examples/aishell/s1/local/align.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: ${0} config_path ckpt_path_prefix"
@@ -9,18 +9,12 @@ ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 config_path=$1
 ckpt_prefix=$2
 
-ckpt_name=$(basename ${ckpt_prefxi})
-
-mkdir -p exp
-
-
-
 batch_size=1
 output_dir=${ckpt_prefix}
 mkdir -p ${output_dir}

examples/aishell/s1/local/data.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 stage=-1
 stop_stage=100

examples/aishell/s1/local/export.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 3 ];then
     echo "usage: $0 config_path ckpt_prefix jit_model_path"
@@ -13,7 +13,7 @@ ckpt_path_prefix=$2
 jit_model_export_path=$3
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 

examples/aishell/s1/local/test.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: ${0} config_path ckpt_path_prefix"
@@ -9,15 +9,17 @@ ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
+
 config_path=$1
 ckpt_prefix=$2
 
-ckpt_name=$(basename ${ckpt_prefxi})
-
-mkdir -p exp
+chunk_mode=false
+if [[ ${config_path} =~ ^chunk_ ]];then
+    chunk_mode=true
+fi
 
 # download language model
 #bash local/download_lm_ch.sh
@@ -28,7 +30,12 @@ mkdir -p exp
 
 for type in attention ctc_greedy_search; do
     echo "decoding ${type}"
-    batch_size=64
+    if [ ${chunk_mode} == true ];then
+        # stream decoding only support batchsize=1
+        batch_size=1
+    else
+        batch_size=64
+    fi
     output_dir=${ckpt_prefix}
     mkdir -p ${output_dir}
     python3 -u ${BIN_DIR}/test.py \

examples/aishell/s1/local/train.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: CUDA_VISIBLE_DEVICES=0 ${0} config_path ckpt_name"
@@ -12,7 +12,7 @@ config_path=$1
 ckpt_name=$2
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 echo "using ${device}..."

examples/librispeech/s0/local/data.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 stage=-1
 stop_stage=100

examples/librispeech/s0/local/download_lm_en.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 . ${MAIN_ROOT}/utils/utility.sh
 

examples/librispeech/s0/local/export.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 3 ];then
     echo "usage: $0 config_path ckpt_prefix jit_model_path"
@@ -13,7 +13,7 @@ ckpt_path_prefix=$2
 jit_model_export_path=$3
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 

examples/librispeech/s0/local/test.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: ${0} config_path ckpt_path_prefix"
@@ -9,7 +9,7 @@ ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 config_path=$1

examples/librispeech/s0/local/train.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: CUDA_VISIBLE_DEVICES=0 ${0} config_path ckpt_name"
@@ -12,7 +12,7 @@ config_path=$1
 ckpt_name=$2
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 echo "using ${device}..."

examples/librispeech/s0/local/tune.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 1 ];then
     echo "usage: tune ckpt_path"

examples/librispeech/s1/README.md

@@ -10,6 +10,18 @@
 | conformer | 47.63 M | conf/conformer.yaml | spec_aug + shift | test-clean | ctc_prefix_beam_search | 6.35 | 0.037761 |  
 | conformer | 47.63 M | conf/conformer.yaml | spec_aug + shift | test-clean | attention_rescoring | 6.35 | 0.032115 |  
 
+
+## Chunk Conformer
+
+| Model | Params | Config | Augmentation| Test set | Decode method | Chunk Size & Left Chunks | Loss | WER |  
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |  
+| conformer | 47.63 M | conf/chunk_conformer.yaml | spec_aug + shift | test-clean | attention | 16, -1 | 7.01250648 |  |  
+| conformer | 47.63 M | conf/chunk_conformer.yaml | spec_aug + shift | test-clean | ctc_greedy_search | 16, -1 | 7.01250648 |  |  
+| conformer | 47.63 M | conf/chunk_conformer.yaml | spec_aug + shift | test-clean | ctc_prefix_beam_search | 16, -1 | 7.01250648 |  |  
+| conformer | 47.63 M | conf/chunk_conformer.yaml | spec_aug + shift | test-clean | attention_rescoring | 16, -1 | 7.01250648 | |  
+
+
+
 ## Transformer
 
 | Model | Params | Config | Augmentation| Test set | Decode method | Loss | WER |  

examples/librispeech/s1/conf/chunk_confermer.yaml

@@ -110,7 +110,7 @@ decoding:
   cutoff_top_n: 0
   num_proc_bsearch: 8
   ctc_weight: 0.5 # ctc weight for attention rescoring decode mode.
-  decoding_chunk_size: -1 # decoding chunk size. Defaults to -1.
+  decoding_chunk_size: 16 # decoding chunk size. Defaults to -1.
       # <0: for decoding, use full chunk.
       # >0: for decoding, use fixed chunk size as set.
       # 0: used for training, it's prohibited here. 

examples/librispeech/s1/local/align.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: ${0} config_path ckpt_path_prefix"
@@ -9,18 +9,12 @@ ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 config_path=$1
 ckpt_prefix=$2
 
-ckpt_name=$(basename ${ckpt_prefxi})
-
-mkdir -p exp
-
-
-
 batch_size=1
 output_dir=${ckpt_prefix}
 mkdir -p ${output_dir}

examples/librispeech/s1/local/data.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 stage=-1
 stop_stage=100

examples/librispeech/s1/local/download_lm_en.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 . ${MAIN_ROOT}/utils/utility.sh
 

examples/librispeech/s1/local/export.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 3 ];then
     echo "usage: $0 config_path ckpt_prefix jit_model_path"
@@ -13,7 +13,7 @@ ckpt_path_prefix=$2
 jit_model_export_path=$3
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 

examples/librispeech/s1/local/test.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: ${0} config_path ckpt_path_prefix"
@@ -9,12 +9,19 @@ ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
+
 config_path=$1
 ckpt_prefix=$2
 
+chunk_mode=false
+if [[ ${config_path} =~ ^chunk_ ]];then
+    chunk_mode=true
+fi
+
+
 # download language model
 #bash local/download_lm_en.sh
 #if [ $? -ne 0 ]; then
@@ -23,7 +30,12 @@ ckpt_prefix=$2
 
 for type in attention ctc_greedy_search; do
     echo "decoding ${type}"
-    batch_size=64
+    if [ ${chunk_mode} == true ];then
+        # stream decoding only support batchsize=1
+        batch_size=1
+    else
+        batch_size=64
+    fi
     python3 -u ${BIN_DIR}/test.py \
     --device ${device} \
     --nproc 1 \

examples/librispeech/s1/local/train.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: CUDA_VISIBLE_DEVICES=0 ${0} config_path ckpt_name"
@@ -12,7 +12,7 @@ config_path=$1
 ckpt_name=$2
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 echo "using ${device}..."

examples/tiny/s0/local/data.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 stage=-1
 stop_stage=100

examples/tiny/s0/local/download_lm_en.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 . ${MAIN_ROOT}/utils/utility.sh
 

examples/tiny/s0/local/export.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 3 ];then
     echo "usage: $0 config_path ckpt_prefix jit_model_path"
@@ -13,7 +13,7 @@ ckpt_path_prefix=$2
 jit_model_export_path=$3
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 

examples/tiny/s0/local/test.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: ${0} config_path ckpt_path_prefix"
@@ -9,7 +9,7 @@ ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 config_path=$1

examples/tiny/s0/local/train.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: CUDA_VISIBLE_DEVICES=0 ${0} config_path ckpt_name"
@@ -12,7 +12,7 @@ config_path=$1
 ckpt_name=$2
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 

examples/tiny/s0/local/tune.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 1 ];then
     echo "usage: tune ckpt_path"

examples/tiny/s1/local/align.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: ${0} config_path ckpt_path_prefix"
@@ -9,18 +9,12 @@ ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 config_path=$1
 ckpt_prefix=$2
 
-ckpt_name=$(basename ${ckpt_prefxi})
-
-mkdir -p exp
-
-
-
 batch_size=1
 output_dir=${ckpt_prefix}
 mkdir -p ${output_dir}

examples/tiny/s1/local/data.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 stage=-1
 stop_stage=100

examples/tiny/s1/local/export.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 3 ];then
     echo "usage: $0 config_path ckpt_prefix jit_model_path"
@@ -13,7 +13,7 @@ ckpt_path_prefix=$2
 jit_model_export_path=$3
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 

examples/tiny/s1/local/test.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: ${0} config_path ckpt_path_prefix"
@@ -9,29 +9,60 @@ ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi
 config_path=$1
 ckpt_prefix=$2
 
+chunk_mode=false
+if [[ ${config_path} =~ ^chunk_ ]];then
+    chunk_mode=true
+fi
+
 # download language model
 #bash local/download_lm_en.sh
 #if [ $? -ne 0 ]; then
 #    exit 1
 #fi
 
-python3 -u ${BIN_DIR}/test.py \
---device ${device} \
---nproc 1 \
---config ${config_path} \
---result_file ${ckpt_prefix}.rsl \
---checkpoint_path ${ckpt_prefix}
+for type in attention ctc_greedy_search; do
+    echo "decoding ${type}"
+    if [ ${chunk_mode} == true ];then
+        # stream decoding only support batchsize=1
+        batch_size=1
+    else
+        batch_size=64
+    fi
+    python3 -u ${BIN_DIR}/test.py \
+    --device ${device} \
+    --nproc 1 \
+    --config ${config_path} \
+    --result_file ${ckpt_prefix}.${type}.rsl \
+    --checkpoint_path ${ckpt_prefix} \
+    --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
 
-if [ $? -ne 0 ]; then
-    echo "Failed in evaluation!"
-    exit 1
-fi
+    if [ $? -ne 0 ]; then
+        echo "Failed in evaluation!"
+        exit 1
+    fi
+done
+
+for type in ctc_prefix_beam_search attention_rescoring; do
+    echo "decoding ${type}"
+    batch_size=1
+    python3 -u ${BIN_DIR}/test.py \
+    --device ${device} \
+    --nproc 1 \
+    --config ${config_path} \
+    --result_file ${ckpt_prefix}.${type}.rsl \
+    --checkpoint_path ${ckpt_prefix} \
+    --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
 
+    if [ $? -ne 0 ]; then
+        echo "Failed in evaluation!"
+        exit 1
+    fi
+done
 
 exit 0

examples/tiny/s1/local/train.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/bin/bash
 
 if [ $# != 2 ];then
     echo "usage: CUDA_VISIBLE_DEVICES=0 ${0} config_path ckpt_name"
@@ -12,7 +12,7 @@ config_path=$1
 ckpt_name=$2
 
 device=gpu
-if [ ngpu == 0 ];then
+if [ ${ngpu} == 0 ];then
     device=cpu
 fi