Merge pull request #881 from LittleChenCc/develop

update timit result

examples/timit/s1/README.md

@@ -2,8 +2,8 @@
 
 ### Transformer
 | Model | Params | Config | Decode method | Loss |  PER |
-| --- | --- | --- | --- | --- |
-| transformer | 5.17M | conf/transformer.yaml | attention              | 49.25688171386719 | 0.510742 |
-| transformer | 5.17M | conf/transformer.yaml | ctc_greedy_search      | 49.25688171386719 | 0.382398 |
-| transformer | 5.17M | conf/transformer.yaml | ctc_prefix_beam_search | 49.25688171386719 | 0.367429 |
-| transformer | 5.17M | conf/transformer.yaml | attention_rescore      | 49.25688171386719 | 0.357173 |  
+| --- | --- | --- | --- | --- | --- |
+| transformer | 5.17M | conf/transformer.yaml | attention              | 46.41119385 | 0.469993 |
+| transformer | 5.17M | conf/transformer.yaml | ctc_greedy_search      | 46.41119385 | 0.297713 |
+| transformer | 5.17M | conf/transformer.yaml | ctc_prefix_beam_search | 46.41119385 | 0.293555 |
+| transformer | 5.17M | conf/transformer.yaml | attention_rescore      | 46.41119385 | 0.281081 |

examples/timit/s1/conf/transformer.yaml

@@ -67,14 +67,14 @@ model:
     model_conf:
         ctc_weight: 0.5
         ctc_dropoutrate: 0.0
-        ctc_grad_norm_type: batch
+        ctc_grad_norm_type: instance
         lsm_weight: 0.1     # label smoothing option
         length_normalized_loss: false
 
 
 training:
   n_epoch: 200
-  accum_grad: 2
+  accum_grad: 1
   global_grad_clip: 5.0
   optim: adam
   optim_conf:
@@ -82,7 +82,7 @@ training:
     weight_decay: 1e-06
   scheduler: warmuplr     # pytorch v1.1.0+ required
   scheduler_conf:
-    warmup_steps: 2000
+    warmup_steps: 1200
     lr_decay: 1.0
   log_interval: 10
   checkpoint:

tests/benchmark/README.md

@@ -1,6 +1,38 @@
 ### Prepare the environment
 Please follow the instructions shown in [here](https://github.com/PaddlePaddle/DeepSpeech/blob/develop/docs/src/install.md) to install the Deepspeech first.
 
+### File list
+└── benchmark                  # 模型名  
+    ├── README.md              # 运行文档  
+    ├── analysis.py        # log解析脚本,每个框架尽量统一,可参考[paddle的analysis.py](https://github.com/mmglove/benchmark/blob/jp_0907/scripts/analysis.py)  
+    ├── recoder_mp_bs16_fp32_ngpu1.txt  # 单卡数据
+    ├── recoder_mp_bs16_fp32_ngpu8.txt  # 8卡数据  
+    ├── prepare.sh             #  竞品PyTorch运行环境搭建  
+    ├── run_benchmark.sh       # 运行脚本（包含性能、收敛性）  
+    ├── run_analysis_mp.sh     # 分析8卡的脚本  
+    ├── run_analysis_sp.sh     # 分析单卡的脚本  
+    ├── log
+    │     ├── log_sp.out    # 单卡的结果
+    │     └── log_mp.out    # 8卡的结果
+    └── run.sh         # 全量运行脚本
+
+
+### The physical environment
+- 单机（单卡、8卡）
+  - 系统：Ubuntu 16.04.6 LTS
+  - GPU：Tesla V100-SXM2-16GB * 8
+  - CPU：Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz * 96
+  - Driver Version: 440.64.00
+  - 内存：440 GB
+  - CUDA、cudnn Version: cuda10.2-cudnn7
+- 多机（32卡） TODO
+
+### Docker 镜像,如:
+
+- **镜像版本**: `registry.baidubce.com/paddlepaddle/paddle:2.1.0-gpu-cuda10.2-cudnn7`  
+- **CUDA 版本**: `10.2`
+- **cuDnn 版本**: `7`
+
 ### Prepare the benchmark environment
 ```
 bash prepare.sh
@@ -20,3 +52,18 @@ bash run_analysis_sp.sh
 ```
 bash run_analysis_mp.sh
 ```
+
+### The log
+{"log_file": "recoder_sp_bs16_fp32_ngpu1.txt",
+ "model_name": "Conformer",
+ "mission_name": "one gpu",
+ "direction_id": 1,
+ "run_mode": "sp",
+ "index": 1,
+ "gpu_num": 1,
+ "FINAL_RESULT": 23.228,
+ "JOB_FAIL_FLAG": 0,
+ "log_with_profiler": null,
+ "profiler_path": null,
+ "UNIT": "sent./sec"
+}

tests/benchmark/prepare.sh

@@ -1,7 +1,7 @@
-source ../tools/venv/bin/activate
+source ../../tools/venv/bin/activate
 
 #Enter the example dir
-pushd ../examples/aishell/s1
+pushd ../../examples/aishell/s1
 
 #Prepare the data
 bash run.sh --stage 0 --stop_stage 0

tests/benchmark/run.sh

@@ -2,9 +2,9 @@
 # 提供可稳定复现性能的脚本，默认在标准docker环境内py37执行： paddlepaddle/paddle:latest-gpu-cuda10.1-cudnn7  paddle=2.1.2  py=37
 # 执行目录：需说明
 CUR_DIR=${PWD}
-source ../tools/venv/bin/activate
+source ../../tools/venv/bin/activate
 #cd **
-pushd ../examples/aishell/s1
+pushd ../../examples/aishell/s1
 # 1 安装该模型需要的依赖 (如需开启优化策略请注明)
 # 2 拷贝该模型需要数据、预训练模型
 
@@ -28,7 +28,13 @@ for fp_item in ${fp_item_list[@]}; do
         echo "index is speed, 1gpus, begin, conformer"
         run_mode=sp
         ngpu=1
-        CUDA_VISIBLE_DEVICES=7 bash ${CUR_DIR}/run_benchmark.sh ${run_mode} ${config_path} ${output} ${seed} ${ngpu} ${profiler_options} ${batch_size} ${fp_item} ${CUR_DIR}
+        CUDA_VISIBLE_DEVICES=0 bash ${CUR_DIR}/run_benchmark.sh ${run_mode} ${config_path} ${output} ${seed} ${ngpu} ${profiler_options} ${batch_size} ${fp_item} ${CUR_DIR}
     done
 done
 
+popd
+
+mkdir -p log
+bash run_analysis_sp.sh > log/log_sp.out
+bash run_analysis_mp.sh > log/log_mp.out
+

tests/benchmark/run_analysis_mp.sh

@@ -9,4 +9,4 @@ python analysis.py \
     --gpu_num 8 \
     --use_num 480 \
     --separator " " \
-
+    --direction_id "1"

tests/benchmark/run_analysis_sp.sh

@@ -9,4 +9,4 @@ python analysis.py \
     --gpu_num 1 \
     --use_num 60 \
     --separator " " \
-
+    --direction_id "1"