#!/usr/bin/env bash
set -xe
# 运行示例：CUDA_VISIBLE_DEVICES=0 bash run_benchmark.sh ${run_mode} ${bs_item} ${fp_item} 500 ${model_mode}
# 参数说明
function _set_params(){

    run_mode=${1:-"sp"}          # 单卡sp|多卡mp
    config_path=${2:-"conf/conformer.yaml"}
    output=${3:-"exp/conformer"}
    seed=${4:-"0"}
    ngpu=${5:-"1"}
    profiler_options=${6:-"None"}
    batch_size=${7:-"32"}
    fp_item=${8:-"fp32"}
    TRAIN_LOG_DIR=${9:-$(pwd)}

    benchmark_max_step=0

    run_log_path=${TRAIN_LOG_DIR:-$(pwd)}  # TRAIN_LOG_DIR 后续QA设置该参数

#   以下不用修改
    device=${CUDA_VISIBLE_DEVICES//,/ }
    arr=(${device})
    num_gpu_devices=${#arr[*]}
    log_file=${run_log_path}/recoder_${run_mode}_bs${batch_size}_${fp_item}_ngpu${ngpu}.txt
}

function _train(){
    echo "Train on ${num_gpu_devices} GPUs"
    echo "current CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES, gpus=$num_gpu_devices, batch_size=$batch_size"
    train_cmd="--config=${config_path}
               --output=${output}
               --seed=${seed}
               --nproc=${ngpu}
               --profiler-options "${profiler_options}"
               --benchmark-batch-size ${batch_size}
               --benchmark-max-step ${benchmark_max_step} "

    echo "run_mode "${run_mode}

    case ${run_mode} in
    sp) train_cmd="python3 -u ${BIN_DIR}/train.py "${train_cmd} ;;
    mp) train_cmd="python3 -u ${BIN_DIR}/train.py "${train_cmd} ;;
    *) echo "choose run_mode(sp or mp)"; exit 1;
    esac
    echo ${train_cmd}
# 以下不用修改
    timeout 15m ${train_cmd} > ${log_file} 2>&1
    if [ $? -ne 0 ];then
        echo -e "${model_name}, FAIL"
        export job_fail_flag=1
    else
        echo -e "${model_name}, SUCCESS"
        export job_fail_flag=0
    fi
    trap 'for pid in $(jobs -pr); do kill -KILL $pid; done' INT QUIT TERM

    if [ $run_mode = "mp" -a -d mylog ]; then
        rm ${log_file}
        cp mylog/workerlog.0 ${log_file}
    fi
}

_set_params $@
_train