@ -26,6 +26,10 @@ if [ ${seed} != 0 ]; then
export FLAGS_cudnn_deterministic=True
fi
# default memeory allocator strategy may case gpu training hang
# for no OOM raised when memory exhaused
export FLAGS_allocator_strategy=naive_best_fit
if [ ${ngpu} == 0 ]; then
python3 -u ${BIN_DIR}/train.py \
--ngpu ${ngpu} \
@ -35,6 +35,10 @@ echo ${ips_config}
mkdir -p exp
@ -29,6 +29,10 @@ fi
# export FLAGS_cudnn_exhaustive_search=true
# export FLAGS_conv_workspace_size_limit=4000
@ -19,6 +19,10 @@ if [ ${seed} != 0 ]; then
@ -32,6 +32,10 @@ fi
@ -34,6 +34,10 @@ fi