msb-public
/
PaddleSpeech
mirror of https://github.com/PaddlePaddle/PaddleSpeech
1
0
Code Issues Projects Releases Wiki Activity

fix profiler (#3323)

Browse Source
pull/3334/head
gmm 1 year ago committed by GitHub
parent b06074853b
commit 5153ac8318
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 50 additions and 24 deletions
Show Stats Download Patch File Download Diff File

42
paddlespeech/t2s/utils/profiler.py
Unescape View File

@ -11,9 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import sys
import paddle import sys
import paddle.profiler as profiler
# A global variable to record the number of calling times for profiler # A global variable to record the number of calling times for profiler
# functions. It is used to specify the tracing range of training steps. # functions. It is used to specify the tracing range of training steps.
@ -21,7 +21,7 @@ _profiler_step_id = 0
# A global variable to avoid parsing from string every time. # A global variable to avoid parsing from string every time.
_profiler_options = None _profiler_options = None
_prof = None
class ProfilerOptions(object): class ProfilerOptions(object):
''' '''
@ -31,6 +31,7 @@ class ProfilerOptions(object):
"profile_path=model.profile" "profile_path=model.profile"
"batch_range=[50, 60]; profile_path=model.profile" "batch_range=[50, 60]; profile_path=model.profile"
"batch_range=[50, 60]; tracer_option=OpDetail; profile_path=model.profile" "batch_range=[50, 60]; tracer_option=OpDetail; profile_path=model.profile"
ProfilerOptions supports following key-value pair: ProfilerOptions supports following key-value pair:
batch_range - a integer list, e.g. [100, 110]. batch_range - a integer list, e.g. [100, 110].
state - a string, the optional values are 'CPU', 'GPU' or 'All'. state - a string, the optional values are 'CPU', 'GPU' or 'All'.
@ -52,7 +53,8 @@ class ProfilerOptions(object):
'sorted_key': 'total', 'sorted_key': 'total',
'tracer_option': 'Default', 'tracer_option': 'Default',
'profile_path': '/tmp/profile', 'profile_path': '/tmp/profile',
'exit_on_finished': True 'exit_on_finished': True,
'timer_only': True
} }
self._parse_from_string(options_str) self._parse_from_string(options_str)
@ -71,6 +73,8 @@ class ProfilerOptions(object):
'state', 'sorted_key', 'tracer_option', 'profile_path' 'state', 'sorted_key', 'tracer_option', 'profile_path'
]: ]:
self._options[key] = value self._options[key] = value
elif key == 'timer_only':
self._options[key] = value
def __getitem__(self, name): def __getitem__(self, name):
if self._options.get(name, None) is None: if self._options.get(name, None) is None:
@ -84,7 +88,6 @@ def add_profiler_step(options_str=None):
Enable the operator-level timing using PaddlePaddle's profiler. Enable the operator-level timing using PaddlePaddle's profiler.
The profiler uses a independent variable to count the profiler steps. The profiler uses a independent variable to count the profiler steps.
One call of this function is treated as a profiler step. One call of this function is treated as a profiler step.
Args: Args:
profiler_options - a string to initialize the ProfilerOptions. profiler_options - a string to initialize the ProfilerOptions.
Default is None, and the profiler is disabled. Default is None, and the profiler is disabled.
@ -92,18 +95,33 @@ def add_profiler_step(options_str=None):
if options_str is None: if options_str is None:
return return
global _prof
global _profiler_step_id global _profiler_step_id
global _profiler_options global _profiler_options
if _profiler_options is None: if _profiler_options is None:
_profiler_options = ProfilerOptions(options_str) _profiler_options = ProfilerOptions(options_str)
# profile : https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/performance_improving/profiling_model.html#chakanxingnengshujudetongjibiaodan
if _profiler_step_id == _profiler_options['batch_range'][0]: # timer_only = True only the model's throughput and time overhead are displayed
paddle.utils.profiler.start_profiler(_profiler_options['state'], # timer_only = False calling summary can print a statistical form that presents performance data from different perspectives.
_profiler_options['tracer_option']) # timer_only = False the output Timeline information can be found in the profiler_log directory
elif _profiler_step_id == _profiler_options['batch_range'][1]: if _prof is None:
paddle.utils.profiler.stop_profiler(_profiler_options['sorted_key'], _timer_only = str(_profiler_options['timer_only']) == str(True)
_profiler_options['profile_path']) _prof = profiler.Profiler(
scheduler = (_profiler_options['batch_range'][0], _profiler_options['batch_range'][1]),
on_trace_ready = profiler.export_chrome_tracing('./profiler_log'),
timer_only = _timer_only)
_prof.start()
else:
_prof.step()
if _profiler_step_id == _profiler_options['batch_range'][1]:
_prof.stop()
_prof.summary(
op_detail=True,
thread_sep=False,
time_unit='ms')
_prof = None
if _profiler_options['exit_on_finished']: if _profiler_options['exit_on_finished']:
sys.exit(0) sys.exit(0)

32
tests/test_tipc/benchmark_train.sh
Unescape View File

@ -110,6 +110,8 @@ repo_name=$(get_repo_name )
SAVE_LOG=${BENCHMARK_LOG_DIR:-$(pwd)} # */benchmark_log SAVE_LOG=${BENCHMARK_LOG_DIR:-$(pwd)} # */benchmark_log
mkdir -p "${SAVE_LOG}/benchmark_log/" mkdir -p "${SAVE_LOG}/benchmark_log/"
status_log="${SAVE_LOG}/benchmark_log/results.log" status_log="${SAVE_LOG}/benchmark_log/results.log"
# get benchmark profiling params : PROFILING_TIMER_ONLY=no|True|False
PROFILING_TIMER_ONLY=${PROFILING_TIMER_ONLY:-"True"}
# The number of lines in which train params can be replaced. # The number of lines in which train params can be replaced.
line_python=3 line_python=3
@ -166,19 +168,25 @@ for batch_size in ${batch_size_list[*]}; do
gpu_id=$(set_gpu_id $device_num) gpu_id=$(set_gpu_id $device_num)
if [ ${#gpu_id} -le 1 ];then if [ ${#gpu_id} -le 1 ];then
log_path="$SAVE_LOG/profiling_log"
mkdir -p $log_path
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_mode}_${device_num}_profiling"
func_sed_params "$FILENAME" "${line_gpuid}" "0" # sed used gpu_id func_sed_params "$FILENAME" "${line_gpuid}" "0" # sed used gpu_id
# set profile_option params if [[ ${PROFILING_TIMER_ONLY} != "no" ]];then
tmp=`sed -i "${line_profile}s/.*/${profile_option}/" "${FILENAME}"` echo "run profile"
# The default value of profile_option's timer_only parameter is True
# run test_train_inference_python.sh if [[ ${PROFILING_TIMER_ONLY} = "False" ]];then
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 " profile_option="${profile_option};timer_only=False"
echo $cmd fi
eval $cmd log_path="$SAVE_LOG/profiling_log"
eval "cat ${log_path}/${log_name}" mkdir -p $log_path
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_mode}_${device_num}_${to_static}profiling"
# set profile_option params
tmp=`sed -i "${line_profile}s/.*/\"${profile_option}\"/" "${FILENAME}"`
# run test_train_inference_python.sh
cmd="timeout 5m bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
echo $cmd
eval ${cmd}
eval "cat ${log_path}/${log_name}"
fi
echo "run without profile"
# without profile # without profile
log_path="$SAVE_LOG/train_log" log_path="$SAVE_LOG/train_log"
speed_log_path="$SAVE_LOG/index" speed_log_path="$SAVE_LOG/index"

Write Preview
Loading…
Cancel
Save